diff --git a/src/dab_tracker_azure_camera.cpp b/src/dab_tracker_azure_camera.cpp index 5b5c9f0..8ca0632 100644 --- a/src/dab_tracker_azure_camera.cpp +++ b/src/dab_tracker_azure_camera.cpp @@ -169,6 +169,24 @@ AzureCamera::init() throw (dab::Exception) } } +bool +AzureCamera::isColorMode() const +{ + return mColorMode; +} + +bool +AzureCamera::isDepthMode() const +{ + return mDepthMode; +} + +bool +AzureCamera::isPointCloudMode() const +{ + return mPointCloudMode; +} + ofxCvImage* AzureCamera::colorFrame() { @@ -187,6 +205,54 @@ AzureCamera::pointCloud() return mPointCloud; } +const k4a::image +AzureCamera::nativeColorFrame() +{ + mLock.lock(); + + k4a::image _nativeColorImage = mNativeColorImage; + + mLock.unlock(); + + return _nativeColorImage; +} + +const k4a::image +AzureCamera::nativeDepthFrame() +{ + mLock.lock(); + + k4a::image _nativeDepthImage = mNativeDepthImage; + + mLock.unlock(); + + return _nativeDepthImage; +} + +const k4a::image +AzureCamera::nativeDepthColorFrame() +{ + mLock.lock(); + + k4a::image _nativeColorDepthImage = mNativeDepthColorImage; + + mLock.unlock(); + + return _nativeColorDepthImage; +} + +const k4a::image +AzureCamera::nativePointCloudFrame() +{ + mLock.lock(); + + k4a::image _nativePointCloudImage = mNativePointCloudImage; + + mLock.unlock(); + + return _nativePointCloudImage; +} + void AzureCamera::setControlValue(const std::string& pControlName) throw (dab::Exception) { @@ -232,6 +298,8 @@ AzureCamera::capture() throw (dab::Exception) { try { + mLock.lock(); + AzureTools::get().capture(mSelf); bool success; @@ -271,9 +339,13 @@ AzureCamera::capture() throw (dab::Exception) mCaptureStill = false; } + + mLock.unlock(); } catch (dab::Exception& e) { + mLock.unlock(); + e += dab::Exception("TRACKER ERROR: failed to capture camera frame", __FILE__, __FUNCTION__, __LINE__); throw e; } diff --git a/src/dab_tracker_azure_camera.h b/src/dab_tracker_azure_camera.h index 423887d..0e4ec47 100644 --- a/src/dab_tracker_azure_camera.h +++ b/src/dab_tracker_azure_camera.h @@ -3,6 +3,7 @@ #include #include +#include #include #include #include @@ -38,10 +39,19 @@ namespace dab ~AzureCamera(); + bool isColorMode() const; + bool isDepthMode() const; + bool isPointCloudMode() const; + ofxCvImage* colorFrame(); ofxCvImage* depthFrame(); PointCloud* pointCloud(); + const k4a::image nativeColorFrame(); + const k4a::image nativeDepthFrame(); + const k4a::image nativeDepthColorFrame(); + const k4a::image nativePointCloudFrame(); + void setControlValue(const std::string& pControlName) throw (dab::Exception); void setControlValue(const std::string& pControlName, const AbstractValue& pValue) throw (dab::Exception); @@ -68,7 +78,11 @@ namespace dab // camera internals for transforming depth image into color image k4a::calibration mDepthColorCalibration; k4a::transformation mDepthColorTransformation; - k4a::image mDepthColorImage; + + k4a::image mNativeColorImage; + k4a::image mNativeDepthImage; + k4a::image mNativeDepthColorImage; + k4a::image mNativePointCloudImage; // camera frame ofxCvColorImage* mColorFrame; @@ -86,6 +100,8 @@ namespace dab bool mDepthMode; bool mPointCloudMode; + std::mutex mLock; + std::shared_ptr mSelf; void init() throw (dab::Exception); diff --git a/src/dab_tracker_azure_tools.cpp b/src/dab_tracker_azure_tools.cpp index d7c54bd..f642b77 100644 --- a/src/dab_tracker_azure_tools.cpp +++ b/src/dab_tracker_azure_tools.cpp @@ -72,6 +72,8 @@ AzureTools::capture(std::shared_ptr pCamera) throw (dab::Exception) const k4a::image colorImage = pCamera->mNativeCapture.get_color_image(); if(colorImage.is_valid() == true) { + pCamera->mNativeColorImage = colorImage; + try { updateColorFrame(pCamera, colorImage); @@ -92,6 +94,8 @@ AzureTools::capture(std::shared_ptr pCamera) throw (dab::Exception) if (depthImage.is_valid() == true) { + pCamera->mNativeDepthImage = depthImage; + try { updateDepthFrame(pCamera, depthImage); @@ -113,12 +117,22 @@ AzureTools::capture(std::shared_ptr pCamera) throw (dab::Exception) if (colorImage.is_valid() == true && depthImage.is_valid() == true) { - pCamera->mDepthColorTransformation.depth_image_to_color_camera(depthImage, &pCamera->mDepthColorImage); + pCamera->mNativeColorImage = colorImage; + pCamera->mNativeDepthImage = depthImage; + + pCamera->mDepthColorTransformation.depth_image_to_color_camera(depthImage, &pCamera->mNativeDepthColorImage); try { updateColorFrame(pCamera, colorImage); - updateDepthFrame(pCamera, pCamera->mDepthColorImage); + updateDepthFrame(pCamera, pCamera->mNativeDepthColorImage); + + if (pCamera->mPointCloudMode == true) + { + pCamera->mDepthColorTransformation.depth_image_to_point_cloud(pCamera->mNativeDepthColorImage, K4A_CALIBRATION_TYPE_COLOR, &(pCamera->mNativePointCloudImage)); + + updatePointCloud(pCamera, pCamera->mNativePointCloudImage); + } if (pCamera->mMovieRecording == true) pCamera->mNativeMovieRecorder.write_capture(pCamera->mNativeCapture); } @@ -246,6 +260,36 @@ AzureTools::updateDepthFrame(std::shared_ptr pCamera, const k4a::im } } +void +AzureTools::updatePointCloud(std::shared_ptr pCamera, const k4a::image& pNativePointCloudFrame) throw (dab::Exception) +{ + PointCloud* pointCloud = pCamera->mPointCloud; + + int frameWidth = pNativePointCloudFrame.get_width_pixels(); + int frameHeight = pNativePointCloudFrame.get_height_pixels(); + unsigned int pointCount = frameWidth * frameHeight; + + if (pointCloud->size()[0] != pointCount) pointCloud->setSize({ pointCount }); + + std::vector& pointCloudPoints = pointCloud->points(); + int16_t *pointCloudImageData = (int16_t *)(void *)pNativePointCloudFrame.get_buffer(); + + for (int pI = 0; pI < pointCount; ++pI) + { + glm::vec3 point; + point.x = pointCloudImageData[3 * pI + 0] / 1000.0f; + point.y = pointCloudImageData[3 * pI + 1] / 1000.0f; + point.z = pointCloudImageData[3 * pI + 2] / 1000.0f; + + //if (point.z == 0) + //{ + // continue; + //} + + pointCloudPoints[pI] = point; + } +} + void AzureTools::startMovieRecording(std::shared_ptr pCamera, const std::string& pFileName) throw (dab::Exception) { @@ -543,11 +587,6 @@ AzureTools::createCameraFrames(std::shared_ptr pCamera) throw (dab: pCamera->mDepthFrame->allocate(frameSize[0], frameSize[1]); pCamera->mFrameSize = frameSize; - - if (pCamera->mPointCloudMode == true) - { - pCamera->mPointCloud = new PointCloud(frameSize); - } } } else if(colorFrameSuccess == true) @@ -567,7 +606,7 @@ AzureTools::createCameraFrames(std::shared_ptr pCamera) throw (dab: pCamera->mDepthColorTransformation = k4a::transformation(pCamera->mDepthColorCalibration); - pCamera->mDepthColorImage = k4a::image::create( + pCamera->mNativeDepthColorImage = k4a::image::create( K4A_IMAGE_FORMAT_DEPTH16, frameSize[0], frameSize[1], @@ -580,7 +619,15 @@ AzureTools::createCameraFrames(std::shared_ptr pCamera) throw (dab: if (pCamera->mPointCloudMode == true) { - pCamera->mPointCloud = new PointCloud(frameSize); + std::cout << "create point cloud\n"; + + pCamera->mNativePointCloudImage = k4a::image::create( + K4A_IMAGE_FORMAT_CUSTOM, + frameSize[0], + frameSize[1], + frameSize[0] * 3 * (int)sizeof(int16_t)); + + pCamera->mPointCloud = new PointCloud({ frameSize[0] * frameSize[1] }); } depthFrameSuccess = true; @@ -649,6 +696,33 @@ AzureTools::initControls(std::shared_ptr pCamera) throw (dab::Excep maxDepthControl->registerParameterListener(pCamera); pCamera->mControls["maxdepth"] = maxDepthControl; } + + // Color Camera Controls + if (pCamera->mColorMode == true && pCamera->mMoviePlaying == false) + { + const k4a::image colorImage = pCamera->mNativeCapture.get_color_image(); + if (colorImage.is_valid() == true) + { + // brightness + int brightness_value; + k4a_color_control_mode_t brightness_mode; + + pCamera->mNativeDevice.get_color_control(K4A_COLOR_CONTROL_BRIGHTNESS, &brightness_mode, &brightness_value); + Parameter* brightnessControl = new Parameter("brightness", brightness_value, 0, 200); + brightnessControl->registerParameterListener(pCamera); + pCamera->mControls["brightness"] = brightnessControl; + + // exposure + int exposure_value; + k4a_color_control_mode_t exposure_mode; + + pCamera->mNativeDevice.get_color_control(K4A_COLOR_CONTROL_EXPOSURE_TIME_ABSOLUTE, &exposure_mode, &exposure_value); + Parameter* exposureControl = new Parameter("exposure", exposure_value, 0, 33330); + exposureControl->registerParameterListener(pCamera); + pCamera->mControls["exposure"] = exposureControl; + } + + } } void @@ -668,6 +742,22 @@ AzureTools::setControlValue(const std::string& pControlName, const AbstractValue pCamera->mMaxDepth = maxdepth; } + else if (pControlName == "brightness") + { + int brightness_value = pValue.get(); + k4a_color_control_mode_t brightness_mode = K4A_COLOR_CONTROL_MODE_MANUAL; + std::cout << "brightness " << brightness_value << "\n"; + + pCamera->mNativeDevice.set_color_control(K4A_COLOR_CONTROL_BRIGHTNESS, brightness_mode, static_cast(brightness_value)); + } + else if (pControlName == "exposure") + { + int exposure_value = pValue.get(); + k4a_color_control_mode_t exposure_mode = K4A_COLOR_CONTROL_MODE_MANUAL; + std::cout << "exposure " << exposure_value << "\n"; + + pCamera->mNativeDevice.set_color_control(K4A_COLOR_CONTROL_EXPOSURE_TIME_ABSOLUTE, exposure_mode, static_cast(exposure_value)); + } } catch (Exception& e) { diff --git a/src/dab_tracker_azure_tools.h b/src/dab_tracker_azure_tools.h index f6ad9b3..a2c4f6b 100644 --- a/src/dab_tracker_azure_tools.h +++ b/src/dab_tracker_azure_tools.h @@ -54,7 +54,7 @@ namespace dab void capture(std::shared_ptr pCamera) throw (dab::Exception); void updateColorFrame(std::shared_ptr pCamera, const k4a::image& pNativeColorFrame) throw (dab::Exception); void updateDepthFrame(std::shared_ptr pCamera, const k4a::image& pNativeDepthFrame) throw (dab::Exception); - //void updatePointCloud(std::shared_ptr pCamera, rs2::depth_frame& pNativeDepthFrame) throw (dab::Exception); + void updatePointCloud(std::shared_ptr pCamera, const k4a::image& pNativePointCloudFrame) throw (dab::Exception); void startMovieRecording(std::shared_ptr pCamera, const std::string& pFileName) throw (dab::Exception); void stopMovieRecording(std::shared_ptr pCamera) throw (dab::Exception); diff --git a/src/dab_tracker_image_save_queue.cpp b/src/dab_tracker_image_save_queue.cpp index 4b2a8d8..0430589 100644 --- a/src/dab_tracker_image_save_queue.cpp +++ b/src/dab_tracker_image_save_queue.cpp @@ -81,7 +81,10 @@ ImageSaveQueue::saveImages() ofxCvImage* _img = mImageQueue.front(); std::string _fileName = mFileNameQueue.front(); - ofSaveImage(_img->getPixels(), _fileName); + bool success = ofSaveImage(_img->getPixels(), _fileName); + + // very hacky, since deleting an image creates a memory leak, scaling the image down to mini size alleviates this problems somewhat + _img->resize(1, 1); delete _img; diff --git a/src/dab_tracker_optical_calibration.cpp b/src/dab_tracker_optical_calibration.cpp index 2f7221f..48d0085 100644 --- a/src/dab_tracker_optical_calibration.cpp +++ b/src/dab_tracker_optical_calibration.cpp @@ -106,14 +106,14 @@ OpticalCalibration::distCoefficients() const return _distCoefficients; } -void +void OpticalCalibration::startCalibration() { mImagePoints.clear(); mCalibrationMode = CALIBRATING; } -void +void OpticalCalibration::captureCalibationPattern() { if (mCalibrationMode != CALIBRATING) return; @@ -555,12 +555,6 @@ OpticalCalibration::runCalibration() std::cout << (success ? "Calibration succeeded" : "Calibration failed") << ". avg re projection error = " << totalAvgErr; - - std::cout << "individual projection errors:\n"; - for (int i = 0; i < reprojErrs.size(); ++i) - { - std::cout << "i " << i << " " << reprojErrs[i] << "\n"; - } if(success == false) return false; diff --git a/src/dab_tracker_optical_calibration.h b/src/dab_tracker_optical_calibration.h index 2a429d0..7b823f8 100644 --- a/src/dab_tracker_optical_calibration.h +++ b/src/dab_tracker_optical_calibration.h @@ -33,7 +33,7 @@ public: void setDistCoefficients(const std::array& pDistCoefficients); void setupGui(ofxDatGui* pGui); - void update() throw (dab::Exception); + void update() throw (dab::Exception); void display() throw (dab::Exception); void saveSettings() throw (dab::Exception); void restoreSettings() throw (dab::Exception); diff --git a/src/dab_tracker_pointcloud.cpp b/src/dab_tracker_pointcloud.cpp index ef46bea..638b424 100644 --- a/src/dab_tracker_pointcloud.cpp +++ b/src/dab_tracker_pointcloud.cpp @@ -38,6 +38,12 @@ PointCloud::operator=(const PointCloud& pPointCloud) return *this; } +unsigned int +PointCloud::pointCount() const +{ + return mPoints.size(); +} + const std::vector& PointCloud::size() const { diff --git a/src/dab_tracker_pointcloud.h b/src/dab_tracker_pointcloud.h index 6b6d683..2463a61 100644 --- a/src/dab_tracker_pointcloud.h +++ b/src/dab_tracker_pointcloud.h @@ -18,15 +18,14 @@ class PointCloud public: PointCloud(); PointCloud(const std::vector& pSize); - template PointCloud(const std::array& pSize); PointCloud(const PointCloud& pPointCloud); ~PointCloud(); const PointCloud& operator=(const PointCloud& pPointCloud); + unsigned int pointCount() const; const std::vector& size() const; void setSize(const std::vector& pSize); - template void setSize(const std::array& pSize); std::vector& points(); const std::vector& points() const; @@ -37,35 +36,6 @@ protected: std::vector mPoints; }; -template -PointCloud::PointCloud(const std::array& pSize) - : mSize(DIM) -{ - int pointCount = 1; - for (int d = 0; d < DIM; ++d) - { - mSize[d] = pSize[d]; - pointCount *= mSize[d]; - } - - mPoints = std::vector(pointCount, glm::vec3(0.0, 0.0, 0.0)); -} - -template -void -PointCloud::setSize(const std::array& pSize) -{ - if (mSize.size() != DIM) mSize.resize(DIM); - - int newPointCount = 1; - for (int d = 0; d < DIM; ++d) - { - mSize[d] = pSize[d]; - newPointCount *= mSize[d]; - } - - if (mPoints.size() != newPointCount) mPoints.resize(newPointCount); -} }; diff --git a/src/dab_tracker_pointcloud_crop.cpp b/src/dab_tracker_pointcloud_crop.cpp index 94607eb..9849ec8 100644 --- a/src/dab_tracker_pointcloud_crop.cpp +++ b/src/dab_tracker_pointcloud_crop.cpp @@ -17,6 +17,10 @@ PointCloudCrop::PointCloudCrop(PointCloud* pPointCloud) , mDisplayZScale(0.06) , mDisplayZPosition(259.0) , mDisplayYRotation(180.0) + , mCloudDisplayScale({ 1.0, 1.0, 0.06 }) + , mCloudDisplayPosition({ 0.0, 0.0, 259.0 }) + , mCloudDisplayRotation({0.0, 180.0, 0.0}) + { mOutputPointCloud = new PointCloud(*mInputPointCloud); @@ -33,6 +37,9 @@ PointCloudCrop::PointCloudCrop(PointCloud* pPointCloud, const std::arrayaddSlider("ZScale", -1.0, 1.0, mDisplayZScale); - mDisplayZPositionControl = mGuiFolder->addSlider("ZPos", -1000.0, 1000.0, mDisplayZPosition); - mDisplayYRotationControl = mGuiFolder->addSlider("YRot", -360.0, 360.0, mDisplayYRotation); - - mDisplayZScaleControl->onSliderEvent(this, &PointCloudCrop::onSliderEvent); - mDisplayZPositionControl->onSliderEvent(this, &PointCloudCrop::onSliderEvent); - mDisplayYRotationControl->onSliderEvent(this, &PointCloudCrop::onSliderEvent); + mCloudDisplayScaleControl = mGuiFolder->addSlider("XScale", -100.0, 100.0, mCloudDisplayScale[0]); + mCloudDisplayScaleZControl = mGuiFolder->addSlider("ZScale", -100.0, 100.0, mCloudDisplayScale[2]); + mCloudDisplayPositionXControl = mGuiFolder->addSlider("XPos", -100.0, 100.0, mCloudDisplayPosition[0]); + mCloudDisplayPositionYControl = mGuiFolder->addSlider("YPos", -100.0, 100.0, mCloudDisplayPosition[1]); + mCloudDisplayPositionZControl = mGuiFolder->addSlider("ZPos", -1000.0, 1000.0, mCloudDisplayPosition[2]); + mCloudDisplayRotationXControl = mGuiFolder->addSlider("XRot", -360.0, 360.0, mCloudDisplayRotation[0]); + mCloudDisplayRotationYControl = mGuiFolder->addSlider("YRot", -360.0, 360.0, mCloudDisplayRotation[1]); + mCloudDisplayRotationZControl = mGuiFolder->addSlider("ZRot", -360.0, 360.0, mCloudDisplayRotation[2]); + + mCloudDisplayScaleControl->onSliderEvent(this, &PointCloudCrop::onSliderEvent); + mCloudDisplayScaleZControl->onSliderEvent(this, &PointCloudCrop::onSliderEvent); + mCloudDisplayPositionXControl->onSliderEvent(this, &PointCloudCrop::onSliderEvent); + mCloudDisplayPositionYControl->onSliderEvent(this, &PointCloudCrop::onSliderEvent); + mCloudDisplayPositionZControl->onSliderEvent(this, &PointCloudCrop::onSliderEvent); + mCloudDisplayRotationXControl->onSliderEvent(this, &PointCloudCrop::onSliderEvent); + mCloudDisplayRotationYControl->onSliderEvent(this, &PointCloudCrop::onSliderEvent); + mCloudDisplayRotationZControl->onSliderEvent(this, &PointCloudCrop::onSliderEvent); + + //mDisplayZScaleControl->onSliderEvent(this, &PointCloudCrop::onSliderEvent); + //mDisplayZPositionControl->onSliderEvent(this, &PointCloudCrop::onSliderEvent); + //mDisplayYRotationControl->onSliderEvent(this, &PointCloudCrop::onSliderEvent); std::cout << "setup crop size controls " << mCropSize[0] << " " << mCropSize[1] << " " << mCropSize[2] << "\n"; std::cout << "min pos " << mMinPos[0] << " " << mMinPos[1] << " " << mMinPos[2] << "\n"; @@ -240,9 +261,15 @@ PointCloudCrop::display() throw (dab::Exception) glPushMatrix(); - glTranslatef(static_cast(mDisplaySize[0]) / 2.0, static_cast(mDisplaySize[1]) / 2.0, mDisplayZPosition); - glScalef(static_cast(mDisplaySize[0]) / cloudSize[0], -1.0 * static_cast(mDisplaySize[1]) / cloudSize[1], mDisplayZScale); - glRotatef(mDisplayYRotation, 0.0, 1.0, 0.0); + glTranslatef(-static_cast(mDisplaySize[0]) / 2.0 + mCloudDisplayPosition[0], -static_cast(mDisplaySize[1]) / 2.0 + mCloudDisplayPosition[1], -mCloudDisplayPosition[2]); + glScalef(mCloudDisplayScale[0], mCloudDisplayScale[1], mCloudDisplayScale[2]); + glRotatef(mCloudDisplayRotation[0], 1.0, 0.0, 0.0); + glRotatef(mCloudDisplayRotation[1], 0.0, 1.0, 0.0); + glRotatef(mCloudDisplayRotation[2], 0.0, 0.0, 2.0); + + //glTranslatef(static_cast(mDisplaySize[0]) / 2.0, static_cast(mDisplaySize[1]) / 2.0, mDisplayZPosition); + //glScalef(static_cast(mDisplaySize[0]) / cloudSize[0], -1.0 * static_cast(mDisplaySize[1]) / cloudSize[1], mDisplayZScale); + //glRotatef(mDisplayYRotation, 0.0, 1.0, 0.0); glColor4f(1.0, 0.0, 0.0, 0.2); inputCloudMesh.draw(); @@ -310,20 +337,39 @@ PointCloudCrop::restoreSettings() throw (dab::Exception) void PointCloudCrop::onSliderEvent(ofxDatGuiSliderEvent e) { - if (e.target == mDisplayZScaleControl) + if (e.target == mCloudDisplayScaleControl) + { + mCloudDisplayScale[0] = e.value; + mCloudDisplayScale[1] = e.value; + } + if (e.target == mCloudDisplayScaleZControl) { float zscale = e.value; - mDisplayZScale = zscale; + mCloudDisplayScale[2] = zscale; + } + else if (e.target == mCloudDisplayPositionXControl) + { + mCloudDisplayPosition[0] = e.value; + } + else if (e.target == mCloudDisplayPositionYControl) + { + mCloudDisplayPosition[1] = e.value; + } + else if (e.target == mCloudDisplayPositionZControl) + { + mCloudDisplayPosition[2] = e.value; + } + else if (e.target == mCloudDisplayRotationXControl) + { + mCloudDisplayRotation[0] = e.value; } - else if (e.target == mDisplayZPositionControl) + else if (e.target == mCloudDisplayRotationYControl) { - float zpos = e.value; - mDisplayZPosition = zpos; + mCloudDisplayRotation[1] = e.value; } - else if (e.target == mDisplayYRotationControl) + else if (e.target == mCloudDisplayRotationZControl) { - float yrot = e.value; - mDisplayYRotation = yrot; + mCloudDisplayRotation[2] = e.value; } else if (e.target == mCropSizeXControl) { diff --git a/src/dab_tracker_pointcloud_crop.h b/src/dab_tracker_pointcloud_crop.h index 45a2614..1bed935 100644 --- a/src/dab_tracker_pointcloud_crop.h +++ b/src/dab_tracker_pointcloud_crop.h @@ -52,12 +52,21 @@ protected: float mDisplayZScale; float mDisplayZPosition; float mDisplayYRotation; + + std::array mCloudDisplayScale; + std::array mCloudDisplayPosition; + std::array mCloudDisplayRotation; // Gui ofxDatGuiFolder* mGuiFolder; - ofxDatGuiSlider* mDisplayZScaleControl; - ofxDatGuiSlider* mDisplayZPositionControl; - ofxDatGuiSlider* mDisplayYRotationControl; + ofxDatGuiSlider* mCloudDisplayScaleControl; + ofxDatGuiSlider* mCloudDisplayScaleZControl; + ofxDatGuiSlider* mCloudDisplayPositionXControl; + ofxDatGuiSlider* mCloudDisplayPositionYControl; + ofxDatGuiSlider* mCloudDisplayPositionZControl; + ofxDatGuiSlider* mCloudDisplayRotationXControl; + ofxDatGuiSlider* mCloudDisplayRotationYControl; + ofxDatGuiSlider* mCloudDisplayRotationZControl; ofxDatGuiSlider* mCropSizeXControl; ofxDatGuiSlider* mCropSizeYControl; diff --git a/src/dab_tracker_pointcloud_render.cpp b/src/dab_tracker_pointcloud_render.cpp new file mode 100644 index 0000000..d947873 --- /dev/null +++ b/src/dab_tracker_pointcloud_render.cpp @@ -0,0 +1,167 @@ +/** \file dab_tracker_pointcloud_render.cpp +*/ + +#include "dab_tracker_pointcloud_render.h" +#include + +using namespace dab; +using namespace dab::tracker; + +PointCloudRender::PointCloudRender(PointCloud* pPointCloud) + : mPointCloud(pPointCloud) + , mMinPos({ 0.0, 0.0, 0.0 }) + , mMaxPos({ 1.0, 1.0, 1.0 }) + , mCloudDisplayScale(100.0) + , mCloudDisplayRotation({0.0, 180.0, 0.0}) +{ + mSettingsFileName = "Tracker_PointCloudRender_Settings.json"; +} + +PointCloudRender::PointCloudRender(PointCloud* pPointCloud, const std::array& pMinPos, const std::array& pMaxPos) + : mPointCloud(pPointCloud) + , mMinPos(pMinPos) + , mMaxPos(pMaxPos) + , mCloudDisplayScale(100.0) + , mCloudDisplayRotation({ 0.0, 180.0, 0.0 }) +{ + mSettingsFileName = "Tracker_PointCloudRender_Settings.json"; +} + +PointCloudRender::~PointCloudRender() +{} + +const std::array& +PointCloudRender::minPos() const +{ + return mMinPos; +} + +const std::array& +PointCloudRender::maxPos() const +{ + return mMaxPos; +} + +PointCloud* +PointCloudRender::pointCloud() +{ + return mPointCloud; +} + +void +PointCloudRender::setMinPos(const std::array& pMinPos) +{ + mMinPos = pMinPos; +} + +void +PointCloudRender::setMaxPos(const std::array& pMaxPos) +{ + mMaxPos = pMaxPos; +} + +void +PointCloudRender::setupGui(ofxDatGui* pGui) +{ + mGuiFolder = pGui->addFolder("PointCloud"); + + mCloudDisplayScaleControl = mGuiFolder->addSlider("Scale", 20.0, 500.0, mCloudDisplayScale); + mCloudDisplayRotationXControl = mGuiFolder->addSlider("XRot", -360.0, 360.0, mCloudDisplayRotation[0]); + mCloudDisplayRotationYControl = mGuiFolder->addSlider("YRot", -360.0, 360.0, mCloudDisplayRotation[1]); + mCloudDisplayRotationZControl = mGuiFolder->addSlider("ZRot", -360.0, 360.0, mCloudDisplayRotation[2]); + + mCloudDisplayScaleControl->onSliderEvent(this, &PointCloudRender::onSliderEvent); + mCloudDisplayRotationXControl->onSliderEvent(this, &PointCloudRender::onSliderEvent); + mCloudDisplayRotationYControl->onSliderEvent(this, &PointCloudRender::onSliderEvent); + mCloudDisplayRotationZControl->onSliderEvent(this, &PointCloudRender::onSliderEvent); + +} + +void +PointCloudRender::update() throw (dab::Exception) +{} + +void +PointCloudRender::display() throw (dab::Exception) +{ + if (mDisplay == false) return; + + mPointCloudMesh = ofMesh(OF_PRIMITIVE_POINTS, mPointCloud->points()); + std::array cloudSize = { mMaxPos[0] - mMinPos[0], mMaxPos[1] - mMinPos[1], mMaxPos[2] - mMinPos[2] }; + //std::cout << "cloud size " << cloudSize[0] << " " << cloudSize[1] << " " << cloudSize[2] << "\n"; + //std::array cloudSize = { 10000.0, 10000.0, 10000.0 }; + //std::array screenScale = { 640.0 / cloudSize[0], 320.0 / cloudSize[1], 1.0 }; + + ofFbo displayFbo; + displayFbo.allocate(mDisplaySize[0], mDisplaySize[1]); + displayFbo.begin(); + + glClearColor(0.0, 0.0, 0.0, 1.0); + glClear(GL_COLOR_BUFFER_BIT | GL_DEPTH_BUFFER_BIT); + + glColor4f(1.0, 1.0, 1.0, 0.5); + glPointSize(2); + + glFogi(GL_FOG_MODE, GL_LINEAR); + GLfloat fogColor[4] = { 1.0f, 0.0f, 0.0f, 1.0f }; + glFogfv(GL_FOG_COLOR, fogColor); + glFogf(GL_FOG_DENSITY, 0.35f); + glFogf(GL_FOG_START, 1.0f); + glFogf(GL_FOG_END, 1000.0f); + glEnable(GL_FOG); + + glPushMatrix(); + + glTranslatef(mDisplaySize[0] / 2.0, mDisplaySize[1] / 2.0, -100.0); + glTranslatef(0.0, 0.0, 150.0); + glRotatef(mCloudDisplayRotation[0], 1.0, 0.0, 0.0); + glRotatef(mCloudDisplayRotation[1], 0.0, 1.0, 0.0); + glRotatef(mCloudDisplayRotation[2], 0.0, 0.0, 1.0); + glTranslatef(0, 0.0, -150.0); + glScalef(-mCloudDisplayScale, mCloudDisplayScale, mCloudDisplayScale); + + mPointCloudMesh.draw(); + + glPopMatrix(); + + glDisable(GL_FOG); + glColor4f(1.0, 1.0, 1.0, 1.0); + glPointSize(1); + + displayFbo.end(); + + displayFbo.draw(mDisplayPosition[0], mDisplayPosition[1], mDisplaySize[0], mDisplaySize[1]); +} + +void +PointCloudRender::saveSettings() throw (dab::Exception) +{ + // TODO +} + +void +PointCloudRender::restoreSettings() throw (dab::Exception) +{ + // TODO +} + +void +PointCloudRender::onSliderEvent(ofxDatGuiSliderEvent e) +{ + if (e.target == mCloudDisplayScaleControl) + { + mCloudDisplayScale = e.value; + } + else if (e.target == mCloudDisplayRotationXControl) + { + mCloudDisplayRotation[0] = e.value; + } + else if (e.target == mCloudDisplayRotationYControl) + { + mCloudDisplayRotation[1] = e.value; + } + else if (e.target == mCloudDisplayRotationZControl) + { + mCloudDisplayRotation[2] = e.value; + } +} \ No newline at end of file diff --git a/src/dab_tracker_pointcloud_render.h b/src/dab_tracker_pointcloud_render.h new file mode 100644 index 0000000..6930e9b --- /dev/null +++ b/src/dab_tracker_pointcloud_render.h @@ -0,0 +1,61 @@ +/** \file dab_tracker_pointcloud_render.h +*/ + +#pragma once + +#include +#include +#include "dab_tracker_pointcloud.h" +#include "dab_tracker_video_processor.h" +#include "ofMesh.h" + +namespace dab +{ + + namespace tracker + { + + class PointCloudRender : public VideoProcessor + { + public: + PointCloudRender(PointCloud* pPointCloud); + PointCloudRender(PointCloud* pPointCloud, const std::array& pMinPos, const std::array& pMaxPos); + ~PointCloudRender(); + + const std::array& minPos() const; + const std::array& maxPos() const; + PointCloud* pointCloud(); + + void setMinPos(const std::array& pMinPos); + void setMaxPos(const std::array& pMaxPos); + + void setupGui(ofxDatGui* pGui); + void update() throw (dab::Exception); + void display() throw (dab::Exception); + void saveSettings() throw (dab::Exception); + void restoreSettings() throw (dab::Exception); + + protected: + void onSliderEvent(ofxDatGuiSliderEvent e); + + std::array mMinPos; + std::array mMaxPos; + + PointCloud* mPointCloud; + ofMesh mPointCloudMesh; + + // Display + float mCloudDisplayScale; + std::array mCloudDisplayRotation; + + // Gui + ofxDatGuiFolder* mGuiFolder; + ofxDatGuiSlider* mCloudDisplayScaleControl; + ofxDatGuiSlider* mCloudDisplayRotationXControl; + ofxDatGuiSlider* mCloudDisplayRotationYControl; + ofxDatGuiSlider* mCloudDisplayRotationZControl; + }; + + }; + +}; \ No newline at end of file diff --git a/src/dab_tracker_realsense_tools.cpp b/src/dab_tracker_realsense_tools.cpp index 2cd5e14..aa09fd3 100644 --- a/src/dab_tracker_realsense_tools.cpp +++ b/src/dab_tracker_realsense_tools.cpp @@ -196,7 +196,7 @@ RealSenseTools::initCamera(std::shared_ptr pCamera) throw (dab: if (pCamera->mPointCloudMode == true) { - pCamera->mPointCloud = new PointCloud(frameSize); + pCamera->mPointCloud = new PointCloud({ frameSize[0], frameSize[1] }); } } } @@ -327,17 +327,45 @@ RealSenseTools::initControls(std::shared_ptr pCamera) throw (da maxDepthControl->registerParameterListener(pCamera); pCamera->mControls["maxdepth"] = maxDepthControl; - // IR Emitter Enabled if (pCamera->mMoviePlaying == false) { - auto depth_sensor = pCamera->mNativeDevice.first(); - - if (depth_sensor.supports(RS2_OPTION_EMITTER_ENABLED)) + std::vector cameraSensors = pCamera->mNativeDevice.query_sensors(); + for (int sI = 0; sI < cameraSensors.size(); ++sI) { - bool emitterActive = true; - Parameter* emitterControl = new Parameter("emitter", emitterActive, false, true); - emitterControl->registerParameterListener(pCamera); - pCamera->mControls["emitter"] = emitterControl; + std::string sensorName = cameraSensors[sI].get_info(RS2_CAMERA_INFO_NAME); + + std::cout << "s " << sI << " name " << sensorName << "\n"; + + if (sensorName == "Stereo Module") + { + rs2::sensor& depthSensor = cameraSensors[sI]; + depthSensor.set_option(RS2_OPTION_ENABLE_AUTO_EXPOSURE, 0); + + bool emitterActive = true; + Parameter* emitterControl = new Parameter("emitter", emitterActive, false, true); + emitterControl->registerParameterListener(pCamera); + pCamera->mControls["emitter"] = emitterControl; + } + + if (sensorName == "RGB Camera") + { + rs2::sensor& colorSensor = cameraSensors[sI]; + colorSensor.set_option(RS2_OPTION_ENABLE_AUTO_EXPOSURE, 0); + + int exposure = colorSensor.get_option(RS2_OPTION_EXPOSURE); + //std::cout << "color exposure " << exposure << "\n"; + + int gain = colorSensor.get_option(RS2_OPTION_GAIN); + //std::cout << "color gain " << gain << "\n"; + + Parameter* exposureControl = new Parameter("exposure", exposure, 0, 1000); + exposureControl->registerParameterListener(pCamera); + pCamera->mControls["exposure"] = exposureControl; + + Parameter* gainControl = new Parameter("gain", gain, 0, 100); + gainControl->registerParameterListener(pCamera); + pCamera->mControls["gain"] = gainControl; + } } } } @@ -378,10 +406,52 @@ RealSenseTools::setControlValue(const std::string& pControlName, const AbstractV } else if (pControlName == "emitter") { - bool emitterActive = pValue.get(); - auto depth_sensor = pCamera->mNativeDevice.first(); - depth_sensor.set_option(RS2_OPTION_EMITTER_ENABLED, static_cast(emitterActive)); + std::vector cameraSensors = pCamera->mNativeDevice.query_sensors(); + for (int sI = 0; sI < cameraSensors.size(); ++sI) + { + std::string sensorName = cameraSensors[sI].get_info(RS2_CAMERA_INFO_NAME); + if (sensorName == "Stereo Module") + { + rs2::sensor& depthSensor = cameraSensors[sI]; + + bool emitterActive = pValue.get(); + depthSensor.set_option(RS2_OPTION_EMITTER_ENABLED, static_cast(emitterActive)); + } + } + } + else if (pControlName == "exposure") + { + std::vector cameraSensors = pCamera->mNativeDevice.query_sensors(); + for (int sI = 0; sI < cameraSensors.size(); ++sI) + { + std::string sensorName = cameraSensors[sI].get_info(RS2_CAMERA_INFO_NAME); + if (sensorName == "RGB Camera") + { + rs2::sensor& colorSensor = cameraSensors[sI]; + + int exposure = pValue.get(); + colorSensor.set_option(RS2_OPTION_EXPOSURE, exposure); + } + } } + else if (pControlName == "gain") + { + std::vector cameraSensors = pCamera->mNativeDevice.query_sensors(); + for (int sI = 0; sI < cameraSensors.size(); ++sI) + { + std::string sensorName = cameraSensors[sI].get_info(RS2_CAMERA_INFO_NAME); + if (sensorName == "RGB Camera") + { + rs2::sensor& colorSensor = cameraSensors[sI]; + + int gain = pValue.get(); + colorSensor.set_option(RS2_OPTION_GAIN, gain); + } + } + } + + + //else if (pControlName == "skeletonsmoothing") //{ // if (pCamera->mSkeletonTracker == nullptr) throw Exception("TRACKER ERROR: camera contains no skeleton tracker", __FILE__, __FUNCTION__, __LINE__); @@ -611,7 +681,7 @@ RealSenseTools::updatePointCloud(std::shared_ptr pCamera, rs2:: const std::vector& cloudSize = pCamera->mPointCloud->size(); std::array nativeFrameSize = { pNativeDepthFrame.get_width(), pNativeDepthFrame.get_height() }; - if (cloudSize[0] != nativeFrameSize[0] || cloudSize[1] != nativeFrameSize[1]) pCamera->mPointCloud->setSize(nativeFrameSize); + if (cloudSize[0] != nativeFrameSize[0] || cloudSize[1] != nativeFrameSize[1]) pCamera->mPointCloud->setSize({ nativeFrameSize[0], nativeFrameSize[1] }); pCamera->mNativePoints = pCamera->mNativePointCloud.calculate(pNativeDepthFrame); diff --git a/src/dab_tracker_video_render.cpp b/src/dab_tracker_video_render.cpp new file mode 100644 index 0000000..f27df29 --- /dev/null +++ b/src/dab_tracker_video_render.cpp @@ -0,0 +1,51 @@ +/** \file dab_tracker_video_render.cpp +*/ + +#include +#include "dab_tracker_video_render.h" +#include "ofxCvColorImage.h" +#include "ofxCvGrayscaleImage.h" +#include "ofxCvFloatImage.h" +#include "ofxCvShortImage.h" +#include "dab_tracker_serialize_helper.h" + +using namespace dab; +using namespace dab::tracker; + +VideoRender::VideoRender(ofxCvImage* pInputImage) + : mInputImage(pInputImage) +{ + mSettingsFileName = "Tracker_VideoRender_Settings.json"; +} + +VideoRender::~VideoRender() +{} + +ofxCvImage* +VideoRender::outputImage() +{ + return mInputImage; +} + +void +VideoRender::setupGui(ofxDatGui* pGui) +{} + +void +VideoRender::update() throw (dab::Exception) +{} + +void +VideoRender::display() throw (dab::Exception) +{ + if (mDisplay == false) return; + mInputImage->draw(mDisplayPosition[0], mDisplayPosition[1], mDisplaySize[0], mDisplaySize[1]); +} + +void +VideoRender::saveSettings() throw (dab::Exception) +{} + +void +VideoRender::restoreSettings() throw (dab::Exception) +{} diff --git a/src/dab_tracker_video_render.h b/src/dab_tracker_video_render.h new file mode 100644 index 0000000..ab067bf --- /dev/null +++ b/src/dab_tracker_video_render.h @@ -0,0 +1,37 @@ +/** \file dab_tracker_video_render.h +*/ + +#pragma once + +#include +#include +#include "ofxCVImage.h" +#include "dab_tracker_video_processor.h" + +namespace dab +{ + + namespace tracker + { + + class VideoRender : public VideoProcessor + { + public: + VideoRender(ofxCvImage* pInputImage); + ~VideoRender(); + + ofxCvImage* outputImage(); + + void setupGui(ofxDatGui* pGui); + void update() throw (dab::Exception); + void display() throw (dab::Exception); + void saveSettings() throw (dab::Exception); + void restoreSettings() throw (dab::Exception); + + protected: + ofxCvImage* mInputImage; + }; + + }; + +};