movie capture with timestamps and calibration improvements

addon_config.mk

@@ -77,18 +77,25 @@ osx:
 	ADDON_INCLUDES_EXCLUDE += src/dab_tracker_zed_%
 
 vs:
+	ADDON_INCLUDES =
 	ADDON_INCLUDES += "C:/Program Files/NVIDIA GPU Computing Toolkit/CUDA/v9.1/include/"
+	ADDON_INCLUDES += libs/astra/include
+	ADDON_INCLUDES += libs/azure/include
+	ADDON_INCLUDES += libs/gige/include
+	ADDON_INCLUDES += libs/nuitrack/include
+	ADDON_INCLUDES += libs/realsense/include
+	ADDON_INCLUDES += libs/ximea/include
 	ADDON_LIBS =
 	ADDON_LIBS += "C:/Program Files/NVIDIA GPU Computing Toolkit/CUDA/v9.1/lib/x64/"
 	ADDON_LIBS += libs/astra/lib/win/astra.lib
 	ADDON_LIBS += libs/astra/lib/win/astra_core.lib
 	ADDON_LIBS += libs/astra/lib/win/astra_core_api.lib
 	ADDON_LIBS += libs/astra/lib/win/OpenNI2.lib
+	ADDON_LIBS += libs/azure/lib/win/win64/k4a.lib
+	ADDON_LIBS += libs/azure/lib/win/win64/k4arecord.lib
 	ADDON_LIBS += libs/gige/lib/win/win64/ImageLib.lib
 	ADDON_LIBS += libs/gige/lib/win/win64/PvAPI.lib
 	ADDON_LIBS += libs/nuitrack/lib/win64/middleware.lib
 	ADDON_LIBS += libs/nuitrack/lib/win64/nuitrack.lib
 	ADDON_LIBS += libs/realsense/lib/win/win64/realsense2.lib
-	ADDON_LIBS += libs/zed/lib/win/win64/sl_core64.lib
-	ADDON_LIBS += libs/zed/lib/win/win64/sl_input64.lib
-	ADDON_LIBS += libs/zed/lib/win/win64/sl_zed64.lib
+	ADDON_LIBS += libs/ximea/lib/win/win64/xiapi64.lib

src/dab_tracker_camera.cpp

@@ -22,7 +22,8 @@ Camera::Camera()
 , mColorConversionFrame(nullptr)
 , mMoviePlayer(nullptr)
 , mMovieRecorder(nullptr)
-, mMovieRecordFileName("capture.mp4")
+, mMovieRecordFileName("capture")
+, mMovieRecordFileSuffix("mp4")
 , mBlocked(false)
 , mCapturing(false)
 , mLoadSettings(false)
@@ -44,7 +45,8 @@ Camera::Camera(std::array<unsigned int, 2> pFrameSize, unsigned int pChannelCoun
 , mColorConversionFrame(nullptr)
 , mMoviePlayer(nullptr)
 , mMovieRecorder(nullptr)
-, mMovieRecordFileName("capture.mp4")
+, mMovieRecordFileName("capture")
+, mMovieRecordFileSuffix("mp4")
 , mBlocked(false)
 , mCapturing(false)
 , mLoadSettings(false)
@@ -382,7 +384,7 @@ Camera::startRecording()  throw (dab::Exception)
     if(mMovieRecording == true) return;
 
     bool success;
-    
+
     if( mMovieRecorder == nullptr )
     {
         mMovieRecorder = new ofxVideoRecorder();
@@ -391,7 +393,9 @@ Camera::startRecording()  throw (dab::Exception)
         //mMovieRecorder->setVideoBitrate("10000k");
     }
 
-    success = mMovieRecorder->setup(mMovieRecordFileName, mFrameSize[0], mFrameSize[1], mFrameRate);
+	std::string fileName = mMovieRecordFileName + "_" + ofGetTimestampString() + "." + mMovieRecordFileSuffix;
+
+    success = mMovieRecorder->setup(fileName, mFrameSize[0], mFrameSize[1], mFrameRate);
     if(success == false) throw dab::Exception("TRACKER ERROR: failed to start recording to file " + mMovieRecordFileName, __FILE__, __FUNCTION__, __LINE__);
     mMovieRecorder->start();
 
@@ -410,7 +414,7 @@ Camera::stopRecording() throw (dab::Exception)
 }
 
 void 
-Camera::captureStill(const std::string& pFileName) throw (dab::Exception)
+Camera::recordStill(const std::string& pFileName) throw (dab::Exception)
 {
 	mCaptureStillFileName = pFileName;
 	mCaptureStill = true;

src/dab_tracker_camera.h

@@ -61,7 +61,7 @@ public:
     virtual void startRecording() throw (dab::Exception);
     virtual void stopRecording() throw (dab::Exception);
 
-	void captureStill(const std::string& pFileName) throw (dab::Exception);
+	void recordStill(const std::string& pFileName) throw (dab::Exception);
 	void saveStill(ofxCvImage* pImage, const std::string& pFileName) throw (dab::Exception);
 
     virtual void saveSettings() throw (dab::Exception);
@@ -103,6 +103,7 @@ protected:
 
     // movie recording
     std::string mMovieRecordFileName;
+	std::string mMovieRecordFileSuffix;
     ofxVideoRecorder* mMovieRecorder;
 };
 

src/dab_tracker_com_manager.cpp

@@ -556,16 +556,30 @@ CameraOscControl::update(std::shared_ptr<OscMessage> pMessage)
 
 		const std::vector<_OscArg*>& arguments = pMessage->arguments();
 
-		if (address.compare("/CaptureStill") == 0) captureStill(arguments);
+		if (address.compare("/RecordStill") == 0) recordStill(arguments);
+		else if (address.compare("/StartRecording") == 0) startRecording(arguments);
+		else if (address.compare("/StopRecording") == 0) stopRecording(arguments);
 	}
 	catch (dab::Exception& e)
 	{
 		std::cout << e << "\n";
 	}
 }
 
+void
+CameraOscControl::startRecording(const std::vector<_OscArg*>& pArgs) throw (dab::Exception)
+{
+	mCamera->startRecording();
+}
+
+void
+CameraOscControl::stopRecording(const std::vector<_OscArg*>& pArgs) throw (dab::Exception)
+{
+	mCamera->stopRecording();
+}
+
 void 
-CameraOscControl::captureStill(const std::vector<_OscArg*>& pArgs) throw (dab::Exception)
+CameraOscControl::recordStill(const std::vector<_OscArg*>& pArgs) throw (dab::Exception)
 {
 	int argCount = pArgs.size();
 
@@ -574,11 +588,16 @@ CameraOscControl::captureStill(const std::vector<_OscArg*>& pArgs) throw (dab::E
 		int timeStamp = *pArgs[0];
 		std::string stillFileName = "still_" + std::to_string(timeStamp);
 
-		std::cout << "CameraOscControl::captureStill " << timeStamp << "\n";
+		std::cout << "CameraOscControl::recordStill " << timeStamp << "\n";
 
-		mCamera->captureStill(stillFileName);
+		mCamera->recordStill(stillFileName);
     }
-    else throw(dab::Exception( "ERR_OSC: Wrong Parameters for /CaptureStill", __FILE__, __FUNCTION__, __LINE__ ));
+	else if (argCount == 1 && pArgs[0]->oscType() == OSC_TYPE_STRING)
+	{
+		std::string fileName = pArgs[0]->operator const std::string&();
+		mCamera->recordStill(fileName);
+	}
+    else throw(dab::Exception( "ERR_OSC: Wrong Parameters for /RecordStill", __FILE__, __FUNCTION__, __LINE__ ));
 }
 
 #pragma mark ComManager implementation

src/dab_tracker_com_manager.h

@@ -168,7 +168,9 @@ protected:
 
 	Camera* mCamera;
 
-	void captureStill(const std::vector<_OscArg*>& pArgs) throw (dab::Exception);
+	void startRecording(const std::vector<_OscArg*>& pArgs) throw (dab::Exception);
+	void stopRecording(const std::vector<_OscArg*>& pArgs) throw (dab::Exception);
+	void recordStill(const std::vector<_OscArg*>& pArgs) throw (dab::Exception);
 
 	std::mutex mLock;
 };

src/dab_tracker_optical_calibration.cpp

@@ -27,6 +27,7 @@ OpticalCalibration::OpticalCalibration(ofxCvImage* pInputImage)
 , mCalibrationFrameCount(sCalibrationFrameCount)
 , mCaptureInterval(sCaptureInterval)
 , mPreviousCaptureTime(0.0)
+, mCapturePatternNow(false)
 {
     mFrameSize = { mInputImage->width, mInputImage->height };
 
@@ -70,6 +71,62 @@ OpticalCalibration::outputImage()
     return mOutputImage;
 }
 
+OpticalCalibration::Mode
+OpticalCalibration::calibrationMode() const
+{
+	return mCalibrationMode;
+}
+
+std::array<float, 9>
+OpticalCalibration::cameraMatrix() const
+{
+	std::array<float, 9> _camMatrix;
+	for (int rI = 0, i = 0; rI < 3; ++rI)
+	{
+		const double* _camData = mCameraMatrix.ptr<double>(rI);
+
+		for (int cI = 0; cI < 3; ++cI, ++i)
+		{
+			_camMatrix[i] = _camData[cI];
+		}
+	}
+
+	return _camMatrix;
+}
+
+std::array<float, 8>
+OpticalCalibration::distCoefficients() const
+{
+	std::array<float, 8> _distCoefficients;
+	for (int rI = 0, i = 0; rI < 8; ++rI, ++i)
+	{
+		const double* _distData = mDistCoeffs.ptr<double>(rI);
+		_distCoefficients[i] = _distData[0];
+	}
+	return _distCoefficients;
+}
+
+void 
+OpticalCalibration::setCameraMatrix(const std::array<float, 9>& pCameraMatrix)
+{
+	for(int r=0, i=0; r<3; ++r)
+	{
+		for(int c=0; c<3; ++c, ++i)
+		{
+			mCameraMatrix.at<double>(r, c) = pCameraMatrix[i];
+		}
+	}
+}
+
+void 
+OpticalCalibration::setDistCoefficients(const std::array<float, 8>& pDistCoefficients)
+{
+	for(int r=0, i=0; r<8; ++r, ++i)
+	{
+		mDistCoeffs.at<double>(r, 0) = pDistCoefficients[i];
+	}
+}
+
 void
 OpticalCalibration::setupGui(ofxDatGui* pGui)
 {
@@ -106,11 +163,15 @@ OpticalCalibration::setupGui(ofxDatGui* pGui)
 
     mFlipVerticalControl = new ofxDatGuiToggle("Flip Vertical", mFlipVertical);
     mGuiFolder->attachItem(mFlipVerticalControl);
-    mFlipVerticalControl->onButtonEvent(this, &OpticalCalibration::onButtonEvent);
+	mFlipVerticalControl->onButtonEvent(this, &OpticalCalibration::onButtonEvent);
 
 	mStartCalibrationControl = new ofxDatGuiButton("Start Calibration");
 	mGuiFolder->attachItem(mStartCalibrationControl);
 	mStartCalibrationControl->onButtonEvent(this, &OpticalCalibration::onButtonEvent);
+
+	mCapturePatternNowControl = new ofxDatGuiToggle("Capture Pattern Now");
+	mGuiFolder->attachItem(mCapturePatternNowControl);
+	mCapturePatternNowControl->onButtonEvent(this, &OpticalCalibration::onButtonEvent);
 }
 
 void
@@ -145,22 +206,24 @@ OpticalCalibration::update() throw (dab::Exception)
         }
         else
         {
-            std::vector<cv::Point2f> patternPoints;
-            bool patternFound = detectCalibrationPattern(inputMat, patternPoints);
+			double captureTime = ofGetElapsedTimeMillis();
+
+			if (mCapturePatternNow == true || (mCaptureInterval > 0.0 && captureTime - mPreviousCaptureTime > mCaptureInterval ) )
+			{
+
+				std::vector<cv::Point2f> patternPoints;
+				bool patternFound = detectCalibrationPattern(inputMat, patternPoints);
 
-            //std::cout << "patternFound " << patternFound << "\n";
+				//std::cout << "patternFound " << patternFound << "\n";
 
-            if(patternFound == true)
-            {
-                double captureTime = ofGetElapsedTimeMillis();
-
-                if( captureTime - mPreviousCaptureTime > mCaptureInterval)
-                {
-                    //std::cout << "add points to image points\n";
-
-                    mPreviousCaptureTime = captureTime;
-                    mImagePoints.push_back(patternPoints);
-                }
+				if (patternFound == true)
+				{
+					//std::cout << "add points to image points\n";   
+					mPreviousCaptureTime = captureTime;
+					mCapturePatternNow = false;
+					mCapturePatternNowControl->setChecked(mCapturePatternNow);
+					mImagePoints.push_back(patternPoints);
+				}
             }
 
             mStartCalibrationControl->setLabel("Capture Frame " + std::to_string(mImagePoints.size()) + " out of " + std::to_string(mCalibrationFrameCount));
@@ -172,7 +235,7 @@ OpticalCalibration::update() throw (dab::Exception)
         cv::Mat inputMat = cv::cvarrToMat(mInputImage->getCvImage(), false);
         cv::Mat outputMat = cv::cvarrToMat(mOutputImage->getCvImage(), false);
         cv::undistort(inputMat, outputMat, mCameraMatrix, mDistCoeffs);
-        
+
 //        void cvUndistort2(const CvArr* src, CvArr* dst, const CvMat* camera_matrix, const CvMat* distortion_coeffs, const CvMat* new_camera_matrix=0 )¶
     }
     else
@@ -328,7 +391,8 @@ OpticalCalibration::restoreSettings() throw (dab::Exception)
         mCalibrationFrameCountControl->setText(std::to_string(mCalibrationFrameCount));
         mCaptureIntervalControl->setText(std::to_string(mCaptureInterval));
         mAspectRatioControl->setText(std::to_string(mAspectRatio));
-        mFlipVerticalControl->setEnabled(mFlipVertical);
+        mFlipVerticalControl->setChecked(mFlipVertical);
+
     }
     catch(Exception& e)
     {
@@ -562,4 +626,10 @@ OpticalCalibration::onButtonEvent(ofxDatGuiButtonEvent e)
 
         //std::cout << "start calibration\n";
     }
+	else if (e.target == mCapturePatternNowControl)
+	{
+		mCapturePatternNow = static_cast<ofxDatGuiToggle*>(e.target)->getChecked();
+
+		//std::cout << "start calibration\n";
+	}
 }

src/dab_tracker_optical_calibration.h

@@ -16,20 +16,26 @@ namespace tracker
 class OpticalCalibration : public VideoProcessor
 {
 public:
+	enum Pattern { NOT_EXISTING, CHESSBOARD, CIRCLES_GRID, ASYMMETRIC_CIRCLES_GRID };
+	enum Mode { CALIBRATING, CALIBRATED };
+
     OpticalCalibration(ofxCvImage* pInputImage);
     ~OpticalCalibration();
 
     ofxCvImage* outputImage();
+	Mode calibrationMode() const;
+	std::array<float, 9> cameraMatrix() const;
+	std::array<float, 8> distCoefficients() const;
+
+	void setCameraMatrix(const std::array<float, 9>& pCameraMatrix);
+	void setDistCoefficients(const std::array<float, 8>& pDistCoefficients);
 
     void setupGui(ofxDatGui* pGui);
     void update() throw (dab::Exception);
     void display() throw (dab::Exception);
     void saveSettings() throw (dab::Exception);
     void restoreSettings() throw (dab::Exception);
 
-    enum Pattern { NOT_EXISTING, CHESSBOARD, CIRCLES_GRID, ASYMMETRIC_CIRCLES_GRID };
-    enum Mode { CALIBRATING, CALIBRATED };
-
 protected:
     static cv::Size sBoardSize;
     static Pattern sBoardPattern;
@@ -48,6 +54,7 @@ protected:
     float mAspectRatio;
     float mCaptureInterval;
     double mPreviousCaptureTime;
+	bool mCapturePatternNow;
     bool mCalibZeroTangentDist; // Assume zero tangential distortion
     bool mCalibFixPrincipalPoint;// Fix the principal point at the center
     bool mFlipVertical;          // Flip the captured images around the horizontal axis
@@ -80,6 +87,7 @@ protected:
     ofxDatGuiTextInput* mAspectRatioControl;
     ofxDatGuiToggle* mFlipVerticalControl;
     ofxDatGuiButton* mStartCalibrationControl;
+	ofxDatGuiToggle* mCapturePatternNowControl;
 
     void onTextInputEvent(ofxDatGuiTextInputEvent e);
     void onDropdownEvent(ofxDatGuiDropdownEvent e);