texture: add image mask support (#967)

apps/DensifyPointCloud/DensifyPointCloud.cpp

@@ -134,7 +134,7 @@ bool Initialize(size_t argc, LPCTSTR* argv)
 		("sub-resolution-levels", boost::program_options::value(&nSubResolutionLevels)->default_value(2), "number of patch-match sub-resolution iterations (0 - disabled)")
 		("number-views", boost::program_options::value(&nNumViews)->default_value(nNumViewsDefault), "number of views used for depth-map estimation (0 - all neighbor views available)")
 		("number-views-fuse", boost::program_options::value(&nMinViewsFuse)->default_value(3), "minimum number of images that agrees with an estimate during fusion in order to consider it inlier (<2 - only merge depth-maps)")
-		("ignore-mask-label", boost::program_options::value(&nIgnoreMaskLabel)->default_value(-1), "integer value for the label to ignore in the segmentation mask; the mask for each image is stored in the MVS scene or next to each image with '.mask.png' extension (<0 - disabled)")
+		("ignore-mask-label", boost::program_options::value(&nIgnoreMaskLabel)->default_value(-1), "label value to ignore in the image mask, stored in the MVS scene or next to each image with '.mask.png' extension (<0 - disabled)")
 		("mask-path", boost::program_options::value<std::string>(&OPT::strMaskPath), "path to folder containing mask images with '.mask.png' extension")
 		("iters", boost::program_options::value(&nEstimationIters)->default_value(numIters), "number of patch-match iterations")
 		("geometric-iters", boost::program_options::value(&nEstimationGeometricIters)->default_value(2), "number of geometric consistent patch-match iterations (0 - disabled)")

apps/TextureMesh/TextureMesh.cpp

@@ -63,6 +63,7 @@ unsigned nTextureSizeMultiple;
 unsigned nRectPackingHeuristic;
 uint32_t nColEmpty;
 float fSharpnessWeight;
+int nIgnoreMaskLabel;
 unsigned nOrthoMapResolution;
 unsigned nArchiveType;
 int nProcessPriority;
@@ -122,6 +123,7 @@ bool Initialize(size_t argc, LPCTSTR* argv)
 		("empty-color", boost::program_options::value(&OPT::nColEmpty)->default_value(0x00FF7F27), "color used for faces not covered by any image")
 		("sharpness-weight", boost::program_options::value(&OPT::fSharpnessWeight)->default_value(0.5f), "amount of sharpness to be applied on the texture (0 - disabled)")
 		("orthographic-image-resolution", boost::program_options::value(&OPT::nOrthoMapResolution)->default_value(0), "orthographic image resolution to be generated from the textured mesh - the mesh is expected to be already geo-referenced or at least properly oriented (0 - disabled)")
+		("ignore-mask-label", boost::program_options::value(&OPT::nIgnoreMaskLabel)->default_value(-1), "label value to ignore in the image mask, stored in the MVS scene or next to each image with '.mask.png' extension (-1 - auto estimate mask for lens distortion, -2 - disabled)")
 		;
 
 	// hidden options, allowed both on command line and
@@ -307,7 +309,7 @@ int main(int argc, LPCTSTR* argv)
 	TD_TIMER_START();
 	if (!scene.TextureMesh(OPT::nResolutionLevel, OPT::nMinResolution, OPT::minCommonCameras, OPT::fOutlierThreshold, OPT::fRatioDataSmoothness,
 						   OPT::bGlobalSeamLeveling, OPT::bLocalSeamLeveling, OPT::nTextureSizeMultiple, OPT::nRectPackingHeuristic, Pixel8U(OPT::nColEmpty),
-						   OPT::fSharpnessWeight, views))
+						   OPT::fSharpnessWeight, OPT::nIgnoreMaskLabel, views))
 		return EXIT_FAILURE;
 	VERBOSE("Mesh texturing completed: %u vertices, %u faces (%s)", scene.mesh.vertices.GetSize(), scene.mesh.faces.GetSize(), TD_TIMER_GET_FMT().c_str());
 

libs/MVS/DepthMap.cpp

@@ -297,8 +297,9 @@ unsigned DepthData::DecRef()
 
 // try to load and apply mask to the depth map;
 // the mask for each image is stored in the MVS scene or next to each image with '.mask.png' extension;
-// the mask marks as false pixels that should be ignored
-bool DepthEstimator::ImportIgnoreMask(const Image& image0, const Image8U::Size& size, BitMatrix& bmask, uint16_t nIgnoreMaskLabel)
+// the mask marks as false (or 0) pixels that should be ignored
+//  - pMask: optional output mask; if defined, the mask is returned in this image instead of the BitMatrix
+bool DepthEstimator::ImportIgnoreMask(const Image& image0, const Image8U::Size& size, uint16_t nIgnoreMaskLabel, BitMatrix& bmask, Image8U* pMask)
 {
 	ASSERT(image0.IsValid() && !image0.image.empty());
 	const String maskFileName(image0.maskName.empty() ? Util::getFileFullName(image0.name)+".mask.png" : image0.maskName);
@@ -308,12 +309,16 @@ bool DepthEstimator::ImportIgnoreMask(const Image& image0, const Image8U::Size&
 		return false;
 	}
 	cv::resize(mask, mask, size, 0, 0, cv::INTER_NEAREST);
-	bmask.create(size);
-	bmask.memset(0xFF);
-	for (int r=0; r<size.height; ++r) {
-		for (int c=0; c<size.width; ++c) {
-			if (mask(r,c) == nIgnoreMaskLabel)
-				bmask.unset(r,c);
+	if (pMask) {
+		*pMask = (mask != nIgnoreMaskLabel);
+	} else {
+		bmask.create(size);
+		bmask.memset(0xFF);
+		for (int r=0; r<size.height; ++r) {
+			for (int c=0; c<size.width; ++c) {
+				if (mask(r,c) == nIgnoreMaskLabel)
+					bmask.unset(r,c);
+			}
 		}
 	}
 	return true;

libs/MVS/DepthMap.h

@@ -452,7 +452,7 @@ struct MVS_API DepthEstimator {
 		ASSERT(ISEQUAL(norm(normal), 1.f));
 	}
 
-	static bool ImportIgnoreMask(const Image&, const Image8U::Size&, BitMatrix&, uint16_t nIgnoreMaskLabel);
+	static bool ImportIgnoreMask(const Image&, const Image8U::Size&, uint16_t nIgnoreMaskLabel, BitMatrix&, Image8U* =NULL);
 	static void MapMatrix2ZigzagIdx(const Image8U::Size& size, DepthEstimator::MapRefArr& coords, const BitMatrix& mask, int rawStride=16);
 
 	const float smoothBonusDepth, smoothBonusNormal;

libs/MVS/PatchMatchCUDA.cpp

@@ -404,7 +404,7 @@ void PatchMatchCUDA::EstimateDepthMap(DepthData& depthData)
 	if (OPTDENSE::nIgnoreMaskLabel >= 0) {
 		const DepthData::ViewData& view = depthData.GetView();
 		BitMatrix mask;
-		if (DepthEstimator::ImportIgnoreMask(*view.pImageData, depthData.depthMap.size(), mask, (uint16_t)OPTDENSE::nIgnoreMaskLabel))
+		if (DepthEstimator::ImportIgnoreMask(*view.pImageData, depthData.depthMap.size(), (uint16_t)OPTDENSE::nIgnoreMaskLabel, mask))
 			depthData.ApplyIgnoreMask(mask);
 	}
 

libs/MVS/Scene.h

@@ -138,7 +138,7 @@ class MVS_API Scene
 	// Mesh texturing
 	bool TextureMesh(unsigned nResolutionLevel, unsigned nMinResolution, unsigned minCommonCameras=0, float fOutlierThreshold=0.f, float fRatioDataSmoothness=0.3f,
 		bool bGlobalSeamLeveling=true, bool bLocalSeamLeveling=true, unsigned nTextureSizeMultiple=0, unsigned nRectPackingHeuristic=3, Pixel8U colEmpty=Pixel8U(255,127,39),
-		float fSharpnessWeight=0.5f, const IIndexArr& views=IIndexArr());
+		float fSharpnessWeight=0.5f, int ignoreMaskLabel = -1, const IIndexArr& views=IIndexArr());
 
 	#ifdef _USE_BOOST
 	// implement BOOST serialization

libs/MVS/SceneDensify.cpp

@@ -672,7 +672,7 @@ bool DepthMapsData::EstimateDepthMap(IIndex idxImage, int nGeometricIter)
 		#endif
 		if (prevDepthMapSize != size || OPTDENSE::nIgnoreMaskLabel >= 0) {
 			BitMatrix mask;
-			if (OPTDENSE::nIgnoreMaskLabel >= 0 && DepthEstimator::ImportIgnoreMask(*image.pImageData, depthData.depthMap.size(), mask, (uint16_t)OPTDENSE::nIgnoreMaskLabel))
+			if (OPTDENSE::nIgnoreMaskLabel >= 0 && DepthEstimator::ImportIgnoreMask(*image.pImageData, depthData.depthMap.size(), (uint16_t)OPTDENSE::nIgnoreMaskLabel, mask))
 				depthData.ApplyIgnoreMask(mask);
 			DepthEstimator::MapMatrix2ZigzagIdx(size, coords, mask, MAXF(64,(int)nMaxThreads*8));
 			#if 0 && !defined(_RELEASE)

libs/MVS/SceneTexture.cpp

@@ -151,7 +151,7 @@ struct MeshTexture {
 		typedef TRasterMesh<RasterMesh> Base;
 		FaceMap& faceMap;
 		FIndex idxFace;
-		Image8U invalidMask;
+		Image8U mask;
 		bool validFace;
 
 		RasterMesh(const Mesh::VertexArr& _vertices, const Camera& _camera, DepthMap& _depthMap, FaceMap& _faceMap)
@@ -167,7 +167,7 @@ struct MeshTexture {
 			Depth& depth = depthMap(pt);
 			if (depth == 0 || depth > z) {
 				depth = z;
-				faceMap(pt) = validFace && (validFace = (invalidMask(pt) == 0)) ? idxFace : NO_ID;
+				faceMap(pt) = validFace && (validFace = (mask(pt) != 0)) ? idxFace : NO_ID;
 			}
 		}
 	};
@@ -326,7 +326,7 @@ struct MeshTexture {
 
 	void ListVertexFaces();
 
-	bool ListCameraFaces(FaceDataViewArr&, float fOutlierThreshold, const IIndexArr& views);
+	bool ListCameraFaces(FaceDataViewArr&, float fOutlierThreshold, int nIgnoreMaskLabel, const IIndexArr& views);
 
 	#if TEXOPT_FACEOUTLIER != TEXOPT_FACEOUTLIER_NA
 	bool FaceOutlierDetection(FaceDataArr& faceDatas, float fOutlierThreshold) const;
@@ -335,7 +335,7 @@ struct MeshTexture {
 	void CreateVirtualFaces(const FaceDataViewArr& facesDatas, FaceDataViewArr& virtualFacesDatas, VirtualFaceIdxsArr& virtualFaces, unsigned minCommonCameras=2, float thMaxNormalDeviation=25.f) const;
 	IIndexArr SelectBestView(const FaceDataArr& faceDatas, FIndex fid, unsigned minCommonCameras, float ratioAngleToQuality) const;
 
-	bool FaceViewSelection(unsigned minCommonCameras, float fOutlierThreshold, float fRatioDataSmoothness, const IIndexArr& views);
+	bool FaceViewSelection(unsigned minCommonCameras, float fOutlierThreshold, float fRatioDataSmoothness, int nIgnoreMaskLabel, const IIndexArr& views);
 
 	void CreateSeamVertices();
 	void GlobalSeamLeveling();
@@ -399,33 +399,33 @@ struct MeshTexture {
 
 // creating an invalid mask for the given image corresponding to
 // the invalid pixels generated during image correction for the lens distortion;
-// the returned mask has the same size as the image and is set to non-zero for invalid pixels
+// the returned mask has the same size as the image and is set to zero for invalid pixels
 static Image8U DetectInvalidImageRegions(const Image8U3& image)
 {
 	const cv::Scalar upDiff(3);
 	const int flags(8 | (255 << 8));
-	Image8U invalidMask(image.rows + 2, image.cols + 2);
-	invalidMask.memset(0);
+	Image8U mask(image.rows + 2, image.cols + 2);
+	mask.memset(0);
 	Image8U imageGray;
 	cv::cvtColor(image, imageGray, cv::COLOR_BGR2GRAY);
 	if (imageGray(0, 0) == 0)
-		cv::floodFill(imageGray, invalidMask, cv::Point(0, 0), 255, NULL, cv::Scalar(0), upDiff, flags);
+		cv::floodFill(imageGray, mask, cv::Point(0, 0), 255, NULL, cv::Scalar(0), upDiff, flags);
 	if (imageGray(image.rows / 2, 0) == 0)
-		cv::floodFill(imageGray, invalidMask, cv::Point(0, image.rows / 2), 255, NULL, cv::Scalar(0), upDiff, flags);
+		cv::floodFill(imageGray, mask, cv::Point(0, image.rows / 2), 255, NULL, cv::Scalar(0), upDiff, flags);
 	if (imageGray(image.rows - 1, 0) == 0)
-		cv::floodFill(imageGray, invalidMask, cv::Point(0, image.rows - 1), 255, NULL, cv::Scalar(0), upDiff, flags);
+		cv::floodFill(imageGray, mask, cv::Point(0, image.rows - 1), 255, NULL, cv::Scalar(0), upDiff, flags);
 	if (imageGray(image.rows - 1, image.cols / 2) == 0)
-		cv::floodFill(imageGray, invalidMask, cv::Point(image.cols / 2, image.rows - 1), 255, NULL, cv::Scalar(0), upDiff, flags);
+		cv::floodFill(imageGray, mask, cv::Point(image.cols / 2, image.rows - 1), 255, NULL, cv::Scalar(0), upDiff, flags);
 	if (imageGray(image.rows - 1, image.cols - 1) == 0)
-		cv::floodFill(imageGray, invalidMask, cv::Point(image.cols - 1, image.rows - 1), 255, NULL, cv::Scalar(0), upDiff, flags);
+		cv::floodFill(imageGray, mask, cv::Point(image.cols - 1, image.rows - 1), 255, NULL, cv::Scalar(0), upDiff, flags);
 	if (imageGray(image.rows / 2, image.cols - 1) == 0)
-		cv::floodFill(imageGray, invalidMask, cv::Point(image.cols - 1, image.rows / 2), 255, NULL, cv::Scalar(0), upDiff, flags);
+		cv::floodFill(imageGray, mask, cv::Point(image.cols - 1, image.rows / 2), 255, NULL, cv::Scalar(0), upDiff, flags);
 	if (imageGray(0, image.cols - 1) == 0)
-		cv::floodFill(imageGray, invalidMask, cv::Point(image.cols - 1, 0), 255, NULL, cv::Scalar(0), upDiff, flags);
+		cv::floodFill(imageGray, mask, cv::Point(image.cols - 1, 0), 255, NULL, cv::Scalar(0), upDiff, flags);
 	if (imageGray(0, image.cols / 2) == 0)
-		cv::floodFill(imageGray, invalidMask, cv::Point(image.cols / 2, 0), 255, NULL, cv::Scalar(0), upDiff, flags);
-	invalidMask = invalidMask(cv::Rect(1,1, imageGray.cols,imageGray.rows));
-	return invalidMask;
+		cv::floodFill(imageGray, mask, cv::Point(image.cols / 2, 0), 255, NULL, cv::Scalar(0), upDiff, flags);
+	mask = (mask(cv::Rect(1,1, imageGray.cols,imageGray.rows)) == 0);
+	return mask;
 }
 
 MeshTexture::MeshTexture(Scene& _scene, unsigned _nResolutionLevel, unsigned _nMinResolution)
@@ -461,7 +461,7 @@ void MeshTexture::ListVertexFaces()
 }
 
 // extract array of faces viewed by each image
-bool MeshTexture::ListCameraFaces(FaceDataViewArr& facesDatas, float fOutlierThreshold, const IIndexArr& _views)
+bool MeshTexture::ListCameraFaces(FaceDataViewArr& facesDatas, float fOutlierThreshold, int nIgnoreMaskLabel, const IIndexArr& _views)
 {
 	// create faces octree
 	Mesh::Octree octree;
@@ -540,15 +540,22 @@ bool MeshTexture::ListCameraFaces(FaceDataViewArr& facesDatas, float fOutlierThr
 		const Frustum frustum(Frustum::MATRIX3x4(((PMatrix::CEMatMap)imageData.camera.P).cast<float>()), (float)imageData.width, (float)imageData.height);
 		octree.Traverse(frustum, inserter);
 		// project all triangles in this view and keep the closest ones
-		faceMap.create(imageData.height, imageData.width);
-		depthMap.create(imageData.height, imageData.width);
+		faceMap.create(imageData.GetSize());
+		depthMap.create(imageData.GetSize());
 		RasterMesh rasterer(vertices, imageData.camera, depthMap, faceMap);
+		if (nIgnoreMaskLabel >= 0) {
+			// import mask
+			BitMatrix bmask;
+			DepthEstimator::ImportIgnoreMask(imageData, imageData.GetSize(), (uint16_t)OPTDENSE::nIgnoreMaskLabel, bmask, &rasterer.mask);
+		} else if (nIgnoreMaskLabel == -1) {
+			// creating mask to discard invalid regions created during image radial undistortion
+			rasterer.mask = DetectInvalidImageRegions(imageData.image);
+			#if TD_VERBOSE != TD_VERBOSE_OFF
+			if (VERBOSITY_LEVEL > 2)
+				cv::imwrite(String::FormatString("umask%04d.png", idxView), rasterer.mask);
+			#endif
+		}
 		rasterer.Clear();
-		rasterer.invalidMask = DetectInvalidImageRegions(imageData.image);
-		#if TD_VERBOSE != TD_VERBOSE_OFF
-		if (VERBOSITY_LEVEL > 2)
-			cv::imwrite(String::FormatString("invalidMask%04d.png", idxView), rasterer.invalidMask);
-		#endif
 		for (auto idxFace : cameraFaces) {
 			rasterer.validFace = true;
 			const Face& facet = faces[idxFace];
@@ -562,6 +569,7 @@ bool MeshTexture::ListCameraFaces(FaceDataViewArr& facesDatas, float fOutlierThr
 		CLISTDEF0IDX(uint32_t,FIndex) areas(faces.GetSize());
 		areas.Memset(0);
 		#endif
+
 		#ifdef TEXOPT_USE_OPENMP
 		#pragma omp critical
 		#endif
@@ -1016,7 +1024,7 @@ bool MeshTexture::FaceOutlierDetection(FaceDataArr& faceDatas, float thOutlier)
 }
 #endif
 
-bool MeshTexture::FaceViewSelection(unsigned minCommonCameras, float fOutlierThreshold, float fRatioDataSmoothness, const IIndexArr& views)
+bool MeshTexture::FaceViewSelection(unsigned minCommonCameras, float fOutlierThreshold, float fRatioDataSmoothness, int nIgnoreMaskLabel, const IIndexArr& views)
 {
 	// extract array of triangles incident to each vertex
 	ListVertexFaces();
@@ -1028,7 +1036,7 @@ bool MeshTexture::FaceViewSelection(unsigned minCommonCameras, float fOutlierThr
 
 		// list all views for each face
 		FaceDataViewArr facesDatas;
-		if (!ListCameraFaces(facesDatas, fOutlierThreshold, views))
+		if (!ListCameraFaces(facesDatas, fOutlierThreshold, nIgnoreMaskLabel, views))
 			return false;
 
 		// create faces graph
@@ -2287,16 +2295,17 @@ void MeshTexture::GenerateTexture(bool bGlobalSeamLeveling, bool bLocalSeamLevel
 // texture mesh
 //  - minCommonCameras: generate texture patches using virtual faces composed of coplanar triangles sharing at least this number of views (0 - disabled, 3 - good value)
 //  - fSharpnessWeight: sharpness weight to be applied on the texture (0 - disabled, 0.5 - good value)
+//  - nIgnoreMaskLabel: label value to ignore in the image mask, stored in the MVS scene or next to each image with '.mask.png' extension (-1 - auto estimate mask for lens distortion, -2 - disabled)
 bool Scene::TextureMesh(unsigned nResolutionLevel, unsigned nMinResolution, unsigned minCommonCameras, float fOutlierThreshold, float fRatioDataSmoothness,
 	bool bGlobalSeamLeveling, bool bLocalSeamLeveling, unsigned nTextureSizeMultiple, unsigned nRectPackingHeuristic, Pixel8U colEmpty, float fSharpnessWeight,
-	const IIndexArr& views)
+	int nIgnoreMaskLabel, const IIndexArr& views)
 {
 	MeshTexture texture(*this, nResolutionLevel, nMinResolution);
 
 	// assign the best view to each face
 	{
 		TD_TIMER_STARTD();
-		if (!texture.FaceViewSelection(minCommonCameras, fOutlierThreshold, fRatioDataSmoothness, views))
+		if (!texture.FaceViewSelection(minCommonCameras, fOutlierThreshold, fRatioDataSmoothness, nIgnoreMaskLabel, views))
 			return false;
 		DEBUG_EXTRA("Assigning the best view to each face completed: %u faces (%s)", mesh.faces.GetSize(), TD_TIMER_GET_FMT().c_str());
 	}