diff --git a/apps/DensifyPointCloud/DensifyPointCloud.cpp b/apps/DensifyPointCloud/DensifyPointCloud.cpp index 9cb89edbf..0e22f8aaa 100644 --- a/apps/DensifyPointCloud/DensifyPointCloud.cpp +++ b/apps/DensifyPointCloud/DensifyPointCloud.cpp @@ -134,7 +134,7 @@ bool Initialize(size_t argc, LPCTSTR* argv) ("sub-resolution-levels", boost::program_options::value(&nSubResolutionLevels)->default_value(2), "number of patch-match sub-resolution iterations (0 - disabled)") ("number-views", boost::program_options::value(&nNumViews)->default_value(nNumViewsDefault), "number of views used for depth-map estimation (0 - all neighbor views available)") ("number-views-fuse", boost::program_options::value(&nMinViewsFuse)->default_value(3), "minimum number of images that agrees with an estimate during fusion in order to consider it inlier (<2 - only merge depth-maps)") - ("ignore-mask-label", boost::program_options::value(&nIgnoreMaskLabel)->default_value(-1), "integer value for the label to ignore in the segmentation mask; the mask for each image is stored in the MVS scene or next to each image with '.mask.png' extension (<0 - disabled)") + ("ignore-mask-label", boost::program_options::value(&nIgnoreMaskLabel)->default_value(-1), "label value to ignore in the image mask, stored in the MVS scene or next to each image with '.mask.png' extension (<0 - disabled)") ("mask-path", boost::program_options::value(&OPT::strMaskPath), "path to folder containing mask images with '.mask.png' extension") ("iters", boost::program_options::value(&nEstimationIters)->default_value(numIters), "number of patch-match iterations") ("geometric-iters", boost::program_options::value(&nEstimationGeometricIters)->default_value(2), "number of geometric consistent patch-match iterations (0 - disabled)") diff --git a/apps/TextureMesh/TextureMesh.cpp b/apps/TextureMesh/TextureMesh.cpp index 6bf906613..acc1c2b51 100644 --- a/apps/TextureMesh/TextureMesh.cpp +++ b/apps/TextureMesh/TextureMesh.cpp @@ -63,6 +63,7 @@ unsigned nTextureSizeMultiple; unsigned nRectPackingHeuristic; uint32_t nColEmpty; float fSharpnessWeight; +int nIgnoreMaskLabel; unsigned nOrthoMapResolution; unsigned nArchiveType; int nProcessPriority; @@ -122,6 +123,7 @@ bool Initialize(size_t argc, LPCTSTR* argv) ("empty-color", boost::program_options::value(&OPT::nColEmpty)->default_value(0x00FF7F27), "color used for faces not covered by any image") ("sharpness-weight", boost::program_options::value(&OPT::fSharpnessWeight)->default_value(0.5f), "amount of sharpness to be applied on the texture (0 - disabled)") ("orthographic-image-resolution", boost::program_options::value(&OPT::nOrthoMapResolution)->default_value(0), "orthographic image resolution to be generated from the textured mesh - the mesh is expected to be already geo-referenced or at least properly oriented (0 - disabled)") + ("ignore-mask-label", boost::program_options::value(&OPT::nIgnoreMaskLabel)->default_value(-1), "label value to ignore in the image mask, stored in the MVS scene or next to each image with '.mask.png' extension (-1 - auto estimate mask for lens distortion, -2 - disabled)") ; // hidden options, allowed both on command line and @@ -307,7 +309,7 @@ int main(int argc, LPCTSTR* argv) TD_TIMER_START(); if (!scene.TextureMesh(OPT::nResolutionLevel, OPT::nMinResolution, OPT::minCommonCameras, OPT::fOutlierThreshold, OPT::fRatioDataSmoothness, OPT::bGlobalSeamLeveling, OPT::bLocalSeamLeveling, OPT::nTextureSizeMultiple, OPT::nRectPackingHeuristic, Pixel8U(OPT::nColEmpty), - OPT::fSharpnessWeight, views)) + OPT::fSharpnessWeight, OPT::nIgnoreMaskLabel, views)) return EXIT_FAILURE; VERBOSE("Mesh texturing completed: %u vertices, %u faces (%s)", scene.mesh.vertices.GetSize(), scene.mesh.faces.GetSize(), TD_TIMER_GET_FMT().c_str()); diff --git a/libs/MVS/DepthMap.cpp b/libs/MVS/DepthMap.cpp index 9ec91bc25..8346349c5 100644 --- a/libs/MVS/DepthMap.cpp +++ b/libs/MVS/DepthMap.cpp @@ -297,8 +297,9 @@ unsigned DepthData::DecRef() // try to load and apply mask to the depth map; // the mask for each image is stored in the MVS scene or next to each image with '.mask.png' extension; -// the mask marks as false pixels that should be ignored -bool DepthEstimator::ImportIgnoreMask(const Image& image0, const Image8U::Size& size, BitMatrix& bmask, uint16_t nIgnoreMaskLabel) +// the mask marks as false (or 0) pixels that should be ignored +// - pMask: optional output mask; if defined, the mask is returned in this image instead of the BitMatrix +bool DepthEstimator::ImportIgnoreMask(const Image& image0, const Image8U::Size& size, uint16_t nIgnoreMaskLabel, BitMatrix& bmask, Image8U* pMask) { ASSERT(image0.IsValid() && !image0.image.empty()); const String maskFileName(image0.maskName.empty() ? Util::getFileFullName(image0.name)+".mask.png" : image0.maskName); @@ -308,12 +309,16 @@ bool DepthEstimator::ImportIgnoreMask(const Image& image0, const Image8U::Size& return false; } cv::resize(mask, mask, size, 0, 0, cv::INTER_NEAREST); - bmask.create(size); - bmask.memset(0xFF); - for (int r=0; r= 0) { const DepthData::ViewData& view = depthData.GetView(); BitMatrix mask; - if (DepthEstimator::ImportIgnoreMask(*view.pImageData, depthData.depthMap.size(), mask, (uint16_t)OPTDENSE::nIgnoreMaskLabel)) + if (DepthEstimator::ImportIgnoreMask(*view.pImageData, depthData.depthMap.size(), (uint16_t)OPTDENSE::nIgnoreMaskLabel, mask)) depthData.ApplyIgnoreMask(mask); } diff --git a/libs/MVS/Scene.h b/libs/MVS/Scene.h index e49f717ef..8c2c35ce5 100644 --- a/libs/MVS/Scene.h +++ b/libs/MVS/Scene.h @@ -138,7 +138,7 @@ class MVS_API Scene // Mesh texturing bool TextureMesh(unsigned nResolutionLevel, unsigned nMinResolution, unsigned minCommonCameras=0, float fOutlierThreshold=0.f, float fRatioDataSmoothness=0.3f, bool bGlobalSeamLeveling=true, bool bLocalSeamLeveling=true, unsigned nTextureSizeMultiple=0, unsigned nRectPackingHeuristic=3, Pixel8U colEmpty=Pixel8U(255,127,39), - float fSharpnessWeight=0.5f, const IIndexArr& views=IIndexArr()); + float fSharpnessWeight=0.5f, int ignoreMaskLabel = -1, const IIndexArr& views=IIndexArr()); #ifdef _USE_BOOST // implement BOOST serialization diff --git a/libs/MVS/SceneDensify.cpp b/libs/MVS/SceneDensify.cpp index b2969c7d3..95038c26f 100644 --- a/libs/MVS/SceneDensify.cpp +++ b/libs/MVS/SceneDensify.cpp @@ -672,7 +672,7 @@ bool DepthMapsData::EstimateDepthMap(IIndex idxImage, int nGeometricIter) #endif if (prevDepthMapSize != size || OPTDENSE::nIgnoreMaskLabel >= 0) { BitMatrix mask; - if (OPTDENSE::nIgnoreMaskLabel >= 0 && DepthEstimator::ImportIgnoreMask(*image.pImageData, depthData.depthMap.size(), mask, (uint16_t)OPTDENSE::nIgnoreMaskLabel)) + if (OPTDENSE::nIgnoreMaskLabel >= 0 && DepthEstimator::ImportIgnoreMask(*image.pImageData, depthData.depthMap.size(), (uint16_t)OPTDENSE::nIgnoreMaskLabel, mask)) depthData.ApplyIgnoreMask(mask); DepthEstimator::MapMatrix2ZigzagIdx(size, coords, mask, MAXF(64,(int)nMaxThreads*8)); #if 0 && !defined(_RELEASE) diff --git a/libs/MVS/SceneTexture.cpp b/libs/MVS/SceneTexture.cpp index e62dbefcc..e8404ba50 100644 --- a/libs/MVS/SceneTexture.cpp +++ b/libs/MVS/SceneTexture.cpp @@ -151,7 +151,7 @@ struct MeshTexture { typedef TRasterMesh Base; FaceMap& faceMap; FIndex idxFace; - Image8U invalidMask; + Image8U mask; bool validFace; RasterMesh(const Mesh::VertexArr& _vertices, const Camera& _camera, DepthMap& _depthMap, FaceMap& _faceMap) @@ -167,7 +167,7 @@ struct MeshTexture { Depth& depth = depthMap(pt); if (depth == 0 || depth > z) { depth = z; - faceMap(pt) = validFace && (validFace = (invalidMask(pt) == 0)) ? idxFace : NO_ID; + faceMap(pt) = validFace && (validFace = (mask(pt) != 0)) ? idxFace : NO_ID; } } }; @@ -326,7 +326,7 @@ struct MeshTexture { void ListVertexFaces(); - bool ListCameraFaces(FaceDataViewArr&, float fOutlierThreshold, const IIndexArr& views); + bool ListCameraFaces(FaceDataViewArr&, float fOutlierThreshold, int nIgnoreMaskLabel, const IIndexArr& views); #if TEXOPT_FACEOUTLIER != TEXOPT_FACEOUTLIER_NA bool FaceOutlierDetection(FaceDataArr& faceDatas, float fOutlierThreshold) const; @@ -335,7 +335,7 @@ struct MeshTexture { void CreateVirtualFaces(const FaceDataViewArr& facesDatas, FaceDataViewArr& virtualFacesDatas, VirtualFaceIdxsArr& virtualFaces, unsigned minCommonCameras=2, float thMaxNormalDeviation=25.f) const; IIndexArr SelectBestView(const FaceDataArr& faceDatas, FIndex fid, unsigned minCommonCameras, float ratioAngleToQuality) const; - bool FaceViewSelection(unsigned minCommonCameras, float fOutlierThreshold, float fRatioDataSmoothness, const IIndexArr& views); + bool FaceViewSelection(unsigned minCommonCameras, float fOutlierThreshold, float fRatioDataSmoothness, int nIgnoreMaskLabel, const IIndexArr& views); void CreateSeamVertices(); void GlobalSeamLeveling(); @@ -399,33 +399,33 @@ struct MeshTexture { // creating an invalid mask for the given image corresponding to // the invalid pixels generated during image correction for the lens distortion; -// the returned mask has the same size as the image and is set to non-zero for invalid pixels +// the returned mask has the same size as the image and is set to zero for invalid pixels static Image8U DetectInvalidImageRegions(const Image8U3& image) { const cv::Scalar upDiff(3); const int flags(8 | (255 << 8)); - Image8U invalidMask(image.rows + 2, image.cols + 2); - invalidMask.memset(0); + Image8U mask(image.rows + 2, image.cols + 2); + mask.memset(0); Image8U imageGray; cv::cvtColor(image, imageGray, cv::COLOR_BGR2GRAY); if (imageGray(0, 0) == 0) - cv::floodFill(imageGray, invalidMask, cv::Point(0, 0), 255, NULL, cv::Scalar(0), upDiff, flags); + cv::floodFill(imageGray, mask, cv::Point(0, 0), 255, NULL, cv::Scalar(0), upDiff, flags); if (imageGray(image.rows / 2, 0) == 0) - cv::floodFill(imageGray, invalidMask, cv::Point(0, image.rows / 2), 255, NULL, cv::Scalar(0), upDiff, flags); + cv::floodFill(imageGray, mask, cv::Point(0, image.rows / 2), 255, NULL, cv::Scalar(0), upDiff, flags); if (imageGray(image.rows - 1, 0) == 0) - cv::floodFill(imageGray, invalidMask, cv::Point(0, image.rows - 1), 255, NULL, cv::Scalar(0), upDiff, flags); + cv::floodFill(imageGray, mask, cv::Point(0, image.rows - 1), 255, NULL, cv::Scalar(0), upDiff, flags); if (imageGray(image.rows - 1, image.cols / 2) == 0) - cv::floodFill(imageGray, invalidMask, cv::Point(image.cols / 2, image.rows - 1), 255, NULL, cv::Scalar(0), upDiff, flags); + cv::floodFill(imageGray, mask, cv::Point(image.cols / 2, image.rows - 1), 255, NULL, cv::Scalar(0), upDiff, flags); if (imageGray(image.rows - 1, image.cols - 1) == 0) - cv::floodFill(imageGray, invalidMask, cv::Point(image.cols - 1, image.rows - 1), 255, NULL, cv::Scalar(0), upDiff, flags); + cv::floodFill(imageGray, mask, cv::Point(image.cols - 1, image.rows - 1), 255, NULL, cv::Scalar(0), upDiff, flags); if (imageGray(image.rows / 2, image.cols - 1) == 0) - cv::floodFill(imageGray, invalidMask, cv::Point(image.cols - 1, image.rows / 2), 255, NULL, cv::Scalar(0), upDiff, flags); + cv::floodFill(imageGray, mask, cv::Point(image.cols - 1, image.rows / 2), 255, NULL, cv::Scalar(0), upDiff, flags); if (imageGray(0, image.cols - 1) == 0) - cv::floodFill(imageGray, invalidMask, cv::Point(image.cols - 1, 0), 255, NULL, cv::Scalar(0), upDiff, flags); + cv::floodFill(imageGray, mask, cv::Point(image.cols - 1, 0), 255, NULL, cv::Scalar(0), upDiff, flags); if (imageGray(0, image.cols / 2) == 0) - cv::floodFill(imageGray, invalidMask, cv::Point(image.cols / 2, 0), 255, NULL, cv::Scalar(0), upDiff, flags); - invalidMask = invalidMask(cv::Rect(1,1, imageGray.cols,imageGray.rows)); - return invalidMask; + cv::floodFill(imageGray, mask, cv::Point(image.cols / 2, 0), 255, NULL, cv::Scalar(0), upDiff, flags); + mask = (mask(cv::Rect(1,1, imageGray.cols,imageGray.rows)) == 0); + return mask; } MeshTexture::MeshTexture(Scene& _scene, unsigned _nResolutionLevel, unsigned _nMinResolution) @@ -461,7 +461,7 @@ void MeshTexture::ListVertexFaces() } // extract array of faces viewed by each image -bool MeshTexture::ListCameraFaces(FaceDataViewArr& facesDatas, float fOutlierThreshold, const IIndexArr& _views) +bool MeshTexture::ListCameraFaces(FaceDataViewArr& facesDatas, float fOutlierThreshold, int nIgnoreMaskLabel, const IIndexArr& _views) { // create faces octree Mesh::Octree octree; @@ -540,15 +540,22 @@ bool MeshTexture::ListCameraFaces(FaceDataViewArr& facesDatas, float fOutlierThr const Frustum frustum(Frustum::MATRIX3x4(((PMatrix::CEMatMap)imageData.camera.P).cast()), (float)imageData.width, (float)imageData.height); octree.Traverse(frustum, inserter); // project all triangles in this view and keep the closest ones - faceMap.create(imageData.height, imageData.width); - depthMap.create(imageData.height, imageData.width); + faceMap.create(imageData.GetSize()); + depthMap.create(imageData.GetSize()); RasterMesh rasterer(vertices, imageData.camera, depthMap, faceMap); + if (nIgnoreMaskLabel >= 0) { + // import mask + BitMatrix bmask; + DepthEstimator::ImportIgnoreMask(imageData, imageData.GetSize(), (uint16_t)OPTDENSE::nIgnoreMaskLabel, bmask, &rasterer.mask); + } else if (nIgnoreMaskLabel == -1) { + // creating mask to discard invalid regions created during image radial undistortion + rasterer.mask = DetectInvalidImageRegions(imageData.image); + #if TD_VERBOSE != TD_VERBOSE_OFF + if (VERBOSITY_LEVEL > 2) + cv::imwrite(String::FormatString("umask%04d.png", idxView), rasterer.mask); + #endif + } rasterer.Clear(); - rasterer.invalidMask = DetectInvalidImageRegions(imageData.image); - #if TD_VERBOSE != TD_VERBOSE_OFF - if (VERBOSITY_LEVEL > 2) - cv::imwrite(String::FormatString("invalidMask%04d.png", idxView), rasterer.invalidMask); - #endif for (auto idxFace : cameraFaces) { rasterer.validFace = true; const Face& facet = faces[idxFace]; @@ -562,6 +569,7 @@ bool MeshTexture::ListCameraFaces(FaceDataViewArr& facesDatas, float fOutlierThr CLISTDEF0IDX(uint32_t,FIndex) areas(faces.GetSize()); areas.Memset(0); #endif + #ifdef TEXOPT_USE_OPENMP #pragma omp critical #endif @@ -1016,7 +1024,7 @@ bool MeshTexture::FaceOutlierDetection(FaceDataArr& faceDatas, float thOutlier) } #endif -bool MeshTexture::FaceViewSelection(unsigned minCommonCameras, float fOutlierThreshold, float fRatioDataSmoothness, const IIndexArr& views) +bool MeshTexture::FaceViewSelection(unsigned minCommonCameras, float fOutlierThreshold, float fRatioDataSmoothness, int nIgnoreMaskLabel, const IIndexArr& views) { // extract array of triangles incident to each vertex ListVertexFaces(); @@ -1028,7 +1036,7 @@ bool MeshTexture::FaceViewSelection(unsigned minCommonCameras, float fOutlierThr // list all views for each face FaceDataViewArr facesDatas; - if (!ListCameraFaces(facesDatas, fOutlierThreshold, views)) + if (!ListCameraFaces(facesDatas, fOutlierThreshold, nIgnoreMaskLabel, views)) return false; // create faces graph @@ -2287,16 +2295,17 @@ void MeshTexture::GenerateTexture(bool bGlobalSeamLeveling, bool bLocalSeamLevel // texture mesh // - minCommonCameras: generate texture patches using virtual faces composed of coplanar triangles sharing at least this number of views (0 - disabled, 3 - good value) // - fSharpnessWeight: sharpness weight to be applied on the texture (0 - disabled, 0.5 - good value) +// - nIgnoreMaskLabel: label value to ignore in the image mask, stored in the MVS scene or next to each image with '.mask.png' extension (-1 - auto estimate mask for lens distortion, -2 - disabled) bool Scene::TextureMesh(unsigned nResolutionLevel, unsigned nMinResolution, unsigned minCommonCameras, float fOutlierThreshold, float fRatioDataSmoothness, bool bGlobalSeamLeveling, bool bLocalSeamLeveling, unsigned nTextureSizeMultiple, unsigned nRectPackingHeuristic, Pixel8U colEmpty, float fSharpnessWeight, - const IIndexArr& views) + int nIgnoreMaskLabel, const IIndexArr& views) { MeshTexture texture(*this, nResolutionLevel, nMinResolution); // assign the best view to each face { TD_TIMER_STARTD(); - if (!texture.FaceViewSelection(minCommonCameras, fOutlierThreshold, fRatioDataSmoothness, views)) + if (!texture.FaceViewSelection(minCommonCameras, fOutlierThreshold, fRatioDataSmoothness, nIgnoreMaskLabel, views)) return false; DEBUG_EXTRA("Assigning the best view to each face completed: %u faces (%s)", mesh.faces.GetSize(), TD_TIMER_GET_FMT().c_str()); }