From f821e265a75a2e7c83dc3ead12b2be1d1a7a8c47 Mon Sep 17 00:00:00 2001 From: Andrew Bell Date: Wed, 9 Dec 2020 14:18:06 -0500 Subject: [PATCH] Add the ability to select fields to load. Make progress more granular during tiling. --- api/QgisTest.cpp | 7 +++-- bu/BuPyramid.cpp | 3 +++ epf/Epf.cpp | 54 +++++++++++++++++++++++++++----------- epf/Epf.hpp | 3 ++- epf/FileProcessor.cpp | 36 ++++++++++++++----------- epf/FileProcessor.hpp | 7 ++++- untwine/Common.hpp | 8 +++++- untwine/ProgressWriter.cpp | 24 ++++++++++++----- untwine/ProgressWriter.hpp | 10 ++++++- untwine/Untwine.cpp | 2 ++ 10 files changed, 110 insertions(+), 44 deletions(-) diff --git a/api/QgisTest.cpp b/api/QgisTest.cpp index df740d6..74746d9 100644 --- a/api/QgisTest.cpp +++ b/api/QgisTest.cpp @@ -17,8 +17,11 @@ int main() // files.push_back("C:\\Users\\andre\\nyc2"); // files.push_back("C:\\Users\\andre\\nyc2\\18TXL075075.las.laz"); - files.push_back("/Users/acbell/nyc/18TXL075075.las.laz"); - files.push_back("/Users/acbell/nyc/18TXL075090.las.laz"); +// files.push_back("/Users/acbell/nyc/18TXL075075.las.laz"); +// files.push_back("/Users/acbell/nyc/18TXL075090.las.laz"); + files.push_back("/Users/acbell/nyc2"); + + options.push_back({"dims", "X, Y, Z, Red, Green, Blue, Intensity"}); // book ok = api.start(files, ".\\out", options); bool ok = api.start(files, "./out", options); if (! ok) diff --git a/bu/BuPyramid.cpp b/bu/BuPyramid.cpp index eed5bea..a5c1037 100644 --- a/bu/BuPyramid.cpp +++ b/bu/BuPyramid.cpp @@ -136,6 +136,8 @@ void BuPyramid::writeInfo() out << "{\n"; pdal::BOX3D& b = m_b.bounds; + std::ios init(NULL); + init.copyfmt(out); out << std::fixed << std::setw(12); out << "\"bounds\": [" << b.minx << ", " << b.miny << ", " << b.minz << ", " << @@ -177,6 +179,7 @@ void BuPyramid::writeInfo() out << "}\n"; out << "}\n"; + out.copyfmt(init); } diff --git a/epf/Epf.cpp b/epf/Epf.cpp index d6978e3..6fbe880 100644 --- a/epf/Epf.cpp +++ b/epf/Epf.cpp @@ -23,12 +23,11 @@ #include #include #include +#include #include #include #include -using namespace pdal; - namespace untwine { namespace epf @@ -39,7 +38,9 @@ void writeMetadata(const std::string& tempDir, const Grid& grid, { std::ofstream out(tempDir + "/" + MetadataFilename); pdal::BOX3D b = grid.processingBounds(); - out.precision(10); + std::ios init(NULL); + init.copyfmt(out); + out << std::setw(10) << std::fixed; out << b.minx << " " << b.miny << " " << b.minz << "\n"; out << b.maxx << " " << b.maxy << " " << b.maxz << "\n"; out << "\n"; @@ -48,11 +49,12 @@ void writeMetadata(const std::string& tempDir, const Grid& grid, out << b.minx << " " << b.miny << " " << b.minz << "\n"; out << b.maxx << " " << b.maxy << " " << b.maxz << "\n"; out << "\n"; + out.copyfmt(init); out << srs << "\n"; out << "\n"; - for (Dimension::Id id : layout->dims()) + for (pdal::Dimension::Id id : layout->dims()) out << layout->dimName(id) << " " << (int)layout->dimType(id) << " " << layout->dimOffset(id) << "\n"; } @@ -69,6 +71,8 @@ Epf::~Epf() void Epf::run(const Options& options, ProgressWriter& progress) { + using namespace pdal; + double millionPoints = 0; BOX3D totalBounds; @@ -78,7 +82,7 @@ void Epf::run(const Options& options, ProgressWriter& progress) m_grid.setCubic(options.doCube); std::vector fileInfos; - createFileInfo(options.inputFiles, fileInfos); + progress.m_total = createFileInfo(options.inputFiles, options.dimNames, fileInfos); if (options.level != -1) m_grid.resetLevel(options.level); @@ -124,7 +128,9 @@ void Epf::run(const Options& options, ProgressWriter& progress) std::sort(fileInfos.begin(), fileInfos.end(), [](const FileInfo& f1, const FileInfo& f2) { return f1.numPoints > f2.numPoints; }); - progress.setIncrement(.4 / fileInfos.size()); + progress.m_threshold = progress.m_total / 40; + progress.setIncrement(.01); + progress.m_current = 0; // Add the files to the processing pool for (const FileInfo& fi : fileInfos) @@ -132,14 +138,14 @@ void Epf::run(const Options& options, ProgressWriter& progress) int pointSize = layout->pointSize(); m_pool.add([&fi, &progress, pointSize, this]() { - FileProcessor fp(fi, pointSize, m_grid, m_writer.get()); + FileProcessor fp(fi, pointSize, m_grid, m_writer.get(), progress); fp.run(); - progress.writeIncrement("Tiled " + fi.filename); }); } // Wait for all the processors to finish and restart. m_pool.cycle(); + progress.setPercent(.4); // Tell the writer that it can exit. stop() will block until the writer threads // are finished. @@ -174,9 +180,24 @@ void Epf::run(const Options& options, ProgressWriter& progress) writeMetadata(options.tempDir, m_grid, srs, layout); } -void Epf::createFileInfo(const pdal::StringList& input, std::vector& fileInfos) +PointCount Epf::createFileInfo(const StringList& input, StringList dimNames, + std::vector& fileInfos) { + using namespace pdal; + std::vector filenames; + PointCount totalPoints = 0; + + // If there are some dim names specified, make sure they contain X, Y and Z and that + // they're all uppercase. + if (!dimNames.empty()) + { + for (std::string& d : dimNames) + d = Utils::toupper(d); + for (const std::string& xyz : { "X", "Y", "Z" }) + if (!Utils::contains(dimNames, xyz)) + dimNames.push_back(xyz); + } // If any of the specified input files is a directory, get the names of the files // in the directory and add them. @@ -213,26 +234,27 @@ void Epf::createFileInfo(const pdal::StringList& input, std::vector& f FileInfo fi; fi.bounds = qi.m_bounds; fi.numPoints = qi.m_pointCount; - for (const std::string& name : qi.m_dimNames) - fi.dimInfo.push_back(FileDimInfo(name)); fi.filename = filename; fi.driver = driver; + // Accept dimension names if there are no limits or this name is in the list + // of desired dimensions. + for (const std::string& name : qi.m_dimNames) + if (dimNames.empty() || Utils::contains(dimNames, Utils::toupper(name))) + fi.dimInfo.push_back(FileDimInfo(name)); + if (m_srsFileInfo.valid() && m_srsFileInfo.srs != qi.m_srs) - { std::cerr << "Files have mismatched SRS values. Using SRS from '" << m_srsFileInfo.filename << "'.\n"; - } fi.srs = qi.m_srs; fileInfos.push_back(fi); if (!m_srsFileInfo.valid() && qi.m_srs.valid()) - { m_srsFileInfo = fi; - std::cerr << "Set SRS file info fo " << m_srsFileInfo.filename << "!\n"; - } m_grid.expand(qi.m_bounds, qi.m_pointCount); + totalPoints += fi.numPoints; } + return totalPoints; } } // namespace epf diff --git a/epf/Epf.hpp b/epf/Epf.hpp index f323f09..4e30f29 100644 --- a/epf/Epf.hpp +++ b/epf/Epf.hpp @@ -44,7 +44,8 @@ class Epf void run(const Options& options, ProgressWriter& progress); private: - void createFileInfo(const pdal::StringList& input, std::vector& fileInfos); + PointCount createFileInfo(const StringList& input, StringList dimNames, + std::vector& fileInfos); Grid m_grid; std::unique_ptr m_writer; diff --git a/epf/FileProcessor.cpp b/epf/FileProcessor.cpp index 112b49b..b997e6b 100644 --- a/epf/FileProcessor.cpp +++ b/epf/FileProcessor.cpp @@ -12,37 +12,35 @@ #include "FileProcessor.hpp" +#include "../untwine/ProgressWriter.hpp" #include #include -using namespace pdal; - namespace untwine { namespace epf { - FileProcessor::FileProcessor(const FileInfo& fi, size_t pointSize, const Grid& grid, - Writer *writer) : - m_fi(fi), m_cellMgr(pointSize, writer), m_grid(grid) + Writer *writer, ProgressWriter& progress) : + m_fi(fi), m_cellMgr(pointSize, writer), m_grid(grid), m_progress(progress) {} void FileProcessor::run() { - Options opts; + pdal::Options opts; opts.add("filename", m_fi.filename); - std::cerr << ("Processing " + m_fi.filename + "!\n"); - - StageFactory factory; - Stage *s = factory.createStage(m_fi.driver); + pdal::StageFactory factory; + pdal::Stage *s = factory.createStage(m_fi.driver); s->setOptions(opts); - StreamCallbackFilter f; + pdal::StreamCallbackFilter f; - size_t count = 0; + const PointCount CountIncrement = 100000; + PointCount count = 0; + PointCount limit = CountIncrement; // We need to move the data from the PointRef to some output buffer. We copy the data // to the end of the *last* output buffer we used in hopes that it's the right one. @@ -52,7 +50,7 @@ void FileProcessor::run() // This is some random cell that ultimately won't get used, but it contains a buffer // into which we can write data. Cell *cell = m_cellMgr.get(VoxelKey()); - f.setCallback([this, &count, &cell](PointRef& point) + f.setCallback([this, &count, &limit, &cell](pdal::PointRef& point) { // Write the data into the point buffer in the cell. This is the *last* // cell buffer that we used. We're hoping that it's the right one. @@ -73,17 +71,23 @@ void FileProcessor::run() // point, we're referring to the next location in the cell's buffer. cell->advance(); count++; + + if (count == limit) + { + m_progress.update(CountIncrement); + limit += CountIncrement; + } + return true; } ); f.setInput(*s); - FixedPointTable t(1000); + pdal::FixedPointTable t(1000); f.prepare(t); f.execute(t); - - std::cerr << ("Done " + m_fi.filename + " - " + std::to_string(count) + " points!\n"); + m_progress.update(count % CountIncrement); // Flush any data remaining in the cells. m_cellMgr.flush(); diff --git a/epf/FileProcessor.hpp b/epf/FileProcessor.hpp index 8f012fa..08daff9 100644 --- a/epf/FileProcessor.hpp +++ b/epf/FileProcessor.hpp @@ -17,6 +17,9 @@ namespace untwine { + +class ProgressWriter; + namespace epf { @@ -26,7 +29,8 @@ class Writer; class FileProcessor { public: - FileProcessor(const FileInfo& fi, size_t pointSize, const Grid& grid, Writer *writer); + FileProcessor(const FileInfo& fi, size_t pointSize, const Grid& grid, Writer *writer, + ProgressWriter& progress); Cell *getCell(const VoxelKey& key); void run(); @@ -35,6 +39,7 @@ class FileProcessor FileInfo m_fi; CellMgr m_cellMgr; Grid m_grid; + ProgressWriter& m_progress; }; } // namespace epf diff --git a/untwine/Common.hpp b/untwine/Common.hpp index c7700ce..28da60f 100644 --- a/untwine/Common.hpp +++ b/untwine/Common.hpp @@ -1,21 +1,27 @@ #pragma once +#include #include +#include namespace untwine { +using PointCount = uint64_t; +using StringList = std::vector; + void fatal(const std::string& err); struct Options { std::string outputDir; - pdal::StringList inputFiles; + StringList inputFiles; std::string tempDir; bool doCube; size_t fileLimit; int level; int progressFd; + StringList dimNames; }; const std::string MetadataFilename {"info2.txt"}; diff --git a/untwine/ProgressWriter.cpp b/untwine/ProgressWriter.cpp index 0c20b25..7130e3c 100644 --- a/untwine/ProgressWriter.cpp +++ b/untwine/ProgressWriter.cpp @@ -13,8 +13,6 @@ namespace untwine { -std::mutex ProgressWriter::s_mutex; - ProgressWriter::ProgressWriter(int fd) : m_progressFd(fd), m_percent(0.0), m_increment(.1) {} @@ -22,7 +20,7 @@ void ProgressWriter::setIncrement(double increment) { if (!m_progressFd) return; - std::unique_lock lock(s_mutex); + std::unique_lock lock(m_mutex); m_increment = increment; } @@ -31,7 +29,7 @@ void ProgressWriter::setPercent(double percent) { if (!m_progressFd) return; - std::unique_lock lock(s_mutex); + std::unique_lock lock(m_mutex); m_percent = (std::max)(0.0, ((std::min)(1.0, percent))); } @@ -40,7 +38,7 @@ void ProgressWriter::writeIncrement(const std::string& message) { if (!m_progressFd) return; - std::unique_lock lock(s_mutex); + std::unique_lock lock(m_mutex); m_percent += m_increment; m_percent = (std::min)(1.0, m_percent); @@ -53,7 +51,7 @@ void ProgressWriter::write(double percent, const std::string& message) { if (!m_progressFd) return; - std::unique_lock lock(s_mutex); + std::unique_lock lock(m_mutex); m_percent = (std::min)(0.0, ((std::max)(1.0, percent))); @@ -78,4 +76,18 @@ void ProgressWriter::writeMessage(uint32_t percent, const std::string& message) #endif } +void ProgressWriter::update(PointCount count) +{ + std::unique_lock lock(m_mutex); + + PointCount inc = m_current / m_threshold; + m_current += count; + PointCount postInc = m_current / m_threshold; + if (inc != postInc) + { + lock.unlock(); + writeIncrement("Processed " + std::to_string(m_current) + " points"); + } +} + } // namespace untwine diff --git a/untwine/ProgressWriter.hpp b/untwine/ProgressWriter.hpp index 71b5b1f..8d75e43 100644 --- a/untwine/ProgressWriter.hpp +++ b/untwine/ProgressWriter.hpp @@ -2,6 +2,8 @@ #include +#include "../untwine/Common.hpp" + namespace untwine { @@ -20,8 +22,14 @@ class ProgressWriter /// Write a message and set the current percentage. void write(double percent, const std::string& message); + void update(PointCount numProcessed); + // Utility fields + PointCount m_total; + PointCount m_threshold; + PointCount m_current; + private: - static std::mutex s_mutex; + std::mutex m_mutex; int m_progressFd; double m_percent; // Current percent. double m_increment; // Current increment. diff --git a/untwine/Untwine.cpp b/untwine/Untwine.cpp index 1bb0af9..c0833e4 100644 --- a/untwine/Untwine.cpp +++ b/untwine/Untwine.cpp @@ -42,6 +42,8 @@ void addArgs(pdal::ProgramArgs& programArgs, Options& options, pdal::Arg * &temp options.fileLimit, (size_t)10000000); programArgs.add("progress_fd", "File descriptor on which to write process messages.", options.progressFd); + programArgs.add("dims", "Dimensions to load. Note that X, Y and Z are always " + "loaded.", options.dimNames); } bool handleOptions(pdal::StringList& arglist, Options& options)