Add the ability to select fields to load.

Make progress more granular during tiling.

api/QgisTest.cpp

@@ -17,8 +17,11 @@ int main()
 
 //    files.push_back("C:\\Users\\andre\\nyc2");
 //    files.push_back("C:\\Users\\andre\\nyc2\\18TXL075075.las.laz");
-    files.push_back("/Users/acbell/nyc/18TXL075075.las.laz");
-    files.push_back("/Users/acbell/nyc/18TXL075090.las.laz");
+//    files.push_back("/Users/acbell/nyc/18TXL075075.las.laz");
+//    files.push_back("/Users/acbell/nyc/18TXL075090.las.laz");
+    files.push_back("/Users/acbell/nyc2");
+
+    options.push_back({"dims", "X, Y, Z, Red, Green, Blue, Intensity"});
 //    book ok = api.start(files, ".\\out", options);
     bool ok = api.start(files, "./out", options);
     if (! ok)

bu/BuPyramid.cpp

@@ -136,6 +136,8 @@ void BuPyramid::writeInfo()
     out << "{\n";
 
     pdal::BOX3D& b = m_b.bounds;
+    std::ios init(NULL);
+    init.copyfmt(out);
     out << std::fixed << std::setw(12);
     out << "\"bounds\": [" <<
         b.minx << ", " << b.miny << ", " << b.minz << ", " <<
@@ -177,6 +179,7 @@ void BuPyramid::writeInfo()
     out << "}\n";
 
     out << "}\n";
+    out.copyfmt(init);
 }
 
 

epf/Epf.cpp

@@ -23,12 +23,11 @@
 #include <pdal/Dimension.hpp>
 #include <pdal/PointLayout.hpp>
 #include <pdal/StageFactory.hpp>
+#include <pdal/util/Algorithm.hpp>
 #include <pdal/util/Bounds.hpp>
 #include <pdal/util/FileUtils.hpp>
 #include <pdal/util/ProgramArgs.hpp>
 
-using namespace pdal;
-
 namespace untwine
 {
 namespace epf
@@ -39,7 +38,9 @@ void writeMetadata(const std::string& tempDir, const Grid& grid,
 {
     std::ofstream out(tempDir + "/" + MetadataFilename);
     pdal::BOX3D b = grid.processingBounds();
-    out.precision(10);
+    std::ios init(NULL);
+    init.copyfmt(out);
+    out << std::setw(10) << std::fixed;
     out << b.minx << " " << b.miny << " " << b.minz << "\n";
     out << b.maxx << " " << b.maxy << " " << b.maxz << "\n";
     out << "\n";
@@ -48,11 +49,12 @@ void writeMetadata(const std::string& tempDir, const Grid& grid,
     out << b.minx << " " << b.miny << " " << b.minz << "\n";
     out << b.maxx << " " << b.maxy << " " << b.maxz << "\n";
     out << "\n";
+    out.copyfmt(init);
 
     out << srs << "\n";
     out << "\n";
 
-    for (Dimension::Id id : layout->dims())
+    for (pdal::Dimension::Id id : layout->dims())
         out << layout->dimName(id) << " " << (int)layout->dimType(id) << " " <<
             layout->dimOffset(id) << "\n";
 }
@@ -69,6 +71,8 @@ Epf::~Epf()
 
 void Epf::run(const Options& options, ProgressWriter& progress)
 {
+    using namespace pdal;
+
     double millionPoints = 0;
     BOX3D totalBounds;
 
@@ -78,7 +82,7 @@ void Epf::run(const Options& options, ProgressWriter& progress)
     m_grid.setCubic(options.doCube);
 
     std::vector<FileInfo> fileInfos;
-    createFileInfo(options.inputFiles, fileInfos);
+    progress.m_total = createFileInfo(options.inputFiles, options.dimNames, fileInfos);
 
     if (options.level != -1)
         m_grid.resetLevel(options.level);
@@ -124,22 +128,24 @@ void Epf::run(const Options& options, ProgressWriter& progress)
     std::sort(fileInfos.begin(), fileInfos.end(), [](const FileInfo& f1, const FileInfo& f2)
         { return f1.numPoints > f2.numPoints; });
 
-    progress.setIncrement(.4 / fileInfos.size());
+    progress.m_threshold = progress.m_total / 40;
+    progress.setIncrement(.01);
+    progress.m_current = 0;
 
     // Add the files to the processing pool
     for (const FileInfo& fi : fileInfos)
     {
         int pointSize = layout->pointSize();
         m_pool.add([&fi, &progress, pointSize, this]()
         {
-            FileProcessor fp(fi, pointSize, m_grid, m_writer.get());
+            FileProcessor fp(fi, pointSize, m_grid, m_writer.get(), progress);
             fp.run();
-            progress.writeIncrement("Tiled " + fi.filename);
         });
     }
 
     // Wait for  all the processors to finish and restart.
     m_pool.cycle();
+    progress.setPercent(.4);
 
     // Tell the writer that it can exit. stop() will block until the writer threads
     // are finished.
@@ -174,9 +180,24 @@ void Epf::run(const Options& options, ProgressWriter& progress)
     writeMetadata(options.tempDir, m_grid, srs, layout);
 }
 
-void Epf::createFileInfo(const pdal::StringList& input, std::vector<FileInfo>& fileInfos)
+PointCount Epf::createFileInfo(const StringList& input, StringList dimNames,
+    std::vector<FileInfo>& fileInfos)
 {
+    using namespace pdal;
+
     std::vector<std::string> filenames;
+    PointCount totalPoints = 0;
+
+    // If there are some dim names specified, make sure they contain X, Y and Z and that
+    // they're all uppercase.
+    if (!dimNames.empty())
+    {
+        for (std::string& d : dimNames)
+            d = Utils::toupper(d);
+        for (const std::string& xyz : { "X", "Y", "Z" })
+            if (!Utils::contains(dimNames, xyz))
+                dimNames.push_back(xyz);
+    }
 
     // If any of the specified input files is a directory, get the names of the files
     // in the directory and add them.
@@ -213,26 +234,27 @@ void Epf::createFileInfo(const pdal::StringList& input, std::vector<FileInfo>& f
         FileInfo fi;
         fi.bounds = qi.m_bounds;
         fi.numPoints = qi.m_pointCount;
-        for (const std::string& name : qi.m_dimNames)
-            fi.dimInfo.push_back(FileDimInfo(name));
         fi.filename = filename;
         fi.driver = driver;
 
+        // Accept dimension names if there are no limits or this name is in the list
+        // of desired dimensions.
+        for (const std::string& name : qi.m_dimNames)
+            if (dimNames.empty() || Utils::contains(dimNames, Utils::toupper(name)))
+                fi.dimInfo.push_back(FileDimInfo(name));
+
         if (m_srsFileInfo.valid() && m_srsFileInfo.srs != qi.m_srs)
-        {
             std::cerr << "Files have mismatched SRS values. Using SRS from '" <<
                 m_srsFileInfo.filename << "'.\n";
-        }
         fi.srs = qi.m_srs;
         fileInfos.push_back(fi);
         if (!m_srsFileInfo.valid() && qi.m_srs.valid())
-        {
             m_srsFileInfo = fi;
-            std::cerr << "Set SRS file info fo " << m_srsFileInfo.filename << "!\n";
-        }
 
         m_grid.expand(qi.m_bounds, qi.m_pointCount);
+        totalPoints += fi.numPoints;
     }
+    return totalPoints;
 }
 
 } // namespace epf

epf/Epf.hpp

@@ -44,7 +44,8 @@ class Epf
     void run(const Options& options, ProgressWriter& progress);
 
 private:
-    void createFileInfo(const pdal::StringList& input, std::vector<FileInfo>& fileInfos);
+    PointCount createFileInfo(const StringList& input, StringList dimNames,
+        std::vector<FileInfo>& fileInfos);
 
     Grid m_grid;
     std::unique_ptr<Writer> m_writer;

epf/FileProcessor.cpp

@@ -12,37 +12,35 @@
 
 
 #include "FileProcessor.hpp"
+#include "../untwine/ProgressWriter.hpp"
 
 #include <pdal/StageFactory.hpp>
 #include <pdal/filters/StreamCallbackFilter.hpp>
 
-using namespace pdal;
-
 namespace untwine
 {
 namespace epf
 {
 
-
 FileProcessor::FileProcessor(const FileInfo& fi, size_t pointSize, const Grid& grid,
-        Writer *writer) :
-    m_fi(fi), m_cellMgr(pointSize, writer), m_grid(grid)
+        Writer *writer, ProgressWriter& progress) :
+    m_fi(fi), m_cellMgr(pointSize, writer), m_grid(grid), m_progress(progress)
 {}
 
 void FileProcessor::run()
 {
-    Options opts;
+    pdal::Options opts;
     opts.add("filename", m_fi.filename);
 
-    std::cerr << ("Processing " + m_fi.filename + "!\n");
-
-    StageFactory factory;
-    Stage *s = factory.createStage(m_fi.driver);
+    pdal::StageFactory factory;
+    pdal::Stage *s = factory.createStage(m_fi.driver);
     s->setOptions(opts);
 
-    StreamCallbackFilter f;
+    pdal::StreamCallbackFilter f;
 
-    size_t count = 0;
+    const PointCount CountIncrement = 100000;
+    PointCount count = 0;
+    PointCount limit = CountIncrement;
 
     // We need to move the data from the PointRef to some output buffer. We copy the data
     // to the end of the *last* output buffer we used in hopes that it's the right one.
@@ -52,7 +50,7 @@ void FileProcessor::run()
     // This is some random cell that ultimately won't get used, but it contains a buffer
     // into which we can write data.
     Cell *cell = m_cellMgr.get(VoxelKey());
-    f.setCallback([this, &count, &cell](PointRef& point)
+    f.setCallback([this, &count, &limit, &cell](pdal::PointRef& point)
         {
             // Write the data into the point buffer in the cell.  This is the *last*
             // cell buffer that we used. We're hoping that it's the right one.
@@ -73,17 +71,23 @@ void FileProcessor::run()
             // point, we're referring to the next location in the cell's buffer.
             cell->advance();
             count++;
+
+            if (count == limit)
+            {
+                m_progress.update(CountIncrement);
+                limit += CountIncrement;
+            }
+
             return true;
         }
     );
     f.setInput(*s);
 
-    FixedPointTable t(1000);
+    pdal::FixedPointTable t(1000);
 
     f.prepare(t);
     f.execute(t);
-
-    std::cerr << ("Done " + m_fi.filename + " - " + std::to_string(count) + " points!\n");
+    m_progress.update(count % CountIncrement);
 
     // Flush any data remaining in the cells.
     m_cellMgr.flush();

epf/FileProcessor.hpp

@@ -17,6 +17,9 @@
 
 namespace untwine
 {
+
+class ProgressWriter;
+
 namespace epf
 {
 
@@ -26,7 +29,8 @@ class Writer;
 class FileProcessor
 {
 public:
-    FileProcessor(const FileInfo& fi, size_t pointSize, const Grid& grid, Writer *writer);
+    FileProcessor(const FileInfo& fi, size_t pointSize, const Grid& grid, Writer *writer,
+        ProgressWriter& progress);
 
     Cell *getCell(const VoxelKey& key);
     void run();
@@ -35,6 +39,7 @@ class FileProcessor
     FileInfo m_fi;
     CellMgr m_cellMgr;
     Grid m_grid;
+    ProgressWriter& m_progress;
 };
 
 } // namespace epf

untwine/Common.hpp

@@ -1,21 +1,27 @@
 #pragma once
 
+#include <stdint.h>
 #include <string>
+#include <vector>
 
 namespace untwine
 {
 
+using PointCount = uint64_t;
+using StringList = std::vector<std::string>;
+
 void fatal(const std::string& err);
 
 struct Options
 {
     std::string outputDir;
-    pdal::StringList inputFiles;
+    StringList inputFiles;
     std::string tempDir;
     bool doCube;
     size_t fileLimit;
     int level;
     int progressFd;
+    StringList dimNames;
 };
 
 const std::string MetadataFilename {"info2.txt"};

untwine/ProgressWriter.cpp

@@ -13,16 +13,14 @@
 namespace untwine
 {
 
-std::mutex ProgressWriter::s_mutex;
-
 ProgressWriter::ProgressWriter(int fd) : m_progressFd(fd), m_percent(0.0), m_increment(.1)
 {}
 
 void ProgressWriter::setIncrement(double increment)
 {
     if (!m_progressFd)
         return;
-    std::unique_lock<std::mutex> lock(s_mutex);
+    std::unique_lock<std::mutex> lock(m_mutex);
 
     m_increment = increment;
 }
@@ -31,7 +29,7 @@ void ProgressWriter::setPercent(double percent)
 {
     if (!m_progressFd)
         return;
-    std::unique_lock<std::mutex> lock(s_mutex);
+    std::unique_lock<std::mutex> lock(m_mutex);
 
     m_percent = (std::max)(0.0, ((std::min)(1.0, percent)));
 }
@@ -40,7 +38,7 @@ void ProgressWriter::writeIncrement(const std::string& message)
 {
     if (!m_progressFd)
         return;
-    std::unique_lock<std::mutex> lock(s_mutex);
+    std::unique_lock<std::mutex> lock(m_mutex);
 
     m_percent += m_increment;
     m_percent = (std::min)(1.0, m_percent);
@@ -53,7 +51,7 @@ void ProgressWriter::write(double percent, const std::string& message)
 {
     if (!m_progressFd)
         return;
-    std::unique_lock<std::mutex> lock(s_mutex);
+    std::unique_lock<std::mutex> lock(m_mutex);
 
     m_percent = (std::min)(0.0, ((std::max)(1.0, percent)));
 
@@ -78,4 +76,18 @@ void ProgressWriter::writeMessage(uint32_t percent, const std::string& message)
 #endif
 }
 
+void ProgressWriter::update(PointCount count)
+{
+    std::unique_lock<std::mutex> lock(m_mutex);
+
+    PointCount inc = m_current / m_threshold;
+    m_current += count;
+    PointCount postInc = m_current / m_threshold;
+    if (inc != postInc)
+    {
+        lock.unlock();
+        writeIncrement("Processed " + std::to_string(m_current) + " points");
+    }
+}
+
 } // namespace untwine