diff --git a/src/qlever-petrimaps/GeomCache.cpp b/src/qlever-petrimaps/GeomCache.cpp index c72f6fd..f0d5552 100644 --- a/src/qlever-petrimaps/GeomCache.cpp +++ b/src/qlever-petrimaps/GeomCache.cpp @@ -7,6 +7,7 @@ #include #include +#include #include #include #include @@ -384,8 +385,8 @@ double GeomCache::getLoadStatusPercent(bool total) { } if (!total) { - return std::atomic(_curRow) / static_cast(_totalSize) * - 100.0; + double percent = _curRow / static_cast(_totalSize) * 100.0; + return std::min(100.0, percent); } double parsePercent = 95.0; @@ -393,22 +394,32 @@ double GeomCache::getLoadStatusPercent(bool total) { double totalPercent = 0.0; switch (_loadStatusStage) { case _LoadStatusStages::Parse: - totalPercent = std::atomic(_curRow) / - static_cast(_totalSize) * parsePercent; + totalPercent = _curRow / static_cast(_totalSize) * parsePercent; break; + case _LoadStatusStages::ParseIds: totalPercent = parsePercent; - totalPercent += std::atomic(_curRow) / - static_cast(_totalSize) * parseIdsPercent; + totalPercent += + _curRow / static_cast(_totalSize) * parseIdsPercent; + break; + + case _LoadStatusStages::FromFile: + totalPercent = _curRow / static_cast(_totalSize) * 100.0; break; } - return totalPercent; + return std::min(100.0, totalPercent); } // _____________________________________________________________________________ int GeomCache::getLoadStatusStage() { return _loadStatusStage; } +// _____________________________________________________________________________ +size_t GeomCache::getTotalProgress() { return _totalSize; } + +// _____________________________________________________________________________ +size_t GeomCache::getCurrentProgress() { return _curRow; } + // _____________________________________________________________________________ void GeomCache::parseIds(const char *c, size_t size) { _loadStatusStage = _LoadStatusStages::ParseIds; @@ -1036,6 +1047,7 @@ std::string GeomCache::indexHashFromDisk(const std::string &fname) { // _____________________________________________________________________________ void GeomCache::fromDisk(const std::string &fname) { + _loadStatusStage = _LoadStatusStages::FromFile; _points.clear(); _linePoints.clear(); _lines.clear(); @@ -1046,27 +1058,73 @@ void GeomCache::fromDisk(const std::string &fname) { char tmp[100]; f.read(tmp, 100); tmp[99] = 0; - _indexHash = util::trim(tmp); size_t numPoints; + size_t numLinePoints; + size_t numLines; + size_t numQidToId; + std::streampos posPoints; + std::streampos posLinePoints; + std::streampos posLines; + std::streampos posQidToId; + // get total num points + // points f.read(reinterpret_cast(&numPoints), sizeof(size_t)); _points.resize(numPoints); - f.read(reinterpret_cast(&_points[0]), - sizeof(util::geo::FPoint) * numPoints); + posPoints = f.tellg(); + f.seekg(sizeof(util::geo::FPoint) * numPoints, f.cur); + + // linePoints + f.read(reinterpret_cast(&numLinePoints), sizeof(size_t)); + _linePoints.resize(numLinePoints); + posLinePoints = f.tellg(); + f.seekg(sizeof(util::geo::Point) * numLinePoints, f.cur); + + // lines + f.read(reinterpret_cast(&numLines), sizeof(size_t)); + _lines.resize(numLines); + posLines = f.tellg(); + f.seekg(sizeof(size_t) * numLines, f.cur); + + // qidToId + f.read(reinterpret_cast(&numQidToId), sizeof(size_t)); + _qidToId.resize(numQidToId); + posQidToId = f.tellg(); + f.seekg(sizeof(IdMapping) * numQidToId, f.cur); + + _totalSize = numPoints + numLinePoints + numLines + numQidToId; + _curRow = 0; - f.read(reinterpret_cast(&numPoints), sizeof(size_t)); - _linePoints.resize(numPoints); - f.read(reinterpret_cast(&_linePoints[0]), - sizeof(util::geo::Point) * numPoints); + // read data from file + // points + f.seekg(posPoints); + for (size_t i = 0; i < numPoints; i++) { + f.read(reinterpret_cast(&_points[i]), sizeof(util::geo::FPoint)); + _curRow += 1; + } - f.read(reinterpret_cast(&numPoints), sizeof(size_t)); - _lines.resize(numPoints); - f.read(reinterpret_cast(&_lines[0]), sizeof(size_t) * numPoints); + // linePoints + f.seekg(posLinePoints); + for (size_t i = 0; i < numLinePoints; i++) { + f.read(reinterpret_cast(&_linePoints[i]), + sizeof(util::geo::Point)); + _curRow += 1; + } - f.read(reinterpret_cast(&numPoints), sizeof(size_t)); - _qidToId.resize(numPoints); - f.read(reinterpret_cast(&_qidToId[0]), sizeof(IdMapping) * numPoints); + // lines + f.seekg(posLines); + for (size_t i = 0; i < numLines; i++) { + f.read(reinterpret_cast(&_lines[i]), sizeof(size_t)); + _curRow += 1; + } + + // qidToId + f.seekg(posQidToId); + for (size_t i = 0; i < numQidToId; i++) { + f.read(reinterpret_cast(&_qidToId[i]), sizeof(IdMapping)); + _curRow += 1; + } f.close(); } diff --git a/src/qlever-petrimaps/GeomCache.h b/src/qlever-petrimaps/GeomCache.h index 1c197cf..211f966 100644 --- a/src/qlever-petrimaps/GeomCache.h +++ b/src/qlever-petrimaps/GeomCache.h @@ -13,6 +13,7 @@ #include #include #include +#include #include "qlever-petrimaps/Misc.h" #include "util/geo/Geo.h" @@ -24,8 +25,7 @@ class GeomCache { GeomCache() : _backendUrl(""), _curl(0) {} explicit GeomCache(const std::string& backendUrl) : _backendUrl(backendUrl), - _curl(curl_easy_init()) { - } + _curl(curl_easy_init()) {} GeomCache& operator=(GeomCache&& o) { _backendUrl = o._backendUrl; @@ -92,6 +92,8 @@ class GeomCache { double getLoadStatusPercent(bool total); double getLoadStatusPercent() { return getLoadStatusPercent(false); }; int getLoadStatusStage(); + size_t getTotalProgress(); + size_t getCurrentProgress(); private: std::string _backendUrl; @@ -100,18 +102,17 @@ class GeomCache { uint8_t _curByte; ID _curId; QLEVER_ID_TYPE _maxQid; - size_t _curRow, _curUniqueGeom; + size_t _totalSize = 0; + std::atomic _curRow; + size_t _curUniqueGeom; - enum _LoadStatusStages {Parse = 1, ParseIds}; + enum _LoadStatusStages {Parse = 1, ParseIds, FromFile}; _LoadStatusStages _loadStatusStage = Parse; static size_t writeCb(void* contents, size_t size, size_t nmemb, void* userp); - static size_t writeCbIds(void* contents, size_t size, size_t nmemb, - void* userp); - static size_t writeCbCount(void* contents, size_t size, size_t nmemb, - void* userp); - static size_t writeCbString(void* contents, size_t size, size_t nmemb, - void* userp); + static size_t writeCbIds(void* contents, size_t size, size_t nmemb, void* userp); + static size_t writeCbCount(void* contents, size_t size, size_t nmemb, void* userp); + static size_t writeCbString(void* contents, size_t size, size_t nmemb, void* userp); // Get the right SPARQL query for the given backend. const std::string& getQuery(const std::string& backendUrl) const; @@ -145,7 +146,6 @@ class GeomCache { std::fstream _linesF; std::fstream _qidToIdF; - size_t _totalSize = 0; size_t _geometryDuplicates = 0; IdMapping _lastQidToId; diff --git a/src/qlever-petrimaps/server/Server.cpp b/src/qlever-petrimaps/server/Server.cpp index 6b138fe..1257563 100755 --- a/src/qlever-petrimaps/server/Server.cpp +++ b/src/qlever-petrimaps/server/Server.cpp @@ -48,6 +48,7 @@ using util::geo::LineSegment; using util::geo::webMercToLatLng; const static double THRESHOLD = 200; +static std::atomic _curRow; // _____________________________________________________________________________ Server::Server(size_t maxMemory, const std::string& cacheDir, int cacheLifetime) @@ -154,7 +155,6 @@ util::http::Answer Server::handleHeatMapReq(const Params& pars, if (box.size() != 4) throw std::invalid_argument("Invalid request."); std::shared_ptr r; - { std::lock_guard guard(_m); bool has = _rs.count(id); @@ -297,7 +297,6 @@ util::http::Answer Server::handleHeatMapReq(const Params& pars, } // LINES - const auto& lgrid = r->getLineGrid(); if (intersects(lgrid.getBBox(), fbbox)) { @@ -638,7 +637,6 @@ util::http::Answer Server::handlePosReq(const Params& pars) const { LOG(INFO) << "[SERVER] Click at " << x << ", " << y; std::shared_ptr reqor; - { std::lock_guard guard(_m); bool has = _rs.count(id); @@ -846,6 +844,10 @@ std::string Server::parseUrl(std::string u, std::string pl, return util::urlDecode(parts.front()); } +void Server::pngWriteRowCb(png_structp png_ptr, png_uint_32 row, int pass) { + _curRow = row; +} + // _____________________________________________________________________________ inline void pngWriteCb(png_structp png_ptr, png_bytep data, png_size_t length) { int sock = *((int*)png_get_io_ptr(png_ptr)); @@ -874,7 +876,7 @@ inline void pngErrorCb(png_structp, png_const_charp error_msg) { } // _____________________________________________________________________________ -void Server::writePNG(const unsigned char* data, size_t w, size_t h, int sock) { +void Server::writePNG(const unsigned char* data, size_t w, size_t h, int sock) const { png_structp png_ptr = png_create_write_struct(PNG_LIBPNG_VER_STRING, nullptr, pngErrorCb, pngWarnCb); if (!png_ptr) return; @@ -890,8 +892,12 @@ void Server::writePNG(const unsigned char* data, size_t w, size_t h, int sock) { return; } - png_set_write_fn(png_ptr, &sock, pngWriteCb, 0); + // Handle Load Status + _totalSize = h; + _curRow = 0; + png_set_write_status_fn(png_ptr, pngWriteRowCb); + png_set_write_fn(png_ptr, &sock, pngWriteCb, 0); png_set_filter(png_ptr, 0, PNG_FILTER_NONE | PNG_FILTER_VALUE_NONE); png_set_compression_level(png_ptr, 7); @@ -901,8 +907,7 @@ void Server::writePNG(const unsigned char* data, size_t w, size_t h, int sock) { png_set_IHDR(png_ptr, info_ptr, w, h, bit_depth, color_type, interlace_type, PNG_COMPRESSION_TYPE_DEFAULT, PNG_FILTER_TYPE_DEFAULT); - png_bytep* row_pointers = - (png_byte**)png_malloc(png_ptr, h * sizeof(png_bytep)); + png_bytep* row_pointers = (png_byte**)png_malloc(png_ptr, h * sizeof(png_bytep)); for (size_t y = 0; y < h; ++y) { row_pointers[y] = const_cast(data + y * w * 4); @@ -1125,12 +1130,30 @@ util::http::Answer Server::handleLoadStatusReq(const Params& pars) const { auto backend = pars.find("backend")->second; createCache(backend); std::shared_ptr cache = _caches[backend]; - double loadStatusPercent = cache->getLoadStatusPercent(true); + + // We have 3 loading stages: + // 1) Filling geometry cache / reading cache from disk + // 2) Fetching geometries + // 3) Rendering result + // 1) + 2) by GeomCache, 3) by Server + // => Merge load status + // 1) + 2) = 95%, 3) = 5% + + double geomCachePercent = 0.95; + double serverPercent = 0.05; + double geomCacheLoadStatusPercent = cache->getLoadStatusPercent(true); + double serverLoadStatusPercent = getLoadStatusPercent(); + double totalPercent = geomCachePercent * geomCacheLoadStatusPercent + serverPercent * serverLoadStatusPercent; + int loadStatusStage = cache->getLoadStatusStage(); + size_t totalProgress = cache->getTotalProgress(); + size_t currentProgress = cache->getCurrentProgress(); std::stringstream json; - json << "{\"percent\": " << loadStatusPercent - << ", \"stage\": " << loadStatusStage << "}"; + json << "{\"percent\": " << totalPercent + << ", \"stage\": " << loadStatusStage + << ", \"totalProgress\": " << totalProgress + << ", \"currentProgress\": " << currentProgress << "}"; util::http::Answer ans = util::http::Answer("200 OK", json.str()); return ans; @@ -1168,6 +1191,17 @@ std::string Server::getSessionId() const { return std::to_string(d(rng)); } +double Server::getLoadStatusPercent() const { + if (_totalSize == 0) { + return 0.0; + } + + double percent = _curRow / static_cast(_totalSize) * 100.0; + assert(percent <= 100.0); + + return percent; +} + void Server::createCache(const std::string& backend) const { std::shared_ptr cache; diff --git a/src/qlever-petrimaps/server/Server.h b/src/qlever-petrimaps/server/Server.h index 8239a73..30a5b1e 100755 --- a/src/qlever-petrimaps/server/Server.h +++ b/src/qlever-petrimaps/server/Server.h @@ -11,6 +11,7 @@ #include #include +#include #include "qlever-petrimaps/GeomCache.h" #include "qlever-petrimaps/server/Requestor.h" #include "util/http/Server.h" @@ -51,7 +52,10 @@ class Server : public util::http::Handler { std::string getSessionId() const; - static void writePNG(const unsigned char* data, size_t w, size_t h, int sock); + double getLoadStatusPercent() const; + + static void pngWriteRowCb(png_structp png_ptr, png_uint_32 row, int pass); + void writePNG(const unsigned char* data, size_t w, size_t h, int sock) const; void drawPoint(std::vector& points, std::vector& points2, int px, int py, int w, int h, MapStyle style, @@ -65,6 +69,9 @@ class Server : public util::http::Handler { int _cacheLifetime; + // Load Status + mutable size_t _totalSize = 0; + mutable std::mutex _m; mutable std::map> _caches; diff --git a/web/index.html b/web/index.html index 15a09a7..bbb0291 100755 --- a/web/index.html +++ b/web/index.html @@ -24,11 +24,15 @@
-
Loading results from QLever
-
- +
Loading results from QLever
+
+
Filling the geometry cache
+
This needs to be done only once after the server has been started and does not have to be repeated for subsequent queries.
+
+
+
-
Parsing geometry... (1/2)
+
0.00%
diff --git a/web/merged.js b/web/merged.js index d501608..b07d83f 100644 --- a/web/merged.js +++ b/web/merged.js @@ -131,10 +131,10 @@ function getGeoJsonLayer(geom) { function showError(error) { document.getElementById("msg").style.display = "block"; document.getElementById("loader").style.display = "none"; - document.getElementById("msg-inner").style.color = "red"; - document.getElementById("msg-inner").style.fontWeight = "bold"; - document.getElementById("msg-inner").style.fontSize = "20px"; - document.getElementById("msg-inner").innerHTML = error; + document.getElementById("msg-heading").style.color = "red"; + document.getElementById("msg-heading").style.fontWeight = "bold"; + document.getElementById("msg-heading").style.fontSize = "20px"; + document.getElementById("msg-heading").innerHTML = error; } function loadMap(id, bounds, numObjects) { diff --git a/web/script.js b/web/script.js index 8ab28d2..381bcab 100755 --- a/web/script.js +++ b/web/script.js @@ -126,16 +126,16 @@ function getGeoJsonLayer(geom) { function showError(error) { error = error.toString(); document.getElementById("msg").style.display = "block"; + document.getElementById("msg-info").style.display = "none"; document.getElementById("load").style.display = "none"; - document.getElementById("msg-inner").style.color = "red"; - document.getElementById("msg-inner").style.fontSize = "20px"; - document.getElementById("msg-inner").innerHTML = error.split("\n")[0]; - if (error.search("\n") > 0) document.getElementById("msg-inner-desc").innerHTML = "
" + error.substring(error.search("\n")) + "
"; - else document.getElementById("msg-inner-desc").innerHTML = ""; + document.getElementById("msg-heading").style.color = "red"; + document.getElementById("msg-heading").style.fontSize = "20px"; + document.getElementById("msg-heading").innerHTML = error.split("\n")[0]; + if (error.search("\n") > 0) document.getElementById("msg-error").innerHTML = "
" + error.substring(error.search("\n")) + "
"; + else document.getElementById("msg-error").innerHTML = ""; } function loadMap(id, bounds, numObjects) { - document.getElementById("msg").style.display = "none"; const ll = L.Projection.SphericalMercator.unproject({"x": bounds[0][0], "y":bounds[0][1]}); const ur = L.Projection.SphericalMercator.unproject({"x": bounds[1][0], "y":bounds[1][1]}); const boundsLatLng = [[ll.lat, ll.lng], [ur.lat, ur.lng]]; @@ -162,41 +162,40 @@ function loadMap(id, bounds, numObjects) { format: 'image/png' }); - const autoLayer = L.layerGroup([ - L.nonTiledLayer.wms('heatmap', { - minZoom: 0, - maxZoom: 15, - opacity: 0.8, - layers: id, - styles: ["heatmap"], - format: 'image/png', - transparent: true, - }), L.nonTiledLayer.wms('heatmap', { - minZoom: 16, - maxZoom: 19, - layers: id, - styles: ["objects"], - format: 'image/png' - }) - ]); + const autoHeatmapLayer = L.nonTiledLayer.wms('heatmap', { + minZoom: 0, + maxZoom: 15, + opacity: 0.8, + layers: id, + styles: ["heatmap"], + format: 'image/png', + transparent: true, + }); + + const autoObjectLayer = L.nonTiledLayer.wms('heatmap', { + minZoom: 16, + maxZoom: 19, + layers: id, + styles: ["objects"], + format: 'image/png' + }); + const autoLayerGroup = L.layerGroup([autoHeatmapLayer, autoObjectLayer]); - heatmapLayer.on('error', function() {showError(genError);}); - objectsLayer.on('error', function() {showError(genError);}); - heatmapLayer.on('load', function() {console.log("Finished loading map!");}); - objectsLayer.on('load', function() {console.log("Finished loading map!");}); - autoLayer.on('error', function() {showError(genError);}); - autoLayer.on('load', function() {console.log("Finished loading map!");}); + heatmapLayer.on('load', _onLayerLoad); + objectsLayer.on('load', _onLayerLoad); + autoHeatmapLayer.on('load', _onLayerLoad); + autoObjectLayer.on('load', _onLayerLoad); layerControl.addBaseLayer(heatmapLayer, "Heatmap"); layerControl.addBaseLayer(objectsLayer, "Objects"); - layerControl.addBaseLayer(autoLayer, "Auto"); + layerControl.addBaseLayer(autoLayerGroup, "Auto"); if (mode == "heatmap") { heatmapLayer.addTo(map).on('error', function() {showError(genError);}); } else if (mode == "objects") { objectsLayer.addTo(map).on('error', function() {showError(genError);}); } else { - autoLayer.addTo(map).on('error', function() {showError(genError);}); + autoLayerGroup.addTo(map).on('error', function() {showError(genError);}); } map.on('click', function(e) { @@ -214,7 +213,6 @@ function loadMap(id, bounds, numObjects) { let styles = "objects"; if (map.hasLayer(heatmapLayer)) styles = "heatmap"; if (map.hasLayer(objectsLayer)) styles = "objects"; - if (map.hasLayer(objectsLayer)) styles = "objects"; fetch('pos?x=' + pos.x + "&y=" + pos.y + "&id=" + id + "&rad=" + (100 * Math.pow(2, 14 - map.getZoom())) + '&width=' + w + '&height=' + h + '&bbox=' + bounds.join(',') + '&styles=' + styles) .then(response => { @@ -239,20 +237,33 @@ function loadMap(id, bounds, numObjects) { }); } -function updateLoad(stage, percent) { +function updateLoad(stage, percent, totalProgress, currentProgress) { + const infoElem = document.getElementById("msg-info"); + const infoHeadingElem = document.getElementById("msg-info-heading"); + const infoDescElem = document.getElementById("msg-info-desc"); const stageElem = document.getElementById("load-stage"); const barElem = document.getElementById("load-bar"); const percentElem = document.getElementById("load-percent"); switch (stage) { case 1: - stageElem.innerHTML = "Parsing geometry... (1/2)"; + infoHeadingElem.innerHTML = "Filling the geometry cache"; + infoDescElem.innerHTML = "This needs to be done only once for each new version of the dataset and does not have to be repeated for subsequent queries."; + stageElem.innerHTML = `Parsing ${currentProgress}/${totalProgress} geometries... (1/2)`; break; case 2: - stageElem.innerHTML = "Parsing geometry Ids... (2/2)"; + infoHeadingElem.innerHTML = "Filling the geometry cache"; + infoDescElem.innerHTML = "This needs to be done only once for each new version of the dataset and does not have to be repeated for subsequent queries."; + stageElem.innerHTML = `Fetching ${currentProgress}/${totalProgress} geometries... (2/2)`; + break; + case 3: + infoHeadingElem.innerHTML = "Reading cached geometries from disk"; + infoDescElem.innerHTML = "This needs to be done only once after the server has been started and does not have to be repeated for subsequent queries."; + stageElem.innerHTML = `Reading ${currentProgress}/${totalProgress} geometries from disk... (1/1)`; break; } barElem.style.width = percent + "%"; percentElem.innerHTML = percent.toString() + "%"; + infoElem.style.display = "block"; } function fetchResults() { @@ -265,7 +276,6 @@ function fetchResults() { }) .then(response => response.json()) .then(data => { - clearInterval(loadStatusIntervalId); loadMap(data["qid"], data["bounds"], data["numobjects"]); }) .catch(error => showError(error)); @@ -283,12 +293,14 @@ async function fetchLoadStatus() { .then(response => { if (!response.ok) return response.text().then(text => {throw new Error(text)}); return response; - }) + }) .then(response => response.json()) .then(data => { var stage = data["stage"]; var percent = parseFloat(data["percent"]).toFixed(2); - updateLoad(stage, percent); + var totalProgress = data["totalProgress"].toLocaleString('en'); + var currentProgress = data["currentProgress"].toLocaleString('en'); + updateLoad(stage, percent, totalProgress, currentProgress); }) .catch(error => { showError(error); @@ -299,6 +311,13 @@ async function fetchLoadStatus() { fetchResults(); fetchLoadStatusInterval(333); +function _onLayerLoad(e) { + console.log("Map finished loading."); + clearInterval(loadStatusIntervalId); + + document.getElementById("msg").style.display = "none"; +} + document.getElementById("ex-geojson").onclick = function() { if (!sessionId) return; let a = document.createElement("a"); diff --git a/web/style.css b/web/style.css index 8ecdab4..cf2621d 100755 --- a/web/style.css +++ b/web/style.css @@ -630,7 +630,7 @@ tt { margin: 5px -10px 5px -6px; } -h1, #msg-inner { +h1, #msg-heading { font: 1.5em sans-serif; margin-bottom: 40px; opacity: 0.8; @@ -650,7 +650,15 @@ h1, #msg-inner { padding: 50px; } -#msg-inner-desc { +#msg-info-heading { + font-weight: bold; +} + +#msg-info-desc { + margin-bottom: 40px; +} + +#msg-error { font: 0.9em mono; text-align: left; background-color: #fefefe;