diff --git a/android/tangram/src/main/cpp/jniExports.cpp b/android/tangram/src/main/cpp/jniExports.cpp index 59e0e45319..96d5b58d91 100644 --- a/android/tangram/src/main/cpp/jniExports.cpp +++ b/android/tangram/src/main/cpp/jniExports.cpp @@ -357,29 +357,29 @@ extern "C" { map->markerRemoveAll(); } - JNIEXPORT jlong JNICALL Java_com_mapzen_tangram_MapController_nativeAddDataSource(JNIEnv* jniEnv, jobject obj, jlong mapPtr, jstring name) { + JNIEXPORT jlong JNICALL Java_com_mapzen_tangram_MapController_nativeAddTileSource(JNIEnv* jniEnv, jobject obj, jlong mapPtr, jstring name) { assert(mapPtr > 0); auto map = reinterpret_cast(mapPtr); auto sourceName = stringFromJString(jniEnv, name); - auto source = std::shared_ptr(new Tangram::ClientGeoJsonSource(sourceName, "")); - map->addDataSource(source); + auto source = std::shared_ptr(new Tangram::ClientGeoJsonSource(sourceName, "")); + map->addTileSource(source); return reinterpret_cast(source.get()); } - JNIEXPORT void JNICALL Java_com_mapzen_tangram_MapController_nativeRemoveDataSource(JNIEnv* jniEnv, jobject obj, jlong mapPtr, jlong sourcePtr) { + JNIEXPORT void JNICALL Java_com_mapzen_tangram_MapController_nativeRemoveTileSource(JNIEnv* jniEnv, jobject obj, jlong mapPtr, jlong sourcePtr) { assert(mapPtr > 0); auto map = reinterpret_cast(mapPtr); assert(sourcePtr > 0); - auto source = reinterpret_cast(sourcePtr); - map->removeDataSource(*source); + auto source = reinterpret_cast(sourcePtr); + map->removeTileSource(*source); } - JNIEXPORT void JNICALL Java_com_mapzen_tangram_MapController_nativeClearDataSource(JNIEnv* jniEnv, jobject obj, jlong mapPtr, jlong sourcePtr) { + JNIEXPORT void JNICALL Java_com_mapzen_tangram_MapController_nativeClearTileSource(JNIEnv* jniEnv, jobject obj, jlong mapPtr, jlong sourcePtr) { assert(mapPtr > 0); auto map = reinterpret_cast(mapPtr); assert(sourcePtr > 0); - auto source = reinterpret_cast(sourcePtr); - map->clearDataSource(*source, true, true); + auto source = reinterpret_cast(sourcePtr); + map->clearTileSource(*source, true, true); } JNIEXPORT void JNICALL Java_com_mapzen_tangram_MapController_nativeAddFeature(JNIEnv* jniEnv, jobject obj, jlong mapPtr, jlong sourcePtr, diff --git a/android/tangram/src/main/java/com/mapzen/tangram/MapController.java b/android/tangram/src/main/java/com/mapzen/tangram/MapController.java index d0699c1bc7..8863b64846 100644 --- a/android/tangram/src/main/java/com/mapzen/tangram/MapController.java +++ b/android/tangram/src/main/java/com/mapzen/tangram/MapController.java @@ -220,14 +220,14 @@ void dispose() { public void run() { // Dispose each data sources by first removing it from the HashMap values and then // calling remove(), so that we don't improperly modify the HashMap while iterating. - for (Iterator it = clientDataSources.values().iterator(); it.hasNext();) { + for (Iterator it = clientTileSources.values().iterator(); it.hasNext();) { MapData mapData = it.next(); it.remove(); mapData.remove(); } nativeDispose(mapPointer); mapPointer = 0; - clientDataSources.clear(); + clientTileSources.clear(); markers.clear(); } }); @@ -490,17 +490,17 @@ public PointF lngLatToScreenPosition(LngLat lngLat) { * object will be returned. */ public MapData addDataLayer(String name) { - MapData mapData = clientDataSources.get(name); + MapData mapData = clientTileSources.get(name); if (mapData != null) { return mapData; } checkPointer(mapPointer); - long pointer = nativeAddDataSource(mapPointer, name); + long pointer = nativeAddTileSource(mapPointer, name); if (pointer <= 0) { throw new RuntimeException("Unable to create new data source"); } mapData = new MapData(name, pointer, this); - clientDataSources.put(name, mapData); + clientTileSources.put(name, mapData); return mapData; } @@ -509,10 +509,10 @@ public MapData addDataLayer(String name) { * @param mapData The {@code MapData} to remove */ void removeDataLayer(MapData mapData) { - clientDataSources.remove(mapData.name); + clientTileSources.remove(mapData.name); checkPointer(mapPointer); checkPointer(mapData.pointer); - nativeRemoveDataSource(mapPointer, mapData.pointer); + nativeRemoveTileSource(mapPointer, mapData.pointer); } /** @@ -841,16 +841,16 @@ public void useCachedGlState(boolean use) { // Package private methods // ======================= - void removeDataSource(long sourcePtr) { + void removeTileSource(long sourcePtr) { checkPointer(mapPointer); checkPointer(sourcePtr); - nativeRemoveDataSource(mapPointer, sourcePtr); + nativeRemoveTileSource(mapPointer, sourcePtr); } - void clearDataSource(long sourcePtr) { + void clearTileSource(long sourcePtr) { checkPointer(mapPointer); checkPointer(sourcePtr); - nativeClearDataSource(mapPointer, sourcePtr); + nativeClearTileSource(mapPointer, sourcePtr); } void addFeature(long sourcePtr, double[] coordinates, int[] rings, String[] properties) { @@ -1007,9 +1007,9 @@ Marker markerById(long markerId) { private native void nativeOnUrlSuccess(byte[] rawDataBytes, long callbackPtr); private native void nativeOnUrlFailure(long callbackPtr); - synchronized native long nativeAddDataSource(long mapPtr, String name); - synchronized native void nativeRemoveDataSource(long mapPtr, long sourcePtr); - synchronized native void nativeClearDataSource(long mapPtr, long sourcePtr); + synchronized native long nativeAddTileSource(long mapPtr, String name); + synchronized native void nativeRemoveTileSource(long mapPtr, long sourcePtr); + synchronized native void nativeClearTileSource(long mapPtr, long sourcePtr); synchronized native void nativeAddFeature(long mapPtr, long sourcePtr, double[] coordinates, int[] rings, String[] properties); synchronized native void nativeAddGeoJson(long mapPtr, long sourcePtr, String geoJson); @@ -1033,7 +1033,7 @@ Marker markerById(long markerId) { private ViewCompleteListener viewCompleteListener; private FrameCaptureCallback frameCaptureCallback; private boolean frameCaptureAwaitCompleteView; - private Map clientDataSources = new HashMap<>(); + private Map clientTileSources = new HashMap<>(); private Map markers = new HashMap<>(); // GLSurfaceView.Renderer methods diff --git a/android/tangram/src/main/java/com/mapzen/tangram/MapData.java b/android/tangram/src/main/java/com/mapzen/tangram/MapData.java index dd04aed4be..e501715d52 100644 --- a/android/tangram/src/main/java/com/mapzen/tangram/MapData.java +++ b/android/tangram/src/main/java/com/mapzen/tangram/MapData.java @@ -113,7 +113,7 @@ public MapData addGeoJson(String data) { * @return This object, for chaining. */ public MapData clear() { - map.clearDataSource(pointer); + map.clearTileSource(pointer); return this; } diff --git a/bench/src/tileLoading.cpp b/bench/src/tileLoading.cpp index 3d8482e286..974cf5fcf8 100644 --- a/bench/src/tileLoading.cpp +++ b/bench/src/tileLoading.cpp @@ -2,7 +2,7 @@ #include "gl.h" #include "platform.h" #include "log.h" -#include "data/dataSource.h" +#include "data/tileSource.h" #include "scene/sceneLoader.h" #include "scene/scene.h" #include "style/style.h" @@ -30,7 +30,7 @@ struct TestContext { std::shared_ptr scene; StyleContext styleContext; - std::shared_ptr source; + std::shared_ptr source; std::vector rawTileData; @@ -54,7 +54,7 @@ struct TestContext { styleContext.initFunctions(*scene); styleContext.setKeywordZoom(0); - source = *scene->dataSources().begin(); + source = *scene->tileSources().begin(); tileBuilder = std::make_unique(scene); } @@ -76,9 +76,9 @@ struct TestContext { void parseTile() { Tile tile({0,0,10,10,0}, s_projection); - source = *scene->dataSources().begin(); + source = *scene->tileSources().begin(); auto task = source->createTask(tile.getID()); - auto& t = dynamic_cast(*task); + auto& t = dynamic_cast(*task); t.rawTileData = std::make_shared>(rawTileData); tileData = source->parse(*task, s_projection); diff --git a/core/src/data/clientGeoJsonSource.cpp b/core/src/data/clientGeoJsonSource.cpp index 6b18168182..1f6e2b9a17 100644 --- a/core/src/data/clientGeoJsonSource.cpp +++ b/core/src/data/clientGeoJsonSource.cpp @@ -34,7 +34,7 @@ Point transformPoint(geojsonvt::TilePoint pt) { // TODO: pass scene's resourcePath to constructor to be used with `stringFromFile` ClientGeoJsonSource::ClientGeoJsonSource(const std::string& _name, const std::string& _url, int32_t _minDisplayZoom, int32_t _maxDisplayZoom, int32_t _maxZoom) - : DataSource(_name, _url, _minDisplayZoom, _maxDisplayZoom, _maxZoom) { + : TileSource(_name, nullptr, _minDisplayZoom, _maxDisplayZoom, _maxZoom) { // TODO: handle network url for client datasource data // TODO: generic uri handling @@ -74,15 +74,20 @@ void ClientGeoJsonSource::addData(const std::string& _data) { } -bool ClientGeoJsonSource::loadTileData(std::shared_ptr&& _task, TileTaskCb _cb) { +void ClientGeoJsonSource::loadTileData(std::shared_ptr _task, TileTaskCb _cb) { if (m_hasPendingData) { - return false; + return; } - _cb.func(std::move(_task)); + if (_task->needsLoading()) { + _task->startedLoading(); - return true; + _cb.func(_task); + } + + // Load subsources + TileSource::loadTileData(_task, _cb); } void ClientGeoJsonSource::clearData() { diff --git a/core/src/data/clientGeoJsonSource.h b/core/src/data/clientGeoJsonSource.h index 7376f4cd96..e3d80afc6a 100644 --- a/core/src/data/clientGeoJsonSource.h +++ b/core/src/data/clientGeoJsonSource.h @@ -1,6 +1,6 @@ #pragma once -#include "dataSource.h" +#include "tileSource.h" #include "util/types.h" #include @@ -19,7 +19,7 @@ using GeoJSONVT = mapbox::util::geojsonvt::GeoJSONVT; struct Properties; -class ClientGeoJsonSource : public DataSource { +class ClientGeoJsonSource : public TileSource { public: @@ -33,7 +33,7 @@ class ClientGeoJsonSource : public DataSource { void addLine(const Properties& _tags, const Coordinates& _line); void addPoly(const Properties& _tags, const std::vector& _poly); - virtual bool loadTileData(std::shared_ptr&& _task, TileTaskCb _cb) override; + virtual void loadTileData(std::shared_ptr _task, TileTaskCb _cb) override; std::shared_ptr createTask(TileID _tileId, int _subTask) override; virtual void cancelLoadingTile(const TileID& _tile) override {}; diff --git a/core/src/data/dataSource.cpp b/core/src/data/dataSource.cpp deleted file mode 100644 index fcb2442bca..0000000000 --- a/core/src/data/dataSource.cpp +++ /dev/null @@ -1,232 +0,0 @@ -#include "dataSource.h" -#include "util/geoJson.h" -#include "platform.h" -#include "tileData.h" -#include "tile/tileID.h" -#include "tile/tileHash.h" -#include "tile/tile.h" -#include "tile/tileManager.h" -#include "tile/tileTask.h" -#include "gl/texture.h" -#include "log.h" - -#include -#include -#include -#include -#include - -namespace Tangram { - -struct RawCache { - - // Used to ensure safe access from async loading threads - std::mutex m_mutex; - - // LRU in-memory cache for raw tile data - using CacheEntry = std::pair>>; - using CacheList = std::list; - using CacheMap = std::unordered_map; - - CacheMap m_cacheMap; - CacheList m_cacheList; - int m_usage = 0; - int m_maxUsage = 0; - - bool get(DownloadTileTask& _task) { - - if (m_maxUsage <= 0) { return false; } - - std::lock_guard lock(m_mutex); - TileID id(_task.tileId().x, _task.tileId().y, _task.tileId().z); - - auto it = m_cacheMap.find(id); - if (it != m_cacheMap.end()) { - // Move cached entry to start of list - m_cacheList.splice(m_cacheList.begin(), m_cacheList, it->second); - _task.rawTileData = m_cacheList.front().second; - - return true; - } - - return false; - } - void put(const TileID& tileID, std::shared_ptr> rawDataRef) { - - if (m_maxUsage <= 0) { return; } - - std::lock_guard lock(m_mutex); - TileID id(tileID.x, tileID.y, tileID.z); - - m_cacheList.push_front({id, rawDataRef}); - m_cacheMap[id] = m_cacheList.begin(); - - m_usage += rawDataRef->size(); - - while (m_usage > m_maxUsage) { - if (m_cacheList.empty()) { - LOGE("Error: invalid cache state!"); - m_usage = 0; - break; - } - - // LOGE("Limit raw cache tiles:%d, %fMB ", m_cacheList.size(), - // double(m_cacheUsage) / (1024*1024)); - - auto& entry = m_cacheList.back(); - m_usage -= entry.second->size(); - - m_cacheMap.erase(entry.first); - m_cacheList.pop_back(); - } - } - - void clear() { - std::lock_guard lock(m_mutex); - m_cacheMap.clear(); - m_cacheList.clear(); - m_usage = 0; - } -}; - -DataSource::DataSource(const std::string& _name, const std::string& _urlTemplate, - int32_t _minDisplayZoom, int32_t _maxDisplayZoom, int32_t _maxZoom) : - m_name(_name), - m_minDisplayZoom(_minDisplayZoom), m_maxDisplayZoom(_maxDisplayZoom), m_maxZoom(_maxZoom), - m_urlTemplate(_urlTemplate), - m_cache(std::make_unique()){ - - static std::atomic s_serial; - - m_id = s_serial++; -} - -DataSource::~DataSource() { - clearData(); -} - -std::shared_ptr DataSource::createTask(TileID _tileId, int _subTask) { - auto task = std::make_shared(_tileId, shared_from_this(), _subTask); - - cacheGet(*task); - - return task; -} - -void DataSource::setCacheSize(size_t _cacheSize) { - m_cache->m_maxUsage = _cacheSize; -} - -bool DataSource::cacheGet(DownloadTileTask& _task) { - return m_cache->get(_task); -} - -void DataSource::cachePut(const TileID& _tileID, std::shared_ptr> _rawDataRef) { - m_cache->put(_tileID, _rawDataRef); -} - -void DataSource::clearData() { - m_cache->clear(); - m_generation++; -} - -void DataSource::constructURL(const TileID& _tileCoord, std::string& _url) const { - _url.assign(m_urlTemplate); - try { - size_t xpos = _url.find("{x}"); - _url.replace(xpos, 3, std::to_string(_tileCoord.x)); - size_t ypos = _url.find("{y}"); - _url.replace(ypos, 3, std::to_string(_tileCoord.y)); - size_t zpos = _url.find("{z}"); - _url.replace(zpos, 3, std::to_string(_tileCoord.z)); - } catch(...) { - LOGE("Bad URL template!"); - } -} - -bool DataSource::equals(const DataSource& other) const { - if (m_name != other.m_name) { return false; } - if (m_urlTemplate != other.m_urlTemplate) { return false; } - if (m_minDisplayZoom != other.m_minDisplayZoom) { return false; } - if (m_maxDisplayZoom != other.m_maxDisplayZoom) { return false; } - if (m_maxZoom != other.m_maxZoom) { return false; } - if (m_rasterSources.size() != other.m_rasterSources.size()) { return false; } - for (size_t i = 0, end = m_rasterSources.size(); i < end; ++i) { - if (!m_rasterSources[i]->equals(*other.m_rasterSources[i])) { return false; } - } - - return true; -} - -void DataSource::onTileLoaded(std::vector&& _rawData, std::shared_ptr&& _task, - TileTaskCb _cb) { - - if (_task->isCanceled()) { return; } - - TileID tileID = _task->tileId(); - - if (!_rawData.empty()) { - - auto rawDataRef = std::make_shared>(); - std::swap(*rawDataRef, _rawData); - - auto& task = static_cast(*_task); - task.rawTileData = rawDataRef; - - _cb.func(std::move(_task)); - - cachePut(tileID, rawDataRef); - } -} - -bool DataSource::loadTileData(std::shared_ptr&& _task, TileTaskCb _cb) { - - std::string url(constructURL(_task->tileId())); - - // lambda captured parameters are const by default, we want "task" (moved) to be non-const, - // hence "mutable" - // Refer: http://en.cppreference.com/w/cpp/language/lambda - return startUrlRequest(url, - [this, _cb, task = std::move(_task)](std::vector&& rawData) mutable { - this->onTileLoaded(std::move(rawData), std::move(task), _cb); - }); - -} - -void DataSource::cancelLoadingTile(const TileID& _tileID) { - cancelUrlRequest(constructURL(_tileID)); - for (auto& raster : m_rasterSources) { - TileID rasterID = _tileID.withMaxSourceZoom(raster->maxZoom()); - raster->cancelLoadingTile(rasterID); - } -} - -void DataSource::clearRasters() { - for (auto& raster : m_rasterSources) { - raster->clearRasters(); - } -} - -void DataSource::clearRaster(const TileID& id) { - for (auto& raster : m_rasterSources) { - TileID rasterID = id.withMaxSourceZoom(raster->maxZoom()); - raster->clearRaster(rasterID); - } -} - -void DataSource::addRasterSource(std::shared_ptr _rasterSource) { - /* - * We limit the parent source by any attached raster source's min/max. - */ - int32_t rasterMinDisplayZoom = _rasterSource->minDisplayZoom(); - int32_t rasterMaxDisplayZoom = _rasterSource->maxDisplayZoom(); - if (rasterMinDisplayZoom > m_minDisplayZoom) { - m_minDisplayZoom = rasterMinDisplayZoom; - } - if (rasterMaxDisplayZoom < m_maxDisplayZoom) { - m_maxDisplayZoom = rasterMaxDisplayZoom; - } - m_rasterSources.push_back(_rasterSource); -} - -} diff --git a/core/src/data/geoJsonSource.cpp b/core/src/data/geoJsonSource.cpp index 6baa745a66..0c167f4883 100644 --- a/core/src/data/geoJsonSource.cpp +++ b/core/src/data/geoJsonSource.cpp @@ -11,15 +11,10 @@ namespace Tangram { -GeoJsonSource::GeoJsonSource(const std::string& _name, const std::string& _urlTemplate, - int32_t _minDisplayZoom, int32_t _maxDisplayZoom, int32_t _maxZoom) : - DataSource(_name, _urlTemplate, _minDisplayZoom, _maxDisplayZoom, _maxZoom) { -} - std::shared_ptr GeoJsonSource::parse(const TileTask& _task, const MapProjection& _projection) const { - auto& task = static_cast(_task); + auto& task = static_cast(_task); std::shared_ptr tileData = std::make_shared(); diff --git a/core/src/data/geoJsonSource.h b/core/src/data/geoJsonSource.h index 4132c0f0e6..be413af838 100644 --- a/core/src/data/geoJsonSource.h +++ b/core/src/data/geoJsonSource.h @@ -1,21 +1,19 @@ #pragma once -#include "dataSource.h" +#include "tileSource.h" namespace Tangram { -class GeoJsonSource: public DataSource { +class GeoJsonSource: public TileSource { + +public: + using TileSource::TileSource; protected: virtual std::shared_ptr parse(const TileTask& _task, const MapProjection& _projection) const override; -public: - - GeoJsonSource(const std::string& _name, const std::string& _urlTemplate, - int32_t _minDisplayZoom, int32_t _maxDisplayZoom, int32_t _maxZoom); - }; } diff --git a/core/src/data/memoryCacheDataSource.cpp b/core/src/data/memoryCacheDataSource.cpp new file mode 100644 index 0000000000..785f62d04b --- /dev/null +++ b/core/src/data/memoryCacheDataSource.cpp @@ -0,0 +1,141 @@ +#include "data/memoryCacheDataSource.h" + +#include "tile/tileHash.h" +#include "tile/tileID.h" +#include "log.h" + +#include +#include +#include + +namespace Tangram { + +struct RawCache { + + // Used to ensure safe access from async loading threads + std::mutex m_mutex; + + // LRU in-memory cache for raw tile data + using CacheEntry = std::pair>>; + using CacheList = std::list; + using CacheMap = std::unordered_map; + + CacheMap m_cacheMap; + CacheList m_cacheList; + int m_usage = 0; + int m_maxUsage = 0; + + bool get(BinaryTileTask& _task) { + + if (m_maxUsage <= 0) { return false; } + + std::lock_guard lock(m_mutex); + TileID id(_task.tileId().x, _task.tileId().y, _task.tileId().z); + + auto it = m_cacheMap.find(id); + if (it != m_cacheMap.end()) { + // Move cached entry to start of list + m_cacheList.splice(m_cacheList.begin(), m_cacheList, it->second); + _task.rawTileData = m_cacheList.front().second; + + return true; + } + + return false; + } + void put(const TileID& tileID, std::shared_ptr> rawDataRef) { + + if (m_maxUsage <= 0) { return; } + + std::lock_guard lock(m_mutex); + TileID id(tileID.x, tileID.y, tileID.z); + + m_cacheList.push_front({id, rawDataRef}); + m_cacheMap[id] = m_cacheList.begin(); + + m_usage += rawDataRef->size(); + + while (m_usage > m_maxUsage) { + if (m_cacheList.empty()) { + LOGE("Error: invalid cache state!"); + m_usage = 0; + break; + } + + // LOGE("Limit raw cache tiles:%d, %fMB ", m_cacheList.size(), + // double(m_cacheUsage) / (1024*1024)); + + auto& entry = m_cacheList.back(); + m_usage -= entry.second->size(); + + m_cacheMap.erase(entry.first); + m_cacheList.pop_back(); + } + } + + void clear() { + std::lock_guard lock(m_mutex); + m_cacheMap.clear(); + m_cacheList.clear(); + m_usage = 0; + } +}; + + +MemoryCacheDataSource::MemoryCacheDataSource() : + m_cache(std::make_unique()) { +} + +MemoryCacheDataSource::~MemoryCacheDataSource() {} + +void MemoryCacheDataSource::setCacheSize(size_t _cacheSize) { + m_cache->m_maxUsage = _cacheSize; +} + +bool MemoryCacheDataSource::cacheGet(BinaryTileTask& _task) { + return m_cache->get(_task); +} + +void MemoryCacheDataSource::cachePut(const TileID& _tileID, std::shared_ptr> _rawDataRef) { + m_cache->put(_tileID, _rawDataRef); +} + +bool MemoryCacheDataSource::loadTileData(std::shared_ptr _task, TileTaskCb _cb) { + + auto& task = static_cast(*_task); + + if (_task->rawSource == this->level) { + + cacheGet(task); + + if (task.hasData()) { + _cb.func(_task); + return true; + } + + // Try next source on subsequent calls + if (next) { _task->rawSource = next->level; } + } + + if (next) { + + return next->loadTileData(_task, {[this, _cb](std::shared_ptr _task) { + + auto& task = static_cast(*_task); + + if (task.hasData()) { cachePut(task.tileId(), task.rawTileData); } + + _cb.func(_task); + }}); + } + + return false; +} + +void MemoryCacheDataSource::clear() { + m_cache->clear(); + + if (next) { next->clear(); } +} + +} diff --git a/core/src/data/memoryCacheDataSource.h b/core/src/data/memoryCacheDataSource.h new file mode 100644 index 0000000000..ba19e98c49 --- /dev/null +++ b/core/src/data/memoryCacheDataSource.h @@ -0,0 +1,31 @@ +#pragma once + +#include "data/tileSource.h" + +namespace Tangram { + +class MemoryCacheDataSource : public TileSource::DataSource { +public: + + MemoryCacheDataSource(); + ~MemoryCacheDataSource(); + + bool loadTileData(std::shared_ptr _task, TileTaskCb _cb) override; + + void clear() override; + + /* @_cacheSize: Set size of in-memory cache for tile data in bytes. + * This cache holds unprocessed tile data for fast recreation of recently used tiles. + */ + void setCacheSize(size_t _cacheSize); + +private: + bool cacheGet(BinaryTileTask& _task); + + void cachePut(const TileID& _tileID, std::shared_ptr> _rawDataRef); + + std::unique_ptr m_cache; + +}; + +} diff --git a/core/src/data/mvtSource.cpp b/core/src/data/mvtSource.cpp index a4c0db76a4..ee909d53b2 100644 --- a/core/src/data/mvtSource.cpp +++ b/core/src/data/mvtSource.cpp @@ -9,17 +9,11 @@ namespace Tangram { - -MVTSource::MVTSource(const std::string& _name, const std::string& _urlTemplate, - int32_t _minDisplayZoom, int32_t _maxDisplayZoom, int32_t _maxZoom) : - DataSource(_name, _urlTemplate, _minDisplayZoom, _maxDisplayZoom, _maxZoom) { -} - std::shared_ptr MVTSource::parse(const TileTask& _task, const MapProjection& _projection) const { auto tileData = std::make_shared(); - auto& task = static_cast(_task); + auto& task = static_cast(_task); protobuf::message item(task.rawTileData->data(), task.rawTileData->size()); PbfParser::ParserContext ctx(m_id); diff --git a/core/src/data/mvtSource.h b/core/src/data/mvtSource.h index c9c3dfeadf..659594c8e5 100644 --- a/core/src/data/mvtSource.h +++ b/core/src/data/mvtSource.h @@ -1,21 +1,19 @@ #pragma once -#include "dataSource.h" +#include "tileSource.h" namespace Tangram { -class MVTSource : public DataSource { +class MVTSource : public TileSource { + +public: + using TileSource::TileSource; protected: virtual std::shared_ptr parse(const TileTask& _task, const MapProjection& _projection) const override; -public: - - MVTSource(const std::string& _name, const std::string& _urlTemplate, - int32_t _minDisplayZoom, int32_t _maxDisplayZoom, int32_t _maxZoom); - }; } diff --git a/core/src/data/networkDataSource.cpp b/core/src/data/networkDataSource.cpp new file mode 100644 index 0000000000..9faa0c044c --- /dev/null +++ b/core/src/data/networkDataSource.cpp @@ -0,0 +1,93 @@ +#include "data/networkDataSource.h" + +#include "log.h" + +#define MAX_DOWNLOADS 4 + +namespace Tangram { + +NetworkDataSource::NetworkDataSource(const std::string& _urlTemplate) + : m_urlTemplate(_urlTemplate), + m_maxDownloads(MAX_DOWNLOADS) {} + +void NetworkDataSource::constructURL(const TileID& _tileCoord, std::string& _url) const { + _url.assign(m_urlTemplate); + + try { + size_t xpos = _url.find("{x}"); + _url.replace(xpos, 3, std::to_string(_tileCoord.x)); + size_t ypos = _url.find("{y}"); + _url.replace(ypos, 3, std::to_string(_tileCoord.y)); + size_t zpos = _url.find("{z}"); + _url.replace(zpos, 3, std::to_string(_tileCoord.z)); + } catch(...) { + LOGE("Bad URL template!"); + } +} + +bool NetworkDataSource::loadTileData(std::shared_ptr _task, TileTaskCb _cb) { + + if (_task->rawSource != this->level) { + LOGE("NetworkDataSource must be last!"); + return false; + } + + if (m_pending.size() >= m_maxDownloads) { + return false; + } + + auto tileId = _task->tileId(); + + { + std::unique_lock lock(m_mutex); + if (std::find(m_pending.begin(), m_pending.end(), tileId) != m_pending.end()) { + return false; + } + m_pending.push_back(tileId); + } + + std::string url(constructURL(_task->tileId())); + + bool started = startUrlRequest(url, + [this, cb = _cb, task = _task](std::vector&& _rawData) mutable { + + removePending(task->tileId()); + + if (task->isCanceled()) { + return; + } + + if (!_rawData.empty()) { + auto rawDataRef = std::make_shared>(); + std::swap(*rawDataRef, _rawData); + + auto& dlTask = static_cast(*task); + // NB: Sets hasData() state true + dlTask.rawTileData = rawDataRef; + } + cb.func(task); + }); + + if (!started) { + removePending(_task->tileId()); + + // Set canceled state, so that tile will not be tried + // for reloading until sourceGeneration increased. + _task->cancel(); + } + + return started; +} + +void NetworkDataSource::removePending(const TileID& _tileId) { + std::unique_lock lock(m_mutex); + auto it = std::find(m_pending.begin(), m_pending.end(), _tileId); + if (it != m_pending.end()) { m_pending.erase(it); } +} + +void NetworkDataSource::cancelLoadingTile(const TileID& _tileId) { + removePending(_tileId); + cancelUrlRequest(constructURL(_tileId)); +} + +} diff --git a/core/src/data/networkDataSource.h b/core/src/data/networkDataSource.h new file mode 100644 index 0000000000..6ded6088ff --- /dev/null +++ b/core/src/data/networkDataSource.h @@ -0,0 +1,38 @@ +#pragma once + +#include "data/tileSource.h" + +namespace Tangram { + +class NetworkDataSource : public TileSource::DataSource { +public: + + NetworkDataSource(const std::string& _urlTemplate); + + bool loadTileData(std::shared_ptr _task, TileTaskCb _cb) override; + + void cancelLoadingTile(const TileID& _tile) override; + +private: + /* Constructs the URL of a tile using */ + void constructURL(const TileID& _tileCoord, std::string& _url) const; + + std::string constructURL(const TileID& _tileCoord) const { + std::string url; + constructURL(_tileCoord, url); + return url; + } + + void removePending(const TileID& _tileId); + + // URL template for requesting tiles from a network or filesystem + std::string m_urlTemplate; + + std::vector m_pending; + + size_t m_maxDownloads; + + std::mutex m_mutex; +}; + +} diff --git a/core/src/data/rasterSource.cpp b/core/src/data/rasterSource.cpp index ff39eb45e9..f882832893 100644 --- a/core/src/data/rasterSource.cpp +++ b/core/src/data/rasterSource.cpp @@ -10,10 +10,10 @@ namespace Tangram { -class RasterTileTask : public DownloadTileTask { +class RasterTileTask : public BinaryTileTask { public: - RasterTileTask(TileID& _tileId, std::shared_ptr _source, int _subTask) - : DownloadTileTask(_tileId, _source, _subTask) {} + RasterTileTask(TileID& _tileId, std::shared_ptr _source, int _subTask) + : BinaryTileTask(_tileId, _source, _subTask) {} std::shared_ptr m_texture; @@ -40,7 +40,7 @@ class RasterTileTask : public DownloadTileTask { // Create tile geometries if (!isSubTask()) { - DownloadTileTask::process(_tileBuilder); + BinaryTileTask::process(_tileBuilder); } } @@ -69,10 +69,12 @@ class RasterTileTask : public DownloadTileTask { }; -RasterSource::RasterSource(const std::string& _name, const std::string& _urlTemplate, +RasterSource::RasterSource(const std::string& _name, std::unique_ptr _sources, int32_t _minDisplayZoom, int32_t _maxDisplayZoom, int32_t _maxZoom, TextureOptions _options, bool _genMipmap) - : DataSource(_name, _urlTemplate, _minDisplayZoom, _maxDisplayZoom, _maxZoom), m_texOptions(_options), m_genMipmap(_genMipmap) { + : TileSource(_name, std::move(_sources), _minDisplayZoom, _maxDisplayZoom, _maxZoom), + m_texOptions(_options), + m_genMipmap(_genMipmap) { std::vector data = {}; m_emptyTexture = std::make_shared(data, m_texOptions, m_genMipmap); @@ -88,6 +90,21 @@ std::shared_ptr RasterSource::createTexture(const std::vector& _r return texture; } +void RasterSource::loadTileData(std::shared_ptr _task, TileTaskCb _cb) { + // TODO, remove this + // Overwrite cb to set empty texture on failure + TileTaskCb cb{[this, _cb](std::shared_ptr _task) { + + if (!_task->hasData()) { + auto& task = static_cast(*_task); + task.m_texture = m_emptyTexture; + } + _cb.func(_task); + }}; + + TileSource::loadTileData(_task, cb); +} + std::shared_ptr RasterSource::parse(const TileTask& _task, const MapProjection& _projection) const { std::shared_ptr tileData = std::make_shared(); @@ -112,6 +129,8 @@ std::shared_ptr RasterSource::parse(const TileTask& _task, const MapPr std::shared_ptr RasterSource::createTask(TileID _tileId, int _subTask) { auto task = std::make_shared(_tileId, shared_from_this(), _subTask); + createSubTasks(task); + // First try existing textures cache { TileID id(_tileId.x, _tileId.y, _tileId.z); @@ -119,58 +138,17 @@ std::shared_ptr RasterSource::createTask(TileID _tileId, int _subTask) auto texIt = m_textures.find(id); if (texIt != m_textures.end()) { task->m_texture = texIt->second; + + // No more loading needed. + task->startedLoading(); + return task; } } - // Try raw data cache - cacheGet(*task); - return task; } -void RasterSource::onTileLoaded(std::vector&& _rawData, std::shared_ptr&& _task, - TileTaskCb _cb) { - - if (_task->isCanceled()) { return; } - - TileID tileID = _task->tileId(); - - auto rawDataRef = std::make_shared>(); - std::swap(*rawDataRef, _rawData); - - auto& task = static_cast(*_task); - task.rawTileData = rawDataRef; - - _cb.func(std::move(_task)); - - cachePut(tileID, rawDataRef); -} - -bool RasterSource::loadTileData(std::shared_ptr&& _task, TileTaskCb _cb) { - - std::string url(constructURL(_task->tileId())); - - auto copyTask = _task; - - // lambda captured parameters are const by default, we want "task" (moved) to be non-const, - // hence "mutable" - // Refer: http://en.cppreference.com/w/cpp/language/lambda - bool status = startUrlRequest(url, - [this, _cb, task = std::move(_task)](std::vector&& rawData) mutable { - this->onTileLoaded(std::move(rawData), std::move(task), _cb); - }); - - // For "dependent" raster datasources if this returns false make sure to create a black texture - // for tileID in this task, and consider dependent raster ready - if (!status) { - auto& task = static_cast(*copyTask); - task.m_texture = m_emptyTexture; - } - - return status; -} - Raster RasterSource::getRaster(const TileTask& _task) { TileID id(_task.tileId().x, _task.tileId().y, _task.tileId().z); diff --git a/core/src/data/rasterSource.h b/core/src/data/rasterSource.h index cc0bd6089b..073b297026 100644 --- a/core/src/data/rasterSource.h +++ b/core/src/data/rasterSource.h @@ -3,7 +3,7 @@ #include #include "tile/tileHash.h" -#include "dataSource.h" +#include "tileSource.h" #include "gl/texture.h" #include @@ -14,7 +14,7 @@ namespace Tangram { class RasterTileTask; -class RasterSource : public DataSource { +class RasterSource : public TileSource { TextureOptions m_texOptions; bool m_genMipmap; @@ -27,18 +27,16 @@ class RasterSource : public DataSource { virtual std::shared_ptr parse(const TileTask& _task, const MapProjection& _projection) const override; - virtual void onTileLoaded(std::vector&& _rawData, std::shared_ptr&& _task, - TileTaskCb _cb) override; - public: - RasterSource(const std::string& _name, const std::string& _urlTemplate, + RasterSource(const std::string& _name, std::unique_ptr _sources, int32_t _minDisplayZoom, int32_t _maxDisplayZoom, int32_t _maxZoom, TextureOptions _options, bool genMipmap = false); - virtual std::shared_ptr createTask(TileID _tile, int _subTask) override; - virtual bool loadTileData(std::shared_ptr&& _task, TileTaskCb _cb) override; + void loadTileData(std::shared_ptr _task, TileTaskCb _cb); + + virtual std::shared_ptr createTask(TileID _tile, int _subTask) override; virtual void clearRasters() override; virtual void clearRaster(const TileID& id) override; diff --git a/core/src/data/tileSource.cpp b/core/src/data/tileSource.cpp new file mode 100644 index 0000000000..2dc2b9a5a2 --- /dev/null +++ b/core/src/data/tileSource.cpp @@ -0,0 +1,128 @@ +#include "tileSource.h" + +#include "platform.h" +#include "tileData.h" +#include "tile/tileID.h" +#include "tile/tile.h" +#include "tile/tileTask.h" +#include "log.h" + +#include +#include + +namespace Tangram { + +TileSource::TileSource(const std::string& _name, std::unique_ptr _sources, + int32_t _minDisplayZoom, int32_t _maxDisplayZoom, int32_t _maxZoom) : + m_name(_name), + m_minDisplayZoom(_minDisplayZoom), m_maxDisplayZoom(_maxDisplayZoom), m_maxZoom(_maxZoom), + m_sources(std::move(_sources)) { + + static std::atomic s_serial; + + m_id = s_serial++; +} + +TileSource::~TileSource() { + clearData(); +} + +std::shared_ptr TileSource::createTask(TileID _tileId, int _subTask) { + auto task = std::make_shared(_tileId, shared_from_this(), _subTask); + + createSubTasks(task); + + return task; +} + +void TileSource::createSubTasks(std::shared_ptr _task) { + size_t index = 0; + + for (auto& subSource : m_rasterSources) { + TileID subTileID = _task->tileId(); + + // get tile for lower zoom if we are past max zoom + if (subTileID.z > subSource->maxZoom()) { + subTileID = subTileID.withMaxSourceZoom(subSource->maxZoom()); + } + + _task->subTasks().push_back(subSource->createTask(subTileID, index++)); + } +} + +void TileSource::clearData() { + + if (m_sources) { m_sources->clear(); } + + m_generation++; +} + +bool TileSource::equals(const TileSource& other) const { + if (m_name != other.m_name) { return false; } + // TODO compare DataSources instead + //if (m_urlTemplate != other.m_urlTemplate) { return false; } + if (m_minDisplayZoom != other.m_minDisplayZoom) { return false; } + if (m_maxDisplayZoom != other.m_maxDisplayZoom) { return false; } + if (m_maxZoom != other.m_maxZoom) { return false; } + if (m_rasterSources.size() != other.m_rasterSources.size()) { return false; } + for (size_t i = 0, end = m_rasterSources.size(); i < end; ++i) { + if (!m_rasterSources[i]->equals(*other.m_rasterSources[i])) { return false; } + } + + return true; +} + +void TileSource::loadTileData(std::shared_ptr _task, TileTaskCb _cb) { + + if (m_sources) { + if (_task->needsLoading()) { + if (m_sources->loadTileData(_task, _cb)) { + _task->startedLoading(); + } + } + } + + for (auto& subTask : _task->subTasks()) { + subTask->source().loadTileData(subTask, _cb); + } +} + +void TileSource::cancelLoadingTile(const TileID& _tileID) { + + if (m_sources) { return m_sources->cancelLoadingTile(_tileID); } + + for (auto& raster : m_rasterSources) { + TileID rasterID = _tileID.withMaxSourceZoom(raster->maxZoom()); + raster->cancelLoadingTile(rasterID); + } +} + +void TileSource::clearRasters() { + for (auto& raster : m_rasterSources) { + raster->clearRasters(); + } +} + +void TileSource::clearRaster(const TileID& id) { + for (auto& raster : m_rasterSources) { + TileID rasterID = id.withMaxSourceZoom(raster->maxZoom()); + raster->clearRaster(rasterID); + } +} + +void TileSource::addRasterSource(std::shared_ptr _rasterSource) { + /* + * We limit the parent source by any attached raster source's min/max. + */ + int32_t rasterMinDisplayZoom = _rasterSource->minDisplayZoom(); + int32_t rasterMaxDisplayZoom = _rasterSource->maxDisplayZoom(); + if (rasterMinDisplayZoom > m_minDisplayZoom) { + m_minDisplayZoom = rasterMinDisplayZoom; + } + if (rasterMaxDisplayZoom < m_maxDisplayZoom) { + m_maxDisplayZoom = rasterMaxDisplayZoom; + } + m_rasterSources.push_back(_rasterSource); +} + +} diff --git a/core/src/data/dataSource.h b/core/src/data/tileSource.h similarity index 65% rename from core/src/data/dataSource.h rename to core/src/data/tileSource.h index ccc5f8e89a..4e1973e1b0 100644 --- a/core/src/data/dataSource.h +++ b/core/src/data/tileSource.h @@ -4,6 +4,7 @@ #include #include #include +#include #include "tile/tileTask.h" @@ -18,19 +19,37 @@ class TileManager; struct RawCache; class Texture; -class DataSource : public std::enable_shared_from_this { +class TileSource : public std::enable_shared_from_this { public: + struct DataSource { + virtual bool loadTileData(std::shared_ptr _task, TileTaskCb _cb) = 0; + + /* Stops any running I/O tasks pertaining to @_tile */ + virtual void cancelLoadingTile(const TileID& _tile) { + if (next) { next->cancelLoadingTile(_tile); } + } + + virtual void clear() { if (next) next->clear(); } + + void setNext(std::unique_ptr _next) { + next = std::move(_next); + next->level = level + 1; + } + std::unique_ptr next; + int level = 0; + }; + /* Tile data sources must have a name and a URL template that defines where to find * a tile based on its coordinates. A URL template includes exactly one occurrance * each of '{x}', '{y}', and '{z}' which will be replaced by the x index, y index, * and zoom level of tiles to produce their URL. */ - DataSource(const std::string& _name, const std::string& _urlTemplate, + TileSource(const std::string& _name, std::unique_ptr _sources, int32_t _minDisplayZoom = -1, int32_t _maxDisplayZoom = -1, int32_t _maxZoom = 18); - virtual ~DataSource(); + virtual ~TileSource(); /* Fetches data for the map tile specified by @_tileID * @@ -38,8 +57,7 @@ class DataSource : public std::enable_shared_from_this { * the I/O task is complete, the tile data is added to a queue in @_tileManager for * further processing before it is renderable. */ - virtual bool loadTileData(std::shared_ptr&& _task, TileTaskCb _cb); - + virtual void loadTileData(std::shared_ptr _task, TileTaskCb _cb); /* Stops any running I/O tasks pertaining to @_tile */ virtual void cancelLoadingTile(const TileID& _tile); @@ -47,7 +65,7 @@ class DataSource : public std::enable_shared_from_this { /* Parse a with data into a , returning an empty TileData on failure */ virtual std::shared_ptr parse(const TileTask& _task, const MapProjection& _projection) const = 0; - /* Clears all data associated with this DataSource */ + /* Clears all data associated with this TileSource */ virtual void clearData(); const std::string& name() const { return m_name; } @@ -55,19 +73,14 @@ class DataSource : public std::enable_shared_from_this { virtual void clearRasters(); virtual void clearRaster(const TileID& id); - bool equals(const DataSource& _other) const; + bool equals(const TileSource& _other) const; virtual std::shared_ptr createTask(TileID _tile, int _subTask = -1); - /* @_cacheSize: Set size of in-memory cache for tile data in bytes. - * This cache holds unprocessed tile data for fast recreation of recently used tiles. - */ - void setCacheSize(size_t _cacheSize); - - /* ID of this DataSource instance */ + /* ID of this TileSource instance */ int32_t id() const { return m_id; } - /* Generation ID of DataSource state (incremented for each update, e.g. on clearData()) */ + /* Generation ID of TileSource state (incremented for each update, e.g. on clearData()) */ int64_t generation() const { return m_generation; } int32_t minDisplayZoom() const { return m_minDisplayZoom; } @@ -79,7 +92,7 @@ class DataSource : public std::enable_shared_from_this { } /* assign/get raster datasources to this datasource */ - void addRasterSource(std::shared_ptr _dataSource); + void addRasterSource(std::shared_ptr _dataSource); auto& rasterSources() { return m_rasterSources; } const auto& rasterSources() const { return m_rasterSources; } @@ -91,21 +104,7 @@ class DataSource : public std::enable_shared_from_this { protected: - virtual void onTileLoaded(std::vector&& _rawData, std::shared_ptr&& _task, - TileTaskCb _cb); - - /* Constructs the URL of a tile using */ - virtual void constructURL(const TileID& _tileCoord, std::string& _url) const; - - std::string constructURL(const TileID& _tileCoord) const { - std::string url; - constructURL(_tileCoord, url); - return url; - } - - bool cacheGet(DownloadTileTask& _task); - - void cachePut(const TileID& _tileID, std::shared_ptr> _rawDataRef); + void createSubTasks(std::shared_ptr _task); // This datasource is used to generate actual tile geometry bool m_generateGeometry = false; @@ -122,19 +121,17 @@ class DataSource : public std::enable_shared_from_this { // Maximum zoom for which tiles will be requested int32_t m_maxZoom; - // Unique id for DataSource + // Unique id for TileSource int32_t m_id; - // Generation of dynamic DataSource state (incremented for each update) + // Generation of dynamic TileSource state (incremented for each update) int64_t m_generation = 1; - // URL template for requesting tiles from a network or filesystem - std::string m_urlTemplate; + /* vector of raster sources (as raster samplers) referenced by this datasource */ + std::vector> m_rasterSources; - std::unique_ptr m_cache; + std::unique_ptr m_sources; - /* vector of raster sources (as raster samplers) referenced by this datasource */ - std::vector> m_rasterSources; }; } diff --git a/core/src/data/topoJsonSource.cpp b/core/src/data/topoJsonSource.cpp index 8f13ddf5b9..35b6ae0229 100644 --- a/core/src/data/topoJsonSource.cpp +++ b/core/src/data/topoJsonSource.cpp @@ -10,15 +10,10 @@ namespace Tangram { -TopoJsonSource::TopoJsonSource(const std::string& _name, const std::string& _urlTemplate, - int32_t minDisplayZoom, int32_t _maxDisplayZoom, int32_t _maxZoom) : - DataSource(_name, _urlTemplate, minDisplayZoom, _maxDisplayZoom, _maxZoom) { -} - std::shared_ptr TopoJsonSource::parse(const TileTask& _task, const MapProjection& _projection) const { - auto& task = static_cast(_task); + auto& task = static_cast(_task); std::shared_ptr tileData = std::make_shared(); diff --git a/core/src/data/topoJsonSource.h b/core/src/data/topoJsonSource.h index efbf6b91d9..cbebc0e314 100644 --- a/core/src/data/topoJsonSource.h +++ b/core/src/data/topoJsonSource.h @@ -1,21 +1,19 @@ #pragma once -#include "dataSource.h" +#include "tileSource.h" namespace Tangram { -class TopoJsonSource : public DataSource { +class TopoJsonSource : public TileSource { + +public: + using TileSource::TileSource; protected: virtual std::shared_ptr parse(const TileTask& _task, const MapProjection& _projection) const override; -public: - - TopoJsonSource(const std::string& _name, const std::string& _urlTemplate, - int32_t minDisplayZoom, int32_t _maxDisplayZoom, int32_t maxZoom); - }; } diff --git a/core/src/scene/dataLayer.h b/core/src/scene/dataLayer.h index 935ce8a3f3..6913f13917 100644 --- a/core/src/scene/dataLayer.h +++ b/core/src/scene/dataLayer.h @@ -6,7 +6,7 @@ namespace Tangram { // DataLayer represents a top-level layer in the stylesheet, distinct from -// SceneLayer by its association with a collection within a DataSource +// SceneLayer by its association with a collection within a TileSource class DataLayer : public SceneLayer { std::string m_source; diff --git a/core/src/scene/scene.cpp b/core/src/scene/scene.cpp index 936f709b82..cf7bf2a3b6 100644 --- a/core/src/scene/scene.cpp +++ b/core/src/scene/scene.cpp @@ -1,6 +1,6 @@ #include "scene.h" -#include "data/dataSource.h" +#include "data/tileSource.h" #include "gl/shaderProgram.h" #include "platform.h" #include "scene/dataLayer.h" @@ -132,10 +132,10 @@ std::shared_ptr Scene::getTexture(const std::string& textureName) const return texIt->second; } -std::shared_ptr Scene::getDataSource(const std::string& name) { - auto it = std::find_if(m_dataSources.begin(), m_dataSources.end(), +std::shared_ptr Scene::getTileSource(const std::string& name) { + auto it = std::find_if(m_tileSources.begin(), m_tileSources.end(), [&](auto& s){ return s->name() == name; }); - if (it != m_dataSources.end()) { + if (it != m_tileSources.end()) { return *it; } return nullptr; diff --git a/core/src/scene/scene.h b/core/src/scene/scene.h index d0feefa20a..121ad032f3 100644 --- a/core/src/scene/scene.h +++ b/core/src/scene/scene.h @@ -17,15 +17,15 @@ namespace Tangram { -class Style; -class Texture; -class DataSource; class DataLayer; +class FeatureSelection; class FontContext; class Light; class MapProjection; class SpriteAtlas; -class FeatureSelection; +class Style; +class Texture; +class TileSource; struct Stops; @@ -66,7 +66,7 @@ class Scene { auto& resourceRoot() { return m_resourceRoot; } auto& config() { return m_config; } - auto& dataSources() { return m_dataSources; }; + auto& tileSources() { return m_tileSources; }; auto& layers() { return m_layers; }; auto& styles() { return m_styles; }; auto& lights() { return m_lights; }; @@ -83,7 +83,7 @@ class Scene { const auto& path() const { return m_path; } const auto& resourceRoot() const { return m_resourceRoot; } const auto& config() const { return m_config; } - const auto& dataSources() const { return m_dataSources; }; + const auto& tileSources() const { return m_tileSources; }; const auto& layers() const { return m_layers; }; const auto& styles() const { return m_styles; }; const auto& lights() const { return m_lights; }; @@ -113,7 +113,7 @@ class Scene { void animated(bool animated) { m_animated = animated ? yes : no; } animate animated() const { return m_animated; } - std::shared_ptr getDataSource(const std::string& name); + std::shared_ptr getTileSource(const std::string& name); std::shared_ptr getTexture(const std::string& name) const; @@ -136,7 +136,7 @@ class Scene { std::unique_ptr m_mapProjection; std::vector m_layers; - std::vector> m_dataSources; + std::vector> m_tileSources; std::vector> m_styles; std::vector> m_lights; std::unordered_map> m_textures; diff --git a/core/src/scene/sceneLoader.cpp b/core/src/scene/sceneLoader.cpp index e45a494e1f..679e04d3d7 100644 --- a/core/src/scene/sceneLoader.cpp +++ b/core/src/scene/sceneLoader.cpp @@ -4,9 +4,11 @@ #include "lights.h" #include "data/clientGeoJsonSource.h" #include "data/geoJsonSource.h" +#include "data/memoryCacheDataSource.h" #include "data/mvtSource.h" -#include "data/topoJsonSource.h" +#include "data/networkDataSource.h" #include "data/rasterSource.h" +#include "data/topoJsonSource.h" #include "gl/shaderProgram.h" #include "style/material.h" #include "style/polygonStyle.h" @@ -905,8 +907,8 @@ bool SceneLoader::loadStyle(const std::string& name, Node config, const std::sha } void SceneLoader::loadSource(const std::string& name, const Node& source, const Node& sources, const std::shared_ptr& _scene) { - if (_scene->getDataSource(name)) { - LOGW("Duplicate DataSource: %s", name.c_str()); + if (_scene->getTileSource(name)) { + LOGW("Duplicate TileSource: %s", name.c_str()); return; } @@ -957,18 +959,27 @@ void SceneLoader::loadSource(const std::string& name, const Node& source, const url.find("{y}") != std::string::npos && url.find("{z}") != std::string::npos; - std::shared_ptr sourcePtr; + std::shared_ptr sourcePtr; + + auto rawSources = std::make_unique(); + rawSources->setCacheSize(CACHE_SIZE); if (type == "GeoJSON") { if (tiled) { - sourcePtr = std::shared_ptr(new GeoJsonSource(name, url, minDisplayZoom, maxDisplayZoom, maxZoom)); + rawSources->setNext(std::make_unique(url)); + sourcePtr = std::make_shared(name, std::move(rawSources), + minDisplayZoom, maxDisplayZoom, maxZoom); } else { - sourcePtr = std::shared_ptr(new ClientGeoJsonSource(name, url, minDisplayZoom, maxDisplayZoom, maxZoom)); + sourcePtr = std::make_shared(name, url, minDisplayZoom, maxDisplayZoom, maxZoom); } } else if (type == "TopoJSON") { - sourcePtr = std::shared_ptr(new TopoJsonSource(name, url, minDisplayZoom, maxDisplayZoom, maxZoom)); + rawSources->setNext(std::make_unique(url)); + sourcePtr = std::make_shared(name, std::move(rawSources), + minDisplayZoom, maxDisplayZoom, maxZoom); } else if (type == "MVT") { - sourcePtr = std::shared_ptr(new MVTSource(name, url, minDisplayZoom, maxDisplayZoom, maxZoom)); + rawSources->setNext(std::make_unique(url)); + sourcePtr = std::make_shared(name, std::move(rawSources), + minDisplayZoom, maxDisplayZoom, maxZoom); } else if (type == "Raster") { TextureOptions options = {GL_RGBA, GL_RGBA, {GL_LINEAR, GL_LINEAR}, {GL_CLAMP_TO_EDGE, GL_CLAMP_TO_EDGE} }; bool generateMipmaps = false; @@ -977,14 +988,16 @@ void SceneLoader::loadSource(const std::string& name, const Node& source, const generateMipmaps = true; } } - sourcePtr = std::shared_ptr(new RasterSource(name, url, minDisplayZoom, maxDisplayZoom, maxZoom, options, generateMipmaps)); + auto rawSources = std::make_unique(url); + sourcePtr = std::make_shared(name, std::move(rawSources), + minDisplayZoom, maxDisplayZoom, maxZoom, + options, generateMipmaps); } else { LOGW("Unrecognized data source type '%s', skipping", type.c_str()); } if (sourcePtr) { - sourcePtr->setCacheSize(CACHE_SIZE); - _scene->dataSources().push_back(sourcePtr); + _scene->tileSources().push_back(sourcePtr); } if (auto rasters = source["rasters"]) { @@ -993,7 +1006,7 @@ void SceneLoader::loadSource(const std::string& name, const Node& source, const } -void SceneLoader::loadSourceRasters(std::shared_ptr &source, Node rasterNode, const Node& sources, +void SceneLoader::loadSourceRasters(std::shared_ptr &source, Node rasterNode, const Node& sources, const std::shared_ptr& scene) { if (rasterNode.IsSequence()) { for (const auto& raster : rasterNode) { @@ -1004,7 +1017,7 @@ void SceneLoader::loadSourceRasters(std::shared_ptr &source, Node ra LOGNode("Parsing sources: '%s'", sources[srcName], e.what()); return; } - source->addRasterSource(scene->getDataSource(srcName)); + source->addRasterSource(scene->getTileSource(srcName)); } } } @@ -1644,7 +1657,7 @@ void SceneLoader::loadLayer(const std::pair& layer, const std::share if (Node data_source = data["source"]) { if (data_source.IsScalar()) { source = data_source.Scalar(); - auto dataSource = scene->getDataSource(source); + auto dataSource = scene->getTileSource(source); if (dataSource) { dataSource->generateGeometry(true); } else { diff --git a/core/src/scene/sceneLoader.h b/core/src/scene/sceneLoader.h index 6f947ad0c5..00d9fd5953 100644 --- a/core/src/scene/sceneLoader.h +++ b/core/src/scene/sceneLoader.h @@ -17,17 +17,17 @@ namespace Tangram { -class TileManager; +class Material; +class PointLight; class SceneLayer; -class View; class ShaderProgram; -class Material; class Style; -struct StyleParam; -struct MaterialTexture; -class PointLight; -class DataSource; +class TileManager; +class TileSource; +class View; struct Filter; +struct MaterialTexture; +struct StyleParam; struct TextureFiltering; struct TextureOptions; @@ -49,7 +49,7 @@ struct SceneLoader { static void loadBackground(Node background, const std::shared_ptr& scene); static void loadSource(const std::string& name, const Node& source, const Node& sources, const std::shared_ptr& scene); - static void loadSourceRasters(std::shared_ptr& source, Node rasterNode, const Node& sources, + static void loadSourceRasters(std::shared_ptr& source, Node rasterNode, const Node& sources, const std::shared_ptr& scene); static void loadTexture(const std::pair& texture, const std::shared_ptr& scene); static void loadLayer(const std::pair& layer, const std::shared_ptr& scene); diff --git a/core/src/style/style.cpp b/core/src/style/style.cpp index c521fd1658..8062f38013 100644 --- a/core/src/style/style.cpp +++ b/core/src/style/style.cpp @@ -10,7 +10,7 @@ #include "scene/scene.h" #include "scene/spriteAtlas.h" #include "tile/tile.h" -#include "data/dataSource.h" +#include "data/tileSource.h" #include "view/view.h" #include "marker/marker.h" #include "tangram.h" @@ -89,7 +89,7 @@ void Style::build(const Scene& _scene) { } } - setupRasters(_scene.dataSources()); + setupRasters(_scene.tileSources()); constructSelectionShaderProgram(); } @@ -161,14 +161,14 @@ void Style::setupSceneShaderUniforms(RenderState& rs, Scene& _scene, UniformBloc } } -void Style::setupRasters(const std::vector>& _dataSources) { +void Style::setupRasters(const std::vector>& _sources) { if (!hasRasters()) { return; } int numRasterSource = 0; - for (const auto& dataSource : _dataSources) { - if (dataSource->isRaster()) { + for (const auto& source : _sources) { + if (source->isRaster()) { numRasterSource++; } } diff --git a/core/src/style/style.h b/core/src/style/style.h index fb5c9096c9..9726036b18 100644 --- a/core/src/style/style.h +++ b/core/src/style/style.h @@ -11,23 +11,23 @@ namespace Tangram { -struct DrawRule; class Label; class LabelCollider; class Light; -struct LightUniforms; -class Tile; class MapProjection; -class Material; -struct MaterialUniforms; class Marker; -class VertexLayout; -class View; +class Material; +class RenderState; class Scene; class ShaderProgram; class Style; -class DataSource; -class RenderState; +class Tile; +class TileSource; +class VertexLayout; +class View; +struct DrawRule; +struct LightUniforms; +struct MaterialUniforms; enum class LightingType : char { none, @@ -294,7 +294,7 @@ using StyleUniform = std::pair; virtual bool hasRasters() const { return m_rasterType != RasterType::none; } - void setupRasters(const std::vector>& _dataSources); + void setupRasters(const std::vector>& _sources); std::vector& styleUniforms() { return m_mainUniforms.styleUniforms; } diff --git a/core/src/tangram.cpp b/core/src/tangram.cpp index eeee4fd64a..5f6372f85c 100644 --- a/core/src/tangram.cpp +++ b/core/src/tangram.cpp @@ -153,8 +153,7 @@ void Map::Impl::setScene(std::shared_ptr& _scene) { } inputHandler.setView(view); - tileManager.setDataSources(_scene->dataSources()); - + tileManager.setTileSources(_scene->tileSources()); tileWorker.setScene(_scene); markerManager.setScene(_scene); setPixelScale(view.pixelScale()); @@ -698,17 +697,17 @@ int Map::getCameraType() { } -void Map::addDataSource(std::shared_ptr _source) { +void Map::addTileSource(std::shared_ptr _source) { std::lock_guard lock(impl->tilesMutex); - impl->tileManager.addClientDataSource(_source); + impl->tileManager.addClientTileSource(_source); } -bool Map::removeDataSource(DataSource& source) { +bool Map::removeTileSource(TileSource& source) { std::lock_guard lock(impl->tilesMutex); - return impl->tileManager.removeClientDataSource(source); + return impl->tileManager.removeClientTileSource(source); } -void Map::clearDataSource(DataSource& _source, bool _data, bool _tiles) { +void Map::clearTileSource(TileSource& _source, bool _data, bool _tiles) { std::lock_guard lock(impl->tilesMutex); if (_tiles) { impl->tileManager.clearTileSet(_source.id()); } diff --git a/core/src/tangram.h b/core/src/tangram.h index 50a8931f6d..714534f7a8 100644 --- a/core/src/tangram.h +++ b/core/src/tangram.h @@ -10,7 +10,7 @@ namespace Tangram { -class DataSource; +class TileSource; enum LabelType { icon, @@ -183,15 +183,15 @@ class Map { // point is not visible on the screen, otherwise returns true bool lngLatToScreenPosition(double _lng, double _lat, double* _x, double* _y); - // Add a data source for adding drawable map data, which will be styled + // Add a tile source for adding drawable map data, which will be styled // according to the scene file using the provided data source name; - void addDataSource(std::shared_ptr _source); + void addTileSource(std::shared_ptr _source); - // Remove a data source from the map; returns true if the source was found + // Remove a tile source from the map; returns true if the source was found // and removed, otherwise returns false. - bool removeDataSource(DataSource& _source); + bool removeTileSource(TileSource& _source); - void clearDataSource(DataSource& _source, bool _data, bool _tiles); + void clearTileSource(TileSource& _source, bool _data, bool _tiles); // Add a marker object to the map and return an ID for it; an ID of 0 indicates an invalid marker; // the marker will not be drawn until both styling and geometry are set using the functions below. diff --git a/core/src/tile/tile.cpp b/core/src/tile/tile.cpp index 96e291a91d..95e8fd2e4f 100644 --- a/core/src/tile/tile.cpp +++ b/core/src/tile/tile.cpp @@ -1,6 +1,6 @@ #include "tile.h" -#include "data/dataSource.h" +#include "data/tileSource.h" #include "style/style.h" #include "view/view.h" #include "tile/tileID.h" @@ -10,7 +10,7 @@ namespace Tangram { -Tile::Tile(TileID _id, const MapProjection& _projection, const DataSource* _source) : +Tile::Tile(TileID _id, const MapProjection& _projection, const TileSource* _source) : m_id(_id), m_projection(&_projection), m_sourceId(_source ? _source->id() : 0), diff --git a/core/src/tile/tile.h b/core/src/tile/tile.h index c36ede3e34..50e16c51e6 100644 --- a/core/src/tile/tile.h +++ b/core/src/tile/tile.h @@ -13,7 +13,7 @@ namespace Tangram { -class DataSource; +class TileSource; class MapProjection; struct Properties; class Style; @@ -40,7 +40,7 @@ class Tile { public: - Tile(TileID _id, const MapProjection& _projection, const DataSource* _source = nullptr); + Tile(TileID _id, const MapProjection& _projection, const TileSource* _source = nullptr); virtual ~Tile(); @@ -110,10 +110,10 @@ class Tile { float m_inverseScale = 1; - /* ID of the DataSource */ + /* ID of the TileSource */ const int32_t m_sourceId; - /* State of the DataSource for which this tile was created */ + /* State of the TileSource for which this tile was created */ const int64_t m_sourceGeneration; bool m_proxyState = false; diff --git a/core/src/tile/tileBuilder.cpp b/core/src/tile/tileBuilder.cpp index 3e39303a15..4ab5ed3ead 100644 --- a/core/src/tile/tileBuilder.cpp +++ b/core/src/tile/tileBuilder.cpp @@ -1,8 +1,8 @@ #include "tile/tileBuilder.h" -#include "data/dataSource.h" #include "data/properties.h" #include "data/propertyItem.h" +#include "data/tileSource.h" #include "gl/mesh.h" #include "log.h" #include "scene/dataLayer.h" @@ -92,7 +92,7 @@ void TileBuilder::applyStyling(const Feature& _feature, const SceneLayer& _layer } } -std::shared_ptr TileBuilder::build(TileID _tileID, const TileData& _tileData, const DataSource& _source) { +std::shared_ptr TileBuilder::build(TileID _tileID, const TileData& _tileData, const TileSource& _source) { m_selectionFeatures.clear(); diff --git a/core/src/tile/tileBuilder.h b/core/src/tile/tileBuilder.h index 2748798866..5def22f519 100644 --- a/core/src/tile/tileBuilder.h +++ b/core/src/tile/tileBuilder.h @@ -1,6 +1,6 @@ #pragma once -#include "data/dataSource.h" +#include "data/tileSource.h" #include "scene/styleContext.h" #include "scene/drawRule.h" #include "labels/labelCollider.h" @@ -8,11 +8,11 @@ namespace Tangram { class DataLayer; -class DataSource; +class StyleBuilder; +class Tile; +class TileSource; struct Feature; struct Properties; -class Tile; -class StyleBuilder; struct TileData; class TileBuilder { @@ -25,7 +25,7 @@ class TileBuilder { StyleBuilder* getStyleBuilder(const std::string& _name); - std::shared_ptr build(TileID _tileID, const TileData& _data, const DataSource& _source); + std::shared_ptr build(TileID _tileID, const TileData& _data, const TileSource& _source); const Scene& scene() const { return *m_scene; } diff --git a/core/src/tile/tileManager.cpp b/core/src/tile/tileManager.cpp index 078f0290fd..8043377471 100644 --- a/core/src/tile/tileManager.cpp +++ b/core/src/tile/tileManager.cpp @@ -1,6 +1,6 @@ #include "tileManager.h" -#include "data/dataSource.h" +#include "data/tileSource.h" #include "platform.h" #include "tile/tile.h" #include "tileCache.h" @@ -21,13 +21,13 @@ TileManager::TileManager(TileTaskQueue& _tileWorker) : m_tileCache = std::unique_ptr(new TileCache(DEFAULT_CACHE_SIZE)); // Callback to pass task from Download-Thread to Worker-Queue - m_dataCallback = TileTaskCb{[this](std::shared_ptr&& task) { + m_dataCallback = TileTaskCb{[this](std::shared_ptr task) { - if (task->isReady()) { + if (task->isReady()) { requestRender(); } else if (task->hasData()) { - m_workers.enqueue(std::move(task)); + m_workers.enqueue(task); } else { task->cancel(); @@ -39,7 +39,7 @@ TileManager::~TileManager() { m_tileSets.clear(); } -void TileManager::setDataSources(const std::vector>& _sources) { +void TileManager::setTileSources(const std::vector>& _sources) { m_tileCache->clear(); @@ -47,7 +47,7 @@ void TileManager::setDataSources(const std::vector>& auto it = std::remove_if( m_tileSets.begin(), m_tileSets.end(), [&](auto& tileSet) { - if (!tileSet.clientDataSource) { + if (!tileSet.clientTileSource) { auto sIt = std::find_if(_sources.begin(), _sources.end(), [&](auto& source){ return source->equals(*tileSet.source); }); @@ -79,17 +79,17 @@ void TileManager::setDataSources(const std::vector>& } } -void TileManager::addClientDataSource(std::shared_ptr _dataSource) { - m_tileSets.push_back({ _dataSource, true }); +void TileManager::addClientTileSource(std::shared_ptr _tileSource) { + m_tileSets.push_back({ _tileSource, true }); } -bool TileManager::removeClientDataSource(DataSource& dataSource) { +bool TileManager::removeClientTileSource(TileSource& _tileSource) { bool removed = false; for (auto it = m_tileSets.begin(); it != m_tileSets.end();) { - if (it->source.get() == &dataSource) { - // Remove the textures for this data source + if (it->source.get() == &_tileSource) { + // Remove the textures for this tile source it->source->clearRasters(); - // Remove the tile set associated with this data source + // Remove the tile set associated with this tile source it = m_tileSets.erase(it); removed = true; } else { @@ -120,7 +120,6 @@ void TileManager::clearTileSet(int32_t _sourceId) { void TileManager::updateTileSets(const ViewState& _view, const std::set& _visibleTiles) { m_tiles.clear(); - m_loadPending = 0; m_tilesInProgress = 0; m_tileSetChanged = false; @@ -224,29 +223,25 @@ void TileManager::updateTileSet(TileSet& _tileSet, const ViewState& _view, if (!entry.isLoading() && (entry.tile->sourceGeneration() < generation)) { // Tile needs update - enqueue for loading + entry.task = _tileSet.source->createTask(visTileId); enqueueTask(_tileSet, visTileId, _view); } - } else { + } else if (entry.needsLoading()) { + // Not yet available - enqueue for loading + enqueueTask(_tileSet, visTileId, _view); + m_tilesInProgress++; - if (entry.isLoading() && entry.rastersPending() == 0) { - if (newTiles) { - // check again for proxies - updateProxyTiles(_tileSet, visTileId, entry); - } - m_tilesInProgress++; - } else if (!bool(entry.task) || - (entry.rastersPending() > 0 && !entry.isCanceled()) || - (entry.isCanceled() && (entry.task->sourceGeneration() < generation))) { - // Start loading when: - // no task is set, - // one of the raster for this task has not been loaded yet - // or the task stems from an older tile source generation. - - // Not yet available - enqueue for loading - enqueueTask(_tileSet, visTileId, _view); + } else if (entry.isCanceled() && + (entry.task->sourceGeneration() < generation)) { + // Tile needs update - enqueue for loading + entry.task = _tileSet.source->createTask(visTileId); + enqueueTask(_tileSet, visTileId, _view); + m_tilesInProgress++; + } - m_tilesInProgress++; - } + if (newTiles && entry.isLoading()) { + // check again for proxies + updateProxyTiles(_tileSet, visTileId, entry); } ++curTilesIt; @@ -306,16 +301,15 @@ void TileManager::updateTileSet(TileSet& _tileSet, const ViewState& _view, for (auto& it : tiles) { auto& entry = it.second; +#if LOG_LEVEL >= 3 size_t rasterLoading = 0; size_t rasterDone = 0; - if (entry.task) { for (auto &raster : entry.task->subTasks()) { if (raster->isReady()) { rasterDone++; } else { rasterLoading++; } } } - DBG("> %s - ready:%d proxy:%d/%d loading:%d rDone:%d rLoading:%d rPending:%d canceled:%d", it.first.toString().c_str(), entry.isReady(), @@ -326,6 +320,7 @@ void TileManager::updateTileSet(TileSet& _tileSet, const ViewState& _view, rasterLoading, entry.rastersPending(), entry.task && entry.task->isCanceled()); +#endif if (entry.isLoading()) { auto& id = it.first; @@ -337,16 +332,6 @@ void TileManager::updateTileSet(TileSet& _tileSet, const ViewState& _view, if (scaleDiv < 1) { scaleDiv = 0.1/scaleDiv; } // prefer parent tiles task->setPriority(glm::length2(tileCenter - _view.center) * scaleDiv); task->setProxyState(entry.getProxyCounter() > 0); - - // Count tiles that are currently being downloaded to - // limit download requests. - if (!task->hasData()) { - m_loadPending++; - } - - for (auto& subTask : task->subTasks()) { - if (!subTask->hasData()) { m_loadPending++; } - } } if (entry.isReady()) { @@ -371,56 +356,10 @@ void TileManager::enqueueTask(TileSet& _tileSet, const TileID& _tileID, m_loadTasks.insert(it, std::make_tuple(distance, &_tileSet, _tileID)); } -// create and download raster references store these -// rastertasks in this datasource' task -void TileManager::loadSubTasks(std::vector>& _subSources, - std::shared_ptr& tileTask, const TileID& tileID) { - - auto& subTasks = tileTask->subTasks(); - - if (subTasks.size() >= _subSources.size()) { return; } - - for (size_t index = 0; index < _subSources.size(); index++) { - auto& subSource = _subSources[index]; - - auto it = std::lower_bound(subTasks.begin(), subTasks.end(), int(index), - [&](auto& task, auto id){ return task->subTaskId() < id; }); - - if (it != subTasks.end() && (*it)->subTaskId() == int(index)) { continue; } - - TileID subTileID = tileID; - - // get tile for lower zoom if we are past max zoom - if (subTileID.z > subSource->maxZoom()) { - subTileID = subTileID.withMaxSourceZoom(subSource->maxZoom()); - } - auto subTask = subSource->createTask(subTileID, index); - if (subTask->isReady()) { - subTasks.insert(it, subTask); - requestRender(); - - } else if (subTask->hasData()) { - subTasks.insert(it, subTask); - m_dataCallback.func(std::move(subTask)); - - } else if (m_loadPending < MAX_DOWNLOADS) { - subTasks.insert(it, subTask); - - if (subSource->loadTileData(std::move(subTask), m_dataCallback)) { - m_loadPending++; - - } else { - // dependent raster's loading failed.. - // this subTask's rasterReady must have been set with black texture - assert(subTask->isReady()); - requestRender(); - } - } - } -} - void TileManager::loadTiles() { + if (m_loadTasks.empty()) { return; } + for (auto& loadTask : m_loadTasks) { auto tileId = std::get<2>(loadTask); @@ -428,33 +367,7 @@ void TileManager::loadTiles() { auto tileIt = tileSet.tiles.find(tileId); auto& entry = tileIt->second; - if (entry.task && entry.rastersPending() > 0 && !entry.isCanceled()) { - // just load the rasters and continue, - // the main tile task has already started loading - loadSubTasks(tileSet.source->rasterSources(), entry.task, tileId); - continue; - } - - auto task = tileSet.source->createTask(tileId); - - if (task->hasData()) { - // Note: Set implicit 'loading' state - entry.task = task; - loadSubTasks(tileSet.source->rasterSources(), entry.task, tileId); - m_dataCallback.func(std::move(task)); - - } else if (m_loadPending < MAX_DOWNLOADS) { - entry.task = task; - if (tileSet.source->loadTileData(std::move(task), m_dataCallback)) { - m_loadPending++; - loadSubTasks(tileSet.source->rasterSources(), entry.task, tileId); - } else { - // Set canceled state, so that tile will not be tried - // for reloading until sourceGeneration increased. - entry.task->cancel(); - continue; - } - } + tileSet.source->loadTileData(entry.task, m_dataCallback); } DBG("loading:%d pending:%d cache: %fMB", @@ -490,6 +403,8 @@ bool TileManager::addTile(TileSet& _tileSet, const TileID& _tileID) { if (!tile) { // Add Proxy if corresponding proxy MapTile ready updateProxyTiles(_tileSet, _tileID, entry.first->second); + + entry.first->second.task = _tileSet.source->createTask(_tileID); } entry.first->second.setVisible(true); @@ -517,7 +432,7 @@ void TileManager::removeTile(TileSet& _tileSet, std::map::ite } } - // Remove rasters from this DataSource + // Remove rasters from this TileSource _tileSet.source->clearRaster(id); // Remove tile from set diff --git a/core/src/tile/tileManager.h b/core/src/tile/tileManager.h index ca389814f9..cf6ad69e9d 100644 --- a/core/src/tile/tileManager.h +++ b/core/src/tile/tileManager.h @@ -13,11 +13,11 @@ #include #include #include -#include +#include namespace Tangram { -class DataSource; +class TileSource; class TileCache; struct ViewState; @@ -29,7 +29,6 @@ struct ViewState; class TileManager { const static size_t DEFAULT_CACHE_SIZE = 32*1024*1024; // 32 MB - const static int MAX_DOWNLOADS = 4; public: @@ -37,8 +36,8 @@ class TileManager { virtual ~TileManager(); - /* Sets the tile DataSources */ - void setDataSources(const std::vector>& _sources); + /* Sets the tile TileSources */ + void setTileSources(const std::vector>& _sources); /* Updates visible tile set and load missing tiles */ void updateTileSets(const ViewState& _view, const std::set& _visibleTiles); @@ -54,9 +53,9 @@ class TileManager { bool hasLoadingTiles() { return m_tilesInProgress > 0; } - void addClientDataSource(std::shared_ptr _dataSource); + void addClientTileSource(std::shared_ptr _source); - bool removeClientDataSource(DataSource& dataSource); + bool removeClientTileSource(TileSource& _source); std::unique_ptr& getTileCache() { return m_tileCache; } @@ -97,12 +96,26 @@ class TileManager { bool isReady() { return bool(tile); } bool isLoading() { return bool(task) && !task->isCanceled(); } - size_t rastersPending() { - if (task) { - return (task->source().rasterSources().size() - task->subTasks().size()); + + bool needsLoading() { + //return !bool(task) || (task->needsLoading() && !task->isCanceled()); + if (isReady()) { return false; } + if (!task) { return true; } + if (task->isCanceled()) { return false; } + if (task->needsLoading()) { return true; } + + for (auto& subtask : task->subTasks()) { + if (subtask->needsLoading()) { return true; } } - return 0; + return false; } + + // size_t rastersPending() { + // if (task) { + // return (task->source().rasterSources().size() - task->subTasks().size()); + // } + // return 0; + // } bool isCanceled() { return bool(task) && task->isCanceled(); } // New Data only when @@ -112,7 +125,7 @@ class TileManager { bool newData() { if (bool(task) && task->isReady()) { - if (rastersPending()) { return false; } + //if (rastersPending()) { return false; } for (auto& rTask : task->subTasks()) { if (!rTask->isReady()) { return false; } @@ -173,13 +186,13 @@ class TileManager { }; struct TileSet { - TileSet(std::shared_ptr _source, bool _clientDataSource) - : source(_source), clientDataSource(_clientDataSource) {} + TileSet(std::shared_ptr _source, bool _clientSource) + : source(_source), clientTileSource(_clientSource) {} - std::shared_ptr source; + std::shared_ptr source; std::map tiles; int64_t sourceGeneration = 0; - bool clientDataSource; + bool clientTileSource; }; void updateTileSet(TileSet& tileSet, const ViewState& _view, const std::set& _visibleTiles); @@ -187,8 +200,6 @@ class TileManager { void enqueueTask(TileSet& _tileSet, const TileID& _tileID, const ViewState& _view); void loadTiles(); - void loadSubTasks(std::vector>& subSources, std::shared_ptr& tileTask, - const TileID& tileID); /* * Constructs a future (async) to load data of a new visible tile this is @@ -215,7 +226,6 @@ class TileManager { */ void clearProxyTiles(TileSet& _tileSet, const TileID& _tileID, TileEntry& _tile, std::vector& _removes); - int32_t m_loadPending = 0; int32_t m_tilesInProgress = 0; std::vector m_tileSets; @@ -229,7 +239,7 @@ class TileManager { bool m_tileSetChanged = false; - /* Callback for DataSource: + /* Callback for TileSource: * Passes TileTask back with data for further processing by s */ TileTaskCb m_dataCallback; diff --git a/core/src/tile/tileTask.cpp b/core/src/tile/tileTask.cpp index 806bbd11d3..005d7f32bc 100644 --- a/core/src/tile/tileTask.cpp +++ b/core/src/tile/tileTask.cpp @@ -1,5 +1,5 @@ #include "tileTask.h" -#include "data/dataSource.h" +#include "data/tileSource.h" #include "tile/tileBuilder.h" #include "scene/scene.h" #include "util/mapProjection.h" @@ -7,7 +7,7 @@ namespace Tangram { -TileTask::TileTask(TileID& _tileId, std::shared_ptr _source, int _subTask) : +TileTask::TileTask(TileID& _tileId, std::shared_ptr _source, int _subTask) : m_tileId(_tileId), m_subTaskId(_subTask), m_source(_source), diff --git a/core/src/tile/tileTask.h b/core/src/tile/tileTask.h index 212a7a4b31..910acdef4c 100644 --- a/core/src/tile/tileTask.h +++ b/core/src/tile/tileTask.h @@ -11,7 +11,7 @@ namespace Tangram { class TileManager; class TileBuilder; -class DataSource; +class TileSource; class Tile; class MapProjection; struct TileData; @@ -21,7 +21,7 @@ class TileTask { public: - TileTask(TileID& _tileId, std::shared_ptr _source, int _subTask); + TileTask(TileID& _tileId, std::shared_ptr _source, int _subTask); // No copies TileTask(const TileTask& _other) = delete; @@ -31,11 +31,15 @@ class TileTask { virtual bool hasData() const { return true; } - virtual bool isReady() const { return bool(m_tile); } + virtual bool isReady() const { + if (needsLoading()) { return false; } + + return bool(m_tile); + } std::shared_ptr& tile() { return m_tile; } - DataSource& source() { return *m_source; } + TileSource& source() { return *m_source; } int64_t sourceGeneration() const { return m_sourceGeneration; } TileID tileId() const { return m_tileId; } @@ -67,6 +71,12 @@ class TileTask { // onDone for sub-tasks virtual void complete(TileTask& _mainTask) {} + int rawSource = 0; + + bool needsLoading() const { return m_needsLoading; } + + void startedLoading() { m_needsLoading = false; } + protected: const TileID m_tileId; @@ -74,7 +84,7 @@ class TileTask { const int m_subTaskId; // Save shared reference to Datasource while building tile - std::shared_ptr m_source; + std::shared_ptr m_source; // Vector of tasks to download raster samplers std::vector> m_subTasks; @@ -85,29 +95,30 @@ class TileTask { std::shared_ptr m_tile; bool m_canceled = false; + bool m_needsLoading = true; std::atomic m_priority; bool m_proxyState = false; }; -class DownloadTileTask : public TileTask { +class BinaryTileTask : public TileTask { public: - DownloadTileTask(TileID& _tileId, std::shared_ptr _source, int _subTask) + BinaryTileTask(TileID& _tileId, std::shared_ptr _source, int _subTask) : TileTask(_tileId, _source, _subTask) {} virtual bool hasData() const override { return rawTileData && !rawTileData->empty(); } - // Raw tile data that will be processed by DataSource. + // Raw tile data that will be processed by TileSource. std::shared_ptr> rawTileData; }; struct TileTaskQueue { - virtual void enqueue(std::shared_ptr&& task) = 0; + virtual void enqueue(std::shared_ptr task) = 0; }; struct TileTaskCb { - std::function&&)> func; + std::function)> func; }; } diff --git a/core/src/tile/tileWorker.cpp b/core/src/tile/tileWorker.cpp index 720d3a83d9..71492fd19a 100644 --- a/core/src/tile/tileWorker.cpp +++ b/core/src/tile/tileWorker.cpp @@ -1,7 +1,7 @@ #include "tileWorker.h" #include "platform.h" -#include "data/dataSource.h" +#include "data/tileSource.h" #include "tile/tileID.h" #include "tile/tileTask.h" #include "tile/tileBuilder.h" @@ -108,7 +108,7 @@ void TileWorker::setScene(std::shared_ptr& _scene) { } } -void TileWorker::enqueue(std::shared_ptr&& task) { +void TileWorker::enqueue(std::shared_ptr task) { { std::unique_lock lock(m_mutex); if (!m_running) { diff --git a/core/src/tile/tileWorker.h b/core/src/tile/tileWorker.h index a47d678794..e14242624f 100644 --- a/core/src/tile/tileWorker.h +++ b/core/src/tile/tileWorker.h @@ -24,7 +24,7 @@ class TileWorker : public TileTaskQueue { ~TileWorker(); - virtual void enqueue(std::shared_ptr&& task) override; + virtual void enqueue(std::shared_ptr task) override; void stop(); diff --git a/linux/src/main.cpp b/linux/src/main.cpp index ab5319b338..fc2b515c73 100644 --- a/linux/src/main.cpp +++ b/linux/src/main.cpp @@ -48,6 +48,7 @@ double last_y_velocity = 0.0; bool scene_editing_mode = false; MarkerID marker = 0; +bool testClientDataSource = true; std::shared_ptr data_source; LngLat last_point; @@ -88,8 +89,11 @@ void mouse_button_callback(GLFWwindow* window, int button, int action, int mods) map->screenPositionToLngLat(x, y, &p.longitude, &p.latitude); // map->setPositionEased(p.longitude, p.latitude, 1.f); - logMsg("pick feature\n"); - map->clearDataSource(*data_source, true, true); + LOG("pick feature"); + + if (testClientDataSource) { + map->clearTileSource(*data_source, true, true); + } map->pickFeatureAt(x, y, [](auto item) { if (!item) { return; } @@ -99,15 +103,30 @@ void mouse_button_callback(GLFWwindow* window, int button, int action, int mods) LOGS("%s", name.c_str()); }); } else if ((time - last_time_pressed) < single_tap_time) { - // LngLat p; - // map->screenPositionToLngLat(x, y, &p.longitude, &p.latitude); + LngLat p; + map->screenPositionToLngLat(x, y, &p.longitude, &p.latitude); - // marker = map->markerAdd(); - // map->markerSetStyling(marker, markerStyling.c_str()); - // map->markerSetPoint(marker, p); - // map->markerSetDrawOrder(marker, mods); - // logMsg("Added marker with zOrder: %d\n", mods); + if (testClientDataSource) { + LOG("Added point %f,%f", p.longitude, p.latitude); + Properties prop; + prop.set("type", "point"); + data_source->addPoint(prop, p); + + if (!(last_point == LngLat{0, 0})) { + LngLat p2 = last_point; + Properties prop1; + prop1.set("type", "line"); + data_source->addLine(prop1, {p, p2}); + } + last_point = p; + } else { + marker = map->markerAdd(); + map->markerSetStyling(marker, markerStyling.c_str()); + map->markerSetPoint(marker, p); + map->markerSetDrawOrder(marker, mods); + LOG("Added marker with zOrder: %d", mods); + } // This updates the tiles (maybe we need a recalcTiles()) requestRender(); } @@ -295,8 +314,10 @@ void init_main_window(bool recreate) { map->setupGL(); map->resize(width, height); - data_source = std::make_shared("touch", ""); - map->addDataSource(data_source); + if (testClientDataSource) { + data_source = std::make_shared("touch", ""); + map->addTileSource(data_source); + } } @@ -310,11 +331,11 @@ int main(int argc, char* argv[]) { // Give it a chance to shutdown cleanly on CTRL-C signal(SIGINT, [](int) { if (keepRunning) { - logMsg("shutdown\n"); + LOG("shutdown"); keepRunning = false; glfwPostEmptyEvent(); } else { - logMsg("killed!\n"); + LOG("killed!"); exit(1); }}); @@ -322,7 +343,7 @@ int main(int argc, char* argv[]) { while (++argi < argc) { if (strcmp(argv[argi - 1], "-f") == 0) { sceneFile = std::string(argv[argi]); - logMsg("File from command line: %s\n", argv[argi]); + LOG("File from command line: %s", argv[argi]); break; } } @@ -334,7 +355,7 @@ int main(int argc, char* argv[]) { struct stat sb; //if (stat(sceneFile.c_str(), &sb) == -1) { - //logMsg("scene file not found!"); + //LOG("scene file not found!"); //exit(EXIT_FAILURE); //} auto last_mod = sb.st_mtime; @@ -375,7 +396,7 @@ int main(int argc, char* argv[]) { } if (recreate_context) { - logMsg("recreate context\n"); + LOG("recreate context"); // Simulate GL context loss init_main_window(true); recreate_context = false; diff --git a/tests/unit/tileManagerTests.cpp b/tests/unit/tileManagerTests.cpp index 6d2dc2f731..755b39499e 100644 --- a/tests/unit/tileManagerTests.cpp +++ b/tests/unit/tileManagerTests.cpp @@ -1,6 +1,6 @@ #include "catch.hpp" -#include "data/dataSource.h" +#include "data/tileSource.h" #include "tile/tileManager.h" #include "tile/tileWorker.h" #include "util/mapProjection.h" @@ -19,11 +19,11 @@ struct TestTileWorker : TileTaskQueue { std::deque> tasks; - virtual void enqueue(std::shared_ptr&& task) { + void enqueue(std::shared_ptr task) override{ tasks.push_back(std::move(task)); } - virtual bool checkProcessedTiles() { + bool checkProcessedTiles() { if (pendingTiles) { pendingTiles = false; return true; @@ -66,29 +66,28 @@ struct TestTileWorker : TileTaskQueue { } }; -struct TestDataSource : DataSource { +struct TestTileSource : TileSource { class Task : public TileTask { public: bool gotData = false; - Task(TileID& _tileId, std::shared_ptr _source, bool _subTask) + Task(TileID& _tileId, std::shared_ptr _source, bool _subTask) : TileTask(_tileId, _source, _subTask) {} bool hasData() const override { return gotData; } - }; int tileTaskCount = 0; - TestDataSource() : DataSource("", "") { + TestTileSource() : TileSource("", nullptr) { m_generateGeometry = true; } - bool loadTileData(std::shared_ptr&& _task, TileTaskCb _cb) override { + void loadTileData(std::shared_ptr _task, TileTaskCb _cb) override { tileTaskCount++; static_cast(_task.get())->gotData = true; + _task->startedLoading(); _cb.func(std::move(_task)); - return true; } void cancelLoadingTile(const TileID& _tile) override {} @@ -110,9 +109,9 @@ TEST_CASE( "Use proxy Tile - Dont remove proxy if it is now visible", "[TileMana TileManager tileManager(worker); ViewState viewState { &s_projection, true, glm::vec2(0), 1 }; - auto source = std::make_shared(); - std::vector> sources = { source }; - tileManager.setDataSources(sources); + auto source = std::make_shared(); + std::vector> sources = { source }; + tileManager.setTileSources(sources); /// Start loading tile 0/0/0 std::set visibleTiles_1 = { TileID{0,0,0} }; @@ -166,9 +165,9 @@ TEST_CASE( "Load visible Tile", "[TileManager][updateTileSets]" ) { TileManager tileManager(worker); ViewState viewState { &s_projection, true, glm::vec2(0), 1 }; - auto source = std::make_shared(); - std::vector> sources = { source }; - tileManager.setDataSources(sources); + auto source = std::make_shared(); + std::vector> sources = { source }; + tileManager.setTileSources(sources); std::set visibleTiles = { TileID{0,0,0} }; tileManager.updateTileSets(viewState, visibleTiles); @@ -192,9 +191,9 @@ TEST_CASE( "Use proxy Tile", "[TileManager][updateTileSets]" ) { TileManager tileManager(worker); ViewState viewState { &s_projection, true, glm::vec2(0), 1 }; - auto source = std::make_shared(); - std::vector> sources = { source }; - tileManager.setDataSources(sources); + auto source = std::make_shared(); + std::vector> sources = { source }; + tileManager.setTileSources(sources); std::set visibleTiles = { TileID{0,0,0} }; tileManager.updateTileSets(viewState, visibleTiles); @@ -231,9 +230,9 @@ TEST_CASE( "Use proxy Tile - circular proxies", "[TileManager][updateTileSets]" TileManager tileManager(worker); ViewState viewState { &s_projection, true, glm::vec2(0), 1 }; - auto source = std::make_shared(); - std::vector> sources = { source }; - tileManager.setDataSources(sources); + auto source = std::make_shared(); + std::vector> sources = { source }; + tileManager.setTileSources(sources); /// Start loading tile 0/0/0 std::set visibleTiles_1 = { TileID{0,0,0} };