From 00d50fc8917d4cab1b545903c6c247c3d26c4f97 Mon Sep 17 00:00:00 2001 From: Zach Thorson Date: Mon, 16 Aug 2021 15:40:11 -0500 Subject: [PATCH] Add ability to delete cache files from tiler - To allow the system to reset after subsequent runs, it can be useful to have the system automatically delete cache tiles when starting a new run. - This is added as a command line option to allow enabling/disabling of the behavior - This is preparation for allowing a job to resume (sans old keypoints) after a crash --- .../include/realm_ortho/tile_cache.h | 10 +++++-- modules/realm_ortho/src/map_tiler.cpp | 3 -- modules/realm_ortho/src/tile_cache.cpp | 29 +++++++++++++++++-- .../include/realm_stages/stage_settings.h | 1 + .../include/realm_stages/tileing.h | 6 ++++ modules/realm_stages/src/tileing.cpp | 17 +++++++++++ 6 files changed, 57 insertions(+), 9 deletions(-) diff --git a/modules/realm_ortho/include/realm_ortho/tile_cache.h b/modules/realm_ortho/include/realm_ortho/tile_cache.h index 69268257..49542aaf 100644 --- a/modules/realm_ortho/include/realm_ortho/tile_cache.h +++ b/modules/realm_ortho/include/realm_ortho/tile_cache.h @@ -55,6 +55,9 @@ class TileCache : public WorkerThreadBase void flushAll(); void loadAll(); + void deleteCache(); + void deleteCache(std::string layer); + private: bool m_has_init_directories; @@ -64,6 +67,7 @@ class TileCache : public WorkerThreadBase std::mutex m_mutex_cache; std::map m_cache; + std::mutex m_mutex_file_write; std::mutex m_mutex_do_update; bool m_do_update; @@ -78,10 +82,10 @@ class TileCache : public WorkerThreadBase void reset() override; - void load(const CacheElement::Ptr &element) const; - void write(const CacheElement::Ptr &element) const; + void load(const CacheElement::Ptr &element); + void write(const CacheElement::Ptr &element); - void flush(const CacheElement::Ptr &element) const; + void flush(const CacheElement::Ptr &element); bool isCached(const CacheElement::Ptr &element) const; diff --git a/modules/realm_ortho/src/map_tiler.cpp b/modules/realm_ortho/src/map_tiler.cpp index f9247972..e873eb3c 100644 --- a/modules/realm_ortho/src/map_tiler.cpp +++ b/modules/realm_ortho/src/map_tiler.cpp @@ -78,11 +78,8 @@ std::map MapTiler::createTiles(const CvGridMap::Ptr &ma // Therefore first identify how many tiles we have to split our map into by computing the tile indices cv::Rect2i tile_bounds_idx = computeTileBounds(roi, zoom_level); - LOG_F(INFO, "SENTERA: Tile bounds idx: %d, %d", tile_bounds_idx.x, tile_bounds_idx.y); - // With the tile indices we can compute the exact region of interest in the geographic frame in meters cv::Rect2d tile_bounds_meters = computeTileBoundsMeters(tile_bounds_idx, zoom_level); - LOG_F(INFO, "SENTERA: Tile bounds meters: %f, %f %f x %f", tile_bounds_meters.x, tile_bounds_meters.y, tile_bounds_meters.width, tile_bounds_meters.height); // Because our map is not yet guaranteed to have exactly the size of the tile region, we have to perform padding to // to fit exactly the tile map boundaries diff --git a/modules/realm_ortho/src/tile_cache.cpp b/modules/realm_ortho/src/tile_cache.cpp index 8484dfb0..bc52fcbf 100644 --- a/modules/realm_ortho/src/tile_cache.cpp +++ b/modules/realm_ortho/src/tile_cache.cpp @@ -257,8 +257,30 @@ void TileCache::loadAll() } } -void TileCache::load(const CacheElement::Ptr &element) const +void TileCache::deleteCache() { + std::lock_guard lock(m_mutex_file_write); + // Remove all cache items + flushAll(); + m_has_init_directories = false; + auto files = io::getFileList(m_dir_toplevel, ""); + for (auto & file : files) { + if (!file.empty()) io::removeFileOrDirectory(file); + } +} + +void TileCache::deleteCache(std::string layer) +{ + std::lock_guard lock(m_mutex_file_write); + // Attempt to remove the specific layer name + flushAll(); + m_has_init_directories = false; + io::removeFileOrDirectory(m_dir_toplevel + "/" + layer); +} + +void TileCache::load(const CacheElement::Ptr &element) +{ + std::lock_guard lock(m_mutex_file_write); for (const auto &meta : element->layer_meta) { std::string filename = m_dir_toplevel + "/" @@ -303,8 +325,9 @@ void TileCache::load(const CacheElement::Ptr &element) const } } -void TileCache::write(const CacheElement::Ptr &element) const +void TileCache::write(const CacheElement::Ptr &element) { + std::lock_guard lock(m_mutex_file_write); for (const auto &meta : element->layer_meta) { cv::Mat data = element->tile->data()->get(meta.name); @@ -341,7 +364,7 @@ void TileCache::write(const CacheElement::Ptr &element) const } } -void TileCache::flush(const CacheElement::Ptr &element) const +void TileCache::flush(const CacheElement::Ptr &element) { if (!element->was_written) write(element); diff --git a/modules/realm_stages/include/realm_stages/stage_settings.h b/modules/realm_stages/include/realm_stages/stage_settings.h index 4c282b08..bb9506f9 100644 --- a/modules/realm_stages/include/realm_stages/stage_settings.h +++ b/modules/realm_stages/include/realm_stages/stage_settings.h @@ -135,6 +135,7 @@ class TileingSettings : public StageSettings add("tms_tiles", Parameter_t{1, "Generate output tiles using TMS standard. If false, Google/OSM standard is used."}); add("min_zoom", Parameter_t{11, "The minimum tile zoom to generate for the output tiles."}); add("max_zoom", Parameter_t{20, "The maximum tile zoom to generate for the output tiles. (Set to -1 for maximum zoom based on GSD)"}); + add("delete_cache_on_init", Parameter_t{0, "If there are leftover cache items in the cache folder, delete them before starting the stage."}); } }; diff --git a/modules/realm_stages/include/realm_stages/tileing.h b/modules/realm_stages/include/realm_stages/tileing.h index 7fa4d0c9..f669b504 100644 --- a/modules/realm_stages/include/realm_stages/tileing.h +++ b/modules/realm_stages/include/realm_stages/tileing.h @@ -61,6 +61,9 @@ class Tileing : public StageBase bool process() override; void saveAll(); + void deleteCache(); + void deleteCache(std::string layer); + private: std::deque m_buffer; std::mutex m_mutex_buffer; @@ -78,6 +81,9 @@ class Tileing : public StageBase /// The maximum zoom to generate. May not be generated if GSD isn't sufficient int m_max_tile_zoom; + /// Indicates we should wipe the cache directory when starting or resetting the stage + bool m_delete_cache_on_init; + /// The directory to store the output map tiles in, defaults to log directory std::string m_cache_path; diff --git a/modules/realm_stages/src/tileing.cpp b/modules/realm_stages/src/tileing.cpp index c860c7d6..4d6398ed 100644 --- a/modules/realm_stages/src/tileing.cpp +++ b/modules/realm_stages/src/tileing.cpp @@ -30,6 +30,7 @@ Tileing::Tileing(const StageSettings::Ptr &stage_set, double rate) m_generate_tms_tiles((*stage_set)["tms_tiles"].toInt() > 0), m_min_tile_zoom((*stage_set)["min_zoom"].toInt()), m_max_tile_zoom((*stage_set)["max_zoom"].toInt()), + m_delete_cache_on_init((*stage_set)["delete_cache_on_init"].toInt() > 0), m_utm_reference(nullptr), m_map_tiler(nullptr), m_tile_cache(nullptr), @@ -246,6 +247,19 @@ bool Tileing::process() return has_processed; } +void Tileing::deleteCache() { + if (m_tile_cache) + { + m_tile_cache->deleteCache(); + } +} + +void Tileing::deleteCache(std::string layer) { + if (m_tile_cache) { + m_tile_cache->deleteCache(layer); + } +} + Tile::Ptr Tileing::merge(const Tile::Ptr &t1, const Tile::Ptr &t2) { if (t2->data()->empty()) @@ -390,6 +404,9 @@ void Tileing::initStageCallback() { m_map_tiler = std::make_shared(true, m_generate_tms_tiles); m_tile_cache = std::make_shared("tile_cache", 500, m_cache_path, false); + if (m_delete_cache_on_init) { + deleteCache(); + } m_tile_cache->start(); } }