diff --git a/src/replicator/replication.cc b/src/replicator/replication.cc index 0cdde75a7..85f202109 100644 --- a/src/replicator/replication.cc +++ b/src/replicator/replication.cc @@ -675,18 +675,19 @@ RemoteURL::updatePath(int _major, int _minor, int _index) } void -RemoteURL::Increment(void) +RemoteURL::increment(void) { + boost::format majorfmt("%03d"); boost::format minorfmt("%03d"); boost::format indexfmt("%03d"); std::string newpath; - if (minor == 999) { + + if (minor == 999 && index == 999) { major++; minor = 0; index = 0; - } - if (index == 999) { + } else if (index == 999) { minor++; index = 0; } else { @@ -701,20 +702,20 @@ RemoteURL::Increment(void) } void -RemoteURL::Decrement(void) +RemoteURL::decrement(void) { boost::format majorfmt("%03d"); boost::format minorfmt("%03d"); boost::format indexfmt("%03d"); std::string newpath; - if (minor == 0) { + + if (minor == 000 && index == 000) { major--; - minor = 0; - index = 0; - } - if (index == 0) { + minor = 999; + index = 999; + } else if (index == 000) { minor--; - index = 0; + index = 999; } else { index--; } @@ -723,10 +724,7 @@ RemoteURL::Decrement(void) minorfmt % (minor); indexfmt % (index); - newpath = majorfmt.str() + "/" + minorfmt.str() + "/" + indexfmt.str(); - boost::algorithm::replace_all(destdir, subpath, newpath); - boost::algorithm::replace_all(filespec, subpath, newpath); - subpath = newpath; + updatePath(major, minor, index); } RemoteURL & diff --git a/src/replicator/replication.hh b/src/replicator/replication.hh index e59c0c1f8..749a12c83 100644 --- a/src/replicator/replication.hh +++ b/src/replicator/replication.hh @@ -228,9 +228,9 @@ class RemoteURL { /// Dump internal data for debugging void dump(void); /// Increment the numerical part of the path by one file - void Increment(void); + void increment(void); /// Decrement the numerical part of the path by one file - void Decrement(void); + void decrement(void); /// Copy one remote object to another RemoteURL &operator=(const RemoteURL &inr); long sequence() const; ///< The sequence number of this path diff --git a/src/replicator/threads.cc b/src/replicator/threads.cc index 9815e0bf2..9d1e8aa1a 100644 --- a/src/replicator/threads.cc +++ b/src/replicator/threads.cc @@ -130,7 +130,7 @@ getClosest(std::shared_ptr> tasks, ptime now) { void startMonitorChangesets(std::shared_ptr &remote, const multipolygon_t &poly, - const UnderpassConfig &config) + const UnderpassConfig config) { #ifdef TIMING_DEBUG boost::timer::auto_cpu_timer timer("startMonitorChangesets: took %w seconds\n"); @@ -172,6 +172,7 @@ startMonitorChangesets(std::shared_ptr &remote, auto last_task = std::make_shared(); bool caughtUpWithNow = false; bool monitoring = true; + while (monitoring) { auto tasks = std::make_shared>(); i = cores*2; @@ -180,7 +181,7 @@ startMonitorChangesets(std::shared_ptr &remote, std::this_thread::sleep_for(delay); if (last_task->status == reqfile_t::success || (last_task->status == reqfile_t::remoteNotFound && !caughtUpWithNow)) { - remote->Increment(); + remote->increment(); if (!config.silent) { remote->dump(); } @@ -314,7 +315,7 @@ startMonitorChanges(std::shared_ptr &remote, std::this_thread::sleep_for(delay); if (last_task->status == reqfile_t::success || (last_task->status == reqfile_t::remoteNotFound && !caughtUpWithNow)) { - remote->Increment(); + remote->increment(); if (!config.silent) { remote->dump(); } diff --git a/src/replicator/threads.hh b/src/replicator/threads.hh index 39004aa79..dca7727a0 100644 --- a/src/replicator/threads.hh +++ b/src/replicator/threads.hh @@ -105,7 +105,7 @@ struct ReplicationTask { extern void startMonitorChangesets(std::shared_ptr &remote, const multipolygon_t &poly, - const underpassconfig::UnderpassConfig &config + const underpassconfig::UnderpassConfig config ); /// This updates several fields in the changesets table, which are part of diff --git a/src/testsuite/libunderpass.all/stats-test.cc b/src/testsuite/libunderpass.all/stats-test.cc index d9caf1c89..79cda99ae 100644 --- a/src/testsuite/libunderpass.all/stats-test.cc +++ b/src/testsuite/libunderpass.all/stats-test.cc @@ -125,7 +125,7 @@ class TestStats { auto stats = change.collectStats(boundary); jsonstr += statsToJSON(stats, osmchange->filespec); - osmchange->Increment(); + osmchange->increment(); } jsonstr.erase(jsonstr.size() - 2); diff --git a/src/underpass.cc b/src/underpass.cc index ee74c60a9..6026c6d62 100644 --- a/src/underpass.cc +++ b/src/underpass.cc @@ -84,9 +84,6 @@ int main(int argc, char *argv[]) { - // The changesets URL path (e.g. "/001/001/999") - std::string starting_url_path; - std::string datadir = "replication/"; std::string boundary = "/etc/underpass/priority.geojson"; @@ -153,19 +150,20 @@ main(int argc, char *argv[]) if (vm.count("debug")) { dbglogfile.setVerbosity(); } + if (vm.count("silent")) { + config.silent = true; + } + // Database if (vm.count("server")) { config.underpass_db_url = vm["server"].as(); } + // Local cache if (vm.count("destdir_base")) { config.destdir_base = vm["destdir_base"].as(); } - if (vm.count("silent")) { - config.silent = true; - } - // Concurrency if (vm.count("concurrency")) { const auto concurrency = vm["concurrency"].as(); @@ -184,6 +182,7 @@ main(int argc, char *argv[]) } if (vm.count("timestamp") || vm.count("url")) { + // Planet server if (vm.count("planet")) { config.planet_server = vm["planet"].as(); @@ -199,15 +198,19 @@ main(int argc, char *argv[]) config.planet_server = config.planet_servers[0].domain; } + // Priority boundary + multipolygon_t poly; if (vm.count("boundary")) { boundary = vm["boundary"].as(); } - - // Boundary geoutil::GeoUtil geou; if (!geou.readFile(boundary)) { log_debug("Could not find '%1%' area file!", boundary); } + multipolygon_t * oscboundary = &poly; + if (!vm.count("oscnoboundary")) { + oscboundary = &geou.boundary; + } // Features if (vm.count("disable-validation")) { @@ -222,15 +225,13 @@ main(int argc, char *argv[]) // Replication planetreplicator::PlanetReplicator replicator; - std::shared_ptr> data; - - if (!starting_url_path.empty() && vm.count("timestamp")) { + if (vm.count("url") && vm.count("timestamp")) { log_debug("ERROR: 'url' takes precedence over 'timestamp' arguments are mutually exclusive!"); exit(-1); } - // This is the default data directory on that server + // Default data directory on the server if (vm.count("datadir")) { datadir = vm["datadir"].as(); } @@ -238,10 +239,9 @@ main(int argc, char *argv[]) if (tmp != 0) { datadir = tmp; } - - // Add datadir to config config.datadir = datadir; + // Frequency: minutely, hourly, daily if (vm.count("frequency")) { const auto strfreq = vm["frequency"].as(); if (strfreq[0] == 'm') { @@ -276,55 +276,52 @@ main(int argc, char *argv[]) } } else if (vm.count("url")) { replicator.connectServer("https://" + config.planet_server); - // This is the changesets path part (ex. 000/075/000), takes precedence over 'timestamp' - // option. This only applies to the osm change files, as it's timestamp is used to - // start the changesets. std::string fullurl = "https://" + config.planet_server + "/replication/" + StateFile::freq_to_string(config.frequency); std::vector parts; boost::split(parts, vm["url"].as(), boost::is_any_of("/")); - // fullurl += "/" + vm["url"].as() + "/" + parts[2] + ".state.txt"; fullurl += "/" + vm["url"].as() + ".state.txt"; + osmchange->parse(fullurl); osmchange->destdir_base = config.destdir_base; auto data = replicator.downloadFile(*osmchange).data; StateFile start(osmchange->filespec, false); - //start.dump(); config.start_time = start.timestamp; boost::algorithm::replace_all(osmchange->filespec, ".state.txt", ".osc.gz"); } - std::thread changesetThread; + // OsmChanges std::thread osmChangeThread; - - multipolygon_t poly; if (!vm.count("changesets")) { multipolygon_t * osmboundary = &poly; if (!vm.count("osmnoboundary")) { osmboundary = &geou.boundary; } osmchange->destdir_base = config.destdir_base; + osmchange->dump(); osmChangeThread = std::thread(replicatorthreads::startMonitorChanges, std::ref(osmchange), - std::ref(*osmboundary), std::ref(config)); + std::ref(*osmboundary), config); } - config.frequency = replication::changeset; - auto changeset = replicator.findRemotePath(config, config.start_time); - changeset->destdir_base = config.destdir_base; - if (vm.count("changeseturl")) { + + // Changesets + std::thread changesetThread; + if (vm.count("changeseturl") || vm.count("timestamp")) { + config.frequency = replication::changeset; + auto changeset = replicator.findRemotePath(config, config.start_time); + changeset->destdir_base = config.destdir_base; std::vector parts; boost::split(parts, vm["changeseturl"].as(), boost::is_any_of("/")); changeset->updatePath(stoi(parts[0]),stoi(parts[1]),stoi(parts[2])); if (!config.silent) { changeset->dump(); } - } - if (!vm.count("osmchanges")) { - multipolygon_t * oscboundary = &poly; - if (!vm.count("oscnoboundary")) { - oscboundary = &geou.boundary; + if (!vm.count("osmchanges")) { + changesetThread = std::thread(replicatorthreads::startMonitorChangesets, std::ref(changeset), + std::ref(*oscboundary), std::ref(config)); } - changesetThread = std::thread(replicatorthreads::startMonitorChangesets, std::ref(changeset), - std::ref(*oscboundary), std::ref(config)); } + + // Start processing + if (changesetThread.joinable()) { changesetThread.join(); } @@ -336,6 +333,7 @@ main(int argc, char *argv[]) } + // Bootstrapping if (vm.count("bootstrap")){ std::thread bootstrapThread; std::cout << "Starting bootstrapping process ..." << std::endl;