diff --git a/data/Dockerfile b/data/Dockerfile index 96c8c947f..a391ce0ed 100644 --- a/data/Dockerfile +++ b/data/Dockerfile @@ -21,7 +21,7 @@ RUN mkdir output \ && cp -r sources/img/* output # put all data into the cdn folder -COPY external/maps/roomfinder output/maps/roomfinder +COPY external/results/maps/roomfinder output/maps/roomfinder # Synonyms are also provided via the cdn for the server container build COPY search_synonyms.json output/ diff --git a/data/external/main.py b/data/external/main.py index bad432b5e..cd5add68e 100644 --- a/data/external/main.py +++ b/data/external/main.py @@ -1,26 +1,26 @@ import os -from external import downloader +from external.scrapers import roomfinder, tumonline +from external.scraping_utils import CACHE_PATH if __name__ == "__main__": # Create cache directory structure - cache_base = os.path.join(os.path.dirname(__file__), "cache") - os.makedirs(cache_base, exist_ok=True) - os.makedirs(os.path.join(cache_base, "filter"), exist_ok=True) - os.makedirs(os.path.join(cache_base, "tumonline"), exist_ok=True) - os.makedirs(os.path.join(cache_base, "room"), exist_ok=True) - os.makedirs(os.path.join(cache_base, "maps/roomfinder"), exist_ok=True) - os.makedirs(os.path.join(cache_base, "maps/roomfinder/kmz"), exist_ok=True) + os.makedirs(CACHE_PATH, exist_ok=True) + os.makedirs(CACHE_PATH / "filter", exist_ok=True) + os.makedirs(CACHE_PATH / "tumonline", exist_ok=True) + os.makedirs(CACHE_PATH / "room", exist_ok=True) + os.makedirs(CACHE_PATH / "maps" / "roomfinder", exist_ok=True) + os.makedirs(CACHE_PATH / "maps" / "roomfinder" / "kmz", exist_ok=True) # You can comment out steps that should be skipped. # The downloader will automatically create a cache in `cache/`. - downloader.roomfinder_buildings() - downloader.tumonline_buildings() + roomfinder.scrape_buildings() + tumonline.scrape_buildings() - downloader.roomfinder_rooms() - downloader.tumonline_rooms() + roomfinder.scrape_rooms() + tumonline.scrape_rooms() - downloader.tumonline_usages() + tumonline.scrape_usages() - downloader.roomfinder_maps() - downloader.tumonline_orgs() + roomfinder.scrape_maps() + tumonline.scrape_orgs() diff --git a/data/external/buildings_roomfinder.json b/data/external/results/buildings_roomfinder.json similarity index 100% rename from data/external/buildings_roomfinder.json rename to data/external/results/buildings_roomfinder.json diff --git a/data/external/buildings_tumonline.json b/data/external/results/buildings_tumonline.json similarity index 100% rename from data/external/buildings_tumonline.json rename to data/external/results/buildings_tumonline.json diff --git a/data/external/maps/roomfinder/rf10.webp b/data/external/results/maps/roomfinder/rf10.webp similarity index 100% rename from data/external/maps/roomfinder/rf10.webp rename to data/external/results/maps/roomfinder/rf10.webp diff --git a/data/external/maps/roomfinder/rf100.webp b/data/external/results/maps/roomfinder/rf100.webp similarity index 100% rename from data/external/maps/roomfinder/rf100.webp rename to data/external/results/maps/roomfinder/rf100.webp diff --git a/data/external/maps/roomfinder/rf101.webp b/data/external/results/maps/roomfinder/rf101.webp similarity index 100% rename from data/external/maps/roomfinder/rf101.webp rename to data/external/results/maps/roomfinder/rf101.webp diff --git a/data/external/maps/roomfinder/rf102.webp b/data/external/results/maps/roomfinder/rf102.webp similarity index 100% rename from data/external/maps/roomfinder/rf102.webp rename to data/external/results/maps/roomfinder/rf102.webp diff --git a/data/external/maps/roomfinder/rf103.webp b/data/external/results/maps/roomfinder/rf103.webp similarity index 100% rename from data/external/maps/roomfinder/rf103.webp rename to data/external/results/maps/roomfinder/rf103.webp diff --git a/data/external/maps/roomfinder/rf104.webp b/data/external/results/maps/roomfinder/rf104.webp similarity index 100% rename from data/external/maps/roomfinder/rf104.webp rename to data/external/results/maps/roomfinder/rf104.webp diff --git a/data/external/maps/roomfinder/rf105.webp b/data/external/results/maps/roomfinder/rf105.webp similarity index 100% rename from data/external/maps/roomfinder/rf105.webp rename to data/external/results/maps/roomfinder/rf105.webp diff --git a/data/external/maps/roomfinder/rf106.webp b/data/external/results/maps/roomfinder/rf106.webp similarity index 100% rename from data/external/maps/roomfinder/rf106.webp rename to data/external/results/maps/roomfinder/rf106.webp diff --git a/data/external/maps/roomfinder/rf107.webp b/data/external/results/maps/roomfinder/rf107.webp similarity index 100% rename from data/external/maps/roomfinder/rf107.webp rename to data/external/results/maps/roomfinder/rf107.webp diff --git a/data/external/maps/roomfinder/rf108.webp b/data/external/results/maps/roomfinder/rf108.webp similarity index 100% rename from data/external/maps/roomfinder/rf108.webp rename to data/external/results/maps/roomfinder/rf108.webp diff --git a/data/external/maps/roomfinder/rf109.webp b/data/external/results/maps/roomfinder/rf109.webp similarity index 100% rename from data/external/maps/roomfinder/rf109.webp rename to data/external/results/maps/roomfinder/rf109.webp diff --git a/data/external/maps/roomfinder/rf110.webp b/data/external/results/maps/roomfinder/rf110.webp similarity index 100% rename from data/external/maps/roomfinder/rf110.webp rename to data/external/results/maps/roomfinder/rf110.webp diff --git a/data/external/maps/roomfinder/rf112.webp b/data/external/results/maps/roomfinder/rf112.webp similarity index 100% rename from data/external/maps/roomfinder/rf112.webp rename to data/external/results/maps/roomfinder/rf112.webp diff --git a/data/external/maps/roomfinder/rf113.webp b/data/external/results/maps/roomfinder/rf113.webp similarity index 100% rename from data/external/maps/roomfinder/rf113.webp rename to data/external/results/maps/roomfinder/rf113.webp diff --git a/data/external/maps/roomfinder/rf114.webp b/data/external/results/maps/roomfinder/rf114.webp similarity index 100% rename from data/external/maps/roomfinder/rf114.webp rename to data/external/results/maps/roomfinder/rf114.webp diff --git a/data/external/maps/roomfinder/rf115.webp b/data/external/results/maps/roomfinder/rf115.webp similarity index 100% rename from data/external/maps/roomfinder/rf115.webp rename to data/external/results/maps/roomfinder/rf115.webp diff --git a/data/external/maps/roomfinder/rf116.webp b/data/external/results/maps/roomfinder/rf116.webp similarity index 100% rename from data/external/maps/roomfinder/rf116.webp rename to data/external/results/maps/roomfinder/rf116.webp diff --git a/data/external/maps/roomfinder/rf117.webp b/data/external/results/maps/roomfinder/rf117.webp similarity index 100% rename from data/external/maps/roomfinder/rf117.webp rename to data/external/results/maps/roomfinder/rf117.webp diff --git a/data/external/maps/roomfinder/rf118.webp b/data/external/results/maps/roomfinder/rf118.webp similarity index 100% rename from data/external/maps/roomfinder/rf118.webp rename to data/external/results/maps/roomfinder/rf118.webp diff --git a/data/external/maps/roomfinder/rf119.webp b/data/external/results/maps/roomfinder/rf119.webp similarity index 100% rename from data/external/maps/roomfinder/rf119.webp rename to data/external/results/maps/roomfinder/rf119.webp diff --git a/data/external/maps/roomfinder/rf12.webp b/data/external/results/maps/roomfinder/rf12.webp similarity index 100% rename from data/external/maps/roomfinder/rf12.webp rename to data/external/results/maps/roomfinder/rf12.webp diff --git a/data/external/maps/roomfinder/rf120.webp b/data/external/results/maps/roomfinder/rf120.webp similarity index 100% rename from data/external/maps/roomfinder/rf120.webp rename to data/external/results/maps/roomfinder/rf120.webp diff --git a/data/external/maps/roomfinder/rf121.webp b/data/external/results/maps/roomfinder/rf121.webp similarity index 100% rename from data/external/maps/roomfinder/rf121.webp rename to data/external/results/maps/roomfinder/rf121.webp diff --git a/data/external/maps/roomfinder/rf122.webp b/data/external/results/maps/roomfinder/rf122.webp similarity index 100% rename from data/external/maps/roomfinder/rf122.webp rename to data/external/results/maps/roomfinder/rf122.webp diff --git a/data/external/maps/roomfinder/rf123.webp b/data/external/results/maps/roomfinder/rf123.webp similarity index 100% rename from data/external/maps/roomfinder/rf123.webp rename to data/external/results/maps/roomfinder/rf123.webp diff --git a/data/external/maps/roomfinder/rf124.webp b/data/external/results/maps/roomfinder/rf124.webp similarity index 100% rename from data/external/maps/roomfinder/rf124.webp rename to data/external/results/maps/roomfinder/rf124.webp diff --git a/data/external/maps/roomfinder/rf125.webp b/data/external/results/maps/roomfinder/rf125.webp similarity index 100% rename from data/external/maps/roomfinder/rf125.webp rename to data/external/results/maps/roomfinder/rf125.webp diff --git a/data/external/maps/roomfinder/rf126.webp b/data/external/results/maps/roomfinder/rf126.webp similarity index 100% rename from data/external/maps/roomfinder/rf126.webp rename to data/external/results/maps/roomfinder/rf126.webp diff --git a/data/external/maps/roomfinder/rf127.webp b/data/external/results/maps/roomfinder/rf127.webp similarity index 100% rename from data/external/maps/roomfinder/rf127.webp rename to data/external/results/maps/roomfinder/rf127.webp diff --git a/data/external/maps/roomfinder/rf128.webp b/data/external/results/maps/roomfinder/rf128.webp similarity index 100% rename from data/external/maps/roomfinder/rf128.webp rename to data/external/results/maps/roomfinder/rf128.webp diff --git a/data/external/maps/roomfinder/rf129.webp b/data/external/results/maps/roomfinder/rf129.webp similarity index 100% rename from data/external/maps/roomfinder/rf129.webp rename to data/external/results/maps/roomfinder/rf129.webp diff --git a/data/external/maps/roomfinder/rf130.webp b/data/external/results/maps/roomfinder/rf130.webp similarity index 100% rename from data/external/maps/roomfinder/rf130.webp rename to data/external/results/maps/roomfinder/rf130.webp diff --git a/data/external/maps/roomfinder/rf131.webp b/data/external/results/maps/roomfinder/rf131.webp similarity index 100% rename from data/external/maps/roomfinder/rf131.webp rename to data/external/results/maps/roomfinder/rf131.webp diff --git a/data/external/maps/roomfinder/rf132.webp b/data/external/results/maps/roomfinder/rf132.webp similarity index 100% rename from data/external/maps/roomfinder/rf132.webp rename to data/external/results/maps/roomfinder/rf132.webp diff --git a/data/external/maps/roomfinder/rf133.webp b/data/external/results/maps/roomfinder/rf133.webp similarity index 100% rename from data/external/maps/roomfinder/rf133.webp rename to data/external/results/maps/roomfinder/rf133.webp diff --git a/data/external/maps/roomfinder/rf134.webp b/data/external/results/maps/roomfinder/rf134.webp similarity index 100% rename from data/external/maps/roomfinder/rf134.webp rename to data/external/results/maps/roomfinder/rf134.webp diff --git a/data/external/maps/roomfinder/rf135.webp b/data/external/results/maps/roomfinder/rf135.webp similarity index 100% rename from data/external/maps/roomfinder/rf135.webp rename to data/external/results/maps/roomfinder/rf135.webp diff --git a/data/external/maps/roomfinder/rf136.webp b/data/external/results/maps/roomfinder/rf136.webp similarity index 100% rename from data/external/maps/roomfinder/rf136.webp rename to data/external/results/maps/roomfinder/rf136.webp diff --git a/data/external/maps/roomfinder/rf137.webp b/data/external/results/maps/roomfinder/rf137.webp similarity index 100% rename from data/external/maps/roomfinder/rf137.webp rename to data/external/results/maps/roomfinder/rf137.webp diff --git a/data/external/maps/roomfinder/rf138.webp b/data/external/results/maps/roomfinder/rf138.webp similarity index 100% rename from data/external/maps/roomfinder/rf138.webp rename to data/external/results/maps/roomfinder/rf138.webp diff --git a/data/external/maps/roomfinder/rf139.webp b/data/external/results/maps/roomfinder/rf139.webp similarity index 100% rename from data/external/maps/roomfinder/rf139.webp rename to data/external/results/maps/roomfinder/rf139.webp diff --git a/data/external/maps/roomfinder/rf14.webp b/data/external/results/maps/roomfinder/rf14.webp similarity index 100% rename from data/external/maps/roomfinder/rf14.webp rename to data/external/results/maps/roomfinder/rf14.webp diff --git a/data/external/maps/roomfinder/rf140.webp b/data/external/results/maps/roomfinder/rf140.webp similarity index 100% rename from data/external/maps/roomfinder/rf140.webp rename to data/external/results/maps/roomfinder/rf140.webp diff --git a/data/external/maps/roomfinder/rf141.webp b/data/external/results/maps/roomfinder/rf141.webp similarity index 100% rename from data/external/maps/roomfinder/rf141.webp rename to data/external/results/maps/roomfinder/rf141.webp diff --git a/data/external/maps/roomfinder/rf142.webp b/data/external/results/maps/roomfinder/rf142.webp similarity index 100% rename from data/external/maps/roomfinder/rf142.webp rename to data/external/results/maps/roomfinder/rf142.webp diff --git a/data/external/maps/roomfinder/rf143.webp b/data/external/results/maps/roomfinder/rf143.webp similarity index 100% rename from data/external/maps/roomfinder/rf143.webp rename to data/external/results/maps/roomfinder/rf143.webp diff --git a/data/external/maps/roomfinder/rf144.webp b/data/external/results/maps/roomfinder/rf144.webp similarity index 100% rename from data/external/maps/roomfinder/rf144.webp rename to data/external/results/maps/roomfinder/rf144.webp diff --git a/data/external/maps/roomfinder/rf145.webp b/data/external/results/maps/roomfinder/rf145.webp similarity index 100% rename from data/external/maps/roomfinder/rf145.webp rename to data/external/results/maps/roomfinder/rf145.webp diff --git a/data/external/maps/roomfinder/rf146.webp b/data/external/results/maps/roomfinder/rf146.webp similarity index 100% rename from data/external/maps/roomfinder/rf146.webp rename to data/external/results/maps/roomfinder/rf146.webp diff --git a/data/external/maps/roomfinder/rf147.webp b/data/external/results/maps/roomfinder/rf147.webp similarity index 100% rename from data/external/maps/roomfinder/rf147.webp rename to data/external/results/maps/roomfinder/rf147.webp diff --git a/data/external/maps/roomfinder/rf148.webp b/data/external/results/maps/roomfinder/rf148.webp similarity index 100% rename from data/external/maps/roomfinder/rf148.webp rename to data/external/results/maps/roomfinder/rf148.webp diff --git a/data/external/maps/roomfinder/rf149.webp b/data/external/results/maps/roomfinder/rf149.webp similarity index 100% rename from data/external/maps/roomfinder/rf149.webp rename to data/external/results/maps/roomfinder/rf149.webp diff --git a/data/external/maps/roomfinder/rf15.webp b/data/external/results/maps/roomfinder/rf15.webp similarity index 100% rename from data/external/maps/roomfinder/rf15.webp rename to data/external/results/maps/roomfinder/rf15.webp diff --git a/data/external/maps/roomfinder/rf150.webp b/data/external/results/maps/roomfinder/rf150.webp similarity index 100% rename from data/external/maps/roomfinder/rf150.webp rename to data/external/results/maps/roomfinder/rf150.webp diff --git a/data/external/maps/roomfinder/rf151.webp b/data/external/results/maps/roomfinder/rf151.webp similarity index 100% rename from data/external/maps/roomfinder/rf151.webp rename to data/external/results/maps/roomfinder/rf151.webp diff --git a/data/external/maps/roomfinder/rf152.webp b/data/external/results/maps/roomfinder/rf152.webp similarity index 100% rename from data/external/maps/roomfinder/rf152.webp rename to data/external/results/maps/roomfinder/rf152.webp diff --git a/data/external/maps/roomfinder/rf153.webp b/data/external/results/maps/roomfinder/rf153.webp similarity index 100% rename from data/external/maps/roomfinder/rf153.webp rename to data/external/results/maps/roomfinder/rf153.webp diff --git a/data/external/maps/roomfinder/rf154.webp b/data/external/results/maps/roomfinder/rf154.webp similarity index 100% rename from data/external/maps/roomfinder/rf154.webp rename to data/external/results/maps/roomfinder/rf154.webp diff --git a/data/external/maps/roomfinder/rf156.webp b/data/external/results/maps/roomfinder/rf156.webp similarity index 100% rename from data/external/maps/roomfinder/rf156.webp rename to data/external/results/maps/roomfinder/rf156.webp diff --git a/data/external/maps/roomfinder/rf16.webp b/data/external/results/maps/roomfinder/rf16.webp similarity index 100% rename from data/external/maps/roomfinder/rf16.webp rename to data/external/results/maps/roomfinder/rf16.webp diff --git a/data/external/maps/roomfinder/rf160.webp b/data/external/results/maps/roomfinder/rf160.webp similarity index 100% rename from data/external/maps/roomfinder/rf160.webp rename to data/external/results/maps/roomfinder/rf160.webp diff --git a/data/external/maps/roomfinder/rf161.webp b/data/external/results/maps/roomfinder/rf161.webp similarity index 100% rename from data/external/maps/roomfinder/rf161.webp rename to data/external/results/maps/roomfinder/rf161.webp diff --git a/data/external/maps/roomfinder/rf162.webp b/data/external/results/maps/roomfinder/rf162.webp similarity index 100% rename from data/external/maps/roomfinder/rf162.webp rename to data/external/results/maps/roomfinder/rf162.webp diff --git a/data/external/maps/roomfinder/rf163.webp b/data/external/results/maps/roomfinder/rf163.webp similarity index 100% rename from data/external/maps/roomfinder/rf163.webp rename to data/external/results/maps/roomfinder/rf163.webp diff --git a/data/external/maps/roomfinder/rf164.webp b/data/external/results/maps/roomfinder/rf164.webp similarity index 100% rename from data/external/maps/roomfinder/rf164.webp rename to data/external/results/maps/roomfinder/rf164.webp diff --git a/data/external/maps/roomfinder/rf165.webp b/data/external/results/maps/roomfinder/rf165.webp similarity index 100% rename from data/external/maps/roomfinder/rf165.webp rename to data/external/results/maps/roomfinder/rf165.webp diff --git a/data/external/maps/roomfinder/rf166.webp b/data/external/results/maps/roomfinder/rf166.webp similarity index 100% rename from data/external/maps/roomfinder/rf166.webp rename to data/external/results/maps/roomfinder/rf166.webp diff --git a/data/external/maps/roomfinder/rf167.webp b/data/external/results/maps/roomfinder/rf167.webp similarity index 100% rename from data/external/maps/roomfinder/rf167.webp rename to data/external/results/maps/roomfinder/rf167.webp diff --git a/data/external/maps/roomfinder/rf168.webp b/data/external/results/maps/roomfinder/rf168.webp similarity index 100% rename from data/external/maps/roomfinder/rf168.webp rename to data/external/results/maps/roomfinder/rf168.webp diff --git a/data/external/maps/roomfinder/rf169.webp b/data/external/results/maps/roomfinder/rf169.webp similarity index 100% rename from data/external/maps/roomfinder/rf169.webp rename to data/external/results/maps/roomfinder/rf169.webp diff --git a/data/external/maps/roomfinder/rf17.webp b/data/external/results/maps/roomfinder/rf17.webp similarity index 100% rename from data/external/maps/roomfinder/rf17.webp rename to data/external/results/maps/roomfinder/rf17.webp diff --git a/data/external/maps/roomfinder/rf170.webp b/data/external/results/maps/roomfinder/rf170.webp similarity index 100% rename from data/external/maps/roomfinder/rf170.webp rename to data/external/results/maps/roomfinder/rf170.webp diff --git a/data/external/maps/roomfinder/rf171.webp b/data/external/results/maps/roomfinder/rf171.webp similarity index 100% rename from data/external/maps/roomfinder/rf171.webp rename to data/external/results/maps/roomfinder/rf171.webp diff --git a/data/external/maps/roomfinder/rf172.webp b/data/external/results/maps/roomfinder/rf172.webp similarity index 100% rename from data/external/maps/roomfinder/rf172.webp rename to data/external/results/maps/roomfinder/rf172.webp diff --git a/data/external/maps/roomfinder/rf173.webp b/data/external/results/maps/roomfinder/rf173.webp similarity index 100% rename from data/external/maps/roomfinder/rf173.webp rename to data/external/results/maps/roomfinder/rf173.webp diff --git a/data/external/maps/roomfinder/rf174.webp b/data/external/results/maps/roomfinder/rf174.webp similarity index 100% rename from data/external/maps/roomfinder/rf174.webp rename to data/external/results/maps/roomfinder/rf174.webp diff --git a/data/external/maps/roomfinder/rf175.webp b/data/external/results/maps/roomfinder/rf175.webp similarity index 100% rename from data/external/maps/roomfinder/rf175.webp rename to data/external/results/maps/roomfinder/rf175.webp diff --git a/data/external/maps/roomfinder/rf176.webp b/data/external/results/maps/roomfinder/rf176.webp similarity index 100% rename from data/external/maps/roomfinder/rf176.webp rename to data/external/results/maps/roomfinder/rf176.webp diff --git a/data/external/maps/roomfinder/rf177.webp b/data/external/results/maps/roomfinder/rf177.webp similarity index 100% rename from data/external/maps/roomfinder/rf177.webp rename to data/external/results/maps/roomfinder/rf177.webp diff --git a/data/external/maps/roomfinder/rf178.webp b/data/external/results/maps/roomfinder/rf178.webp similarity index 100% rename from data/external/maps/roomfinder/rf178.webp rename to data/external/results/maps/roomfinder/rf178.webp diff --git a/data/external/maps/roomfinder/rf179.webp b/data/external/results/maps/roomfinder/rf179.webp similarity index 100% rename from data/external/maps/roomfinder/rf179.webp rename to data/external/results/maps/roomfinder/rf179.webp diff --git a/data/external/maps/roomfinder/rf18.webp b/data/external/results/maps/roomfinder/rf18.webp similarity index 100% rename from data/external/maps/roomfinder/rf18.webp rename to data/external/results/maps/roomfinder/rf18.webp diff --git a/data/external/maps/roomfinder/rf180.webp b/data/external/results/maps/roomfinder/rf180.webp similarity index 100% rename from data/external/maps/roomfinder/rf180.webp rename to data/external/results/maps/roomfinder/rf180.webp diff --git a/data/external/maps/roomfinder/rf181.webp b/data/external/results/maps/roomfinder/rf181.webp similarity index 100% rename from data/external/maps/roomfinder/rf181.webp rename to data/external/results/maps/roomfinder/rf181.webp diff --git a/data/external/maps/roomfinder/rf182.webp b/data/external/results/maps/roomfinder/rf182.webp similarity index 100% rename from data/external/maps/roomfinder/rf182.webp rename to data/external/results/maps/roomfinder/rf182.webp diff --git a/data/external/maps/roomfinder/rf183.webp b/data/external/results/maps/roomfinder/rf183.webp similarity index 100% rename from data/external/maps/roomfinder/rf183.webp rename to data/external/results/maps/roomfinder/rf183.webp diff --git a/data/external/maps/roomfinder/rf184.webp b/data/external/results/maps/roomfinder/rf184.webp similarity index 100% rename from data/external/maps/roomfinder/rf184.webp rename to data/external/results/maps/roomfinder/rf184.webp diff --git a/data/external/maps/roomfinder/rf185.webp b/data/external/results/maps/roomfinder/rf185.webp similarity index 100% rename from data/external/maps/roomfinder/rf185.webp rename to data/external/results/maps/roomfinder/rf185.webp diff --git a/data/external/maps/roomfinder/rf186.webp b/data/external/results/maps/roomfinder/rf186.webp similarity index 100% rename from data/external/maps/roomfinder/rf186.webp rename to data/external/results/maps/roomfinder/rf186.webp diff --git a/data/external/maps/roomfinder/rf187.webp b/data/external/results/maps/roomfinder/rf187.webp similarity index 100% rename from data/external/maps/roomfinder/rf187.webp rename to data/external/results/maps/roomfinder/rf187.webp diff --git a/data/external/maps/roomfinder/rf188.webp b/data/external/results/maps/roomfinder/rf188.webp similarity index 100% rename from data/external/maps/roomfinder/rf188.webp rename to data/external/results/maps/roomfinder/rf188.webp diff --git a/data/external/maps/roomfinder/rf189.webp b/data/external/results/maps/roomfinder/rf189.webp similarity index 100% rename from data/external/maps/roomfinder/rf189.webp rename to data/external/results/maps/roomfinder/rf189.webp diff --git a/data/external/maps/roomfinder/rf19.webp b/data/external/results/maps/roomfinder/rf19.webp similarity index 100% rename from data/external/maps/roomfinder/rf19.webp rename to data/external/results/maps/roomfinder/rf19.webp diff --git a/data/external/maps/roomfinder/rf190.webp b/data/external/results/maps/roomfinder/rf190.webp similarity index 100% rename from data/external/maps/roomfinder/rf190.webp rename to data/external/results/maps/roomfinder/rf190.webp diff --git a/data/external/maps/roomfinder/rf191.webp b/data/external/results/maps/roomfinder/rf191.webp similarity index 100% rename from data/external/maps/roomfinder/rf191.webp rename to data/external/results/maps/roomfinder/rf191.webp diff --git a/data/external/maps/roomfinder/rf192.webp b/data/external/results/maps/roomfinder/rf192.webp similarity index 100% rename from data/external/maps/roomfinder/rf192.webp rename to data/external/results/maps/roomfinder/rf192.webp diff --git a/data/external/maps/roomfinder/rf193.webp b/data/external/results/maps/roomfinder/rf193.webp similarity index 100% rename from data/external/maps/roomfinder/rf193.webp rename to data/external/results/maps/roomfinder/rf193.webp diff --git a/data/external/maps/roomfinder/rf194.webp b/data/external/results/maps/roomfinder/rf194.webp similarity index 100% rename from data/external/maps/roomfinder/rf194.webp rename to data/external/results/maps/roomfinder/rf194.webp diff --git a/data/external/maps/roomfinder/rf195.webp b/data/external/results/maps/roomfinder/rf195.webp similarity index 100% rename from data/external/maps/roomfinder/rf195.webp rename to data/external/results/maps/roomfinder/rf195.webp diff --git a/data/external/maps/roomfinder/rf196.webp b/data/external/results/maps/roomfinder/rf196.webp similarity index 100% rename from data/external/maps/roomfinder/rf196.webp rename to data/external/results/maps/roomfinder/rf196.webp diff --git a/data/external/maps/roomfinder/rf197.webp b/data/external/results/maps/roomfinder/rf197.webp similarity index 100% rename from data/external/maps/roomfinder/rf197.webp rename to data/external/results/maps/roomfinder/rf197.webp diff --git a/data/external/maps/roomfinder/rf198.webp b/data/external/results/maps/roomfinder/rf198.webp similarity index 100% rename from data/external/maps/roomfinder/rf198.webp rename to data/external/results/maps/roomfinder/rf198.webp diff --git a/data/external/maps/roomfinder/rf199.webp b/data/external/results/maps/roomfinder/rf199.webp similarity index 100% rename from data/external/maps/roomfinder/rf199.webp rename to data/external/results/maps/roomfinder/rf199.webp diff --git a/data/external/maps/roomfinder/rf20.webp b/data/external/results/maps/roomfinder/rf20.webp similarity index 100% rename from data/external/maps/roomfinder/rf20.webp rename to data/external/results/maps/roomfinder/rf20.webp diff --git a/data/external/maps/roomfinder/rf201.webp b/data/external/results/maps/roomfinder/rf201.webp similarity index 100% rename from data/external/maps/roomfinder/rf201.webp rename to data/external/results/maps/roomfinder/rf201.webp diff --git a/data/external/maps/roomfinder/rf202.webp b/data/external/results/maps/roomfinder/rf202.webp similarity index 100% rename from data/external/maps/roomfinder/rf202.webp rename to data/external/results/maps/roomfinder/rf202.webp diff --git a/data/external/maps/roomfinder/rf203.webp b/data/external/results/maps/roomfinder/rf203.webp similarity index 100% rename from data/external/maps/roomfinder/rf203.webp rename to data/external/results/maps/roomfinder/rf203.webp diff --git a/data/external/maps/roomfinder/rf204.webp b/data/external/results/maps/roomfinder/rf204.webp similarity index 100% rename from data/external/maps/roomfinder/rf204.webp rename to data/external/results/maps/roomfinder/rf204.webp diff --git a/data/external/maps/roomfinder/rf205.webp b/data/external/results/maps/roomfinder/rf205.webp similarity index 100% rename from data/external/maps/roomfinder/rf205.webp rename to data/external/results/maps/roomfinder/rf205.webp diff --git a/data/external/maps/roomfinder/rf206.webp b/data/external/results/maps/roomfinder/rf206.webp similarity index 100% rename from data/external/maps/roomfinder/rf206.webp rename to data/external/results/maps/roomfinder/rf206.webp diff --git a/data/external/maps/roomfinder/rf207.webp b/data/external/results/maps/roomfinder/rf207.webp similarity index 100% rename from data/external/maps/roomfinder/rf207.webp rename to data/external/results/maps/roomfinder/rf207.webp diff --git a/data/external/maps/roomfinder/rf208.webp b/data/external/results/maps/roomfinder/rf208.webp similarity index 100% rename from data/external/maps/roomfinder/rf208.webp rename to data/external/results/maps/roomfinder/rf208.webp diff --git a/data/external/maps/roomfinder/rf209.webp b/data/external/results/maps/roomfinder/rf209.webp similarity index 100% rename from data/external/maps/roomfinder/rf209.webp rename to data/external/results/maps/roomfinder/rf209.webp diff --git a/data/external/maps/roomfinder/rf21.webp b/data/external/results/maps/roomfinder/rf21.webp similarity index 100% rename from data/external/maps/roomfinder/rf21.webp rename to data/external/results/maps/roomfinder/rf21.webp diff --git a/data/external/maps/roomfinder/rf210.webp b/data/external/results/maps/roomfinder/rf210.webp similarity index 100% rename from data/external/maps/roomfinder/rf210.webp rename to data/external/results/maps/roomfinder/rf210.webp diff --git a/data/external/maps/roomfinder/rf211.webp b/data/external/results/maps/roomfinder/rf211.webp similarity index 100% rename from data/external/maps/roomfinder/rf211.webp rename to data/external/results/maps/roomfinder/rf211.webp diff --git a/data/external/maps/roomfinder/rf212.webp b/data/external/results/maps/roomfinder/rf212.webp similarity index 100% rename from data/external/maps/roomfinder/rf212.webp rename to data/external/results/maps/roomfinder/rf212.webp diff --git a/data/external/maps/roomfinder/rf213.webp b/data/external/results/maps/roomfinder/rf213.webp similarity index 100% rename from data/external/maps/roomfinder/rf213.webp rename to data/external/results/maps/roomfinder/rf213.webp diff --git a/data/external/maps/roomfinder/rf214.webp b/data/external/results/maps/roomfinder/rf214.webp similarity index 100% rename from data/external/maps/roomfinder/rf214.webp rename to data/external/results/maps/roomfinder/rf214.webp diff --git a/data/external/maps/roomfinder/rf215.webp b/data/external/results/maps/roomfinder/rf215.webp similarity index 100% rename from data/external/maps/roomfinder/rf215.webp rename to data/external/results/maps/roomfinder/rf215.webp diff --git a/data/external/maps/roomfinder/rf216.webp b/data/external/results/maps/roomfinder/rf216.webp similarity index 100% rename from data/external/maps/roomfinder/rf216.webp rename to data/external/results/maps/roomfinder/rf216.webp diff --git a/data/external/maps/roomfinder/rf217.webp b/data/external/results/maps/roomfinder/rf217.webp similarity index 100% rename from data/external/maps/roomfinder/rf217.webp rename to data/external/results/maps/roomfinder/rf217.webp diff --git a/data/external/maps/roomfinder/rf22.webp b/data/external/results/maps/roomfinder/rf22.webp similarity index 100% rename from data/external/maps/roomfinder/rf22.webp rename to data/external/results/maps/roomfinder/rf22.webp diff --git a/data/external/maps/roomfinder/rf228.webp b/data/external/results/maps/roomfinder/rf228.webp similarity index 100% rename from data/external/maps/roomfinder/rf228.webp rename to data/external/results/maps/roomfinder/rf228.webp diff --git a/data/external/maps/roomfinder/rf229.webp b/data/external/results/maps/roomfinder/rf229.webp similarity index 100% rename from data/external/maps/roomfinder/rf229.webp rename to data/external/results/maps/roomfinder/rf229.webp diff --git a/data/external/maps/roomfinder/rf230.webp b/data/external/results/maps/roomfinder/rf230.webp similarity index 100% rename from data/external/maps/roomfinder/rf230.webp rename to data/external/results/maps/roomfinder/rf230.webp diff --git a/data/external/maps/roomfinder/rf231.webp b/data/external/results/maps/roomfinder/rf231.webp similarity index 100% rename from data/external/maps/roomfinder/rf231.webp rename to data/external/results/maps/roomfinder/rf231.webp diff --git a/data/external/maps/roomfinder/rf232.webp b/data/external/results/maps/roomfinder/rf232.webp similarity index 100% rename from data/external/maps/roomfinder/rf232.webp rename to data/external/results/maps/roomfinder/rf232.webp diff --git a/data/external/maps/roomfinder/rf233.webp b/data/external/results/maps/roomfinder/rf233.webp similarity index 100% rename from data/external/maps/roomfinder/rf233.webp rename to data/external/results/maps/roomfinder/rf233.webp diff --git a/data/external/maps/roomfinder/rf235.webp b/data/external/results/maps/roomfinder/rf235.webp similarity index 100% rename from data/external/maps/roomfinder/rf235.webp rename to data/external/results/maps/roomfinder/rf235.webp diff --git a/data/external/maps/roomfinder/rf236.webp b/data/external/results/maps/roomfinder/rf236.webp similarity index 100% rename from data/external/maps/roomfinder/rf236.webp rename to data/external/results/maps/roomfinder/rf236.webp diff --git a/data/external/maps/roomfinder/rf237.webp b/data/external/results/maps/roomfinder/rf237.webp similarity index 100% rename from data/external/maps/roomfinder/rf237.webp rename to data/external/results/maps/roomfinder/rf237.webp diff --git a/data/external/maps/roomfinder/rf238.webp b/data/external/results/maps/roomfinder/rf238.webp similarity index 100% rename from data/external/maps/roomfinder/rf238.webp rename to data/external/results/maps/roomfinder/rf238.webp diff --git a/data/external/maps/roomfinder/rf239.webp b/data/external/results/maps/roomfinder/rf239.webp similarity index 100% rename from data/external/maps/roomfinder/rf239.webp rename to data/external/results/maps/roomfinder/rf239.webp diff --git a/data/external/maps/roomfinder/rf240.webp b/data/external/results/maps/roomfinder/rf240.webp similarity index 100% rename from data/external/maps/roomfinder/rf240.webp rename to data/external/results/maps/roomfinder/rf240.webp diff --git a/data/external/maps/roomfinder/rf241.webp b/data/external/results/maps/roomfinder/rf241.webp similarity index 100% rename from data/external/maps/roomfinder/rf241.webp rename to data/external/results/maps/roomfinder/rf241.webp diff --git a/data/external/maps/roomfinder/rf242.webp b/data/external/results/maps/roomfinder/rf242.webp similarity index 100% rename from data/external/maps/roomfinder/rf242.webp rename to data/external/results/maps/roomfinder/rf242.webp diff --git a/data/external/maps/roomfinder/rf243.webp b/data/external/results/maps/roomfinder/rf243.webp similarity index 100% rename from data/external/maps/roomfinder/rf243.webp rename to data/external/results/maps/roomfinder/rf243.webp diff --git a/data/external/maps/roomfinder/rf244.webp b/data/external/results/maps/roomfinder/rf244.webp similarity index 100% rename from data/external/maps/roomfinder/rf244.webp rename to data/external/results/maps/roomfinder/rf244.webp diff --git a/data/external/maps/roomfinder/rf245.webp b/data/external/results/maps/roomfinder/rf245.webp similarity index 100% rename from data/external/maps/roomfinder/rf245.webp rename to data/external/results/maps/roomfinder/rf245.webp diff --git a/data/external/maps/roomfinder/rf246.webp b/data/external/results/maps/roomfinder/rf246.webp similarity index 100% rename from data/external/maps/roomfinder/rf246.webp rename to data/external/results/maps/roomfinder/rf246.webp diff --git a/data/external/maps/roomfinder/rf247.webp b/data/external/results/maps/roomfinder/rf247.webp similarity index 100% rename from data/external/maps/roomfinder/rf247.webp rename to data/external/results/maps/roomfinder/rf247.webp diff --git a/data/external/maps/roomfinder/rf25.webp b/data/external/results/maps/roomfinder/rf25.webp similarity index 100% rename from data/external/maps/roomfinder/rf25.webp rename to data/external/results/maps/roomfinder/rf25.webp diff --git a/data/external/maps/roomfinder/rf26.webp b/data/external/results/maps/roomfinder/rf26.webp similarity index 100% rename from data/external/maps/roomfinder/rf26.webp rename to data/external/results/maps/roomfinder/rf26.webp diff --git a/data/external/maps/roomfinder/rf27.webp b/data/external/results/maps/roomfinder/rf27.webp similarity index 100% rename from data/external/maps/roomfinder/rf27.webp rename to data/external/results/maps/roomfinder/rf27.webp diff --git a/data/external/maps/roomfinder/rf28.webp b/data/external/results/maps/roomfinder/rf28.webp similarity index 100% rename from data/external/maps/roomfinder/rf28.webp rename to data/external/results/maps/roomfinder/rf28.webp diff --git a/data/external/maps/roomfinder/rf29.webp b/data/external/results/maps/roomfinder/rf29.webp similarity index 100% rename from data/external/maps/roomfinder/rf29.webp rename to data/external/results/maps/roomfinder/rf29.webp diff --git a/data/external/maps/roomfinder/rf30.webp b/data/external/results/maps/roomfinder/rf30.webp similarity index 100% rename from data/external/maps/roomfinder/rf30.webp rename to data/external/results/maps/roomfinder/rf30.webp diff --git a/data/external/maps/roomfinder/rf31.webp b/data/external/results/maps/roomfinder/rf31.webp similarity index 100% rename from data/external/maps/roomfinder/rf31.webp rename to data/external/results/maps/roomfinder/rf31.webp diff --git a/data/external/maps/roomfinder/rf32.webp b/data/external/results/maps/roomfinder/rf32.webp similarity index 100% rename from data/external/maps/roomfinder/rf32.webp rename to data/external/results/maps/roomfinder/rf32.webp diff --git a/data/external/maps/roomfinder/rf33.webp b/data/external/results/maps/roomfinder/rf33.webp similarity index 100% rename from data/external/maps/roomfinder/rf33.webp rename to data/external/results/maps/roomfinder/rf33.webp diff --git a/data/external/maps/roomfinder/rf34.webp b/data/external/results/maps/roomfinder/rf34.webp similarity index 100% rename from data/external/maps/roomfinder/rf34.webp rename to data/external/results/maps/roomfinder/rf34.webp diff --git a/data/external/maps/roomfinder/rf35.webp b/data/external/results/maps/roomfinder/rf35.webp similarity index 100% rename from data/external/maps/roomfinder/rf35.webp rename to data/external/results/maps/roomfinder/rf35.webp diff --git a/data/external/maps/roomfinder/rf36.webp b/data/external/results/maps/roomfinder/rf36.webp similarity index 100% rename from data/external/maps/roomfinder/rf36.webp rename to data/external/results/maps/roomfinder/rf36.webp diff --git a/data/external/maps/roomfinder/rf37.webp b/data/external/results/maps/roomfinder/rf37.webp similarity index 100% rename from data/external/maps/roomfinder/rf37.webp rename to data/external/results/maps/roomfinder/rf37.webp diff --git a/data/external/maps/roomfinder/rf38.webp b/data/external/results/maps/roomfinder/rf38.webp similarity index 100% rename from data/external/maps/roomfinder/rf38.webp rename to data/external/results/maps/roomfinder/rf38.webp diff --git a/data/external/maps/roomfinder/rf39.webp b/data/external/results/maps/roomfinder/rf39.webp similarity index 100% rename from data/external/maps/roomfinder/rf39.webp rename to data/external/results/maps/roomfinder/rf39.webp diff --git a/data/external/maps/roomfinder/rf40.webp b/data/external/results/maps/roomfinder/rf40.webp similarity index 100% rename from data/external/maps/roomfinder/rf40.webp rename to data/external/results/maps/roomfinder/rf40.webp diff --git a/data/external/maps/roomfinder/rf41.webp b/data/external/results/maps/roomfinder/rf41.webp similarity index 100% rename from data/external/maps/roomfinder/rf41.webp rename to data/external/results/maps/roomfinder/rf41.webp diff --git a/data/external/maps/roomfinder/rf42.webp b/data/external/results/maps/roomfinder/rf42.webp similarity index 100% rename from data/external/maps/roomfinder/rf42.webp rename to data/external/results/maps/roomfinder/rf42.webp diff --git a/data/external/maps/roomfinder/rf43.webp b/data/external/results/maps/roomfinder/rf43.webp similarity index 100% rename from data/external/maps/roomfinder/rf43.webp rename to data/external/results/maps/roomfinder/rf43.webp diff --git a/data/external/maps/roomfinder/rf44.webp b/data/external/results/maps/roomfinder/rf44.webp similarity index 100% rename from data/external/maps/roomfinder/rf44.webp rename to data/external/results/maps/roomfinder/rf44.webp diff --git a/data/external/maps/roomfinder/rf45.webp b/data/external/results/maps/roomfinder/rf45.webp similarity index 100% rename from data/external/maps/roomfinder/rf45.webp rename to data/external/results/maps/roomfinder/rf45.webp diff --git a/data/external/maps/roomfinder/rf46.webp b/data/external/results/maps/roomfinder/rf46.webp similarity index 100% rename from data/external/maps/roomfinder/rf46.webp rename to data/external/results/maps/roomfinder/rf46.webp diff --git a/data/external/maps/roomfinder/rf47.webp b/data/external/results/maps/roomfinder/rf47.webp similarity index 100% rename from data/external/maps/roomfinder/rf47.webp rename to data/external/results/maps/roomfinder/rf47.webp diff --git a/data/external/maps/roomfinder/rf48.webp b/data/external/results/maps/roomfinder/rf48.webp similarity index 100% rename from data/external/maps/roomfinder/rf48.webp rename to data/external/results/maps/roomfinder/rf48.webp diff --git a/data/external/maps/roomfinder/rf49.webp b/data/external/results/maps/roomfinder/rf49.webp similarity index 100% rename from data/external/maps/roomfinder/rf49.webp rename to data/external/results/maps/roomfinder/rf49.webp diff --git a/data/external/maps/roomfinder/rf50.webp b/data/external/results/maps/roomfinder/rf50.webp similarity index 100% rename from data/external/maps/roomfinder/rf50.webp rename to data/external/results/maps/roomfinder/rf50.webp diff --git a/data/external/maps/roomfinder/rf51.webp b/data/external/results/maps/roomfinder/rf51.webp similarity index 100% rename from data/external/maps/roomfinder/rf51.webp rename to data/external/results/maps/roomfinder/rf51.webp diff --git a/data/external/maps/roomfinder/rf52.webp b/data/external/results/maps/roomfinder/rf52.webp similarity index 100% rename from data/external/maps/roomfinder/rf52.webp rename to data/external/results/maps/roomfinder/rf52.webp diff --git a/data/external/maps/roomfinder/rf53.webp b/data/external/results/maps/roomfinder/rf53.webp similarity index 100% rename from data/external/maps/roomfinder/rf53.webp rename to data/external/results/maps/roomfinder/rf53.webp diff --git a/data/external/maps/roomfinder/rf54.webp b/data/external/results/maps/roomfinder/rf54.webp similarity index 100% rename from data/external/maps/roomfinder/rf54.webp rename to data/external/results/maps/roomfinder/rf54.webp diff --git a/data/external/maps/roomfinder/rf57.webp b/data/external/results/maps/roomfinder/rf57.webp similarity index 100% rename from data/external/maps/roomfinder/rf57.webp rename to data/external/results/maps/roomfinder/rf57.webp diff --git a/data/external/maps/roomfinder/rf58.webp b/data/external/results/maps/roomfinder/rf58.webp similarity index 100% rename from data/external/maps/roomfinder/rf58.webp rename to data/external/results/maps/roomfinder/rf58.webp diff --git a/data/external/maps/roomfinder/rf59.webp b/data/external/results/maps/roomfinder/rf59.webp similarity index 100% rename from data/external/maps/roomfinder/rf59.webp rename to data/external/results/maps/roomfinder/rf59.webp diff --git a/data/external/maps/roomfinder/rf60.webp b/data/external/results/maps/roomfinder/rf60.webp similarity index 100% rename from data/external/maps/roomfinder/rf60.webp rename to data/external/results/maps/roomfinder/rf60.webp diff --git a/data/external/maps/roomfinder/rf61.webp b/data/external/results/maps/roomfinder/rf61.webp similarity index 100% rename from data/external/maps/roomfinder/rf61.webp rename to data/external/results/maps/roomfinder/rf61.webp diff --git a/data/external/maps/roomfinder/rf62.webp b/data/external/results/maps/roomfinder/rf62.webp similarity index 100% rename from data/external/maps/roomfinder/rf62.webp rename to data/external/results/maps/roomfinder/rf62.webp diff --git a/data/external/maps/roomfinder/rf63.webp b/data/external/results/maps/roomfinder/rf63.webp similarity index 100% rename from data/external/maps/roomfinder/rf63.webp rename to data/external/results/maps/roomfinder/rf63.webp diff --git a/data/external/maps/roomfinder/rf64.webp b/data/external/results/maps/roomfinder/rf64.webp similarity index 100% rename from data/external/maps/roomfinder/rf64.webp rename to data/external/results/maps/roomfinder/rf64.webp diff --git a/data/external/maps/roomfinder/rf65.webp b/data/external/results/maps/roomfinder/rf65.webp similarity index 100% rename from data/external/maps/roomfinder/rf65.webp rename to data/external/results/maps/roomfinder/rf65.webp diff --git a/data/external/maps/roomfinder/rf66.webp b/data/external/results/maps/roomfinder/rf66.webp similarity index 100% rename from data/external/maps/roomfinder/rf66.webp rename to data/external/results/maps/roomfinder/rf66.webp diff --git a/data/external/maps/roomfinder/rf67.webp b/data/external/results/maps/roomfinder/rf67.webp similarity index 100% rename from data/external/maps/roomfinder/rf67.webp rename to data/external/results/maps/roomfinder/rf67.webp diff --git a/data/external/maps/roomfinder/rf68.webp b/data/external/results/maps/roomfinder/rf68.webp similarity index 100% rename from data/external/maps/roomfinder/rf68.webp rename to data/external/results/maps/roomfinder/rf68.webp diff --git a/data/external/maps/roomfinder/rf69.webp b/data/external/results/maps/roomfinder/rf69.webp similarity index 100% rename from data/external/maps/roomfinder/rf69.webp rename to data/external/results/maps/roomfinder/rf69.webp diff --git a/data/external/maps/roomfinder/rf70.webp b/data/external/results/maps/roomfinder/rf70.webp similarity index 100% rename from data/external/maps/roomfinder/rf70.webp rename to data/external/results/maps/roomfinder/rf70.webp diff --git a/data/external/maps/roomfinder/rf71.webp b/data/external/results/maps/roomfinder/rf71.webp similarity index 100% rename from data/external/maps/roomfinder/rf71.webp rename to data/external/results/maps/roomfinder/rf71.webp diff --git a/data/external/maps/roomfinder/rf72.webp b/data/external/results/maps/roomfinder/rf72.webp similarity index 100% rename from data/external/maps/roomfinder/rf72.webp rename to data/external/results/maps/roomfinder/rf72.webp diff --git a/data/external/maps/roomfinder/rf73.webp b/data/external/results/maps/roomfinder/rf73.webp similarity index 100% rename from data/external/maps/roomfinder/rf73.webp rename to data/external/results/maps/roomfinder/rf73.webp diff --git a/data/external/maps/roomfinder/rf74.webp b/data/external/results/maps/roomfinder/rf74.webp similarity index 100% rename from data/external/maps/roomfinder/rf74.webp rename to data/external/results/maps/roomfinder/rf74.webp diff --git a/data/external/maps/roomfinder/rf75.webp b/data/external/results/maps/roomfinder/rf75.webp similarity index 100% rename from data/external/maps/roomfinder/rf75.webp rename to data/external/results/maps/roomfinder/rf75.webp diff --git a/data/external/maps/roomfinder/rf76.webp b/data/external/results/maps/roomfinder/rf76.webp similarity index 100% rename from data/external/maps/roomfinder/rf76.webp rename to data/external/results/maps/roomfinder/rf76.webp diff --git a/data/external/maps/roomfinder/rf77.webp b/data/external/results/maps/roomfinder/rf77.webp similarity index 100% rename from data/external/maps/roomfinder/rf77.webp rename to data/external/results/maps/roomfinder/rf77.webp diff --git a/data/external/maps/roomfinder/rf78.webp b/data/external/results/maps/roomfinder/rf78.webp similarity index 100% rename from data/external/maps/roomfinder/rf78.webp rename to data/external/results/maps/roomfinder/rf78.webp diff --git a/data/external/maps/roomfinder/rf79.webp b/data/external/results/maps/roomfinder/rf79.webp similarity index 100% rename from data/external/maps/roomfinder/rf79.webp rename to data/external/results/maps/roomfinder/rf79.webp diff --git a/data/external/maps/roomfinder/rf80.webp b/data/external/results/maps/roomfinder/rf80.webp similarity index 100% rename from data/external/maps/roomfinder/rf80.webp rename to data/external/results/maps/roomfinder/rf80.webp diff --git a/data/external/maps/roomfinder/rf81.webp b/data/external/results/maps/roomfinder/rf81.webp similarity index 100% rename from data/external/maps/roomfinder/rf81.webp rename to data/external/results/maps/roomfinder/rf81.webp diff --git a/data/external/maps/roomfinder/rf82.webp b/data/external/results/maps/roomfinder/rf82.webp similarity index 100% rename from data/external/maps/roomfinder/rf82.webp rename to data/external/results/maps/roomfinder/rf82.webp diff --git a/data/external/maps/roomfinder/rf83.webp b/data/external/results/maps/roomfinder/rf83.webp similarity index 100% rename from data/external/maps/roomfinder/rf83.webp rename to data/external/results/maps/roomfinder/rf83.webp diff --git a/data/external/maps/roomfinder/rf84.webp b/data/external/results/maps/roomfinder/rf84.webp similarity index 100% rename from data/external/maps/roomfinder/rf84.webp rename to data/external/results/maps/roomfinder/rf84.webp diff --git a/data/external/maps/roomfinder/rf85.webp b/data/external/results/maps/roomfinder/rf85.webp similarity index 100% rename from data/external/maps/roomfinder/rf85.webp rename to data/external/results/maps/roomfinder/rf85.webp diff --git a/data/external/maps/roomfinder/rf86.webp b/data/external/results/maps/roomfinder/rf86.webp similarity index 100% rename from data/external/maps/roomfinder/rf86.webp rename to data/external/results/maps/roomfinder/rf86.webp diff --git a/data/external/maps/roomfinder/rf87.webp b/data/external/results/maps/roomfinder/rf87.webp similarity index 100% rename from data/external/maps/roomfinder/rf87.webp rename to data/external/results/maps/roomfinder/rf87.webp diff --git a/data/external/maps/roomfinder/rf88.webp b/data/external/results/maps/roomfinder/rf88.webp similarity index 100% rename from data/external/maps/roomfinder/rf88.webp rename to data/external/results/maps/roomfinder/rf88.webp diff --git a/data/external/maps/roomfinder/rf89.webp b/data/external/results/maps/roomfinder/rf89.webp similarity index 100% rename from data/external/maps/roomfinder/rf89.webp rename to data/external/results/maps/roomfinder/rf89.webp diff --git a/data/external/maps/roomfinder/rf9.webp b/data/external/results/maps/roomfinder/rf9.webp similarity index 100% rename from data/external/maps/roomfinder/rf9.webp rename to data/external/results/maps/roomfinder/rf9.webp diff --git a/data/external/maps/roomfinder/rf90.webp b/data/external/results/maps/roomfinder/rf90.webp similarity index 100% rename from data/external/maps/roomfinder/rf90.webp rename to data/external/results/maps/roomfinder/rf90.webp diff --git a/data/external/maps/roomfinder/rf91.webp b/data/external/results/maps/roomfinder/rf91.webp similarity index 100% rename from data/external/maps/roomfinder/rf91.webp rename to data/external/results/maps/roomfinder/rf91.webp diff --git a/data/external/maps/roomfinder/rf92.webp b/data/external/results/maps/roomfinder/rf92.webp similarity index 100% rename from data/external/maps/roomfinder/rf92.webp rename to data/external/results/maps/roomfinder/rf92.webp diff --git a/data/external/maps/roomfinder/rf93.webp b/data/external/results/maps/roomfinder/rf93.webp similarity index 100% rename from data/external/maps/roomfinder/rf93.webp rename to data/external/results/maps/roomfinder/rf93.webp diff --git a/data/external/maps/roomfinder/rf94.webp b/data/external/results/maps/roomfinder/rf94.webp similarity index 100% rename from data/external/maps/roomfinder/rf94.webp rename to data/external/results/maps/roomfinder/rf94.webp diff --git a/data/external/maps/roomfinder/rf95.webp b/data/external/results/maps/roomfinder/rf95.webp similarity index 100% rename from data/external/maps/roomfinder/rf95.webp rename to data/external/results/maps/roomfinder/rf95.webp diff --git a/data/external/maps/roomfinder/rf96.webp b/data/external/results/maps/roomfinder/rf96.webp similarity index 100% rename from data/external/maps/roomfinder/rf96.webp rename to data/external/results/maps/roomfinder/rf96.webp diff --git a/data/external/maps/roomfinder/rf97.webp b/data/external/results/maps/roomfinder/rf97.webp similarity index 100% rename from data/external/maps/roomfinder/rf97.webp rename to data/external/results/maps/roomfinder/rf97.webp diff --git a/data/external/maps/roomfinder/rf98.webp b/data/external/results/maps/roomfinder/rf98.webp similarity index 100% rename from data/external/maps/roomfinder/rf98.webp rename to data/external/results/maps/roomfinder/rf98.webp diff --git a/data/external/maps/roomfinder/rf99.webp b/data/external/results/maps/roomfinder/rf99.webp similarity index 100% rename from data/external/maps/roomfinder/rf99.webp rename to data/external/results/maps/roomfinder/rf99.webp diff --git a/data/external/maps_roomfinder.json b/data/external/results/maps_roomfinder.json similarity index 100% rename from data/external/maps_roomfinder.json rename to data/external/results/maps_roomfinder.json diff --git a/data/external/rooms_roomfinder.json b/data/external/results/rooms_roomfinder.json similarity index 100% rename from data/external/rooms_roomfinder.json rename to data/external/results/rooms_roomfinder.json diff --git a/data/external/rooms_tumonline.json b/data/external/results/rooms_tumonline.json similarity index 100% rename from data/external/rooms_tumonline.json rename to data/external/results/rooms_tumonline.json diff --git a/data/external/usages_tumonline.json b/data/external/results/usages_tumonline.json similarity index 100% rename from data/external/usages_tumonline.json rename to data/external/results/usages_tumonline.json diff --git a/data/external/scrapers/__init__.py b/data/external/scrapers/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/data/external/scrapers/roomfinder.py b/data/external/scrapers/roomfinder.py new file mode 100644 index 000000000..609657c90 --- /dev/null +++ b/data/external/scrapers/roomfinder.py @@ -0,0 +1,233 @@ +# This script takes care of downloading data from the Roomfinder and TUMonline +# and caching the results +import string +import urllib +import xmlrpc.client +import zipfile + +from defusedxml import ElementTree as ET +from progress.bar import Bar # type: ignore + +from external.scraping_utils import maybe_sleep, _write_cache_json, _cached_json, CACHE_PATH +from utils import convert_to_webp + +ROOMFINDER_API_URL = "http://roomfinder.ze.tum.de:8192" + + +def scrape_buildings(): + """ + Retrieve the (extended, i.e. with coordinates) buildings data from the Roomfinder API + + :returns: A list of buildings, each building is a dict + """ + cache_name = "buildings_roomfinder.json" + + buildings = _cached_json(cache_name) + if buildings is not None: + return buildings + + with xmlrpc.client.ServerProxy(ROOMFINDER_API_URL) as proxy: + buildings = proxy.getBuildings() + bar = Bar("Retrieving", suffix="%(index)d / %(max)d buildings", max=len(buildings)) + for i, building in enumerate(buildings): + bar.next() + # Make sure b_id is numeric. There is an incorrect entry with the value + # 'CiO/SGInstitute West, Bibliot' which causes a crash + try: + int(building["b_id"]) + except ValueError: + continue + extended_data = proxy.getBuildingData(building["b_id"]) + for key, value in extended_data.items(): + buildings[i][key] = value + buildings[i]["maps"] = proxy.getBuildingMaps(building["b_id"]) + buildings[i]["default_map"] = proxy.getBuildingDefaultMap(building["b_id"]) + maybe_sleep(0.05) + + _write_cache_json(cache_name, buildings) + return buildings + + +def scrape_rooms(): + """ + Retrieve the (extended, i.e. with coordinates) rooms data from the Roomfinder API. + This may retrieve the Roomfinder buildings. + + :returns: A list of rooms, each room is a dict + """ + cache_name = "rooms_roomfinder.json" + + rooms = _cached_json(cache_name) + if rooms is not None: + return rooms + + buildings = scrape_buildings() + rooms_list = [] + + # Get all rooms in a building + # The API does not provide such a function directly, so we + # have to use search for this. Since search returns a max + # of 50 results we need to guess to collect all rooms. + print("Searching for rooms in buildings") + b_cnt = 0 + with xmlrpc.client.ServerProxy(ROOMFINDER_API_URL) as proxy: + for building in buildings: + if "b_roomCount" in building and building["b_roomCount"] > 0: + search_results = proxy.searchRoom("", {"r_building": building["b_id"]}) + b_rooms = {room["r_id"] for room in search_results} + + if len(b_rooms) < building["b_roomCount"]: + # Collect guess queries that are executed until + # all buildings are found or the query list is exhausted + for guessed_query in _guess_queries(b_rooms, building["b_roomCount"]): + search_results = proxy.searchRoom(guessed_query, {"r_building": building["b_id"]}) + b_rooms |= {r["r_id"] for r in search_results} + + if len(b_rooms) < building["b_roomCount"]: + print("Could not guess all queries for:") + + b_cnt += 1 + print(f"{building['b_id']} -> {len(b_rooms)} / {building['b_roomCount']}") + + rooms_list.extend(list(b_rooms)) + + rooms = [] + for room in Bar("Retrieving", suffix=f"%(index)d / %(max)d rooms for {b_cnt} buildings").iter(rooms_list): + extended_data = proxy.getRoomData(room) + # for k, v in extended_data.items(): + # rooms[i][k] = v + extended_data["maps"] = proxy.getRoomMaps(room) + extended_data["default_map"] = proxy.getDefaultMap(room) + extended_data["metas"] = proxy.getRoomMetas(room) + rooms.append(extended_data) + maybe_sleep(0.05) + + _write_cache_json(cache_name, rooms) + return rooms + + +def _guess_queries(rooms, n_rooms): + # First try: all single-digit numbers + for i in range(10): + if len(rooms) < n_rooms: + maybe_sleep(0.05) + yield str(i) + else: + return + + # Second try: all double-digit numbers + for i in range(100): + if len(rooms) < n_rooms: + maybe_sleep(0.05) + yield str(i).zfill(2) + else: + return + + # Thirs try: all characters + for char in string.ascii_lowercase: + if len(rooms) < n_rooms: + maybe_sleep(0.05) + yield char + else: + return + + +def scrape_maps(): + """ + Retrieve the maps including the data about them from Roomfinder. + Map files will be stored in 'cache/maps/roomfinder'. + + This may retrieve Roomfinder rooms and buildings. + + :returns: A list of maps + """ + cache_name = "maps_roomfinder.json" + + cached_maps = _cached_json(cache_name) + if cached_maps is not None: + return cached_maps + + # The only way to get the map boundaries seems to be to download the kml with overlaid map. + # For this api we need a room or building for each map available. + rooms = scrape_rooms() + buildings = scrape_buildings() + + used_maps = {} + for building_entity in rooms + buildings: + for _map in building_entity.get("maps", []): + # _map[1] is the map id + if _map[1] not in used_maps: + if "r_id" in building_entity: + used_maps[_map[1]] = ("room", building_entity["r_id"], _map) + else: + used_maps[_map[1]] = ("building", building_entity["b_id"], _map) + maps = _download_maps(used_maps) + + # Not all maps are used somewhere. + # TODO: Download the rest + + _write_cache_json(cache_name, maps) + return maps + + +def _download_maps(used_maps): + maps = [] + for e_type, e_id, _map in used_maps.values(): + # Download as file + url = f"{ROOMFINDER_API_URL}/getMapImage?m_id={_map[1]}" + filepath = CACHE_PATH / "maps" / "roomfinder" / f"rf{_map[1]}.gif" + _download_file(url, filepath) + convert_to_webp(filepath) + + map_data = { + "id": _map[1], + "scale": _map[0], + "desc": _map[2], + "width": _map[3], + "height": _map[4], + } + maps.append(map_data) + + # Download as kmz to get the map boundary coordinates. + # The world map (id 9) does not support kmz download + if _map[1] == 9: + continue + + f_path = _download_map(_map, e_id, e_type) + + with zipfile.ZipFile(f_path, "r") as zip_f, zip_f.open("RoomFinder.kml") as file: + root = ET.fromstring(file.read()) + # [0] gives , + # [3] gives , + # [3] gives + latlonbox = root[0][3][3] + map_data["latlonbox"] = { + "north": latlonbox[0].text, + "east": latlonbox[1].text, + "west": latlonbox[2].text, + "south": latlonbox[3].text, + "rotation": latlonbox[4].text, + } + return maps + + +def _download_map(_map, e_id, e_type): + filepath = CACHE_PATH / "maps" / "roomfinder" / "kmz" / f"{_map[1]}.kmz" + if e_type == "room": + base_url = "https://portal.mytum.de/campus/roomfinder/getRoomPlacemark" + url = f"{base_url}?roomid={urllib.parse.quote_plus(e_id)}&mapid={_map[1]}" + return _download_file(url, filepath) + if e_type == "building": + base_url = "https://portal.mytum.de/campus/roomfinder/getBuildingPlacemark" + url = f"{base_url}?b_id={e_id}&mapid={_map[1]}" + return _download_file(url, filepath) + raise RuntimeError(f"Unknown entity type: {e_type}") + + +def _download_file(url, target_cache_file): + if not target_cache_file.exists(): + print(f"Retrieving: '{url}'") + # url parameter does not allow path traversal, because we build it further up in the callstack + urllib.request.urlretrieve(url, target_cache_file) # nosec: B310 + + return target_cache_file diff --git a/data/external/downloader.py b/data/external/scrapers/tumonline.py similarity index 57% rename from data/external/downloader.py rename to data/external/scrapers/tumonline.py index 00dd026ab..04872ed87 100644 --- a/data/external/downloader.py +++ b/data/external/scrapers/tumonline.py @@ -2,243 +2,17 @@ # and caching the results import json import random -import string -import time -import urllib -import xmlrpc.client -import zipfile -from pathlib import Path import requests from bs4 import BeautifulSoup, element from defusedxml import ElementTree as ET +from external.scraping_utils import _cached_json, _write_cache_json, CACHE_PATH, maybe_sleep from progress.bar import Bar # type: ignore -from utils import convert_to_webp TUMONLINE_URL = "https://campus.tum.de/tumonline" -ROOMFINDER_API_URL = "http://roomfinder.ze.tum.de:8192" -CACHE_PATH = Path(__file__).parent / "cache" -def maybe_sleep(duration): - """ - Sleep for the given duration, but only if the script was called during a workday and working hours. - """ - if time.gmtime().tm_wday not in [5, 6] and 5 <= time.gmtime().tm_hour <= 22: - time.sleep(duration) - - -def roomfinder_buildings(): - """ - Retrieve the (extended, i.e. with coordinates) buildings data from the Roomfinder API - - :returns: A list of buildings, each building is a dict - """ - cache_name = "buildings_roomfinder.json" - - buildings = _cached_json(cache_name) - if buildings is not None: - return buildings - - with xmlrpc.client.ServerProxy(ROOMFINDER_API_URL) as proxy: - buildings = proxy.getBuildings() - bar = Bar("Retrieving", suffix="%(index)d / %(max)d buildings", max=len(buildings)) - for i, building in enumerate(buildings): - bar.next() - # Make sure b_id is numeric. There is an incorrect entry with the value - # 'CiO/SGInstitute West, Bibliot' which causes a crash - try: - int(building["b_id"]) - except ValueError: - continue - extended_data = proxy.getBuildingData(building["b_id"]) - for key, value in extended_data.items(): - buildings[i][key] = value - buildings[i]["maps"] = proxy.getBuildingMaps(building["b_id"]) - buildings[i]["default_map"] = proxy.getBuildingDefaultMap(building["b_id"]) - maybe_sleep(0.05) - - _write_cache_json(cache_name, buildings) - return buildings - - -def roomfinder_rooms(): - """ - Retrieve the (extended, i.e. with coordinates) rooms data from the Roomfinder API. - This may retrieve the Roomfinder buildings. - - :returns: A list of rooms, each room is a dict - """ - cache_name = "rooms_roomfinder.json" - - rooms = _cached_json(cache_name) - if rooms is not None: - return rooms - - buildings = roomfinder_buildings() - rooms_list = [] - - # Get all rooms in a building - # The API does not provide such a function directly, so we - # have to use search for this. Since search returns a max - # of 50 results we need to guess to collect all rooms. - print("Searching for rooms in buildings") - b_cnt = 0 - with xmlrpc.client.ServerProxy(ROOMFINDER_API_URL) as proxy: - for building in buildings: - if "b_roomCount" in building and building["b_roomCount"] > 0: - search_results = proxy.searchRoom("", {"r_building": building["b_id"]}) - b_rooms = {room["r_id"] for room in search_results} - - if len(b_rooms) < building["b_roomCount"]: - # Collect guess queries that are executed until - # all buildings are found or the query list is exhausted - for guessed_query in _guess_queries(b_rooms, building["b_roomCount"]): - search_results = proxy.searchRoom(guessed_query, {"r_building": building["b_id"]}) - b_rooms |= {r["r_id"] for r in search_results} - - if len(b_rooms) < building["b_roomCount"]: - print("Could not guess all queries for:") - - b_cnt += 1 - print(f"{building['b_id']} -> {len(b_rooms)} / {building['b_roomCount']}") - - rooms_list.extend(list(b_rooms)) - - rooms = [] - for room in Bar("Retrieving", suffix=f"%(index)d / %(max)d rooms for {b_cnt} buildings").iter(rooms_list): - extended_data = proxy.getRoomData(room) - # for k, v in extended_data.items(): - # rooms[i][k] = v - extended_data["maps"] = proxy.getRoomMaps(room) - extended_data["default_map"] = proxy.getDefaultMap(room) - extended_data["metas"] = proxy.getRoomMetas(room) - rooms.append(extended_data) - maybe_sleep(0.05) - - _write_cache_json(cache_name, rooms) - return rooms - - -def _guess_queries(rooms, n_rooms): - # First try: all single-digit numbers - for i in range(10): - if len(rooms) < n_rooms: - maybe_sleep(0.05) - yield str(i) - else: - return - - # Second try: all double-digit numbers - for i in range(100): - if len(rooms) < n_rooms: - maybe_sleep(0.05) - yield str(i).zfill(2) - else: - return - - # Thirs try: all characters - for char in string.ascii_lowercase: - if len(rooms) < n_rooms: - maybe_sleep(0.05) - yield char - else: - return - - -def roomfinder_maps(): - """ - Retrieve the maps including the data about them from Roomfinder. - Map files will be stored in 'cache/maps/roomfinder'. - - This may retrieve Roomfinder rooms and buildings. - - :returns: A list of maps - """ - cache_name = "maps_roomfinder.json" - - cached_maps = _cached_json(cache_name) - if cached_maps is not None: - return cached_maps - - # The only way to get the map boundaries seems to be to download the kml with overlaid map. - # For this api we need a room or building for each map available. - rooms = roomfinder_rooms() - buildings = roomfinder_buildings() - - used_maps = {} - for building_entity in rooms + buildings: - for _map in building_entity.get("maps", []): - # _map[1] is the map id - if _map[1] not in used_maps: - if "r_id" in building_entity: - used_maps[_map[1]] = ("room", building_entity["r_id"], _map) - else: - used_maps[_map[1]] = ("building", building_entity["b_id"], _map) - maps = _download_maps(used_maps) - - # Not all maps are used somewhere. - # TODO: Download the rest - - _write_cache_json(cache_name, maps) - return maps - - -def _download_maps(used_maps): - maps = [] - for e_type, e_id, _map in used_maps.values(): - # Download as file - url = f"{ROOMFINDER_API_URL}/getMapImage?m_id={_map[1]}" - filepath = CACHE_PATH / "maps" / "roomfinder" / f"rf{_map[1]}.gif" - _download_file(url, filepath) - convert_to_webp(filepath) - - map_data = { - "id": _map[1], - "scale": _map[0], - "desc": _map[2], - "width": _map[3], - "height": _map[4], - } - maps.append(map_data) - - # Download as kmz to get the map boundary coordinates. - # The world map (id 9) does not support kmz download - if _map[1] == 9: - continue - - f_path = _download_map(_map, e_id, e_type) - - with zipfile.ZipFile(f_path, "r") as zip_f, zip_f.open("RoomFinder.kml") as file: - root = ET.fromstring(file.read()) - # [0] gives , - # [3] gives , - # [3] gives - latlonbox = root[0][3][3] - map_data["latlonbox"] = { - "north": latlonbox[0].text, - "east": latlonbox[1].text, - "west": latlonbox[2].text, - "south": latlonbox[3].text, - "rotation": latlonbox[4].text, - } - return maps - - -def _download_map(_map, e_id, e_type): - filepath = CACHE_PATH / "maps" / "roomfinder" / "kmz" / f"{_map[1]}.kmz" - if e_type == "room": - base_url = "https://portal.mytum.de/campus/roomfinder/getRoomPlacemark" - url = f"{base_url}?roomid={urllib.parse.quote_plus(e_id)}&mapid={_map[1]}" - return _download_file(url, filepath) - if e_type == "building": - base_url = "https://portal.mytum.de/campus/roomfinder/getBuildingPlacemark" - url = f"{base_url}?b_id={e_id}&mapid={_map[1]}" - return _download_file(url, filepath) - raise RuntimeError(f"Unknown entity type: {e_type}") - - -def tumonline_areas(): +def scrape_areas(): """ Retrieve the building areas as in TUMonline. @@ -253,7 +27,7 @@ def tumonline_areas(): return [{"id": int(e[0]), "name": e[1]} for e in _parse_filter_options(filters, "areas")] -def tumonline_usages_filter(): +def scrape_usages_filter(): """ Retrieve the room usage types that are available as a filter in TUMonline. These are not all usage types known to TUMonline! @@ -269,7 +43,7 @@ def tumonline_usages_filter(): return [{"id": int(e[0]), "name": e[1]} for e in _parse_filter_options(filters, "usages")] -def tumonline_buildings(): +def scrape_buildings(): """ Retrieve the buildings as in TUMonline with their assigned TUMonline area. This may retrieve TUMonline areas. @@ -289,7 +63,7 @@ def tumonline_buildings(): ) all_buildings = _parse_filter_options(filters, "buildings") - areas = tumonline_areas() + areas = scrape_areas() buildings = [] for area in areas: filters_area = _get_roomsearch_xml( @@ -309,7 +83,7 @@ def tumonline_buildings(): return buildings -def tumonline_rooms(): +def scrape_rooms(): """ Retrieve the rooms as in TUMonline including building and usage type. For some room types (e.g. lecture halls) additional information is retrieved. @@ -340,9 +114,9 @@ def tumonline_rooms(): room_index = {} - buildings = tumonline_buildings() + buildings = scrape_buildings() for building in buildings: - b_rooms = _retrieve_tumonline_roomlist("b", "building", "pGebaeude", building["filter_id"], building["area_id"]) + b_rooms = _retrieve_roomlist("b", "building", "pGebaeude", building["filter_id"], building["area_id"]) for room in b_rooms: room["b_filter_id"] = building["filter_id"] room["b_area_id"] = building["area_id"] @@ -353,12 +127,12 @@ def tumonline_rooms(): rooms = [] usage_id = 1 # Observed: usage ids go up to 223, the limit below is for safety while not (usage_id > 300 or len(rooms) >= len(room_index)): - u_rooms = _retrieve_tumonline_roomlist("u", "usage", "pVerwendung", usage_id) + u_rooms = _retrieve_roomlist("u", "usage", "pVerwendung", usage_id) for room in u_rooms: room_index[room["roomcode"]]["usage"] = usage_id if usage_id in extend_for_usages: system_id = room_index[room["roomcode"]]["room_link"][24:] - room_index[room["roomcode"]]["extended"] = _retrieve_tumonline_roominfo(system_id) + room_index[room["roomcode"]]["extended"] = _retrieve_roominfo(system_id) rooms.append(room_index[room["roomcode"]]) usage_id += 1 @@ -366,7 +140,7 @@ def tumonline_rooms(): return rooms -def tumonline_usages(): +def scrape_usages(): """ Retrieve all usage types available in TUMonline. This may retrieve TUMonline rooms. @@ -379,7 +153,7 @@ def tumonline_usages(): if usages is not None: return usages - rooms = tumonline_rooms() + rooms = scrape_rooms() used_usage_types = {} for room in rooms: @@ -391,7 +165,7 @@ def tumonline_usages(): for usage_type, example_room in used_usage_types.items(): # room links start with "wbRaum.editRaum?pRaumNr=..." system_id = example_room["room_link"][24:] - roominfo = _retrieve_tumonline_roominfo(system_id) + roominfo = _retrieve_roominfo(system_id) usage = roominfo["Basisdaten"]["Verwendung"] parts = [] @@ -412,7 +186,7 @@ def tumonline_usages(): return usages -def tumonline_orgs(): +def scrape_orgs(): """ Retrieve all organisations in TUMonline, that may operate rooms. @@ -462,7 +236,7 @@ def tumonline_orgs(): return orgs -def _retrieve_tumonline_roomlist(f_prefix, f_type, f_name, f_value, area_id=0): +def _retrieve_roomlist(f_prefix, f_type, f_name, f_value, area_id=0): """Retrieve all rooms (multi-page) from the TUMonline room search list""" cache_name = f"tumonline/{f_prefix}_{f_value}.{area_id}.json" @@ -499,7 +273,7 @@ def _retrieve_tumonline_roomlist(f_prefix, f_type, f_name, f_value, area_id=0): return all_rooms -def _retrieve_tumonline_roominfo(system_id): +def _retrieve_roominfo(system_id): """Retrieve the extended room information from TUMonline for one room""" html_parser: BeautifulSoup = _get_html( f"{TUMONLINE_URL}/wbRaum.editRaum?pRaumNr={system_id}", @@ -596,15 +370,6 @@ def _parse_rooms_list(lxml_parser: BeautifulSoup): return rooms, num_pages, current_page -def _cached_json(fname): - path = CACHE_PATH / fname - if path.exists(): - with open(path, encoding="utf-8") as file: - return json.load(file) - else: - return None - - def _get_roomsearch_xml(url: str, params: dict, cache_fname: str) -> BeautifulSoup: root = _get_xml(url, params, cache_fname) elem = root.find('.//instruction[@jsid="raumSucheKontainerID"]') @@ -638,21 +403,8 @@ def _get_html(url: str, params: dict, cache_fname: str) -> BeautifulSoup: return BeautifulSoup(result, "lxml") -def _download_file(url, target_cache_file): - if not target_cache_file.exists(): - print(f"Retrieving: '{url}'") - # url parameter does not allow path traversal, because we build it further up in the callstack - urllib.request.urlretrieve(url, target_cache_file) # nosec: B310 - - return target_cache_file - - -def _write_cache_json(fname, data): - with open(CACHE_PATH / fname, "w", encoding="utf-8") as file: - json.dump(data, file) - - def _get_tumonline_api_url(base_target): - # I have no idea, what this magic_string is, or why it exists.. Usage is the same as from TUMonline.. + # I have no idea, what this magic_string is, or why it exists.. + # Usage is the same as from TUMonline.. magic_string = f"NC_{str(random.randint(0, 9999)).zfill(4)}" # nosec: random is not used security/crypto purposes return f"{TUMONLINE_URL}/{base_target}/{magic_string}" diff --git a/data/external/scraping_utils.py b/data/external/scraping_utils.py new file mode 100644 index 000000000..ef0f813b5 --- /dev/null +++ b/data/external/scraping_utils.py @@ -0,0 +1,26 @@ +import json +import time +from pathlib import Path + +CACHE_PATH = Path(__file__).parent / "cache" + + +def maybe_sleep(duration): + """ + Sleep for the given duration, but only if the script was called during a workday and working hours. + """ + if time.gmtime().tm_wday not in [5, 6] and 5 <= time.gmtime().tm_hour <= 22: + time.sleep(duration) + + +def _write_cache_json(fname, data): + with open(CACHE_PATH / fname, "w", encoding="utf-8") as file: + json.dump(data, file) + + +def _cached_json(fname): + path = CACHE_PATH / fname + if path.exists(): + with open(path, encoding="utf-8") as file: + return json.load(file) + return None diff --git a/data/processors/maps.py b/data/processors/maps.py index eb9ac1b11..3706dbb17 100644 --- a/data/processors/maps.py +++ b/data/processors/maps.py @@ -8,8 +8,8 @@ import yaml from PIL import Image -EXTERNAL_PATH = Path(__file__).parent.parent / "external" -RF_MAPS_PATH = EXTERNAL_PATH / "maps" / "roomfinder" +EXTERNAL_RESULTS_PATH = Path(__file__).parent.parent / "external" / "results" +RF_MAPS_PATH = EXTERNAL_RESULTS_PATH / "maps" / "roomfinder" def _assign_roomfinder_maps(data): @@ -175,7 +175,7 @@ def _deduplicate_maps(maps_list): def _load_maps_list(): """Read the Roomfinder maps. The world-map is not used""" - with open("external/maps_roomfinder.json", encoding="utf-8") as file: + with open("external/results/maps_roomfinder.json", encoding="utf-8") as file: maps_list: list[dict[str, Any]] = json.load(file) world_map = None for _map in maps_list: @@ -334,7 +334,7 @@ def _assign_default_roomfinder_map(data): def _generate_assignment_data(): # Read the Roomfinder and custom maps - with open("external/maps_roomfinder.json", encoding="utf-8") as file: + with open("external/results/maps_roomfinder.json", encoding="utf-8") as file: maps_list = json.load(file) custom_maps = _load_custom_maps() # For each map, we calculate the boundaries in UTM beforehand diff --git a/data/processors/roomfinder.py b/data/processors/roomfinder.py index cc1e680bd..d86d4094b 100644 --- a/data/processors/roomfinder.py +++ b/data/processors/roomfinder.py @@ -11,7 +11,7 @@ def merge_roomfinder_buildings(data): Merge the buildings in Roomfinder with the existing data. This will not overwrite the existing data, but act directly on the provided data. """ - with open("external/buildings_roomfinder.json", encoding="utf-8") as file: + with open("external/results/buildings_roomfinder.json", encoding="utf-8") as file: buildings = json.load(file) with open("sources/10_patches-roomfinder-buildings.yaml", encoding="utf-8") as file: @@ -80,7 +80,7 @@ def merge_roomfinder_rooms(data): This will not overwrite the existing data, but act directly on the provided data. """ - with open("external/rooms_roomfinder.json", encoding="utf-8") as file: + with open("external/results/rooms_roomfinder.json", encoding="utf-8") as file: rooms = json.load(file) with open("sources/16_roomfinder-merge-patches.yaml", encoding="utf-8") as file: diff --git a/data/processors/tumonline.py b/data/processors/tumonline.py index deb859e22..d8f26584b 100644 --- a/data/processors/tumonline.py +++ b/data/processors/tumonline.py @@ -15,7 +15,7 @@ def merge_tumonline_buildings(data): Merge the buildings in TUMonline with the existing data. This will not overwrite the existing data, but act directly on the provided data. """ - with open("external/buildings_tumonline.json", encoding="utf-8") as file: + with open("external/results/buildings_tumonline.json", encoding="utf-8") as file: buildings = json.load(file) error = False @@ -76,10 +76,10 @@ def merge_tumonline_rooms(data): Merge the rooms in TUMonline with the existing data. This will not overwrite the existing data, but act directly on the provided data. """ - with open("external/rooms_tumonline.json", encoding="utf-8") as file: + with open("external/results/rooms_tumonline.json", encoding="utf-8") as file: rooms = json.load(file) - with open("external/usages_tumonline.json", encoding="utf-8") as file: + with open("external/results/usages_tumonline.json", encoding="utf-8") as file: usages = json.load(file) usages_lookup = {usage["id"]: usage for usage in usages} diff --git a/resources/scripts/regen.sh b/resources/scripts/regen.sh index 6ab9896d9..9e4918be0 100755 --- a/resources/scripts/regen.sh +++ b/resources/scripts/regen.sh @@ -25,7 +25,7 @@ cd ./webclient || exit rm -fr cdn mkdir cdn rsync -r --exclude '*.yaml' ../data/sources/img/ cdn/ -cp -r ../data/external/maps/roomfinder/* cdn/maps/roomfinder +cp -r ../data/external/results/maps/roomfinder/* cdn/maps/roomfinder ) diff --git a/webclient/README.md b/webclient/README.md index 7e94b3511..1bb0b0236 100644 --- a/webclient/README.md +++ b/webclient/README.md @@ -21,7 +21,7 @@ into it: mkdir cdn rsync -r --exclude '*.yaml' ../data/sources/img/ cdn/ mkdir -p cdn/maps/roomfinder -cp -r ../data/external/maps/roomfinder/* cdn/maps/roomfinder/ +cp -r ../data/external/results/maps/roomfinder/* cdn/maps/roomfinder/ ``` ### Building