Backend threading to avoid commit conflicts (+ website updates)

backend/app.py

@@ -1,15 +1,24 @@
 import json
 import os
+import time
 from datetime import datetime
+from threading import Thread
+from typing import Any, Dict
 
 from flask import Flask, jsonify, request
 from googlemaps import Client as GoogleMaps
 from haversine import haversine
 from thefuzz import process as fuzzysearch
+from werkzeug.datastructures import FileStorage
 
-from pennyme.github_update import push_newmachine_to_github
+from pennyme.github_update import isbusy, push_newmachine_to_github
 from pennyme.locations import COUNTRIES
-from pennyme.slack import image_slack, message_slack, process_uploaded_image
+from pennyme.slack import (
+    image_slack,
+    message_slack,
+    message_slack_raw,
+    process_uploaded_image,
+)
 
 app = Flask(__name__)
 
@@ -111,6 +120,71 @@ def save_comment(comment: str, ip: str, machine_id: int):
         json.dump(IP_COMMENT_DICT, f, indent=4)
 
 
+def process_machine_entry(
+    new_machine_entry: Dict[str, Any],
+    image: FileStorage,
+    ip_address: str,
+    title: str,
+    address: str,
+):
+    """
+    Process a new machine entry (upload image, send message to slack, etc.)
+    Typically executed from a thread to avoid clash with cron job
+
+    Args:
+        new_machine_entry: The new machine entry to process.
+        image: The image to save, obtained via Flask's request.files["image"].
+        ip_address: The IP address of the user.
+        title: The title of the machine.
+        address: The address of the machine.
+    """
+    try:
+        # Optional waiting if cron job is running
+        if isbusy():
+            message_slack_raw(
+                ip=ip_address,
+                text="Found conflicting cron job, waiting for it to finish...",
+            )
+            counter = 0
+            while isbusy() and counter < 60:
+                time.sleep(300)  # Retry every 5min
+                counter += 1
+            if counter == 60:
+                message_slack_raw(
+                    ip=ip_address,
+                    text="Timeout of 5h reached, cron job still runs, aborting...",
+                )
+                return
+
+        # Cron job has finished, we can add machine
+        new_machine_id = push_newmachine_to_github(new_machine_entry)
+
+        # Upload the image
+        if image:
+            img_path = os.path.join(PATH_IMAGES, f"{new_machine_id}.jpg")
+            process_uploaded_image(image, img_path)
+
+            # Send message to slack
+            image_slack(
+                new_machine_id,
+                ip=ip_address,
+                m_name=title,
+                img_slack_text="New machine proposed:",
+            )
+        else:
+            message_slack(
+                new_machine_id,
+                ip=ip_address,
+                m_name=title,
+                img_slack_text="Picture missing for machine!",
+            )
+    except Exception as e:
+        message_slack_raw(
+            ip=ip_address,
+            text=f"Error when processing machine entry: {title}, {address} ({e})",
+        )
+
+
 @app.route("/create_machine", methods=["POST"])
 def create_machine():
     """Receives a comment and adds it to the json file."""
@@ -220,26 +294,18 @@ def create_machine():
         "geometry": {"type": "Point", "coordinates": location},
         "properties": properties_dict,
     }
-    # If pushing to new branch: set unique branch name
-    # branch_name = f"new_machine_{round(time.time())}"
-    new_machine_id = push_newmachine_to_github(new_machine_entry)
-
-    # Upload the image
-    if "image" not in request.files:
-        return "No image file", 400
-    image = request.files["image"]
     ip_address = request.remote_addr
-    # crop and save the image
-    img_path = os.path.join(PATH_IMAGES, f"{new_machine_id}.jpg")
-    process_uploaded_image(image, img_path)
+    image = request.files.get("image", None)
+    message_slack_raw(
+        ip=ip_address, text=f"New machine proposed: {title}, {address} ({area})"
+    )
 
-    # send message to slack
-    image_slack(
-        new_machine_id,
-        ip=ip_address,
-        m_name=title,
-        img_slack_text="New machine proposed:",
+    # Create and start the thread
+    thread = Thread(
+        target=process_machine_entry,
+        args=(new_machine_entry, image, ip_address, title, address),
     )
+    thread.start()
 
     return jsonify({"message": "Success!"}), 200
 

backend/pennyme/github_update.py

@@ -1,6 +1,7 @@
 import base64
 import json
 import logging
+import os
 from typing import Any, Dict, List, Optional, Tuple
 
 import requests
@@ -416,3 +417,16 @@ def get_pr_id(
             if pr["head"]["ref"] == branch_name:
                 return pr["number"]
     return None
+
+
+def isbusy() -> bool:
+    """
+    Check whether the cronjob is running.
+
+    Returns:
+        True if the cronjob is running, False otherwise.
+    """
+    # pennyme package directory
+    current_script_dir = os.path.dirname(os.path.abspath(__file__))
+    running_tmp_path = os.path.join(current_script_dir, "../../new_data/running.tmp")
+    return os.path.exists(running_tmp_path)

backend/pennyme/slack.py

@@ -148,6 +148,18 @@ def message_slack(machine_id: str, comment_text: str, ip: str):
     prefix = m_name.split("Status=")[0]
     postfix = "Status=" + m_name.split("Status=")[-1]
     text = f"New comment for machine {machine_id} - {prefix}: {comment_text} (from {ip}. Machine: {postfix}"
+
+    message_slack_raw(text, ip)
+
+
+def message_slack_raw(text: str, ip: str):
+    """
+    Send a message to Slack, unspecific to a machine.
+
+    Args:
+        text: The message to send.
+        ip: The IP address of the user.
+    """
     try:
         CLIENT.chat_postMessage(
             channel="#pennyme_uploads", text=text, username="PennyMe"

backend/scripts/location_differ.py

@@ -74,6 +74,10 @@ def location_differ(
     logger.info(f"======Location differ joblog from {start_time}=======")
     os.makedirs(output_folder, exist_ok=True)
 
+    # Create empty file to indicate that the job is running
+    with open(os.path.join(output_folder, "running.tmp", "w")) as file:
+        pass
+
     today = f"{YEAR}-{MONTH}-{DAY}"
 
     gmaps = GoogleMaps(api_key)
@@ -506,6 +510,7 @@ def location_differ(
             json.dump(problem_data, f, ensure_ascii=False, indent=4)
 
     end_time = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
+
     logger.info(f"======Location differ completed at {end_time}=======")
 
 

backend/scripts/open_diff_pull_request.py

@@ -1,5 +1,6 @@
 import argparse
 import json
+import os
 
 from pennyme.github_update import (
     DATA_BRANCH,
@@ -84,3 +85,7 @@
                 comment="No website updates today!",
                 headers=HEADER_LOCATION_DIFF,
             )
+
+    # Remove the running file to indicate that the job is done
+    os.remove(os.path.join(os.path.dirname(args.file), "running.tmp"))
+    print("Done")

data/server_locations.json

@@ -8115,12 +8115,12 @@
                 "name": "Donner Memorial State Park - Museum",
                 "area": "California",
                 "address": "12593 Donner Pass Rd, Truckee",
-                "status": "unvisited",
+                "status": "retired",
                 "external_url": "http://209.221.138.252/Details.aspx?location=315132",
                 "internal_url": "null",
                 "latitude": "39.32413390000001",
                 "longitude": "-120.2321081",
-                "machine_status": "available",
+                "machine_status": "retired",
                 "id": 6126,
                 "last_updated": "2023-10-14"
             }
@@ -9523,9 +9523,9 @@
                 "internal_url": "null",
                 "latitude": "36.642321",
                 "longitude": "-93.2466481",
-                "machine_status": "retired",
+                "machine_status": "available",
                 "id": 5325,
-                "last_updated": "2023-10-14"
+                "last_updated": "2023-12-22"
             }
         },
         {
@@ -13387,9 +13387,9 @@
                 "internal_url": "null",
                 "latitude": "37.7538033",
                 "longitude": "-100.0217447",
-                "machine_status": "retired",
+                "machine_status": "available",
                 "id": 3926,
-                "last_updated": "2023-10-14"
+                "last_updated": "2023-12-21"
             }
         },
         {
@@ -41345,6 +41345,99 @@
                 "id": 7010,
                 "last_updated": "2023-12-20"
             }
+        },
+        {
+            "type": "Feature",
+            "geometry": {
+                "type": "Point",
+                "coordinates": [
+                    -77.12628030915306,
+                    34.69220822848988
+                ]
+            },
+            "properties": {
+                "name": "Candy Edcentures",
+                "area": "North Carolina",
+                "address": "650 W Corbett Ave, Swansboro, NC 28584, USA",
+                "status": "unvisited",
+                "external_url": "null",
+                "internal_url": "null",
+                "latitude": 34.69220822848988,
+                "longitude": -77.12628030915306,
+                "machine_status": "available",
+                "id": 7011,
+                "last_updated": "2023-12-23",
+                "paywall": true
+            }
+        },
+        {
+            "type": "Feature",
+            "geometry": {
+                "type": "Point",
+                "coordinates": [
+                    -93.29229889999999,
+                    37.2089572
+                ]
+            },
+            "properties": {
+                "name": "Buc-ee's Springfield",
+                "area": "Missouri",
+                "address": "3284 N Beaver Road, Springfield",
+                "status": "unvisited",
+                "external_url": "http://209.221.138.252/Details.aspx?location=419302",
+                "internal_url": "null",
+                "latitude": "37.2089572",
+                "longitude": "-93.29229889999999",
+                "machine_status": "available",
+                "id": 7013,
+                "last_updated": "2023-12-22"
+            }
+        },
+        {
+            "type": "Feature",
+            "geometry": {
+                "type": "Point",
+                "coordinates": [
+                    -105.6564371,
+                    33.32305760000001
+                ]
+            },
+            "properties": {
+                "name": "Pillow's Funtrackers",
+                "area": "New Mexico",
+                "address": "101 Carrizo Canyon Rd, Ruidoso",
+                "status": "unvisited",
+                "external_url": "http://209.221.138.252/Details.aspx?location=419305",
+                "internal_url": "null",
+                "latitude": "33.32305760000001",
+                "longitude": "-105.6564371",
+                "machine_status": "available",
+                "id": 7012,
+                "last_updated": "2023-12-22"
+            }
+        },
+        {
+            "type": "Feature",
+            "geometry": {
+                "type": "Point",
+                "coordinates": [
+                    4.9165321,
+                    52.3660003
+                ]
+            },
+            "properties": {
+                "name": "Artis Royal Zoo",
+                "area": "Netherlands",
+                "address": "Plantage Kerklaan 38-40, Amsterdam",
+                "external_url": "http://209.221.138.252/Details.aspx?location=334704",
+                "internal_url": "null",
+                "latitude": "52.3660003",
+                "longitude": "4.9165321",
+                "machine_status": "retired",
+                "status": "retired",
+                "id": 5860,
+                "last_updated": "2023-12-23"
+            }
         }
     ]
 }