diff --git a/poetry.lock b/poetry.lock index 3124f59..354cf64 100644 --- a/poetry.lock +++ b/poetry.lock @@ -34,13 +34,13 @@ test = ["astroid (>=1,<2)", "astroid (>=2,<4)", "pytest"] [[package]] name = "azure-core" -version = "1.30.0" +version = "1.30.1" description = "Microsoft Azure Core Library for Python" optional = false python-versions = ">=3.7" files = [ - {file = "azure-core-1.30.0.tar.gz", hash = "sha256:6f3a7883ef184722f6bd997262eddaf80cfe7e5b3e0caaaf8db1695695893d35"}, - {file = "azure_core-1.30.0-py3-none-any.whl", hash = "sha256:3dae7962aad109610e68c9a7abb31d79720e1d982ddf61363038d175a5025e89"}, + {file = "azure-core-1.30.1.tar.gz", hash = "sha256:26273a254131f84269e8ea4464f3560c731f29c0c1f69ac99010845f239c1a8f"}, + {file = "azure_core-1.30.1-py3-none-any.whl", hash = "sha256:7c5ee397e48f281ec4dd773d67a0a47a0962ed6fa833036057f9ea067f688e74"}, ] [package.dependencies] @@ -693,13 +693,13 @@ ipython = {version = ">=7.31.1", markers = "python_version >= \"3.11\""} [[package]] name = "ipython" -version = "8.22.1" +version = "8.22.2" description = "IPython: Productive Interactive Computing" optional = false python-versions = ">=3.10" files = [ - {file = "ipython-8.22.1-py3-none-any.whl", hash = "sha256:869335e8cded62ffb6fac8928e5287a05433d6462e3ebaac25f4216474dd6bc4"}, - {file = "ipython-8.22.1.tar.gz", hash = "sha256:39c6f9efc079fb19bfb0f17eee903978fe9a290b1b82d68196c641cecb76ea22"}, + {file = "ipython-8.22.2-py3-none-any.whl", hash = "sha256:3c86f284c8f3d8f2b6c662f885c4889a91df7cd52056fd02b7d8d6195d7f56e9"}, + {file = "ipython-8.22.2.tar.gz", hash = "sha256:2dcaad9049f9056f1fef63514f176c7d41f930daa78d05b82a176202818f2c14"}, ] [package.dependencies] @@ -988,13 +988,13 @@ windows-terminal = ["colorama (>=0.4.6)"] [[package]] name = "python-dateutil" -version = "2.8.2" +version = "2.9.0.post0" description = "Extensions to the standard Python datetime module" optional = false python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,>=2.7" files = [ - {file = "python-dateutil-2.8.2.tar.gz", hash = "sha256:0123cacc1627ae19ddf3c27a5de5bd67ee4586fbdd6440d9748f8abb483d3e86"}, - {file = "python_dateutil-2.8.2-py2.py3-none-any.whl", hash = "sha256:961d03dc3453ebbc59dbdea9e4e11c5651520a876d0f4db161e8674aae935da9"}, + {file = "python-dateutil-2.9.0.post0.tar.gz", hash = "sha256:37dd54208da7e1cd875388217d5e00ebd4179249f90fb72437e91a35459a0ad3"}, + {file = "python_dateutil-2.9.0.post0-py2.py3-none-any.whl", hash = "sha256:a8b2bc7bffae282281c8140a97d3aa9c14da0b136dfe83f850eea9a5f7470427"}, ] [package.dependencies] @@ -1097,13 +1097,13 @@ use-chardet-on-py3 = ["chardet (>=3.0.2,<6)"] [[package]] name = "sentry-sdk" -version = "1.40.5" +version = "1.40.6" description = "Python client for Sentry (https://sentry.io)" optional = false python-versions = "*" files = [ - {file = "sentry-sdk-1.40.5.tar.gz", hash = "sha256:d2dca2392cc5c9a2cc9bb874dd7978ebb759682fe4fe889ee7e970ee8dd1c61e"}, - {file = "sentry_sdk-1.40.5-py2.py3-none-any.whl", hash = "sha256:d188b407c9bacbe2a50a824e1f8fb99ee1aeb309133310488c570cb6d7056643"}, + {file = "sentry-sdk-1.40.6.tar.gz", hash = "sha256:f143f3fb4bb57c90abef6e2ad06b5f6f02b2ca13e4060ec5c0549c7a9ccce3fa"}, + {file = "sentry_sdk-1.40.6-py2.py3-none-any.whl", hash = "sha256:becda09660df63e55f307570e9817c664392655a7328bbc414b507e9cb874c67"}, ] [package.dependencies] @@ -1130,7 +1130,7 @@ huey = ["huey (>=2)"] loguru = ["loguru (>=0.5)"] opentelemetry = ["opentelemetry-distro (>=0.35b0)"] opentelemetry-experimental = ["opentelemetry-distro (>=0.40b0,<1.0)", "opentelemetry-instrumentation-aiohttp-client (>=0.40b0,<1.0)", "opentelemetry-instrumentation-django (>=0.40b0,<1.0)", "opentelemetry-instrumentation-fastapi (>=0.40b0,<1.0)", "opentelemetry-instrumentation-flask (>=0.40b0,<1.0)", "opentelemetry-instrumentation-requests (>=0.40b0,<1.0)", "opentelemetry-instrumentation-sqlite3 (>=0.40b0,<1.0)", "opentelemetry-instrumentation-urllib (>=0.40b0,<1.0)"] -pure-eval = ["asttokens", "executing", "pure-eval"] +pure-eval = ["asttokens", "executing", "pure_eval"] pymongo = ["pymongo (>=3.1)"] pyspark = ["pyspark (>=2.4.4)"] quart = ["blinker (>=1.1)", "quart (>=0.16.1)"] @@ -1318,4 +1318,4 @@ brotli = ["Brotli"] [metadata] lock-version = "2.0" python-versions = "^3.11" -content-hash = "a01bb29d98cdbbd198fde4db0e6e01294fa0371ab759cf0df7f01212707e54c8" +content-hash = "52b061b3fd28e76560ffc0d6050a10ddf070de64072dc5fc0fccd6e41a253b6e" diff --git a/pyproject.toml b/pyproject.toml index ed4de1e..4052e8f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -19,12 +19,12 @@ django-geojson = "4.0.0" unicodecsv = "0.14.1" whitenoise = {version = "6.6.0", extras = ["brotli"]} azure-storage-blob = "12.19.0" -sentry-sdk = {version = "1.40.5", extras = ["django"]} +sentry-sdk = {version = "1.40.6", extras = ["django"]} [tool.poetry.group.dev.dependencies] -ipython = "^8.22.1" +ipython = "^8.22.2" ipdb = "^0.13.13" -black = "^24.1.1" +black = "^24.2.0" pre-commit = "^3.6.2" mixer = "^7.2.2" diff --git a/tracking/admin.py b/tracking/admin.py index 9cf552d..ccd47ad 100644 --- a/tracking/admin.py +++ b/tracking/admin.py @@ -53,7 +53,8 @@ def has_add_permission(self, request, obj=None): return False def has_change_permission(self, request, obj=None): - if obj is not None and obj.source_device_type == 'tracplus': + # TracPlus and DFES vehicles have an external source of truth regarding metadata. + if obj is not None and obj.source_device_type in ["tracplus", "dfes"]: return False else: return super(DeviceAdmin, self).has_change_permission(request, obj=obj) @@ -62,13 +63,13 @@ def has_change_permission(self, request, obj=None): class DeviceSSSAdmin(DeviceAdmin): def add_view(self, request, obj=None): - return HttpResponseRedirect(reverse('sss_admin:tracking_device_changelist')) + return HttpResponseRedirect(reverse("sss_admin:tracking_device_changelist")) class TrackingAdminSite(AdminSite): - site_header = 'SSS administration' + site_header = "SSS administration" site_url = None -tracking_admin_site = TrackingAdminSite(name='sss_admin') +tracking_admin_site = TrackingAdminSite(name="sss_admin") tracking_admin_site.register(Device, DeviceSSSAdmin) diff --git a/tracking/harvest.py b/tracking/harvest.py index 2d36044..0fa96a0 100644 --- a/tracking/harvest.py +++ b/tracking/harvest.py @@ -289,7 +289,7 @@ def save_dplus(message): def save_dfes_feed(): - """Download and process the DFES API endpoint (returns GeoJSON). + """Download and process the DFES API endpoint (returns GeoJSON), create new devices, update existing. """ LOGGER.info("Querying DFES API") resp = requests.get(url=settings.DFES_URL, auth=(settings.DFES_USER, settings.DFES_PASS)) @@ -317,25 +317,20 @@ def save_dfes_feed(): continue device, created = Device.objects.get_or_create(deviceid=data["device_id"]) + properties = feature["properties"] + if created: created_device += 1 device.source_device_type = "dfes" - # Set some additional values on the Device from the feature data. - properties = feature["properties"] - device.callsign = properties["VehicleName"] - device.callsign_display = properties["VehicleName"] - device.model = properties["Model"] - if properties["Registration"]: - rego = properties["Registration"][:32].strip() - device.registration = f"DFES - {rego}" - else: - device.registration = "DFES - No Rego" - vehicle_type = properties["VehicleType"].strip() - if vehicle_type in DFES_SYMBOL_MAP: - device.symbol = DFES_SYMBOL_MAP[vehicle_type] - else: - device.symbol = "unknown" - device.save() + else: + updated_device += 1 + + device.callsign = properties["VehicleName"] + device.callsign_display = properties["VehicleName"] + if properties["Registration"]: + device.registration = properties["Registration"][:32].strip() + if properties["VehicleType"].strip() in DFES_SYMBOL_MAP: + device.symbol = DFES_SYMBOL_MAP[properties["VehicleType"].strip()] seen = data["timestamp"] point = f"POINT({data['longitude']} {data['latitude']})" @@ -346,8 +341,8 @@ def save_dfes_feed(): device.heading = data["heading"] device.velocity = data["velocity"] device.altitude = data["altitude"] - device.save() - updated_device += 1 + + device.save() loggedpoint, created = LoggedPoint.objects.get_or_create(device=device, seen=seen, point=point) if created: @@ -357,14 +352,12 @@ def save_dfes_feed(): loggedpoint.altitude = data["altitude"] loggedpoint.save() logged_points += 1 - else: - skipped_device += 1 LOGGER.info(f"Created {created_device}, updated {updated_device}, skipped {skipped_device}, {logged_points} new logged points") def save_tracplus_feed(): - """Query the TracPlus API, create logged points per device. + """Query the TracPlus API, create logged points per device, update existing devices. """ LOGGER.info("Harvesting TracPlus feed") content = requests.get(settings.TRACPLUS_URL).content.decode("utf-8") @@ -395,17 +388,18 @@ def save_tracplus_feed(): continue device, created = Device.objects.get_or_create(deviceid=data["device_id"]) + rego = row["Asset Regn"][:32].strip() + symbol = row["Asset Type"] if row["Asset Type"] in tracplus_symbol_map else None + if created: created_device += 1 device.source_device_type = "tracplus" - # Set some additional values on the Device from the CSV row data. - device.callsign = row["Asset Name"] - device.callsign_display = row["Asset Name"] - device.model = row["Asset Model"] - device.registration = row["Asset Regn"][:32] - if row["Asset Type"] in tracplus_symbol_map: - device.symbol = tracplus_symbol_map[row["Asset Type"]] - device.save() + else: + updated_device += 1 + + device.registration = rego + if symbol: + device.symbol = symbol seen = data["timestamp"] point = f"POINT({data['longitude']} {data['latitude']})" @@ -416,8 +410,8 @@ def save_tracplus_feed(): device.heading = data["heading"] device.velocity = data["velocity"] device.altitude = data["altitude"] - updated_device += 1 - device.save() + + device.save() loggedpoint, created = LoggedPoint.objects.get_or_create(device=device, seen=seen, point=point) if created: