From a00ec4b205edbf4b659ce992fadb5b22a8ac0081 Mon Sep 17 00:00:00 2001 From: Petya Date: Thu, 4 Jan 2024 15:41:36 +0000 Subject: [PATCH 01/11] Create FUNDING.yml Adding the sponsorship button to fAIr --- .github/FUNDING.yml | 13 +++++++++++++ 1 file changed, 13 insertions(+) create mode 100644 .github/FUNDING.yml diff --git a/.github/FUNDING.yml b/.github/FUNDING.yml new file mode 100644 index 00000000..ed5e88b1 --- /dev/null +++ b/.github/FUNDING.yml @@ -0,0 +1,13 @@ +# These are supported funding model platforms + +github: hotosm +patreon: # Replace with a single Patreon username +open_collective: # Replace with a single Open Collective username +ko_fi: # Replace with a single Ko-fi username +tidelift: # Replace with a single Tidelift platform-name/package-name e.g., npm/babel +community_bridge: # Replace with a single Community Bridge project-name e.g., cloud-foundry +liberapay: # Replace with a single Liberapay username +issuehunt: # Replace with a single IssueHunt username +otechie: # Replace with a single Otechie username +lfx_crowdfunding: # Replace with a single LFX Crowdfunding project-name e.g., cloud-foundry +custom: # Replace with up to 4 custom sponsorship URLs e.g., ['link1', 'link2'] From 98ccb7a34eba2aedd743132f80e873d755bb4363 Mon Sep 17 00:00:00 2001 From: kshtiijrajsharma Date: Tue, 9 Jan 2024 18:56:03 +0545 Subject: [PATCH 02/11] Added method to dump aoi.geojson and labels .geojson --- backend/core/models.py | 10 +++++++-- backend/core/serializers.py | 4 ++-- backend/core/tasks.py | 42 ++++++++++++++++++++++++++++--------- backend/core/utils.py | 10 +++++++-- 4 files changed, 50 insertions(+), 16 deletions(-) diff --git a/backend/core/models.py b/backend/core/models.py index b1d67be1..5ad0e284 100644 --- a/backend/core/models.py +++ b/backend/core/models.py @@ -41,6 +41,7 @@ class Label(models.Model): aoi = models.ForeignKey(AOI, to_field="id", on_delete=models.CASCADE) geom = geomodels.GeometryField(srid=4326) osm_id = models.BigIntegerField(null=True, blank=True) + tags = models.JSONField(null=True, blank=True) created_at = models.DateTimeField(auto_now_add=True) @@ -101,7 +102,7 @@ class Feedback(models.Model): validators=[MinValueValidator(18), MaxValueValidator(23)] ) feedback_type = models.CharField(choices=FEEDBACK_TYPE, max_length=10) - comments = models.TextField(max_length=100,null=True,blank=True) + comments = models.TextField(max_length=100, null=True, blank=True) user = models.ForeignKey(OsmUser, to_field="osm_id", on_delete=models.CASCADE) source_imagery = models.URLField() @@ -111,6 +112,7 @@ class DownloadStatus(models.IntegerChoices): DOWNLOADED = 1 NOT_DOWNLOADED = -1 RUNNING = 0 + training = models.ForeignKey(Training, to_field="id", on_delete=models.CASCADE) geom = geomodels.PolygonField(srid=4326) label_status = models.IntegerField(default=-1, choices=DownloadStatus.choices) @@ -123,6 +125,10 @@ class DownloadStatus(models.IntegerChoices): class FeedbackLabel(models.Model): osm_id = models.BigIntegerField(null=True, blank=True) - feedback_aoi = models.ForeignKey(FeedbackAOI, to_field="id", on_delete=models.CASCADE) + feedback_aoi = models.ForeignKey( + FeedbackAOI, to_field="id", on_delete=models.CASCADE + ) + tags = models.JSONField(null=True, blank=True) + geom = geomodels.PolygonField(srid=4326) created_at = models.DateTimeField(auto_now_add=True) diff --git a/backend/core/serializers.py b/backend/core/serializers.py index 9b08ff8b..24939aa9 100644 --- a/backend/core/serializers.py +++ b/backend/core/serializers.py @@ -126,7 +126,7 @@ class Meta: model = Label geo_field = "geom" # auto_bbox = True - fields = ("osm_id",) + fields = ("osm_id", "tags") class FeedbackLabelFileSerializer(GeoFeatureModelSerializer): @@ -134,7 +134,7 @@ class Meta: model = FeedbackLabel geo_field = "geom" # auto_bbox = True - fields = ("osm_id",) + fields = ("osm_id", "tags") class FeedbackFileSerializer(GeoFeatureModelSerializer): diff --git a/backend/core/tasks.py b/backend/core/tasks.py index b78d4a96..cf74195b 100644 --- a/backend/core/tasks.py +++ b/backend/core/tasks.py @@ -12,15 +12,13 @@ from celery import shared_task from core.models import AOI, Feedback, FeedbackAOI, FeedbackLabel, Label, Training from core.serializers import ( + AOISerializer, + FeedbackAOISerializer, FeedbackFileSerializer, FeedbackLabelFileSerializer, LabelFileSerializer, ) -from predictor import download_imagery,get_start_end_download_coords -from core.utils import ( - bbox, - is_dir_empty, -) +from core.utils import bbox, is_dir_empty from django.conf import settings from django.contrib.gis.db.models.aggregates import Extent from django.contrib.gis.geos import GEOSGeometry @@ -28,6 +26,7 @@ from django.utils import timezone from hot_fair_utilities import preprocess, train from hot_fair_utilities.training import run_feedback +from predictor import download_imagery, get_start_end_download_coords logger = logging.getLogger(__name__) @@ -56,8 +55,8 @@ def train_model( try: ## -----------IMAGE DOWNLOADER--------- os.makedirs(settings.LOG_PATH, exist_ok=True) - if training_instance.task_id is None or training_instance.task_id.strip() == '': - training_instance.task_id=train_model.request.id + if training_instance.task_id is None or training_instance.task_id.strip() == "": + training_instance.task_id = train_model.request.id training_instance.save() log_file = os.path.join( settings.LOG_PATH, f"run_{train_model.request.id}_log.txt" @@ -77,6 +76,8 @@ def train_model( if feedback: try: aois = FeedbackAOI.objects.filter(training=feedback) + aoi_serializer = FeedbackAOISerializer(aois, many=True) + except FeedbackAOI.DoesNotExist: raise ValueError( f"No Feedback AOI is attached with supplied training id:{dataset_id}, Create AOI first", @@ -85,11 +86,12 @@ def train_model( else: try: aois = AOI.objects.filter(dataset=dataset_id) + aoi_serializer = AOISerializer(aois, many=True) + except AOI.DoesNotExist: raise ValueError( f"No AOI is attached with supplied dataset id:{dataset_id}, Create AOI first", ) - for obj in aois: bbox_coords = bbox(obj.geom.coords[0]) for z in zoom_level: @@ -223,15 +225,35 @@ def train_model( logger.info(model.inputs) logger.info(model.outputs) - + # Convert the model to tflite for android/ios. converter = tf.lite.TFLiteConverter.from_keras_model(model) tflite_model = converter.convert() # Save the model. - with open(os.path.join(output_path, "checkpoint.tflite"), 'wb') as f: + with open(os.path.join(output_path, "checkpoint.tflite"), "wb") as f: f.write(tflite_model) + # dump labels to output folder as well + with open( + os.path.join(output_path, "labels.geojson"), + "w", + encoding="utf-8", + ) as f: + f.write(json.dumps(serialized_field.data)) + + # dump used aois as featurecollection in output + aois_geojson = { + "type": "FeatureCollection", + "features": aoi_serializer.data, + } + with open( + os.path.join(output_path, "aois.geojson"), + "w", + encoding="utf-8", + ) as f: + f.write(json.dumps(aois_geojson)) + # now remove the ramp-data all our outputs are copied to our training workspace shutil.rmtree(base_path) training_instance.accuracy = float(final_accuracy) diff --git a/backend/core/utils.py b/backend/core/utils.py index 9d0a9f3b..3181eec7 100644 --- a/backend/core/utils.py +++ b/backend/core/utils.py @@ -189,6 +189,7 @@ def process_feature(feature, aoi_id, foreign_key_id, feedback=False): """Multi thread process of features""" properties = feature["properties"] osm_id = properties["osm_id"] + tags = properties["tags"] geometry = feature["geometry"] if feedback: if FeedbackLabel.objects.filter( @@ -199,7 +200,12 @@ def process_feature(feature, aoi_id, foreign_key_id, feedback=False): ).delete() label = FeedbackLabelSerializer( - data={"osm_id": int(osm_id), "geom": geometry, "feedback_aoi": aoi_id} + data={ + "osm_id": int(osm_id), + "tags": tags, + "geom": geometry, + "feedback_aoi": aoi_id, + } ) else: @@ -211,7 +217,7 @@ def process_feature(feature, aoi_id, foreign_key_id, feedback=False): ).delete() label = LabelSerializer( - data={"osm_id": int(osm_id), "geom": geometry, "aoi": aoi_id} + data={"osm_id": int(osm_id), "tags": tags, "geom": geometry, "aoi": aoi_id} ) if label.is_valid(): label.save() From 841a572b7a73efa72f589b226dba7ad8ddd524e0 Mon Sep 17 00:00:00 2001 From: kshtiijrajsharma Date: Wed, 10 Jan 2024 17:37:12 +0545 Subject: [PATCH 03/11] Fix bug on aoi serializer --- backend/core/tasks.py | 23 ++++++++++------------- 1 file changed, 10 insertions(+), 13 deletions(-) diff --git a/backend/core/tasks.py b/backend/core/tasks.py index cf74195b..a66e4748 100644 --- a/backend/core/tasks.py +++ b/backend/core/tasks.py @@ -10,6 +10,15 @@ import ramp.utils import tensorflow as tf from celery import shared_task +from django.conf import settings +from django.contrib.gis.db.models.aggregates import Extent +from django.contrib.gis.geos import GEOSGeometry +from django.shortcuts import get_object_or_404 +from django.utils import timezone +from hot_fair_utilities import preprocess, train +from hot_fair_utilities.training import run_feedback +from predictor import download_imagery, get_start_end_download_coords + from core.models import AOI, Feedback, FeedbackAOI, FeedbackLabel, Label, Training from core.serializers import ( AOISerializer, @@ -19,14 +28,6 @@ LabelFileSerializer, ) from core.utils import bbox, is_dir_empty -from django.conf import settings -from django.contrib.gis.db.models.aggregates import Extent -from django.contrib.gis.geos import GEOSGeometry -from django.shortcuts import get_object_or_404 -from django.utils import timezone -from hot_fair_utilities import preprocess, train -from hot_fair_utilities.training import run_feedback -from predictor import download_imagery, get_start_end_download_coords logger = logging.getLogger(__name__) @@ -243,16 +244,12 @@ def train_model( f.write(json.dumps(serialized_field.data)) # dump used aois as featurecollection in output - aois_geojson = { - "type": "FeatureCollection", - "features": aoi_serializer.data, - } with open( os.path.join(output_path, "aois.geojson"), "w", encoding="utf-8", ) as f: - f.write(json.dumps(aois_geojson)) + f.write(json.dumps(aoi_serializer.data)) # now remove the ramp-data all our outputs are copied to our training workspace shutil.rmtree(base_path) From 367e00222b3f1adbf1f8fe613ca52975f9a57bf8 Mon Sep 17 00:00:00 2001 From: Joseph Munyenze <59202325+Joseph-Munyenze@users.noreply.github.com> Date: Wed, 10 Jan 2024 16:14:52 +0300 Subject: [PATCH 04/11] Update CONTRIBUTING.md Added the welcome section --- CONTRIBUTING.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 3cf2eb75..6bb90883 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -4,3 +4,10 @@ :+1::tada: First off, I'm really glad you're reading this, because we need volunteer developers to help with the development of fAIr! :tada::+1: We welcome and encourage contributors of all skill levels and we are committed to making sure your participation in our tech collective is inclusive, enjoyable and rewarding. If you have never contributed to an open-source project before, we are a good place to start and will make sure you are supported every step of the way. If you have **any** questions, please ask! +## :hugs: Welcome + +:+1::tada: First off, we are really glad you're reading this, because we need volunteer developers to help improve the fAIr! :tada::+1: + +We welcome and encourage contributors of all skill levels and we are committed to making sure your participation is inclusive, enjoyable and rewarding. If you have never contributed to an open source project before, we are a good place to start and will make sure you are supported every step of the way. If you have **any** questions, please ask! + +There are many ways to contribute to this repo: From 3b612f53821caab52a1a89c14621f86ed7d5cbc0 Mon Sep 17 00:00:00 2001 From: Bestor~Jnr Date: Wed, 10 Jan 2024 14:14:53 +0100 Subject: [PATCH 05/11] Update CONTRIBUTING.md Added the Documentation messaged --- CONTRIBUTING.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 3cf2eb75..8d32283e 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -4,3 +4,7 @@ :+1::tada: First off, I'm really glad you're reading this, because we need volunteer developers to help with the development of fAIr! :tada::+1: We welcome and encourage contributors of all skill levels and we are committed to making sure your participation in our tech collective is inclusive, enjoyable and rewarding. If you have never contributed to an open-source project before, we are a good place to start and will make sure you are supported every step of the way. If you have **any** questions, please ask! + +## Documentation contributions + +Create pull requests (PRs) for changes that you think are needed to the documentation of fAIr.As of now you can find the documentation work at the [docs](https://github.com/hotosm/fAIr/tree/master/docs) directory. From 515a74bfc9cf675d4b9cd00436e2e68d6aad96eb Mon Sep 17 00:00:00 2001 From: Victor Ademoyero <53705305+vickystickz@users.noreply.github.com> Date: Wed, 10 Jan 2024 14:15:01 +0100 Subject: [PATCH 06/11] Update CONTRIBUTING.md Added Code Contributions message --- CONTRIBUTING.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 3cf2eb75..2621bd36 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -4,3 +4,7 @@ :+1::tada: First off, I'm really glad you're reading this, because we need volunteer developers to help with the development of fAIr! :tada::+1: We welcome and encourage contributors of all skill levels and we are committed to making sure your participation in our tech collective is inclusive, enjoyable and rewarding. If you have never contributed to an open-source project before, we are a good place to start and will make sure you are supported every step of the way. If you have **any** questions, please ask! + +## Code contributions + +Fork repo, Maintain your local changes on branch and Create pull requests (PRs) for changes that you think are needed. We would really appreciate your help! From 256bfa5e0daaacacb8a4cd0e8603807d745dcf9e Mon Sep 17 00:00:00 2001 From: Joseph Munyenze <59202325+Joseph-Munyenze@users.noreply.github.com> Date: Wed, 10 Jan 2024 17:05:29 +0300 Subject: [PATCH 07/11] Update CONTRIBUTING.md Added welcome section --- CONTRIBUTING.md | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 6bb90883..c4a25c91 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -1,13 +1,8 @@ # Contributing to fAIr -## Welcome +## :hugs: Welcome :+1::tada: First off, I'm really glad you're reading this, because we need volunteer developers to help with the development of fAIr! :tada::+1: We welcome and encourage contributors of all skill levels and we are committed to making sure your participation in our tech collective is inclusive, enjoyable and rewarding. If you have never contributed to an open-source project before, we are a good place to start and will make sure you are supported every step of the way. If you have **any** questions, please ask! -## :hugs: Welcome - -:+1::tada: First off, we are really glad you're reading this, because we need volunteer developers to help improve the fAIr! :tada::+1: - -We welcome and encourage contributors of all skill levels and we are committed to making sure your participation is inclusive, enjoyable and rewarding. If you have never contributed to an open source project before, we are a good place to start and will make sure you are supported every step of the way. If you have **any** questions, please ask! There are many ways to contribute to this repo: From 3c62e638a0ae91e36d9af601a9662ba276cef934 Mon Sep 17 00:00:00 2001 From: kshitijrajsharma Date: Mon, 22 Jan 2024 17:25:44 +0545 Subject: [PATCH 08/11] Fixes bug for old osm data --- backend/core/utils.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/backend/core/utils.py b/backend/core/utils.py index 3181eec7..3c09b559 100644 --- a/backend/core/utils.py +++ b/backend/core/utils.py @@ -192,6 +192,7 @@ def process_feature(feature, aoi_id, foreign_key_id, feedback=False): tags = properties["tags"] geometry = feature["geometry"] if feedback: + FeedbackLabel.objects.filter(aoi__id=aoi_id).delete() if FeedbackLabel.objects.filter( osm_id=int(osm_id), feedback_aoi__training=foreign_key_id ).exists(): @@ -209,6 +210,7 @@ def process_feature(feature, aoi_id, foreign_key_id, feedback=False): ) else: + Label.objects.filter(aoi__id=aoi_id).delete() if Label.objects.filter( osm_id=int(osm_id), aoi__dataset=foreign_key_id ).exists(): From 5351d4a3ff2480c5ee3fb0a94c0039417781fbf0 Mon Sep 17 00:00:00 2001 From: kshitijrajsharma Date: Mon, 22 Jan 2024 17:39:53 +0545 Subject: [PATCH 09/11] Fix bug of removing layer inside thread --- backend/core/utils.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/backend/core/utils.py b/backend/core/utils.py index 3c09b559..82a403a8 100644 --- a/backend/core/utils.py +++ b/backend/core/utils.py @@ -192,7 +192,6 @@ def process_feature(feature, aoi_id, foreign_key_id, feedback=False): tags = properties["tags"] geometry = feature["geometry"] if feedback: - FeedbackLabel.objects.filter(aoi__id=aoi_id).delete() if FeedbackLabel.objects.filter( osm_id=int(osm_id), feedback_aoi__training=foreign_key_id ).exists(): @@ -210,7 +209,6 @@ def process_feature(feature, aoi_id, foreign_key_id, feedback=False): ) else: - Label.objects.filter(aoi__id=aoi_id).delete() if Label.objects.filter( osm_id=int(osm_id), aoi__dataset=foreign_key_id ).exists(): @@ -247,7 +245,10 @@ def process_geojson(geojson_file_path, aoi_id, feedback=False): max_workers = ( (os.cpu_count() - 1) if os.cpu_count() != 1 else 1 ) # leave one cpu free always - + if feedback: + FeedbackLabel.objects.filter(aoi__id=aoi_id).delete() + else : + Label.objects.filter(aoi__id=aoi_id).delete() # max_workers = os.cpu_count() # get total cpu count available on the with open(geojson_file_path) as f: From 04010c26c537f735343989ddeab465dc48126c0c Mon Sep 17 00:00:00 2001 From: kshitijrajsharma Date: Mon, 22 Jan 2024 18:42:22 +0545 Subject: [PATCH 10/11] fix for the feedback aoi id --- backend/core/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/core/utils.py b/backend/core/utils.py index 82a403a8..fe8c5619 100644 --- a/backend/core/utils.py +++ b/backend/core/utils.py @@ -246,7 +246,7 @@ def process_geojson(geojson_file_path, aoi_id, feedback=False): (os.cpu_count() - 1) if os.cpu_count() != 1 else 1 ) # leave one cpu free always if feedback: - FeedbackLabel.objects.filter(aoi__id=aoi_id).delete() + FeedbackLabel.objects.filter(feedback_aoi__id=aoi_id).delete() else : Label.objects.filter(aoi__id=aoi_id).delete() # max_workers = os.cpu_count() # get total cpu count available on the From a79ddea17e2983e2fe1dea4fc52bc64bb2c01e47 Mon Sep 17 00:00:00 2001 From: kshitijrajsharma Date: Tue, 23 Jan 2024 19:45:49 +0545 Subject: [PATCH 11/11] Adds django filter backend to feedback labels --- backend/core/views.py | 1 + 1 file changed, 1 insertion(+) diff --git a/backend/core/views.py b/backend/core/views.py index 80389981..03f4ffe4 100644 --- a/backend/core/views.py +++ b/backend/core/views.py @@ -192,6 +192,7 @@ class FeedbackLabelViewset(viewsets.ModelViewSet): bbox_filter_field = "geom" filter_backends = ( InBBoxFilter, # it will take bbox like this api/v1/label/?in_bbox=-90,29,-89,35 , + DjangoFilterBackend ) bbox_filter_include_overlapping = True filterset_fields = ["feedback_aoi", "feedback_aoi__training"]