Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[3.3.x][Fixes #8689] Extend the ResourceBase metadata model with an opaque JSONField #8727

Merged
merged 25 commits into from
Feb 3, 2022
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
25 commits
Select commit Hold shift + click to select a range
0f048db
[Fixes #8689] Extend the ResourceBase metadata model with an opaque J…
mattiagiupponi Jan 26, 2022
20a1c14
[Fixes #8689] Fix missing resource_type for new form instances
mattiagiupponi Jan 27, 2022
95aa9c7
[Fixes #8689] Add test and UI fix for doc, maps and geoapps
mattiagiupponi Jan 27, 2022
38de0f0
[Fixes #8689] Fix flakee8 formatting
mattiagiupponi Jan 27, 2022
b2ba2c6
[Fixes #8689] Extra metadata json saved with format
mattiagiupponi Jan 27, 2022
318a2f0
[Fixes #8689] Refactor validation def, start defining endpoint for API
mattiagiupponi Jan 27, 2022
f108344
[Fixes #8689] Definition of extra-metadata endpoints for resources
mattiagiupponi Jan 27, 2022
e20c656
[Fixes #8689] Converting metadata from jsonfield to manytomany relation
mattiagiupponi Jan 28, 2022
0ba2ebd
[Fixes #8689] Fix views with new relation and prettify json on UI
mattiagiupponi Jan 28, 2022
19319a2
[Fixes #8689] Fix serializer
mattiagiupponi Jan 31, 2022
bd04373
Merge branch '3.3.x' of https://github.com/GeoNode/geonode into ISSUE…
mattiagiupponi Feb 1, 2022
f06c954
[Fixes #8689] Fix custom metadata endpoint, update metadata schema
mattiagiupponi Feb 1, 2022
43c77df
[Fixes #8689] Fix flake8 issues
mattiagiupponi Feb 1, 2022
4685630
[Fixes #8689] Remove endpoint from each resorce, keep it only on base…
mattiagiupponi Feb 1, 2022
8cb942f
[Fixes #8689] Fix broken tests
mattiagiupponi Feb 1, 2022
499ea10
[Fixes #8689] Add metadata filtering in API v1
mattiagiupponi Feb 1, 2022
bf819b4
[Fixes #8689] Add test for metadata filtering in API v1
mattiagiupponi Feb 1, 2022
17cf50e
[Fixes #8689] Fix some of broken tests
mattiagiupponi Feb 2, 2022
0d425b1
[Fixes #8689] fix flake8
mattiagiupponi Feb 2, 2022
2f197f1
[Fixes #8689] fix tests
mattiagiupponi Feb 2, 2022
d7b5c55
[Fixes #8689] removed typo on settings.py
mattiagiupponi Feb 2, 2022
a6498e2
[Fixes #8689] fix broken build
mattiagiupponi Feb 2, 2022
9a911f6
[Fixes #8689] fix minor error on filter convertion
mattiagiupponi Feb 2, 2022
cf7c4ec
[Fixes #8689] fix flake8
mattiagiupponi Feb 2, 2022
8d8480a
[Fixes #8689] Update default schema structure
mattiagiupponi Feb 2, 2022
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
15 changes: 14 additions & 1 deletion geonode/api/resourcebase_api.py
Original file line number Diff line number Diff line change
Expand Up @@ -95,7 +95,8 @@ class CommonMetaApi:
'date': ALL,
'purpose': ALL,
'uuid': ALL_WITH_RELATIONS,
'abstract': ALL
'abstract': ALL,
'metadata': ALL_WITH_RELATIONS
}
ordering = ['date', 'title', 'popular_count']
max_limit = None
Expand Down Expand Up @@ -168,6 +169,9 @@ def build_filters(self, filters=None, ignore_bad_filters=False, **kwargs):
orm_filters.update({'polymorphic_ctype__model__in': [filt.lower() for filt in filters.getlist('app_type__in')]})
if 'extent' in filters:
orm_filters.update({'extent': filters['extent']})
_metadata = {f"metadata__{_k}": _v for _k, _v in filters.items() if _k.startswith('metadata__')}
if _metadata:
orm_filters.update({"metadata_filters": _metadata})
orm_filters['f_method'] = filters['f_method'] if 'f_method' in filters else 'and'
if not settings.SEARCH_RESOURCES_EXTENDED:
return self._remove_additional_filters(orm_filters)
Expand All @@ -186,6 +190,9 @@ def apply_filters(self, request, applicable_filters):
metadata_only = applicable_filters.pop('metadata_only', False)
filtering_method = applicable_filters.pop('f_method', 'and')
polyphormic_model = applicable_filters.pop('polymorphic_ctype__model__in', None)

metadata_filters = applicable_filters.pop('metadata_filters', None)

if filtering_method == 'or':
filters = Q()
for f in applicable_filters.items():
Expand Down Expand Up @@ -234,6 +241,9 @@ def apply_filters(self, request, applicable_filters):
if keywords:
filtered = self.filter_h_keywords(filtered, keywords)

if metadata_filters:
filtered = filtered.filter(**metadata_filters)

# return filtered
return get_visible_resources(
filtered,
Expand Down Expand Up @@ -589,6 +599,9 @@ def format_objects(self, objects):
except Exception as e:
logger.exception(e)

if formatted_obj.get('metadata', None):
formatted_obj['metadata'] = [model_to_dict(_m) for _m in formatted_obj['metadata']]

formatted_objects.append(formatted_obj)

return formatted_objects
Expand Down
30 changes: 30 additions & 0 deletions geonode/api/tests.py
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,7 @@
from guardian.shortcuts import get_anonymous_user

from geonode import geoserver
from geonode.base.models import ExtraMetadata
from geonode.maps.models import Map
from geonode.layers.models import Layer
from geonode.utils import check_ogc_backend
Expand Down Expand Up @@ -360,6 +361,35 @@ def test_category_filters(self):
self.assertValidJSONResponse(resp)
self.assertEqual(len(self.deserialize(resp)['objects']), 5)

def test_metadata_filters(self):
"""Test category filtering"""
_r = Layer.objects.first()
_m = ExtraMetadata.objects.create(
resource=_r,
metadata={
"name": "metadata-updated",
"slug": "metadata-slug-updated",
"help_text": "this is the help text-updated",
"field_type": "str-updated",
"value": "my value-updated",
"category": "category"
}
)
_r.metadata.add(_m)
# check we get the correct layers number returnered filtering on one
# and then two different categories
filter_url = f"{self.list_url}?metadata__category=category"

resp = self.api_client.get(filter_url)
self.assertValidJSONResponse(resp)
self.assertEqual(len(self.deserialize(resp)['objects']), 1)

filter_url = f"{self.list_url}?metadata__category=not-existing-category"

resp = self.api_client.get(filter_url)
self.assertValidJSONResponse(resp)
self.assertEqual(len(self.deserialize(resp)['objects']), 0)

def test_tag_filters(self):
"""Test keywords filtering"""

Expand Down
32 changes: 27 additions & 5 deletions geonode/base/api/serializers.py
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
#########################################################################
from django.db.models.query import QuerySet
from slugify import slugify
from urllib.parse import urljoin

Expand All @@ -35,15 +36,16 @@

from geonode.favorite.models import Favorite
from geonode.base.models import (
ResourceBase,
ExtraMetadata,
HierarchicalKeyword,
License,
Region,
ResourceBase,
RestrictionCodeType,
License,
TopicCategory,
SpatialRepresentationType,
ThesaurusKeyword,
ThesaurusKeywordLabel
ThesaurusKeywordLabel,
TopicCategory,
)
from geonode.groups.models import (
GroupCategory,
Expand Down Expand Up @@ -263,6 +265,24 @@ def get_attribute(self, instance):
return build_absolute_uri(instance.detail_url)


class ExtraMetadataSerializer(DynamicModelSerializer):
class Meta:
model = ExtraMetadata
name = 'ExtraMetadata'
fields = ('pk', 'metadata')

def to_representation(self, obj):

if isinstance(obj, QuerySet):
out = []
for el in obj:
out.append({**{"id": el.id}, **el.metadata})
return out
elif isinstance(obj, list):
return obj
return {**{"id": obj.id}, **obj.metadata}


class ThumbnailUrlField(DynamicComputedField):

def __init__(self, **kwargs):
Expand Down Expand Up @@ -381,6 +401,8 @@ def __init__(self, *args, **kwargs):
self.fields['spatial_representation_type'] = DynamicRelationField(
SpatialRepresentationTypeSerializer, embed=True, many=False)

metadata = DynamicRelationField(ExtraMetadataSerializer, embed=False, many=True, deferred=True)

class Meta:
model = ResourceBase
name = 'resource'
Expand All @@ -397,7 +419,7 @@ class Meta:
'popular_count', 'share_count', 'rating', 'featured', 'is_published', 'is_approved',
'detail_url', 'embed_url', 'created', 'last_updated',
'raw_abstract', 'raw_purpose', 'raw_constraints_other',
'raw_supplemental_information', 'raw_data_quality_statement', 'metadata_only', 'processed'
'raw_supplemental_information', 'raw_data_quality_statement', 'metadata_only', 'processed', "metadata"
# TODO
# csw_typename, csw_schema, csw_mdsource, csw_insert_date, csw_type, csw_anytext, csw_wkt_geometry,
# metadata_uploaded, metadata_uploaded_preserve, metadata_xml,
Expand Down
65 changes: 64 additions & 1 deletion geonode/base/api/tests.py
Original file line number Diff line number Diff line change
Expand Up @@ -36,16 +36,18 @@

from geonode import geoserver
from geonode.layers.models import Layer
from geonode.tests.base import GeoNodeBaseTestSupport
from geonode.utils import check_ogc_backend, set_resource_default_links
from geonode.favorite.models import Favorite
from geonode.documents.models import Document
from geonode.base.utils import build_absolute_uri
from geonode.thumbs.exceptions import ThumbnailError
from geonode.base.populate_test_data import create_models
from geonode.base.populate_test_data import create_models, create_single_layer
from geonode.security.utils import get_resources_with_perms

from geonode.base.models import (
CuratedThumbnail,
ExtraMetadata,
HierarchicalKeyword,
Region,
ResourceBase,
Expand Down Expand Up @@ -1021,3 +1023,64 @@ def test_set_thumbnail_from_bbox_from_logged_user_for_existing_dataset_raise_exp
}
self.assertEqual(response.status_code, 500)
self.assertEqual(expected, response.json())


class TestExtraMetadataBaseApi(GeoNodeBaseTestSupport):
def setUp(self):
self.layer = create_single_layer('single_layer')
self.metadata = {
"filter_header": "Foo Filter header",
"field_name": "metadata-name",
"field_label": "this is the help text",
"field_value": "foo"
}
m = ExtraMetadata.objects.create(
resource=self.layer,
metadata=self.metadata
)
self.layer.metadata.add(m)
self.mdata = ExtraMetadata.objects.first()

def test_get_will_return_the_list_of_extra_metadata(self):
self.client.login(username="admin", password="admin")
url = reverse('base-resources-extra-metadata', args=[self.layer.id])
response = self.client.get(url, content_type='application/json')
self.assertTrue(200, response.status_code)
expected = [
{**{"id": self.mdata.id}, **self.metadata}
]
self.assertEqual(expected, response.json())

def test_put_will_update_the_whole_metadata(self):
self.client.login(username="admin", password="admin")
url = reverse('base-resources-extra-metadata', args=[self.layer.id])
input_metadata = {
"id": self.mdata.id,
"filter_header": "Foo Filter header",
"field_name": "metadata-updated",
"field_label": "this is the help text",
"field_value": "foo"
}
response = self.client.put(url, data=[input_metadata], content_type='application/json')
self.assertTrue(200, response.status_code)
self.assertEqual([input_metadata], response.json())

def test_post_will_add_new_metadata(self):
self.client.login(username="admin", password="admin")
url = reverse('base-resources-extra-metadata', args=[self.layer.id])
input_metadata = {
"filter_header": "Foo Filter header",
"field_name": "metadata-updated",
"field_label": "this is the help text",
"field_value": "foo"
}
response = self.client.post(url, data=[input_metadata], content_type='application/json')
self.assertTrue(201, response.status_code)
self.assertEqual(2, len(response.json()))

def test_delete_will_delete_single_metadata(self):
self.client.login(username="admin", password="admin")
url = reverse('base-resources-extra-metadata', args=[self.layer.id])
response = self.client.delete(url, data=[self.mdata.id], content_type='application/json')
self.assertTrue(200, response.status_code)
self.assertEqual([], response.json())
85 changes: 84 additions & 1 deletion geonode/base/api/views.py
Original file line number Diff line number Diff line change
Expand Up @@ -38,7 +38,7 @@
from geonode.favorite.models import Favorite
from geonode.thumbs.exceptions import ThumbnailError
from geonode.thumbs.thumbnails import create_thumbnail
from geonode.base.models import HierarchicalKeyword, Region, ResourceBase, TopicCategory, ThesaurusKeyword
from geonode.base.models import ExtraMetadata, HierarchicalKeyword, Region, ResourceBase, TopicCategory, ThesaurusKeyword
from geonode.base.api.filters import DynamicSearchFilter, ExtentFilter, FavoriteFilter
from geonode.groups.models import GroupProfile, GroupMember
from geonode.layers.models import Layer
Expand Down Expand Up @@ -70,6 +70,8 @@
ThesaurusKeywordSerializer,
)
from .pagination import GeoNodeApiPagination
from geonode.base.api.serializers import ExtraMetadataSerializer
from geonode.base.utils import validate_extra_metadata

import logging

Expand Down Expand Up @@ -466,3 +468,84 @@ def set_thumbnail_from_bbox(self, request, resource_id):
traceback.print_exc()
logger.error(e)
return Response(data={"message": e.args[0], "success": False}, status=500, exception=True)

@extend_schema(
methods=["get", "put", "delete", "post"], description="Get/Update/Delete/Add extra metadata for resource"
)
@action(
detail=True,
methods=["get", "put", "delete", "post"],
permission_classes=[
IsOwnerOrAdmin,
],
url_path=r"extra_metadata", # noqa
url_name="extra-metadata",
)
def extra_metadata(self, request, pk=None):
_obj = self.get_object()
if request.method == "GET":
# get list of available metadata
queryset = _obj.metadata.all()
_filters = [{f"metadata__{key}": value} for key, value in request.query_params.items()]
if _filters:
queryset = queryset.filter(**_filters[0])
return Response(ExtraMetadataSerializer().to_representation(queryset))
if not request.method == "DELETE":
try:
extra_metadata = validate_extra_metadata(request.data, _obj)
except Exception as e:
return Response(status=500, data=e.args[0])

if request.method == "PUT":
'''
update specific metadata. The ID of the metadata is required to perform the update
[
{
"id": 1,
"name": "foo_name",
"slug": "foo_sug",
"help_text": "object",
"field_type": "int",
"value": "object",
"category": "object"
}
]
'''
for _m in extra_metadata:
_id = _m.pop('id')
ResourceBase.objects.filter(id=_obj.id).first().metadata.filter(id=_id).update(metadata=_m)
logger.info("metadata updated for the selected resource")
_obj.refresh_from_db()
return Response(ExtraMetadataSerializer().to_representation(_obj.metadata.all()))
elif request.method == "DELETE":
# delete single metadata
'''
Expect a payload with the IDs of the metadata that should be deleted. Payload be like:
[4, 3]
'''
ResourceBase.objects.filter(id=_obj.id).first().metadata.filter(id__in=request.data).delete()
_obj.refresh_from_db()
return Response(ExtraMetadataSerializer().to_representation(_obj.metadata.all()))
elif request.method == "POST":
# add new metadata
'''
[
{
"name": "foo_name",
"slug": "foo_sug",
"help_text": "object",
"field_type": "int",
"value": "object",
"category": "object"
}
]
'''
for _m in extra_metadata:
new_m = ExtraMetadata.objects.create(
resource=_obj,
metadata=_m
)
new_m.save()
_obj.metadata.add(new_m)
_obj.refresh_from_db()
return Response(ExtraMetadataSerializer().to_representation(_obj.metadata.all()), status=201)
Loading