Skip to content

Commit

Permalink
[Fixes #4959] By using UPLOADER=geonode.rest we are facing a timing i…
Browse files Browse the repository at this point in the history 
…ssues on signals

 - Fix test cases
  • Loading branch information
afabiani committed Sep 30, 2019
1 parent 025546d commit 5ee005c
Show file tree
Hide file tree
Showing 3 changed files with 66 additions and 135 deletions.
61 changes: 36 additions & 25 deletions geonode/geoserver/signals.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,7 @@

import errno
import logging

from time import sleep
from django.conf import settings
from django.forms.models import model_to_dict

Expand Down Expand Up @@ -122,6 +122,7 @@ def geoserver_post_save_local(instance, *args, **kwargs):

gs_resource = None
values = None
_tries = 0

# If the store in None then it's a new instance from an upload,
# only in this case run the geoserver_upload method
Expand All @@ -141,30 +142,40 @@ def geoserver_post_save_local(instance, *args, **kwargs):
# keywords=instance.keywords,
charset=instance.charset)

gs_resource = gs_catalog.get_resource(
name=instance.name,
store=instance.store,
workspace=instance.workspace)
if not gs_resource:
gs_resource = gs_catalog.get_resource(name=instance.alternate)

if gs_resource:
gs_resource.title = instance.title or ""
gs_resource.abstract = instance.abstract or ""
gs_resource.name = instance.name or ""

if not values:
values = dict(store=gs_resource.store.name,
storeType=gs_resource.store.resource_type,
alternate=gs_resource.store.workspace.name + ':' + gs_resource.name,
title=gs_resource.title or gs_resource.store.name,
abstract=gs_resource.abstract or '',
owner=instance.owner)
else:
msg = "There isn't a geoserver resource for this layer: %s" % instance.name
logger.exception(msg)
# raise Exception(msg)
gs_resource = None
def fetch_gs_resource(values, tries):
_max_tries = getattr(ogc_server_settings, "MAX_RETRIES", 5)
gs_resource = gs_catalog.get_resource(
name=instance.name,
store=instance.store,
workspace=instance.workspace)
if not gs_resource:
gs_resource = gs_catalog.get_resource(name=instance.alternate)

if gs_resource:
gs_resource.title = instance.title or ""
gs_resource.abstract = instance.abstract or ""
gs_resource.name = instance.name or ""

if not values:
values = dict(store=gs_resource.store.name,
storeType=gs_resource.store.resource_type,
alternate=gs_resource.store.workspace.name + ':' + gs_resource.name,
title=gs_resource.title or gs_resource.store.name,
abstract=gs_resource.abstract or '',
owner=instance.owner)
else:
msg = "There isn't a geoserver resource for this layer: %s" % instance.name
logger.exception(msg)
if tries >= _max_tries:
raise GeoNodeException(msg)
gs_resource = None
sleep(3.00)

return (values, gs_resource)

while not gs_resource:
values, gs_resource = fetch_gs_resource(values, _tries)
_tries += 1

# Get metadata links
metadata_links = []
Expand Down
4 changes: 2 additions & 2 deletions geonode/messaging/consumer.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -132,7 +132,7 @@ def on_geoserver_catalog(self, body, message):
_update_layer_data(body, self.last_message)
self.last_message = json.loads(body)
except BaseException:
logger.info("Could not encode message {!r}".format(body))
logger.debug("Could not encode message {!r}".format(body))
message.ack()
logger.debug("on_geoserver_catalog: finished")
self._check_message_limit()
Expand All @@ -143,7 +143,7 @@ def on_geoserver_data(self, body, message):
_update_layer_data(body, self.last_message)
self.last_message = json.loads(body)
except BaseException:
logger.info("Could not encode message {!r}".format(body))
logger.debug("Could not encode message {!r}".format(body))
message.ack()
logger.debug("on_geoserver_data: finished")
self._check_message_limit()
Expand Down
136 changes: 28 additions & 108 deletions geonode/tests/csw.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -115,12 +115,10 @@ def test_csw_outputschema_dc(self):
record = csw.catalogue.records.values()[0]

# test that the ISO title maps correctly in Dublin Core
self.assertEqual(record.title, 'San Andres Y Providencia Location',
'Expected a specific title in Dublin Core model')
self.assertEquals(record.title, 'San Andres Y Providencia Location')

# test that the ISO abstract maps correctly in Dublin Core
self.assertEqual(record.abstract, 'No abstract provided',
'Expected a specific abstract in Dublin Core model')
self.assertEquals(record.abstract, None)

# test for correct service link articulation
for link in record.references:
Expand Down Expand Up @@ -160,35 +158,17 @@ def test_csw_outputschema_iso(self):
record = csw.catalogue.records.values()[0]

# test that the ISO title maps correctly in Dublin Core
self.assertEqual(
record.identification.title,
'San Andres Y Providencia Location',
'Expected a specific title in ISO model')
self.assertEquals(record.identification.title, 'San Andres Y Providencia Location')

# test that the ISO abstract maps correctly in Dublin Core
self.assertEqual(
record.identification.abstract,
'No abstract provided',
'Expected a specific abstract in ISO model')
self.assertEquals(record.identification.abstract, None)

# test BBOX properties in Dublin Core
from decimal import Decimal
self.assertEqual(
Decimal(record.identification.bbox.minx),
Decimal('-81.8593555'),
'Expected a specific minx coordinate value in ISO model')
self.assertEqual(
Decimal(record.identification.bbox.miny),
Decimal('12.1665322'),
'Expected a specific minx coordinate value in ISO model')
self.assertEqual(
Decimal(record.identification.bbox.maxx),
Decimal('-81.356409'),
'Expected a specific maxx coordinate value in ISO model')
self.assertEqual(
Decimal(record.identification.bbox.maxy),
Decimal('13.396306'),
'Expected a specific maxy coordinate value in ISO model')
self.assertEquals(Decimal(record.identification.bbox.minx), Decimal('-81.8593555'))
self.assertEquals(Decimal(record.identification.bbox.miny), Decimal('12.1665322'))
self.assertEquals(Decimal(record.identification.bbox.maxx), Decimal('-81.356409'))
self.assertEquals(Decimal(record.identification.bbox.maxy), Decimal('13.396306'))

# test for correct link articulation
for link in record.distribution.online:
Expand Down Expand Up @@ -237,30 +217,15 @@ def test_csw_outputschema_dc_bbox(self):
record = csw.catalogue.records.values()[0]

# test CRS constructs in Dublin Core
self.assertEqual(
record.bbox.crs.code,
4326,
'Expected a specific CRS code value in Dublin Core model')
self.assertEquals(record.bbox.crs.code, 4326)
# test BBOX properties in Dublin Core
from decimal import Decimal
logger.debug([Decimal(record.bbox.minx), Decimal(record.bbox.miny),
Decimal(record.bbox.maxx), Decimal(record.bbox.maxy)])
self.assertEqual(
Decimal(record.bbox.minx),
Decimal('-81.8593555'),
'Expected a specific minx coordinate value in Dublin Core model')
self.assertEqual(
Decimal(record.bbox.miny),
Decimal('12.1665322'),
'Expected a specific minx coordinate value in Dublin Core model')
self.assertEqual(
Decimal(record.bbox.maxx),
Decimal('-81.356409'),
'Expected a specific maxx coordinate value in Dublin Core model')
self.assertEqual(
Decimal(record.bbox.maxy),
Decimal('13.396306'),
'Expected a specific maxy coordinate value in Dublin Core model')
self.assertEquals(Decimal(record.bbox.minx), Decimal('-81.8593555'))
self.assertEquals(Decimal(record.bbox.miny), Decimal('12.1665322'))
self.assertEquals(Decimal(record.bbox.maxx), Decimal('-81.356409'))
self.assertEquals(Decimal(record.bbox.maxy), Decimal('13.396306'))

def test_csw_outputschema_fgdc(self):
"""Verify that GeoNode CSW can handle ISO metadata with FGDC outputSchema"""
Expand All @@ -279,27 +244,18 @@ def test_csw_outputschema_fgdc(self):
record = csw.catalogue.records.values()[0]

# test that the ISO title maps correctly in FGDC
self.assertEqual(
record.idinfo.citation.citeinfo['title'],
'San Andres Y Providencia Location',
'Expected a specific title in FGDC model')
self.assertEquals(record.idinfo.citation.citeinfo['title'], 'San Andres Y Providencia Location')

# test that the ISO abstract maps correctly in FGDC
self.assertEqual(
record.idinfo.descript.abstract,
'No abstract provided',
'Expected a specific abstract in FGDC model')
self.assertEquals(record.idinfo.descript.abstract, None)

def test_csw_query_bbox(self):
"""Verify that GeoNode CSW can handle bbox queries"""

csw = get_catalogue()
csw.catalogue.getrecords(bbox=[-140, -70, 80, 70])
logger.debug(csw.catalogue.results)
self.assertEqual(
csw.catalogue.results,
{'matches': 7, 'nextrecord': 0, 'returned': 7},
'Expected a specific bbox query result set')
self.assertEquals(csw.catalogue.results, {'matches': 7, 'nextrecord': 0, 'returned': 7})

def test_csw_upload_fgdc(self):
"""Verify that GeoNode CSW can handle FGDC metadata upload"""
Expand All @@ -324,84 +280,48 @@ def test_csw_upload_fgdc(self):
record=md_doc)

# test that FGDC document was successfully inserted
self.assertEqual(
csw.catalogue.results['inserted'],
1,
'Expected 1 inserted record in FGDC model')
self.assertEquals(csw.catalogue.results['inserted'], 1)

# query against FGDC typename, output FGDC
csw.catalogue.getrecords(typenames='fgdc:metadata')
self.assertEqual(
csw.catalogue.results['matches'],
1,
'Expected 1 record in FGDC model')
self.assertEquals(csw.catalogue.results['matches'], 1)

record = csw.catalogue.records.values()[0]

# test that the FGDC title maps correctly in DC
self.assertEqual(
record.title,
'Census_Blockgroup_Pop_Housing',
'Expected a specific title in DC model')
self.assertEquals(record.title, 'Census_Blockgroup_Pop_Housing')

# test that the FGDC type maps correctly in DC
self.assertEqual(
record.type,
'vector digital data',
'Expected a specific type in DC model')
self.assertEquals(record.type, 'vector digital data')

# test CRS constructs in Dublin Core
self.assertEqual(
record.bbox.crs.code,
4326,
'Expected a specific CRS code value in Dublin Core model')
self.assertEquals(record.bbox.crs.code, 4326)

# test BBOX properties in Dublin Core
from decimal import Decimal
self.assertEqual(
Decimal(record.bbox.minx),
Decimal('-117.6'),
'Expected a specific minx coordinate value in Dublin Core model')
self.assertEqual(
Decimal(record.bbox.miny),
Decimal('32.53'),
'Expected a specific minx coordinate value in Dublin Core model')
self.assertEqual(
Decimal(record.bbox.maxx),
Decimal('-116.08'),
'Expected a specific maxx coordinate value in Dublin Core model')
self.assertEqual(
Decimal(record.bbox.maxy),
Decimal('33.51'),
'Expected a specific maxy coordinate value in Dublin Core model')
self.assertEquals(Decimal(record.bbox.minx), Decimal('-117.6'))
self.assertEquals(Decimal(record.bbox.miny), Decimal('32.53'))
self.assertEquals(Decimal(record.bbox.maxx), Decimal('-116.08'))
self.assertEquals(Decimal(record.bbox.maxy), Decimal('33.51'))

# query against FGDC typename, return in ISO
csw.catalogue.getrecords(
typenames='fgdc:metadata',
esn='brief',
outputschema='http://www.isotc211.org/2005/gmd')
self.assertEqual(
csw.catalogue.results['matches'],
1,
'Expected 1 record in ISO model')
self.assertEquals(csw.catalogue.results['matches'], 1)

record = csw.catalogue.records.values()[0]

# test that the FGDC title maps correctly in ISO
self.assertEqual(
record.identification.title,
'Census_Blockgroup_Pop_Housing',
'Expected a specific title in ISO model')
self.assertEquals(record.identification.title, 'Census_Blockgroup_Pop_Housing')

# cleanup and delete inserted FGDC metadata document
csw.catalogue.transaction(
ttype='delete',
typename='fgdc:metadata',
cql='fgdc:Title like "Census_Blockgroup_Pop_Housing"')
self.assertEqual(
csw.catalogue.results['deleted'],
1,
'Expected 1 deleted record in FGDC model')
self.assertEquals(csw.catalogue.results['deleted'], 1)

def test_csw_bulk_upload(self):
"""Verify that GeoNode CSW can handle bulk upload of ISO and FGDC metadata"""
Expand Down

0 comments on commit 5ee005c

Please sign in to comment.