From 5f1ae12b59f8faa3c2cedd09e37b3d56fcd19778 Mon Sep 17 00:00:00 2001 From: Jon Hill Date: Mon, 19 Nov 2018 08:13:33 +1100 Subject: [PATCH 1/6] Adding support for nans in the raster data --- hrds/raster_buffer.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/hrds/raster_buffer.py b/hrds/raster_buffer.py index 167c375..1afdbd1 100644 --- a/hrds/raster_buffer.py +++ b/hrds/raster_buffer.py @@ -105,8 +105,10 @@ def make_buffer(self, output_file): # fill with edge value dist = np.full((ncols, nrows), 0.0) - # then fill in the middle + # then fill in the middle, + # except where there is no data dist[1:-1, 1:-1] = 1 + dist[ self.raster.get_array == self.raster.ds.GetNoDataValue() ] = 0 # calc euclidian distance and convert to 0 -> 1 scale dist = distance_transform_edt(dist, sampling=[dx[0], dx[1]]) dist = dist / self.distance From 9081e15d34ea5256f660592f422a71883a3c5a78 Mon Sep 17 00:00:00 2001 From: Jon Hill Date: Mon, 19 Nov 2018 08:26:45 +1100 Subject: [PATCH 2/6] the raster holds the nodata val, not the data source --- hrds/raster.py | 2 ++ hrds/raster_buffer.py | 4 +++- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/hrds/raster.py b/hrds/raster.py index 7309967..a318cea 100644 --- a/hrds/raster.py +++ b/hrds/raster.py @@ -141,6 +141,7 @@ def __init__(self, filename): self.interpolator = None self.extent = None self.dx = 0.0 + self.nodata = None def get_extent(self): """Return list of corner coordinates from a geotransform @@ -175,6 +176,7 @@ def set_band(self, band_no=1): Usually 1, which is default""" self.band = band_no raster = self.ds.GetRasterBand(self.band) + self.nodata = raster.GetNoDataValue() self.val = np.flipud(np.array(raster.ReadAsArray())) self.extent = self.get_extent() origin = np.amin(self.extent, axis=0) diff --git a/hrds/raster_buffer.py b/hrds/raster_buffer.py index 1afdbd1..29dbc14 100644 --- a/hrds/raster_buffer.py +++ b/hrds/raster_buffer.py @@ -108,7 +108,9 @@ def make_buffer(self, output_file): # then fill in the middle, # except where there is no data dist[1:-1, 1:-1] = 1 - dist[ self.raster.get_array == self.raster.ds.GetNoDataValue() ] = 0 + orig_raster = self.raster.get_array() + nodata = self.raster.nodata + dist[ orig_raster == nodata ] = 0 # calc euclidian distance and convert to 0 -> 1 scale dist = distance_transform_edt(dist, sampling=[dx[0], dx[1]]) dist = dist / self.distance From 02a138637b2beb8b0cfa72e35080a690bac9db57 Mon Sep 17 00:00:00 2001 From: Jon Hill Date: Mon, 19 Nov 2018 08:58:55 +1100 Subject: [PATCH 3/6] Only deal with NaNs if buffer matches raster. Skipping test automatically --- hrds/raster_buffer.py | 8 +++++--- tests/test_hrds.py | 3 ++- tests/test_interpolation.py | 3 ++- 3 files changed, 9 insertions(+), 5 deletions(-) diff --git a/hrds/raster_buffer.py b/hrds/raster_buffer.py index 29dbc14..387797e 100644 --- a/hrds/raster_buffer.py +++ b/hrds/raster_buffer.py @@ -108,9 +108,11 @@ def make_buffer(self, output_file): # then fill in the middle, # except where there is no data dist[1:-1, 1:-1] = 1 - orig_raster = self.raster.get_array() - nodata = self.raster.nodata - dist[ orig_raster == nodata ] = 0 + if self.over == None: + orig_raster = self.raster.get_array() + nodata = self.raster.nodata + #TODO this will only work if dist and orig_raster are the same size + dist[ orig_raster == nodata ] = 0 # calc euclidian distance and convert to 0 -> 1 scale dist = distance_transform_edt(dist, sampling=[dx[0], dx[1]]) dist = dist / self.distance diff --git a/tests/test_hrds.py b/tests/test_hrds.py index 7078516..ae01c3f 100644 --- a/tests/test_hrds.py +++ b/tests/test_hrds.py @@ -54,7 +54,8 @@ def test_simple_rasters(self): for p,e in zip(points, expected): self.assertAlmostEqual(bathy.get_val(p),e,delta=0.1) -@unittest.skip("Skipping this by default. Uses proprietary data.") +@unittest.skipUnless(os.path.isfile("tests/real_data/gebco_uk.tif"), + "Skipping as proprietary data missing.") class RealDataTest(unittest.TestCase): def test_real_world(self): diff --git a/tests/test_interpolation.py b/tests/test_interpolation.py index 8ff0794..2acc51c 100644 --- a/tests/test_interpolation.py +++ b/tests/test_interpolation.py @@ -78,7 +78,8 @@ def test_point_in(self): self.assertTrue(rci.point_in(point4)) self.assertFalse(rci.point_in(point5)) - @unittest.skip("Skipping this by default. Uses proprietary data.") + @unittest.skipUnless(os.path.isfile("tests/real_data/gebco_uk.tif"), + "Skipping as proprietary data missing.") def test_real_data(self): """Using gebco cut out to test interpolation """ From 8f8b3534e0a9b08f51e2c8a9010736f459f63bde Mon Sep 17 00:00:00 2001 From: Jon Hill Date: Mon, 19 Nov 2018 09:00:13 +1100 Subject: [PATCH 4/6] Fixing lint --- hrds/raster_buffer.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/hrds/raster_buffer.py b/hrds/raster_buffer.py index 387797e..180d8ad 100644 --- a/hrds/raster_buffer.py +++ b/hrds/raster_buffer.py @@ -105,14 +105,15 @@ def make_buffer(self, output_file): # fill with edge value dist = np.full((ncols, nrows), 0.0) - # then fill in the middle, - # except where there is no data + # then fill in the middle, + # except where there is no data dist[1:-1, 1:-1] = 1 - if self.over == None: + if self.over is None: orig_raster = self.raster.get_array() nodata = self.raster.nodata - #TODO this will only work if dist and orig_raster are the same size - dist[ orig_raster == nodata ] = 0 + # TODO this will only work if dist and orig_raster + # are the same size + dist[orig_raster == nodata] = 0 # calc euclidian distance and convert to 0 -> 1 scale dist = distance_transform_edt(dist, sampling=[dx[0], dx[1]]) dist = dist / self.distance From 799220fc26489f7c725a34b6a8065ec8a59330ae Mon Sep 17 00:00:00 2001 From: Jon Hill Date: Mon, 19 Nov 2018 09:21:29 +1100 Subject: [PATCH 5/6] Adding test for dealing with nodata --- tests/test_buffer.py | 28 ++++++++++++++++++++++++++++ tests/test_raster_large_nodata.tif | Bin 0 -> 2360 bytes 2 files changed, 28 insertions(+) create mode 100644 tests/test_raster_large_nodata.tif diff --git a/tests/test_buffer.py b/tests/test_buffer.py index 5e00956..cc50379 100644 --- a/tests/test_buffer.py +++ b/tests/test_buffer.py @@ -19,6 +19,7 @@ # Copyright Jon Hill, University of York, jon.hill@york.ac.uk test_file_name1 = "tests/test_raster_large.tif" +test_file_name2 = "tests/test_raster_nodata_centre.tif" temp_file = "temp.tif" class TestBufferCreator(unittest.TestCase): @@ -87,5 +88,32 @@ def test_simple_distance_setres(self): self.assertEqual(rci.get_val(point3),1.0) self.assertAlmostEqual(rci.get_val(point4),0.5,delta=0.01) + def test_simple_distance_nan(self): + """ Very simple test with a raster object like thus: + 1 2 3 4 + 5 6 7 8 + 9 NAN NAN 12 + 13 14 15 16 + (on a 20x20 grid, not 4x4 as above...only so much space + in comments!). The slightly-left-of-centre has been + replaced by NaN + + LLC is 0,0 and upper right is 4,4. (hence dx is 0.2) + The data are stored in cell centres and we ask for a few coords. + We check that the value is 1 at the distance, 0 at the boundary + and, for this test, the value is 0 in the centre. + """ + rbuff = CreateBuffer(test_file_name2, 1.0) + point1 = [0.2, 3.85] # should return ~0 + point2 = [0.7, 2] # should be 0.4 + point3 = [1.7, 2] # should be 0 (in the centre) + rbuff.make_buffer(temp_file) + # we now read in the buffer using the rasterinterpolator class + rci = RasterInterpolator(temp_file) + rci.set_band() + self.assertAlmostEqual(rci.get_val(point1),0.0, delta=0.03) + self.assertAlmostEqual(rci.get_val(point2),0.4, delta=0.03) + self.assertEqual(rci.get_val(point3),0.0) + if __name__ == '__main__': unittest.main() diff --git a/tests/test_raster_large_nodata.tif b/tests/test_raster_large_nodata.tif new file mode 100644 index 0000000000000000000000000000000000000000..88d498d356ace9b5acad4689eee8c65f5ca5d600 GIT binary patch literal 2360 zcmeHIJxjw-6umDgHN_$+Xzid7a8s1h)RL{~B;ACHI=bklh&VdL4~Ubas8fgHsP-Q? z>f>hJK}G>^ucLJQ)Sz{x%L#TE@X|VUY~*UGRQr zkOh9kFNEvqk+09Du0wU&yY1^q75y97l7kmG4X=)hLvWa)fvv25h^Lf~`n#2a^%nB; z*{BlT9-rSY_s)2q|Gcx`+S}?lUi|`X&`g7(Ii4NC2bTz-rI79W!2kAVYCq6$sIhbEEjOMAy&O7FOr-f_P@1_#h2 zPN7Kz&?K^bY46xe=^gjdyP#D<{QNzoPt@d;_7T04QaV8T>6Cr|*D3u0u2b47j8eJ+ se%Ujqf=}N8=SwHGu0wU&`~N5Qy1P-Y*1VcqZ!}zQv*vk?!!_5VAB7)0&;S4c literal 0 HcmV?d00001 From 655f96d6784f16a8c5e234d0ff562ca6c45547c4 Mon Sep 17 00:00:00 2001 From: Jon Hill Date: Mon, 19 Nov 2018 09:29:31 +1100 Subject: [PATCH 6/6] Helps if you add he right file for the test --- ...data.tif => test_raster_nodata_centre.tif} | Bin 2360 -> 2360 bytes 1 file changed, 0 insertions(+), 0 deletions(-) rename tests/{test_raster_large_nodata.tif => test_raster_nodata_centre.tif} (81%) diff --git a/tests/test_raster_large_nodata.tif b/tests/test_raster_nodata_centre.tif similarity index 81% rename from tests/test_raster_large_nodata.tif rename to tests/test_raster_nodata_centre.tif index 88d498d356ace9b5acad4689eee8c65f5ca5d600..a3cd4df14c5899b6b8bb0bd0a5615054347fa1c4 100644 GIT binary patch delta 173 pcmdlXv_ojaCT5xc|Lgzb!jtuwO(;~5$3nn_eFPM2=3~`j0RVEBgCGC^ delta 28 jcmdlXv_ojaCg#aP%nN|nZt?^cp~+6H_ck}M?qCK0m0}7E