Merge branch 'master' into geowave-adapter

locationtech · Apr 8, 2021 · 7fc4a7c · 7fc4a7c
2 parents 84825f7 + 3da1236
commit 7fc4a7c
Show file tree

Hide file tree

Showing 21 changed files with 109 additions and 61 deletions.
diff --git a/.locationtech/deploy-211.sh b/.locationtech/deploy-211.sh
diff --git a/.locationtech/deploy.sh b/.locationtech/deploy.sh
@@ -3,4 +3,4 @@
  set -e
  set -x
 
-./.locationtech/deploy-211.sh && ./.locationtech/deploy-212.sh
+./.locationtech/deploy-212.sh
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -17,6 +17,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 - S3LayerDeleter cannot handle over 1000 objects to delete [#3371](https://github.com/locationtech/geotrellis/issues/3371)
 - Drop Scala 2.11 cross compilation [#3259](https://github.com/locationtech/geotrellis/issues/3259)
 - Replace geowave subproject with GeoTrellis/GeoWave data adapter [#3364](https://github.com/locationtech/geotrellis/pull/3364)
+- Fix MosaicRasterSource.tileToLayout behavior [#3338](https://github.com/locationtech/geotrellis/pull/3338)
+
 
 ## [3.5.2] - 2021-02-01
 

diff --git a/layer/src/main/scala/geotrellis/layer/LayoutTileSource.scala b/layer/src/main/scala/geotrellis/layer/LayoutTileSource.scala
@@ -82,10 +82,22 @@ class LayoutTileSource[K: SpatialComponent](
       if (raster.tile.cols == layout.tileCols && raster.tile.rows == layout.tileRows) {
         raster.tile
       } else {
-        // raster is smaller but not bigger than I think ...
-        // its offset is relative to the raster we wished we had
-        val colOffset = bounds.colMin - sourcePixelBounds.colMin
-        val rowOffset = bounds.rowMin - sourcePixelBounds.rowMin
+        /** Raster is smaller but not bigger than I think ...
+          * Its offset is relative to the raster we wished we had.
+          * When the source represents a single [[RasterSource]], it is possible co compute offsets this way:
+          *   val colOffset = bounds.colMin - sourcePixelBounds.colMin
+          *   val rowOffset = bounds.rowMin - sourcePixelBounds.rowMin
+          *
+          * However, if the source is a [[MosaicRasterSource]], offsets are still relative to the underlying [[RasterSource]].
+          * In this case it is possible to compute offsets through the returned extent.
+          */
+
+        // the actual returned extent
+        val actualExtent = raster.extent
+        // expected extent
+        val expectedRasterExtent = RasterExtent(layout.mapTransform(spatialComponent), layout.tileCols, layout.tileRows)
+        val (colOffset, rowOffset) = expectedRasterExtent.mapToGrid(actualExtent.xmin, actualExtent.ymax)
+
         raster.tile.mapBands { (_, band) =>
           PaddedTile(band, colOffset.toInt, rowOffset.toInt, layout.tileCols, layout.tileRows)
         }

diff --git a/layer/src/main/scala/geotrellis/layer/buffer/BufferTiles.scala b/layer/src/main/scala/geotrellis/layer/buffer/BufferTiles.scala
@@ -84,7 +84,7 @@ trait BufferTiles {
     addSlice(SpatialKey(col+1, row+1), TopLeft)
     addSlice(SpatialKey(col-1, row+1), TopRight)
 
-    parts
+    parts.toSeq
   }
 
   def bufferWithNeighbors[

diff --git a/layer/src/test/scala/geotrellis/layer/LayoutTileSourceSpec.scala b/layer/src/test/scala/geotrellis/layer/LayoutTileSourceSpec.scala
@@ -23,12 +23,15 @@ import geotrellis.raster.geotiff._
 import geotrellis.raster.io.geotiff.reader._
 import geotrellis.raster.resample._
 import geotrellis.vector._
+import geotrellis.raster.io.geotiff.AutoHigherResolution
 
-import java.io.File
+import cats.data.NonEmptyList
 
+import java.io.File
 import org.scalatest.funspec.AnyFunSpec
 
 class LayoutTileSourceSpec extends AnyFunSpec with RasterMatchers {
+
   /** Function to get paths from the raster/data dir.  */
   def rasterGeoTiffPath(name: String): String = {
     def baseDataPath = "raster/data"
@@ -268,13 +271,54 @@ class LayoutTileSourceSpec extends AnyFunSpec with RasterMatchers {
           .get
           .band(0)
         val arr = tile.toArray
-        val ones = tile.mapIfSet(i => 1).toArray()
-        ones.sum shouldBe(arr.size)
+        val ones = tile.mapIfSet(_ => 1).toArray()
+        ones.sum shouldBe arr.size
       }
 
       neighborhood map {
         case (x, y) => checkExtent(x, y, rs)
       }
     }
+
+    // https://github.com/locationtech/geotrellis/issues/3253
+    it("MosaicRasterSource should be properly tiled to layout") {
+      val ld = LayoutDefinition(
+        Extent(-2.003750834278925E7, -2.003750834278925E7, 2.003750834278925E7, 2.003750834278925E7),
+        TileLayout(256, 256, 256, 256)
+      )
+
+      // For SpatialKey(74, 96):
+      // rs1: GridBounds(133,0,388,12)
+      // rs2: GridBounds(355,105,512,360)
+      val rasterSources = List(
+        rasterGeoTiffPath("vlm/lc8-utm-1.tif"),
+        rasterGeoTiffPath("vlm/lc8-utm-2.tif")
+      ).map(GeoTiffRasterSource(_))
+
+      val mosaic = MosaicRasterSource.instance(NonEmptyList.fromListUnsafe(rasterSources), rasterSources.head.crs)
+
+      val layout = mosaic
+        .reproject(WebMercator)
+        .tileToLayout(ld, identity, NearestNeighbor, AutoHigherResolution)
+
+      val mosaicReprojected = mosaic
+        .reproject(WebMercator)
+        .resampleToGrid(ld, NearestNeighbor, AutoHigherResolution)
+
+      layout.keys.foreach { key =>
+        val extent = ld.mapTransform.keyToExtent(key)
+        val ltile = layout.read(key).map(_.band(0).toArrayTile)
+        val mtile = mosaicReprojected.read(extent.buffer(mosaicReprojected.cellSize.resolution / 2)).map(_.mapTile(_.band(0).toArrayTile))
+
+        (ltile, mtile) match {
+          case (Some(ltile), Some(mtile)) =>
+            val sample = Raster(ArrayTile.alloc(ltile.cellType, 256, 256), extent)
+            val rtile = sample.merge(mtile)
+            assertTilesEqual(ltile, rtile.tile)
+          case (None, None) =>
+          case (l, r) => throw new Exception(s"$l is not equal to $r")
+        }
+      }
+    }
   }
 }
diff --git a/proj4/src/main/scala/geotrellis/proj4/io/wkt/WKT.scala b/proj4/src/main/scala/geotrellis/proj4/io/wkt/WKT.scala
@@ -22,7 +22,7 @@ import scala.io.Source
 
 object WKT {
   private val wktResourcePath = "/proj4/wkt/epsg.properties"
-  lazy val parsed: Map[Int, WktCS] = records.mapValues(WKTParser.apply)
+  lazy val parsed: Map[Int, WktCS] = records.mapValues(WKTParser.apply).toMap
   lazy val projections: Set[WktCS] = parsed.values.toSet
   lazy val records: Map[Int, String] = parseWktEpsgResource
 

diff --git a/publish/publish-to-sonatype-211.sh b/publish/publish-to-sonatype-211.sh
diff --git a/publish/publish-to-sonatype.sh b/publish/publish-to-sonatype.sh
@@ -1,4 +1,3 @@
 #!/bin/bash
 
-./publish/publish-to-sonatype-211.sh
 ./publish/publish-to-sonatype-212.sh
diff --git a/raster/data/vlm/lc8-utm-2.tif b/raster/data/vlm/lc8-utm-2.tif
diff --git a/raster/src/main/scala/geotrellis/raster/GridBounds.scala b/raster/src/main/scala/geotrellis/raster/GridBounds.scala
@@ -162,7 +162,7 @@ case class GridBounds[@specialized(Int, Long) N: Integral](
       if(overlapRowMax < rowMax) {
         result += GridBounds(overlapColMin, overlapRowMax + 1, overlapColMax, rowMax)
       }
-      result
+      result.toSeq
     }
 
   /**

diff --git a/raster/src/main/scala/geotrellis/raster/GridExtent.scala b/raster/src/main/scala/geotrellis/raster/GridExtent.scala
@@ -90,8 +90,8 @@ class GridExtent[@specialized(Int, Long) N: Integral](
 
   /** Convert map coordinates (x, y) to grid coordinates (col, row). */
   final def mapToGrid(x: Double, y: Double): (N, N) = {
-    val col = floorWithTolerance((x - extent.xmin) / cellwidth).toInt
-    val row = floorWithTolerance((extent.ymax - y) / cellheight).toInt
+    val col = Integral[N].fromDouble(floorWithTolerance((x - extent.xmin) / cellwidth))
+    val row = Integral[N].fromDouble(floorWithTolerance((extent.ymax - y) / cellheight))
     (col, row)
   }
 

diff --git a/raster/src/main/scala/geotrellis/raster/MosaicRasterSource.scala b/raster/src/main/scala/geotrellis/raster/MosaicRasterSource.scala
@@ -93,7 +93,31 @@ abstract class MosaicRasterSource extends RasterSource {
   }
 
   def read(bounds: GridBounds[Long], bands: Seq[Int]): Option[Raster[MultibandTile]] = {
-    val rasters = sources map { _.read(bounds, bands) }
+    /** The passed bounds are relative to the [[MosaicRasterSource]] bounds.
+      * However, each [[RasterSource]] has its own [[GridBounds]].
+      * Before passing [[GridBounds]] into each underlying [[RasterSource]]
+      * we need to map them into the each [[RasterSource]] relative grid space.
+      *
+      * This is done by calculating the relative offset using each [[RasterSource]]
+      * underlying [[Extent]].
+
+      * @param gb     global bounds, relative to the [[MosaicRasterSource]]
+      * @param extent extent of the [[RasterSource]]
+      * @return       relative to the extent [[GridBounds]]
+      */
+    def relativeGridBounds(gb: GridBounds[Long], extent: Extent): GridBounds[Long] = {
+      val GridBounds(colMin, rowMin, colMax, rowMax) = gb
+      val (sourceColOffset, sourceRowOffset) = gridExtent.mapToGrid(extent.xmin, extent.ymax)
+
+      GridBounds(
+        colMin - sourceColOffset,
+        rowMin - sourceRowOffset,
+        colMax - sourceColOffset,
+        rowMax - sourceRowOffset
+      )
+    }
+
+    val rasters = sources.map { rs => rs.read(relativeGridBounds(bounds, rs.extent), bands) }
     rasters.reduce
   }
 

diff --git a/raster/src/main/scala/geotrellis/raster/rasterize/polygon/FractionalRasterizer.scala b/raster/src/main/scala/geotrellis/raster/rasterize/polygon/FractionalRasterizer.scala
@@ -30,7 +30,7 @@ object FractionalRasterizer {
 
   private type Segment = (Double, Double, Double, Double)
 
-  private def polygonToEdges(poly: Polygon, re: RasterExtent): Seq[Segment] = {
+  private def polygonToEdges(poly: Polygon, re: RasterExtent): mutable.ArrayBuffer[Segment] = {
 
     val arrayBuffer = mutable.ArrayBuffer.empty[Segment]
 

diff --git a/raster/src/main/scala/geotrellis/raster/rasterize/polygon/PolygonRasterizer.scala b/raster/src/main/scala/geotrellis/raster/rasterize/polygon/PolygonRasterizer.scala
@@ -72,7 +72,7 @@ object PolygonRasterizer {
 
   private def intervalDifference(a : List[Interval], b: Interval) : List[Interval] = a.flatMap(intervalDifference(_,b))
 
-  private def mergeIntervals(sortedIntervals : Seq[Interval]) : Array[Double] = {
+  private def mergeIntervals(sortedIntervals : mutable.ListBuffer[Interval]) : Array[Double] = {
     if (sortedIntervals.length > 0) {
       val head = sortedIntervals.head
       val stack = mutable.Stack(head._1, head._2)

diff --git a/store/src/main/scala/geotrellis/store/AttributeStore.scala b/store/src/main/scala/geotrellis/store/AttributeStore.scala
@@ -56,7 +56,7 @@ trait AttributeStore extends AttributeCaching with LayerAttributeStore {
    * This function should be re-implemented by AttributeStore subclasses so that
    * catalogs with large numbers of layers can be queried efficiently.
    */
-  def layersWithZoomLevels: Map[String, Seq[Int]] = layerIds.groupBy(_.name).mapValues(_.map(_.zoom))
+  def layersWithZoomLevels: Map[String, Seq[Int]] = layerIds.groupBy(_.name).mapValues(_.map(_.zoom)).toMap
 
   /** Return a sequence of available zoom levels for a named layer.
    *

diff --git a/store/src/main/scala/geotrellis/store/hadoop/formats/FilterMapFileInputFormat.scala b/store/src/main/scala/geotrellis/store/hadoop/formats/FilterMapFileInputFormat.scala
@@ -115,8 +115,8 @@ class FilterMapFileInputFormat() extends FileInputFormat[BigIntWritable, BytesWr
 
     val possibleMatches =
       FilterMapFileInputFormat
-        .mapFileRanges(dataFileStatus.asScala.map(_.getPath.getParent), conf)
-        .filter { case (file, iMin, iMax) =>
+        .mapFileRanges(dataFileStatus.asScala.map(_.getPath.getParent).toSeq, conf)
+        .filter { case (_, iMin: BigInt, iMax: BigInt) =>
           // both file ranges and query ranges are sorted, use in-sync traversal
           while (it.hasNext && it.head._2 < iMin) it.next
           if (it.hasNext) iMin <= it.head._2 && (iMax == -1 || it.head._1 <= iMax)

diff --git a/vector/src/main/scala/geotrellis/vector/triangulation/DelaunayTriangulation.scala b/vector/src/main/scala/geotrellis/vector/triangulation/DelaunayTriangulation.scala
@@ -501,7 +501,7 @@ case class DelaunayTriangulation(
     removeIncidentEdge(vi)
   }
 
-  private def removeVertexAndFill(vi: Int, tris: Map[(Int, Int, Int), HalfEdge[Int, Int]], bnd: Option[Int]): Seq[Int] = {
+  private def removeVertexAndFill(vi: Int, tris: Map[(Int, Int, Int), HalfEdge[Int, Int]], bnd: Option[Int]): ListBuffer[Int] = {
     val exteriorRing = ListBuffer.empty[Int]
 
     decoupleVertex(vi)

diff --git a/vector/src/main/scala/geotrellis/vector/voronoi/VoronoiDiagram.scala b/vector/src/main/scala/geotrellis/vector/voronoi/VoronoiDiagram.scala
@@ -66,7 +66,7 @@ object VoronoiDiagram {
 
   private final val EPSILON = 1e-10
 
-  private def cellBoundsNew(het: HalfEdgeTable, verts: Int => Coordinate, extent: Extent)(incidentEdge: Int): Seq[CellBound] = {
+  private def cellBoundsNew(het: HalfEdgeTable, verts: Int => Coordinate, extent: Extent)(incidentEdge: Int): ListBuffer[CellBound] = {
     import het._
 
     var e = incidentEdge
@@ -118,7 +118,7 @@ object VoronoiDiagram {
     l
   }
 
-  private def cellExtentIntersection(het: HalfEdgeTable, verts: Int => Coordinate, incidentEdge: Int)(cell: Seq[CellBound], extent: Extent) = {
+  private def cellExtentIntersection(het: HalfEdgeTable, verts: Int => Coordinate, incidentEdge: Int)(cell: ListBuffer[CellBound], extent: Extent) = {
     val Extent(xmin, ymin, xmax, ymax) = extent
     val expts = ListBuffer((xmin, ymin), (xmax, ymin), (xmax, ymax), (xmin, ymax)).map{ case (x, y) => new Coordinate(x, y) }
 
@@ -185,7 +185,7 @@ object VoronoiDiagram {
       None
     } else {
       clippedCorners += clippedCorners.head
-      val poly = Polygon(clippedCorners.map(Point(_)))
+      val poly = Polygon(clippedCorners.map(Point(_)).toSeq)
       Some(poly)
     }
   }

diff --git a/vectortile/src/main/scala/geotrellis/vectortile/internal/Command.scala b/vectortile/src/main/scala/geotrellis/vectortile/internal/Command.scala
@@ -68,7 +68,7 @@ private[vectortile] case object ClosePath extends Command
 /** Contains convenience functions for handling [[Command]]s. */
 private[vectortile] object Command {
   /** Attempt to parse a list of Command/Parameter Integers. */
-  def commands(cmds: Seq[Int]): ListBuffer[Command] = {
+  def commands(cmds: Seq[Int]): Seq[Command] = {
     @tailrec def work(cmds: Seq[Int], curr: ListBuffer[Command]): ListBuffer[Command] = cmds match {
       case Nil => curr
       case ns => (parseCmd(ns.head): @unchecked) match {
@@ -102,7 +102,7 @@ private[vectortile] object Command {
       }
     }
 
-    work(cmds, new ListBuffer[Command])
+    work(cmds, new ListBuffer[Command]).toSeq
   }
 
   /** Convert a list of parsed Commands back into their original Command

diff --git a/vectortile/src/main/scala/geotrellis/vectortile/internal/package.scala b/vectortile/src/main/scala/geotrellis/vectortile/internal/package.scala
@@ -199,7 +199,7 @@ package object internal {
     ): Seq[Command] = {
       def work(lines: Array[LineString]): Seq[Command] = {
         var curs: (Int, Int) = (0, 0)
-        var buff = new ListBuffer[Command]
+        val buff = new ListBuffer[Command]
 
         lines.foreach({l =>
           val diffs: Array[(Int, Int)] = collapse(
@@ -313,7 +313,7 @@ package object internal {
           buff.appendAll(Seq(MoveTo(Array(diffs.head)), LineTo(diffs.tail), ClosePath))
         })
 
-        buff
+        buff.toSeq
       }
 
       poly match {