Skip to content

Commit

Permalink
Move GeoWave into a separate executor, rm BlockingThreadPool implemen…
Browse files Browse the repository at this point in the history
…tation from benchmarks, generate missing headers
  • Loading branch information
pomadchin committed Apr 30, 2021
1 parent 2c1246b commit 8241774
Show file tree
Hide file tree
Showing 23 changed files with 280 additions and 82 deletions.
3 changes: 1 addition & 2 deletions .circleci/build-and-test-cassandra.sh
Original file line number Diff line number Diff line change
Expand Up @@ -4,5 +4,4 @@

./sbt -Dsbt.supershell=false "++$SCALA_VERSION" \
"project cassandra" test \
"project cassandra-spark" test \
"project geowave" test || { exit 1; }
"project cassandra-spark" test || { exit 1; }
6 changes: 6 additions & 0 deletions .circleci/build-and-test-geowave.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,6 @@
#!/bin/bash

.circleci/unzip-rasters.sh

./sbt -Dsbt.supershell=false "++$SCALA_VERSION" \
"project geowave" test || { exit 1; }
25 changes: 25 additions & 0 deletions .circleci/config.yml
Original file line number Diff line number Diff line change
Expand Up @@ -137,6 +137,21 @@ jobs:
.circleci/build-and-test-accumulo.sh
- save_cache: *save_build_cache

geowave:
parameters:
scala-version:
type: string
executor: executor-cassandra
steps:
- checkout
- restore_cache: *restore_build_cache
- run:
name: Test Cassandra
command: |
export SCALA_VERSION=<< parameters.scala-version >>
.circleci/build-and-test-geowave.sh
- save_cache: *save_build_cache

scaladocs:
parameters:
scala-version:
Expand Down Expand Up @@ -223,6 +238,16 @@ workflows:
tags:
only: /^v.*/

- geowave:
matrix:
parameters:
scala-version: [ "2.12.13", "2.13.5" ]
filters:
branches:
only: /.*/
tags:
only: /^v.*/

- scaladocs:
matrix:
parameters:
Expand Down
2 changes: 1 addition & 1 deletion CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -18,8 +18,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
- GDALWarpOptions incorrect rpc flag [#3370](https://github.com/locationtech/geotrellis/issues/3370)
- S3LayerDeleter cannot handle over 1000 objects to delete [#3371](https://github.com/locationtech/geotrellis/issues/3371)
- Drop Scala 2.11 cross compilation [#3259](https://github.com/locationtech/geotrellis/issues/3259)
- Replace geowave subproject with GeoTrellis/GeoWave data adapter [#3364](https://github.com/locationtech/geotrellis/pull/3364)
- Fix MosaicRasterSource.tileToLayout behavior [#3338](https://github.com/locationtech/geotrellis/pull/3338)
- Replace geowave subproject with GeoTrellis/GeoWave data adapter [#3364](https://github.com/locationtech/geotrellis/pull/3364)

## [3.5.2] - 2021-02-01

Expand Down
8 changes: 3 additions & 5 deletions geowave/benchmark/src/main/resources/application.conf
Original file line number Diff line number Diff line change
Expand Up @@ -7,8 +7,6 @@ geotrellis.geowave.connection.store {
}
}

geotrellis.geowave {
blocking-thread-pool {
threads = default
}
}
geotrellis.blocking-thread-pool {
threads = default
}
Original file line number Diff line number Diff line change
@@ -1,12 +1,30 @@
/*
* Copyright 2021 Azavea
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/

package geotrellis.geowave

import cats.effect.{ContextShift, IO, Timer}
import geotrellis.store.util.BlockingThreadPool

import scala.util.Properties

trait BenchmarkEnvironment {
val kafka: String = Properties.envOrElse("KAFKA_HOST", "localhost:9092")
val cassandra: String = Properties.envOrElse("CASSANDRA_HOST", "localhost")

implicit val contextShift: ContextShift[IO] = BlockingThreadPool.contextShiftIO
implicit val timer: Timer[IO] = BlockingThreadPool.timerIO
implicit val contextShift: ContextShift[IO] = IO.contextShift(BlockingThreadPool.executionContext)
implicit val timer: Timer[IO] = IO.timer(BlockingThreadPool.executionContext)
}

This file was deleted.

Original file line number Diff line number Diff line change
@@ -1,3 +1,19 @@
/*
* Copyright 2021 Azavea
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/

package geotrellis.geowave

import geotrellis.geowave.dsl.syntax._
Expand All @@ -13,6 +29,7 @@ import cats.effect.IO
import geotrellis.geowave.adapter.geotiff.GeoTiffAdapter
import geotrellis.raster.io.geotiff.GeoTiff
import geotrellis.raster.MultibandTile
import geotrellis.store.util.BlockingThreadPool

object IngestBenchmarkData extends BenchmarkEnvironment {
def main(args: Array[String]): Unit = {
Expand Down
Original file line number Diff line number Diff line change
@@ -1,3 +1,19 @@
/*
* Copyright 2021 Azavea
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/

package geotrellis.geowave

import geotrellis.geowave.dsl.syntax._
Expand All @@ -7,6 +23,9 @@ import geotrellis.geowave.index.query.ExplicitSpatialTemporalElevationQuery
import geotrellis.geowave.adapter.TypeName
import geotrellis.geowave.api._
import geotrellis.vector._
import geotrellis.raster.io.geotiff.GeoTiff
import geotrellis.raster.MultibandTile
import geotrellis.store.util.BlockingThreadPool

import org.locationtech.geowave.core.geotime.index.api.{SpatialIndexBuilder, SpatialTemporalIndexBuilder}
import org.locationtech.geowave.core.store.api.{DataStore, DataStoreFactory, Index}
Expand All @@ -24,8 +43,6 @@ import scala.collection.JavaConverters._
import java.util.concurrent.TimeUnit
import java.time.{LocalDate, ZoneOffset}
import java.util.Date
import geotrellis.raster.io.geotiff.GeoTiff
import geotrellis.raster.MultibandTile

@BenchmarkMode(Array(Mode.AverageTime))
@State(Scope.Benchmark)
Expand Down
16 changes: 16 additions & 0 deletions geowave/src/test/scala/geotrellis/geowave/TestEnvironment.scala
Original file line number Diff line number Diff line change
@@ -1,3 +1,19 @@
/*
* Copyright 2021 Azavea
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/

package geotrellis.geowave

import org.scalatest.funspec.AnyFunSpec
Expand Down
Original file line number Diff line number Diff line change
@@ -1,3 +1,19 @@
/*
* Copyright 2021 Azavea
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/

package geotrellis.geowave.adapter

import geotrellis.geowave.dsl.syntax._
Expand Down
Original file line number Diff line number Diff line change
@@ -1,3 +1,19 @@
/*
* Copyright 2021 Azavea
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/

package geotrellis.geowave.adapter.geotiff

import geotrellis.geowave.dsl.syntax._
Expand Down
Original file line number Diff line number Diff line change
@@ -1,3 +1,19 @@
/*
* Copyright 2021 Azavea
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/

package geotrellis.geowave.adapter.geotiff

import geotrellis.geowave.dsl.syntax._
Expand Down
Original file line number Diff line number Diff line change
@@ -1,3 +1,19 @@
/*
* Copyright 2021 Azavea
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/

package geotrellis.geowave.adapter.geotiff

import geotrellis.geowave.dsl.syntax._
Expand Down
Original file line number Diff line number Diff line change
@@ -1,3 +1,19 @@
/*
* Copyright 2021 Azavea
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/

package geotrellis.geowave.adapter.geotiff

import geotrellis.geowave.dsl.syntax._
Expand Down
Original file line number Diff line number Diff line change
@@ -1,3 +1,19 @@
/*
* Copyright 2021 Azavea
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/

package geotrellis.geowave.adapter.raster

import geotrellis.geowave.dsl.syntax._
Expand Down
Original file line number Diff line number Diff line change
@@ -1,3 +1,19 @@
/*
* Copyright 2021 Azavea
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/

package geotrellis.geowave.adapter.raster

import geotrellis.geowave.dsl.syntax._
Expand Down
Loading

0 comments on commit 8241774

Please sign in to comment.