diff --git a/.travis.yml b/.travis.yml index 08a53cd..883a606 100644 --- a/.travis.yml +++ b/.travis.yml @@ -12,17 +12,17 @@ before_deploy: deploy: - provider: script skip_cleanup: true - script: release-manager --config ./.travis/release.yml --check-version --make-version --make-artifact --upload-artifact + script: ./.travis/deploy_template.sh $TRAVIS_TAG on: tags: true - provider: script skip_cleanup: true - script: ./.travis/deploy_docker.sh $TRAVIS_TAG + script: release-manager --config ./.travis/release.yml --check-version --make-version --make-artifact --upload-artifact on: tags: true - provider: script skip_cleanup: true - script: ./.travis/deploy_template.sh $TRAVIS_TAG + script: ./.travis/deploy_docker.sh $TRAVIS_TAG on: tags: true env: diff --git a/.travis/deploy_docker.sh b/.travis/deploy_docker.sh index d9194d8..d41168b 100755 --- a/.travis/deploy_docker.sh +++ b/.travis/deploy_docker.sh @@ -13,7 +13,7 @@ cd ${TRAVIS_BUILD_DIR} project_version=$(sbt -no-colors version | perl -ne 'print "$1\n" if /info.*(\d+\.\d+\.\d+[^\r\n]*)/' | tail -n 1 | tr -d '\n') if [[ "${tag}" = "${project_version}" ]]; then sbt docker:publishLocal - docker push "${docker_repo}/snowplow/snowplow-cloud-storage-loader:${tag}" + docker push "${docker_repo}/snowplow/snowplow-google-cloud-storage-loader:${tag}" else echo "Tag version '${tag}' doesn't match version in scala project ('${project_version}'). aborting!" exit 1 diff --git a/.travis/deploy_template.sh b/.travis/deploy_template.sh index 1dc368e..23a1360 100755 --- a/.travis/deploy_template.sh +++ b/.travis/deploy_template.sh @@ -10,9 +10,9 @@ cd ${TRAVIS_BUILD_DIR} project_version=$(sbt -no-colors version | perl -ne 'print "$1\n" if /info.*(\d+\.\d+\.\d+[^\r\n]*)/' | tail -n 1 | tr -d '\n') if [[ "${tag}" = *"${project_version}" ]]; then - sbt "runMain com.snowplowanalytics.storage.cloudstorage.loader.CloudStorageLoader --project=engineering-sandbox \ - --templateLocation=gs://snowplow-hosted-assets/4-storage/cloud-storage-loader/${tag}/CloudStorageLoaderTemplate-${tag} \ - --stagingLocation=gs://snowplow-hosted-assets/4-storage/cloud-storage-loader/${tag}/staging \ + sbt "runMain com.snowplowanalytics.storage.googlecloudstorage.loader.CloudStorageLoader --project=engineering-sandbox \ + --templateLocation=gs://snowplow-hosted-assets/4-storage/snowplow-google-cloud-storage-loader/${tag}/SnowplowGoogleCloudStorageLoaderTemplate-${tag} \ + --stagingLocation=gs://snowplow-hosted-assets/4-storage/snowplow-google-cloud-storage-loader/${tag}/staging \ --runner=DataflowRunner \ --tempLocation=gs://snowplow-hosted-assets/tmp" else diff --git a/.travis/release.yml b/.travis/release.yml index 85aeb7d..4d37cc7 100644 --- a/.travis/release.yml +++ b/.travis/release.yml @@ -15,7 +15,7 @@ targets : # Required: packages to be deployed packages : - repo : "snowplow-generic" - name : "snowplow-cloud-storage-loader" + name : "snowplow-google-cloud-storage-loader" user_org : "snowplow" publish : true override : false @@ -26,7 +26,7 @@ packages : - sbt universal:packageBin artifacts : - type : "asis" - prefix : "snowplow_cloud_storage_loader_" + prefix : "snowplow_google_cloud_storage_loader_" suffix : ".zip" binary_paths : - - "target/universal/cloud-storage-loader-{{ packages.0.build_version }}.zip" + - "target/universal/snowplow-google-cloud-storage-loader-{{ packages.0.build_version }}.zip" diff --git a/README.md b/README.md index 43b39af..abab513 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ -# Cloud Storage Loader +# Snowplow Google Cloud Storage Loader ## Introduction @@ -15,9 +15,9 @@ additional flexibility when running your pipeline. To upload the template to your own bucket, run: ```bash -sbt "runMain com.snowplowanalytics.storage.cloudstorage.loader.CloudStorageLoader \ +sbt "runMain com.snowplowanalytics.storage.googlecloudstorage.loader.CloudStorageLoader \ --project=[PROJECT] \ - --templateLocation=gs://[BUCKET]/CloudStorageLoaderTemplate \ + --templateLocation=gs://[BUCKET]/SnowplowGoogleCloudStorageLoaderTemplate \ --stagingLocation=gs://[BUCKET]/staging \ --runner=DataflowRunner \ --tempLocation=gs://[BUCKET]/tmp" @@ -55,7 +55,7 @@ Here, we provide an example using `gcloud`: ```bash gcloud dataflow jobs run [JOB-NAME] \ - --gcs-location gs://snowplow-hosted-assets/4-storage/cloud-storage-loader/0.1.0/CloudStorageLoaderTemplate-0.1.0 \ + --gcs-location gs://sp-hosted-assets/4-storage/snowplow-google-cloud-storage-loader/0.1.0/SnowplowGoogleCloudStorageLoaderTemplate-0.1.0 \ --parameters \ inputSubscription=projects/[PROJECT]/subscriptions/[SUBSCRIPTION],\ outputDirectory=gs://[BUCKET]/YYYY/MM/dd/HH/,\ # partitions by date @@ -67,14 +67,14 @@ gcloud dataflow jobs run [JOB-NAME] \ numShards=1 # optional ``` -### Directly +### Through the zip archive You can find the archive hosted on [our Bintray][bintray]. Once unzipped the artifact can be run as follows: ```bash -./bin/cloud-storage-loader \ +./bin/snowplow-google-cloud-storage-loader \ --runner=DataFlowRunner \ --project=[PROJECT] \ --streaming=true \ @@ -92,25 +92,28 @@ Once unzipped the artifact can be run as follows: To display the help message: ```bash -./bin/cloud-storage-loader --help +./bin/snowplow-google-cloud-storage-loader --help ``` To display documentation about Cloud Storage Loader-specific options: ```bash -./bin/cloud-storage-loader --help=com.snowplowanalytics.storage.cloudstorage.loader.Options +./bin/snowplow-google-cloud-storage-loader --help=com.snowplowanalytics.storage.googlecloudstorage.loader.Options ``` ### Through a docker container +You can also find the image on [our Bintray][bintray-docker]. + A container can be run as follows: ```bash docker run \ + -v $PWD/config:/snowplow/config \ -e GOOGLE_APPLICATION_CREDENTIALS=/snowplow/config/credentials.json \ # if running outside GCP - snowplow-docker-registry.bintray.io/snowplow/cloud-storage-loader:0.1.0 \ + snowplow-docker-registry.bintray.io/snowplow/snowplow-google-cloud-storage-loader:0.1.0 \ --runner=DataFlowRunner \ - --job-name=[JOB-NAME] \ + --jobName=[JOB-NAME] \ --project=[PROJECT] \ --streaming=true \ --zone=[ZONE] \ @@ -127,15 +130,15 @@ docker run \ To display the help message: ```bash -docker run snowplow-docker-registry.bintray.io/snowplow/cloud-storage-loader:0.1.0 \ +docker run snowplow-docker-registry.bintray.io/snowplow/snowplow-google-cloud-storage-loader:0.1.0 \ --help ``` To display documentation about Cloud Storage Loader-specific options: ```bash -docker run snowplow-docker-registry.bintray.io/snowplow/cloud-storage-loader:0.1.0 \ - --help=com.snowplowanalytics.storage.cloudstorage.loader.Options +docker run snowplow-docker-registry.bintray.io/snowplow/snowplow-google-cloud-storage-loader:0.1.0 \ + --help=com.snowplowanalytics.storage.googlecloudstorage.loader.Options ``` ### Additional information @@ -186,11 +189,12 @@ limitations under the License. [templates]: https://cloud.google.com/dataflow/docs/templates/overview [executing-templates]: https://cloud.google.com/dataflow/docs/templates/executing-templates -[bintray]: https://bintray.com/snowplow/snowplow-generic/snowplow-cloud-storage-loader +[bintray]: https://bintray.com/snowplow/snowplow-generic/snowplow-google-cloud-storage-loader +[bintray-docker]: https://bintray.com/snowplow/registry/snowplow%3Asnowplow-google-cloud-storage-loader [license]: http://www.apache.org/licenses/LICENSE-2.0 [techdocs-image]: https://d3i6fms1cm1j0i.cloudfront.net/github/images/techdocs.png [setup-image]: https://d3i6fms1cm1j0i.cloudfront.net/github/images/setup.png -[techdocs]: https://github.com/snowplow/snowplow/wiki/Cloud-Storage-Loader -[setup]: https://github.com/snowplow/snowplow/wiki/setting-up-cloud-storage-loader +[techdocs]: https://github.com/snowplow/snowplow/wiki/Snowplow-Google-Cloud-Storage-Loader +[setup]: https://github.com/snowplow/snowplow/wiki/setting-up-snowplow-google-cloud-storage-loader diff --git a/build.sbt b/build.sbt index 83b618c..bd32c87 100644 --- a/build.sbt +++ b/build.sbt @@ -51,7 +51,7 @@ lazy val root: Project = project .settings(commonSettings) .settings(macroSettings) .settings( - name := "cloud-storage-loader", + name := "snowplow-google-cloud-storage-loader", description := "Snowplow Google Cloud Storage Loader", publish / skip := true, libraryDependencies ++= Seq( @@ -71,7 +71,7 @@ lazy val repl: Project = project .settings(macroSettings) .settings( name := "repl", - description := "Scio REPL for snowplow-cloud-storage-loader", + description := "Scio REPL for snowplow-google-cloud-storage-loader", libraryDependencies ++= Seq( "com.spotify" %% "scio-repl" % scioVersion ), diff --git a/src/main/scala/com/snowplowanalytics/storage/cloudstorage/loader/CloudStorageLoader.scala b/src/main/scala/com/snowplowanalytics/storage/googlecloudstorage/loader/CloudStorageLoader.scala similarity index 98% rename from src/main/scala/com/snowplowanalytics/storage/cloudstorage/loader/CloudStorageLoader.scala rename to src/main/scala/com/snowplowanalytics/storage/googlecloudstorage/loader/CloudStorageLoader.scala index f628998..c7d0bcf 100644 --- a/src/main/scala/com/snowplowanalytics/storage/cloudstorage/loader/CloudStorageLoader.scala +++ b/src/main/scala/com/snowplowanalytics/storage/googlecloudstorage/loader/CloudStorageLoader.scala @@ -12,7 +12,7 @@ * See the Apache License Version 2.0 for the specific language governing permissions and * limitations there under. */ -package com.snowplowanalytics.storage.cloudstorage.loader +package com.snowplowanalytics.storage.googlecloudstorage.loader import com.spotify.scio._ import org.apache.beam.sdk.io.{Compression, FileBasedSink, TextIO} diff --git a/src/main/scala/com/snowplowanalytics/storage/cloudstorage/loader/Options.scala b/src/main/scala/com/snowplowanalytics/storage/googlecloudstorage/loader/Options.scala similarity index 92% rename from src/main/scala/com/snowplowanalytics/storage/cloudstorage/loader/Options.scala rename to src/main/scala/com/snowplowanalytics/storage/googlecloudstorage/loader/Options.scala index d3c6049..b9c21d5 100644 --- a/src/main/scala/com/snowplowanalytics/storage/cloudstorage/loader/Options.scala +++ b/src/main/scala/com/snowplowanalytics/storage/googlecloudstorage/loader/Options.scala @@ -12,14 +12,14 @@ * See the Apache License Version 2.0 for the specific language governing permissions and * limitations there under. */ -package com.snowplowanalytics.storage.cloudstorage.loader +package com.snowplowanalytics.storage.googlecloudstorage.loader import org.apache.beam.sdk.options._ import org.apache.beam.sdk.options.Validation.Required /** Trait regrouping the loader's configuration. */ trait Options extends PipelineOptions with StreamingOptions { - @Description("The Cloud Pub/Sub subscription to read from") + @Description("The Cloud Pub/Sub subscription to read from, formatted as projects/[PROJECT]/subscriptions/[SUB]") @Required def getInputSubscription: ValueProvider[String] def setInputSubscription(value: ValueProvider[String]): Unit diff --git a/src/main/scala/com/snowplowanalytics/storage/cloudstorage/loader/WindowedFilenamePolicy.scala b/src/main/scala/com/snowplowanalytics/storage/googlecloudstorage/loader/WindowedFilenamePolicy.scala similarity index 98% rename from src/main/scala/com/snowplowanalytics/storage/cloudstorage/loader/WindowedFilenamePolicy.scala rename to src/main/scala/com/snowplowanalytics/storage/googlecloudstorage/loader/WindowedFilenamePolicy.scala index e9b4cc4..f4c5c8f 100644 --- a/src/main/scala/com/snowplowanalytics/storage/cloudstorage/loader/WindowedFilenamePolicy.scala +++ b/src/main/scala/com/snowplowanalytics/storage/googlecloudstorage/loader/WindowedFilenamePolicy.scala @@ -12,7 +12,7 @@ * See the Apache License Version 2.0 for the specific language governing permissions and * limitations there under. */ -package com.snowplowanalytics.storage.cloudstorage.loader +package com.snowplowanalytics.storage.googlecloudstorage.loader import org.apache.beam.sdk.io.DefaultFilenamePolicy import org.apache.beam.sdk.io.FileBasedSink.{FilenamePolicy, OutputFileHints} diff --git a/src/test/scala/com/snowplowanalytics/storage/cloudstorage/loader/CloudStorageLoaderSpec.scala b/src/test/scala/com/snowplowanalytics/storage/googlecloudstorage/loader/CloudStorageLoaderSpec.scala similarity index 94% rename from src/test/scala/com/snowplowanalytics/storage/cloudstorage/loader/CloudStorageLoaderSpec.scala rename to src/test/scala/com/snowplowanalytics/storage/googlecloudstorage/loader/CloudStorageLoaderSpec.scala index 986f5d0..3d95eb7 100644 --- a/src/test/scala/com/snowplowanalytics/storage/cloudstorage/loader/CloudStorageLoaderSpec.scala +++ b/src/test/scala/com/snowplowanalytics/storage/googlecloudstorage/loader/CloudStorageLoaderSpec.scala @@ -12,7 +12,7 @@ * See the Apache License Version 2.0 for the specific language governing permissions and * limitations there under. */ -package com.snowplowanalytics.storage.cloudstorage.loader +package com.snowplowanalytics.storage.googlecloudstorage.loader import com.spotify.scio.testing._ diff --git a/src/test/scala/com/snowplowanalytics/storage/cloudstorage/loader/WindowedFilenamePolicySpec.scala b/src/test/scala/com/snowplowanalytics/storage/googlecloudstorage/loader/WindowedFilenamePolicySpec.scala similarity index 98% rename from src/test/scala/com/snowplowanalytics/storage/cloudstorage/loader/WindowedFilenamePolicySpec.scala rename to src/test/scala/com/snowplowanalytics/storage/googlecloudstorage/loader/WindowedFilenamePolicySpec.scala index 79c36f4..6d88cb0 100644 --- a/src/test/scala/com/snowplowanalytics/storage/cloudstorage/loader/WindowedFilenamePolicySpec.scala +++ b/src/test/scala/com/snowplowanalytics/storage/googlecloudstorage/loader/WindowedFilenamePolicySpec.scala @@ -12,7 +12,7 @@ * See the Apache License Version 2.0 for the specific language governing permissions and * limitations there under. */ -package com.snowplowanalytics.storage.cloudstorage.loader +package com.snowplowanalytics.storage.googlecloudstorage.loader import java.io.File import java.nio.file.Files