diff --git a/.github/workflows/build-push.yaml b/.github/workflows/build-push.yaml new file mode 100644 index 00000000..f7d3dc92 --- /dev/null +++ b/.github/workflows/build-push.yaml @@ -0,0 +1,37 @@ +name: Docker Publish + +on: + push: + branches: [ main ] + tags: + - '*' + +jobs: + docker_publish: + name: Build, Tag, and Push Image + runs-on: ubuntu-latest + + strategy: + matrix: + execution_ids: [ + "mapt-event-time-filter-function", "flat-map-stream", "map-flatmap", + "even-odd", "simple-sink", "reduce-sum", "reduce-stream-sum", + "map-forward-message", "reduce-counter", "sideinput-example", + "udf-sideinput-example", "source-simple-source", "session-reduce-count" + ] + + steps: + - name: Check out repository + uses: actions/checkout@v3 + - name: Set up QEMU + uses: docker/setup-qemu-action@v3 + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + - name: Login to Quay.io registry + uses: docker/login-action@v3 + with: + registry: quay.io + username: ${{ secrets.NUMAIO_USERNAME }} + password: ${{ secrets.NUMAIO_PASSWORD }} + - name: Build, tag, and push images + run: ./update_examples.sh --build-push-example ${{ matrix.execution_ids }} diff --git a/.github/workflows/ci.yaml b/.github/workflows/run-tests.yaml similarity index 100% rename from .github/workflows/ci.yaml rename to .github/workflows/run-tests.yaml diff --git a/development.md b/development.md new file mode 100644 index 00000000..ad3abf8c --- /dev/null +++ b/development.md @@ -0,0 +1,73 @@ +# Development + +This document explains the development process for the Numaflow Java SDK. + +### Testing + +After you make certain changes to the Java SDK implementation, you would like to use it to build a +brand new UDF image, and run it on a pipeline to verify the image continues working well with the Numaflow platform. +In order to do this you can follow these steps: + +1. Make the changes. +2. Update your local repository using maven, to reflect your latest changes. This can be done in the root +directory by running: `mvn clean install` (make sure that the version in the `pom.xml in the root and examples +directories match). +3. Build your test images, and test them in a pipeline. + +There exists a script `update_examples.sh`, which you can also use while testing, for your convenience. + +If you would like to build and push a specific example, you can run: + +```shell +./update_examples -bpe -t + ``` +The -bpe flag builds and pushes to the remote registry. must be +the id tag used within the execution element of the desired example. It is recommended to specify a tag, +like `test`, as the default is `stable`, which is used by the Github Actions CI. For example, in order to push +the example, for the reducer SDK, specifically the sum function, you would run: +```shell +./update_examples -bpe reduce-sum -t test +``` +If you would like to build and push all the examples at once, you can run: +```shell +./update_examples -bp -t test +``` +Both `bpe` and `bp` first build a local image with the naming convention +`numaflow-java-examples/:`, which then gets pushed as +`quay.io/numaio/numaflow-java/:`. If while testing, you would like +to build your image locally, without pushing to quay.io, you can run the following in the +examples directory: + +```shell +mvn jib:dockerBuild@ -Ddocker.tag= +``` + +If you would like to build all example images without pushing, in the examples directory you can also run: +```shell +mvn clean install -Ddocker.tag= +``` + +### Deploying + +After confirming that your changes pass local testing: + +1. Clean up testing artifacts. +2. Create a PR. Once your PR has been merged a Github Actions workflow will be triggered to build, tag (with `stable`), and push +all example images, so that they are using the most up-to-date version of the SDK, i.e. the one including your +changes. + +### Before Release + +Before releasing a new SDK version, make sure to update all references from the old version to the new one. +For example, the version in the `pom.xml` in the root and example directories. For [reference +](https://github.com/numaproj/numaflow-java/pull/89/files#diff-9c5fb3d1b7e3b0f54bc5c4182965c4fe1f9023d449017cece3005d3f90e8e4d8). + +### Adding a New Example + +If you add a new example, there are a few steps to follow in order for it to be used by the update script and the Docker +Publish workflow: + +1. Add the example to the `pom.xml` file in the examples directory, within an execution element. Note that the +id tag you specify must be exactly the same as the quay.io repository name for the example. +2. Add the id tag you specified in step 1 to the executionIDs array in `update_examples.sh`. +3. Add the id tag you specified in step 1 to the execution_ids matrix in `build-push.yaml`. diff --git a/examples/pom.xml b/examples/pom.xml index d9ed9fd5..51d3c6c1 100644 --- a/examples/pom.xml +++ b/examples/pom.xml @@ -10,6 +10,7 @@ UTF-8 + stable @@ -34,7 +35,7 @@ 3.3.0 - event-time-filter-function + mapt-event-time-filter-function package dockerBuild @@ -46,7 +47,7 @@ - numaflow-java-examples/event-time-filter + numaflow-java-examples/mapt-event-time-filter-function:${docker.tag} @@ -63,12 +64,12 @@ - numaflow-java-examples/flat-map-stream + numaflow-java-examples/flat-map-stream:${docker.tag} - flat-map + map-flatmap package dockerBuild @@ -80,7 +81,7 @@ - numaflow-java-examples/map-flatmap + numaflow-java-examples/map-flatmap:${docker.tag} @@ -97,7 +98,7 @@ - numaflow-java-examples/even-odd + numaflow-java-examples/even-odd:${docker.tag} @@ -113,7 +114,7 @@ - numaflow-java-examples/simple-sink + numaflow-java-examples/simple-sink:${docker.tag} @@ -130,7 +131,7 @@ - numaflow-java-examples/reduce-sum + numaflow-java-examples/reduce-sum:${docker.tag} @@ -147,7 +148,92 @@ - numaflow-java-examples/reduce-stream-sum + numaflow-java-examples/reduce-stream-sum:${docker.tag} + + + + + map-forward-message + package + + dockerBuild + + + + + io.numaproj.numaflow.examples.map.forward.ForwardFunction + + + + numaflow-java-examples/map-forward-message:${docker.tag} + + + + + reduce-counter + package + + dockerBuild + + + + + io.numaproj.numaflow.examples.reduce.count.CounterFactory + + + + numaflow-java-examples/reduce-counter:${docker.tag} + + + + + sideinput-example + package + + dockerBuild + + + + + io.numaproj.numaflow.examples.sideinput.simple.SimpleSideInput + + + + numaflow-java-examples/sideinput-example:${docker.tag} + + + + + udf-sideinput-example + package + + dockerBuild + + + + + io.numaproj.numaflow.examples.sideinput.udf.SimpleMapWithSideInput + + + + numaflow-java-examples/udf-sideinput-example:${docker.tag} + + + + + source-simple-source + package + + dockerBuild + + + + + io.numaproj.numaflow.examples.source.simple.SimpleSource + + + + numaflow-java-examples/source-simple-source:${docker.tag} @@ -164,7 +250,7 @@ - numaflow-java-examples/session-reduce-count + numaflow-java-examples/session-reduce-count:${docker.tag} diff --git a/update_examples.sh b/update_examples.sh new file mode 100755 index 00000000..b4c04cd1 --- /dev/null +++ b/update_examples.sh @@ -0,0 +1,136 @@ +#!/bin/bash + +function show_help () { + echo "Usage: $0 [-h|--help | -t|--tag ] (-bp|--build-push | -bpe|--build-push-example )" + echo " -h, --help Display help message and exit" + echo " -bp, --build-push Build all the examples and push them to the quay.io registry" + echo " -bpe, --build-push-example Build the given example id (found in examples/pom.xml), and push it to the quay.io registry" + echo " -t, --tag To be optionally used with -bpe or -bp. Specify the tag to build with. Default tag: stable" +} + +if [ $# -eq 0 ]; then + echo "Error: provide at least one argument" >&2 + show_help + exit 1 +fi + +usingHelp=0 +usingBuildPush=0 +usingBuildPushExample=0 +usingTag=0 +executionID="" +tag="stable" + +function handle_options () { + while [ $# -gt 0 ]; do + case "$1" in + -h | --help) + usingHelp=1 + ;; + -bp | --build-push) + usingBuildPush=1 + ;; + -bpe | --build-push-example) + if [ -z "$2" ]; then + echo "execution ID of example not specified." >&2 + show_help + exit 1 + fi + + usingBuildPushExample=1 + executionID=$2 + shift + ;; + -t | --tag) + if [ -z "$2" ]; then + echo "Tag not specified." >&2 + show_help + exit 1 + fi + + usingTag=1 + tag=$2 + shift + ;; + *) + echo "Invalid option: $1" >&2 + show_help + exit 1 + ;; + esac + shift + done +} + +handle_options "$@" + +if (( usingBuildPush + usingBuildPushExample + usingHelp > 1 )); then + echo "Only one of '-h', '-bp', or '-bpe' is allowed at a time" >&2 + show_help + exit 1 +fi + +if (( (usingTag + usingHelp > 1) || (usingTag && usingBuildPush + usingBuildPushExample == 0) )); then + echo "Can only use -t with -bp or -bpe" >&2 + show_help + exit 1 +fi + +if [ -n "$version" ]; then + echo "Using version: $version" +fi + +if [ -n "$executionID" ]; then + echo "Updating example: $executionID" +fi + +if [ -n "$tag" ] && (( ! usingHelp )); then + echo "Using tag: $tag" +fi + +executionIDs=("mapt-event-time-filter-function" "flat-map-stream" "map-flatmap" \ + "even-odd" "simple-sink" "reduce-sum" "reduce-stream-sum" \ + "map-forward-message" "reduce-counter" "sideinput-example" \ + "udf-sideinput-example" "source-simple-source" "session-reduce-count" + ) + +function dockerPublish () { + echo "Docker publish for example: $1" + if ! docker tag numaflow-java-examples/"$1":"$tag" quay.io/numaio/numaflow-java/"$1":"$tag"; then + echo "Error: failed to tag example $1 with tag $tag" >&2 + exit 1 + fi + if ! docker push quay.io/numaio/numaflow-java/"$1":"$tag"; then + echo "Error: failed to push example $1 with tag $tag" >&2 + exit 1 + fi +} + +if (( usingBuildPush )); then + if ! mvn clean install; then + echo "Error: failed to mvn clean install in root directory" >&2 + exit 1 + fi + cd examples || exit + if ! mvn clean install -Ddocker.tag="$tag"; then + echo "Error: failed to build images in examples directory" >&2 + exit 1 + fi + for id in "${executionIDs[@]}" + do + dockerPublish "$id" + done +elif (( usingBuildPushExample )); then + if ! mvn clean install; then + echo "Error: failed to mvn clean install in root directory" >&2 + exit 1 + fi + cd examples || exit + if ! mvn jib:dockerBuild@"$executionID" -Ddocker.tag="$tag"; then + echo "Error: failed to build example image $executionID" >&2 + exit 1 + fi + dockerPublish "$executionID" +elif (( usingHelp )); then + show_help +fi