config.yml

---
# CI Overview
# -----------
#
# Every commit:
#
# The CI for every PR and merge to main runs tests, fmt, lints and compiles debug binaries
#
# On main if all these checks pass it will then additionally compile in "release" mode and
# publish a docker image to quay.io/influxdb/influxdb3:$COMMIT_SHA
#
# Manually trigger build and push of container image for a branch:
#
# Navigate to https://app.circleci.com/pipelines/github/influxdata/influxdb?branch=<branch-name> (<- change this!)
# Then:
#
# - Click "Run Pipeline" in the top-right
# - Expand "Add Parameters"
# - Add a "boolean" parameter called "release_branch" with the value true
# - Click "Run Pipeline"
#
# You can also do this using the CircleCI API:
#
# Using `xh`:
#
#   # e.g. using 'xh' (https://github.com/ducaale/xh)
#   $ xh -a '<your personal circleCI token>:' POST \
#       https://circleci.com/api/v2/project/github/influxdata/influxdb/pipeline \
#       parameters:='{"release_branch": true}' branch=chore/ci-tidy-up
#
# ...or equivalent with `curl`:
#   $ curl -XPOST -H "Content-Type: application/json" -H "Circle-Token: <your personal circleCI token>" \
#       -d '{"parameters": {"release_branch": true}, "branch": "chore/ci-tidy-up"}' \
#       https://circleci.com/api/v2/project/github/influxdata/influxdb/pipeline

version: 2.1

orbs:
  aws-s3:    circleci/aws-s3@2.0.0
  terraform: circleci/terraform@2.1.0
  rust: circleci/rust@1.6.1

# Unlike when a commit is pushed to a branch, CircleCI does not automatically
# execute a workflow when a tag is pushed to a repository. These filters
# allow the corresponding workflow to execute on any branch or tag.
any_filter: &any_filter
  filters:
    tags:
      only: /.*/
    branches:
      only: /.*/

release_filter: &release_filter
  filters:
    tags:
      # This regex matches what is found in 'scripts/get-version' with the
      # '[[:digit:]]' transformed into '\d'. This also excludes release
      # candidate detection, because this filter only matches against
      # full releases.
      only: /^v(\d+)(?:\.(\d+))?(?:\.(\d+))?$/
    branches:
      ignore: /.*/

nofork_filter: &nofork_filter
  filters:
    branches:
      ignore: /pull\/[0-9]+/


commands:
  rust_components:
    description: Verify installed components
    steps:
      - run:
          name: Verify installed components
          command: |
            rustup --version
            rustup show
            cargo fmt --version
            cargo clippy --version
  gcloud-docker-login:
    steps:
      - run:
          name: configure-gar
          command: |
            gcloud auth activate-service-account "${GCLOUD_SERVICE_ACCOUNT_EMAIL}" --key-file <(echo "${GCLOUD_SERVICE_ACCOUNT_KEY}")
            gcloud auth configure-docker us-docker.pkg.dev

jobs:
  fmt:
    docker:
      - image: quay.io/influxdb/rust:ci
    environment:
      # Disable incremental compilation to avoid overhead. We are not preserving these files anyway.
      CARGO_INCREMENTAL: "0"
      # Disable full debug symbol generation to speed up CI build
      # "1" means line tables only, which is useful for panic tracebacks.
      CARGO_PROFILE_DEV_DEBUG: "1"
      # https://github.com/rust-lang/cargo/issues/10280
      CARGO_NET_GIT_FETCH_WITH_CLI: "true"
    steps:
      - checkout
      - rust_components
      - run:
          name: Rust fmt
          command: cargo fmt --all -- --check
  lint:
    docker:
      - image: quay.io/influxdb/rust:ci
    environment:
      # Disable incremental compilation to avoid overhead. We are not preserving these files anyway.
      CARGO_INCREMENTAL: "0"
      # Disable full debug symbol generation to speed up CI build
      # "1" means line tables only, which is useful for panic tracebacks.
      CARGO_PROFILE_DEV_DEBUG: "1"
      # https://github.com/rust-lang/cargo/issues/10280
      CARGO_NET_GIT_FETCH_WITH_CLI: "true"
    steps:
      - checkout
      - rust_components
      - run:
          name: Clippy
          command: cargo clippy --all-targets --all-features --workspace -- -D warnings
      - run:
          name: Yamllint
          command: yamllint --config-file .circleci/yamllint.yml --strict .
  cargo-audit:
    docker:
      - image: quay.io/influxdb/rust:ci
    environment:
      # Disable incremental compilation to avoid overhead. We are not preserving these files anyway.
      CARGO_INCREMENTAL: "0"
      # Disable full debug symbol generation to speed up CI build
      # "1" means line tables only, which is useful for panic tracebacks.
      CARGO_PROFILE_DEV_DEBUG: "1"
      # https://github.com/rust-lang/cargo/issues/10280
      CARGO_NET_GIT_FETCH_WITH_CLI: "true"
    steps:
      - checkout
      - rust_components
      - run:
          name: Install cargo-deny
          command: cargo install cargo-deny --locked
      - run:
          name: cargo-deny Checks
          command: cargo deny check -s
  doc:
    docker:
      - image: quay.io/influxdb/rust:ci
    resource_class: large # use of a smaller executor runs out of memory
    environment:
      # Disable incremental compilation to avoid overhead. We are not preserving these files anyway.
      CARGO_INCREMENTAL: "0"
      # Disable full debug symbol generation to speed up CI build
      # "1" means line tables only, which is useful for panic tracebacks.
      CARGO_PROFILE_DEV_DEBUG: "1"
      # https://github.com/rust-lang/cargo/issues/10280
      CARGO_NET_GIT_FETCH_WITH_CLI: "true"
      # Turn warnings into errors
      RUSTDOCFLAGS: "-D warnings -A rustdoc::private-intra-doc-links"
    steps:
      - checkout
      - rust_components
      - run:
          name: Cargo doc
          command: cargo doc --document-private-items --no-deps --workspace
      - run:
          name: Compress Docs
          command: tar -cvzf rustdoc.tar.gz target/doc/
      - store_artifacts:
          path: rustdoc.tar.gz

  # Run all tests
  test:
    docker:
      - image: quay.io/influxdb/rust:ci
    resource_class: 2xlarge+ # use of a smaller executor tends crashes on link
    environment:
      # Disable incremental compilation to avoid overhead. We are not preserving these files anyway.
      CARGO_INCREMENTAL: "0"
      # Disable full debug symbol generation to speed up CI build
      # "1" means line tables only, which is useful for panic tracebacks.
      CARGO_PROFILE_DEV_DEBUG: "1"
      # https://github.com/rust-lang/cargo/issues/10280
      CARGO_NET_GIT_FETCH_WITH_CLI: "true"
      RUST_BACKTRACE: "1"
    steps:
      - checkout
      - rust_components
      - run:
          name: cargo nextest
          command: TEST_LOG=1 RUST_LOG=info RUST_LOG_SPAN_EVENTS=full RUST_BACKTRACE=1 cargo nextest run --workspace --failure-output immediate-final --no-fail-fast

  # Build a dev binary.
  #
  # Compiles a binary with the default ("dev") cargo profile from the influxdb3 source
  # using the latest ci_image (influxdb/rust) and ensures various targets compile successfully
  # Build a dev binary.
  build-dev:
    docker:
      - image: us-east1-docker.pkg.dev/influxdata-team-edge/ci-support/ci-cross-influxdb3:latest
        auth:
          username: _json_key
          password: $CISUPPORT_GCS_AUTHORIZATION
    resource_class: 2xlarge+ # use of a smaller executor tends crashes on link
    environment:
      TARGET: << parameters.target >>
      # Disable incremental compilation to avoid overhead. We are not preserving these files anyway.
      CARGO_INCREMENTAL: "0"
      # Disable full debug symbol generation to speed up CI build
      # "1" means line tables only, which is useful for panic tracebacks.
      CARGO_PROFILE_DEV_DEBUG: "1"
      # https://github.com/rust-lang/cargo/issues/10280
      CARGO_NET_GIT_FETCH_WITH_CLI: "true"
      # The `2xlarge` resource class that we use has 32GB RAM but also 16 CPUs. This means we have 2GB RAM per core on
      # avarage. At peak this is a bit tight, so lower the CPU count for cargo a bit.
      CARGO_BUILD_JOBS: "12"
    parameters:
      target:
        type: string
    steps:
      - checkout
      - run:
          name: Install Target
          command: rustup target add << parameters.target >>
      - run:
          name: Cargo build
          command: target-env cargo build --target=<< parameters.target >> --workspace
      - when:
          condition:
            not:
              equal: [ << parameters.target >>, x86_64-pc-windows-gnu ]
          steps:
          - run:
              name: Check benches compile
              command: target-env cargo check --target=<< parameters.target >> --workspace --benches
          - run:
              name: Check extra features (like prod image)
              command: target-env cargo check --target=<< parameters.target >> --no-default-features --features="aws,gcp,azure,jemalloc_replacing_malloc,tokio_console"
      - when:
          condition:
            equal: [ << parameters.target >>, x86_64-pc-windows-gnu ]
          steps:
          - run:
              name: Check extra features (like prod image)
              command: target-env cargo check --target=<< parameters.target >> --no-default-features --features="aws,gcp,azure,jemalloc_replacing_malloc,tokio_console"

  # Compile cargo "release" profile binaries for influxdb3 edge releases
  build-release:
    docker:
      - image: us-east1-docker.pkg.dev/influxdata-team-edge/ci-support/ci-cross-influxdb3:latest
        auth:
          username: _json_key
          password: $CISUPPORT_GCS_AUTHORIZATION
    resource_class: 2xlarge+
    environment:
      TARGET: << parameters.target >>
      # Disable incremental compilation to avoid overhead. We are not preserving these files anyway.
      CARGO_INCREMENTAL: "0"
      # Disable full debug symbol generation to speed up CI build
      # "1" means line tables only, which is useful for panic tracebacks.
      CARGO_PROFILE_DEV_DEBUG: "1"
      # https://github.com/rust-lang/cargo/issues/10280
      CARGO_NET_GIT_FETCH_WITH_CLI: "true"
      # The `2xlarge` resource class that we use has 32GB RAM but also 16 CPUs. This means we have 2GB RAM per core on
      # avarage. At peak this is a bit tight, so lower the CPU count for cargo a bit.
      CARGO_BUILD_JOBS: "12"
    parameters:
      target:
        type: string
    steps:
      - checkout
      - run:
          name: Install Target
          command: rustup target add << parameters.target >>
      - run:
          name: Cargo release build
          command: target-env cargo build --target=<< parameters.target >> --workspace --release
          # linking might take a while and doesn't produce CLI output
          no_output_timeout: 30m
      - run:
          name: tar and gzip build artifacts
          command: |
            mkdir -p artifacts
            tar --ignore-failed-read -czvf "${PWD}/artifacts/influxdb3-edge_<< parameters.target >>.tar.gz" -C "${PWD}/target/<< parameters.target >>/release" influxdb3{,.exe}
      - store_artifacts:
          path: artifacts
      - persist_to_workspace:
          root: .
          paths:
            - artifacts
  build-packages:
    docker:
      - image: us-east1-docker.pkg.dev/influxdata-team-edge/ci-support/ci-packager-next:latest
        auth:
          username: _json_key
          password: $CISUPPORT_GCS_AUTHORIZATION
    steps:
      - checkout
      - attach_workspace:
          at: /tmp/workspace
      - run: packager .circleci/packages/config.yaml
      - persist_to_workspace:
          root: .
          paths:
            - artifacts
      - store_artifacts:
          path: artifacts/
  check_package_deb_amd64:
    machine:
      image: ubuntu-2204:current
      resource_class: medium
    steps:
      - attach_workspace:
          at: /tmp/workspace
      - checkout
      - run:
          name: Validate Debian Package (AMD64)
          command: |
            sudo .circleci/scripts/package-validation/debian \
              /tmp/workspace/artifacts/influxdb3*amd64.deb
  check_package_deb_arm64:
    machine:
      image: ubuntu-2204:current
      resource_class: arm.medium
    steps:
      - attach_workspace:
          at: /tmp/workspace
      - checkout
      - run:
          name: Validate Debian Package (ARM64)
          command: |
            sudo .circleci/scripts/package-validation/debian \
              /tmp/workspace/artifacts/influxdb3*arm64.deb
  check_package_rpm:
    machine:
      image: ubuntu-2204:current
      resource_class: arm.medium
    parameters:
      arch:
        type: string
    steps:
      - attach_workspace:
          at: /tmp/workspace
      - add_ssh_keys:
          fingerprints:
            - 3a:d1:7a:b7:57:d7:85:0b:76:79:85:51:38:f3:e4:67
      - checkout
      - run: |
          AWS_ACCESS_KEY_ID=$TEST_AWS_ACCESS_KEY_ID \
          AWS_SECRET_ACCESS_KEY=$TEST_AWS_SECRET_ACCESS_KEY \
          .circleci/scripts/package-validation/redhat << parameters.arch >> /tmp/workspace/artifacts/influxdb3*.<< parameters.arch >>.rpm
  sign-packages:
    circleci_ip_ranges: true
    docker:
      - image: quay.io/influxdb/rsign:latest
        auth:
          username: $QUAY_RSIGN_USERNAME
          password: $QUAY_RSIGN_PASSWORD
    steps:
      - add_ssh_keys:
          fingerprints:
            - fc:7b:6e:a6:38:7c:63:5a:13:be:cb:bb:fa:33:b3:3c
      - attach_workspace:
          at: /tmp/workspace
      - run: |
          # We need this base so that we can filter it out of our checksums
          # output and if that changes at all we only need to update it here
          WORK_DIR="/tmp/workspace/artifacts/"
          for target in ${WORK_DIR}*
          do
            case "${target}"
            in
              # rsign is shipped on Alpine Linux which uses "busybox ash" instead
              # of bash. ash is somewhat more posix compliant and is missing some
              # extensions and niceties from bash.
              *.deb|*.rpm|*.tar.gz|*.zip)
                rsign "${target}"
              ;;
            esac

            if [ -f "${target}" ]
            then
              # Since all artifacts are present, sign them here. This saves Circle
              # credits over spinning up another instance just to separate out the
              # checksum job.
              sha256sum "${target}" | sed "s#$WORK_DIR##" >> "/tmp/workspace/artifacts/influxdb3-edge.${CIRCLE_TAG}.digests"

              # write individual checksums
              md5sum    "${target}" | sed "s#$WORK_DIR##" >> "${target}.md5"
              sha256sum "${target}" | sed "s#$WORK_DIR##" >> "${target}.sha256"
            fi
          done
      - persist_to_workspace:
          root: /tmp/workspace
          paths:
            - artifacts
      - store_artifacts:
          path: /tmp/workspace/artifacts
  publish-packages:
    docker:
      - image: cimg/python:3.12.2
    parameters:
      # "destination" should be one of:
      #   - releases
      #   - nightlies
      #   - snapshots
      destination:
        type: string
    steps:
      - attach_workspace:
          at: /tmp/workspace
      - aws-s3/sync:
          arguments:             --acl public-read
          aws-region:            RELEASE_AWS_REGION
          aws-access-key-id:     RELEASE_AWS_ACCESS_KEY_ID
          aws-secret-access-key: RELEASE_AWS_SECRET_ACCESS_KEY
          from:                  /tmp/workspace/artifacts
          to:                    s3://dl.influxdata.com/influxdb/<< parameters.destination >>
  build-docker:
    # need a machine executor to have a full-powered docker daemon (the `setup_remote_docker` system just provides a
    # kinda small node)
    machine:
      image: default
    resource_class: 2xlarge+ # CPU bound, so make it fast
    steps:
      - checkout
      - run:
          name: Build the docker image
          command: |
            .circleci/scripts/docker_build_release.bash \
              "influxdb3" \
              "aws,gcp,azure,jemalloc_replacing_malloc,tokio_console" \
              "influxdb3-edge:latest"

          # linking might take a while and doesn't produce CLI output
          no_output_timeout: 30m
      - run: |
          docker save influxdb3-edge:latest >influxdb3-edge.tar
      - persist_to_workspace:
          root: .
          paths:
            - influxdb3-edge.tar
  publish-docker:
    docker:
      - image: cimg/gcp:2023.02
    resource_class: medium
    steps:
      - checkout
      - setup_remote_docker
      - gcloud-docker-login
      - attach_workspace:
          at: .
      - run: |
          docker load <influxdb3-edge.tar
          .circleci/scripts/publish.bash influxdb3-edge

  wait-for-docker:
    resource_class: small
    docker:
      - image: busybox
    steps:
      - run: |
          echo build executed successfully


workflows:
  version: 2
  ci:
    jobs:
      - fmt:
          <<: *any_filter
      - lint:
          <<: *any_filter
      - cargo-audit:
          <<: *any_filter
      - test:
          <<: *any_filter
      - build-dev:
          # This workflow requires secrets stored in the environment.
          # These are not passed to workflows executed on forked
          # repositories. In this case, skip the workflow, as it
          # will be executed on merge to main anyways.
          <<: *nofork_filter
          name: build-dev-<< matrix.target >>
          matrix:
            parameters:
              target:
                - aarch64-apple-darwin
                - aarch64-unknown-linux-gnu
                - aarch64-unknown-linux-musl
                - x86_64-pc-windows-gnu
                - x86_64-unknown-linux-gnu
                - x86_64-unknown-linux-musl
      - doc:
          <<: *any_filter
      - build-release:
          <<: *release_filter
          name: build-release-<< matrix.target >>
          matrix:
            parameters:
              target:
                - aarch64-apple-darwin
                - aarch64-unknown-linux-gnu
                - aarch64-unknown-linux-musl
                - x86_64-pc-windows-gnu
                - x86_64-unknown-linux-gnu
                - x86_64-unknown-linux-musl
      - build-packages:
          <<: *release_filter
          requires:
            - build-release
      - check_package_deb_arm64:
          <<: *release_filter
          requires:
            - build-packages
      - check_package_deb_amd64:
          <<: *release_filter
          requires:
            - build-packages
      - check_package_rpm:
          <<: *nofork_filter
          name:
            check_package_rpm-<< matrix.arch >>
          matrix:
            parameters:
              arch: [ x86_64, aarch64 ]
          requires:
            - build-packages

      - sign-packages:
          <<: *release_filter
          requires:
            - build-packages
            - check_package_rpm
            - check_package_deb_arm64
            - check_package_deb_amd64
            - test
      - publish-packages:
          <<: *release_filter
          matrix:
            parameters:
              destination: [ releases ]
          requires:
            - build-release
            - sign-packages
            - test
            - doc
            - lint
            - fmt
            - cargo-audit
      - build-docker:
          <<: *release_filter
      - publish-docker:
          <<: *release_filter
          requires:
            - build-docker
      - wait-for-docker:
          <<: *release_filter
          requires:
            - build-docker
            - publish-docker