diff --git a/.github/install-openvas-dependencies.sh b/.github/install-openvas-dependencies.sh index afbbbf2232..e4f1bc6fbe 100755 --- a/.github/install-openvas-dependencies.sh +++ b/.github/install-openvas-dependencies.sh @@ -10,6 +10,7 @@ apt-get update && apt-get install --no-install-recommends --no-install-suggests clang-tools \ cmake \ curl \ + git \ lcov \ libgnutls28-dev \ libgpgme-dev \ diff --git a/.github/workflows/build-container.yml b/.github/workflows/build-container.yml index 5526873f98..9ce440c9b6 100644 --- a/.github/workflows/build-container.yml +++ b/.github/workflows/build-container.yml @@ -1,22 +1,6 @@ name: Build Container -on: - push: - branches: [ main, stable, oldstable, middleware ] - tags: ["v*"] - paths: - - .github/workflows/build-container.yml - - .docker/build.Dockerfile - pull_request: - branches: [ main, stable, oldstable, middleware ] - paths: - - .github/workflows/build-container.yml - - .docker/build.Dockerfile - workflow_dispatch: - repository_dispatch: - schedule: - # rebuild image every sunday - - cron: "0 0 * * 0" +on: [workflow_call] jobs: build: diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml new file mode 100644 index 0000000000..5a1d056391 --- /dev/null +++ b/.github/workflows/build.yml @@ -0,0 +1,25 @@ +name: "Build" + +on: [workflow_call] + +jobs: + OpenVAS: + runs-on: ubuntu-latest + container: greenbone/gvm-libs:stable + steps: + - uses: actions/checkout@v4 + - name: install dependencies + run: | + sh .github/install-openvas-dependencies.sh + - name: build + run: | + cmake -Bbuild -DCMAKE_C_COMPILER=/usr/share/clang/scan-build-14/libexec/ccc-analyzer + scan-build -o ~/scan-build-report cmake --build build + - name: Upload scan-build report + uses: actions/upload-artifact@v3 + with: + name: scan-build-report + path: ~/scan-build-report/ + retention-days: 7 + OpenVAS_Daemon: + uses: ./.github/workflows/build-rust.yml diff --git a/.github/workflows/build_and_test.yml b/.github/workflows/build_and_test.yml deleted file mode 100644 index d605f9edde..0000000000 --- a/.github/workflows/build_and_test.yml +++ /dev/null @@ -1,61 +0,0 @@ -name: "openvas-c" - -on: - push: - branches: [ main, stable ] - pull_request: - branches: [ main, stable ] - -jobs: - formatting: - runs-on: ubuntu-latest - steps: - - name: Check out openvas-scanner - uses: actions/checkout@v4 - - name: Check Source Format - run: | - clang-format -i -style=file {src,misc,nasl}/*.{c,h} - git diff --exit-code - compile: - runs-on: ubuntu-latest - strategy: - matrix: - # With the upcoming changes, we require both downwards and upwards compatibility between the OpenVAS C - # code and GVM-libs. This is because, even though we will be using semantic versioning, as long as - # OpenVAS and GVM-libs remain separate repositories, we want to be notified of every change. - gvm-libs-version: - - stable - - unstable - container: greenbone/gvm-libs:${{ matrix.gvm-libs-version }} - steps: - - uses: actions/checkout@v4 - - name: install dependencies - run: | - sh .github/install-openvas-dependencies.sh - - name: Configure and Scan Build - run: | - cmake -Bbuild -DCMAKE_C_COMPILER=/usr/share/clang/scan-build-14/libexec/ccc-analyzer - scan-build -o ~/scan-build-report cmake --build build - - name: Upload scan-build report - uses: actions/upload-artifact@v3 - with: - name: scan-build-report - path: ~/scan-build-report/ - retention-days: 7 - unit-tests: - runs-on: ubuntu-latest - strategy: - matrix: - gvm-libs-version: - - stable - - unstable - container: greenbone/gvm-libs:${{ matrix.gvm-libs-version }} - steps: - - uses: actions/checkout@v4 - - name: install dependencies - run: | - sh .github/install-openvas-dependencies.sh - - name: unit-tests - run: | - cmake -Bbuild -DCMAKE_BUILD_TYPE=Release - CTEST_OUTPUT_ON_FAILURE=1 cmake --build build -- tests test diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000000..6ea816ab78 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,40 @@ +name: "Linting" + +on: [workflow_call] + +jobs: + OpenVAS: + runs-on: ubuntu-latest + container: greenbone/gvm-libs:stable + steps: + - uses: actions/checkout@v4 + - name: install dependencies + run: | + sh .github/install-openvas-dependencies.sh + - name: Formatting + run: | + clang-format --dry-run --Werror -i -style=file {src,misc,nasl}/*.{c,h} + - name: unit-tests + run: | + cmake -Bbuild -DCMAKE_BUILD_TYPE=Release + CTEST_OUTPUT_ON_FAILURE=1 cmake --build build -- tests test + OpenVAS_Daemon: + runs-on: ubuntu-latest + defaults: + run: + working-directory: rust + steps: + - uses: actions/checkout@v4 + - run: sudo apt update && sudo apt-get install -y libpcap-dev + - run: rustup update stable && rustup default stable || rustup default stable + - run: cargo install cargo-audit + - run: cargo install typos-cli + - name: unit-tests + run: cargo test --lib --tests --workspace + - name: Clippy + run: cargo clippy -- -D warnings + - name: Audit + run: cargo audit + - run: typos + - name: Formatting + run: cargo fmt --check diff --git a/.github/workflows/codeql-analysis-c.yml b/.github/workflows/codeql.yml similarity index 100% rename from .github/workflows/codeql-analysis-c.yml rename to .github/workflows/codeql.yml diff --git a/.github/workflows/control.yml b/.github/workflows/control.yml new file mode 100644 index 0000000000..e52a9f33d9 --- /dev/null +++ b/.github/workflows/control.yml @@ -0,0 +1,24 @@ +name: CI + +on: + push: + branches: [ main] + tags: ["v*"] + pull_request: + workflow_dispatch: + repository_dispatch: + schedule: + # rebuild image every sunday + - cron: "0 0 * * 0" + +jobs: + build: + uses: ./.github/workflows/build.yml + linting: + uses: ./.github/workflows/ci.yml + smoketests: + needs: [build] + uses: ./.github/workflows/smoketest.yml + container: + needs: [smoketests] + uses: ./.github/workflows/push-container.yml diff --git a/.github/workflows/ddependabot.yml b/.github/workflows/ddependabot.yml new file mode 100644 index 0000000000..e69de29bb2 diff --git a/.github/workflows/nasl.yml b/.github/workflows/nasl.yml deleted file mode 100644 index c1599250f3..0000000000 --- a/.github/workflows/nasl.yml +++ /dev/null @@ -1,41 +0,0 @@ -name: "NASL" - -on: - push: - branches: [ main ] - pull_request: - branches: [ main ] - -jobs: - nasl-test: - name: test - runs-on: ubuntu-latest - container: greenbone/gvm-libs:unstable - steps: - - uses: actions/checkout@v4 - - name: install dependencies - run: | - sh .github/install-openvas-dependencies.sh - - name: build openvas - run: | - cmake -Bbuild -DCMAKE_BUILD_TYPE=Release - cmake --build build - - name: redis - run: | - apt-get update && apt-get install --no-install-recommends --no-install-suggests -y redis - mkdir /run/redis-openvas - redis-server config/redis-openvas.conf || exit 1 - - name: scripttests - run: | - mkdir -p /etc/openvas - echo "db_address = /run/redis-openvas/redis.sock" >> /etc/openvas/openvas.conf - cd nasl/tests - OPENVAS_NASL=../../build/nasl/openvas-nasl make check - - uses: actions/setup-go@v5 - with: - go-version: '>=1.16.0' - - name: smoketest/lint - run: | - make build - ./run -e ../../build/nasl/openvas-nasl-lint - working-directory: smoketest_lint diff --git a/.github/workflows/push-container.yml b/.github/workflows/push-container.yml new file mode 100644 index 0000000000..0360f22850 --- /dev/null +++ b/.github/workflows/push-container.yml @@ -0,0 +1,144 @@ +name: Container + +on: [workflow_call] + +jobs: + production-image: + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v4 + - name: "set IS_VERSION_TAG" + run: | + echo "IS_VERSION_TAG=${{ github.ref_type == 'tag' && startsWith(github.ref_name, 'v') }}" >> $GITHUB_ENV + # set defaults + echo "IS_LATEST_TAG=false" >> $GITHUB_ENV + - name: "set IS_LATEST_TAG" + if: ( env.IS_VERSION_TAG ) + run: | + # find the latest version that is not ourself + export LATEST_VERSION=$(git tag -l | grep -v '${{ github.ref_name }}' | sort -r --version-sort) + # get major minor patch versions + IFS='.' read -r latest_major latest_minor latest_patch << EOF + $LATEST_VERSION + EOF + IFS='.' read -r tag_major tag_minor tag_patch << EOF + ${{ github.ref_name }} + EOF + # remove leading v + latest_major=$(echo $latest_major | cut -c2-) + tag_major=$(echo $tag_major | cut -c2-) + echo "$tag_major >= $latest_major" + if [[ $tag_major -ge $latest_major && ($tag_minor -ne 0 || $tag_patch -ne 0) ]]; then + # set this tag to latest and stable + echo "IS_LATEST_TAG=true" >> $GITHUB_ENV + fi + - name: "Setup meta information (IS_VERSION_TAG: ${{ env.IS_VERSION_TAG }}, IS_LATEST_TAG: ${{ env.IS_LATEST_TAG }} )" + id: meta + uses: docker/metadata-action@v5 + with: + images: ${{ github.repository }} + labels: | + org.opencontainers.image.vendor=Greenbone + org.opencontainers.image.base.name=greenbone/gvm-libs + flavor: latest=false # no auto latest container tag for git tags + tags: | + # when IS_LATEST_TAG is set create a stable and a latest tag + type=raw,value=latest,enable=${{ env.IS_LATEST_TAG }} + type=raw,value=stable,enable=${{ env.IS_LATEST_TAG }} + # if tag version is set than create a version tags + type=semver,pattern={{version}},enable=${{ env.IS_VERSION_TAG }} + type=semver,pattern={{major}}.{{minor}},enable=${{ env.IS_VERSION_TAG }} + type=semver,pattern={{major}},enable=${{ env.IS_VERSION_TAG }} + # if we are on the main branch set edge + type=edge,branch=main + # use branch-sha otherwise for pushes to branches other then main (will not be uploaded) + type=raw,value={{branch}}-{{sha}},enable=${{ github.ref_type == 'branch' && github.event_name == 'push' && github.ref_name != 'main' }} + # use pr-$PR_ID for pull requests (will not be uploaded) + type=ref,event=pr + - name: Login to DockerHub + if: github.event_name != 'pull_request' + uses: docker/login-action@v3 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} + + - uses: actions/download-artifact@v3 + with: + name: rs-binaries + path: assets + - run: mkdir -p assets/linux/amd64 + - run: mkdir -p assets/linux/arm64 + - run: mv assets/openvasd-aarch64-unknown-linux-gnu assets/linux/arm64/openvasd + - run: mv assets/openvasd-x86_64-unknown-linux-gnu assets/linux/amd64/openvasd + - run: mv assets/nasl-cli-aarch64-unknown-linux-gnu assets/linux/arm64/nasl-cli + - run: mv assets/nasl-cli-x86_64-unknown-linux-gnu assets/linux/amd64/nasl-cli + - name: Set up QEMU + uses: docker/setup-qemu-action@v3 + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + - name: Build and push + uses: docker/build-push-action@v5 + with: + context: . + push: ${{ github.event_name != 'pull_request' && (github.ref_type == 'tag' || github.ref_name == 'main') }} + file: .docker/prod.Dockerfile + build-args: | + REPOSITORY=${{ github.repository }} + platforms: linux/amd64,linux/aarch64 + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} + + - name: "Setup meta information debian:oldstable" + id: old_stable_meta + uses: docker/metadata-action@v5 + with: + images: ${{ github.repository }} + labels: | + org.opencontainers.image.vendor=Greenbone + org.opencontainers.image.base.name=greenbone/gvm-libs + flavor: latest=false # no auto latest container tag for git tags + tags: | + # for the images provided for debian:oldstable we just provide + # oldstable on an new version or oldstable-edge when it is on main. + # oldstable-branch-sha on a branch + type=raw,value=oldstable,enable=${{ env.IS_LATEST_TAG }} + type=raw,value=oldstable-edge,enable=${{ github.ref_name == 'main' }} + type=raw,value=oldstable-{{branch}}-{{sha}},enable=${{ github.ref_type == 'branch' && github.event_name == 'push' && github.ref_name != 'main' }} + type=ref,event=pr + - name: Build and push Container image + uses: docker/build-push-action@v5 + with: + context: . + push: ${{ github.event_name != 'pull_request' && (github.ref_type == 'tag' || github.ref_name == 'main') }} + file: .docker/prod-oldstable.Dockerfile + platforms: linux/amd64,linux/arm64 + tags: ${{ steps.old_stable_meta.outputs.tags }} + labels: ${{ steps.old_stable_meta.outputs.labels }} + + - name: "Setup meta information debian:testing" + id: test_meta + uses: docker/metadata-action@v5 + with: + images: ${{ github.repository }} + labels: | + org.opencontainers.image.vendor=Greenbone + org.opencontainers.image.base.name=greenbone/gvm-libs + flavor: latest=false # no auto latest container tag for git tags + tags: | + # for the images provided for debian:testing we just provide + # testing on an new version or testing-edge when it is on main. + # testing-branch-sha on a branch + type=raw,value=testing,enable=${{ env.IS_LATEST_TAG }} + type=raw,value=testing-edge,enable=${{ github.ref_name == 'main' }} + type=raw,value=testing-{{branch}}-{{sha}},enable=${{ github.ref_type == 'branch' && github.event_name == 'push' && github.ref_name != 'main' }} + type=ref,event=pr + - name: Build and push Container image + uses: docker/build-push-action@v5 + with: + context: . + push: ${{ github.event_name != 'pull_request' && (github.ref_type == 'tag' || github.ref_name == 'main') }} + file: .docker/prod-testing.Dockerfile + platforms: linux/amd64,linux/arm64 + tags: ${{ steps.test_meta.outputs.tags }} + labels: ${{ steps.test_meta.outputs.labels }} diff --git a/.github/workflows/push-helm-chart.yml b/.github/workflows/push-helm-chart.yml new file mode 100644 index 0000000000..39bc31cf4c --- /dev/null +++ b/.github/workflows/push-helm-chart.yml @@ -0,0 +1,17 @@ +name: "Helm Push" + +on: [workflow_call] + +jobs: + helm: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: greenbone/actions/helm-build-push@v3 + if: github.event_name == 'workflow_dispatch' + with: + chart-name: openvasd + registry: ${{ vars.IMAGE_REGISTRY }} + registry-subpath: helm-charts/ + registry-user: ${{ secrets.GREENBONE_BOT }} + registry-token: ${{ secrets.GREENBONE_BOT_PACKAGES_WRITE_TOKEN }} diff --git a/.github/workflows/rustification.yaml b/.github/workflows/rustification.yaml index a41b102381..3804c7136b 100644 --- a/.github/workflows/rustification.yaml +++ b/.github/workflows/rustification.yaml @@ -9,69 +9,6 @@ env: CARGO_TERM_COLOR: always jobs: - unittests: - runs-on: ubuntu-latest - defaults: - run: - working-directory: rust - strategy: - matrix: - toolchain: - - stable - - beta - - nightly - steps: - - uses: actions/checkout@v4 - - run: sudo apt update && sudo apt-get install -y libpcap-dev - - run: rustup update ${{ matrix.toolchain }} && rustup default ${{ matrix.toolchain }} || rustup default ${{ matrix.toolchain }} - - run: cargo test --lib --tests --workspace - clippy: - runs-on: ubuntu-latest - defaults: - run: - working-directory: rust - steps: - - uses: actions/checkout@v4 - - run: rustup update stable && rustup default stable && rustup component add clippy - - run: cargo clippy -- -D warnings - audit: - runs-on: ubuntu-latest - defaults: - run: - working-directory: rust - steps: - - uses: actions/checkout@v4 - - run: rustup update stable && rustup default stable - - run: cargo install cargo-audit - - run: cargo audit - typos: - runs-on: ubuntu-latest - defaults: - run: - working-directory: rust - steps: - - uses: actions/checkout@v4 - - run: rustup update stable && rustup default stable - - run: cargo install typos-cli - - run: typos - formatting: - runs-on: ubuntu-latest - defaults: - run: - working-directory: rust - strategy: - matrix: - crates: - # we verify each dir separately to make it easier to verify formatting issues or even ignore - # crates we deem not important for checking (e.g. feed-verifier) - - nasl-syntax - - storage - - nasl-interpreter - - redis-storage - - json-storage - - nasl-cli - steps: - - uses: actions/checkout@v4 rs-build-binaries: uses: ./.github/workflows/build-rust.yml verify-syntax: diff --git a/.github/workflows/smoketest.yaml b/.github/workflows/smoketest.yaml new file mode 100644 index 0000000000..f9673c1960 --- /dev/null +++ b/.github/workflows/smoketest.yaml @@ -0,0 +1,110 @@ +name: Smoketests + +on: [workflow_call] +# smoke test definition. +# It depends on build.yml that is controlled via control.yml +# +jobs: + OpenVAS_Daemon: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - name: Start a local k8s cluster + uses: jupyterhub/action-k3s-helm@v3 + with: + k3s-channel: latest + metrics-enabled: false + - name: deploy openvasd + run: | + cd rust/examples/tls/Self-Signed\ mTLS\ Method + make delete deploy + cd - + helm uninstall openvasd --namespace openvasd|| true + helm install --namespace openvasd --create-namespace openvasd charts/openvasd/ --values charts/openvasd/values.yaml --values charts/openvasd/mtls-wo-ingress.yaml + kubectl rollout status --watch --timeout 600s deployment/openvasd --namespace openvasd + echo "OPENVASD_SERVER=https://$(kubectl get svc -n openvasd | awk 'FNR == 2 {print $(3)}')" >> $GITHUB_ENV + - name: smoketest + working-directory: rust/smoketest + env: + SCAN_CONFIG: configs/simple_scan_ssh_only.json + CLIENT_KEY: ../examples/tls/Self-Signed mTLS Method/client.rsa + CLIENT_CERT: ../examples/tls/Self-Signed mTLS Method/client.pem + run: | + make build run + + - run: FEED_DIR="feed/" sh .github/prepare-feed.sh + - uses: actions/download-artifact@v3 + with: + name: rs-binaries + path: assets + - run: mv assets/nasl-cli-x86_64-unknown-linux-gnu ./nasl-cli + - run: chmod +x ./nasl-cli + - name: verify syntax parsing + run: ./nasl-cli syntax --quiet feed/ + # find a way to include it... + verify-feed-update: + runs-on: ubuntu-latest + needs: [build] + container: + # maybe better to use builder, build openvas to have + # the version of this checkout rather than a dated official one? + image: greenbone/openvas-scanner:unstable + options: --privileged + services: + redis: + image: redis + options: >- + --health-cmd "redis-cli ping" + --health-interval 10s + --health-timeout 5s + --health-retries 5 + steps: + - uses: actions/checkout@v4 + - run: apt-get update && apt-get install -y docker.io + - run: FEED_DIR="feed/" sh .github/prepare-feed.sh + - uses: actions/download-artifact@v3 + with: + name: rs-binaries + path: assets + - run: mv assets/nasl-cli-x86_64-unknown-linux-gnu ./nasl-cli + - run: mv assets/feed-verifier-x86_64-unknown-linux-gnu ./feed-verifier + - name: prepare setup + run: | + install -m 755 feed-verifier /usr/local/bin/ + install -m 755 nasl-cli /usr/local/bin/ + echo "db_address = tcp://redis:6379" >> /etc/openvas/openvas.conf + mv ./feed/* "$(openvas -s | grep plugins_folder | sed 's/plugins_folder = //')/" + - run: openvas -s + - run: feed-verifier || (cat /var/log/gvm/openvas.log && false) + nasl-test: + name: test + runs-on: ubuntu-latest + container: greenbone/gvm-libs:stable + steps: + - uses: actions/checkout@v4 + - name: install dependencies + run: | + sh .github/install-openvas-dependencies.sh + - name: build openvas + run: | + cmake -Bbuild -DCMAKE_BUILD_TYPE=Release + cmake --build build + - name: redis + run: | + apt-get update && apt-get install --no-install-recommends --no-install-suggests -y redis + mkdir /run/redis-openvas + redis-server config/redis-openvas.conf || exit 1 + - name: scripttests + run: | + mkdir -p /etc/openvas + echo "db_address = /run/redis-openvas/redis.sock" >> /etc/openvas/openvas.conf + cd nasl/tests + OPENVAS_NASL=../../build/nasl/openvas-nasl make check + - uses: actions/setup-go@v5 + with: + go-version: '>=1.16.0' + - name: smoketest/lint + run: | + make build + ./run -e ../../build/nasl/openvas-nasl-lint + working-directory: smoketest_lint