Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[fix][ci] Fix OWASP Dependency Check download by using NVD API key #22999

Merged
merged 12 commits into from
Jul 4, 2024
Merged

[fix][ci] Fix OWASP Dependency Check download by using NVD API key #22999

Show file tree
Hide file tree
Changes from 11 commits
Commits
Show all changes
12 commits
Select commit Hold shift + click to select a range
848c525
Use pluginManagement to manage plugin version
lhotari Jul 3, 2024
ef13818
Use NIST_NVD_API_KEY as nvdApiKeyEnvironmentVariable
lhotari Jul 3, 2024
6fd6497
Upgrade plugin version
lhotari Jul 3, 2024
9377871
Increase max heap size for maven
lhotari Jul 3, 2024
b4c5aea
Fix cache lookup
lhotari Jul 3, 2024
755ee86
Add -DnarPluginPhase=none to skip nar file building
lhotari Jul 3, 2024
45d0b11
Fix building without building nar files
lhotari Jul 3, 2024
53da04e
Scan only maintained branches
lhotari Jul 3, 2024
53200b4
Add nvdDatafeedUrl to speed up database download
lhotari Jul 3, 2024
ddafa6a
Upgrade to v4 version
lhotari Jul 3, 2024
585912e
Comment out nvdDatafeedUrl
lhotari Jul 3, 2024
388d55c
Merge remote-tracking branch 'origin/master' into lh-owasp-dep-check-…
lhotari Jul 4, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
20 changes: 8 additions & 12 deletions .github/workflows/ci-owasp-dependency-check.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -24,8 +24,9 @@ on:
workflow_dispatch:

env:
MAVEN_OPTS: -Xss1500k -Xmx1024m -Daether.connector.http.reuseConnections=false -Daether.connector.requestTimeout=60000 -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false -Dmaven.wagon.http.retryHandler.class=standard -Dmaven.wagon.http.retryHandler.count=3 -Dmaven.wagon.http.retryHandler.requestSentEnabled=true -Dmaven.wagon.http.serviceUnavailableRetryStrategy.class=standard -Dmaven.wagon.rto=60000
MAVEN_OPTS: -Xss1500k -Xmx1500m -Daether.connector.http.reuseConnections=false -Daether.connector.requestTimeout=60000 -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false -Dmaven.wagon.http.retryHandler.class=standard -Dmaven.wagon.http.retryHandler.count=3 -Dmaven.wagon.http.retryHandler.requestSentEnabled=true -Dmaven.wagon.http.serviceUnavailableRetryStrategy.class=standard -Dmaven.wagon.rto=60000
JDK_DISTRIBUTION: corretto
NIST_NVD_API_KEY: ${{ secrets.NIST_NVD_API_KEY }}

jobs:
run-owasp-dependency-check:
Expand All @@ -42,12 +43,9 @@ jobs:
matrix:
include:
- branch: master
- branch: branch-3.3
- branch: branch-3.2
- branch: branch-3.1
- branch: branch-3.0
- branch: branch-2.11
- branch: branch-2.10
jdk: 11

steps:
- name: checkout
Expand All @@ -58,16 +56,14 @@ jobs:
- name: Tune Runner VM
uses: ./.github/actions/tune-runner-vm

- name: Cache local Maven repository
uses: actions/cache@v4
- name: Restore Maven repository cache
uses: actions/cache/restore@v4
timeout-minutes: 5
with:
path: |
~/.m2/repository/*/*/*
!~/.m2/repository/org/apache/pulsar
!~/.m2/repository/org/owasp/dependency-check-data
key: ${{ runner.os }}-m2-dependencies-all-${{ hashFiles('**/pom.xml') }}
lookup-only: true
restore-keys: |
${{ runner.os }}-m2-dependencies-core-modules-${{ hashFiles('**/pom.xml') }}
${{ runner.os }}-m2-dependencies-core-modules-
Expand All @@ -79,7 +75,7 @@ jobs:
java-version: ${{ matrix.jdk || '17' }}

- name: run install by skip tests
run: mvn -B -ntp clean install -DskipTests -Dspotbugs.skip=true -Dlicense.skip=true -Dcheckstyle.skip=true -Drat.skip=true -DskipDocker=true
run: mvn -B -ntp clean install -DskipTests -Dspotbugs.skip=true -Dlicense.skip=true -Dcheckstyle.skip=true -Drat.skip=true -DskipDocker=true -DnarPluginPhase=none -pl '!distribution/io,!distribution/offloaders'

- name: OWASP cache key weeknum
id: get-weeknum
Expand All @@ -89,7 +85,7 @@ jobs:

- name: Restore OWASP Dependency Check data
id: restore-owasp-dependency-check-data
uses: actions/cache/restore@v3
uses: actions/cache/restore@v4
timeout-minutes: 5
with:
path: ~/.m2/repository/org/owasp/dependency-check-data
Expand All @@ -105,7 +101,7 @@ jobs:

- name: Save OWASP Dependency Check data
if: ${{ steps.update-owasp-dependency-check-data.outcome == 'success' }}
uses: actions/cache/save@v3
uses: actions/cache/save@v4
timeout-minutes: 5
with:
path: ~/.m2/repository/org/owasp/dependency-check-data
Expand Down
9 changes: 4 additions & 5 deletions .github/workflows/pulsar-ci.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1427,6 +1427,7 @@ jobs:
env:
GRADLE_ENTERPRISE_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }}
CI_JDK_MAJOR_VERSION: ${{ needs.preconditions.outputs.jdk_major_version }}
NIST_NVD_API_KEY: ${{ secrets.NIST_NVD_API_KEY }}
steps:
- name: checkout
uses: actions/checkout@v4
Expand All @@ -1442,16 +1443,14 @@ jobs:
with:
limit-access-to-actor: true

- name: Cache Maven dependencies
uses: actions/cache@v4
- name: Restore Maven repository cache
uses: actions/cache/restore@v4
timeout-minutes: 5
with:
path: |
~/.m2/repository/*/*/*
!~/.m2/repository/org/apache/pulsar
!~/.m2/repository/org/owasp/dependency-check-data
key: ${{ runner.os }}-m2-dependencies-core-modules-${{ hashFiles('**/pom.xml') }}
lookup-only: true
restore-keys: |
${{ runner.os }}-m2-dependencies-core-modules-

Expand Down Expand Up @@ -1480,7 +1479,7 @@ jobs:

- name: Restore OWASP Dependency Check data
id: restore-owasp-dependency-check-data
uses: actions/cache/restore@v3
uses: actions/cache/restore@v4
timeout-minutes: 5
with:
path: ~/.m2/repository/org/owasp/dependency-check-data
Expand Down
1 change: 0 additions & 1 deletion distribution/io/pom.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -136,7 +136,6 @@
<plugin>
<groupId>org.owasp</groupId>
<artifactId>dependency-check-maven</artifactId>
<version>${dependency-check-maven.version}</version>
<executions>
<execution>
<goals>
Expand Down
14 changes: 11 additions & 3 deletions pom.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -316,7 +316,7 @@ flexible messaging model and an intuitive client API.</description>
<errorprone-slf4j.version>0.1.21</errorprone-slf4j.version>
<j2objc-annotations.version>1.3</j2objc-annotations.version>
<lightproto-maven-plugin.version>0.4</lightproto-maven-plugin.version>
<dependency-check-maven.version>9.1.0</dependency-check-maven.version>
<dependency-check-maven.version>10.0.1</dependency-check-maven.version>
<roaringbitmap.version>1.0.6</roaringbitmap.version>
<extra-enforcer-rules.version>1.6.1</extra-enforcer-rules.version>
<oshi.version>6.4.0</oshi.version>
Expand Down Expand Up @@ -2192,6 +2192,16 @@ flexible messaging model and an intuitive client API.</description>
<artifactId>build-helper-maven-plugin</artifactId>
<version>${build-helper-maven-plugin.version}</version>
</plugin>
<plugin>
<groupId>org.owasp</groupId>
<artifactId>dependency-check-maven</artifactId>
<version>${dependency-check-maven.version}</version>
<configuration>
<nvdApiKeyEnvironmentVariable>NIST_NVD_API_KEY</nvdApiKeyEnvironmentVariable>
<!-- Uncomment the following to use the NVD data feed provided by the Dependency-Check project -->
<!-- <nvdDatafeedUrl>https://jeremylong.github.io/DependencyCheck/hb_nvd/</nvdDatafeedUrl> -->
dave2wave marked this conversation as resolved.
Show resolved Hide resolved
</configuration>
</plugin>
</plugins>
</pluginManagement>
<extensions>
Expand Down Expand Up @@ -2639,7 +2649,6 @@ flexible messaging model and an intuitive client API.</description>
<plugin>
<groupId>org.owasp</groupId>
<artifactId>dependency-check-maven</artifactId>
<version>${dependency-check-maven.version}</version>
<configuration>
<suppressionFiles>
<suppressionFile>${pulsar.basedir}/src/owasp-dependency-check-false-positives.xml</suppressionFile>
Expand Down Expand Up @@ -2674,7 +2683,6 @@ flexible messaging model and an intuitive client API.</description>
<plugin>
<groupId>org.owasp</groupId>
<artifactId>dependency-check-maven</artifactId>
<version>${dependency-check-maven.version}</version>
<reportSets>
<reportSet>
<reports>
Expand Down
1 change: 0 additions & 1 deletion pulsar-io/docs/pom.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -258,7 +258,6 @@
<plugin>
<groupId>org.owasp</groupId>
<artifactId>dependency-check-maven</artifactId>
<version>${dependency-check-maven.version}</version>
<executions>
<execution>
<goals>
Expand Down
1 change: 0 additions & 1 deletion pulsar-io/flume/pom.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -141,7 +141,6 @@
<plugin>
<groupId>org.owasp</groupId>
<artifactId>dependency-check-maven</artifactId>
<version>${dependency-check-maven.version}</version>
<executions>
<execution>
<goals>
Expand Down
1 change: 0 additions & 1 deletion pulsar-io/hbase/pom.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -108,7 +108,6 @@
<plugin>
<groupId>org.owasp</groupId>
<artifactId>dependency-check-maven</artifactId>
<version>${dependency-check-maven.version}</version>
<executions>
<execution>
<goals>
Expand Down
7 changes: 3 additions & 4 deletions pulsar-io/hdfs2/pom.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -27,14 +27,14 @@
</parent>
<artifactId>pulsar-io-hdfs2</artifactId>
<name>Pulsar IO :: Hdfs2</name>

<dependencies>
<dependency>
<groupId>${project.groupId}</groupId>
<artifactId>pulsar-io-core</artifactId>
<version>${project.version}</version>
</dependency>

<dependency>
<groupId>com.fasterxml.jackson.core</groupId>
<artifactId>jackson-databind</artifactId>
Expand Down Expand Up @@ -74,7 +74,7 @@
<artifactId>commons-lang3</artifactId>
</dependency>
</dependencies>

<build>
<plugins>
<plugin>
Expand Down Expand Up @@ -113,7 +113,6 @@
<plugin>
<groupId>org.owasp</groupId>
<artifactId>dependency-check-maven</artifactId>
<version>${dependency-check-maven.version}</version>
<executions>
<execution>
<goals>
Expand Down
9 changes: 4 additions & 5 deletions pulsar-io/hdfs3/pom.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -27,14 +27,14 @@
</parent>
<artifactId>pulsar-io-hdfs3</artifactId>
<name>Pulsar IO :: Hdfs3</name>

<dependencies>
<dependency>
<groupId>${project.groupId}</groupId>
<artifactId>pulsar-io-core</artifactId>
<version>${project.version}</version>
</dependency>

<dependency>
<groupId>com.fasterxml.jackson.core</groupId>
<artifactId>jackson-databind</artifactId>
Expand All @@ -49,7 +49,7 @@
<groupId>org.apache.commons</groupId>
<artifactId>commons-collections4</artifactId>
</dependency>

<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-client</artifactId>
Expand Down Expand Up @@ -80,7 +80,7 @@
</dependency>

</dependencies>

<build>
<plugins>
<plugin>
Expand Down Expand Up @@ -119,7 +119,6 @@
<plugin>
<groupId>org.owasp</groupId>
<artifactId>dependency-check-maven</artifactId>
<version>${dependency-check-maven.version}</version>
<executions>
<execution>
<goals>
Expand Down
1 change: 0 additions & 1 deletion tiered-storage/file-system/pom.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -208,7 +208,6 @@
<plugin>
<groupId>org.owasp</groupId>
<artifactId>dependency-check-maven</artifactId>
<version>${dependency-check-maven.version}</version>
<executions>
<execution>
<goals>
Expand Down
Loading