build

New field-list, field-rm commands (#3017) #30

Workflow file for this run

	name: "build"

	on:
	push:
	branches: [ "main" ]

	pull_request:
	# The branches below must be a subset of the branches above
	branches: [ "main" ]
	paths-ignore:
	- '*.md'
	- '.*'

	permissions:
	contents: read
	packages: read

	jobs:
	generate-matrix:
	runs-on: ubuntu-latest
	outputs:
	matrix: ${{ steps.set-matrix.outputs.matrix }}
	steps:
	- name: Checkout repository
	uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
	with:
	fetch-depth: 0

	- name: Generate Matrix JSON with yq
	id: set-matrix
	run: \|
	if [ "${{ github.event_name }}" != "push" ]; then
	MATRIX=$(yq -o json eval '{"include": [.include[] \| select(.version \| test("(aarch64\|_arm64)$") \| not)]}' .github/workflows/versions \| jq -c)
	else
	MATRIX=$(yq -o json .github/workflows/versions \| jq -c)
	fi
	echo "matrix=$MATRIX" >> $GITHUB_OUTPUT

	build:
	needs: generate-matrix
	strategy:
	fail-fast: false
	matrix: ${{fromJson(needs.generate-matrix.outputs.matrix)}}

	permissions:
	contents: write
	packages: write

	concurrency:
	group: ${{ github.workflow }}-${{ matrix.version }}-${{ github.event.pull_request.number \|\| github.ref }}
	cancel-in-progress: true

	env:
	PYTHON: ${{ matrix.python \|\| 'python' }}

	runs-on: ${{ matrix.runson \|\| 'ubuntu-latest' }}

	container:
	image: ${{ matrix.container }}

	steps:
	- name: Checkout repository
	uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
	with:
	fetch-depth: 0

	# - name: debug
	# run: \|
	# echo "matrix.version: ${{ matrix.version }}"
	# echo "github.event_name: ${{ github.event_name }}"
	# echo "github.ref: ${{ github.ref }}"
	# echo "github.head_ref: ${{ github.head_ref }}"
	# echo "github.base_ref: ${{ github.base_ref }}"

	- name: thirdparty
	run: \|
	(cd / ; curl https://raw.githubusercontent.com/arkime/arkime-test-data/main/snfmin.tar.gz \| tar -zxvf -)
	if [ -d "/thirdparty" ]; then
	ln -s /thirdparty .
	fi
	git config --global --add safe.directory `pwd`

	- name: build and install
	run: \|
	./easybutton-build.sh ${{ matrix.buildopt }} --rminstall
	export PATH=/opt/arkime/bin:$PATH
	INSTALL_BUNDLE=bundle make install

	- name: lint
	run: \|
	export PATH=/opt/arkime/bin:$PATH
	npm run lint
	cp -r capture/plugins/lua/samples /opt/arkime/lua

	- name: capture tests
	run: \|
	(cd tests; ./tests.pl)

	- name: ui test
	if: ${{ matrix.uitest }}
	run: \|
	export PATH=/opt/arkime/bin:$PATH
	npm run test

	- name: viewer test
	if: ${{ matrix.viewertest == 'normal' }}
	run: \|
	export PATH=/opt/arkime/bin:$PATH
	(chown -R opensearch /opensearch-2.7.0; cd /opensearch-2.7.0 ; su opensearch -c "ES_JAVA_OPTS='-Xms1000m -Xmx1000m' bin/opensearch" > /tmp/os &)
	sleep 30
	cat /tmp/os
	(cd tests ; G_SLICE=always-malloc ./tests.pl --viewer)

	- name: package arch
	if: ${{ matrix.package == 'arch' }}
	run: \|
	gem install --no-document fpm rexml erb
	export ARKIME_VERSION=`sed 's/."$.$\".*$/\1/' /opt/arkime/common/version.js \| head -n 1`
	/root/.local/share/gem/ruby/3.3.0/bin/fpm -s dir -t pacman -n arkime -x opt/arkime/logs -x opt/arkime/raw -v $ARKIME_VERSION --iteration ${{ github.run_id }} --template-scripts --after-install "release/afterinstall.sh" --url "https://arkime.com" --description "Arkime Full Packet System" ${{ matrix.fpmdeps }} -p arkime-${ARKIME_VERSION}-${{github.run_id}}_${{matrix.version}}.pkg.tar.zst /opt/arkime
	ls -l *.zst
	mv *.zst arkime-main.${{matrix.version}}.pkg.tar.zst

	- name: package rpm
	if: ${{ matrix.package == 'rpm' }}
	run: \|
	export ARKIME_VERSION=`sed 's/."$.$\".*$/\1/' /opt/arkime/common/version.js \| head -n 1 \| tr "-" "_"`
	fpm -s dir -t rpm --rpm-digest sha256 -n arkime -x opt/arkime/logs -x opt/arkime/raw -v $ARKIME_VERSION --iteration ${{ github.run_id }} --template-scripts --after-install "release/afterinstall.sh" --url "https://arkime.com" --description "Arkime Full Packet System" ${{ matrix.fpmdeps }} -p arkime-${ARKIME_VERSION}-${{github.run_id}}.${{matrix.version}}.rpm /opt/arkime
	mv *.rpm arkime-main.${{matrix.version}}.rpm
	ls -l *.rpm

	- name: Arkime ENV Variables
	run: \|
	export ARKIME_MAJOR_VERSION=`sed 's/."$.$\".*$/\1/' /opt/arkime/common/version.js \| head -n 1 \| cut -c 1-1`
	echo "ARKIME_MAJOR_VERSION=$ARKIME_MAJOR_VERSION" >> $GITHUB_ENV
	if [ "$ARKIME_MAJOR_VERSION" = "6" ]; then
	echo "ARKIME_DOCKER_TAG=last-commit6" >> $GITHUB_ENV
	else
	echo "ARKIME_DOCKER_TAG=last-commit" >> $GITHUB_ENV
	fi
	arch=$(arch \| sed s/aarch64/arm64/ \| sed s/x86_64/amd64/)
	echo "ARKIME_ARCH=$arch" >> $GITHUB_ENV

	- name: ja4plus
	if: ${{ matrix.ja4plus }}
	run: \|
	(cd .. ; git clone https://github.com/arkime/ja4)
	cp ../ja4/ja4plus.c capture/plugins
	(cd capture/plugins; make)
	(cd ../ja4; make)
	(cd tests; ./tests.pl --extra "-o plugins=ja4plus.so" ../../ja4/pcap/*.pcap)
	mv capture/plugins/ja4plus.so ja4plus.${{ env.ARKIME_ARCH }}.so
	rm -f capture/plugins/ja4plus.c

	- name: moloch el8 package rpm
	if: ${{ matrix.version == 'el8.x86_64' }}
	run: \|
	export ARKIME_VERSION=`sed 's/."$.$\".*$/\1/' /opt/arkime/common/version.js \| head -n 1 \| tr "-" "_"`
	rm -rf /data/moloch; mkdir -p /data
	mv /opt/arkime /data/moloch
	./easybutton-build.sh ${{ matrix.buildopt }} --dir /data/moloch
	(cd capture ; make clean; make; make install)
	(cd release ; make installmoloch)
	mv /data/moloch/bin/capture /data/moloch/bin/moloch-capture
	/bin/cp -f common/version.js /data/moloch/common/

	fpm -s dir -t rpm -n moloch -x data/moloch/logs -x data/molcoh/raw -v $ARKIME_VERSION --iteration ${{ github.run_id }} --template-scripts --url "https://arkime.com" --description "Moloch Full Packet System" ${{ matrix.fpmdeps }} --rpm-rpmbuild-define "_build_id_links none" -p moloch-${ARKIME_VERSION}-${{github.run_id}}.${{matrix.version}}.rpm /data/moloch
	mv moloch*.rpm moloch-main.${{matrix.version}}.rpm
	ls -l *.rpm

	- name: package deb
	if: ${{ matrix.package == 'deb' }}
	run: \|
	export ARKIME_VERSION=`sed 's/."$.$\".*$/\1/' /opt/arkime/common/version.js \| head -n 1`
	fpm -s dir -t deb -n arkime -x opt/arkime/logs -x opt/arkime/raw -v $ARKIME_VERSION --iteration ${{ github.run_id }} --template-scripts --after-install "release/afterinstall.sh" --url "https://arkime.com" --description "Arkime Full Packet System" ${{ matrix.fpmdeps }} -p arkime_${ARKIME_VERSION}-${{github.run_id}}.${{matrix.version}}.deb /opt/arkime
	mv *.deb arkime_main.${{matrix.version}}.deb
	ls -l *.deb

	- name: moloch ubuntu2004 package rpm
	if: ${{ matrix.version == 'ubuntu2004_amd64' }}
	run: \|
	export ARKIME_VERSION=`sed 's/."$.$\".*$/\1/' /opt/arkime/common/version.js \| head -n 1`
	rm -rf /data/moloch; mkdir -p /data
	mv /opt/arkime /data/moloch
	./easybutton-build.sh ${{ matrix.buildopt }} --dir /data/moloch
	(cd capture ; make clean; make; make install)
	(cd release ; make installmoloch)
	mv /data/moloch/bin/capture /data/moloch/bin/moloch-capture
	/bin/cp -f common/version.js /data/moloch/common/

	fpm -s dir -t deb -n moloch -x data/moloch/logs -x data/moloch/raw -v $ARKIME_VERSION --iteration ${{ github.run_id }} --template-scripts --after-install "release/afterinstall.sh" --url "https://arkime.com" --description "Moloch Full Packet System" ${{ matrix.fpmdeps }} -p moloch_${ARKIME_VERSION}-${{github.run_id}}.${{matrix.version}}.deb /data/moloch
	mv moloch*.deb moloch_main.${{matrix.version}}.deb
	ls -l *.deb

	- name: sanitize
	if: ${{ matrix.viewertest == 'sanitize' }}
	run: \|
	(chown -R opensearch /opensearch-2.7.0; cd /opensearch-2.7.0 ; su opensearch -c "ES_JAVA_OPTS='-Xms1000m -Xmx1000m' bin/opensearch" > /tmp/os &)
	(cd capture ; make sanitize CC=clang-18)
	(cd tests; ./tests.pl)
	cat /tmp/os
	export PATH=/opt/arkime/bin:$PATH
	(cd tests ; G_SLICE=always-malloc ./tests.pl --viewer)

	- name: notice
	if: ${{ matrix.notice }}
	run: \|
	export PATH=/opt/arkime/bin:$PATH
	(npm install license-checker; release/notice.txt.pl /opt/arkime NOTICE release/CAPTURENOTICE > NOTICE.txt)
	ls -l NOT*

	- name: upload github
	if: github.event_name == 'push' && github.ref == 'refs/heads/main'
	uses: softprops/[email protected]
	with:
	tag_name: last-commit
	draft: false
	prerelease: true
	generate_release_notes: false
	files: \|
	*.deb
	*.so
	*.rpm
	*.zst
	NOTICE.txt

	- name: tag last-commit
	if: github.event_name == 'push' && github.ref == 'refs/heads/main' && matrix.notice
	run: \|
	if [ "$ARKIME_MAJOR_VERSION" = "5" ]; then
	git tag --force last-commit
	git push --force origin last-commit
	else
	git tag --force last-commit6
	git push --force origin last-commit6
	fi

	############# DOCKER DOCKER DOCKER #############
	- name: Set up Docker Buildx
	if: matrix.docker
	uses: docker/setup-buildx-action@v3

	- name: Log in to registry
	if: matrix.docker
	uses: docker/login-action@v3
	with:
	registry: ghcr.io
	username: ${{ github.actor }}
	password: ${{ secrets.GITHUB_TOKEN }}

	- name: Build and push
	if: matrix.docker
	uses: docker/build-push-action@v5
	with:
	context: .
	file: ./release/Dockerfile
	push: ${{ github.event_name == 'push' }}
	provenance: false
	tags: \|
	ghcr.io/${{ github.repository_owner }}/arkime/arkime:snapshot-v${{ env.ARKIME_MAJOR_VERSION }}-${{ env.ARKIME_ARCH }}-${{ github.run_id }}
	ghcr.io/${{ github.repository_owner }}/arkime/arkime:snapshot-v${{ env.ARKIME_MAJOR_VERSION }}-${{ env.ARKIME_ARCH }}-latest
	build-args: \|
	ARKIME_TAG=${{ env.ARKIME_DOCKER_TAG }}
	ARKIME_BUILD=main

	############# Docker Manifest #############
	create-manifest:
	needs: build
	runs-on: ubuntu-latest
	if: github.event_name == 'push'
	permissions:
	contents: write
	packages: write
	steps:
	- name: Checkout repository
	uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
	with:
	fetch-depth: 0

	- name: Log in to registry
	uses: docker/login-action@v3
	with:
	registry: ghcr.io
	username: ${{ github.actor }}
	password: ${{ secrets.GITHUB_TOKEN }}

	- name: Create and push manifest
	run: \|
	export ARKIME_MAJOR_VERSION=`grep AC_INIT configure.ac \| sed "s/.\[$.$\].*$/\1/" \| cut -c 1-1`

	package="ghcr.io/${{ github.repository_owner }}/arkime/arkime"

	# Create and push the 'snapshot-v{major_version}-{run_id}' tag manifest
	docker manifest create ${package}:snapshot-v${ARKIME_MAJOR_VERSION}-${{ github.run_id }} \
	${package}:snapshot-v${ARKIME_MAJOR_VERSION}-amd64-${{ github.run_id }} \
	${package}:snapshot-v${ARKIME_MAJOR_VERSION}-arm64-${{ github.run_id }}

	docker manifest push ${package}:snapshot-v${ARKIME_MAJOR_VERSION}-${{ github.run_id }}

	# Create and push the 'snapshot-v{major_version}-latest' tag manifest
	docker manifest create ${package}:snapshot-v${ARKIME_MAJOR_VERSION}-latest \
	${package}:snapshot-v${ARKIME_MAJOR_VERSION}-amd64-${{ github.run_id }} \
	${package}:snapshot-v${ARKIME_MAJOR_VERSION}-arm64-${{ github.run_id }}
	docker manifest annotate ${package}:snapshot-v${ARKIME_MAJOR_VERSION}-latest \
	${package}:snapshot-v${ARKIME_MAJOR_VERSION}-amd64-${{ github.run_id }} --arch amd64
	docker manifest annotate ${package}:snapshot-v${ARKIME_MAJOR_VERSION}-latest \
	${package}:snapshot-v${ARKIME_MAJOR_VERSION}-arm64-${{ github.run_id }} --arch arm64
	docker manifest push ${package}:snapshot-v${ARKIME_MAJOR_VERSION}-latest

	############# Send Slack#############
	slack:
	runs-on: ubuntu-latest
	needs: build
	steps:
	- name: Checkout repository
	uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
	with:
	fetch-depth: 0

	- name: send
	env:
	SLACK_URL: ${{ secrets.SLACK_URL }}
	run: \|
	echo "Sending msg"
	export ARKIME_VERSION=`grep AC_INIT configure.ac \| sed "s/.\[$.$\].*$/\1/"`
	echo ARKIME_VERSION: $ARKIME_VERSION
	MSG=`git log -1 --format=%s`
	BODY="{\"icon_emoji\": \":sushi:\", \"username\": \"GitAction\", \"text\":\"Worked: $GIT_BRANCH - arkime-$ARKIME_VERSION-${{github.run_id}} - $MSG\"}"
	curl -XPOST -H "Content-type: application/json" --data "$BODY" $SLACK_URL

Provide feedback

Saved searches

Use saved searches to filter your results more quickly

New field-list, field-rm commands (#3017) #30

Workflow file

New field-list, field-rm commands (#3017) #30

Jobs

Run details

Workflow file for this run