v6.4.0 development

Squashed commit of the following: commit 9f03bda Author: Seth Grover <[email protected]> Date: Mon Oct 17 10:56:57 2022 -0600 Fixes for creating Views in Arkime v4.0 commit 3ce9918 Author: Seth Grover <[email protected]> Date: Thu Oct 13 13:12:28 2022 -0600 fixes to ISO build commit b40df91 Merge: 6edeff3 2e5f653 Author: Seth Grover <[email protected]> Date: Thu Oct 13 10:16:41 2022 -0600 now that arkime v4.0.0 is out, bump internal arkime version commit 6edeff3 Author: Seth Grover <[email protected]> Date: Tue Oct 11 16:10:15 2022 -0600 fix idaholab#120, capa hits parsing (replaced att&ck with attack in parsing) commit 594cb2f Author: Seth Grover <[email protected]> Date: Tue Oct 11 14:30:35 2022 -0600 set bacnet instance_number to long instead of integer commit 106bf61 Author: SG <[email protected]> Date: Mon Oct 10 13:38:20 2022 -0600 add s7comm upload/download log support commit 515c170 Author: SG <[email protected]> Date: Mon Oct 10 13:32:24 2022 -0600 add s7comm upload/download log support commit 143bfcb Author: SG <[email protected]> Date: Mon Oct 10 10:27:13 2022 -0600 i don't think we need to actually run auth_setup prior to packaging commit 4da36a9 Author: SG <[email protected]> Date: Mon Oct 10 08:14:39 2022 -0600 exclusions for documentation commit dfcccc9 Author: SG <[email protected]> Date: Mon Oct 10 08:03:39 2022 -0600 minor documentation tweaks commit da6fad5 Author: Seth Grover <[email protected]> Date: Thu Oct 6 15:23:39 2022 -0600 fix missing link of license.txt commit dccf3f4 Author: Seth Grover <[email protected]> Date: Thu Oct 6 14:51:50 2022 -0600 workflow changes for nginx for docs commit 02fd8a2 Author: Seth Grover <[email protected]> Date: Thu Oct 6 14:49:41 2022 -0600 tweak tgz download commit a093739 Author: Seth Grover <[email protected]> Date: Thu Oct 6 14:39:16 2022 -0600 fix nginx docs commit f55a592 Author: Seth Grover <[email protected]> Date: Thu Oct 6 14:37:00 2022 -0600 fix nginx docs commit 86582a2 Author: Seth Grover <[email protected]> Date: Thu Oct 6 14:30:52 2022 -0600 fix nginx docs commit 5b4ada7 Author: Seth Grover <[email protected]> Date: Thu Oct 6 11:17:17 2022 -0600 add seo tags commit 0dcec0e Author: SG <[email protected]> Date: Thu Oct 6 10:31:51 2022 -0600 fix sensor build workflow commit 99daec2 Author: SG <[email protected]> Date: Thu Oct 6 10:28:10 2022 -0600 fix sensor build workflow commit bcaa9a9 Author: SG <[email protected]> Date: Thu Oct 6 10:10:40 2022 -0600 tweak ignore paths for documenation build commit e273d76 Author: SG <[email protected]> Date: Thu Oct 6 10:08:04 2022 -0600 Tweaks to building documentation commit 19b7ff9 Author: SG <[email protected]> Date: Thu Oct 6 07:09:57 2022 -0600 Tweaks to building documentation commit 9a0661c Author: SG <[email protected]> Date: Thu Oct 6 06:57:29 2022 -0600 don't include the kitchen sink when building documentation commit 964669b Author: SG <[email protected]> Date: Wed Oct 5 15:46:47 2022 -0600 more work on development of documentation to split out into github pages commit d661336 Author: SG <[email protected]> Date: Wed Oct 5 15:27:24 2022 -0600 more work on development of documentation to split out into github pages commit 48ac4f4 Author: SG <[email protected]> Date: Wed Oct 5 15:23:19 2022 -0600 more work on development of documentation to split out into github pages commit f558cd2 Author: SG <[email protected]> Date: Wed Oct 5 15:06:12 2022 -0600 more work on development of documentation to split out into github pages commit 7203a0b Author: SG <[email protected]> Date: Wed Oct 5 13:46:48 2022 -0600 more work on development of documentation to split out into github pages commit 0cc0c38 Author: SG <[email protected]> Date: Wed Oct 5 13:45:22 2022 -0600 more work on development of documentation to split out into github pages commit ac2d3f0 Author: SG <[email protected]> Date: Wed Oct 5 13:36:06 2022 -0600 more work on development of documentation to split out into github pages commit 6573c6d Author: SG <[email protected]> Date: Wed Oct 5 12:37:56 2022 -0600 more work on development of documentation to split out into github pages commit 554e605 Author: SG <[email protected]> Date: Wed Oct 5 12:33:38 2022 -0600 more work on development of documentation to split out into github pages commit 8b4b2c9 Author: SG <[email protected]> Date: Wed Oct 5 12:07:11 2022 -0600 more work on development of documentation to split out into github pages commit ba203f1 Author: Seth Grover <[email protected]> Date: Wed Oct 5 07:36:12 2022 -0600 Add package java-1.8.0-openjdk Seems to be a change in the base Docker image and the current version of logstash fails to build with a reference to missing javac. Add java-1.8.0-openjdk according to error message. commit f73b663 Author: Seth Grover <[email protected]> Date: Mon Oct 3 12:28:32 2022 -0600 bump fluent-bit version in ps1 commit 7bc9ad2 Author: Seth Grover <[email protected]> Date: Fri Sep 23 17:45:59 2022 -0600 for web development commit 6cae2e6 Author: Seth Grover <[email protected]> Date: Fri Sep 23 17:33:41 2022 -0600 minor tweaks for documentation commit 568da6c Author: Seth Grover <[email protected]> Date: Fri Sep 23 17:29:29 2022 -0600 Reworked development for using GitHub pages instead of one monolithic README file Squashed commit of the following: commit 76f4508 Author: Seth Grover <[email protected]> Date: Thu Sep 22 22:56:54 2022 -0600 fix hedgehog images commit 5758e6f Author: Seth Grover <[email protected]> Date: Thu Sep 22 22:54:28 2022 -0600 fix hedgehog images commit c576497 Author: Seth Grover <[email protected]> Date: Thu Sep 22 22:45:13 2022 -0600 experimenting with github pages commit 5029669 Author: Seth Grover <[email protected]> Date: Thu Sep 22 22:40:07 2022 -0600 experimenting with github pages commit b85fec2 Author: Seth Grover <[email protected]> Date: Thu Sep 22 22:33:07 2022 -0600 experimenting with github pages commit 061d2ac Author: Seth Grover <[email protected]> Date: Thu Sep 22 22:29:05 2022 -0600 experimenting with github pages commit 3b5e26a Author: Seth Grover <[email protected]> Date: Thu Sep 22 22:18:37 2022 -0600 experimenting with github pages commit 3f20469 Author: Seth Grover <[email protected]> Date: Thu Sep 22 22:07:39 2022 -0600 experimenting with github pages commit ce521e7 Author: Seth Grover <[email protected]> Date: Thu Sep 22 21:52:45 2022 -0600 experimenting with github pages commit 811a35d Author: Seth Grover <[email protected]> Date: Thu Sep 22 21:37:30 2022 -0600 experimenting with github pages commit e6f4471 Author: Seth Grover <[email protected]> Date: Thu Sep 22 21:32:15 2022 -0600 experimenting with github pages commit f70fd95 Author: Seth Grover <[email protected]> Date: Thu Sep 22 21:23:37 2022 -0600 experimenting with github pages commit 48752eb Author: Seth Grover <[email protected]> Date: Thu Sep 22 16:01:51 2022 -0600 experimenting with github pages commit 6230783 Author: Seth Grover <[email protected]> Date: Thu Sep 22 16:00:35 2022 -0600 experimenting with github pages commit 6321f68 Author: Seth Grover <[email protected]> Date: Thu Sep 22 15:55:58 2022 -0600 experimenting with github pages commit 74a8e8e Author: Seth Grover <[email protected]> Date: Thu Sep 22 15:51:52 2022 -0600 experimenting with github pages commit 216aed2 Author: Seth Grover <[email protected]> Date: Thu Sep 22 15:50:52 2022 -0600 experimenting with github pages commit 7fa1e76 Author: Seth Grover <[email protected]> Date: Thu Sep 22 15:49:01 2022 -0600 experimenting with github pages commit 1c72362 Author: Seth Grover <[email protected]> Date: Thu Sep 22 15:47:31 2022 -0600 experimenting with github pages commit 6ccf841 Author: Seth Grover <[email protected]> Date: Thu Sep 22 15:45:06 2022 -0600 experimenting with github pages commit adc6360 Author: Seth Grover <[email protected]> Date: Thu Sep 22 15:42:42 2022 -0600 experimenting with github pages commit 25964a8 Author: Seth Grover <[email protected]> Date: Thu Sep 22 15:35:02 2022 -0600 experimenting with github pages commit c43e2ac Author: Seth Grover <[email protected]> Date: Thu Sep 22 15:21:01 2022 -0600 experimenting with github pages commit 9871deb Author: Seth Grover <[email protected]> Date: Thu Sep 22 15:19:24 2022 -0600 experimenting with github pages commit 760a1f9 Author: Seth Grover <[email protected]> Date: Thu Sep 22 15:16:40 2022 -0600 experimenting with github pages commit 6ae5032 Author: Seth Grover <[email protected]> Date: Thu Sep 22 15:11:46 2022 -0600 experimenting with github pages commit 0ea9c94 Author: Seth Grover <[email protected]> Date: Thu Sep 22 15:08:57 2022 -0600 experimenting with github pages commit b95b060 Author: Seth Grover <[email protected]> Date: Thu Sep 22 15:05:54 2022 -0600 experimenting with github pages commit 3195c4e Author: Seth Grover <[email protected]> Date: Thu Sep 22 15:04:48 2022 -0600 experimenting with github pages commit a07bc5e Author: Seth Grover <[email protected]> Date: Thu Sep 22 15:02:25 2022 -0600 experimenting with github pages commit d77099f Author: Seth Grover <[email protected]> Date: Thu Sep 22 14:59:06 2022 -0600 experimenting with github pages commit 18f4647 Author: Seth Grover <[email protected]> Date: Thu Sep 22 14:57:31 2022 -0600 experimenting with github pages commit 7a08476 Author: Seth Grover <[email protected]> Date: Thu Sep 22 14:52:19 2022 -0600 experimenting with github pages commit acf2a6d Author: Seth Grover <[email protected]> Date: Thu Sep 22 14:47:56 2022 -0600 experimenting with github pages commit 26029bc Author: Seth Grover <[email protected]> Date: Thu Sep 22 14:46:35 2022 -0600 experimenting with github pages commit 60cdab0 Author: Seth Grover <[email protected]> Date: Thu Sep 22 14:42:13 2022 -0600 experimenting with github pages commit 39e88b6 Author: Seth Grover <[email protected]> Date: Thu Sep 22 14:41:13 2022 -0600 experimenting with github pages commit 651acd3 Author: Seth Grover <[email protected]> Date: Thu Sep 22 14:39:38 2022 -0600 experimenting with github pages commit df96e0e Author: Seth Grover <[email protected]> Date: Thu Sep 22 14:36:26 2022 -0600 experimenting with github pages commit 5016081 Author: Seth Grover <[email protected]> Date: Thu Sep 22 14:22:46 2022 -0600 experimenting with github pages commit f1bff36 Author: Seth Grover <[email protected]> Date: Thu Sep 22 14:21:24 2022 -0600 experimenting with github pages commit 0e0d9f0 Author: Seth Grover <[email protected]> Date: Thu Sep 22 14:18:34 2022 -0600 experimenting with github pages commit e170422 Author: Seth Grover <[email protected]> Date: Thu Sep 22 14:03:54 2022 -0600 experimenting with github pages commit 63de7bb Author: Seth Grover <[email protected]> Date: Thu Sep 22 14:00:54 2022 -0600 Revert "experimenting with github pages" This reverts commit f43a4aa. commit f43a4aa Author: Seth Grover <[email protected]> Date: Thu Sep 22 13:59:10 2022 -0600 experimenting with github pages commit b9925dc Author: Seth Grover <[email protected]> Date: Thu Sep 22 13:38:46 2022 -0600 experimenting with github pages commit 41528fb Author: Seth Grover <[email protected]> Date: Thu Sep 22 13:34:51 2022 -0600 experimenting with github pages commit efd3c88 Author: Seth Grover <[email protected]> Date: Thu Sep 22 13:32:01 2022 -0600 experimenting with github pages commit e0f4466 Author: Seth Grover <[email protected]> Date: Thu Sep 22 13:30:37 2022 -0600 experimenting with github pages commit 8b8d469 Author: Seth Grover <[email protected]> Date: Thu Sep 22 13:28:06 2022 -0600 experimenting with github pages commit 9c00ea2 Author: Seth Grover <[email protected]> Date: Thu Sep 22 13:21:43 2022 -0600 experimenting with github pages commit 1a0df24 Author: Seth Grover <[email protected]> Date: Thu Sep 22 13:16:25 2022 -0600 experimenting with github pages commit b7ae1d2 Author: Seth Grover <[email protected]> Date: Thu Sep 22 12:54:49 2022 -0600 basic config commit 208ef01 Author: Seth Grover <[email protected]> Date: Thu Sep 22 12:53:30 2022 -0600 experimenting with jekyll commit 8aea3e2 Author: Seth Grover <[email protected]> Date: Thu Sep 22 12:43:41 2022 -0600 links work in progress commit 1605844 Author: Seth Grover <[email protected]> Date: Thu Sep 22 12:37:25 2022 -0600 Added github pages config commit 599eb83 Author: Seth Grover <[email protected]> Date: Thu Sep 22 12:25:14 2022 -0600 Added github pages config commit 73754a4 Author: Seth Grover <[email protected]> Date: Thu Sep 22 12:18:58 2022 -0600 documentation links work in progress commit 03012af Author: Seth Grover <[email protected]> Date: Thu Sep 22 12:18:19 2022 -0600 documentation links work in progress commit 3b8cd74 Author: Seth Grover <[email protected]> Date: Thu Sep 22 12:00:57 2022 -0600 documentation links work in progress commit 7b13fa7 Author: Seth Grover <[email protected]> Date: Thu Sep 22 11:59:20 2022 -0600 documentation links work in progress commit 52df01b Author: Seth Grover <[email protected]> Date: Thu Sep 22 11:54:45 2022 -0600 documentation links work in progress commit b7ac174 Author: Seth Grover <[email protected]> Date: Thu Sep 22 11:02:37 2022 -0600 testing relative links commit 952936d Author: Seth Grover <[email protected]> Date: Thu Sep 22 10:47:47 2022 -0600 split all the .md documentation into different files commit 6a7003a Author: Seth Grover <[email protected]> Date: Wed Sep 21 13:54:31 2022 -0600 simplify adjustment of netbox unit file commit 1896e24 Author: Seth Grover <[email protected]> Date: Wed Sep 21 13:29:05 2022 -0600 Using /assets/ for netbox conflicts with Arkime's /assets/, so use /netbox/ for the NetBox path commit 5ca383d Author: Seth Grover <[email protected]> Date: Wed Sep 21 13:26:09 2022 -0600 Using /assets/ for netbox conflicts with Arkime's /assets/, so use /netbox/ for the NetBox path commit 2cf383b Author: Seth Grover <[email protected]> Date: Wed Sep 21 12:03:26 2022 -0600 Fix depends commit 06a1369 Author: Seth Grover <[email protected]> Date: Tue Sep 20 19:15:28 2022 -0600 tweaks for ISO commit 32caf88 Author: Seth Grover <[email protected]> Date: Tue Sep 20 15:10:50 2022 -0600 fix packaging commit 28969ea Author: Seth Grover <[email protected]> Date: Tue Sep 20 14:36:40 2022 -0600 readme update commit bc6c9ea Author: Seth Grover <[email protected]> Date: Tue Sep 20 14:32:38 2022 -0600 update poster commit ff402b3 Author: Seth Grover <[email protected]> Date: Tue Sep 20 14:10:41 2022 -0600 slide update commit 8cd74d7 Author: Seth Grover <[email protected]> Date: Tue Sep 20 13:30:52 2022 -0600 build fixes commit 87d8b0d Merge: 90f40dc 85c764e Author: Seth Grover <[email protected]> Date: Tue Sep 20 13:14:36 2022 -0600 Merge branch 'netbox' of https://github.com/mmguero-dev/Malcolm into development commit 85c764e Author: Seth Grover <[email protected]> Date: Tue Sep 20 13:13:56 2022 -0600 readme commit 12d99f7 Author: Seth Grover <[email protected]> Date: Tue Sep 20 13:12:21 2022 -0600 readme commit 94948fb Author: Seth Grover <[email protected]> Date: Tue Sep 20 12:26:30 2022 -0600 readme commit 357fde6 Author: Seth Grover <[email protected]> Date: Tue Sep 20 12:16:21 2022 -0600 netbox wip commit f6ecce5 Author: Seth Grover <[email protected]> Date: Tue Sep 20 11:58:15 2022 -0600 netbox wip commit bcd0ee9 Author: Seth Grover <[email protected]> Date: Tue Sep 20 11:20:52 2022 -0600 netbox wip commit 41a1706 Author: Seth Grover <[email protected]> Date: Tue Sep 20 11:08:21 2022 -0600 initialize netbox on startup commit 7c0d37a Author: Seth Grover <[email protected]> Date: Tue Sep 20 10:21:21 2022 -0600 cleaner disabling of netbox by default commit 58c21aa Author: Seth Grover <[email protected]> Date: Tue Sep 20 09:27:51 2022 -0600 use tini for init on docker containers; allow netbox to be toggled commit d048835 Author: Seth Grover <[email protected]> Date: Tue Sep 20 06:56:09 2022 -0600 Bump zeek to v5.0.2 commit 90f40dc Author: Seth Grover <[email protected]> Date: Tue Sep 20 06:56:09 2022 -0600 Bump zeek to v5.0.2 commit 9e6fbee Author: Seth Grover <[email protected]> Date: Mon Sep 19 21:52:31 2022 -0600 netbox wip commit b2a043f Author: Seth Grover <[email protected]> Date: Mon Sep 19 21:44:29 2022 -0600 netbox wip commit 481e01f Author: Seth Grover <[email protected]> Date: Mon Sep 19 21:34:19 2022 -0600 netbox WIP commit d348641 Author: Seth Grover <[email protected]> Date: Mon Sep 19 20:35:56 2022 -0600 build docker images specifically for malcolm commit cc907c2 Author: SG <[email protected]> Date: Mon Sep 19 15:40:30 2022 -0600 Fix envs for netbox commit 382d8ea Author: SG <[email protected]> Date: Mon Sep 19 15:17:21 2022 -0600 initial swag at netbox integration commit 639d251 Author: SG <[email protected]> Date: Mon Sep 19 14:27:26 2022 -0600 bump to version 6.4.0 for idaholab#17 commit 29ed8dd Author: Seth Grover <[email protected]> Date: Thu Sep 15 07:10:15 2022 -0600 fix broken visualization commit ccd1e25 Author: Seth Grover <[email protected]> Date: Wed Sep 14 15:08:16 2022 -0600 bump opensearch and dashboards to v2.3.0 commit 18f9cbf Author: SG <[email protected]> Date: Tue Sep 13 08:55:47 2022 -0600 update audit rules commit 60b3ff0 Author: SG <[email protected]> Date: Tue Sep 13 08:44:15 2022 -0600 set boot grub permission at the end of preseed commit 7c1a011 Author: Seth Grover <[email protected]> Date: Mon Sep 12 16:17:30 2022 -0600 documentation tweaks for hardening (see idaholab#111) commit 332e7d1 Author: Seth Grover <[email protected]> Date: Mon Sep 12 16:16:57 2022 -0600 documentation tweaks for hardening (see idaholab#111) commit 2a26bba Author: Seth Grover <[email protected]> Date: Mon Sep 12 16:16:32 2022 -0600 documentation tweaks for hardening (see idaholab#111) commit a48b12e Author: Seth Grover <[email protected]> Date: Mon Sep 12 16:13:19 2022 -0600 documentation tweaks for hardening (see idaholab#111) commit f3cb433 Author: Seth Grover <[email protected]> Date: Mon Sep 12 15:35:14 2022 -0600 work in progress for updates to documentation for hardening, see idaholab#111 commit 6ac88e0 Author: Seth Grover <[email protected]> Date: Mon Sep 12 14:35:20 2022 -0600 documentation and script changes for switching to just using hte harbian-audit project commit 5eee892 Author: Seth Grover <[email protected]> Date: Thu Sep 8 13:10:36 2022 -0600 bump version to v6.3.1 for development commit 59e24b1 Author: Seth Grover <[email protected]> Date: Thu Sep 8 08:21:25 2022 -0600 Added template slide commit 2e5f653 Author: Seth Grover <[email protected]> Date: Mon Aug 15 09:41:34 2022 -0600 set userAuthIps to work with docker commit 8185a91 Author: Seth Grover <[email protected]> Date: Mon Aug 15 09:12:18 2022 -0600 fix build commit 7481a86 Author: Seth Grover <[email protected]> Date: Mon Aug 15 08:54:33 2022 -0600 trying build of main branch
cisagov · Oct 17, 2022 · e331d49 · e331d49
1 parent 0ed90ba
commit e331d49
Show file tree

Hide file tree

Showing 302 changed files with 7,765 additions and 6,075 deletions.
diff --git a/.dockerignore b/.dockerignore
@@ -17,13 +17,20 @@
 .tmp
 docker-compose*yml
 Dockerfiles
+Gemfile.lock
 opensearch
 opensearch-backup
 arkime-logs
 arkime-raw
+malcolm-iso
+sensor-iso
 nginx/nginx_ldap.conf
 pcap
+_site
 scripts
 !scripts/malcolm_common.py
 zeek-logs
 suricata-logs
+netbox/netbox/media
+netbox/netbox/postgres
+netbox/netbox/redis
diff --git a/.github/workflows/malcolm-iso-build-docker-wrap-push-ghcr.yml b/.github/workflows/malcolm-iso-build-docker-wrap-push-ghcr.yml
@@ -59,7 +59,6 @@ jobs:
                gnupg2 \
                imagemagick \
                jq \
-               pandoc \
                po4a \
                rsync \
                software-properties-common \

diff --git a/.github/workflows/netbox-build-and-push-ghcr.yml b/.github/workflows/netbox-build-and-push-ghcr.yml
@@ -0,0 +1,62 @@
+name: netbox-build-and-push-ghcr
+
+on:
+  push:
+    branches:
+      - main
+      - development
+    paths:
+      - 'netbox/**'
+      - 'Dockerfiles/netbox.Dockerfile'
+      - 'shared/bin/*'
+      - '.trigger_workflow_build'
+  workflow_dispatch:
+  repository_dispatch:
+
+jobs:
+  docker:
+    runs-on: ubuntu-latest
+    permissions:
+      actions: write
+      packages: write
+      contents: read
+    steps:
+      -
+        name: Cancel previous run in progress
+        uses: styfle/[email protected]
+        with:
+          ignore_sha: true
+          all_but_latest: true
+          access_token: ${{ secrets.GITHUB_TOKEN }}
+      -
+        name: Checkout
+        uses: actions/checkout@v2
+      -
+        name: Extract branch name
+        shell: bash
+        run: echo "##[set-output name=branch;]$(echo ${GITHUB_REF#refs/heads/})"
+        id: extract_branch
+      -
+        name: Set up QEMU
+        uses: docker/setup-qemu-action@v1
+      -
+        name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v1
+        with:
+          driver-opts: |
+            image=moby/buildkit:master
+      -
+        name: Log in to registry
+        uses: docker/login-action@v1
+        with:
+          registry: ghcr.io
+          username: ${{ github.repository_owner }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+      -
+        name: Build and push
+        uses: docker/build-push-action@v2
+        with:
+          context: .
+          file: ./Dockerfiles/netbox.Dockerfile
+          push: true
+          tags: ghcr.io/${{ github.repository_owner }}/malcolmnetsec/netbox:${{ steps.extract_branch.outputs.branch }}
diff --git a/.github/workflows/nginx-build-and-push-ghcr.yml b/.github/workflows/nginx-build-and-push-ghcr.yml
@@ -10,6 +10,12 @@ on:
       - 'Dockerfiles/nginx.Dockerfile'
       - 'shared/bin/*'
       - '.trigger_workflow_build'
+      - '_config.yml'
+      - '_includes/**'
+      - '_layouts/**'
+      - 'docs/**'
+      - 'Gemfile'
+      - 'README.md'
   workflow_dispatch:
   repository_dispatch:
 

diff --git a/.github/workflows/postgresql-build-and-push-ghcr.yml b/.github/workflows/postgresql-build-and-push-ghcr.yml
@@ -0,0 +1,61 @@
+name: postgresql-build-and-push-ghcr
+
+on:
+  push:
+    branches:
+      - main
+      - development
+    paths:
+      - 'Dockerfiles/postgresql.Dockerfile'
+      - 'shared/bin/*'
+      - '.trigger_workflow_build'
+  workflow_dispatch:
+  repository_dispatch:
+
+jobs:
+  docker:
+    runs-on: ubuntu-latest
+    permissions:
+      actions: write
+      packages: write
+      contents: read
+    steps:
+      -
+        name: Cancel previous run in progress
+        uses: styfle/[email protected]
+        with:
+          ignore_sha: true
+          all_but_latest: true
+          access_token: ${{ secrets.GITHUB_TOKEN }}
+      -
+        name: Checkout
+        uses: actions/checkout@v2
+      -
+        name: Extract branch name
+        shell: bash
+        run: echo "##[set-output name=branch;]$(echo ${GITHUB_REF#refs/heads/})"
+        id: extract_branch
+      -
+        name: Set up QEMU
+        uses: docker/setup-qemu-action@v1
+      -
+        name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v1
+        with:
+          driver-opts: |
+            image=moby/buildkit:master
+      -
+        name: Log in to registry
+        uses: docker/login-action@v1
+        with:
+          registry: ghcr.io
+          username: ${{ github.repository_owner }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+      -
+        name: Build and push
+        uses: docker/build-push-action@v2
+        with:
+          context: .
+          file: ./Dockerfiles/postgresql.Dockerfile
+          push: true
+          tags: ghcr.io/${{ github.repository_owner }}/malcolmnetsec/postgresql:${{ steps.extract_branch.outputs.branch }}
diff --git a/.github/workflows/redis-build-and-push-ghcr.yml b/.github/workflows/redis-build-and-push-ghcr.yml
@@ -0,0 +1,61 @@
+name: redis-build-and-push-ghcr
+
+on:
+  push:
+    branches:
+      - main
+      - development
+    paths:
+      - 'Dockerfiles/redis.Dockerfile'
+      - 'shared/bin/*'
+      - '.trigger_workflow_build'
+  workflow_dispatch:
+  repository_dispatch:
+
+jobs:
+  docker:
+    runs-on: ubuntu-latest
+    permissions:
+      actions: write
+      packages: write
+      contents: read
+    steps:
+      -
+        name: Cancel previous run in progress
+        uses: styfle/[email protected]
+        with:
+          ignore_sha: true
+          all_but_latest: true
+          access_token: ${{ secrets.GITHUB_TOKEN }}
+      -
+        name: Checkout
+        uses: actions/checkout@v2
+      -
+        name: Extract branch name
+        shell: bash
+        run: echo "##[set-output name=branch;]$(echo ${GITHUB_REF#refs/heads/})"
+        id: extract_branch
+      -
+        name: Set up QEMU
+        uses: docker/setup-qemu-action@v1
+      -
+        name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v1
+        with:
+          driver-opts: |
+            image=moby/buildkit:master
+      -
+        name: Log in to registry
+        uses: docker/login-action@v1
+        with:
+          registry: ghcr.io
+          username: ${{ github.repository_owner }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+      -
+        name: Build and push
+        uses: docker/build-push-action@v2
+        with:
+          context: .
+          file: ./Dockerfiles/redis.Dockerfile
+          push: true
+          tags: ghcr.io/${{ github.repository_owner }}/malcolmnetsec/redis:${{ steps.extract_branch.outputs.branch }}
diff --git a/.github/workflows/sensor-iso-build-docker-wrap-push-ghcr.yml b/.github/workflows/sensor-iso-build-docker-wrap-push-ghcr.yml
@@ -59,7 +59,6 @@ jobs:
                gnupg2 \
                imagemagick \
                jq \
-               pandoc \
                po4a \
                rsync \
                software-properties-common \
@@ -86,16 +85,18 @@ jobs:
       -
         name: Build image
         run: |
-          cp -r ./shared ./sensor-iso
+          cp -r ./shared ./docs ./_config.yml ./_includes ./_layouts ./Gemfile ./README.md ./sensor-iso
+          cp ./scripts/documentation_build.sh ./sensor-iso/docs/
           cp -r ./arkime/patch ./sensor-iso/shared/arkime_patch
           pushd ./sensor-iso
           echo "${{ steps.extract_malcolm_version.outputs.mversion }}" > ./shared/version.txt
           echo "${{ secrets.MAXMIND_GEOIP_DB_LICENSE_KEY }}" > ./shared/maxmind_license.txt
           echo "GITHUB_TOKEN=${{ secrets.GITHUB_TOKEN }}" > ./shared/environment.chroot
           sudo /usr/bin/env bash ./build.sh
-          rm -rf ./shared/
+          rm -rf ./shared/ ./docs/ ./_config.yml ./_includes ./_layouts /Gemfile ./README.md
           sudo chmod 644 ./hedgehog-*.*
           popd
+
       -
         name: ghcr.io login
         uses: docker/login-action@v1

diff --git a/.gitignore b/.gitignore
@@ -24,6 +24,15 @@
 malcolm_*images.tar.gz
 *.iso
 *-build.log
+Gemfile.lock
+_site
+sensor-iso/_config.yml
+sensor-iso/_includes
+sensor-iso/_layouts
+sensor-iso/_site
+sensor-iso/docs
+sensor-iso/Gemfile
+sensor-iso/README.md
 
 # Byte-compiled / optimized / DLL files
 __pycache__/

diff --git a/Dockerfiles/api.Dockerfile b/Dockerfiles/api.Dockerfile
@@ -76,7 +76,7 @@ COPY shared/bin/opensearch_status.sh "${APP_HOME}"/
 ADD shared/bin/docker-uid-gid-setup.sh /usr/local/bin/
 RUN    apt-get -q update \
     && apt-get -y -q --no-install-recommends upgrade \
-    && apt-get -y -q --no-install-recommends install curl netcat \
+    && apt-get -y -q --no-install-recommends install curl netcat tini \
     && python3 -m pip install --upgrade pip \
     && python3 -m pip install --no-cache /wheels/* \
     && chmod 755 /usr/local/bin/docker-uid-gid-setup.sh \
@@ -89,7 +89,7 @@ RUN    apt-get -q update \
 
 EXPOSE 5000
 
-ENTRYPOINT ["/usr/local/bin/docker-uid-gid-setup.sh", "${APP_HOME}/entrypoint.sh"]
+ENTRYPOINT ["/usr/bin/tini", "--", "/usr/local/bin/docker-uid-gid-setup.sh", "${APP_HOME}/entrypoint.sh"]
 
 # to be populated at build-time:
 ARG BUILD_DATE

diff --git a/Dockerfiles/arkime.Dockerfile b/Dockerfiles/arkime.Dockerfile
@@ -4,17 +4,14 @@ FROM debian:11-slim AS build
 
 ENV DEBIAN_FRONTEND noninteractive
 
-ENV ARKIME_VERSION "3.4.2"
+ENV ARKIME_VERSION "v4.0.0"
 ENV ARKIME_DIR "/opt/arkime"
 ENV ARKIME_URL "https://github.com/arkime/arkime.git"
 ENV ARKIME_LOCALELASTICSEARCH no
 ENV ARKIME_INET yes
 
 ADD arkime/scripts/bs4_remove_div.py /opt/
 ADD arkime/patch/* /opt/patches/
-ADD README.md $ARKIME_DIR/doc/
-ADD docs/doc.css $ARKIME_DIR/doc/
-ADD docs/images $ARKIME_DIR/doc/images/
 
 RUN apt-get -q update && \
     apt-get -y -q --no-install-recommends upgrade && \
@@ -31,7 +28,6 @@ RUN apt-get -q update && \
         git-core \
         groff \
         groff-base \
-        imagemagick \
         libcap-dev \
         libjson-perl \
         libkrb5-dev \
@@ -44,32 +40,20 @@ RUN apt-get -q update && \
         make \
         meson \
         ninja-build \
-        pandoc \
         patch \
         python3-dev \
         python3-pip \
         python3-setuptools \
         python3-wheel \
-        rename \
         sudo \
         swig \
         wget \
         zlib1g-dev && \
   pip3 install --no-cache-dir beautifulsoup4 && \
-  cd $ARKIME_DIR/doc/images && \
-    find . -name "*.png" -exec bash -c 'convert "{}" -fuzz 2% -transparent white -background white -alpha remove -strip -interlace Plane -quality 85% "{}.jpg" && rename "s/\.png//" "{}.jpg"' \; && \
-    cd $ARKIME_DIR/doc && \
-    sed -i "s/^# Malcolm$//" README.md && \
-    sed -i '/./,$!d' README.md && \
-    sed -i "s/.png/.jpg/g" README.md && \
-    sed -i "s@docs/images@images@g" README.md && \
-    sed -i 's/\!\[.*\](.*\/badge.svg)//g' README.md && \
-    pandoc -s --self-contained --metadata title="Malcolm README" --css $ARKIME_DIR/doc/doc.css -o $ARKIME_DIR/doc/README.html $ARKIME_DIR/doc/README.md && \
   cd /opt && \
-    git clone --depth=1 --single-branch --recurse-submodules --shallow-submodules --no-tags --branch="v$ARKIME_VERSION" "$ARKIME_URL" "./arkime-"$ARKIME_VERSION && \
+    git clone --recurse-submodules --branch="$ARKIME_VERSION" "$ARKIME_URL" "./arkime-"$ARKIME_VERSION && \
     cd "./arkime-"$ARKIME_VERSION && \
     bash -c 'for i in /opt/patches/*; do patch -p 1 -r - --no-backup-if-mismatch < $i || true; done' && \
-    find $ARKIME_DIR/doc/images/screenshots -name "*.png" -delete && \
     export PATH="$ARKIME_DIR/bin:${PATH}" && \
     ln -sfr $ARKIME_DIR/bin/npm /usr/local/bin/npm && \
     ln -sfr $ARKIME_DIR/bin/node /usr/local/bin/node && \
@@ -156,6 +140,7 @@ RUN sed -i "s/bullseye main/bullseye main contrib non-free/g" /etc/apt/sources.l
       file \
       geoip-bin \
       gettext \
+      jq \
       libcap2-bin \
       libjson-perl \
       libkrb5-3 \
@@ -178,6 +163,7 @@ RUN sed -i "s/bullseye main/bullseye main contrib non-free/g" /etc/apt/sources.l
       supervisor \
       vim-tiny \
       wget \
+      tini \
       tar gzip unzip cpio bzip2 lzma xz-utils p7zip-full unrar zlib1g && \
     pip3 install --no-cache-dir beautifulsoup4 pyzmq && \
     ln -sfr $ARKIME_DIR/bin/npm /usr/local/bin/npm && \
@@ -231,7 +217,7 @@ ENV PATH="/opt:$ARKIME_DIR/bin:${PATH}"
 EXPOSE 8000 8005 8081
 WORKDIR $ARKIME_DIR
 
-ENTRYPOINT ["/usr/local/bin/docker-uid-gid-setup.sh", "/opt/docker_entrypoint.sh"]
+ENTRYPOINT ["/usr/bin/tini", "--", "/usr/local/bin/docker-uid-gid-setup.sh", "/opt/docker_entrypoint.sh"]
 
 CMD ["/usr/bin/supervisord", "-c", "/etc/supervisord.conf", "-n"]
 

diff --git a/Dockerfiles/dashboards-helper.Dockerfile b/Dockerfiles/dashboards-helper.Dockerfile
@@ -69,7 +69,7 @@ ADD scripts/malcolm_common.py /data/
 
 RUN apk update --no-cache && \
     apk upgrade --no-cache && \
-    apk --no-cache add bash python3 py3-pip curl openssl procps psmisc npm shadow jq && \
+    apk --no-cache add bash python3 py3-pip curl openssl procps psmisc npm shadow jq tini && \
     npm install -g http-server && \
     pip3 install supervisor humanfriendly requests && \
     curl -fsSLO "$SUPERCRONIC_URL" && \
@@ -97,7 +97,7 @@ RUN apk update --no-cache && \
 
 EXPOSE $OFFLINE_REGION_MAPS_PORT
 
-ENTRYPOINT ["/usr/local/bin/docker-uid-gid-setup.sh"]
+ENTRYPOINT ["/sbin/tini", "--", "/usr/local/bin/docker-uid-gid-setup.sh"]
 
 CMD ["/usr/bin/supervisord", "-c", "/etc/supervisord.conf", "-n"]