diff --git a/Dockerfiles/arkime.Dockerfile b/Dockerfiles/arkime.Dockerfile index a3d999985..82aea7952 100644 --- a/Dockerfiles/arkime.Dockerfile +++ b/Dockerfiles/arkime.Dockerfile @@ -4,7 +4,7 @@ FROM debian:11-slim AS build ENV DEBIAN_FRONTEND noninteractive -ENV ARKIME_VERSION "3.4.2" +ENV ARKIME_VERSION "v4.0.0" ENV ARKIME_DIR "/opt/arkime" ENV ARKIME_URL "https://github.com/arkime/arkime.git" ENV ARKIME_LOCALELASTICSEARCH no @@ -51,7 +51,7 @@ RUN apt-get -q update && \ zlib1g-dev && \ pip3 install --no-cache-dir beautifulsoup4 && \ cd /opt && \ - git clone --depth=1 --single-branch --recurse-submodules --shallow-submodules --no-tags --branch="v$ARKIME_VERSION" "$ARKIME_URL" "./arkime-"$ARKIME_VERSION && \ + git clone --recurse-submodules --branch="$ARKIME_VERSION" "$ARKIME_URL" "./arkime-"$ARKIME_VERSION && \ cd "./arkime-"$ARKIME_VERSION && \ bash -c 'for i in /opt/patches/*; do patch -p 1 -r - --no-backup-if-mismatch < $i || true; done' && \ export PATH="$ARKIME_DIR/bin:${PATH}" && \ diff --git a/Dockerfiles/pcap-capture.Dockerfile b/Dockerfiles/pcap-capture.Dockerfile index 7c5a0a2b0..399900328 100644 --- a/Dockerfiles/pcap-capture.Dockerfile +++ b/Dockerfiles/pcap-capture.Dockerfile @@ -34,8 +34,8 @@ ARG PCAP_IFACE=lo ARG PCAP_IFACE_TWEAK=false ARG PCAP_NETSNIFF_MAGIC=0xa1b2c3d4 ARG PCAP_TCPDUMP_FILENAME_PATTERN=%Y%m%d%H%M%S.pcap -ARG PCAP_ROTATE_MINUTES=30 -ARG PCAP_ROTATE_MEGABYTES=500 +ARG PCAP_ROTATE_MINUTES=10 +ARG PCAP_ROTATE_MEGABYTES=4096 ARG PCAP_PATH=/pcap ARG PCAP_FILTER= ARG PCAP_SNAPLEN=0 diff --git a/arkime/etc/config.ini b/arkime/etc/config.ini index e4b5ee36a..1b86eea47 100644 --- a/arkime/etc/config.ini +++ b/arkime/etc/config.ini @@ -9,12 +9,13 @@ cronQueries=true rotateIndex=daily passwordSecret=Malcolm httpRealm=Arkime +userAuthIps=::,0.0.0.0/0 interface=eth0 wiseHost=127.0.0.1 wisePort=8081 pcapDir=/data/pcap/processed readTruncatedPackets=true -maxFileSizeG=1 +maxFileSizeG=4 tcpTimeout=600 tcpSaveTimeout=720 udpTimeout=30 diff --git a/arkime/patch/capture_event_dataset.patch b/arkime/patch/capture_event_dataset.patch deleted file mode 100644 index a8862c7d6..000000000 --- a/arkime/patch/capture_event_dataset.patch +++ /dev/null @@ -1,34 +0,0 @@ -diff --git a/capture/db.c b/capture/db.c -index 2515b177..0a6d06bd 100644 ---- a/capture/db.c -+++ b/capture/db.c -@@ -71,6 +71,7 @@ LOCAL int dbExit; - LOCAL char *esBulkQuery; - LOCAL int esBulkQueryLen; - LOCAL char *ecsEventProvider; -+LOCAL char *ecsEventDataset; - - extern uint64_t packetStats[MOLOCH_PACKET_MAX]; - -@@ -906,8 +907,12 @@ void moloch_db_save_session(MolochSession_t *session, int final) - } - BSB_EXPORT_cstr(jbsb, "],"); - -- if (ecsEventProvider) { -+ if (ecsEventProvider && ecsEventDataset) { -+ BSB_EXPORT_sprintf(jbsb, "\"event\":{\"provider\":\"%s\", \"dataset\":\"%s\"},", ecsEventProvider, ecsEventDataset); -+ } else if (ecsEventProvider) { - BSB_EXPORT_sprintf(jbsb, "\"event\":{\"provider\":\"%s\"},", ecsEventProvider); -+ } else if (ecsEventDataset) { -+ BSB_EXPORT_sprintf(jbsb, "\"event\":{\"dataset\":\"%s\"},", ecsEventDataset); - } - - int inGroupNum = 0; -@@ -2661,6 +2666,7 @@ void moloch_db_init() - } - - ecsEventProvider = moloch_config_str(NULL, "ecsEventProvider", NULL); -+ ecsEventDataset = moloch_config_str(NULL, "ecsEventDataset", NULL); - - int thread; - for (thread = 0; thread < config.packetThreads; thread++) { diff --git a/arkime/patch/db_pl_quiet_backup_warning.patch b/arkime/patch/db_pl_quiet_backup_warning.patch index 4e43ebb2f..49248c6be 100644 --- a/arkime/patch/db_pl_quiet_backup_warning.patch +++ b/arkime/patch/db_pl_quiet_backup_warning.patch @@ -1,8 +1,8 @@ diff --git a/db/db.pl b/db/db.pl -index 1b24b6a1..c8034ec4 100755 +index bcb3948a..995c7e4e 100755 --- a/db/db.pl +++ b/db/db.pl -@@ -7111,7 +7111,7 @@ my $health = dbCheckHealth(); +@@ -7383,7 +7383,7 @@ my $health = dbCheckHealth(); my $nodes = esGet("/_nodes"); $main::numberOfNodes = dataNodes($nodes->{nodes}); diff --git a/arkime/patch/field_best_priority.patch b/arkime/patch/field_best_priority.patch deleted file mode 100644 index 799be8d9e..000000000 --- a/arkime/patch/field_best_priority.patch +++ /dev/null @@ -1,13 +0,0 @@ -diff --git a/capture/db.c b/capture/db.c -index c1350507..b29a41ac 100644 ---- a/capture/db.c -+++ b/capture/db.c -@@ -2411,7 +2411,7 @@ void moloch_db_add_field(char *group, char *kind, char *expression, char *friend - } - - BSB_EXPORT_u08(bsb, '}'); -- moloch_http_schedule(esServer, "POST", key, key_len, json, BSB_LENGTH(bsb), NULL, MOLOCH_HTTP_PRIORITY_NORMAL, NULL, NULL); -+ moloch_http_schedule(esServer, "POST", key, key_len, json, BSB_LENGTH(bsb), NULL, MOLOCH_HTTP_PRIORITY_BEST, NULL, NULL); - } - /******************************************************************************/ - void moloch_db_update_field(char *expression, char *name, char *value) diff --git a/arkime/patch/fields_db_max_5000.patch b/arkime/patch/fields_db_max_5000.patch index b4799e580..cf5b5e0cb 100644 --- a/arkime/patch/fields_db_max_5000.patch +++ b/arkime/patch/fields_db_max_5000.patch @@ -1,8 +1,8 @@ diff --git a/capture/db.c b/capture/db.c -index c1350507..1ae57143 100644 +index 70be2c19..dd3bfad9 100644 --- a/capture/db.c +++ b/capture/db.c -@@ -2325,7 +2325,7 @@ LOCAL void moloch_db_load_fields() +@@ -2321,7 +2321,7 @@ LOCAL void moloch_db_load_fields() char key[100]; int key_len; @@ -12,7 +12,7 @@ index c1350507..1ae57143 100644 if (!data) { diff --git a/capture/moloch.h b/capture/moloch.h -index 2a63d913..4b7dc4f0 100644 +index b5d866bf..b7bb2f09 100644 --- a/capture/moloch.h +++ b/capture/moloch.h @@ -357,7 +357,7 @@ enum MolochRotate { @@ -25,7 +25,7 @@ index 2a63d913..4b7dc4f0 100644 #define MOLOCH_FIELDS_CNT_MAX (MOLOCH_FIELDS_DB_MAX*2) #define MOLOCH_FIELD_EXSPECIAL_START (MOLOCH_FIELDS_CNT_MAX) diff --git a/capture/plugins/wise.c b/capture/plugins/wise.c -index 2f98ebe1..0e7b4002 100644 +index 252683a0..e26d3ddb 100644 --- a/capture/plugins/wise.c +++ b/capture/plugins/wise.c @@ -111,7 +111,7 @@ typedef struct wiseitem_head { @@ -38,10 +38,10 @@ index 2f98ebe1..0e7b4002 100644 BSB bsb; WiseItem_t *items[WISE_MAX_REQUEST_ITEMS]; diff --git a/viewer/db.js b/viewer/db.js -index 1eebc559..2046f17b 100644 +index 299d2f31..e5b853db 100644 --- a/viewer/db.js +++ b/viewer/db.js -@@ -1835,7 +1835,7 @@ exports.sid2Index = function (id, options) { +@@ -1729,7 +1729,7 @@ exports.sid2Index = function (id, options) { }; exports.loadFields = async () => { diff --git a/arkime/patch/footer_links.patch b/arkime/patch/footer_links.patch index aaf2635af..3c92d51c7 100644 --- a/arkime/patch/footer_links.patch +++ b/arkime/patch/footer_links.patch @@ -1,11 +1,11 @@ diff --git a/viewer/vueapp/src/components/utils/Footer.vue b/viewer/vueapp/src/components/utils/Footer.vue -index 50e30c99..d67dfb36 100644 +index 84b28a45..320dd70c 100644 --- a/viewer/vueapp/src/components/utils/Footer.vue +++ b/viewer/vueapp/src/components/utils/Footer.vue @@ -4,9 +4,11 @@
- Arkime v{{molochVersion}} | + Arkime v{{ version }} | - arkime.com + arkime.com 🦉 + | Malc⦿lm 📄 diff --git a/arkime/patch/hide_pcap_download_without_file.patch b/arkime/patch/hide_pcap_download_without_file.patch deleted file mode 100644 index 9dd5a8468..000000000 --- a/arkime/patch/hide_pcap_download_without_file.patch +++ /dev/null @@ -1,55 +0,0 @@ -diff --git a/viewer/views/sessionDetail.pug b/viewer/views/sessionDetail.pug -index 7a97fde7..89d56aee 100644 ---- a/viewer/views/sessionDetail.pug -+++ b/viewer/views/sessionDetail.pug -@@ -2,30 +2,33 @@ ul.nav.nav-pills.mb-3 - if (session.rootId) - li.nav-item - a.nav-link.cursor-pointer(@click='allSessions(' + '"' + session.rootId + '"' + ', ' + session.firstPacket + ')') All Sessions -+ if (session.packetPos && session.packetPos.length > 0) -+ if (session.rootId) -+ li.nav-item -+ a.nav-link(href=session.node + '/pcap/' + session.id + '.pcap', target="_blank", download=session.id + '-segment.pcap') Download Segment Pcap -+ li.nav-item -+ a.nav-link(href=session.node + '/entirePcap/' + session.rootId + '.pcap', target="_blank", download=session.id + '.pcap') Download Entire Pcap -+ else -+ li.nav-item -+ a.nav-link(href=session.node + '/pcap/' + session.id + '.pcap', target="_blank", v-has-permission="'!disablePcapDownload'", v-b-tooltip.hover.bottom.d300="'Download the PCAP file for this session.'", download=session.id + '.pcap') -+ span.fa.fa-download -+ | Download PCAP - li.nav-item -- a.nav-link(href=session.node + '/pcap/' + session.id + '.pcap', target="_blank", download=session.id + '-segment.pcap') Download Segment Pcap -+ a.nav-link(href=session.node + '/raw/' + session.id + '?type=src', target="_blank", v-b-tooltip.hover.bottom.d300="'Download the raw source packets for this session.'", download=session.id + '-src-raw') -+ span.fa.fa-arrow-circle-up -+ | Source Raw - li.nav-item -- a.nav-link(href=session.node + '/entirePcap/' + session.rootId + '.pcap', target="_blank", download=session.id + '.pcap') Download Entire Pcap -- else -- li.nav-item -- a.nav-link(href=session.node + '/pcap/' + session.id + '.pcap', target="_blank", v-has-permission="'!disablePcapDownload'", v-b-tooltip.hover.bottom.d300="'Download the PCAP file for this session.'", download=session.id + '.pcap') -- span.fa.fa-download -- | Download PCAP -- li.nav-item -- a.nav-link(href=session.node + '/raw/' + session.id + '?type=src', target="_blank", v-b-tooltip.hover.bottom.d300="'Download the raw source packets for this session.'", download=session.id + '-src-raw') -- span.fa.fa-arrow-circle-up -- | Source Raw -- li.nav-item -- a.nav-link(href=session.node + '/raw/' + session.id + '?type=dst', target="_blank", v-b-tooltip.hover.bottom.d300="'Download the raw destination packets for this session.'", download=session.id + '-dst-raw') -- span.fa.fa-arrow-circle-down -- | Destination Raw -+ a.nav-link(href=session.node + '/raw/' + session.id + '?type=dst', target="_blank", v-b-tooltip.hover.bottom.d300="'Download the raw destination packets for this session.'", download=session.id + '-dst-raw') -+ span.fa.fa-arrow-circle-down -+ | Destination Raw - li.nav-item - a.nav-link.cursor-pointer(@click="openPermalink", v-b-tooltip.hover.bottom.d300="'Navigate to the sessions page containing just this session. You can use this link to share this session with other users.'") - span.fa.fa-link - | Link - b-dropdown.nav-item(text="Actions", size="sm") -- b-dropdown-item(@click="exportPCAP", v-has-permission="'!disablePcapDownload'") -- | Export PCAP -+ if (session.packetPos && session.packetPos.length > 0) -+ b-dropdown-item(@click="exportPCAP", v-has-permission="'!disablePcapDownload'") -+ | Export PCAP - b-dropdown-item(@click="addTags") - | Add Tags - b-dropdown-item(@click="removeTags", v-has-permission="'removeEnabled'") diff --git a/arkime/patch/packetpos_arkime_issues_1952_1953.patch b/arkime/patch/packetpos_arkime_issues_1952_1953.patch deleted file mode 100644 index 9f96e3b04..000000000 --- a/arkime/patch/packetpos_arkime_issues_1952_1953.patch +++ /dev/null @@ -1,15 +0,0 @@ -diff --git a/viewer/apiSessions.js b/viewer/apiSessions.js -index 367ef7d9..4ec216b7 100644 ---- a/viewer/apiSessions.js -+++ b/viewer/apiSessions.js -@@ -1124,6 +1124,10 @@ module.exports = (Config, Db, internals, ViewerUtils) => { - - const fields = session.fields; - -+ if (!fields.packetPos) { -+ return endCb(null); -+ } -+ - if (maxPackets && fields.packetPos.length > maxPackets) { - fields.packetPos.length = maxPackets; - } diff --git a/arkime/patch/remove_upload.patch b/arkime/patch/remove_upload.patch index 8eb19d01d..4c17cc35c 100644 --- a/arkime/patch/remove_upload.patch +++ b/arkime/patch/remove_upload.patch @@ -1,8 +1,8 @@ diff --git a/viewer/vueapp/src/router/index.js b/viewer/vueapp/src/router/index.js -index 02494926..a3c974f9 100644 +index 104506cf..0850f2cd 100644 --- a/viewer/vueapp/src/router/index.js +++ b/viewer/vueapp/src/router/index.js -@@ -11,7 +11,6 @@ import Spiview from '@/components/spiview/Spiview'; +@@ -12,7 +12,6 @@ import Spiview from '@/components/spiview/Spiview'; import Spigraph from '@/components/spigraph/Spigraph'; import Connections from '@/components/connections/Connections'; import Settings from '@/components/settings/Settings'; @@ -10,7 +10,7 @@ index 02494926..a3c974f9 100644 import Hunt from '@/components/hunt/Hunt'; import Moloch404 from '@/components/utils/404'; -@@ -86,11 +85,6 @@ const router = new Router({ +@@ -92,11 +91,6 @@ const router = new Router({ name: 'Settings', component: Settings }, diff --git a/arkime/patch/spi_sort_zeek.patch b/arkime/patch/spi_sort_malcolm.patch similarity index 88% rename from arkime/patch/spi_sort_zeek.patch rename to arkime/patch/spi_sort_malcolm.patch index 73afa84bc..1a16f13cf 100644 --- a/arkime/patch/spi_sort_zeek.patch +++ b/arkime/patch/spi_sort_malcolm.patch @@ -1,13 +1,13 @@ diff --git a/viewer/vueapp/src/components/spiview/Spiview.vue b/viewer/vueapp/src/components/spiview/Spiview.vue -index 66e08ecf..f7156b80 100644 +index 76dd613b..b8117034 100644 --- a/viewer/vueapp/src/components/spiview/Spiview.vue +++ b/viewer/vueapp/src/components/spiview/Spiview.vue -@@ -915,6 +915,8 @@ export default { - +@@ -960,6 +960,8 @@ export default { + // sorted list of categories for the view this.categoryList = Object.keys(this.categoryObjects).sort(); + this.categoryList.splice(this.categoryList.indexOf('malcolm'), 1); + this.categoryList.unshift('malcolm'); this.categoryList.splice(this.categoryList.indexOf('general'), 1); this.categoryList.unshift('general'); - + diff --git a/arkime/patch/viewer_db_opensearchv2_keyword_hack.patch b/arkime/patch/viewer_db_opensearchv2_keyword_hack.patch index 4f93580cf..9cf920a01 100644 --- a/arkime/patch/viewer_db_opensearchv2_keyword_hack.patch +++ b/arkime/patch/viewer_db_opensearchv2_keyword_hack.patch @@ -1,8 +1,8 @@ diff --git a/viewer/db.js b/viewer/db.js -index 1eebc559..45645fd3 100644 +index 299d2f31..1b31cace 100644 --- a/viewer/db.js +++ b/viewer/db.js -@@ -382,7 +382,13 @@ function fixSessionFields (fields, unflatten) { +@@ -386,7 +386,13 @@ function fixSessionFields (fields, unflatten) { delete fields[f]; for (let i = 0; i < path.length; i++) { if (i === path.length - 1) { diff --git a/arkime/patch/viewer_wider_field_detail.patch b/arkime/patch/viewer_wider_field_detail.patch index 3543771f2..ec0cc5376 100644 --- a/arkime/patch/viewer_wider_field_detail.patch +++ b/arkime/patch/viewer_wider_field_detail.patch @@ -1,8 +1,8 @@ diff --git a/viewer/vueapp/src/components/sessions/SessionDetail.vue b/viewer/vueapp/src/components/sessions/SessionDetail.vue -index a15f3b52..e7b0cd7b 100644 +index 572c41f3..60143d82 100644 --- a/viewer/vueapp/src/components/sessions/SessionDetail.vue +++ b/viewer/vueapp/src/components/sessions/SessionDetail.vue -@@ -844,7 +844,7 @@ export default { +@@ -846,7 +846,7 @@ export default { .session-detail dt { float: left; clear: left; @@ -11,7 +11,7 @@ index a15f3b52..e7b0cd7b 100644 text-align: right; margin-right: 6px; line-height: 1.7; -@@ -852,7 +852,7 @@ export default { +@@ -854,7 +854,7 @@ export default { } .session-detail dd { diff --git a/arkime/scripts/arkime-needs-upgrade.sh b/arkime/scripts/arkime-needs-upgrade.sh index 6bd5e474d..12fe4cfbf 100755 --- a/arkime/scripts/arkime-needs-upgrade.sh +++ b/arkime/scripts/arkime-needs-upgrade.sh @@ -28,9 +28,12 @@ set -e [arkime_files_v]=arkime_files_v30 [arkime_hunts_v]=arkime_hunts_v30 [arkime_lookups_v]=arkime_lookups_v30 + [arkime_notifiers_v]=arkime_notifiers_v40 [arkime_queries_v]=arkime_queries_v30 + [arkime_sequence_v]=arkime_sequence_v30 [arkime_stats_v]=arkime_stats_v30 [arkime_users_v]=arkime_users_v30 + [arkime_views_v]=arkime_views_v40 ) # get a list of all current indices and loop over them diff --git a/docker-compose-standalone.yml b/docker-compose-standalone.yml index 0b10c0e3a..4c91d1495 100644 --- a/docker-compose-standalone.yml +++ b/docker-compose-standalone.yml @@ -342,7 +342,7 @@ x-pcap-capture-variables: &pcap-capture-variables PCAP_IFACE_TWEAK : 'false' # Specifies how large a locally-captured PCAP file can become (in megabytes) before # it is closed for processing and a new PCAP file created - PCAP_ROTATE_MEGABYTES : 1024 + PCAP_ROTATE_MEGABYTES : 4096 # Specifies a time interval (in minutes) after which a locally-captured PCAP file # will be closed for processing and a new PCAP file created PCAP_ROTATE_MINUTES : 10 diff --git a/docker-compose.yml b/docker-compose.yml index 652d5340c..81a0a104f 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -342,7 +342,7 @@ x-pcap-capture-variables: &pcap-capture-variables PCAP_IFACE_TWEAK : 'false' # Specifies how large a locally-captured PCAP file can become (in megabytes) before # it is closed for processing and a new PCAP file created - PCAP_ROTATE_MEGABYTES : 1024 + PCAP_ROTATE_MEGABYTES : 4096 # Specifies a time interval (in minutes) after which a locally-captured PCAP file # will be closed for processing and a new PCAP file created PCAP_ROTATE_MINUTES : 10 diff --git a/sensor-iso/arkime/Dockerfile b/sensor-iso/arkime/Dockerfile index c0c2fe345..333c1ccf6 100644 --- a/sensor-iso/arkime/Dockerfile +++ b/sensor-iso/arkime/Dockerfile @@ -6,7 +6,7 @@ LABEL maintainer="malcolm@inl.gov" ENV DEBIAN_FRONTEND noninteractive -ENV ARKIME_VERSION "3.4.2" +ENV ARKIME_VERSION "v4.0.0" ENV ARKIME_DIR "/opt/arkime" RUN sed -i "s/bullseye main/bullseye main contrib non-free/g" /etc/apt/sources.list && \ diff --git a/sensor-iso/arkime/build-arkime-deb.sh b/sensor-iso/arkime/build-arkime-deb.sh index ada7e8fd1..494048a15 100755 --- a/sensor-iso/arkime/build-arkime-deb.sh +++ b/sensor-iso/arkime/build-arkime-deb.sh @@ -21,7 +21,7 @@ fi apt-get -q update cd /tmp -git clone --depth=1 --single-branch --recurse-submodules --shallow-submodules --no-tags --branch="v$ARKIME_VERSION" "$ARKIME_URL" "./arkime-"$ARKIME_VERSION +git clone --recurse-submodules --branch="$ARKIME_VERSION" "$ARKIME_URL" "./arkime-"$ARKIME_VERSION cd "./arkime-"$ARKIME_VERSION for i in /opt/patches/*; do patch -p 1 -r - --no-backup-if-mismatch < $i || true diff --git a/sensor-iso/interface/sensor_ctl/arkime/config.ini b/sensor-iso/interface/sensor_ctl/arkime/config.ini index 5b4670924..501620047 100644 --- a/sensor-iso/interface/sensor_ctl/arkime/config.ini +++ b/sensor-iso/interface/sensor_ctl/arkime/config.ini @@ -8,7 +8,7 @@ passwordSecret=Malcolm httpRealm=Arkime interface=enp0s1 pcapDir=/tmp -maxFileSizeG=2 +maxFileSizeG=4 maxFileTimeM=180 tcpTimeout=600 tcpSaveTimeout=720 diff --git a/sensor-iso/interface/sensor_ctl/control_vars.conf b/sensor-iso/interface/sensor_ctl/control_vars.conf index 7a6f602ca..3c10ad312 100644 --- a/sensor-iso/interface/sensor_ctl/control_vars.conf +++ b/sensor-iso/interface/sensor_ctl/control_vars.conf @@ -4,8 +4,8 @@ export CAPTURE_FILTER="" export PCAP_PATH=/home/sensor/net_cap export PCAP_TCPDUMP_FILENAME_PATTERN=%Y%m%d_%H%M%S.pcap export PCAP_NETSNIFF_MAGIC=0xa1b2c3d4 -export PCAP_ROTATE_SECONDS=10800 -export PCAP_ROTATE_MEGABYTES=500 +export PCAP_ROTATE_SECONDS=3600 +export PCAP_ROTATE_MEGABYTES=4096 export PCAP_SNAPLEN=0 export PCAP_MAX_DISK_FILL=90 export PCAP_PRUNE_CHECK_SECONDS=60