diff --git a/Dockerfiles/arkime.Dockerfile b/Dockerfiles/arkime.Dockerfile index abae5edea..35b6d20c6 100644 --- a/Dockerfiles/arkime.Dockerfile +++ b/Dockerfiles/arkime.Dockerfile @@ -4,7 +4,7 @@ FROM debian:11-slim AS build ENV DEBIAN_FRONTEND noninteractive -ENV ARKIME_VERSION "3.4.2" +ENV ARKIME_VERSION "main" ENV ARKIMEDIR "/opt/arkime" ENV ARKIME_URL "https://github.com/arkime/arkime.git" ENV ARKIME_LOCALELASTICSEARCH no @@ -66,7 +66,7 @@ RUN apt-get -q update && \ sed -i 's/\!\[.*\](.*\/badge.svg)//g' README.md && \ pandoc -s --self-contained --metadata title="Malcolm README" --css $ARKIMEDIR/doc/doc.css -o $ARKIMEDIR/doc/README.html $ARKIMEDIR/doc/README.md && \ cd /opt && \ - git clone --depth=1 --single-branch --recurse-submodules --shallow-submodules --no-tags --branch="v$ARKIME_VERSION" "$ARKIME_URL" "./arkime-"$ARKIME_VERSION && \ + git clone --depth=1 --single-branch --recurse-submodules --shallow-submodules --no-tags --branch="$ARKIME_VERSION" "$ARKIME_URL" "./arkime-"$ARKIME_VERSION && \ cd "./arkime-"$ARKIME_VERSION && \ bash -c 'for i in /opt/patches/*; do patch -p 1 -r - --no-backup-if-mismatch < $i || true; done' && \ find $ARKIMEDIR/doc/images/screenshots -name "*.png" -delete && \ @@ -80,10 +80,15 @@ RUN apt-get -q update && \ rm -rf ./viewer/vueapp/src/components/upload ./capture/plugins/suricata* && \ sed -i "s/^\(ARKIME_LOCALELASTICSEARCH=\).*/\1"$ARKIME_LOCALELASTICSEARCH"/" ./release/Configure && \ sed -i "s/^\(ARKIME_INET=\).*/\1"$ARKIME_INET"/" ./release/Configure && \ + echo '>>>> easybutton-build.sh' && \ ./easybutton-build.sh && \ + echo '>>>> npm -g config set user root' && \ npm -g config set user root && \ + echo '>>>> make install' && \ make install && \ + echo '>>>> npm cache clean --force' && \ npm cache clean --force && \ + echo '>>>> cleanup' && \ rm -f ${ARKIMEDIR}/wiseService/source.* && \ bash -c "file ${ARKIMEDIR}/bin/* ${ARKIMEDIR}/node-v*/bin/* | grep 'ELF 64-bit' | sed 's/:.*//' | xargs -l -r strip -v --strip-unneeded" diff --git a/arkime/patch/capture_event_dataset.patch b/arkime/patch/capture_event_dataset.patch deleted file mode 100644 index a8862c7d6..000000000 --- a/arkime/patch/capture_event_dataset.patch +++ /dev/null @@ -1,34 +0,0 @@ -diff --git a/capture/db.c b/capture/db.c -index 2515b177..0a6d06bd 100644 ---- a/capture/db.c -+++ b/capture/db.c -@@ -71,6 +71,7 @@ LOCAL int dbExit; - LOCAL char *esBulkQuery; - LOCAL int esBulkQueryLen; - LOCAL char *ecsEventProvider; -+LOCAL char *ecsEventDataset; - - extern uint64_t packetStats[MOLOCH_PACKET_MAX]; - -@@ -906,8 +907,12 @@ void moloch_db_save_session(MolochSession_t *session, int final) - } - BSB_EXPORT_cstr(jbsb, "],"); - -- if (ecsEventProvider) { -+ if (ecsEventProvider && ecsEventDataset) { -+ BSB_EXPORT_sprintf(jbsb, "\"event\":{\"provider\":\"%s\", \"dataset\":\"%s\"},", ecsEventProvider, ecsEventDataset); -+ } else if (ecsEventProvider) { - BSB_EXPORT_sprintf(jbsb, "\"event\":{\"provider\":\"%s\"},", ecsEventProvider); -+ } else if (ecsEventDataset) { -+ BSB_EXPORT_sprintf(jbsb, "\"event\":{\"dataset\":\"%s\"},", ecsEventDataset); - } - - int inGroupNum = 0; -@@ -2661,6 +2666,7 @@ void moloch_db_init() - } - - ecsEventProvider = moloch_config_str(NULL, "ecsEventProvider", NULL); -+ ecsEventDataset = moloch_config_str(NULL, "ecsEventDataset", NULL); - - int thread; - for (thread = 0; thread < config.packetThreads; thread++) { diff --git a/arkime/patch/field_best_priority.patch b/arkime/patch/field_best_priority.patch deleted file mode 100644 index 799be8d9e..000000000 --- a/arkime/patch/field_best_priority.patch +++ /dev/null @@ -1,13 +0,0 @@ -diff --git a/capture/db.c b/capture/db.c -index c1350507..b29a41ac 100644 ---- a/capture/db.c -+++ b/capture/db.c -@@ -2411,7 +2411,7 @@ void moloch_db_add_field(char *group, char *kind, char *expression, char *friend - } - - BSB_EXPORT_u08(bsb, '}'); -- moloch_http_schedule(esServer, "POST", key, key_len, json, BSB_LENGTH(bsb), NULL, MOLOCH_HTTP_PRIORITY_NORMAL, NULL, NULL); -+ moloch_http_schedule(esServer, "POST", key, key_len, json, BSB_LENGTH(bsb), NULL, MOLOCH_HTTP_PRIORITY_BEST, NULL, NULL); - } - /******************************************************************************/ - void moloch_db_update_field(char *expression, char *name, char *value) diff --git a/arkime/patch/fields_db_max_5000.patch b/arkime/patch/fields_db_max_5000.patch index b4799e580..525d5dff1 100644 --- a/arkime/patch/fields_db_max_5000.patch +++ b/arkime/patch/fields_db_max_5000.patch @@ -1,8 +1,8 @@ diff --git a/capture/db.c b/capture/db.c -index c1350507..1ae57143 100644 +index a21d853a..c3d79c56 100644 --- a/capture/db.c +++ b/capture/db.c -@@ -2325,7 +2325,7 @@ LOCAL void moloch_db_load_fields() +@@ -2321,7 +2321,7 @@ LOCAL void moloch_db_load_fields() char key[100]; int key_len; @@ -12,7 +12,7 @@ index c1350507..1ae57143 100644 if (!data) { diff --git a/capture/moloch.h b/capture/moloch.h -index 2a63d913..4b7dc4f0 100644 +index b5d866bf..b7bb2f09 100644 --- a/capture/moloch.h +++ b/capture/moloch.h @@ -357,7 +357,7 @@ enum MolochRotate { @@ -25,7 +25,7 @@ index 2a63d913..4b7dc4f0 100644 #define MOLOCH_FIELDS_CNT_MAX (MOLOCH_FIELDS_DB_MAX*2) #define MOLOCH_FIELD_EXSPECIAL_START (MOLOCH_FIELDS_CNT_MAX) diff --git a/capture/plugins/wise.c b/capture/plugins/wise.c -index 2f98ebe1..0e7b4002 100644 +index 252683a0..e26d3ddb 100644 --- a/capture/plugins/wise.c +++ b/capture/plugins/wise.c @@ -111,7 +111,7 @@ typedef struct wiseitem_head { @@ -38,10 +38,10 @@ index 2f98ebe1..0e7b4002 100644 BSB bsb; WiseItem_t *items[WISE_MAX_REQUEST_ITEMS]; diff --git a/viewer/db.js b/viewer/db.js -index 1eebc559..2046f17b 100644 +index afd0ef70..cda80e14 100644 --- a/viewer/db.js +++ b/viewer/db.js -@@ -1835,7 +1835,7 @@ exports.sid2Index = function (id, options) { +@@ -1736,7 +1736,7 @@ exports.sid2Index = function (id, options) { }; exports.loadFields = async () => { diff --git a/arkime/patch/footer_links.patch b/arkime/patch/footer_links.patch index 0c1b2a39e..c08bb98d2 100644 --- a/arkime/patch/footer_links.patch +++ b/arkime/patch/footer_links.patch @@ -1,10 +1,10 @@ diff --git a/viewer/vueapp/src/components/utils/Footer.vue b/viewer/vueapp/src/components/utils/Footer.vue -index 50e30c99..d54542dc 100644 +index 84b28a45..85f778bc 100644 --- a/viewer/vueapp/src/components/utils/Footer.vue +++ b/viewer/vueapp/src/components/utils/Footer.vue @@ -5,6 +5,8 @@ - Arkime v{{molochVersion}} | + Arkime v{{ version }} | arkime.com + | Malcolm + | Dashboards diff --git a/arkime/patch/hide_pcap_download_without_file.patch b/arkime/patch/hide_pcap_download_without_file.patch deleted file mode 100644 index 9dd5a8468..000000000 --- a/arkime/patch/hide_pcap_download_without_file.patch +++ /dev/null @@ -1,55 +0,0 @@ -diff --git a/viewer/views/sessionDetail.pug b/viewer/views/sessionDetail.pug -index 7a97fde7..89d56aee 100644 ---- a/viewer/views/sessionDetail.pug -+++ b/viewer/views/sessionDetail.pug -@@ -2,30 +2,33 @@ ul.nav.nav-pills.mb-3 - if (session.rootId) - li.nav-item - a.nav-link.cursor-pointer(@click='allSessions(' + '"' + session.rootId + '"' + ', ' + session.firstPacket + ')') All Sessions -+ if (session.packetPos && session.packetPos.length > 0) -+ if (session.rootId) -+ li.nav-item -+ a.nav-link(href=session.node + '/pcap/' + session.id + '.pcap', target="_blank", download=session.id + '-segment.pcap') Download Segment Pcap -+ li.nav-item -+ a.nav-link(href=session.node + '/entirePcap/' + session.rootId + '.pcap', target="_blank", download=session.id + '.pcap') Download Entire Pcap -+ else -+ li.nav-item -+ a.nav-link(href=session.node + '/pcap/' + session.id + '.pcap', target="_blank", v-has-permission="'!disablePcapDownload'", v-b-tooltip.hover.bottom.d300="'Download the PCAP file for this session.'", download=session.id + '.pcap') -+ span.fa.fa-download -+ |  Download PCAP - li.nav-item -- a.nav-link(href=session.node + '/pcap/' + session.id + '.pcap', target="_blank", download=session.id + '-segment.pcap') Download Segment Pcap -+ a.nav-link(href=session.node + '/raw/' + session.id + '?type=src', target="_blank", v-b-tooltip.hover.bottom.d300="'Download the raw source packets for this session.'", download=session.id + '-src-raw') -+ span.fa.fa-arrow-circle-up -+ |  Source Raw - li.nav-item -- a.nav-link(href=session.node + '/entirePcap/' + session.rootId + '.pcap', target="_blank", download=session.id + '.pcap') Download Entire Pcap -- else -- li.nav-item -- a.nav-link(href=session.node + '/pcap/' + session.id + '.pcap', target="_blank", v-has-permission="'!disablePcapDownload'", v-b-tooltip.hover.bottom.d300="'Download the PCAP file for this session.'", download=session.id + '.pcap') -- span.fa.fa-download -- |  Download PCAP -- li.nav-item -- a.nav-link(href=session.node + '/raw/' + session.id + '?type=src', target="_blank", v-b-tooltip.hover.bottom.d300="'Download the raw source packets for this session.'", download=session.id + '-src-raw') -- span.fa.fa-arrow-circle-up -- |  Source Raw -- li.nav-item -- a.nav-link(href=session.node + '/raw/' + session.id + '?type=dst', target="_blank", v-b-tooltip.hover.bottom.d300="'Download the raw destination packets for this session.'", download=session.id + '-dst-raw') -- span.fa.fa-arrow-circle-down -- |  Destination Raw -+ a.nav-link(href=session.node + '/raw/' + session.id + '?type=dst', target="_blank", v-b-tooltip.hover.bottom.d300="'Download the raw destination packets for this session.'", download=session.id + '-dst-raw') -+ span.fa.fa-arrow-circle-down -+ |  Destination Raw - li.nav-item - a.nav-link.cursor-pointer(@click="openPermalink", v-b-tooltip.hover.bottom.d300="'Navigate to the sessions page containing just this session. You can use this link to share this session with other users.'") - span.fa.fa-link - |  Link - b-dropdown.nav-item(text="Actions", size="sm") -- b-dropdown-item(@click="exportPCAP", v-has-permission="'!disablePcapDownload'") -- | Export PCAP -+ if (session.packetPos && session.packetPos.length > 0) -+ b-dropdown-item(@click="exportPCAP", v-has-permission="'!disablePcapDownload'") -+ | Export PCAP - b-dropdown-item(@click="addTags") - | Add Tags - b-dropdown-item(@click="removeTags", v-has-permission="'removeEnabled'") diff --git a/arkime/patch/packetpos_arkime_issues_1952_1953.patch b/arkime/patch/packetpos_arkime_issues_1952_1953.patch deleted file mode 100644 index 9f96e3b04..000000000 --- a/arkime/patch/packetpos_arkime_issues_1952_1953.patch +++ /dev/null @@ -1,15 +0,0 @@ -diff --git a/viewer/apiSessions.js b/viewer/apiSessions.js -index 367ef7d9..4ec216b7 100644 ---- a/viewer/apiSessions.js -+++ b/viewer/apiSessions.js -@@ -1124,6 +1124,10 @@ module.exports = (Config, Db, internals, ViewerUtils) => { - - const fields = session.fields; - -+ if (!fields.packetPos) { -+ return endCb(null); -+ } -+ - if (maxPackets && fields.packetPos.length > maxPackets) { - fields.packetPos.length = maxPackets; - } diff --git a/arkime/patch/remove_upload.patch b/arkime/patch/remove_upload.patch index 8eb19d01d..4c17cc35c 100644 --- a/arkime/patch/remove_upload.patch +++ b/arkime/patch/remove_upload.patch @@ -1,8 +1,8 @@ diff --git a/viewer/vueapp/src/router/index.js b/viewer/vueapp/src/router/index.js -index 02494926..a3c974f9 100644 +index 104506cf..0850f2cd 100644 --- a/viewer/vueapp/src/router/index.js +++ b/viewer/vueapp/src/router/index.js -@@ -11,7 +11,6 @@ import Spiview from '@/components/spiview/Spiview'; +@@ -12,7 +12,6 @@ import Spiview from '@/components/spiview/Spiview'; import Spigraph from '@/components/spigraph/Spigraph'; import Connections from '@/components/connections/Connections'; import Settings from '@/components/settings/Settings'; @@ -10,7 +10,7 @@ index 02494926..a3c974f9 100644 import Hunt from '@/components/hunt/Hunt'; import Moloch404 from '@/components/utils/404'; -@@ -86,11 +85,6 @@ const router = new Router({ +@@ -92,11 +91,6 @@ const router = new Router({ name: 'Settings', component: Settings }, diff --git a/arkime/patch/spi_sort_zeek.patch b/arkime/patch/spi_sort_malcolm.patch similarity index 88% rename from arkime/patch/spi_sort_zeek.patch rename to arkime/patch/spi_sort_malcolm.patch index 73afa84bc..1a16f13cf 100644 --- a/arkime/patch/spi_sort_zeek.patch +++ b/arkime/patch/spi_sort_malcolm.patch @@ -1,13 +1,13 @@ diff --git a/viewer/vueapp/src/components/spiview/Spiview.vue b/viewer/vueapp/src/components/spiview/Spiview.vue -index 66e08ecf..f7156b80 100644 +index 76dd613b..b8117034 100644 --- a/viewer/vueapp/src/components/spiview/Spiview.vue +++ b/viewer/vueapp/src/components/spiview/Spiview.vue -@@ -915,6 +915,8 @@ export default { - +@@ -960,6 +960,8 @@ export default { + // sorted list of categories for the view this.categoryList = Object.keys(this.categoryObjects).sort(); + this.categoryList.splice(this.categoryList.indexOf('malcolm'), 1); + this.categoryList.unshift('malcolm'); this.categoryList.splice(this.categoryList.indexOf('general'), 1); this.categoryList.unshift('general'); - + diff --git a/arkime/patch/viewer_db_opensearchv2_keyword_hack.patch b/arkime/patch/viewer_db_opensearchv2_keyword_hack.patch index 4f93580cf..89c85d5e3 100644 --- a/arkime/patch/viewer_db_opensearchv2_keyword_hack.patch +++ b/arkime/patch/viewer_db_opensearchv2_keyword_hack.patch @@ -1,8 +1,8 @@ diff --git a/viewer/db.js b/viewer/db.js -index 1eebc559..45645fd3 100644 +index afd0ef70..ca707dee 100644 --- a/viewer/db.js +++ b/viewer/db.js -@@ -382,7 +382,13 @@ function fixSessionFields (fields, unflatten) { +@@ -386,7 +386,13 @@ function fixSessionFields (fields, unflatten) { delete fields[f]; for (let i = 0; i < path.length; i++) { if (i === path.length - 1) { diff --git a/arkime/patch/viewer_wider_field_detail.patch b/arkime/patch/viewer_wider_field_detail.patch index 3543771f2..ec0cc5376 100644 --- a/arkime/patch/viewer_wider_field_detail.patch +++ b/arkime/patch/viewer_wider_field_detail.patch @@ -1,8 +1,8 @@ diff --git a/viewer/vueapp/src/components/sessions/SessionDetail.vue b/viewer/vueapp/src/components/sessions/SessionDetail.vue -index a15f3b52..e7b0cd7b 100644 +index 572c41f3..60143d82 100644 --- a/viewer/vueapp/src/components/sessions/SessionDetail.vue +++ b/viewer/vueapp/src/components/sessions/SessionDetail.vue -@@ -844,7 +844,7 @@ export default { +@@ -846,7 +846,7 @@ export default { .session-detail dt { float: left; clear: left; @@ -11,7 +11,7 @@ index a15f3b52..e7b0cd7b 100644 text-align: right; margin-right: 6px; line-height: 1.7; -@@ -852,7 +852,7 @@ export default { +@@ -854,7 +854,7 @@ export default { } .session-detail dd {