From 7481a86c8f9e566fc268bd325e14e641b1f402d9 Mon Sep 17 00:00:00 2001 From: Seth Grover Date: Mon, 15 Aug 2022 08:54:33 -0600 Subject: [PATCH 1/3] trying build of main branch --- Dockerfiles/arkime.Dockerfile | 9 ++- arkime/patch/capture_event_dataset.patch | 34 ------------ arkime/patch/field_best_priority.patch | 13 ----- arkime/patch/fields_db_max_5000.patch | 12 ++-- arkime/patch/footer_links.patch | 4 +- .../hide_pcap_download_without_file.patch | 55 ------------------- .../packetpos_arkime_issues_1952_1953.patch | 15 ----- arkime/patch/remove_upload.patch | 6 +- ...sort_zeek.patch => spi_sort_malcolm.patch} | 8 +-- .../viewer_db_opensearchv2_keyword_hack.patch | 4 +- arkime/patch/viewer_wider_field_detail.patch | 6 +- 11 files changed, 27 insertions(+), 139 deletions(-) delete mode 100644 arkime/patch/capture_event_dataset.patch delete mode 100644 arkime/patch/field_best_priority.patch delete mode 100644 arkime/patch/hide_pcap_download_without_file.patch delete mode 100644 arkime/patch/packetpos_arkime_issues_1952_1953.patch rename arkime/patch/{spi_sort_zeek.patch => spi_sort_malcolm.patch} (88%) diff --git a/Dockerfiles/arkime.Dockerfile b/Dockerfiles/arkime.Dockerfile index abae5edea..35b6d20c6 100644 --- a/Dockerfiles/arkime.Dockerfile +++ b/Dockerfiles/arkime.Dockerfile @@ -4,7 +4,7 @@ FROM debian:11-slim AS build ENV DEBIAN_FRONTEND noninteractive -ENV ARKIME_VERSION "3.4.2" +ENV ARKIME_VERSION "main" ENV ARKIMEDIR "/opt/arkime" ENV ARKIME_URL "https://github.com/arkime/arkime.git" ENV ARKIME_LOCALELASTICSEARCH no @@ -66,7 +66,7 @@ RUN apt-get -q update && \ sed -i 's/\!\[.*\](.*\/badge.svg)//g' README.md && \ pandoc -s --self-contained --metadata title="Malcolm README" --css $ARKIMEDIR/doc/doc.css -o $ARKIMEDIR/doc/README.html $ARKIMEDIR/doc/README.md && \ cd /opt && \ - git clone --depth=1 --single-branch --recurse-submodules --shallow-submodules --no-tags --branch="v$ARKIME_VERSION" "$ARKIME_URL" "./arkime-"$ARKIME_VERSION && \ + git clone --depth=1 --single-branch --recurse-submodules --shallow-submodules --no-tags --branch="$ARKIME_VERSION" "$ARKIME_URL" "./arkime-"$ARKIME_VERSION && \ cd "./arkime-"$ARKIME_VERSION && \ bash -c 'for i in /opt/patches/*; do patch -p 1 -r - --no-backup-if-mismatch < $i || true; done' && \ find $ARKIMEDIR/doc/images/screenshots -name "*.png" -delete && \ @@ -80,10 +80,15 @@ RUN apt-get -q update && \ rm -rf ./viewer/vueapp/src/components/upload ./capture/plugins/suricata* && \ sed -i "s/^\(ARKIME_LOCALELASTICSEARCH=\).*/\1"$ARKIME_LOCALELASTICSEARCH"/" ./release/Configure && \ sed -i "s/^\(ARKIME_INET=\).*/\1"$ARKIME_INET"/" ./release/Configure && \ + echo '>>>> easybutton-build.sh' && \ ./easybutton-build.sh && \ + echo '>>>> npm -g config set user root' && \ npm -g config set user root && \ + echo '>>>> make install' && \ make install && \ + echo '>>>> npm cache clean --force' && \ npm cache clean --force && \ + echo '>>>> cleanup' && \ rm -f ${ARKIMEDIR}/wiseService/source.* && \ bash -c "file ${ARKIMEDIR}/bin/* ${ARKIMEDIR}/node-v*/bin/* | grep 'ELF 64-bit' | sed 's/:.*//' | xargs -l -r strip -v --strip-unneeded" diff --git a/arkime/patch/capture_event_dataset.patch b/arkime/patch/capture_event_dataset.patch deleted file mode 100644 index a8862c7d6..000000000 --- a/arkime/patch/capture_event_dataset.patch +++ /dev/null @@ -1,34 +0,0 @@ -diff --git a/capture/db.c b/capture/db.c -index 2515b177..0a6d06bd 100644 ---- a/capture/db.c -+++ b/capture/db.c -@@ -71,6 +71,7 @@ LOCAL int dbExit; - LOCAL char *esBulkQuery; - LOCAL int esBulkQueryLen; - LOCAL char *ecsEventProvider; -+LOCAL char *ecsEventDataset; - - extern uint64_t packetStats[MOLOCH_PACKET_MAX]; - -@@ -906,8 +907,12 @@ void moloch_db_save_session(MolochSession_t *session, int final) - } - BSB_EXPORT_cstr(jbsb, "],"); - -- if (ecsEventProvider) { -+ if (ecsEventProvider && ecsEventDataset) { -+ BSB_EXPORT_sprintf(jbsb, "\"event\":{\"provider\":\"%s\", \"dataset\":\"%s\"},", ecsEventProvider, ecsEventDataset); -+ } else if (ecsEventProvider) { - BSB_EXPORT_sprintf(jbsb, "\"event\":{\"provider\":\"%s\"},", ecsEventProvider); -+ } else if (ecsEventDataset) { -+ BSB_EXPORT_sprintf(jbsb, "\"event\":{\"dataset\":\"%s\"},", ecsEventDataset); - } - - int inGroupNum = 0; -@@ -2661,6 +2666,7 @@ void moloch_db_init() - } - - ecsEventProvider = moloch_config_str(NULL, "ecsEventProvider", NULL); -+ ecsEventDataset = moloch_config_str(NULL, "ecsEventDataset", NULL); - - int thread; - for (thread = 0; thread < config.packetThreads; thread++) { diff --git a/arkime/patch/field_best_priority.patch b/arkime/patch/field_best_priority.patch deleted file mode 100644 index 799be8d9e..000000000 --- a/arkime/patch/field_best_priority.patch +++ /dev/null @@ -1,13 +0,0 @@ -diff --git a/capture/db.c b/capture/db.c -index c1350507..b29a41ac 100644 ---- a/capture/db.c -+++ b/capture/db.c -@@ -2411,7 +2411,7 @@ void moloch_db_add_field(char *group, char *kind, char *expression, char *friend - } - - BSB_EXPORT_u08(bsb, '}'); -- moloch_http_schedule(esServer, "POST", key, key_len, json, BSB_LENGTH(bsb), NULL, MOLOCH_HTTP_PRIORITY_NORMAL, NULL, NULL); -+ moloch_http_schedule(esServer, "POST", key, key_len, json, BSB_LENGTH(bsb), NULL, MOLOCH_HTTP_PRIORITY_BEST, NULL, NULL); - } - /******************************************************************************/ - void moloch_db_update_field(char *expression, char *name, char *value) diff --git a/arkime/patch/fields_db_max_5000.patch b/arkime/patch/fields_db_max_5000.patch index b4799e580..525d5dff1 100644 --- a/arkime/patch/fields_db_max_5000.patch +++ b/arkime/patch/fields_db_max_5000.patch @@ -1,8 +1,8 @@ diff --git a/capture/db.c b/capture/db.c -index c1350507..1ae57143 100644 +index a21d853a..c3d79c56 100644 --- a/capture/db.c +++ b/capture/db.c -@@ -2325,7 +2325,7 @@ LOCAL void moloch_db_load_fields() +@@ -2321,7 +2321,7 @@ LOCAL void moloch_db_load_fields() char key[100]; int key_len; @@ -12,7 +12,7 @@ index c1350507..1ae57143 100644 if (!data) { diff --git a/capture/moloch.h b/capture/moloch.h -index 2a63d913..4b7dc4f0 100644 +index b5d866bf..b7bb2f09 100644 --- a/capture/moloch.h +++ b/capture/moloch.h @@ -357,7 +357,7 @@ enum MolochRotate { @@ -25,7 +25,7 @@ index 2a63d913..4b7dc4f0 100644 #define MOLOCH_FIELDS_CNT_MAX (MOLOCH_FIELDS_DB_MAX*2) #define MOLOCH_FIELD_EXSPECIAL_START (MOLOCH_FIELDS_CNT_MAX) diff --git a/capture/plugins/wise.c b/capture/plugins/wise.c -index 2f98ebe1..0e7b4002 100644 +index 252683a0..e26d3ddb 100644 --- a/capture/plugins/wise.c +++ b/capture/plugins/wise.c @@ -111,7 +111,7 @@ typedef struct wiseitem_head { @@ -38,10 +38,10 @@ index 2f98ebe1..0e7b4002 100644 BSB bsb; WiseItem_t *items[WISE_MAX_REQUEST_ITEMS]; diff --git a/viewer/db.js b/viewer/db.js -index 1eebc559..2046f17b 100644 +index afd0ef70..cda80e14 100644 --- a/viewer/db.js +++ b/viewer/db.js -@@ -1835,7 +1835,7 @@ exports.sid2Index = function (id, options) { +@@ -1736,7 +1736,7 @@ exports.sid2Index = function (id, options) { }; exports.loadFields = async () => { diff --git a/arkime/patch/footer_links.patch b/arkime/patch/footer_links.patch index 0c1b2a39e..c08bb98d2 100644 --- a/arkime/patch/footer_links.patch +++ b/arkime/patch/footer_links.patch @@ -1,10 +1,10 @@ diff --git a/viewer/vueapp/src/components/utils/Footer.vue b/viewer/vueapp/src/components/utils/Footer.vue -index 50e30c99..d54542dc 100644 +index 84b28a45..85f778bc 100644 --- a/viewer/vueapp/src/components/utils/Footer.vue +++ b/viewer/vueapp/src/components/utils/Footer.vue @@ -5,6 +5,8 @@ - Arkime v{{molochVersion}} | + Arkime v{{ version }} | arkime.com + | Malcolm + | Dashboards diff --git a/arkime/patch/hide_pcap_download_without_file.patch b/arkime/patch/hide_pcap_download_without_file.patch deleted file mode 100644 index 9dd5a8468..000000000 --- a/arkime/patch/hide_pcap_download_without_file.patch +++ /dev/null @@ -1,55 +0,0 @@ -diff --git a/viewer/views/sessionDetail.pug b/viewer/views/sessionDetail.pug -index 7a97fde7..89d56aee 100644 ---- a/viewer/views/sessionDetail.pug -+++ b/viewer/views/sessionDetail.pug -@@ -2,30 +2,33 @@ ul.nav.nav-pills.mb-3 - if (session.rootId) - li.nav-item - a.nav-link.cursor-pointer(@click='allSessions(' + '"' + session.rootId + '"' + ', ' + session.firstPacket + ')') All Sessions -+ if (session.packetPos && session.packetPos.length > 0) -+ if (session.rootId) -+ li.nav-item -+ a.nav-link(href=session.node + '/pcap/' + session.id + '.pcap', target="_blank", download=session.id + '-segment.pcap') Download Segment Pcap -+ li.nav-item -+ a.nav-link(href=session.node + '/entirePcap/' + session.rootId + '.pcap', target="_blank", download=session.id + '.pcap') Download Entire Pcap -+ else -+ li.nav-item -+ a.nav-link(href=session.node + '/pcap/' + session.id + '.pcap', target="_blank", v-has-permission="'!disablePcapDownload'", v-b-tooltip.hover.bottom.d300="'Download the PCAP file for this session.'", download=session.id + '.pcap') -+ span.fa.fa-download -+ |  Download PCAP - li.nav-item -- a.nav-link(href=session.node + '/pcap/' + session.id + '.pcap', target="_blank", download=session.id + '-segment.pcap') Download Segment Pcap -+ a.nav-link(href=session.node + '/raw/' + session.id + '?type=src', target="_blank", v-b-tooltip.hover.bottom.d300="'Download the raw source packets for this session.'", download=session.id + '-src-raw') -+ span.fa.fa-arrow-circle-up -+ |  Source Raw - li.nav-item -- a.nav-link(href=session.node + '/entirePcap/' + session.rootId + '.pcap', target="_blank", download=session.id + '.pcap') Download Entire Pcap -- else -- li.nav-item -- a.nav-link(href=session.node + '/pcap/' + session.id + '.pcap', target="_blank", v-has-permission="'!disablePcapDownload'", v-b-tooltip.hover.bottom.d300="'Download the PCAP file for this session.'", download=session.id + '.pcap') -- span.fa.fa-download -- |  Download PCAP -- li.nav-item -- a.nav-link(href=session.node + '/raw/' + session.id + '?type=src', target="_blank", v-b-tooltip.hover.bottom.d300="'Download the raw source packets for this session.'", download=session.id + '-src-raw') -- span.fa.fa-arrow-circle-up -- |  Source Raw -- li.nav-item -- a.nav-link(href=session.node + '/raw/' + session.id + '?type=dst', target="_blank", v-b-tooltip.hover.bottom.d300="'Download the raw destination packets for this session.'", download=session.id + '-dst-raw') -- span.fa.fa-arrow-circle-down -- |  Destination Raw -+ a.nav-link(href=session.node + '/raw/' + session.id + '?type=dst', target="_blank", v-b-tooltip.hover.bottom.d300="'Download the raw destination packets for this session.'", download=session.id + '-dst-raw') -+ span.fa.fa-arrow-circle-down -+ |  Destination Raw - li.nav-item - a.nav-link.cursor-pointer(@click="openPermalink", v-b-tooltip.hover.bottom.d300="'Navigate to the sessions page containing just this session. You can use this link to share this session with other users.'") - span.fa.fa-link - |  Link - b-dropdown.nav-item(text="Actions", size="sm") -- b-dropdown-item(@click="exportPCAP", v-has-permission="'!disablePcapDownload'") -- | Export PCAP -+ if (session.packetPos && session.packetPos.length > 0) -+ b-dropdown-item(@click="exportPCAP", v-has-permission="'!disablePcapDownload'") -+ | Export PCAP - b-dropdown-item(@click="addTags") - | Add Tags - b-dropdown-item(@click="removeTags", v-has-permission="'removeEnabled'") diff --git a/arkime/patch/packetpos_arkime_issues_1952_1953.patch b/arkime/patch/packetpos_arkime_issues_1952_1953.patch deleted file mode 100644 index 9f96e3b04..000000000 --- a/arkime/patch/packetpos_arkime_issues_1952_1953.patch +++ /dev/null @@ -1,15 +0,0 @@ -diff --git a/viewer/apiSessions.js b/viewer/apiSessions.js -index 367ef7d9..4ec216b7 100644 ---- a/viewer/apiSessions.js -+++ b/viewer/apiSessions.js -@@ -1124,6 +1124,10 @@ module.exports = (Config, Db, internals, ViewerUtils) => { - - const fields = session.fields; - -+ if (!fields.packetPos) { -+ return endCb(null); -+ } -+ - if (maxPackets && fields.packetPos.length > maxPackets) { - fields.packetPos.length = maxPackets; - } diff --git a/arkime/patch/remove_upload.patch b/arkime/patch/remove_upload.patch index 8eb19d01d..4c17cc35c 100644 --- a/arkime/patch/remove_upload.patch +++ b/arkime/patch/remove_upload.patch @@ -1,8 +1,8 @@ diff --git a/viewer/vueapp/src/router/index.js b/viewer/vueapp/src/router/index.js -index 02494926..a3c974f9 100644 +index 104506cf..0850f2cd 100644 --- a/viewer/vueapp/src/router/index.js +++ b/viewer/vueapp/src/router/index.js -@@ -11,7 +11,6 @@ import Spiview from '@/components/spiview/Spiview'; +@@ -12,7 +12,6 @@ import Spiview from '@/components/spiview/Spiview'; import Spigraph from '@/components/spigraph/Spigraph'; import Connections from '@/components/connections/Connections'; import Settings from '@/components/settings/Settings'; @@ -10,7 +10,7 @@ index 02494926..a3c974f9 100644 import Hunt from '@/components/hunt/Hunt'; import Moloch404 from '@/components/utils/404'; -@@ -86,11 +85,6 @@ const router = new Router({ +@@ -92,11 +91,6 @@ const router = new Router({ name: 'Settings', component: Settings }, diff --git a/arkime/patch/spi_sort_zeek.patch b/arkime/patch/spi_sort_malcolm.patch similarity index 88% rename from arkime/patch/spi_sort_zeek.patch rename to arkime/patch/spi_sort_malcolm.patch index 73afa84bc..1a16f13cf 100644 --- a/arkime/patch/spi_sort_zeek.patch +++ b/arkime/patch/spi_sort_malcolm.patch @@ -1,13 +1,13 @@ diff --git a/viewer/vueapp/src/components/spiview/Spiview.vue b/viewer/vueapp/src/components/spiview/Spiview.vue -index 66e08ecf..f7156b80 100644 +index 76dd613b..b8117034 100644 --- a/viewer/vueapp/src/components/spiview/Spiview.vue +++ b/viewer/vueapp/src/components/spiview/Spiview.vue -@@ -915,6 +915,8 @@ export default { - +@@ -960,6 +960,8 @@ export default { + // sorted list of categories for the view this.categoryList = Object.keys(this.categoryObjects).sort(); + this.categoryList.splice(this.categoryList.indexOf('malcolm'), 1); + this.categoryList.unshift('malcolm'); this.categoryList.splice(this.categoryList.indexOf('general'), 1); this.categoryList.unshift('general'); - + diff --git a/arkime/patch/viewer_db_opensearchv2_keyword_hack.patch b/arkime/patch/viewer_db_opensearchv2_keyword_hack.patch index 4f93580cf..89c85d5e3 100644 --- a/arkime/patch/viewer_db_opensearchv2_keyword_hack.patch +++ b/arkime/patch/viewer_db_opensearchv2_keyword_hack.patch @@ -1,8 +1,8 @@ diff --git a/viewer/db.js b/viewer/db.js -index 1eebc559..45645fd3 100644 +index afd0ef70..ca707dee 100644 --- a/viewer/db.js +++ b/viewer/db.js -@@ -382,7 +382,13 @@ function fixSessionFields (fields, unflatten) { +@@ -386,7 +386,13 @@ function fixSessionFields (fields, unflatten) { delete fields[f]; for (let i = 0; i < path.length; i++) { if (i === path.length - 1) { diff --git a/arkime/patch/viewer_wider_field_detail.patch b/arkime/patch/viewer_wider_field_detail.patch index 3543771f2..ec0cc5376 100644 --- a/arkime/patch/viewer_wider_field_detail.patch +++ b/arkime/patch/viewer_wider_field_detail.patch @@ -1,8 +1,8 @@ diff --git a/viewer/vueapp/src/components/sessions/SessionDetail.vue b/viewer/vueapp/src/components/sessions/SessionDetail.vue -index a15f3b52..e7b0cd7b 100644 +index 572c41f3..60143d82 100644 --- a/viewer/vueapp/src/components/sessions/SessionDetail.vue +++ b/viewer/vueapp/src/components/sessions/SessionDetail.vue -@@ -844,7 +844,7 @@ export default { +@@ -846,7 +846,7 @@ export default { .session-detail dt { float: left; clear: left; @@ -11,7 +11,7 @@ index a15f3b52..e7b0cd7b 100644 text-align: right; margin-right: 6px; line-height: 1.7; -@@ -852,7 +852,7 @@ export default { +@@ -854,7 +854,7 @@ export default { } .session-detail dd { From 8185a9190b3aa40a2b697bf973cc363f201624b5 Mon Sep 17 00:00:00 2001 From: Seth Grover Date: Mon, 15 Aug 2022 09:12:18 -0600 Subject: [PATCH 2/3] fix build --- Dockerfiles/arkime.Dockerfile | 2 +- arkime/patch/bulkfields.patch | 163 ++++++++++++++++++++++++++++++++++ 2 files changed, 164 insertions(+), 1 deletion(-) create mode 100644 arkime/patch/bulkfields.patch diff --git a/Dockerfiles/arkime.Dockerfile b/Dockerfiles/arkime.Dockerfile index 35b6d20c6..1a077361b 100644 --- a/Dockerfiles/arkime.Dockerfile +++ b/Dockerfiles/arkime.Dockerfile @@ -66,7 +66,7 @@ RUN apt-get -q update && \ sed -i 's/\!\[.*\](.*\/badge.svg)//g' README.md && \ pandoc -s --self-contained --metadata title="Malcolm README" --css $ARKIMEDIR/doc/doc.css -o $ARKIMEDIR/doc/README.html $ARKIMEDIR/doc/README.md && \ cd /opt && \ - git clone --depth=1 --single-branch --recurse-submodules --shallow-submodules --no-tags --branch="$ARKIME_VERSION" "$ARKIME_URL" "./arkime-"$ARKIME_VERSION && \ + git clone --recurse-submodules --branch="$ARKIME_VERSION" "$ARKIME_URL" "./arkime-"$ARKIME_VERSION && \ cd "./arkime-"$ARKIME_VERSION && \ bash -c 'for i in /opt/patches/*; do patch -p 1 -r - --no-backup-if-mismatch < $i || true; done' && \ find $ARKIMEDIR/doc/images/screenshots -name "*.png" -delete && \ diff --git a/arkime/patch/bulkfields.patch b/arkime/patch/bulkfields.patch new file mode 100644 index 000000000..2230980b6 --- /dev/null +++ b/arkime/patch/bulkfields.patch @@ -0,0 +1,163 @@ +diff --git a/CHANGELOG b/CHANGELOG +index 3878d951..f276fcfb 100644 +--- a/CHANGELOG ++++ b/CHANGELOG +@@ -39,6 +39,7 @@ NOTICE: Cross-cluster Shortcuts require you to not restart all your viewers at o + - capture - save sessions not saving packets for across restarts + - capture - afpacket rewrite, improve performance & less out of order packets + - capture - fix quic crash ++ - capture - make creating fields from config/parsers/wise/tagger use ES bulk call + - capture/viewer - new outer fields replace gre fields (PR #1889) + - cont3xt - new Cont3xt application, see https://arkime.com/cont3xt + - cont3xt/viewer - share new user UI +diff --git a/capture/bsb.h b/capture/bsb.h +index 282e81d4..8ac4df26 100644 +--- a/capture/bsb.h ++++ b/capture/bsb.h +@@ -13,12 +13,12 @@ + #define BSB_INIT(b, buffer, size) \ + do { \ + (b).buf = (unsigned char*)buffer; \ +- (b).ptr = (unsigned char*)buffer; \ ++ (b).ptr = (b).buf; \ + int s = (int)size; \ +- if ((buffer == NULL) || (s < 0)) \ ++ if (((b).buf == NULL) || (s < 0)) \ + (b).end = 0; \ + else \ +- (b).end = (unsigned char*)buffer + size; \ ++ (b).end = (b).buf + size; \ + } while (0) + + #define BSB_SET_ERROR(b) ((b).end = NULL) +diff --git a/capture/db.c b/capture/db.c +index a21d853a..70be2c19 100644 +--- a/capture/db.c ++++ b/capture/db.c +@@ -2366,22 +2366,39 @@ LOCAL void moloch_db_load_fields() + free(data); + } + /******************************************************************************/ ++LOCAL BSB fieldBSB; ++LOCAL int fieldBSBTimeout; ++LOCAL gboolean moloch_db_fieldsbsb_timeout(gpointer UNUSED(user_data)) ++{ ++ if (fieldBSB.buf && BSB_LENGTH(fieldBSB) > 0) { ++ moloch_http_schedule(esServer, "POST", "/_bulk", 6, (char *)fieldBSB.buf, BSB_LENGTH(fieldBSB), NULL, MOLOCH_HTTP_PRIORITY_BEST, NULL, NULL); ++ BSB_INIT(fieldBSB, moloch_http_get_buffer(config.dbBulkSize), config.dbBulkSize); ++ } ++ fieldBSBTimeout = 0; ++ return G_SOURCE_REMOVE; ++} ++/******************************************************************************/ ++LOCAL void moloch_db_fieldbsb_make() ++{ ++ if (!fieldBSB.buf) { ++ BSB_INIT(fieldBSB, moloch_http_get_buffer(config.dbBulkSize), config.dbBulkSize); ++ fieldBSBTimeout = g_timeout_add_seconds(1, moloch_db_fieldsbsb_timeout, 0); ++ } else if (BSB_REMAINING(fieldBSB) < 1000) { ++ g_source_remove(fieldBSBTimeout); ++ moloch_db_fieldsbsb_timeout(0); ++ fieldBSBTimeout = g_timeout_add_seconds(1, moloch_db_fieldsbsb_timeout, 0); ++ } ++} ++/******************************************************************************/ + void moloch_db_add_field(char *group, char *kind, char *expression, char *friendlyName, char *dbField, char *help, int haveap, va_list ap) + { +- char key[100]; +- int key_len; +- BSB bsb; +- + if (config.dryRun) + return; + +- char *json = moloch_http_get_buffer(10000); ++ moloch_db_fieldbsb_make(); + +- BSB_INIT(bsb, json, 10000); +- +- key_len = snprintf(key, sizeof(key), "/%sfields/_doc/%s", config.prefix, expression); +- +- BSB_EXPORT_sprintf(bsb, "{\"friendlyName\": \"%s\", \"group\": \"%s\", \"help\": \"%s\", \"dbField2\": \"%s\", \"type\": \"%s\"", ++ BSB_EXPORT_sprintf(fieldBSB, "{\"index\": {\"_index\": \"%sfields\", \"_id\": \"%s\"}}\n", config.prefix, expression); ++ BSB_EXPORT_sprintf(fieldBSB, "{\"friendlyName\": \"%s\", \"group\": \"%s\", \"help\": \"%s\", \"dbField2\": \"%s\", \"type\": \"%s\"", + friendlyName, + group, + help, +@@ -2398,41 +2415,33 @@ void moloch_db_add_field(char *group, char *kind, char *expression, char *friend + if (!value) + break; + +- BSB_EXPORT_sprintf(bsb, ", \"%s\": ", field); ++ BSB_EXPORT_sprintf(fieldBSB, ", \"%s\": ", field); + if (*value == '{' || *value == '[') +- BSB_EXPORT_sprintf(bsb, "%s", value); ++ BSB_EXPORT_sprintf(fieldBSB, "%s", value); + else +- BSB_EXPORT_sprintf(bsb, "\"%s\"", value); ++ BSB_EXPORT_sprintf(fieldBSB, "\"%s\"", value); + } + } + +- BSB_EXPORT_u08(bsb, '}'); +- moloch_http_schedule(esServer, "POST", key, key_len, json, BSB_LENGTH(bsb), NULL, MOLOCH_HTTP_PRIORITY_NORMAL, NULL, NULL); ++ BSB_EXPORT_cstr(fieldBSB, "}\n"); + } + /******************************************************************************/ + void moloch_db_update_field(char *expression, char *name, char *value) + { +- char key[1000]; +- int key_len; +- BSB bsb; +- + if (config.dryRun) + return; + +- char *json = moloch_http_get_buffer(1000); ++ moloch_db_fieldbsb_make(); + +- BSB_INIT(bsb, json, 1000); ++ BSB_EXPORT_sprintf(fieldBSB, "{\"update\": {\"_index\": \"%sfields\", \"_id\": \"%s\"}}\n", config.prefix, expression); + +- key_len = snprintf(key, sizeof(key), "/%sfields/_update/%s", config.prefix, expression); +- +- BSB_EXPORT_sprintf(bsb, "{\"doc\": {\"%s\":", name); ++ BSB_EXPORT_sprintf(fieldBSB, "{\"doc\": {\"%s\":", name); + if (*value == '[') { +- BSB_EXPORT_sprintf(bsb, "%s", value); ++ BSB_EXPORT_sprintf(fieldBSB, "%s", value); + } else { +- moloch_db_js0n_str(&bsb, (unsigned char*)value, TRUE); ++ moloch_db_js0n_str(&fieldBSB, (unsigned char*)value, TRUE); + } +- BSB_EXPORT_sprintf(bsb, "}}"); +- moloch_http_schedule(esServer, "POST", key, key_len, json, BSB_LENGTH(bsb), NULL, MOLOCH_HTTP_PRIORITY_NORMAL, NULL, NULL); ++ BSB_EXPORT_cstr(fieldBSB, "}}\n"); + } + /******************************************************************************/ + void moloch_db_update_filesize(uint32_t fileid, uint64_t filesize, uint64_t packetsSize, uint32_t packets) +@@ -2678,6 +2687,12 @@ void moloch_db_init() + void moloch_db_exit() + { + if (!config.dryRun) { ++ if (fieldBSB.buf && BSB_LENGTH(fieldBSB) > 0) { ++ if (fieldBSBTimeout) ++ g_source_remove(fieldBSBTimeout); ++ moloch_db_fieldsbsb_timeout(0); ++ } ++ + for (int i = 0; timers[i]; i++) { + g_source_remove(timers[i]); + } +diff --git a/capture/http.c b/capture/http.c +index 1a66e5d9..62b89d76 100644 +--- a/capture/http.c ++++ b/capture/http.c +@@ -354,7 +354,6 @@ LOCAL void moloch_http_add_request(MolochHttpServer_t *server, MolochHttpRequest + + if (!requestsTimer) + requestsTimer = g_timeout_add(0, moloch_http_send_timer_callback, NULL); +- } else { + } + } + /******************************************************************************/ From 2e5f653f7cfc7cfa7fdd892a74583a8acbeab07d Mon Sep 17 00:00:00 2001 From: Seth Grover Date: Mon, 15 Aug 2022 09:41:34 -0600 Subject: [PATCH 3/3] set userAuthIps to work with docker --- Dockerfiles/arkime.Dockerfile | 5 ----- arkime/etc/config.ini | 1 + 2 files changed, 1 insertion(+), 5 deletions(-) diff --git a/Dockerfiles/arkime.Dockerfile b/Dockerfiles/arkime.Dockerfile index 1a077361b..081257549 100644 --- a/Dockerfiles/arkime.Dockerfile +++ b/Dockerfiles/arkime.Dockerfile @@ -80,15 +80,10 @@ RUN apt-get -q update && \ rm -rf ./viewer/vueapp/src/components/upload ./capture/plugins/suricata* && \ sed -i "s/^\(ARKIME_LOCALELASTICSEARCH=\).*/\1"$ARKIME_LOCALELASTICSEARCH"/" ./release/Configure && \ sed -i "s/^\(ARKIME_INET=\).*/\1"$ARKIME_INET"/" ./release/Configure && \ - echo '>>>> easybutton-build.sh' && \ ./easybutton-build.sh && \ - echo '>>>> npm -g config set user root' && \ npm -g config set user root && \ - echo '>>>> make install' && \ make install && \ - echo '>>>> npm cache clean --force' && \ npm cache clean --force && \ - echo '>>>> cleanup' && \ rm -f ${ARKIMEDIR}/wiseService/source.* && \ bash -c "file ${ARKIMEDIR}/bin/* ${ARKIMEDIR}/node-v*/bin/* | grep 'ELF 64-bit' | sed 's/:.*//' | xargs -l -r strip -v --strip-unneeded" diff --git a/arkime/etc/config.ini b/arkime/etc/config.ini index c23d4c923..f3aa625bc 100644 --- a/arkime/etc/config.ini +++ b/arkime/etc/config.ini @@ -9,6 +9,7 @@ cronQueries=true rotateIndex=daily passwordSecret=Malcolm httpRealm=Arkime +userAuthIps=::,0.0.0.0/0 interface=eth0 wiseHost=127.0.0.1 wisePort=8081