diff --git a/scripts/install.py b/scripts/install.py index dd012b1f9..19ef651d1 100755 --- a/scripts/install.py +++ b/scripts/install.py @@ -30,6 +30,7 @@ except ImportError: getpwuid = None from collections import defaultdict, namedtuple +from enum import IntEnum from malcolm_common import ( AskForString, @@ -38,6 +39,8 @@ ChooseOne, DetermineYamlFileFormat, DialogInit, + DialogBackException, + DialogCanceledException, DisplayMessage, DOCKER_COMPOSE_INSTALL_URLS, DOCKER_INSTALL_URLS, @@ -100,6 +103,8 @@ MAC_BREW_DOCKER_COMPOSE_PACKAGE = 'docker-compose' MAC_BREW_DOCKER_SETTINGS = '/Users/{}/Library/Group Containers/group.com.docker/settings.json' +BACK_LABEL = 'Back' + LOGSTASH_JAVA_OPTS_DEFAULT = '-server -Xmx2500m -Xms2500m -Xss1536k -XX:-HeapDumpOnOutOfMemoryError -Djava.security.egd=file:/dev/./urandom -Dlog4j.formatMsgNoLookups=true' OPENSEARCH_JAVA_OPTS_DEFAULT = '-server -Xmx10g -Xms10g -Xss256k -XX:-HeapDumpOnOutOfMemoryError -Djava.security.egd=file:/dev/./urandom -Dlog4j.formatMsgNoLookups=true' @@ -122,6 +127,35 @@ str2percent = lambda val: max(min(100, int(remove_suffix(val, '%'))), 0) if val else 0 +class ConfigOptions(IntEnum): + Preconfig = 1 + UidGuid = 2 + NodeName = 3 + RunProfile = 4 + DatabaseMode = 5 + LogstashRemote = 6 + ContainerResources = 7 + RestartMode = 8 + RequireHTTPS = 9 + DockerNetworking = 10 + AuthMethod = 11 + StorageLocations = 12 + ILMISM = 13 + StorageManagement = 14 + AutoArkime = 15 + AutoSuricata = 16 + SuricataRuleUpdate = 17 + AutoZeek = 18 + ICS = 19 + Enrichment = 20 + OpenPorts = 21 + FileCarving = 22 + NetBox = 23 + Capture = 24 + DarkMode = 25 + PostConfig = 26 + + ################################################################################################### # get interactive user response to Y/N question def InstallerYesOrNo( @@ -132,6 +166,7 @@ def InstallerYesOrNo( uiMode=UserInterfaceMode.InteractionInput | UserInterfaceMode.InteractionDialog, yesLabel='Yes', noLabel='No', + extraLabel=BACK_LABEL, ): global args defBehavior = defaultBehavior @@ -145,6 +180,7 @@ def InstallerYesOrNo( uiMode=uiMode, yesLabel=yesLabel, noLabel=noLabel, + extraLabel=extraLabel, ) @@ -156,6 +192,7 @@ def InstallerAskForString( forceInteraction=False, defaultBehavior=UserInputDefaultsBehavior.DefaultsPrompt | UserInputDefaultsBehavior.DefaultsAccept, uiMode=UserInterfaceMode.InteractionInput | UserInterfaceMode.InteractionDialog, + extraLabel=BACK_LABEL, ): global args defBehavior = defaultBehavior @@ -167,6 +204,7 @@ def InstallerAskForString( default=default, defaultBehavior=defBehavior, uiMode=uiMode, + extraLabel=extraLabel, ) @@ -178,6 +216,7 @@ def InstallerChooseOne( forceInteraction=False, defaultBehavior=UserInputDefaultsBehavior.DefaultsPrompt | UserInputDefaultsBehavior.DefaultsAccept, uiMode=UserInterfaceMode.InteractionInput | UserInterfaceMode.InteractionDialog, + extraLabel=BACK_LABEL, ): global args defBehavior = defaultBehavior @@ -189,6 +228,7 @@ def InstallerChooseOne( choices=choices, defaultBehavior=defBehavior, uiMode=uiMode, + extraLabel=extraLabel, ) @@ -200,6 +240,7 @@ def InstallerChooseMultiple( forceInteraction=False, defaultBehavior=UserInputDefaultsBehavior.DefaultsPrompt | UserInputDefaultsBehavior.DefaultsAccept, uiMode=UserInterfaceMode.InteractionInput | UserInterfaceMode.InteractionDialog, + extraLabel=BACK_LABEL, ): global args defBehavior = defaultBehavior @@ -211,6 +252,7 @@ def InstallerChooseMultiple( choices=choices, defaultBehavior=defBehavior, uiMode=uiMode, + extraLabel=extraLabel, ) @@ -221,6 +263,7 @@ def InstallerDisplayMessage( forceInteraction=False, defaultBehavior=UserInputDefaultsBehavior.DefaultsPrompt | UserInputDefaultsBehavior.DefaultsAccept, uiMode=UserInterfaceMode.InteractionInput | UserInterfaceMode.InteractionDialog, + extraLabel=BACK_LABEL, ): global args defBehavior = defaultBehavior @@ -231,6 +274,7 @@ def InstallerDisplayMessage( message, defaultBehavior=defBehavior, uiMode=uiMode, + extraLabel=extraLabel, ) @@ -497,31 +541,6 @@ def tweak_malcolm_runtime(self, malcolm_install_path): if (not args.configDir) or (not os.path.isdir(args.configDir)): raise Exception("Could not determine configuration directory containing Malcolm's .env files") - # figure out what UID/GID to run non-root processes under docker as - puid, pgid = DetermineUid(self.scriptUser, self.platform, malcolm_install_path) - - loopBreaker = CountUntilException(MaxAskForValueCount, 'Invalid UID/GID') - while ( - (not puid.isdigit()) - or (not pgid.isdigit()) - or ( - not InstallerYesOrNo( - f'Malcolm processes will run as UID {puid} and GID {pgid}. Is this OK?', default=True - ) - ) - ) and loopBreaker.increment(): - puid = InstallerAskForString( - 'Enter user ID (UID) for running non-root Malcolm processes', default=defaultUid - ) - pgid = InstallerAskForString( - 'Enter group ID (GID) for running non-root Malcolm processes', default=defaultGid - ) - - pcapNodeName = InstallerAskForString( - f'Enter the node name to associate with network traffic metadata', - default=args.pcapNodeName, - ) - if self.orchMode is OrchestrationFramework.DOCKER_COMPOSE: # guestimate how much memory we should use based on total system memory @@ -597,1010 +616,1189 @@ def tweak_malcolm_runtime(self, malcolm_install_path): dashboardsUrl = 'http://dashboards:5601/dashboards' logstashHost = 'logstash:5044' indexSnapshotCompressed = False - malcolmProfile = ( - PROFILE_MALCOLM - if InstallerYesOrNo( - 'Run with Malcolm (all containers) or Hedgehog (capture only) profile?', - default=args.malcolmProfile, - yesLabel='Malcolm', - noLabel='Hedgehog', - ) - else PROFILE_HEDGEHOG - ) - - if (malcolmProfile == PROFILE_MALCOLM) and InstallerYesOrNo( - 'Should Malcolm use and maintain its own OpenSearch instance?', - default=DATABASE_MODE_ENUMS[args.opensearchPrimaryMode] == DatabaseMode.OpenSearchLocal, - ): - opensearchPrimaryMode = DatabaseMode.OpenSearchLocal + behindReverseProxy = False + dockerNetworkExternalName = "" - else: - databaseModeChoice = '' - allowedDatabaseModes = { - DATABASE_MODE_LABELS[DatabaseMode.OpenSearchLocal]: [DatabaseMode.OpenSearchLocal, 'local OpenSearch'], - DATABASE_MODE_LABELS[DatabaseMode.OpenSearchRemote]: [ - DatabaseMode.OpenSearchRemote, - 'remote OpenSearch', - ], - DATABASE_MODE_LABELS[DatabaseMode.ElasticsearchRemote]: [ - DatabaseMode.ElasticsearchRemote, - 'remote Elasticsearch', - ], - } - if malcolmProfile != PROFILE_MALCOLM: - del allowedDatabaseModes[DATABASE_MODE_LABELS[DatabaseMode.OpenSearchLocal]] - loopBreaker = CountUntilException(MaxAskForValueCount, 'Invalid primary document store mode') - while databaseModeChoice not in list(allowedDatabaseModes.keys()) and loopBreaker.increment(): - databaseModeChoice = InstallerChooseOne( - 'Select primary Malcolm document store', - choices=[ - (x, allowedDatabaseModes[x][1], x == args.opensearchPrimaryMode) - for x in list(allowedDatabaseModes.keys()) - ], - ) - opensearchPrimaryMode = allowedDatabaseModes[databaseModeChoice][0] - opensearchPrimaryLabel = allowedDatabaseModes[databaseModeChoice][1] - - if opensearchPrimaryMode in (DatabaseMode.OpenSearchRemote, DatabaseMode.ElasticsearchRemote): - loopBreaker = CountUntilException(MaxAskForValueCount, f'Invalid {opensearchPrimaryLabel} URL') - opensearchPrimaryUrl = '' - while (len(opensearchPrimaryUrl) <= 1) and loopBreaker.increment(): - opensearchPrimaryUrl = InstallerAskForString( - f'Enter primary {opensearchPrimaryLabel} connection URL (e.g., https://192.168.1.123:9200)', - default=args.opensearchPrimaryUrl, - ) - opensearchPrimarySslVerify = opensearchPrimaryUrl.lower().startswith('https') and InstallerYesOrNo( - f'Require SSL certificate validation for communication with {opensearchPrimaryLabel} instance?', - default=args.opensearchPrimarySslVerify, - ) - else: - indexSnapshotCompressed = InstallerYesOrNo( - f'Compress {opensearchPrimaryLabel} index snapshots?', - default=args.indexSnapshotCompressed, - ) + prevStep = None + currentStep = ConfigOptions.Preconfig + while True: + prevStep = currentStep + currentStep = ConfigOptions(int(currentStep) + 1) + eprint(f'{prevStep} -> {currentStep}') + try: + ################################################################################### + if currentStep == ConfigOptions.Preconfig: + pass - if opensearchPrimaryMode == DatabaseMode.ElasticsearchRemote: - loopBreaker = CountUntilException(MaxAskForValueCount, f'Invalid Kibana connection URL') - dashboardsUrl = '' - while (len(dashboardsUrl) <= 1) and loopBreaker.increment(): - dashboardsUrl = InstallerAskForString( - f'Enter Kibana connection URL (e.g., https://192.168.1.123:5601)', - default=args.dashboardsUrl, - ) - if malcolmProfile != PROFILE_MALCOLM: - loopBreaker = CountUntilException(MaxAskForValueCount, f'Invalid Logstash host and port') - logstashHost = '' - while (len(logstashHost) <= 1) and loopBreaker.increment(): - logstashHost = InstallerAskForString( - f'Enter Logstash host and port (e.g., 192.168.1.123:5044)', - default=args.logstashHost, - ) + ################################################################################### + elif currentStep == ConfigOptions.UidGuid: + # figure out what UID/GID to run non-root processes under docker as + puid, pgid = DetermineUid(self.scriptUser, self.platform, malcolm_install_path) + defaultUid, defaultGid = puid, pgid - if (malcolmProfile == PROFILE_MALCOLM) and InstallerYesOrNo( - 'Forward Logstash logs to a secondary remote document store?', - default=( - DATABASE_MODE_ENUMS[args.opensearchSecondaryMode] - in (DatabaseMode.OpenSearchRemote, DatabaseMode.ElasticsearchRemote) - ), - ): - databaseModeChoice = '' - allowedDatabaseModes = { - DATABASE_MODE_LABELS[DatabaseMode.OpenSearchRemote]: [ - DatabaseMode.OpenSearchRemote, - 'remote OpenSearch', - ], - DATABASE_MODE_LABELS[DatabaseMode.ElasticsearchRemote]: [ - DatabaseMode.ElasticsearchRemote, - 'remote Elasticsearch', - ], - } - loopBreaker = CountUntilException(MaxAskForValueCount, 'Invalid secondary document store mode') - while databaseModeChoice not in list(allowedDatabaseModes.keys()) and loopBreaker.increment(): - databaseModeChoice = InstallerChooseOne( - 'Select secondary Malcolm document store', - choices=[ - (x, allowedDatabaseModes[x][1], x == args.opensearchSecondaryMode) - for x in list(allowedDatabaseModes.keys()) - ], - ) - opensearchSecondaryMode = allowedDatabaseModes[databaseModeChoice][0] - opensearchSecondaryLabel = allowedDatabaseModes[databaseModeChoice][1] - - if opensearchSecondaryMode in (DatabaseMode.OpenSearchRemote, DatabaseMode.ElasticsearchRemote): - loopBreaker = CountUntilException(MaxAskForValueCount, f'Invalid {opensearchSecondaryLabel} URL') - opensearchSecondaryUrl = '' - while (len(opensearchSecondaryUrl) <= 1) and loopBreaker.increment(): - opensearchSecondaryUrl = InstallerAskForString( - f'Enter secondary {opensearchSecondaryLabel} connection URL (e.g., https://192.168.1.123:9200)', - default=args.opensearchSecondaryUrl, - ) - opensearchSecondarySslVerify = opensearchSecondaryUrl.lower().startswith('https') and InstallerYesOrNo( - f'Require SSL certificate validation for communication with secondary {opensearchSecondaryLabel} instance?', - default=args.opensearchSecondarySslVerify, - ) + loopBreaker = CountUntilException(MaxAskForValueCount, 'Invalid UID/GID') + while ( + (not puid.isdigit()) + or (not pgid.isdigit()) + or ( + not InstallerYesOrNo( + f'Malcolm processes will run as UID {puid} and GID {pgid}. Is this OK?', + default=True, + ) + ) + ) and loopBreaker.increment(): + puid = InstallerAskForString( + 'Enter user ID (UID) for running non-root Malcolm processes', default=defaultUid + ) + pgid = InstallerAskForString( + 'Enter group ID (GID) for running non-root Malcolm processes', default=defaultGid + ) - if (opensearchPrimaryMode in (DatabaseMode.OpenSearchRemote, DatabaseMode.ElasticsearchRemote)) or ( - opensearchSecondaryMode in (DatabaseMode.OpenSearchRemote, DatabaseMode.ElasticsearchRemote) - ): - InstallerDisplayMessage( - f'You must run auth_setup after {ScriptName} to store data store connection credentials.', - ) + ################################################################################### + elif currentStep == ConfigOptions.NodeName: + pcapNodeName = InstallerAskForString( + f'Enter the node name to associate with network traffic metadata', + default=args.pcapNodeName, + ) - if malcolmProfile == PROFILE_MALCOLM: - loopBreaker = CountUntilException( - MaxAskForValueCount, - f'Invalid {"OpenSearch/" if opensearchPrimaryMode == DatabaseMode.OpenSearchLocal else ""}Logstash memory setting(s)', - ) - while ( - not InstallerYesOrNo( - ( - f'Setting {osMemory} for OpenSearch and {lsMemory} for Logstash. Is this OK?' - if opensearchPrimaryMode == DatabaseMode.OpenSearchLocal - else f'Setting {lsMemory} for Logstash. Is this OK?' - ), - default=True, - ) - and loopBreaker.increment() - ): - if opensearchPrimaryMode == DatabaseMode.OpenSearchLocal: - osMemory = InstallerAskForString('Enter memory for OpenSearch (e.g., 16g, 9500m, etc.)') - lsMemory = InstallerAskForString('Enter memory for Logstash (e.g., 4g, 2500m, etc.)') - - loopBreaker = CountUntilException(MaxAskForValueCount, 'Invalid Logstash worker setting(s)') - while ( - (not str(lsWorkers).isdigit()) - or ( - not InstallerYesOrNo( - f'Setting {lsWorkers} workers for Logstash pipelines. Is this OK?', default=True + ################################################################################### + elif currentStep == ConfigOptions.RunProfile: + malcolmProfile = ( + PROFILE_MALCOLM + if InstallerYesOrNo( + 'Run with Malcolm (all containers) or Hedgehog (capture only) profile?', + default=args.malcolmProfile, + yesLabel='Malcolm', + noLabel='Hedgehog', + ) + else PROFILE_HEDGEHOG ) - ) - ) and loopBreaker.increment(): - lsWorkers = InstallerAskForString('Enter number of Logstash workers (e.g., 4, 8, etc.)') - restartMode = None - allowedRestartModes = ('no', 'on-failure', 'always', 'unless-stopped') - if (self.orchMode is OrchestrationFramework.DOCKER_COMPOSE) and InstallerYesOrNo( - 'Restart Malcolm upon system or Docker daemon restart?', default=args.malcolmAutoRestart - ): - loopBreaker = CountUntilException(MaxAskForValueCount, 'Invalid restart mode') - while restartMode not in allowedRestartModes and loopBreaker.increment(): - restartMode = InstallerChooseOne( - 'Select Malcolm restart behavior', - choices=[(x, '', x == 'unless-stopped') for x in allowedRestartModes], - ) - else: - restartMode = 'no' + ################################################################################### + elif currentStep == ConfigOptions.DatabaseMode: + if (malcolmProfile == PROFILE_MALCOLM) and InstallerYesOrNo( + 'Should Malcolm use and maintain its own OpenSearch instance?', + default=DATABASE_MODE_ENUMS[args.opensearchPrimaryMode] == DatabaseMode.OpenSearchLocal, + ): + opensearchPrimaryMode = DatabaseMode.OpenSearchLocal - if malcolmProfile == PROFILE_MALCOLM: - nginxSSL = InstallerYesOrNo('Require encrypted HTTPS connections?', default=args.nginxSSL) - if (not nginxSSL) and (not args.acceptDefaultsNonInteractive): - nginxSSL = not InstallerYesOrNo( - 'Unencrypted connections are NOT recommended. Are you sure?', default=False - ) - else: - nginxSSL = True + else: + databaseModeChoice = '' + allowedDatabaseModes = { + DATABASE_MODE_LABELS[DatabaseMode.OpenSearchLocal]: [ + DatabaseMode.OpenSearchLocal, + 'local OpenSearch', + ], + DATABASE_MODE_LABELS[DatabaseMode.OpenSearchRemote]: [ + DatabaseMode.OpenSearchRemote, + 'remote OpenSearch', + ], + DATABASE_MODE_LABELS[DatabaseMode.ElasticsearchRemote]: [ + DatabaseMode.ElasticsearchRemote, + 'remote Elasticsearch', + ], + } + if malcolmProfile != PROFILE_MALCOLM: + del allowedDatabaseModes[DATABASE_MODE_LABELS[DatabaseMode.OpenSearchLocal]] + loopBreaker = CountUntilException(MaxAskForValueCount, 'Invalid primary document store mode') + while databaseModeChoice not in list(allowedDatabaseModes.keys()) and loopBreaker.increment(): + databaseModeChoice = InstallerChooseOne( + 'Select primary Malcolm document store', + choices=[ + (x, allowedDatabaseModes[x][1], x == args.opensearchPrimaryMode) + for x in list(allowedDatabaseModes.keys()) + ], + ) + opensearchPrimaryMode = allowedDatabaseModes[databaseModeChoice][0] + opensearchPrimaryLabel = allowedDatabaseModes[databaseModeChoice][1] + + if opensearchPrimaryMode in (DatabaseMode.OpenSearchRemote, DatabaseMode.ElasticsearchRemote): + loopBreaker = CountUntilException(MaxAskForValueCount, f'Invalid {opensearchPrimaryLabel} URL') + opensearchPrimaryUrl = '' + while (len(opensearchPrimaryUrl) <= 1) and loopBreaker.increment(): + opensearchPrimaryUrl = InstallerAskForString( + f'Enter primary {opensearchPrimaryLabel} connection URL (e.g., https://192.168.1.123:9200)', + default=args.opensearchPrimaryUrl, + ) + opensearchPrimarySslVerify = opensearchPrimaryUrl.lower().startswith( + 'https' + ) and InstallerYesOrNo( + f'Require SSL certificate validation for communication with {opensearchPrimaryLabel} instance?', + default=args.opensearchPrimarySslVerify, + ) + else: + indexSnapshotCompressed = InstallerYesOrNo( + f'Compress {opensearchPrimaryLabel} index snapshots?', + default=args.indexSnapshotCompressed, + ) - behindReverseProxy = False - dockerNetworkExternalName = "" - traefikLabels = False - traefikHost = "" - traefikOpenSearchHost = "" - traefikEntrypoint = "" - traefikResolver = "" - - behindReverseProxy = (self.orchMode is OrchestrationFramework.KUBERNETES) or ( - (malcolmProfile == PROFILE_MALCOLM) - and InstallerYesOrNo( - 'Will Malcolm be running behind another reverse proxy (Traefik, Caddy, etc.)?', - default=args.behindReverseProxy or (not nginxSSL), - ) - ) + if opensearchPrimaryMode == DatabaseMode.ElasticsearchRemote: + loopBreaker = CountUntilException(MaxAskForValueCount, f'Invalid Kibana connection URL') + dashboardsUrl = '' + while (len(dashboardsUrl) <= 1) and loopBreaker.increment(): + dashboardsUrl = InstallerAskForString( + f'Enter Kibana connection URL (e.g., https://192.168.1.123:5601)', + default=args.dashboardsUrl, + ) - if self.orchMode is OrchestrationFramework.DOCKER_COMPOSE: - if behindReverseProxy: - traefikLabels = InstallerYesOrNo('Configure labels for Traefik?', default=bool(args.traefikHost)) - if traefikLabels: - loopBreaker = CountUntilException(MaxAskForValueCount, 'Invalid Traefik request domain') - while (len(traefikHost) <= 1) and loopBreaker.increment(): - traefikHost = InstallerAskForString( - 'Enter request domain (host header value) for Malcolm interface Traefik router (e.g., malcolm.example.org)', - default=args.traefikHost, + ################################################################################### + elif currentStep == ConfigOptions.LogstashRemote: + if malcolmProfile != PROFILE_MALCOLM: + loopBreaker = CountUntilException(MaxAskForValueCount, f'Invalid Logstash host and port') + logstashHost = '' + while (len(logstashHost) <= 1) and loopBreaker.increment(): + logstashHost = InstallerAskForString( + f'Enter Logstash host and port (e.g., 192.168.1.123:5044)', + default=args.logstashHost, + ) + + if (malcolmProfile == PROFILE_MALCOLM) and InstallerYesOrNo( + 'Forward Logstash logs to a secondary remote document store?', + default=( + DATABASE_MODE_ENUMS[args.opensearchSecondaryMode] + in (DatabaseMode.OpenSearchRemote, DatabaseMode.ElasticsearchRemote) + ), + ): + databaseModeChoice = '' + allowedDatabaseModes = { + DATABASE_MODE_LABELS[DatabaseMode.OpenSearchRemote]: [ + DatabaseMode.OpenSearchRemote, + 'remote OpenSearch', + ], + DATABASE_MODE_LABELS[DatabaseMode.ElasticsearchRemote]: [ + DatabaseMode.ElasticsearchRemote, + 'remote Elasticsearch', + ], + } + loopBreaker = CountUntilException(MaxAskForValueCount, 'Invalid secondary document store mode') + while databaseModeChoice not in list(allowedDatabaseModes.keys()) and loopBreaker.increment(): + databaseModeChoice = InstallerChooseOne( + 'Select secondary Malcolm document store', + choices=[ + (x, allowedDatabaseModes[x][1], x == args.opensearchSecondaryMode) + for x in list(allowedDatabaseModes.keys()) + ], + ) + opensearchSecondaryMode = allowedDatabaseModes[databaseModeChoice][0] + opensearchSecondaryLabel = allowedDatabaseModes[databaseModeChoice][1] + + if opensearchSecondaryMode in (DatabaseMode.OpenSearchRemote, DatabaseMode.ElasticsearchRemote): + loopBreaker = CountUntilException( + MaxAskForValueCount, f'Invalid {opensearchSecondaryLabel} URL' ) - if opensearchPrimaryMode == DatabaseMode.OpenSearchLocal: + opensearchSecondaryUrl = '' + while (len(opensearchSecondaryUrl) <= 1) and loopBreaker.increment(): + opensearchSecondaryUrl = InstallerAskForString( + f'Enter secondary {opensearchSecondaryLabel} connection URL (e.g., https://192.168.1.123:9200)', + default=args.opensearchSecondaryUrl, + ) + opensearchSecondarySslVerify = opensearchSecondaryUrl.lower().startswith( + 'https' + ) and InstallerYesOrNo( + f'Require SSL certificate validation for communication with secondary {opensearchSecondaryLabel} instance?', + default=args.opensearchSecondarySslVerify, + ) + + if (opensearchPrimaryMode in (DatabaseMode.OpenSearchRemote, DatabaseMode.ElasticsearchRemote)) or ( + opensearchSecondaryMode in (DatabaseMode.OpenSearchRemote, DatabaseMode.ElasticsearchRemote) + ): + InstallerDisplayMessage( + f'You must run auth_setup after {ScriptName} to store data store connection credentials.', + ) + + ################################################################################### + elif currentStep == ConfigOptions.ContainerResources: + if malcolmProfile == PROFILE_MALCOLM: loopBreaker = CountUntilException( - MaxAskForValueCount, 'Invalid Traefik OpenSearch request domain' + MaxAskForValueCount, + f'Invalid {"OpenSearch/" if opensearchPrimaryMode == DatabaseMode.OpenSearchLocal else ""}Logstash memory setting(s)', ) while ( - (len(traefikOpenSearchHost) <= 1) or (traefikOpenSearchHost == traefikHost) + not InstallerYesOrNo( + ( + f'Setting {osMemory} for OpenSearch and {lsMemory} for Logstash. Is this OK?' + if opensearchPrimaryMode == DatabaseMode.OpenSearchLocal + else f'Setting {lsMemory} for Logstash. Is this OK?' + ), + default=True, + ) + and loopBreaker.increment() + ): + if opensearchPrimaryMode == DatabaseMode.OpenSearchLocal: + osMemory = InstallerAskForString('Enter memory for OpenSearch (e.g., 16g, 9500m, etc.)') + lsMemory = InstallerAskForString('Enter memory for Logstash (e.g., 4g, 2500m, etc.)') + + loopBreaker = CountUntilException(MaxAskForValueCount, 'Invalid Logstash worker setting(s)') + while ( + (not str(lsWorkers).isdigit()) + or ( + not InstallerYesOrNo( + f'Setting {lsWorkers} workers for Logstash pipelines. Is this OK?', + default=True, + ) + ) ) and loopBreaker.increment(): - traefikOpenSearchHost = InstallerAskForString( - f'Enter request domain (host header value) for OpenSearch Traefik router (e.g., opensearch.{traefikHost})', - default=args.traefikOpenSearchHost, + lsWorkers = InstallerAskForString('Enter number of Logstash workers (e.g., 4, 8, etc.)') + + ################################################################################### + elif currentStep == ConfigOptions.RestartMode: + restartMode = None + allowedRestartModes = ('no', 'on-failure', 'always', 'unless-stopped') + if (self.orchMode is OrchestrationFramework.DOCKER_COMPOSE) and InstallerYesOrNo( + 'Restart Malcolm upon system or Docker daemon restart?', + default=args.malcolmAutoRestart, + ): + loopBreaker = CountUntilException(MaxAskForValueCount, 'Invalid restart mode') + while restartMode not in allowedRestartModes and loopBreaker.increment(): + restartMode = InstallerChooseOne( + 'Select Malcolm restart behavior', + choices=[(x, '', x == 'unless-stopped') for x in allowedRestartModes], ) - loopBreaker = CountUntilException(MaxAskForValueCount, 'Invalid Traefik router entrypoint') - while (len(traefikEntrypoint) <= 1) and loopBreaker.increment(): - traefikEntrypoint = InstallerAskForString( - 'Enter Traefik router entrypoint (e.g., websecure)', - default=args.traefikEntrypoint, - ) - loopBreaker = CountUntilException(MaxAskForValueCount, 'Invalid Traefik router resolver') - while (len(traefikResolver) <= 1) and loopBreaker.increment(): - traefikResolver = InstallerAskForString( - 'Enter Traefik router resolver (e.g., myresolver)', - default=args.traefikResolver, + else: + restartMode = 'no' + + ################################################################################### + elif currentStep == ConfigOptions.RequireHTTPS: + if malcolmProfile == PROFILE_MALCOLM: + nginxSSL = InstallerYesOrNo( + 'Require encrypted HTTPS connections?', + default=args.nginxSSL, ) + if (not nginxSSL) and (not args.acceptDefaultsNonInteractive): + nginxSSL = not InstallerYesOrNo( + 'Unencrypted connections are NOT recommended. Are you sure?', default=False + ) + else: + nginxSSL = True - dockerNetworkExternalName = InstallerAskForString( - 'Specify external Docker network name (or leave blank for default networking)', - default=args.dockerNetworkName, - ) - - allowedAuthModes = { - 'Basic': 'true', - 'Lightweight Directory Access Protocol (LDAP)': 'false', - 'None': 'no_authentication', - } - authMode = None if (malcolmProfile == PROFILE_MALCOLM) else 'Basic' - loopBreaker = CountUntilException(MaxAskForValueCount, 'Invalid authentication method') - while authMode not in list(allowedAuthModes.keys()) and loopBreaker.increment(): - authMode = InstallerChooseOne( - 'Select authentication method', - choices=[ - (x, '', x == ('Lightweight Directory Access Protocol (LDAP)' if args.authModeLDAP else 'Basic')) - for x in list(allowedAuthModes.keys()) - ], - ) + ################################################################################### + elif currentStep == ConfigOptions.DockerNetworking: + behindReverseProxy = (self.orchMode is OrchestrationFramework.KUBERNETES) or ( + (malcolmProfile == PROFILE_MALCOLM) + and InstallerYesOrNo( + 'Will Malcolm be running behind another reverse proxy (Traefik, Caddy, etc.)?', + default=args.behindReverseProxy or (not nginxSSL), + ) + ) - ldapStartTLS = False - ldapServerTypeDefault = args.ldapServerType if args.ldapServerType else 'winldap' - ldapServerType = ldapServerTypeDefault - if 'ldap' in authMode.lower(): - allowedLdapModes = ('winldap', 'openldap') - ldapServerType = args.ldapServerType if args.ldapServerType else None - loopBreaker = CountUntilException(MaxAskForValueCount, 'Invalid LDAP server compatibility type') - while ldapServerType not in allowedLdapModes and loopBreaker.increment(): - ldapServerType = InstallerChooseOne( - 'Select LDAP server compatibility type', - choices=[(x, '', x == ldapServerTypeDefault) for x in allowedLdapModes], - ) - ldapStartTLS = InstallerYesOrNo( - 'Use StartTLS (rather than LDAPS) for LDAP connection security?', default=args.ldapStartTLS - ) - try: - with open( - os.path.join(os.path.realpath(os.path.join(ScriptPath, "..")), ".ldap_config_defaults"), "w" - ) as ldapDefaultsFile: - print(f"LDAP_SERVER_TYPE='{ldapServerType}'", file=ldapDefaultsFile) - print( - f"LDAP_PROTO='{'ldap://' if ldapStartTLS else 'ldaps://'}'", - file=ldapDefaultsFile, + traefikLabels = False + traefikHost = "" + traefikOpenSearchHost = "" + traefikEntrypoint = "" + traefikResolver = "" + if self.orchMode is OrchestrationFramework.DOCKER_COMPOSE: + if behindReverseProxy: + traefikLabels = InstallerYesOrNo( + 'Configure labels for Traefik?', + default=bool(args.traefikHost), + ) + if traefikLabels: + loopBreaker = CountUntilException(MaxAskForValueCount, 'Invalid Traefik request domain') + while (len(traefikHost) <= 1) and loopBreaker.increment(): + traefikHost = InstallerAskForString( + 'Enter request domain (host header value) for Malcolm interface Traefik router (e.g., malcolm.example.org)', + default=args.traefikHost, + ) + if opensearchPrimaryMode == DatabaseMode.OpenSearchLocal: + loopBreaker = CountUntilException( + MaxAskForValueCount, 'Invalid Traefik OpenSearch request domain' + ) + while ( + (len(traefikOpenSearchHost) <= 1) or (traefikOpenSearchHost == traefikHost) + ) and loopBreaker.increment(): + traefikOpenSearchHost = InstallerAskForString( + f'Enter request domain (host header value) for OpenSearch Traefik router (e.g., opensearch.{traefikHost})', + default=args.traefikOpenSearchHost, + ) + loopBreaker = CountUntilException( + MaxAskForValueCount, 'Invalid Traefik router entrypoint' + ) + while (len(traefikEntrypoint) <= 1) and loopBreaker.increment(): + traefikEntrypoint = InstallerAskForString( + 'Enter Traefik router entrypoint (e.g., websecure)', + default=args.traefikEntrypoint, + ) + loopBreaker = CountUntilException( + MaxAskForValueCount, 'Invalid Traefik router resolver' + ) + while (len(traefikResolver) <= 1) and loopBreaker.increment(): + traefikResolver = InstallerAskForString( + 'Enter Traefik router resolver (e.g., myresolver)', + default=args.traefikResolver, + ) + + dockerNetworkExternalName = InstallerAskForString( + 'Specify external Docker network name (or leave blank for default networking)', + default=args.dockerNetworkName, ) - print(f"LDAP_PORT='{3268 if ldapStartTLS else 3269}'", file=ldapDefaultsFile) - except Exception: - pass - # directories for data volume mounts (PCAP storage, Zeek log storage, OpenSearch indexes, etc.) + ################################################################################### + elif currentStep == ConfigOptions.AuthMethod: + allowedAuthModes = { + 'Basic': 'true', + 'Lightweight Directory Access Protocol (LDAP)': 'false', + 'None': 'no_authentication', + } + authMode = None if (malcolmProfile == PROFILE_MALCOLM) else 'Basic' + loopBreaker = CountUntilException(MaxAskForValueCount, 'Invalid authentication method') + while authMode not in list(allowedAuthModes.keys()) and loopBreaker.increment(): + authMode = InstallerChooseOne( + 'Select authentication method', + choices=[ + ( + x, + '', + x + == ( + 'Lightweight Directory Access Protocol (LDAP)' if args.authModeLDAP else 'Basic' + ), + ) + for x in list(allowedAuthModes.keys()) + ], + ) - # if the file .os-disk-config-defaults was created by the environment (os-disk-config.py) - # we'll use those as defaults, otherwise base things underneath the malcolm_install_path - diskFormatInfo = {} - try: - diskFormatInfoFile = os.path.join( - os.path.realpath(os.path.join(ScriptPath, "..")), ".os-disk-config-defaults" - ) - if os.path.isfile(diskFormatInfoFile): - with open(diskFormatInfoFile) as f: - diskFormatInfo = LoadFileIfJson(f) - except Exception: - pass - diskFormatInfo = {k: v for k, v in diskFormatInfo.items() if os.path.isdir(v)} - - if MALCOLM_DB_DIR in diskFormatInfo: - for subDir in ['opensearch', 'opensearch-backup']: - pathlib.Path(os.path.join(diskFormatInfo[MALCOLM_DB_DIR], subDir)).mkdir(parents=False, exist_ok=True) - if MALCOLM_LOGS_DIR in diskFormatInfo: - for subDir in ['zeek-logs', 'suricata-logs']: - pathlib.Path(os.path.join(diskFormatInfo[MALCOLM_LOGS_DIR], subDir)).mkdir(parents=False, exist_ok=True) - - if args.indexDir: - indexDirDefault = args.indexDir - indexDir = indexDirDefault - else: - indexDir = './opensearch' - if (MALCOLM_DB_DIR in diskFormatInfo) and os.path.isdir( - os.path.join(diskFormatInfo[MALCOLM_DB_DIR], indexDir) - ): - indexDirDefault = os.path.join(diskFormatInfo[MALCOLM_DB_DIR], indexDir) - indexDir = indexDirDefault - else: - indexDirDefault = os.path.join(malcolm_install_path, indexDir) - indexDirFull = os.path.realpath(indexDirDefault) + ldapStartTLS = False + ldapServerTypeDefault = args.ldapServerType if args.ldapServerType else 'winldap' + ldapServerType = ldapServerTypeDefault + if 'ldap' in authMode.lower(): + allowedLdapModes = ('winldap', 'openldap') + ldapServerType = args.ldapServerType if args.ldapServerType else None + loopBreaker = CountUntilException(MaxAskForValueCount, 'Invalid LDAP server compatibility type') + while ldapServerType not in allowedLdapModes and loopBreaker.increment(): + ldapServerType = InstallerChooseOne( + 'Select LDAP server compatibility type', + choices=[(x, '', x == ldapServerTypeDefault) for x in allowedLdapModes], + ) + ldapStartTLS = InstallerYesOrNo( + 'Use StartTLS (rather than LDAPS) for LDAP connection security?', + default=args.ldapStartTLS, + ) + try: + with open( + os.path.join(os.path.realpath(os.path.join(ScriptPath, "..")), ".ldap_config_defaults"), + "w", + ) as ldapDefaultsFile: + print(f"LDAP_SERVER_TYPE='{ldapServerType}'", file=ldapDefaultsFile) + print( + f"LDAP_PROTO='{'ldap://' if ldapStartTLS else 'ldaps://'}'", + file=ldapDefaultsFile, + ) + print(f"LDAP_PORT='{3268 if ldapStartTLS else 3269}'", file=ldapDefaultsFile) + except Exception: + pass - indexSnapshotCompressed = False - if args.indexSnapshotDir: - indexSnapshotDirDefault = args.indexSnapshotDir - indexSnapshotDir = indexSnapshotDirDefault - else: - indexSnapshotDir = './opensearch-backup' - if (MALCOLM_DB_DIR in diskFormatInfo) and os.path.isdir( - os.path.join(diskFormatInfo[MALCOLM_DB_DIR], indexSnapshotDir) - ): - indexSnapshotDirDefault = os.path.join(diskFormatInfo[MALCOLM_DB_DIR], indexSnapshotDir) - indexSnapshotDir = indexSnapshotDirDefault - else: - indexSnapshotDirDefault = os.path.join(malcolm_install_path, indexSnapshotDir) - indexSnapshotDirFull = os.path.realpath(indexSnapshotDirDefault) + ################################################################################### + elif currentStep == ConfigOptions.StorageLocations: + # directories for data volume mounts (PCAP storage, Zeek log storage, OpenSearch indexes, etc.) - if args.pcapDir: - pcapDirDefault = args.pcapDir - pcapDir = pcapDirDefault - else: - if MALCOLM_PCAP_DIR in diskFormatInfo: - pcapDirDefault = diskFormatInfo[MALCOLM_PCAP_DIR] - pcapDir = pcapDirDefault - else: - pcapDir = './pcap' - pcapDirDefault = os.path.join(malcolm_install_path, pcapDir) - pcapDirFull = os.path.realpath(pcapDirDefault) + # if the file .os-disk-config-defaults was created by the environment (os-disk-config.py) + # we'll use those as defaults, otherwise base things underneath the malcolm_install_path + diskFormatInfo = {} + try: + diskFormatInfoFile = os.path.join( + os.path.realpath(os.path.join(ScriptPath, "..")), ".os-disk-config-defaults" + ) + if os.path.isfile(diskFormatInfoFile): + with open(diskFormatInfoFile) as f: + diskFormatInfo = LoadFileIfJson(f) + except Exception: + pass + diskFormatInfo = {k: v for k, v in diskFormatInfo.items() if os.path.isdir(v)} - if args.suricataLogDir: - suricataLogDirDefault = args.suricataLogDir - suricataLogDir = suricataLogDirDefault - else: - suricataLogDir = './suricata-logs' - if (MALCOLM_LOGS_DIR in diskFormatInfo) and os.path.isdir( - os.path.join(diskFormatInfo[MALCOLM_LOGS_DIR], suricataLogDir) - ): - suricataLogDirDefault = os.path.join(diskFormatInfo[MALCOLM_LOGS_DIR], suricataLogDir) - suricataLogDir = suricataLogDirDefault - else: - suricataLogDirDefault = os.path.join(malcolm_install_path, suricataLogDir) - suricataLogDirFull = os.path.realpath(suricataLogDirDefault) + if MALCOLM_DB_DIR in diskFormatInfo: + for subDir in ['opensearch', 'opensearch-backup']: + pathlib.Path(os.path.join(diskFormatInfo[MALCOLM_DB_DIR], subDir)).mkdir( + parents=False, exist_ok=True + ) + if MALCOLM_LOGS_DIR in diskFormatInfo: + for subDir in ['zeek-logs', 'suricata-logs']: + pathlib.Path(os.path.join(diskFormatInfo[MALCOLM_LOGS_DIR], subDir)).mkdir( + parents=False, exist_ok=True + ) - if args.zeekLogDir: - zeekLogDirDefault = args.zeekLogDir - zeekLogDir = zeekLogDirDefault - else: - zeekLogDir = './zeek-logs' - if (MALCOLM_LOGS_DIR in diskFormatInfo) and os.path.isdir( - os.path.join(diskFormatInfo[MALCOLM_LOGS_DIR], zeekLogDir) - ): - zeekLogDirDefault = os.path.join(diskFormatInfo[MALCOLM_LOGS_DIR], zeekLogDir) - zeekLogDir = zeekLogDirDefault - else: - zeekLogDirDefault = os.path.join(malcolm_install_path, zeekLogDir) - zeekLogDirFull = os.path.realpath(zeekLogDirDefault) + if args.indexDir: + indexDirDefault = args.indexDir + indexDir = indexDirDefault + else: + indexDir = './opensearch' + if (MALCOLM_DB_DIR in diskFormatInfo) and os.path.isdir( + os.path.join(diskFormatInfo[MALCOLM_DB_DIR], indexDir) + ): + indexDirDefault = os.path.join(diskFormatInfo[MALCOLM_DB_DIR], indexDir) + indexDir = indexDirDefault + else: + indexDirDefault = os.path.join(malcolm_install_path, indexDir) + indexDirFull = os.path.realpath(indexDirDefault) - if self.orchMode is OrchestrationFramework.DOCKER_COMPOSE: - if diskFormatInfo or not InstallerYesOrNo( - f'Store {"PCAP, log and index" if (malcolmProfile == PROFILE_MALCOLM) else "PCAP and log"} files in {malcolm_install_path}?', - default=not args.acceptDefaultsNonInteractive, - ): - # PCAP directory - if not InstallerYesOrNo( - 'Store PCAP files in {}?'.format(pcapDirDefault), - default=not bool(args.pcapDir), - ): - loopBreaker = CountUntilException(MaxAskForValueCount, 'Invalid PCAP directory') - while loopBreaker.increment(): - pcapDir = InstallerAskForString('Enter PCAP directory', default=pcapDirDefault) - if (len(pcapDir) > 1) and os.path.isdir(pcapDir): - pcapDirFull = os.path.realpath(pcapDir) - pcapDir = ( - f"./{os.path.relpath(pcapDirDefault, malcolm_install_path)}" - if same_file_or_dir(pcapDirDefault, pcapDirFull) - else pcapDirFull - ) - break + indexSnapshotCompressed = False + if args.indexSnapshotDir: + indexSnapshotDirDefault = args.indexSnapshotDir + indexSnapshotDir = indexSnapshotDirDefault + else: + indexSnapshotDir = './opensearch-backup' + if (MALCOLM_DB_DIR in diskFormatInfo) and os.path.isdir( + os.path.join(diskFormatInfo[MALCOLM_DB_DIR], indexSnapshotDir) + ): + indexSnapshotDirDefault = os.path.join(diskFormatInfo[MALCOLM_DB_DIR], indexSnapshotDir) + indexSnapshotDir = indexSnapshotDirDefault + else: + indexSnapshotDirDefault = os.path.join(malcolm_install_path, indexSnapshotDir) + indexSnapshotDirFull = os.path.realpath(indexSnapshotDirDefault) - # Zeek log directory - if not InstallerYesOrNo( - 'Store Zeek logs in {}?'.format(zeekLogDirDefault), - default=not bool(args.zeekLogDir), - ): - loopBreaker = CountUntilException(MaxAskForValueCount, 'Invalid Zeek directory') - while loopBreaker.increment(): - zeekLogDir = InstallerAskForString('Enter Zeek log directory', default=zeekLogDirDefault) - if (len(zeekLogDir) > 1) and os.path.isdir(zeekLogDir): - zeekLogDirFull = os.path.realpath(zeekLogDir) - zeekLogDir = ( - f"./{os.path.relpath(zeekLogDirDefault, malcolm_install_path)}" - if same_file_or_dir(zeekLogDirDefault, zeekLogDirFull) - else zeekLogDirFull - ) - break + if args.pcapDir: + pcapDirDefault = args.pcapDir + pcapDir = pcapDirDefault + else: + if MALCOLM_PCAP_DIR in diskFormatInfo: + pcapDirDefault = diskFormatInfo[MALCOLM_PCAP_DIR] + pcapDir = pcapDirDefault + else: + pcapDir = './pcap' + pcapDirDefault = os.path.join(malcolm_install_path, pcapDir) + pcapDirFull = os.path.realpath(pcapDirDefault) - # Suricata log directory - if not InstallerYesOrNo( - 'Store Suricata logs in {}?'.format(suricataLogDirDefault), - default=not bool(args.suricataLogDir), - ): - loopBreaker = CountUntilException(MaxAskForValueCount, 'Invalid Suricata directory') - while loopBreaker.increment(): - suricataLogDir = InstallerAskForString( - 'Enter Suricata log directory', default=suricataLogDirDefault - ) - if (len(suricataLogDir) > 1) and os.path.isdir(suricataLogDir): - suricataLogDirFull = os.path.realpath(suricataLogDir) - suricataLogDir = ( - f"./{os.path.relpath(suricataLogDirDefault, malcolm_install_path)}" - if same_file_or_dir(suricataLogDirDefault, suricataLogDirFull) - else suricataLogDirFull - ) - break + if args.suricataLogDir: + suricataLogDirDefault = args.suricataLogDir + suricataLogDir = suricataLogDirDefault + else: + suricataLogDir = './suricata-logs' + if (MALCOLM_LOGS_DIR in diskFormatInfo) and os.path.isdir( + os.path.join(diskFormatInfo[MALCOLM_LOGS_DIR], suricataLogDir) + ): + suricataLogDirDefault = os.path.join(diskFormatInfo[MALCOLM_LOGS_DIR], suricataLogDir) + suricataLogDir = suricataLogDirDefault + else: + suricataLogDirDefault = os.path.join(malcolm_install_path, suricataLogDir) + suricataLogDirFull = os.path.realpath(suricataLogDirDefault) - if (malcolmProfile == PROFILE_MALCOLM) and (opensearchPrimaryMode == DatabaseMode.OpenSearchLocal): - # opensearch index directory - if not InstallerYesOrNo( - 'Store OpenSearch indices in {}?'.format(indexDirDefault), - default=not bool(args.indexDir), - ): - loopBreaker = CountUntilException(MaxAskForValueCount, 'Invalid OpenSearch index directory') + if args.zeekLogDir: + zeekLogDirDefault = args.zeekLogDir + zeekLogDir = zeekLogDirDefault + else: + zeekLogDir = './zeek-logs' + if (MALCOLM_LOGS_DIR in diskFormatInfo) and os.path.isdir( + os.path.join(diskFormatInfo[MALCOLM_LOGS_DIR], zeekLogDir) + ): + zeekLogDirDefault = os.path.join(diskFormatInfo[MALCOLM_LOGS_DIR], zeekLogDir) + zeekLogDir = zeekLogDirDefault + else: + zeekLogDirDefault = os.path.join(malcolm_install_path, zeekLogDir) + zeekLogDirFull = os.path.realpath(zeekLogDirDefault) + + if self.orchMode is OrchestrationFramework.DOCKER_COMPOSE: + if diskFormatInfo or not InstallerYesOrNo( + f'Store {"PCAP, log and index" if (malcolmProfile == PROFILE_MALCOLM) else "PCAP and log"} files in {malcolm_install_path}?', + default=not args.acceptDefaultsNonInteractive, + ): + # PCAP directory + if not InstallerYesOrNo( + 'Store PCAP files in {}?'.format(pcapDirDefault), + default=not bool(args.pcapDir), + ): + loopBreaker = CountUntilException(MaxAskForValueCount, 'Invalid PCAP directory') + while loopBreaker.increment(): + pcapDir = InstallerAskForString( + 'Enter PCAP directory', + default=pcapDirDefault, + ) + if (len(pcapDir) > 1) and os.path.isdir(pcapDir): + pcapDirFull = os.path.realpath(pcapDir) + pcapDir = ( + f"./{os.path.relpath(pcapDirDefault, malcolm_install_path)}" + if same_file_or_dir(pcapDirDefault, pcapDirFull) + else pcapDirFull + ) + break + + # Zeek log directory + if not InstallerYesOrNo( + 'Store Zeek logs in {}?'.format(zeekLogDirDefault), + default=not bool(args.zeekLogDir), + ): + loopBreaker = CountUntilException(MaxAskForValueCount, 'Invalid Zeek directory') + while loopBreaker.increment(): + zeekLogDir = InstallerAskForString( + 'Enter Zeek log directory', + default=zeekLogDirDefault, + ) + if (len(zeekLogDir) > 1) and os.path.isdir(zeekLogDir): + zeekLogDirFull = os.path.realpath(zeekLogDir) + zeekLogDir = ( + f"./{os.path.relpath(zeekLogDirDefault, malcolm_install_path)}" + if same_file_or_dir(zeekLogDirDefault, zeekLogDirFull) + else zeekLogDirFull + ) + break + + # Suricata log directory + if not InstallerYesOrNo( + 'Store Suricata logs in {}?'.format(suricataLogDirDefault), + default=not bool(args.suricataLogDir), + ): + loopBreaker = CountUntilException(MaxAskForValueCount, 'Invalid Suricata directory') + while loopBreaker.increment(): + suricataLogDir = InstallerAskForString( + 'Enter Suricata log directory', + default=suricataLogDirDefault, + ) + if (len(suricataLogDir) > 1) and os.path.isdir(suricataLogDir): + suricataLogDirFull = os.path.realpath(suricataLogDir) + suricataLogDir = ( + f"./{os.path.relpath(suricataLogDirDefault, malcolm_install_path)}" + if same_file_or_dir(suricataLogDirDefault, suricataLogDirFull) + else suricataLogDirFull + ) + break + + if (malcolmProfile == PROFILE_MALCOLM) and ( + opensearchPrimaryMode == DatabaseMode.OpenSearchLocal + ): + # opensearch index directory + if not InstallerYesOrNo( + 'Store OpenSearch indices in {}?'.format(indexDirDefault), + default=not bool(args.indexDir), + ): + loopBreaker = CountUntilException( + MaxAskForValueCount, 'Invalid OpenSearch index directory' + ) + while loopBreaker.increment(): + indexDir = InstallerAskForString( + 'Enter OpenSearch index directory', + default=indexDirDefault, + ) + if (len(indexDir) > 1) and os.path.isdir(indexDir): + indexDirFull = os.path.realpath(indexDir) + indexDir = ( + f"./{os.path.relpath(indexDirDefault, malcolm_install_path)}" + if same_file_or_dir(indexDirDefault, indexDirFull) + else indexDirFull + ) + break + + # opensearch snapshot repository directory and compression + if not InstallerYesOrNo( + 'Store OpenSearch index snapshots in {}?'.format(indexSnapshotDirDefault), + default=not bool(args.indexSnapshotDir), + ): + loopBreaker = CountUntilException( + MaxAskForValueCount, 'Invalid OpenSearch snapshots directory' + ) + while loopBreaker.increment(): + indexSnapshotDir = InstallerAskForString( + 'Enter OpenSearch index snapshot directory', + default=indexSnapshotDirDefault, + ) + if (len(indexSnapshotDir) > 1) and os.path.isdir(indexSnapshotDir): + indexSnapshotDirFull = os.path.realpath(indexSnapshotDir) + indexSnapshotDir = ( + f"./{os.path.relpath(indexSnapshotDirDefault, malcolm_install_path)}" + if same_file_or_dir(indexSnapshotDirDefault, indexSnapshotDirFull) + else indexSnapshotDirFull + ) + break + + # make sure paths specified (and their necessary children) exist + for pathToCreate in ( + malcolm_install_path, + indexDirFull, + indexSnapshotDirFull, + os.path.join(pcapDirFull, 'arkime-live'), + os.path.join(pcapDirFull, 'processed'), + os.path.join(pcapDirFull, os.path.join('upload', os.path.join('tmp', 'spool'))), + os.path.join(pcapDirFull, os.path.join('upload', 'variants')), + os.path.join(suricataLogDirFull, 'live'), + os.path.join(zeekLogDirFull, 'current'), + os.path.join(zeekLogDirFull, 'live'), + os.path.join(zeekLogDirFull, 'upload'), + os.path.join(zeekLogDirFull, os.path.join('extract_files', 'preserved')), + os.path.join(zeekLogDirFull, os.path.join('extract_files', 'quarantine')), + ): + try: + if args.debug: + eprint(f"Creating {pathToCreate}") + pathlib.Path(pathToCreate).mkdir(parents=True, exist_ok=True) + if ( + ((self.platform == PLATFORM_LINUX) or (self.platform == PLATFORM_MAC)) + and (self.scriptUser == "root") + and (getpwuid(os.stat(pathToCreate).st_uid).pw_name == self.scriptUser) + ): + if args.debug: + eprint(f"Setting permissions of {pathToCreate} to {puid}:{pgid}") + # change ownership of newly-created directory to match puid/pgid + os.chown(pathToCreate, int(puid), int(pgid)) + except Exception as e: + eprint(f"Creating {pathToCreate} failed: {e}") + + ################################################################################### + elif currentStep == ConfigOptions.ILMISM: + indexManagementPolicy = False + indexManagementHotWarm = False + indexManagementOptimizationTimePeriod = '30d' + indexManagementSpiDataRetention = '90d' + indexManagementReplicas = 0 + indexManagementHistoryInWeeks = 13 + indexManagementOptimizeSessionSegments = 1 + + loopBreaker = CountUntilException( + MaxAskForValueCount, + f'Invalid ILM/ISM setting(s)', + ) + indexManagementPolicy = InstallerYesOrNo( + f'Enable index management policies (ILM/ISM) in Arkime?', + default=args.indexManagementPolicy, + ) + if indexManagementPolicy: while loopBreaker.increment(): - indexDir = InstallerAskForString( - 'Enter OpenSearch index directory', default=indexDirDefault + # Set 'hot' for 'node.attr.molochtype' on new indices, warm on non sessions indices + indexManagementHotWarm = InstallerYesOrNo( + f'Should Arkime use a hot/warm design in which non-session data is stored in a warm index?', + default=args.indexManagementHotWarm, ) - if (len(indexDir) > 1) and os.path.isdir(indexDir): - indexDirFull = os.path.realpath(indexDir) - indexDir = ( - f"./{os.path.relpath(indexDirDefault, malcolm_install_path)}" - if same_file_or_dir(indexDirDefault, indexDirFull) - else indexDirFull - ) + if indexManagementHotWarm: + if opensearchPrimaryMode == DatabaseMode.ElasticsearchRemote: + InstallerDisplayMessage( + f'You must configure "hot" and "warm" nodes types in the remote Elasticsearch instance (https://arkime.com/faq#ilm)' + ) + else: + InstallerDisplayMessage( + f'You must configure "hot" and "warm" nodes types in the OpenSearch instance' + ) + # Time in hours/days before (moving Arkime indexes to warm) and force merge (number followed by h or d), default 30d + indexManagementOptimizationTimePeriod = InstallerAskForString( + "How long should Arkime keep an index in the hot node? (e.g. 25h, 5d, etc.)", + default=args.indexManagementOptimizationTimePeriod, + ) + # Time in hours/days before deleting Arkime indexes (number followed by h or d), default 90d + indexManagementSpiDataRetention = InstallerAskForString( + "How long should Arkime retain SPI data before deleting it? (e.g. 25h, 90d, etc.)", + default=str(args.indexManagementSpiDataRetention), + ) + # Number of segments to optimize sessions to in the ILM policy, default 1 + indexManagementOptimizeSessionSegments = InstallerAskForString( + "How many segments should Arkime use to optimize?", + default=str(args.indexManagementOptimizeSessionSegments), + ) + # Number of replicas for older sessions indices in the ILM policy, default 0 + indexManagementReplicas = InstallerAskForString( + "How many replicas should Arkime maintain for older session indices?", + default=str(args.indexManagementReplicas), + ) + # Number of weeks of history to keep, default 13 + indexManagementHistoryInWeeks = InstallerAskForString( + "How many weeks of history should Arkime keep?", + default=str(args.indexManagementHistoryInWeeks), + ) + if ( + (re.match(r"\d+(h|d)", indexManagementOptimizationTimePeriod)) + and (re.match(r"\d+(h|d)", indexManagementSpiDataRetention)) + and str(indexManagementOptimizeSessionSegments).isdigit() + and str(indexManagementReplicas).isdigit() + and str(indexManagementHistoryInWeeks).isdigit() + ): break - # opensearch snapshot repository directory and compression - if not InstallerYesOrNo( - 'Store OpenSearch index snapshots in {}?'.format(indexSnapshotDirDefault), - default=not bool(args.indexSnapshotDir), - ): - loopBreaker = CountUntilException(MaxAskForValueCount, 'Invalid OpenSearch snapshots directory') - while loopBreaker.increment(): - indexSnapshotDir = InstallerAskForString( - 'Enter OpenSearch index snapshot directory', default=indexSnapshotDirDefault + ################################################################################### + elif currentStep == ConfigOptions.StorageManagement: + # storage management (deleting oldest indices and/or PCAP files) + indexPruneSizeLimit = '0' + indexPruneNameSort = False + arkimeManagePCAP = False + arkimeFreeSpaceG = '10%' + extractedFileMaxSizeThreshold = '1TB' + extractedFileMaxPercentThreshold = 0 + + diskUsageManagementPrompt = InstallerYesOrNo( + ( + 'Should Malcolm delete the oldest database indices and capture artifacts based on available storage?' + if ( + (opensearchPrimaryMode == DatabaseMode.OpenSearchLocal) + and (malcolmProfile == PROFILE_MALCOLM) + ) + else 'Should Malcolm delete the oldest capture artifacts based on available storage?' + ), + default=args.arkimeManagePCAP + or bool(args.indexPruneSizeLimit) + or bool(args.extractedFileMaxSizeThreshold) + or (args.extractedFileMaxPercentThreshold > 0), + ) + if diskUsageManagementPrompt: + + # delete oldest indexes based on index pattern size + if ( + (malcolmProfile == PROFILE_MALCOLM) + and (opensearchPrimaryMode == DatabaseMode.OpenSearchLocal) + and InstallerYesOrNo( + 'Delete the oldest indices when the database exceeds a certain size?', + default=bool(args.indexPruneSizeLimit), ) - if (len(indexSnapshotDir) > 1) and os.path.isdir(indexSnapshotDir): - indexSnapshotDirFull = os.path.realpath(indexSnapshotDir) - indexSnapshotDir = ( - f"./{os.path.relpath(indexSnapshotDirDefault, malcolm_install_path)}" - if same_file_or_dir(indexSnapshotDirDefault, indexSnapshotDirFull) - else indexSnapshotDirFull + ): + indexPruneSizeLimit = '' + loopBreaker = CountUntilException(MaxAskForValueCount, 'Invalid index threshold') + while ( + (not re.match(r'^\d+(\.\d+)?\s*[kmgtp%]?b?$', indexPruneSizeLimit, flags=re.IGNORECASE)) + and (indexPruneSizeLimit != '0') + and loopBreaker.increment() + ): + indexPruneSizeLimit = InstallerAskForString( + 'Enter index threshold (e.g., 250GB, 1TB, 60%, etc.)', + default=args.indexPruneSizeLimit, ) - break + indexPruneNameSort = InstallerYesOrNo( + 'Determine oldest indices by name (instead of creation time)?', default=False + ) - # make sure paths specified (and their necessary children) exist - for pathToCreate in ( - malcolm_install_path, - indexDirFull, - indexSnapshotDirFull, - os.path.join(pcapDirFull, 'arkime-live'), - os.path.join(pcapDirFull, 'processed'), - os.path.join(pcapDirFull, os.path.join('upload', os.path.join('tmp', 'spool'))), - os.path.join(pcapDirFull, os.path.join('upload', 'variants')), - os.path.join(suricataLogDirFull, 'live'), - os.path.join(zeekLogDirFull, 'current'), - os.path.join(zeekLogDirFull, 'live'), - os.path.join(zeekLogDirFull, 'upload'), - os.path.join(zeekLogDirFull, os.path.join('extract_files', 'preserved')), - os.path.join(zeekLogDirFull, os.path.join('extract_files', 'quarantine')), - ): - try: - if args.debug: - eprint(f"Creating {pathToCreate}") - pathlib.Path(pathToCreate).mkdir(parents=True, exist_ok=True) - if ( - ((self.platform == PLATFORM_LINUX) or (self.platform == PLATFORM_MAC)) - and (self.scriptUser == "root") - and (getpwuid(os.stat(pathToCreate).st_uid).pw_name == self.scriptUser) - ): - if args.debug: - eprint(f"Setting permissions of {pathToCreate} to {puid}:{pgid}") - # change ownership of newly-created directory to match puid/pgid - os.chown(pathToCreate, int(puid), int(pgid)) - except Exception as e: - eprint(f"Creating {pathToCreate} failed: {e}") - - # storage management (deleting oldest indices and/or PCAP files) - indexPruneSizeLimit = '0' - indexPruneNameSort = False - arkimeManagePCAP = False - arkimeFreeSpaceG = '10%' - extractedFileMaxSizeThreshold = '1TB' - extractedFileMaxPercentThreshold = 0 - indexManagementPolicy = False - indexManagementHotWarm = False - indexManagementOptimizationTimePeriod = '30d' - indexManagementSpiDataRetention = '90d' - indexManagementReplicas = 0 - indexManagementHistoryInWeeks = 13 - indexManagementOptimizeSessionSegments = 1 - - loopBreaker = CountUntilException( - MaxAskForValueCount, - f'Invalid ILM/ISM setting(s)', - ) - indexManagementPolicy = InstallerYesOrNo( - f'Enable index management policies (ILM/ISM) in Arkime?', default=args.indexManagementPolicy - ) - if indexManagementPolicy: - while loopBreaker.increment(): - # Set 'hot' for 'node.attr.molochtype' on new indices, warm on non sessions indices - indexManagementHotWarm = InstallerYesOrNo( - f'Should Arkime use a hot/warm design in which non-session data is stored in a warm index?', - default=args.indexManagementHotWarm, - ) - if indexManagementHotWarm: - if opensearchPrimaryMode == DatabaseMode.ElasticsearchRemote: - InstallerDisplayMessage( - f'You must configure "hot" and "warm" nodes types in the remote Elasticsearch instance (https://arkime.com/faq#ilm)' - ) - else: - InstallerDisplayMessage( - f'You must configure "hot" and "warm" nodes types in the OpenSearch instance' + # let Arkime delete old PCAP files based on available storage + arkimeManagePCAP = ( + (opensearchPrimaryMode != DatabaseMode.OpenSearchLocal) + or (malcolmProfile != PROFILE_MALCOLM) + or InstallerYesOrNo( + 'Should Arkime delete uploaded PCAP files based on available storage (see https://arkime.com/faq#pcap-deletion)?', + default=args.arkimeManagePCAP, + ) ) - # Time in hours/days before (moving Arkime indexes to warm) and force merge (number followed by h or d), default 30d - indexManagementOptimizationTimePeriod = InstallerAskForString( - "How long should Arkime keep an index in the hot node? (e.g. 25h, 5d, etc.)", - default=args.indexManagementOptimizationTimePeriod, - ) - # Time in hours/days before deleting Arkime indexes (number followed by h or d), default 90d - indexManagementSpiDataRetention = InstallerAskForString( - "How long should Arkime retain SPI data before deleting it? (e.g. 25h, 90d, etc.)", - default=str(args.indexManagementSpiDataRetention), - ) - # Number of segments to optimize sessions to in the ILM policy, default 1 - indexManagementOptimizeSessionSegments = InstallerAskForString( - "How many segments should Arkime use to optimize?", - default=str(args.indexManagementOptimizeSessionSegments), - ) - # Number of replicas for older sessions indices in the ILM policy, default 0 - indexManagementReplicas = InstallerAskForString( - "How many replicas should Arkime maintain for older session indices?", - default=str(args.indexManagementReplicas), - ) - # Number of weeks of history to keep, default 13 - indexManagementHistoryInWeeks = InstallerAskForString( - "How many weeks of history should Arkime keep?", default=str(args.indexManagementHistoryInWeeks) - ) - if ( - (re.match(r"\d+(h|d)", indexManagementOptimizationTimePeriod)) - and (re.match(r"\d+(h|d)", indexManagementSpiDataRetention)) - and str(indexManagementOptimizeSessionSegments).isdigit() - and str(indexManagementReplicas).isdigit() - and str(indexManagementHistoryInWeeks).isdigit() - ): - break + if arkimeManagePCAP: + arkimeFreeSpaceGTmp = '' + loopBreaker = CountUntilException(MaxAskForValueCount, 'Invalid PCAP deletion threshold') + while ( + not re.match(r'^\d+%?$', arkimeFreeSpaceGTmp, flags=re.IGNORECASE) + ) and loopBreaker.increment(): + arkimeFreeSpaceGTmp = InstallerAskForString( + 'Enter PCAP deletion threshold in gigabytes or as a percentage (e.g., 500, 10%, etc.)', + default=args.arkimeFreeSpaceG, + ) + if arkimeFreeSpaceGTmp: + arkimeFreeSpaceG = arkimeFreeSpaceGTmp + ################################################################################### + elif currentStep == ConfigOptions.AutoArkime: + autoArkime = InstallerYesOrNo( + 'Automatically analyze all PCAP files with Arkime?', default=args.autoArkime + ) + ################################################################################### + elif currentStep == ConfigOptions.AutoSuricata: + autoSuricata = InstallerYesOrNo( + 'Automatically analyze all PCAP files with Suricata?', default=args.autoSuricata + ) + ################################################################################### + elif currentStep == ConfigOptions.SuricataRuleUpdate: + suricataRuleUpdate = autoSuricata and InstallerYesOrNo( + 'Download updated Suricata signatures periodically?', default=args.suricataRuleUpdate + ) + ################################################################################### + elif currentStep == ConfigOptions.AutoZeek: + autoZeek = InstallerYesOrNo( + 'Automatically analyze all PCAP files with Zeek?', default=args.autoZeek + ) + ################################################################################### + elif currentStep == ConfigOptions.ICS: + malcolmIcs = InstallerYesOrNo( + 'Is Malcolm being used to monitor an Operational Technology/Industrial Control Systems (OT/ICS) network?', + default=args.malcolmIcs, + ) - diskUsageManagementPrompt = InstallerYesOrNo( - ( - 'Should Malcolm delete the oldest database indices and capture artifacts based on available storage?' - if ((opensearchPrimaryMode == DatabaseMode.OpenSearchLocal) and (malcolmProfile == PROFILE_MALCOLM)) - else 'Should Malcolm delete the oldest capture artifacts based on available storage?' - ), - default=args.arkimeManagePCAP - or bool(args.indexPruneSizeLimit) - or bool(args.extractedFileMaxSizeThreshold) - or (args.extractedFileMaxPercentThreshold > 0), - ) - if diskUsageManagementPrompt: + zeekICSBestGuess = ( + autoZeek + and malcolmIcs + and InstallerYesOrNo( + 'Should Malcolm use "best guess" to identify potential OT/ICS traffic with Zeek?', + default=args.zeekICSBestGuess, + ) + ) - # delete oldest indexes based on index pattern size - if ( - (malcolmProfile == PROFILE_MALCOLM) - and (opensearchPrimaryMode == DatabaseMode.OpenSearchLocal) - and InstallerYesOrNo( - 'Delete the oldest indices when the database exceeds a certain size?', - default=bool(args.indexPruneSizeLimit), - ) - ): - indexPruneSizeLimit = '' - loopBreaker = CountUntilException(MaxAskForValueCount, 'Invalid index threshold') - while ( - (not re.match(r'^\d+(\.\d+)?\s*[kmgtp%]?b?$', indexPruneSizeLimit, flags=re.IGNORECASE)) - and (indexPruneSizeLimit != '0') - and loopBreaker.increment() - ): - indexPruneSizeLimit = InstallerAskForString( - 'Enter index threshold (e.g., 250GB, 1TB, 60%, etc.)', default=args.indexPruneSizeLimit + ################################################################################### + elif currentStep == ConfigOptions.Enrichment: + reverseDns = (malcolmProfile == PROFILE_MALCOLM) and InstallerYesOrNo( + 'Perform reverse DNS lookup locally for source and destination IP addresses in logs?', + default=args.reverseDns, + ) + autoOui = (malcolmProfile == PROFILE_MALCOLM) and InstallerYesOrNo( + 'Perform hardware vendor OUI lookups for MAC addresses?', default=args.autoOui + ) + autoFreq = (malcolmProfile == PROFILE_MALCOLM) and InstallerYesOrNo( + 'Perform string randomness scoring on some fields?', default=args.autoFreq ) - indexPruneNameSort = InstallerYesOrNo( - 'Determine oldest indices by name (instead of creation time)?', default=False - ) - # let Arkime delete old PCAP files based on available storage - arkimeManagePCAP = ( - (opensearchPrimaryMode != DatabaseMode.OpenSearchLocal) - or (malcolmProfile != PROFILE_MALCOLM) - or InstallerYesOrNo( - 'Should Arkime delete uploaded PCAP files based on available storage (see https://arkime.com/faq#pcap-deletion)?', - default=args.arkimeManagePCAP, - ) - ) - if arkimeManagePCAP: - arkimeFreeSpaceGTmp = '' - loopBreaker = CountUntilException(MaxAskForValueCount, 'Invalid PCAP deletion threshold') - while (not re.match(r'^\d+%?$', arkimeFreeSpaceGTmp, flags=re.IGNORECASE)) and loopBreaker.increment(): - arkimeFreeSpaceGTmp = InstallerAskForString( - 'Enter PCAP deletion threshold in gigabytes or as a percentage (e.g., 500, 10%, etc.)', - default=args.arkimeFreeSpaceG, + ################################################################################### + elif currentStep == ConfigOptions.OpenPorts: + openPortsSelection = ( + 'c' + if (args.exposeLogstash or args.exposeOpenSearch or args.exposeFilebeatTcp or args.exposeSFTP) + else 'unset' ) - if arkimeFreeSpaceGTmp: - arkimeFreeSpaceG = arkimeFreeSpaceGTmp + if self.orchMode is OrchestrationFramework.DOCKER_COMPOSE: + if malcolmProfile == PROFILE_MALCOLM: + openPortsOptions = ('no', 'yes', 'customize') + loopBreaker = CountUntilException(MaxAskForValueCount) + while ( + openPortsSelection not in [x[0] for x in openPortsOptions] and loopBreaker.increment() + ): + openPortsSelection = InstallerChooseOne( + 'Should Malcolm accept logs and metrics from a Hedgehog Linux sensor or other forwarder?', + choices=[(x, '', x == openPortsOptions[0]) for x in openPortsOptions], + )[0] + if openPortsSelection == 'n': + opensearchOpen = False + logstashOpen = False + filebeatTcpOpen = False + elif openPortsSelection == 'y': + opensearchOpen = opensearchPrimaryMode == DatabaseMode.OpenSearchLocal + logstashOpen = True + filebeatTcpOpen = True + else: + openPortsSelection = 'c' + opensearchOpen = ( + opensearchPrimaryMode == DatabaseMode.OpenSearchLocal + ) and InstallerYesOrNo( + 'Expose OpenSearch port to external hosts?', default=args.exposeOpenSearch + ) + logstashOpen = InstallerYesOrNo( + 'Expose Logstash port to external hosts?', default=args.exposeLogstash + ) + filebeatTcpOpen = InstallerYesOrNo( + 'Expose Filebeat TCP port to external hosts?', default=args.exposeFilebeatTcp + ) + else: + opensearchOpen = False + openPortsSelection = 'n' + logstashOpen = False + filebeatTcpOpen = False - autoArkime = InstallerYesOrNo('Automatically analyze all PCAP files with Arkime?', default=args.autoArkime) - autoSuricata = InstallerYesOrNo( - 'Automatically analyze all PCAP files with Suricata?', default=args.autoSuricata - ) - suricataRuleUpdate = autoSuricata and InstallerYesOrNo( - 'Download updated Suricata signatures periodically?', default=args.suricataRuleUpdate - ) - autoZeek = InstallerYesOrNo('Automatically analyze all PCAP files with Zeek?', default=args.autoZeek) + else: + opensearchOpen = opensearchPrimaryMode == DatabaseMode.OpenSearchLocal + openPortsSelection = 'y' + logstashOpen = True + filebeatTcpOpen = True + + filebeatTcpFormat = 'json' + filebeatTcpSourceField = 'message' + filebeatTcpTargetField = 'miscbeat' + filebeatTcpDropField = filebeatTcpSourceField + filebeatTcpTag = '_malcolm_beats' + if ( + filebeatTcpOpen + and (openPortsSelection == 'c') + and not InstallerYesOrNo('Use default field values for Filebeat TCP listener?', default=True) + ): + allowedFilebeatTcpFormats = ('json', 'raw') + filebeatTcpFormat = 'unset' + loopBreaker = CountUntilException(MaxAskForValueCount, f'Invalid log format') + while filebeatTcpFormat not in allowedFilebeatTcpFormats and loopBreaker.increment(): + filebeatTcpFormat = InstallerChooseOne( + 'Select log format for messages sent to Filebeat TCP listener', + choices=[(x, '', x == allowedFilebeatTcpFormats[0]) for x in allowedFilebeatTcpFormats], + ) + if filebeatTcpFormat == 'json': + filebeatTcpSourceField = InstallerAskForString( + 'Source field to parse for messages sent to Filebeat TCP listener', + default=filebeatTcpSourceField, + ) + filebeatTcpTargetField = InstallerAskForString( + 'Target field under which to store decoded JSON fields for messages sent to Filebeat TCP listener', + default=filebeatTcpTargetField, + ) + filebeatTcpDropField = InstallerAskForString( + 'Field to drop from events sent to Filebeat TCP listener', + default=filebeatTcpSourceField, + ) + filebeatTcpTag = InstallerAskForString( + 'Tag to apply to messages sent to Filebeat TCP listener', + default=filebeatTcpTag, + ) - malcolmIcs = InstallerYesOrNo( - 'Is Malcolm being used to monitor an Operational Technology/Industrial Control Systems (OT/ICS) network?', - default=args.malcolmIcs, - ) + sftpOpen = ( + (self.orchMode is OrchestrationFramework.DOCKER_COMPOSE) + and (malcolmProfile == PROFILE_MALCOLM) + and (openPortsSelection == 'c') + and InstallerYesOrNo( + 'Expose SFTP server (for PCAP upload) to external hosts?', default=args.exposeSFTP + ) + ) - zeekICSBestGuess = ( - autoZeek - and malcolmIcs - and InstallerYesOrNo( - 'Should Malcolm use "best guess" to identify potential OT/ICS traffic with Zeek?', - default=args.zeekICSBestGuess, - ) - ) + ################################################################################### + elif currentStep == ConfigOptions.FileCarving: + # input file extraction parameters + allowedFileCarveModes = { + 'none': 'No file extraction', + 'known': 'Extract recognized MIME types', + 'mapped': 'Extract MIME types for which file extensions are known', + 'all': 'Extract all files', + 'interesting': 'Extract MIME types of common attack vectors', + 'notcommtxt': 'Extract all except common plain text files', + } + allowedFilePreserveModes = ('quarantined', 'all', 'none') + + fileCarveMode = None + fileCarveModeDefault = args.fileCarveMode.lower() if args.fileCarveMode else None + filePreserveMode = None + filePreserveModeDefault = args.filePreserveMode.lower() if args.filePreserveMode else None + vtotApiKey = '0' + yaraScan = False + capaScan = False + clamAvScan = False + fileScanRuleUpdate = False + fileCarveHttpServer = False + fileCarveHttpServerZip = False + fileCarveHttpServeEncryptKey = '' + + if InstallerYesOrNo('Enable file extraction with Zeek?', default=bool(fileCarveModeDefault)): + loopBreaker = CountUntilException(MaxAskForValueCount, 'Invalid file extraction behavior') + while fileCarveMode not in allowedFileCarveModes.keys() and loopBreaker.increment(): + fileCarveMode = InstallerChooseOne( + 'Select file extraction behavior', + choices=[ + ( + x, + allowedFileCarveModes[x], + x == fileCarveModeDefault if fileCarveModeDefault else 'none', + ) + for x in allowedFileCarveModes.keys() + ], + ) + if fileCarveMode and (fileCarveMode != 'none'): + + loopBreaker = CountUntilException(MaxAskForValueCount, 'Invalid file preservation behavior') + while filePreserveMode not in allowedFilePreserveModes and loopBreaker.increment(): + filePreserveMode = InstallerChooseOne( + 'Select file preservation behavior', + choices=[ + ( + x, + '', + ( + x == filePreserveModeDefault + if filePreserveModeDefault + else allowedFilePreserveModes[0] + ), + ) + for x in allowedFilePreserveModes + ], + ) - reverseDns = (malcolmProfile == PROFILE_MALCOLM) and InstallerYesOrNo( - 'Perform reverse DNS lookup locally for source and destination IP addresses in logs?', - default=args.reverseDns, - ) - autoOui = (malcolmProfile == PROFILE_MALCOLM) and InstallerYesOrNo( - 'Perform hardware vendor OUI lookups for MAC addresses?', default=args.autoOui - ) - autoFreq = (malcolmProfile == PROFILE_MALCOLM) and InstallerYesOrNo( - 'Perform string randomness scoring on some fields?', default=args.autoFreq - ) + if diskUsageManagementPrompt: + loopBreaker = CountUntilException( + MaxAskForValueCount, 'Invalid Zeek extracted file prune threshold' + ) + extractedFilePruneThresholdTemp = '' + while ( + not re.match( + r'^\d+(\.\d+)?\s*[kmgtp%]?b?$', + extractedFilePruneThresholdTemp, + flags=re.IGNORECASE, + ) + ) and loopBreaker.increment(): + extractedFilePruneThresholdTemp = InstallerAskForString( + 'Enter maximum allowed space for Zeek-extracted files (e.g., 250GB) or file system fill threshold (e.g., 90%)', + default=( + args.extractedFileMaxPercentThreshold + if args.extractedFileMaxPercentThreshold + else args.extractedFileMaxSizeThreshold + ), + ) + if extractedFilePruneThresholdTemp: + if '%' in extractedFilePruneThresholdTemp: + extractedFileMaxPercentThreshold = str2percent(extractedFilePruneThresholdTemp) + extractedFileMaxSizeThreshold = '0' + else: + extractedFileMaxPercentThreshold = 0 + extractedFileMaxSizeThreshold = extractedFilePruneThresholdTemp + + fileCarveHttpServer = (malcolmProfile == PROFILE_MALCOLM) and InstallerYesOrNo( + 'Expose web interface for downloading preserved files?', + default=args.fileCarveHttpServer, + ) + if fileCarveHttpServer: + fileCarveHttpServerZip = InstallerYesOrNo( + 'ZIP downloaded preserved files?', default=args.fileCarveHttpServerZip + ) + fileCarveHttpServeEncryptKey = InstallerAskForString( + ( + 'Enter ZIP archive password for downloaded preserved files (or leave blank for unprotected)' + if fileCarveHttpServerZip + else 'Enter AES-256-CBC encryption password for downloaded preserved files (or leave blank for unencrypted)' + ), + default=args.fileCarveHttpServeEncryptKey, + ) + if fileCarveMode is not None: + if InstallerYesOrNo('Scan extracted files with ClamAV?', default=args.clamAvScan): + clamAvScan = True + if InstallerYesOrNo('Scan extracted files with Yara?', default=args.yaraScan): + yaraScan = True + if InstallerYesOrNo('Scan extracted PE files with Capa?', default=args.capaScan): + capaScan = True + if InstallerYesOrNo( + 'Lookup extracted file hashes with VirusTotal?', default=(len(args.vtotApiKey) > 1) + ): + loopBreaker = CountUntilException(MaxAskForValueCount, 'Invalid VirusTotal API key') + while (len(vtotApiKey) <= 1) and loopBreaker.increment(): + vtotApiKey = InstallerAskForString( + 'Enter VirusTotal API key', default=args.vtotApiKey + ) + fileScanRuleUpdate = InstallerYesOrNo( + 'Download updated file scanner signatures periodically?', + default=args.fileScanRuleUpdate, + ) - openPortsSelection = ( - 'c' - if (args.exposeLogstash or args.exposeOpenSearch or args.exposeFilebeatTcp or args.exposeSFTP) - else 'unset' - ) - if self.orchMode is OrchestrationFramework.DOCKER_COMPOSE: - if malcolmProfile == PROFILE_MALCOLM: - openPortsOptions = ('no', 'yes', 'customize') - loopBreaker = CountUntilException(MaxAskForValueCount) - while openPortsSelection not in [x[0] for x in openPortsOptions] and loopBreaker.increment(): - openPortsSelection = InstallerChooseOne( - 'Should Malcolm accept logs and metrics from a Hedgehog Linux sensor or other forwarder?', - choices=[(x, '', x == openPortsOptions[0]) for x in openPortsOptions], - )[0] - if openPortsSelection == 'n': - opensearchOpen = False - logstashOpen = False - filebeatTcpOpen = False - elif openPortsSelection == 'y': - opensearchOpen = opensearchPrimaryMode == DatabaseMode.OpenSearchLocal - logstashOpen = True - filebeatTcpOpen = True - else: - openPortsSelection = 'c' - opensearchOpen = (opensearchPrimaryMode == DatabaseMode.OpenSearchLocal) and InstallerYesOrNo( - 'Expose OpenSearch port to external hosts?', default=args.exposeOpenSearch + if fileCarveMode not in allowedFileCarveModes.keys(): + fileCarveMode = 'none' + if filePreserveMode not in allowedFilePreserveModes: + filePreserveMode = allowedFilePreserveModes[0] + if (vtotApiKey is None) or (len(vtotApiKey) <= 1): + vtotApiKey = '0' + + ################################################################################### + elif currentStep == ConfigOptions.NetBox: + # NetBox + netboxEnabled = (malcolmProfile == PROFILE_MALCOLM) and InstallerYesOrNo( + 'Should Malcolm run and maintain an instance of NetBox, an infrastructure resource modeling tool?', + default=args.netboxEnabled, ) - logstashOpen = InstallerYesOrNo( - 'Expose Logstash port to external hosts?', default=args.exposeLogstash + netboxLogstashEnrich = netboxEnabled and InstallerYesOrNo( + 'Should Malcolm enrich network traffic using NetBox?', + default=args.netboxLogstashEnrich, ) - filebeatTcpOpen = InstallerYesOrNo( - 'Expose Filebeat TCP port to external hosts?', default=args.exposeFilebeatTcp + netboxAutoPopulate = netboxEnabled and InstallerYesOrNo( + 'Should Malcolm automatically populate NetBox inventory based on observed network traffic?', + default=args.netboxAutoPopulate, ) - else: - opensearchOpen = False - openPortsSelection = 'n' - logstashOpen = False - filebeatTcpOpen = False - - else: - opensearchOpen = opensearchPrimaryMode == DatabaseMode.OpenSearchLocal - openPortsSelection = 'y' - logstashOpen = True - filebeatTcpOpen = True - - filebeatTcpFormat = 'json' - filebeatTcpSourceField = 'message' - filebeatTcpTargetField = 'miscbeat' - filebeatTcpDropField = filebeatTcpSourceField - filebeatTcpTag = '_malcolm_beats' - if ( - filebeatTcpOpen - and (openPortsSelection == 'c') - and not InstallerYesOrNo('Use default field values for Filebeat TCP listener?', default=True) - ): - allowedFilebeatTcpFormats = ('json', 'raw') - filebeatTcpFormat = 'unset' - loopBreaker = CountUntilException(MaxAskForValueCount, f'Invalid log format') - while filebeatTcpFormat not in allowedFilebeatTcpFormats and loopBreaker.increment(): - filebeatTcpFormat = InstallerChooseOne( - 'Select log format for messages sent to Filebeat TCP listener', - choices=[(x, '', x == allowedFilebeatTcpFormats[0]) for x in allowedFilebeatTcpFormats], - ) - if filebeatTcpFormat == 'json': - filebeatTcpSourceField = InstallerAskForString( - 'Source field to parse for messages sent to Filebeat TCP listener', - default=filebeatTcpSourceField, - ) - filebeatTcpTargetField = InstallerAskForString( - 'Target field under which to store decoded JSON fields for messages sent to Filebeat TCP listener', - default=filebeatTcpTargetField, - ) - filebeatTcpDropField = InstallerAskForString( - 'Field to drop from events sent to Filebeat TCP listener', - default=filebeatTcpSourceField, - ) - filebeatTcpTag = InstallerAskForString( - 'Tag to apply to messages sent to Filebeat TCP listener', - default=filebeatTcpTag, - ) - - sftpOpen = ( - (self.orchMode is OrchestrationFramework.DOCKER_COMPOSE) - and (malcolmProfile == PROFILE_MALCOLM) - and (openPortsSelection == 'c') - and InstallerYesOrNo('Expose SFTP server (for PCAP upload) to external hosts?', default=args.exposeSFTP) - ) - - # input file extraction parameters - allowedFileCarveModes = { - 'none': 'No file extraction', - 'known': 'Extract recognized MIME types', - 'mapped': 'Extract MIME types for which file extensions are known', - 'all': 'Extract all files', - 'interesting': 'Extract MIME types of common attack vectors', - 'notcommtxt': 'Extract all except common plain text files', - } - allowedFilePreserveModes = ('quarantined', 'all', 'none') - - fileCarveMode = None - fileCarveModeDefault = args.fileCarveMode.lower() if args.fileCarveMode else None - filePreserveMode = None - filePreserveModeDefault = args.filePreserveMode.lower() if args.filePreserveMode else None - vtotApiKey = '0' - yaraScan = False - capaScan = False - clamAvScan = False - fileScanRuleUpdate = False - fileCarveHttpServer = False - fileCarveHttpServerZip = False - fileCarveHttpServeEncryptKey = '' - - if InstallerYesOrNo('Enable file extraction with Zeek?', default=bool(fileCarveModeDefault)): - loopBreaker = CountUntilException(MaxAskForValueCount, 'Invalid file extraction behavior') - while fileCarveMode not in allowedFileCarveModes.keys() and loopBreaker.increment(): - fileCarveMode = InstallerChooseOne( - 'Select file extraction behavior', - choices=[ - ( - x, - allowedFileCarveModes[x], - x == fileCarveModeDefault if fileCarveModeDefault else 'none', + netboxLogstashAutoSubnets = netboxLogstashEnrich and InstallerYesOrNo( + 'Should Malcolm automatically create missing NetBox subnet prefixes based on observed network traffic?', + default=args.netboxLogstashAutoSubnets, + ) + netboxSiteName = ( + InstallerAskForString( + 'Specify default NetBox site name', + default=args.netboxSiteName, ) - for x in allowedFileCarveModes.keys() - ], - ) - if fileCarveMode and (fileCarveMode != 'none'): - - loopBreaker = CountUntilException(MaxAskForValueCount, 'Invalid file preservation behavior') - while filePreserveMode not in allowedFilePreserveModes and loopBreaker.increment(): - filePreserveMode = InstallerChooseOne( - 'Select file preservation behavior', - choices=[ - ( - x, - '', - ( - x == filePreserveModeDefault - if filePreserveModeDefault - else allowedFilePreserveModes[0] - ), - ) - for x in allowedFilePreserveModes - ], + if netboxEnabled + else '' ) - - if diskUsageManagementPrompt: - loopBreaker = CountUntilException( - MaxAskForValueCount, 'Invalid Zeek extracted file prune threshold' + if len(netboxSiteName) == 0: + netboxSiteName = 'Malcolm' + + ################################################################################### + elif currentStep == ConfigOptions.Capture: + # input packet capture parameters + pcapNetSniff = False + pcapTcpDump = False + liveArkime = False + liveArkimeNodeHost = '' + liveZeek = False + liveSuricata = False + pcapIface = 'lo' + tweakIface = False + pcapFilter = '' + captureSelection = ( + 'c' + if ( + args.pcapNetSniff + or args.pcapTcpDump + or args.liveZeek + or args.liveSuricata + or (malcolmProfile == PROFILE_HEDGEHOG) + ) + else 'unset' ) - extractedFilePruneThresholdTemp = '' - while ( - not re.match( - r'^\d+(\.\d+)?\s*[kmgtp%]?b?$', extractedFilePruneThresholdTemp, flags=re.IGNORECASE + + captureOptions = ('no', 'yes', 'customize') + loopBreaker = CountUntilException(MaxAskForValueCount) + while captureSelection not in [x[0] for x in captureOptions] and loopBreaker.increment(): + captureSelection = InstallerChooseOne( + 'Should Malcolm capture live network traffic?', + choices=[(x, '', x == captureOptions[0]) for x in captureOptions], + )[0] + if captureSelection == 'y': + liveArkime = (malcolmProfile == PROFILE_HEDGEHOG) or ( + opensearchPrimaryMode != DatabaseMode.OpenSearchLocal ) - ) and loopBreaker.increment(): - extractedFilePruneThresholdTemp = InstallerAskForString( - 'Enter maximum allowed space for Zeek-extracted files (e.g., 250GB) or file system fill threshold (e.g., 90%)', - default=( - args.extractedFileMaxPercentThreshold - if args.extractedFileMaxPercentThreshold - else args.extractedFileMaxSizeThreshold - ), + pcapNetSniff = not liveArkime + liveSuricata = True + liveZeek = True + tweakIface = True + elif captureSelection == 'c': + if InstallerYesOrNo( + 'Should Malcolm capture live network traffic to PCAP files for analysis with Arkime?', + default=args.pcapNetSniff + or args.pcapTcpDump + or args.liveArkime + or (malcolmProfile == PROFILE_HEDGEHOG), + ): + liveArkime = (opensearchPrimaryMode != DatabaseMode.OpenSearchLocal) and ( + (malcolmProfile == PROFILE_HEDGEHOG) + or InstallerYesOrNo('Capture packets using Arkime capture?', default=args.liveArkime) + ) + pcapNetSniff = (not liveArkime) and InstallerYesOrNo( + 'Capture packets using netsniff-ng?', default=args.pcapNetSniff + ) + pcapTcpDump = ( + (not liveArkime) + and (not pcapNetSniff) + and InstallerYesOrNo('Capture packets using tcpdump?', default=args.pcapTcpDump) + ) + liveSuricata = InstallerYesOrNo( + 'Should Malcolm analyze live network traffic with Suricata?', default=args.liveSuricata ) - if extractedFilePruneThresholdTemp: - if '%' in extractedFilePruneThresholdTemp: - extractedFileMaxPercentThreshold = str2percent(extractedFilePruneThresholdTemp) - extractedFileMaxSizeThreshold = '0' - else: - extractedFileMaxPercentThreshold = 0 - extractedFileMaxSizeThreshold = extractedFilePruneThresholdTemp + liveZeek = InstallerYesOrNo( + 'Should Malcolm analyze live network traffic with Zeek?', default=args.liveZeek + ) + if pcapNetSniff or pcapTcpDump or liveArkime or liveZeek or liveSuricata: + pcapFilter = InstallerAskForString( + 'Capture filter (tcpdump-like filter expression; leave blank to capture all traffic)', + default=args.pcapFilter, + ) + # Arkime requires disabling NIC offloading: https://arkime.com/faq#arkime_requires_full_packet_captures_error + tweakIface = liveArkime or InstallerYesOrNo( + 'Disable capture interface hardware offloading and adjust ring buffer sizes?', + default=args.tweakIface, + ) - fileCarveHttpServer = (malcolmProfile == PROFILE_MALCOLM) and InstallerYesOrNo( - 'Expose web interface for downloading preserved files?', default=args.fileCarveHttpServer - ) - if fileCarveHttpServer: - fileCarveHttpServerZip = InstallerYesOrNo( - 'ZIP downloaded preserved files?', default=args.fileCarveHttpServerZip - ) - fileCarveHttpServeEncryptKey = InstallerAskForString( - ( - 'Enter ZIP archive password for downloaded preserved files (or leave blank for unprotected)' - if fileCarveHttpServerZip - else 'Enter AES-256-CBC encryption password for downloaded preserved files (or leave blank for unencrypted)' - ), - default=args.fileCarveHttpServeEncryptKey, - ) - if fileCarveMode is not None: - if InstallerYesOrNo('Scan extracted files with ClamAV?', default=args.clamAvScan): - clamAvScan = True - if InstallerYesOrNo('Scan extracted files with Yara?', default=args.yaraScan): - yaraScan = True - if InstallerYesOrNo('Scan extracted PE files with Capa?', default=args.capaScan): - capaScan = True - if InstallerYesOrNo( - 'Lookup extracted file hashes with VirusTotal?', default=(len(args.vtotApiKey) > 1) - ): - loopBreaker = CountUntilException(MaxAskForValueCount, 'Invalid VirusTotal API key') - while (len(vtotApiKey) <= 1) and loopBreaker.increment(): - vtotApiKey = InstallerAskForString('Enter VirusTotal API key', default=args.vtotApiKey) - fileScanRuleUpdate = InstallerYesOrNo( - 'Download updated file scanner signatures periodically?', default=args.fileScanRuleUpdate - ) + if pcapNetSniff or pcapTcpDump or liveArkime or liveZeek or liveSuricata: + pcapIface = '' + loopBreaker = CountUntilException(MaxAskForValueCount, 'Invalid capture interface(s)') + while (len(pcapIface) <= 0) and loopBreaker.increment(): + pcapIface = InstallerAskForString( + 'Specify capture interface(s) (comma-separated)', default=args.pcapIface + ) - if fileCarveMode not in allowedFileCarveModes.keys(): - fileCarveMode = 'none' - if filePreserveMode not in allowedFilePreserveModes: - filePreserveMode = allowedFilePreserveModes[0] - if (vtotApiKey is None) or (len(vtotApiKey) <= 1): - vtotApiKey = '0' - - # NetBox - netboxEnabled = (malcolmProfile == PROFILE_MALCOLM) and InstallerYesOrNo( - 'Should Malcolm run and maintain an instance of NetBox, an infrastructure resource modeling tool?', - default=args.netboxEnabled, - ) - netboxLogstashEnrich = netboxEnabled and InstallerYesOrNo( - 'Should Malcolm enrich network traffic using NetBox?', - default=args.netboxLogstashEnrich, - ) - netboxAutoPopulate = netboxEnabled and InstallerYesOrNo( - 'Should Malcolm automatically populate NetBox inventory based on observed network traffic?', - default=args.netboxAutoPopulate, - ) - netboxLogstashAutoSubnets = netboxLogstashEnrich and InstallerYesOrNo( - 'Should Malcolm automatically create missing NetBox subnet prefixes based on observed network traffic?', - default=args.netboxLogstashAutoSubnets, - ) - netboxSiteName = ( - InstallerAskForString( - 'Specify default NetBox site name', - default=args.netboxSiteName, - ) - if netboxEnabled - else '' - ) - if len(netboxSiteName) == 0: - netboxSiteName = 'Malcolm' - - # input packet capture parameters - pcapNetSniff = False - pcapTcpDump = False - liveArkime = False - liveArkimeNodeHost = '' - liveZeek = False - liveSuricata = False - pcapIface = 'lo' - tweakIface = False - pcapFilter = '' - captureSelection = ( - 'c' - if ( - args.pcapNetSniff - or args.pcapTcpDump - or args.liveZeek - or args.liveSuricata - or (malcolmProfile == PROFILE_HEDGEHOG) - ) - else 'unset' - ) + if liveArkime: + liveArkimeNodeHost = InstallerAskForString( + f"Enter this node's hostname or IP to associate with network traffic metadata", + default=args.liveArkimeNodeHost, + ) - captureOptions = ('no', 'yes', 'customize') - loopBreaker = CountUntilException(MaxAskForValueCount) - while captureSelection not in [x[0] for x in captureOptions] and loopBreaker.increment(): - captureSelection = InstallerChooseOne( - 'Should Malcolm capture live network traffic?', - choices=[(x, '', x == captureOptions[0]) for x in captureOptions], - )[0] - if captureSelection == 'y': - liveArkime = (malcolmProfile == PROFILE_HEDGEHOG) or (opensearchPrimaryMode != DatabaseMode.OpenSearchLocal) - pcapNetSniff = not liveArkime - liveSuricata = True - liveZeek = True - tweakIface = True - elif captureSelection == 'c': - if InstallerYesOrNo( - 'Should Malcolm capture live network traffic to PCAP files for analysis with Arkime?', - default=args.pcapNetSniff - or args.pcapTcpDump - or args.liveArkime - or (malcolmProfile == PROFILE_HEDGEHOG), - ): - liveArkime = (opensearchPrimaryMode != DatabaseMode.OpenSearchLocal) and ( - (malcolmProfile == PROFILE_HEDGEHOG) - or InstallerYesOrNo('Capture packets using Arkime capture?', default=args.liveArkime) - ) - pcapNetSniff = (not liveArkime) and InstallerYesOrNo( - 'Capture packets using netsniff-ng?', default=args.pcapNetSniff - ) - pcapTcpDump = ( - (not liveArkime) - and (not pcapNetSniff) - and InstallerYesOrNo('Capture packets using tcpdump?', default=args.pcapTcpDump) - ) - liveSuricata = InstallerYesOrNo( - 'Should Malcolm analyze live network traffic with Suricata?', default=args.liveSuricata - ) - liveZeek = InstallerYesOrNo('Should Malcolm analyze live network traffic with Zeek?', default=args.liveZeek) - if pcapNetSniff or pcapTcpDump or liveArkime or liveZeek or liveSuricata: - pcapFilter = InstallerAskForString( - 'Capture filter (tcpdump-like filter expression; leave blank to capture all traffic)', - default=args.pcapFilter, - ) - # Arkime requires disabling NIC offloading: https://arkime.com/faq#arkime_requires_full_packet_captures_error - tweakIface = liveArkime or InstallerYesOrNo( - 'Disable capture interface hardware offloading and adjust ring buffer sizes?', - default=args.tweakIface, - ) + if ( + (malcolmProfile == PROFILE_HEDGEHOG) + and (not pcapNetSniff) + and (not pcapTcpDump) + and (not liveZeek) + and (not liveSuricata) + and (not liveArkime) + ): + InstallerDisplayMessage( + f'Warning: Running with the {malcolmProfile} profile but no capture methods are enabled.', + ) - if pcapNetSniff or pcapTcpDump or liveArkime or liveZeek or liveSuricata: - pcapIface = '' - loopBreaker = CountUntilException(MaxAskForValueCount, 'Invalid capture interface(s)') - while (len(pcapIface) <= 0) and loopBreaker.increment(): - pcapIface = InstallerAskForString( - 'Specify capture interface(s) (comma-separated)', default=args.pcapIface - ) + ################################################################################### + elif currentStep == ConfigOptions.DarkMode: + dashboardsDarkMode = ( + (malcolmProfile == PROFILE_MALCOLM) + and (opensearchPrimaryMode != DatabaseMode.ElasticsearchRemote) + and InstallerYesOrNo( + 'Enable dark mode for OpenSearch Dashboards?', default=args.dashboardsDarkMode + ) + ) - if liveArkime: - liveArkimeNodeHost = InstallerAskForString( - f"Enter this node's hostname or IP to associate with network traffic metadata", - default=args.liveArkimeNodeHost, - ) + ################################################################################### + elif currentStep == ConfigOptions.PostConfig: + break - if ( - (malcolmProfile == PROFILE_HEDGEHOG) - and (not pcapNetSniff) - and (not pcapTcpDump) - and (not liveZeek) - and (not liveSuricata) - and (not liveArkime) - ): - InstallerDisplayMessage( - f'Warning: Running with the {malcolmProfile} profile but no capture methods are enabled.', - ) + except DialogBackException: + if int(currentStep) >= 2: + currentStep = ConfigOptions(int(currentStep) - 2) + else: + currentStep = ConfigOptions.Preconfig + eprint(f'back: {currentStep}') - dashboardsDarkMode = ( - (malcolmProfile == PROFILE_MALCOLM) - and (opensearchPrimaryMode != DatabaseMode.ElasticsearchRemote) - and InstallerYesOrNo('Enable dark mode for OpenSearch Dashboards?', default=args.dashboardsDarkMode) - ) + except DialogCanceledException: + raise Exception("Canceled by the user") # modify values in .env files in args.configDir diff --git a/scripts/malcolm_common.py b/scripts/malcolm_common.py index fdf001cfc..f4d5a3af8 100644 --- a/scripts/malcolm_common.py +++ b/scripts/malcolm_common.py @@ -22,11 +22,10 @@ LoadStrIfJson, remove_suffix, run_process, - str2bool, ) from collections import defaultdict, namedtuple -from enum import Flag, IntFlag, auto +from enum import IntEnum, Flag, IntFlag, auto try: from pwd import getpwuid @@ -102,6 +101,20 @@ class UserInterfaceMode(IntFlag): InteractionInput = auto() +class DialogBackException(Exception): + pass + + +class DialogCanceledException(Exception): + pass + + +class BoolOrExtra(IntEnum): + FALSE = 0 + TRUE = 1 + EXTRA = 2 + + BoundPath = namedtuple( "BoundPath", ["service", "target", "files", "relative_dirs", "clean_empty_dirs"], @@ -226,6 +239,23 @@ def ClearScreen(): pass +################################################################################################### +def str2boolorextra(v): + if isinstance(v, bool): + return BoolOrExtra.TRUE if v else BoolOrExtra.FALSE + elif isinstance(v, str): + if v.lower() in ("yes", "true", "t", "y", "1"): + return BoolOrExtra.TRUE + elif v.lower() in ("no", "false", "f", "n", "0"): + return BoolOrExtra.FALSE + elif v.lower() in ("b", "back", "p", "previous", "e", "extra"): + return BoolOrExtra.EXTRA + else: + raise ValueError("BoolOrExtra value expected") + else: + raise ValueError("BoolOrExtra value expected") + + ################################################################################################### # get interactive user response to Y/N question def YesOrNo( @@ -236,9 +266,11 @@ def YesOrNo( clearScreen=False, yesLabel='Yes', noLabel='No', + extraLabel=None, ): global Dialog global MainDialog + result = None if (default is not None) and ( (defaultBehavior & UserInputDefaultsBehavior.DefaultsAccept) @@ -247,20 +279,22 @@ def YesOrNo( reply = "" elif (uiMode & UserInterfaceMode.InteractionDialog) and (MainDialog is not None): - defaultYes = (default is not None) and str2bool(default) + defaultYes = (default is not None) and str2boolorextra(default) reply = MainDialog.yesno( question, yes_label=yesLabel.capitalize() if defaultYes else noLabel.capitalize(), no_label=decapitalize(noLabel) if defaultYes else decapitalize(yesLabel), + extra_button=(extraLabel is not None), + extra_label=extraLabel, ) if defaultYes: - reply = 'y' if (reply == Dialog.OK) else 'n' + reply = 'y' if (reply == Dialog.OK) else ('e' if (reply == Dialog.EXTRA) else 'n') else: - reply = 'n' if (reply == Dialog.OK) else 'y' + reply = 'n' if (reply == Dialog.OK) else ('e' if (reply == Dialog.EXTRA) else 'y') elif uiMode & UserInterfaceMode.InteractionInput: if (default is not None) and defaultBehavior & UserInputDefaultsBehavior.DefaultsPrompt: - if str2bool(default): + if str2boolorextra(default): questionStr = f"\n{question} (Y{'' if yesLabel == 'Yes' else ' (' + yesLabel + ')'} / n{'' if noLabel == 'No' else ' (' + noLabel + ')'}): " else: questionStr = f"\n{question} (y{'' if yesLabel == 'Yes' else ' (' + yesLabel + ')'} / N{'' if noLabel == 'No' else ' (' + noLabel + ')'}): " @@ -271,7 +305,7 @@ def YesOrNo( reply = str(input(questionStr)).lower().strip() if len(reply) > 0: try: - str2bool(reply) + str2boolorextra(reply) break except ValueError: pass @@ -282,22 +316,30 @@ def YesOrNo( raise RuntimeError("No user interfaces available") if (len(reply) == 0) and (defaultBehavior & UserInputDefaultsBehavior.DefaultsAccept): - reply = "y" if (default is not None) and str2bool(default) else "n" + reply = "y" if (default is not None) and str2boolorextra(default) else "n" if clearScreen is True: ClearScreen() try: - return str2bool(reply) + result = str2boolorextra(reply) except ValueError: - return YesOrNo( + result = YesOrNo( question, default=default, uiMode=uiMode, defaultBehavior=defaultBehavior - UserInputDefaultsBehavior.DefaultsAccept, clearScreen=clearScreen, + yesLabel=yesLabel, + noLabel=noLabel, + extraLabel=extraLabel, ) + if result == BoolOrExtra.EXTRA: + raise DialogBackException(question) + + return bool(result) + ################################################################################################### # get interactive user response @@ -307,6 +349,7 @@ def AskForString( defaultBehavior=UserInputDefaultsBehavior.DefaultsPrompt, uiMode=UserInterfaceMode.InteractionDialog | UserInterfaceMode.InteractionInput, clearScreen=False, + extraLabel=None, ): global Dialog global MainDialog @@ -325,9 +368,13 @@ def AskForString( if (default is not None) and (defaultBehavior & UserInputDefaultsBehavior.DefaultsPrompt) else "" ), + extra_button=(extraLabel is not None), + extra_label=extraLabel, ) if (code == Dialog.CANCEL) or (code == Dialog.ESC): - raise RuntimeError("Operation cancelled") + raise DialogCanceledException(question) + elif code == Dialog.EXTRA: + raise DialogBackException(question) else: reply = reply.strip() @@ -357,6 +404,7 @@ def AskForPassword( defaultBehavior=UserInputDefaultsBehavior.DefaultsPrompt, uiMode=UserInterfaceMode.InteractionDialog | UserInterfaceMode.InteractionInput, clearScreen=False, + extraLabel=None, ): global Dialog global MainDialog @@ -368,9 +416,16 @@ def AskForPassword( reply = default elif (uiMode & UserInterfaceMode.InteractionDialog) and (MainDialog is not None): - code, reply = MainDialog.passwordbox(prompt, insecure=True) + code, reply = MainDialog.passwordbox( + prompt, + insecure=True, + extra_button=(extraLabel is not None), + extra_label=extraLabel, + ) if (code == Dialog.CANCEL) or (code == Dialog.ESC): - raise RuntimeError("Operation cancelled") + raise DialogCanceledException(prompt) + elif code == Dialog.EXTRA: + raise DialogBackException(prompt) elif uiMode & UserInterfaceMode.InteractionInput: reply = getpass.getpass(prompt=f"{prompt}: ") @@ -396,6 +451,7 @@ def ChooseOne( defaultBehavior=UserInputDefaultsBehavior.DefaultsPrompt, uiMode=UserInterfaceMode.InteractionDialog | UserInterfaceMode.InteractionInput, clearScreen=False, + extraLabel=None, ): global Dialog global MainDialog @@ -412,9 +468,13 @@ def ChooseOne( code, reply = MainDialog.radiolist( prompt, choices=validChoices, + extra_button=(extraLabel is not None), + extra_label=extraLabel, ) if code == Dialog.CANCEL or code == Dialog.ESC: - raise RuntimeError("Operation cancelled") + raise DialogCanceledException(prompt) + elif code == Dialog.EXTRA: + raise DialogBackException(prompt) elif uiMode & UserInterfaceMode.InteractionInput: index = 0 @@ -460,6 +520,7 @@ def ChooseMultiple( defaultBehavior=UserInputDefaultsBehavior.DefaultsPrompt, uiMode=UserInterfaceMode.InteractionDialog | UserInterfaceMode.InteractionInput, clearScreen=False, + extraLabel=None, ): global Dialog global MainDialog @@ -476,9 +537,13 @@ def ChooseMultiple( code, reply = MainDialog.checklist( prompt, choices=validChoices, + extra_button=(extraLabel is not None), + extra_label=extraLabel, ) if code == Dialog.CANCEL or code == Dialog.ESC: - raise RuntimeError("Operation cancelled") + raise DialogCanceledException(prompt) + elif code == Dialog.EXTRA: + raise DialogBackException(prompt) elif uiMode & UserInterfaceMode.InteractionInput: allowedChars = set(string.digits + ',' + ' ') @@ -529,6 +594,7 @@ def DisplayMessage( defaultBehavior=UserInputDefaultsBehavior.DefaultsPrompt, uiMode=UserInterfaceMode.InteractionDialog | UserInterfaceMode.InteractionInput, clearScreen=False, + extraLabel=None, ): global Dialog global MainDialog @@ -543,9 +609,13 @@ def DisplayMessage( elif (uiMode & UserInterfaceMode.InteractionDialog) and (MainDialog is not None): code = MainDialog.msgbox( message, + extra_button=(extraLabel is not None), + extra_label=extraLabel, ) if (code == Dialog.CANCEL) or (code == Dialog.ESC): - raise RuntimeError("Operation cancelled") + raise DialogCanceledException(message) + elif code == Dialog.EXTRA: + raise DialogBackException(message) else: reply = True @@ -567,6 +637,7 @@ def DisplayProgramBox( fileDescriptor=None, text=None, clearScreen=False, + extraLabel=None, ): global Dialog global MainDialog @@ -581,9 +652,13 @@ def DisplayProgramBox( text=text, width=78, height=20, + extra_button=(extraLabel is not None), + extra_label=extraLabel, ) if (code == Dialog.CANCEL) or (code == Dialog.ESC): - raise RuntimeError("Operation cancelled") + raise DialogCanceledException() + elif code == Dialog.EXTRA: + raise DialogBackException() else: reply = True