diff --git a/malcolm-iso/config/package-lists/python.list.chroot b/malcolm-iso/config/package-lists/python.list.chroot index a7be31ca4..2cf6d89cf 100644 --- a/malcolm-iso/config/package-lists/python.list.chroot +++ b/malcolm-iso/config/package-lists/python.list.chroot @@ -7,4 +7,5 @@ python3-psutil python3-pycryptodome python3-dialog python3-requests -python3-ruamel.yaml \ No newline at end of file +python3-ruamel.yaml +python3-yaml \ No newline at end of file diff --git a/scripts/control.py b/scripts/control.py index eed575719..53e631f0a 100755 --- a/scripts/control.py +++ b/scripts/control.py @@ -48,6 +48,8 @@ def __exit__(self, *args): dockerBin = None dockerComposeBin = None opensslBin = None +yamlImported = None +dockerComposeYaml = None ################################################################################################### try: @@ -495,6 +497,7 @@ def stop(wipe=False): global args global dockerBin global dockerComposeBin + global dockerComposeYaml # docker-compose use local temporary path osEnv = os.environ.copy() @@ -515,47 +518,67 @@ def stop(wipe=False): exit(err) if wipe: - # delete OpenSearch database - shutil.rmtree(os.path.join(MalcolmPath, os.path.join('opensearch', 'nodes')), ignore_errors=True) - - # delete Zeek live-related spool files - shutil.rmtree( - os.path.join(MalcolmPath, os.path.join('zeek-logs', os.path.join('live', 'spool'))), ignore_errors=True + # there is some overlap here among some of these containers, but it doesn't matter + boundPathsToWipe = ( + BoundPath("arkime", "/opt/arkime/logs", True, None, None), + BoundPath("arkime", "/opt/arkime/raw", True, None, None), + BoundPath("filebeat", "/zeek", True, None, None), + BoundPath("file-monitor", "/zeek/logs", True, None, None), + BoundPath("netbox", "/opt/netbox/netbox/media", True, None, ["."]), + BoundPath("netbox-postgres", "/var/lib/postgresql/data", True, None, ["."]), + BoundPath("netbox-redis", "/data", True, None, ["."]), + BoundPath("opensearch", "/usr/share/opensearch/data", True, ["nodes"], None), + BoundPath("pcap-capture", "/pcap", True, None, None), + BoundPath("pcap-monitor", "/pcap", True, ["processed", "upload"], None), + BoundPath("suricata", "/var/log/suricata", True, None, ["."]), + BoundPath("upload", "/var/www/upload/server/php/chroot/files", True, None, None), + BoundPath("zeek", "/zeek/extract_files", True, None, None), + BoundPath("zeek", "/zeek/upload", True, None, None), + BoundPath("zeek-live", "/zeek/live", True, ["spool"], None), + BoundPath( + "filebeat", + "/zeek", + False, + ["processed", "current", "live"], + ["processed", "current", "live"], + ), ) - - # delete data files (backups, zeek logs, arkime logs, PCAP files, captured PCAP files) - for dataDir in [ - 'opensearch-backup', - 'zeek-logs', - 'suricata-logs', - 'arkime-logs', - 'pcap', - 'arkime-raw', - os.path.join('netbox', 'media'), - os.path.join('netbox', 'postgres'), - os.path.join('netbox', 'redis'), - ]: - for root, dirnames, filenames in os.walk(os.path.join(MalcolmPath, dataDir), topdown=True, onerror=None): - for file in filenames: - fileSpec = os.path.join(root, file) - if (os.path.isfile(fileSpec) or os.path.islink(fileSpec)) and (not file.startswith('.git')): - try: - os.remove(fileSpec) - except: - pass - - # clean up empty directories - for dataDir in [ - os.path.join('opensearch-backup', 'logs'), - os.path.join('zeek-logs', 'processed'), - os.path.join('zeek-logs', 'current'), - os.path.join('zeek-logs', 'live'), - os.path.join('suricata-logs'), - os.path.join('netbox', 'media'), - os.path.join('netbox', 'postgres'), - os.path.join('netbox', 'redis'), - ]: - RemoveEmptyFolders(dataDir, removeRoot=False) + for boundPath in boundPathsToWipe: + localPath = LocalPathForContainerBindMount( + boundPath.service, + dockerComposeYaml, + boundPath.container_dir, + MalcolmPath, + ) + if localPath and os.path.isdir(localPath): + # delete files + if boundPath.files: + if args.debug: + eprint(f'Walking "{localPath}" for file deletion') + for root, dirnames, filenames in os.walk(localPath, topdown=True, onerror=None): + for file in filenames: + fileSpec = os.path.join(root, file) + if (os.path.isfile(fileSpec) or os.path.islink(fileSpec)) and (not file.startswith('.git')): + try: + os.remove(fileSpec) + except: + pass + # delete whole directories + if boundPath.relative_dirs: + for relDir in GetIterable(boundPath.relative_dirs): + tmpPath = os.path.join(localPath, relDir) + if os.path.isdir(tmpPath): + if args.debug: + eprint(f'Performing rmtree on "{tmpPath}"') + shutil.rmtree(tmpPath, ignore_errors=True) + # cleanup empty directories + if boundPath.clean_empty_dirs: + for cleanDir in GetIterable(boundPath.clean_empty_dirs): + tmpPath = os.path.join(localPath, cleanDir) + if os.path.isdir(tmpPath): + if args.debug: + eprint(f'Performing RemoveEmptyFolders on "{tmpPath}"') + RemoveEmptyFolders(tmpPath, removeRoot=False) eprint("Malcolm has been stopped and its data cleared\n") @@ -607,31 +630,54 @@ def start(): os.chmod(authFile, stat.S_IRUSR | stat.S_IWUSR) # make sure some directories exist before we start - for path in [ - os.path.join(MalcolmPath, 'opensearch'), - os.path.join(MalcolmPath, 'opensearch-backup'), - os.path.join(MalcolmPath, os.path.join('nginx', 'ca-trust')), - os.path.join(MalcolmPath, os.path.join('netbox', 'media')), - os.path.join(MalcolmPath, os.path.join('netbox', 'postgres')), - os.path.join(MalcolmPath, os.path.join('netbox', 'redis')), - os.path.join(MalcolmPath, os.path.join('pcap', 'processed')), - os.path.join(MalcolmPath, os.path.join('pcap', 'upload')), - os.path.join(MalcolmPath, os.path.join('suricata-logs', 'live')), - os.path.join(MalcolmPath, os.path.join('zeek', os.path.join('intel', 'MISP'))), - os.path.join(MalcolmPath, os.path.join('zeek', os.path.join('intel', 'STIX'))), - os.path.join(MalcolmPath, os.path.join('zeek-logs', 'current')), - os.path.join(MalcolmPath, os.path.join('zeek-logs', 'live')), - os.path.join(MalcolmPath, os.path.join('zeek-logs', 'extract_files')), - os.path.join(MalcolmPath, os.path.join('zeek-logs', 'processed')), - os.path.join(MalcolmPath, os.path.join('zeek-logs', 'upload')), - ]: - try: - os.makedirs(path) - except OSError as exc: - if (exc.errno == errno.EEXIST) and os.path.isdir(path): - pass - else: - raise + boundPathsToCreate = ( + BoundPath("arkime", "/opt/arkime/logs", False, None, None), + BoundPath("arkime", "/opt/arkime/raw", False, None, None), + BoundPath("file-monitor", "/zeek/logs", False, None, None), + BoundPath("nginx-proxy", "/var/local/ca-trust", False, None, None), + BoundPath("netbox", "/opt/netbox/netbox/media", False, None, None), + BoundPath("netbox-postgres", "/var/lib/postgresql/data", False, None, None), + BoundPath("netbox-redis", "/data", False, None, None), + BoundPath("opensearch", "/usr/share/opensearch/data", False, ["nodes"], None), + BoundPath("opensearch", "/opt/opensearch/backup", False, None, None), + BoundPath("pcap-capture", "/pcap", False, ["processed", "upload"], None), + BoundPath("suricata", "/var/log/suricata", False, ["live"], None), + BoundPath("upload", "/var/www/upload/server/php/chroot/files", False, None, None), + BoundPath("zeek", "/zeek/extract_files", False, None, None), + BoundPath("zeek", "/zeek/upload", False, None, None), + BoundPath("zeek", "/opt/zeek/share/zeek/site/intel", False, ["MISP", "STIX"], None), + BoundPath("zeek-live", "/zeek/live", False, ["spool"], None), + BoundPath("filebeat", "/zeek", False, ["processed", "current", "live", "extract_files", "upload"], None), + ) + for boundPath in boundPathsToCreate: + localPath = LocalPathForContainerBindMount( + boundPath.service, + dockerComposeYaml, + boundPath.container_dir, + MalcolmPath, + ) + if localPath: + try: + if args.debug: + eprint(f'Ensuring "{localPath}" exists') + os.makedirs(localPath) + except OSError as exc: + if (exc.errno == errno.EEXIST) and os.path.isdir(localPath): + pass + else: + raise + if boundPath.relative_dirs: + for relDir in GetIterable(boundPath.relative_dirs): + tmpPath = os.path.join(localPath, relDir) + try: + if args.debug: + eprint(f'Ensuring "{tmpPath}" exists') + os.makedirs(tmpPath) + except OSError as exc: + if (exc.errno == errno.EEXIST) and os.path.isdir(tmpPath): + pass + else: + raise # touch the zeek intel file open(os.path.join(MalcolmPath, os.path.join('zeek', os.path.join('intel', '__load__.zeek'))), 'a').close() @@ -1208,6 +1254,8 @@ def main(): global dockerBin global dockerComposeBin global opensslBin + global yamlImported + global dockerComposeYaml # extract arguments from the command line # print (sys.argv[1:]); @@ -1298,6 +1346,12 @@ def main(): else: sys.tracebacklimit = 0 + yamlImported = YAMLDynamic(debug=args.debug) + if args.debug: + eprint(f"Imported yaml: {yamlImported}") + if not yamlImported: + exit(2) + with pushd(MalcolmPath): # don't run this as root @@ -1333,6 +1387,10 @@ def main(): if err != 0: raise Exception(f'{ScriptName} requires docker-compose, please run install.py') + # load compose file YAML (used to find some volume bind mount locations) + with open(args.composeFile, 'r') as cf: + dockerComposeYaml = yamlImported.safe_load(cf) + # identify openssl binary opensslBin = 'openssl.exe' if ((pyPlatform == PLATFORM_WINDOWS) and Which('openssl.exe')) else 'openssl' diff --git a/scripts/install.py b/scripts/install.py index 50da6db33..2079aa8be 100755 --- a/scripts/install.py +++ b/scripts/install.py @@ -45,6 +45,7 @@ ################################################################################################### args = None requests_imported = None +yaml_imported = None ################################################################################################### # get interactive user response to Y/N question @@ -665,14 +666,21 @@ def tweak_malcolm_runtime( os.path.join(zeekLogDirFull, os.path.join('extract_files', 'preserved')), os.path.join(zeekLogDirFull, os.path.join('extract_files', 'quarantine')), ): - pathlib.Path(pathToCreate).mkdir(parents=True, exist_ok=True) - if ( - ((self.platform == PLATFORM_LINUX) or (self.platform == PLATFORM_MAC)) - and (self.scriptUser == "root") - and (getpwuid(os.stat(pathToCreate).st_uid).pw_name == self.scriptUser) - ): - # change ownership of newly-created directory to match puid/pgid - os.chown(pathToCreate, int(puid), int(pgid)) + try: + if args.debug: + eprint(f"Creating {pathToCreate}") + pathlib.Path(pathToCreate).mkdir(parents=True, exist_ok=True) + if ( + ((self.platform == PLATFORM_LINUX) or (self.platform == PLATFORM_MAC)) + and (self.scriptUser == "root") + and (getpwuid(os.stat(pathToCreate).st_uid).pw_name == self.scriptUser) + ): + if args.debug: + eprint(f"Setting permissions of {pathToCreate} to {puid}:{pgid}") + # change ownership of newly-created directory to match puid/pgid + os.chown(pathToCreate, int(puid), int(pgid)) + except Exception as e: + eprint(f"Creating {pathToCreate} failed: {e}") indexSnapshotCompressed = InstallerYesOrNo('Compress OpenSearch index snapshots?', default=False) @@ -1315,6 +1323,14 @@ def tweak_malcolm_runtime( sectionIndents[currentSection] * 3, ) + elif re.match(r'^\s*-.+:/zeek(:.+)?\s*$', line): + # pcap-monitor's reference to the zeek-logs directory + line = ReplaceBindMountLocation( + line, + zeekLogDir, + sectionIndents[currentSection] * 3, + ) + elif currentService == 'suricata': # stuff specifically in the suricata section if re.match(r'^\s*-.+:/data/pcap(:.+)?\s*$', line): @@ -2338,6 +2354,7 @@ def install_docker(self): def main(): global args global requests_imported + global yaml_imported # extract arguments from the command line # print (sys.argv[1:]); @@ -2463,9 +2480,11 @@ def main(): sys.tracebacklimit = 0 requests_imported = RequestsDynamic(debug=args.debug, forceInteraction=(not args.acceptDefaultsNonInteractive)) + yaml_imported = YAMLDynamic(debug=args.debug, forceInteraction=(not args.acceptDefaultsNonInteractive)) if args.debug: eprint(f"Imported requests: {requests_imported}") - if not requests_imported: + eprint(f"Imported yaml: {yaml_imported}") + if (not requests_imported) or (not yaml_imported): exit(2) # If Malcolm and images tarballs are provided, we will use them. diff --git a/scripts/malcolm_common.py b/scripts/malcolm_common.py index ffb142845..43b1eb713 100644 --- a/scripts/malcolm_common.py +++ b/scripts/malcolm_common.py @@ -14,7 +14,6 @@ import sys import time -from collections import defaultdict from enum import IntFlag, auto try: @@ -23,6 +22,14 @@ getpwuid = None from subprocess import PIPE, STDOUT, Popen, CalledProcessError + +from collections import defaultdict, namedtuple + +try: + from collections.abc import Iterable +except ImportError: + from collections import Iterable + try: from dialog import Dialog @@ -57,6 +64,12 @@ class UserInterfaceMode(IntFlag): InteractionInput = auto() +BoundPath = namedtuple( + "BoundPath", + ["service", "container_dir", "files", "relative_dirs", "clean_empty_dirs"], + rename=False, +) + # URLS for figuring things out if something goes wrong DOCKER_INSTALL_URLS = defaultdict(lambda: 'https://docs.docker.com/install/') DOCKER_INSTALL_URLS[PLATFORM_WINDOWS] = [ @@ -136,6 +149,16 @@ def UnescapeForCurl(s): ) +################################################################################################### +# if the object is an iterable, return it, otherwise return a tuple with it as a single element. +# useful if you want to user either a scalar or an array in a loop, etc. +def GetIterable(x): + if isinstance(x, Iterable) and not isinstance(x, str): + return x + else: + return (x,) + + ################################################################################################## def ReplaceBindMountLocation(line, location, linePrefix): if os.path.isdir(location): @@ -146,6 +169,24 @@ def ReplaceBindMountLocation(line, location, linePrefix): return line +################################################################################################## +def LocalPathForContainerBindMount(service, dockerComposeContents, containerPath, localBasePath=None): + localPath = None + if service and dockerComposeContents and containerPath: + vols = DeepGet(dockerComposeContents, ['services', service, 'volumes']) + if (vols is not None) and (len(vols) > 0): + for vol in vols: + volSplit = vol.split(':') + if (len(volSplit) >= 2) and (volSplit[1] == containerPath): + if localBasePath and not os.path.isabs(volSplit[0]): + localPath = os.path.realpath(os.path.join(localBasePath, volSplit[0])) + else: + localPath = volSplit[0] + break + + return localPath + + ################################################################################################### def same_file_or_dir(path1, path2): try: @@ -558,6 +599,23 @@ def LoadStrIfJson(jsonStr): return None +################################################################################################### +# safe deep get for a dictionary +# +# Example: +# d = {'meta': {'status': 'OK', 'status_code': 200}} +# DeepGet(d, ['meta', 'status_code']) # => 200 +# DeepGet(d, ['garbage', 'status_code']) # => None +# DeepGet(d, ['meta', 'garbage'], default='-') # => '-' +def DeepGet(d, keys, default=None): + assert type(keys) is list + if d is None: + return default + if not keys: + return d + return DeepGet(d.get(keys[0]), keys[1:], default) + + ################################################################################################### # run command with arguments and return its exit code, stdout, and stderr def check_output_input(*popenargs, **kwargs): @@ -696,6 +754,10 @@ def RequestsDynamic(debug=False, forceInteraction=False): return DoDynamicImport("requests", "requests", interactive=forceInteraction, debug=debug) +def YAMLDynamic(debug=False, forceInteraction=False): + return DoDynamicImport("yaml", "pyyaml", interactive=forceInteraction, debug=debug) + + ################################################################################################### # do the required auth files for Malcolm exist? def MalcolmAuthFilesExist(): diff --git a/sensor-iso/config/package-lists/python.list.chroot b/sensor-iso/config/package-lists/python.list.chroot index 4660ff230..385f990d6 100644 --- a/sensor-iso/config/package-lists/python.list.chroot +++ b/sensor-iso/config/package-lists/python.list.chroot @@ -16,5 +16,6 @@ python3-semantic-version python3-setuptools python3-tz python3-wheel +python3-yaml python3-yara python3-zmq \ No newline at end of file