Skip to content

Commit

Permalink
for idaholab#127, apply 'wipe' logic to directories that are mounted …
Browse files Browse the repository at this point in the history 
…in locations other than under Malcolm install repo. adds 'pyyaml' dependency to install/control scripts
  • Loading branch information
@mmguero
mmguero committed Dec 5, 2022
1 parent 8f25215 commit 63282c4
Show file tree
Hide file tree
Showing 5 changed files with 217 additions and 76 deletions.
3 changes: 2 additions & 1 deletion malcolm-iso/config/package-lists/python.list.chroot
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,4 +7,5 @@ python3-psutil
python3-pycryptodome
python3-dialog
python3-requests
python3-ruamel.yaml
python3-ruamel.yaml
python3-yaml
188 changes: 123 additions & 65 deletions scripts/control.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -48,6 +48,8 @@ def __exit__(self, *args):
dockerBin = None
dockerComposeBin = None
opensslBin = None
yamlImported = None
dockerComposeYaml = None

###################################################################################################
try:
Expand Down Expand Up @@ -495,6 +497,7 @@ def stop(wipe=False):
global args
global dockerBin
global dockerComposeBin
global dockerComposeYaml

# docker-compose use local temporary path
osEnv = os.environ.copy()
Expand All @@ -515,47 +518,67 @@ def stop(wipe=False):
exit(err)

if wipe:
# delete OpenSearch database
shutil.rmtree(os.path.join(MalcolmPath, os.path.join('opensearch', 'nodes')), ignore_errors=True)

# delete Zeek live-related spool files
shutil.rmtree(
os.path.join(MalcolmPath, os.path.join('zeek-logs', os.path.join('live', 'spool'))), ignore_errors=True
# there is some overlap here among some of these containers, but it doesn't matter
boundPathsToWipe = (
BoundPath("arkime", "/opt/arkime/logs", True, None, None),
BoundPath("arkime", "/opt/arkime/raw", True, None, None),
BoundPath("filebeat", "/zeek", True, None, None),
BoundPath("file-monitor", "/zeek/logs", True, None, None),
BoundPath("netbox", "/opt/netbox/netbox/media", True, None, ["."]),
BoundPath("netbox-postgres", "/var/lib/postgresql/data", True, None, ["."]),
BoundPath("netbox-redis", "/data", True, None, ["."]),
BoundPath("opensearch", "/usr/share/opensearch/data", True, ["nodes"], None),
BoundPath("pcap-capture", "/pcap", True, None, None),
BoundPath("pcap-monitor", "/pcap", True, ["processed", "upload"], None),
BoundPath("suricata", "/var/log/suricata", True, None, ["."]),
BoundPath("upload", "/var/www/upload/server/php/chroot/files", True, None, None),
BoundPath("zeek", "/zeek/extract_files", True, None, None),
BoundPath("zeek", "/zeek/upload", True, None, None),
BoundPath("zeek-live", "/zeek/live", True, ["spool"], None),
BoundPath(
"filebeat",
"/zeek",
False,
["processed", "current", "live"],
["processed", "current", "live"],
),
)

# delete data files (backups, zeek logs, arkime logs, PCAP files, captured PCAP files)
for dataDir in [
'opensearch-backup',
'zeek-logs',
'suricata-logs',
'arkime-logs',
'pcap',
'arkime-raw',
os.path.join('netbox', 'media'),
os.path.join('netbox', 'postgres'),
os.path.join('netbox', 'redis'),
]:
for root, dirnames, filenames in os.walk(os.path.join(MalcolmPath, dataDir), topdown=True, onerror=None):
for file in filenames:
fileSpec = os.path.join(root, file)
if (os.path.isfile(fileSpec) or os.path.islink(fileSpec)) and (not file.startswith('.git')):
try:
os.remove(fileSpec)
except:
pass

# clean up empty directories
for dataDir in [
os.path.join('opensearch-backup', 'logs'),
os.path.join('zeek-logs', 'processed'),
os.path.join('zeek-logs', 'current'),
os.path.join('zeek-logs', 'live'),
os.path.join('suricata-logs'),
os.path.join('netbox', 'media'),
os.path.join('netbox', 'postgres'),
os.path.join('netbox', 'redis'),
]:
RemoveEmptyFolders(dataDir, removeRoot=False)
for boundPath in boundPathsToWipe:
localPath = LocalPathForContainerBindMount(
boundPath.service,
dockerComposeYaml,
boundPath.container_dir,
MalcolmPath,
)
if localPath and os.path.isdir(localPath):
# delete files
if boundPath.files:
if args.debug:
eprint(f'Walking "{localPath}" for file deletion')
for root, dirnames, filenames in os.walk(localPath, topdown=True, onerror=None):
for file in filenames:
fileSpec = os.path.join(root, file)
if (os.path.isfile(fileSpec) or os.path.islink(fileSpec)) and (not file.startswith('.git')):
try:
os.remove(fileSpec)
except:
pass
# delete whole directories
if boundPath.relative_dirs:
for relDir in GetIterable(boundPath.relative_dirs):
tmpPath = os.path.join(localPath, relDir)
if os.path.isdir(tmpPath):
if args.debug:
eprint(f'Performing rmtree on "{tmpPath}"')
shutil.rmtree(tmpPath, ignore_errors=True)
# cleanup empty directories
if boundPath.clean_empty_dirs:
for cleanDir in GetIterable(boundPath.clean_empty_dirs):
tmpPath = os.path.join(localPath, cleanDir)
if os.path.isdir(tmpPath):
if args.debug:
eprint(f'Performing RemoveEmptyFolders on "{tmpPath}"')
RemoveEmptyFolders(tmpPath, removeRoot=False)

eprint("Malcolm has been stopped and its data cleared\n")

Expand Down Expand Up @@ -607,31 +630,54 @@ def start():
os.chmod(authFile, stat.S_IRUSR | stat.S_IWUSR)

# make sure some directories exist before we start
for path in [
os.path.join(MalcolmPath, 'opensearch'),
os.path.join(MalcolmPath, 'opensearch-backup'),
os.path.join(MalcolmPath, os.path.join('nginx', 'ca-trust')),
os.path.join(MalcolmPath, os.path.join('netbox', 'media')),
os.path.join(MalcolmPath, os.path.join('netbox', 'postgres')),
os.path.join(MalcolmPath, os.path.join('netbox', 'redis')),
os.path.join(MalcolmPath, os.path.join('pcap', 'processed')),
os.path.join(MalcolmPath, os.path.join('pcap', 'upload')),
os.path.join(MalcolmPath, os.path.join('suricata-logs', 'live')),
os.path.join(MalcolmPath, os.path.join('zeek', os.path.join('intel', 'MISP'))),
os.path.join(MalcolmPath, os.path.join('zeek', os.path.join('intel', 'STIX'))),
os.path.join(MalcolmPath, os.path.join('zeek-logs', 'current')),
os.path.join(MalcolmPath, os.path.join('zeek-logs', 'live')),
os.path.join(MalcolmPath, os.path.join('zeek-logs', 'extract_files')),
os.path.join(MalcolmPath, os.path.join('zeek-logs', 'processed')),
os.path.join(MalcolmPath, os.path.join('zeek-logs', 'upload')),
]:
try:
os.makedirs(path)
except OSError as exc:
if (exc.errno == errno.EEXIST) and os.path.isdir(path):
pass
else:
raise
boundPathsToCreate = (
BoundPath("arkime", "/opt/arkime/logs", False, None, None),
BoundPath("arkime", "/opt/arkime/raw", False, None, None),
BoundPath("file-monitor", "/zeek/logs", False, None, None),
BoundPath("nginx-proxy", "/var/local/ca-trust", False, None, None),
BoundPath("netbox", "/opt/netbox/netbox/media", False, None, None),
BoundPath("netbox-postgres", "/var/lib/postgresql/data", False, None, None),
BoundPath("netbox-redis", "/data", False, None, None),
BoundPath("opensearch", "/usr/share/opensearch/data", False, ["nodes"], None),
BoundPath("opensearch", "/opt/opensearch/backup", False, None, None),
BoundPath("pcap-capture", "/pcap", False, ["processed", "upload"], None),
BoundPath("suricata", "/var/log/suricata", False, ["live"], None),
BoundPath("upload", "/var/www/upload/server/php/chroot/files", False, None, None),
BoundPath("zeek", "/zeek/extract_files", False, None, None),
BoundPath("zeek", "/zeek/upload", False, None, None),
BoundPath("zeek", "/opt/zeek/share/zeek/site/intel", False, ["MISP", "STIX"], None),
BoundPath("zeek-live", "/zeek/live", False, ["spool"], None),
BoundPath("filebeat", "/zeek", False, ["processed", "current", "live", "extract_files", "upload"], None),
)
for boundPath in boundPathsToCreate:
localPath = LocalPathForContainerBindMount(
boundPath.service,
dockerComposeYaml,
boundPath.container_dir,
MalcolmPath,
)
if localPath:
try:
if args.debug:
eprint(f'Ensuring "{localPath}" exists')
os.makedirs(localPath)
except OSError as exc:
if (exc.errno == errno.EEXIST) and os.path.isdir(localPath):
pass
else:
raise
if boundPath.relative_dirs:
for relDir in GetIterable(boundPath.relative_dirs):
tmpPath = os.path.join(localPath, relDir)
try:
if args.debug:
eprint(f'Ensuring "{tmpPath}" exists')
os.makedirs(tmpPath)
except OSError as exc:
if (exc.errno == errno.EEXIST) and os.path.isdir(tmpPath):
pass
else:
raise

# touch the zeek intel file
open(os.path.join(MalcolmPath, os.path.join('zeek', os.path.join('intel', '__load__.zeek'))), 'a').close()
Expand Down Expand Up @@ -1208,6 +1254,8 @@ def main():
global dockerBin
global dockerComposeBin
global opensslBin
global yamlImported
global dockerComposeYaml

# extract arguments from the command line
# print (sys.argv[1:]);
Expand Down Expand Up @@ -1298,6 +1346,12 @@ def main():
else:
sys.tracebacklimit = 0

yamlImported = YAMLDynamic(debug=args.debug)
if args.debug:
eprint(f"Imported yaml: {yamlImported}")
if not yamlImported:
exit(2)

with pushd(MalcolmPath):

# don't run this as root
Expand Down Expand Up @@ -1333,6 +1387,10 @@ def main():
if err != 0:
raise Exception(f'{ScriptName} requires docker-compose, please run install.py')

# load compose file YAML (used to find some volume bind mount locations)
with open(args.composeFile, 'r') as cf:
dockerComposeYaml = yamlImported.safe_load(cf)

# identify openssl binary
opensslBin = 'openssl.exe' if ((pyPlatform == PLATFORM_WINDOWS) and Which('openssl.exe')) else 'openssl'

Expand Down
37 changes: 28 additions & 9 deletions scripts/install.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -45,6 +45,7 @@
###################################################################################################
args = None
requests_imported = None
yaml_imported = None

###################################################################################################
# get interactive user response to Y/N question
Expand Down Expand Up @@ -665,14 +666,21 @@ def tweak_malcolm_runtime(
os.path.join(zeekLogDirFull, os.path.join('extract_files', 'preserved')),
os.path.join(zeekLogDirFull, os.path.join('extract_files', 'quarantine')),
):
pathlib.Path(pathToCreate).mkdir(parents=True, exist_ok=True)
if (
((self.platform == PLATFORM_LINUX) or (self.platform == PLATFORM_MAC))
and (self.scriptUser == "root")
and (getpwuid(os.stat(pathToCreate).st_uid).pw_name == self.scriptUser)
):
# change ownership of newly-created directory to match puid/pgid
os.chown(pathToCreate, int(puid), int(pgid))
try:
if args.debug:
eprint(f"Creating {pathToCreate}")
pathlib.Path(pathToCreate).mkdir(parents=True, exist_ok=True)
if (
((self.platform == PLATFORM_LINUX) or (self.platform == PLATFORM_MAC))
and (self.scriptUser == "root")
and (getpwuid(os.stat(pathToCreate).st_uid).pw_name == self.scriptUser)
):
if args.debug:
eprint(f"Setting permissions of {pathToCreate} to {puid}:{pgid}")
# change ownership of newly-created directory to match puid/pgid
os.chown(pathToCreate, int(puid), int(pgid))
except Exception as e:
eprint(f"Creating {pathToCreate} failed: {e}")

indexSnapshotCompressed = InstallerYesOrNo('Compress OpenSearch index snapshots?', default=False)

Expand Down Expand Up @@ -1315,6 +1323,14 @@ def tweak_malcolm_runtime(
sectionIndents[currentSection] * 3,
)

elif re.match(r'^\s*-.+:/zeek(:.+)?\s*$', line):
# pcap-monitor's reference to the zeek-logs directory
line = ReplaceBindMountLocation(
line,
zeekLogDir,
sectionIndents[currentSection] * 3,
)

elif currentService == 'suricata':
# stuff specifically in the suricata section
if re.match(r'^\s*-.+:/data/pcap(:.+)?\s*$', line):
Expand Down Expand Up @@ -2338,6 +2354,7 @@ def install_docker(self):
def main():
global args
global requests_imported
global yaml_imported

# extract arguments from the command line
# print (sys.argv[1:]);
Expand Down Expand Up @@ -2463,9 +2480,11 @@ def main():
sys.tracebacklimit = 0

requests_imported = RequestsDynamic(debug=args.debug, forceInteraction=(not args.acceptDefaultsNonInteractive))
yaml_imported = YAMLDynamic(debug=args.debug, forceInteraction=(not args.acceptDefaultsNonInteractive))
if args.debug:
eprint(f"Imported requests: {requests_imported}")
if not requests_imported:
eprint(f"Imported yaml: {yaml_imported}")
if (not requests_imported) or (not yaml_imported):
exit(2)

# If Malcolm and images tarballs are provided, we will use them.
Expand Down
Loading

0 comments on commit 63282c4

Please sign in to comment.