0910-sensor-build.hook.chroot

#!/bin/bash

# Copyright (c) 2024 Battelle Energy Alliance, LLC.  All rights reserved.

# some environment variables needed for build
export CCACHE_DIR="/var/spool/ccache"
export CCACHE_COMPRESS=1
export PYTHONDONTWRITEBYTECODE=1
export PYTHONUNBUFFERED=1
ARCH=$(dpkg --print-architecture)

cat > /etc/environment << EOF
PYTHONDONTWRITEBYTECODE=1
PYTHONUNBUFFERED=1
EOF

# we'll use cURL to get release information and assets from GitHub using the GitHub API
GITHUB_API_CURL_ARGS=()
GITHUB_API_CURL_ARGS+=( -fsSL )
GITHUB_API_CURL_ARGS+=( -H )
GITHUB_API_CURL_ARGS+=( "Accept: application/vnd.github.v3+json" )
[[ -n "$GITHUB_TOKEN" ]] && GITHUB_API_CURL_ARGS+=( -H ) && GITHUB_API_CURL_ARGS+=( "Authorization: token $GITHUB_TOKEN" )

SURICATA_RULES_DIR="/etc/suricata/rules"

CAPA_RELEASE_URL="https://api.github.com/repos/fireeye/capa/releases/latest"
YQ_RELEASE_URL="https://api.github.com/repos/mikefarah/yq/releases/latest"
SUPERCRONIC_RELEASE_URL="https://api.github.com/repos/aptible/supercronic/releases/latest"
CROC_RELEASE_URL="https://api.github.com/repos/schollz/croc/releases/latest"
YARA_RULES_SRC_DIR="/opt/yara-rules-src"
YARA_RULES_DIR="/opt/yara-rules"

mkdir -p /opt/hedgehog_install_artifacts/

# Download and install spicy and 3rd-party zeek plugins
ZEEK_DIR="/opt/zeek"
export PATH="${ZEEK_DIR}"/bin:$PATH
cd /tmp
zkg autoconfig --force
bash /usr/local/bin/zeek_install_plugins.sh
for FILE in /opt/zeek/var/lib/zkg/logs/*; do
  [[ -r "$FILE" ]] && echo "$FILE" && cat "$FILE" && echo ""
done

# cleanup/strip build artifacts
find "${ZEEK_DIR}"/lib "${ZEEK_DIR}"/var/lib/zkg \( -path "*/build/*" -o -path "*/CMakeFiles/*" \) -type f -name "*.*" -print0 | xargs -0 -I XXX bash -c 'file "XXX" | sed "s/^.*:[[:space:]]//" | grep -Pq "(ELF|gzip)" && rm -f "XXX"'
find "${ZEEK_DIR}"/var/lib/zkg/clones -type d -name .git -execdir bash -c "pwd; du -sh; git pull --depth=1 --ff-only; git reflog expire --expire=all --all; git tag -l | xargs -r git tag -d; git gc --prune=all; du -sh" \;
rm -rf "${ZEEK_DIR}"/var/lib/zkg/scratch
rm -rf "${ZEEK_DIR}"/lib/zeek/python/zeekpkg/__pycache__
find "${ZEEK_DIR}/" -type f -exec file "{}" \; | grep -Pi "ELF 64-bit.*not stripped" | sed 's/:.*//' | xargs -l -r strip --strip-unneeded

cat << 'EOF' >> "${ZEEK_DIR}"/etc/zeekctl.cfg
# all interfaces using lb_method=custom should use AF_PACKET
lb_custom.InterfacePrefix=af_packet::

EOF

# set up default zeek local policy and sensor-related directories
cp -f /usr/local/etc/zeek/*.zeek /usr/local/etc/zeek/*.txt "${ZEEK_DIR}"/share/zeek/site/
mkdir -p /opt/sensor/sensor_ctl/zeek/custom /opt/sensor/sensor_ctl/zeek/intel/STIX /opt/sensor/sensor_ctl/zeek/intel/MISP /opt/sensor/sensor_ctl/fluentbit
touch /opt/sensor/sensor_ctl/zeek/intel/__load__.zeek
touch /opt/sensor/sensor_ctl/zeek/custom/__load__.zeek
[[ -f /usr/local/bin/zeek_intel_setup.sh ]] && mv /usr/local/bin/zeek_intel_setup.sh "${ZEEK_DIR}"/bin/
[[ -f /usr/local/bin/zeek_intel_from_threat_feed.py ]] && mv /usr/local/bin/zeek*threat*.py "${ZEEK_DIR}"/bin/

# cleanup
cd /usr/local/src
tar czf zeek-$ZEEK_VER-hedgehog.tar.gz -C /opt zeek spicy
mv ./zeek-$ZEEK_VER-hedgehog.tar.gz /opt/hedgehog_install_artifacts/
rm -Rf zeek-$ZEEK_VER*
###

if [[ ! "${ARCH,,}" =~ ^arm ]]; then
	# tweak some stuff for aide
	chmod a-x /etc/cron.daily/aide
	chattr +i /etc/cron.daily/aide
	mkdir -p /etc/aide/aide.conf.d /var/lib/aide
	touch /var/lib/aide/aide.db
	chmod 600 /var/lib/aide/aide.db
	sed -r -i "s/(Checksums\s*=\s*).*/\1 sha512/" /etc/aide/aide.conf
	cat << 'EOF' >> /etc/aide/aide.conf.d/00_local_excludes
/etc/at\.allow$ f Full-n-c-m
/etc/clamav/.+\.conf$ f VarFile-n-c-m
/etc/clamav$ d VarDir-n-c-m
/etc/cron\.allow$ f Full-n-c-m
/etc/cron\.\w+$ d VarDir-n-c-m
/etc/crontab$ f VarFile-n-c-m
/etc/group-?$ f VarFile-n-c-m
/etc/gshadow-?$ f VarFile-n-c-m
/etc/hostname$ f VarFile-n-c-m
/etc/hosts$ f VarFile-n-c-m
/etc/installer-?$ f Full-n-c-m
/etc/ntp\.conf$ f VarFile-n-c-m
/etc/passwd-?$ f VarFile-n-c-m
/etc/rc\d*\.d/.+$ l VarInode
/etc/rc\d*\.d$ d VarDir-n-c-m
/etc/shadow-?$ f VarFile-n-c-m
/etc/systemd/system/multi-user\.target\.wants/.+$ l VarInode
/etc/systemd/system/multi-user\.target\.wants(/|$) d VarDir-n-c-m
/etc/ufw/.+\.rules$ f VarFile-n-c-m
/etc/ufw$ d VarDir-n-c-m
/opt$ d VarDir-n-c-m
/usr/local/bin$ d VarDirTime-n
/var/lib/aide/aide\.db(\.new)?$ f VarFile-n-c-m-i
/var/lib/aide$ d VarDirInode-n-c-m
/var/lib/samba$ d VarDir-n-c-m

!/(.+/)?__pycache__(/|$)
!/capture(/|$)
!/dev(/|$)
!/etc/.*\.bak$
!/etc/aide/.*\.swp$
!/etc/cron\.d/htpdate$
!/etc/ld\.so\.cache$
!/etc/lvm/archive(/|$)
!/etc/lvm/backup(/|$)
!/etc/network/interfaces.d(/|$)
!/etc/NetworkManager/system-connections(/|$)
!/etc/rc\d*\.d/.*ntp$
!/etc/suricata/rules(/|$)
!/etc/systemd/system/multi-user\.target\.wants/ntp.service$
!/home(/|$)
!/opt/harbianaudit(/|$)
!/opt/sensor(/|$)
!/opt/yara-rules(/|$)
!/opt/zeek(/|$)
!/root(/|$)
!/run(/|$)
!/tmp(/|$)
!/usr/local/lib/python.*/dist-packages/debinterface(/|$)
!/var/backups(/|$)
!/var/cache(/|$)
!/var/lib/AccountsService(/|$)
!/var/lib/apt/daily_lock$
!/var/lib/apt/lists(/|$)
!/var/lib/clamav(/|$)
!/var/lib/colord(/|$)
!/var/lib/containerd(/|$)
!/var/lib/dhcp(/|$)
!/var/lib/docker(/|$)
!/var/lib/dpkg/info(/|$)
!/var/lib/dpkg/triggers/Lock$
!/var/lib/lightdm(/|$)
!/var/lib/logrotate$
!/var/lib/NetworkManager(/|$)
!/var/lib/plymouth(/|$)
!/var/lib/samba(/|$)
!/var/lib/sudo/lectured(/|$)
!/var/lib/suricata(/|$)
!/var/lib/systemd-coredump$
!/var/log(/|$)
!/var/mail(/|$)
!/var/run(/|$)
!/var/spool(/|$)
!/var/tmp(/|$)

EOF
fi
###

# update suricata rules
/usr/bin/suricata-update -v -D /var/lib/suricata --etopen

cd "${SURICATA_RULES_DIR}"/..
tar czf suricata-rules-hedgehog.tar.gz "$(basename "${SURICATA_RULES_DIR}")"
mv ./suricata-rules-hedgehog.tar.gz /opt/hedgehog_install_artifacts/
###

# update yara rules

mkdir -p "${YARA_RULES_DIR}"/custom "${YARA_RULES_SRC_DIR}"
/usr/local/bin/yara_rules_setup.sh -r "${YARA_RULES_SRC_DIR}" -y "${YARA_RULES_DIR}"

cd "${YARA_RULES_DIR}"/..
tar czf yara-rules-hedgehog.tar.gz "$(basename "${YARA_RULES_DIR}")"
mv ./yara-rules-hedgehog.tar.gz /opt/hedgehog_install_artifacts/

cd "${YARA_RULES_SRC_DIR}"/..
tar czf yara-rules-src-hedgehog.tar.gz "$(basename "${YARA_RULES_SRC_DIR}")"
mv ./yara-rules-src-hedgehog.tar.gz /opt/hedgehog_install_artifacts/
###

# capa
cd /tmp

capa_assets_url="$(curl "${GITHUB_API_CURL_ARGS[@]}" "$CAPA_RELEASE_URL" | jq '.assets_url' | tr -d '"')"

if [[ "${ARCH,,}" =~ ^arm ]]; then
	#Build from source for ARM...
	#Not sure if there is an easier way to get the latest release tag
	capa_latest_ver=$(curl "${GITHUB_API_CURL_ARGS[@]}" "$capa_assets_url" | jq ".[] | select(.name | contains(\"-linux.zip\")) | .name")
	# Retrieves strings like "capa-v6.1.0-linux.zip"; below trims out the x.x.x
	capa_latest_ver=$(echo ${capa_latest_ver#*v})
	capa_latest_ver=$(echo ${capa_latest_ver%%-*})
	capa_latest_src_url="https://github.com/mandiant/capa/archive/refs/tags/v${capa_latest_ver}.zip"

	python3 -m venv capa
	source capa/bin/activate
	cd capa

	curl "${GITHUB_API_CURL_ARGS[@]}" "${capa_latest_src_url}" -o capa.zip
	unzip -q capa.zip

	cd capa-${capa_latest_ver}
	python3 -m pip install -e .[build]
	python scripts/cache-ruleset.py rules/ cache/
	pyinstaller .github/pyinstaller/pyinstaller.spec
	mv dist/capa /usr/local/bin/capa

	deactivate

else
	# Assume 64-bit Linux otherwise
	capa_zip_url=$(curl "${GITHUB_API_CURL_ARGS[@]}" "$capa_assets_url" | jq ".[] | select(.browser_download_url | contains(\"-linux.zip\")) | .browser_download_url" | tr -d '"')
	curl -o capa.zip "${GITHUB_API_CURL_ARGS[@]}" "${capa_zip_url}"
	unzip ./capa.zip
	mv ./capa /usr/local/bin/capa

fi

chmod 755 /usr/local/bin/capa
rm -rf /tmp/capa*

cp /usr/local/bin/capa /opt/hedgehog_install_artifacts/
###

# yq
cd /tmp
yq_assets_url="$(curl "${GITHUB_API_CURL_ARGS[@]}" "$YQ_RELEASE_URL" | jq '.assets_url' | tr -d '"')"
yq_tar_url="$(curl "${GITHUB_API_CURL_ARGS[@]}" "$yq_assets_url" | jq ".[] | select(.browser_download_url | contains(\"yq_linux_${ARCH}.tar.gz\")) | .browser_download_url" | tr -d '"')"
curl "${GITHUB_API_CURL_ARGS[@]}" "${yq_tar_url}" | tar -xzf - ./yq_linux_${ARCH}

mv ./yq_linux_${ARCH} /usr/bin/yq
chmod 755 /usr/bin/yq
###

# supercronic
supercronic_assets_url="$(curl "${GITHUB_API_CURL_ARGS[@]}" "$SUPERCRONIC_RELEASE_URL" | jq '.assets_url' | tr -d '"')"
supercronic_bin_url="$(curl "${GITHUB_API_CURL_ARGS[@]}" "$supercronic_assets_url" | jq ".[] | select(.browser_download_url | contains(\"supercronic-linux-${ARCH}\")) | .browser_download_url" | tr -d '"')"
curl -o /usr/local/bin/supercronic "${GITHUB_API_CURL_ARGS[@]}" "$supercronic_bin_url"

chmod 755 /usr/local/bin/supercronic
###

# croc
cd /tmp
[[ $ARCH =~ ^arm ]] && CROC_ASSET_ARCH="${ARCH^^}" || CROC_ASSET_ARCH=64bit
croc_assets_url="$(curl "${GITHUB_API_CURL_ARGS[@]}" "$CROC_RELEASE_URL" | jq '.assets_url' | tr -d '"')"
croc_tar_url="$(curl "${GITHUB_API_CURL_ARGS[@]}" "$croc_assets_url" | jq ".[] | select(.browser_download_url | contains(\"_Linux-${CROC_ASSET_ARCH}.tar.gz\")) | .browser_download_url" | tr -d '"')"
curl "${GITHUB_API_CURL_ARGS[@]}" "${croc_tar_url}" | tar -xzf - croc

mv ./croc /usr/local/bin/croc
chmod 755 /usr/local/bin/croc
###

# update clamav signatures
freshclam --stdout --quiet --no-warnings

###

# set up capabilities for network-related tools
chown root:netdev /usr/sbin/netsniff-ng && \
  setcap 'CAP_NET_RAW+eip CAP_NET_ADMIN+eip CAP_IPC_LOCK+eip CAP_SYS_ADMIN+eip' /usr/sbin/netsniff-ng
chown root:netdev "${ZEEK_DIR}"/bin/zeek && \
  setcap 'CAP_NET_RAW+eip CAP_NET_ADMIN+eip' "${ZEEK_DIR}"/bin/zeek
chown root:netdev /sbin/ethtool && \
  setcap 'CAP_NET_RAW+eip CAP_NET_ADMIN+eip' /sbin/ethtool
chown root:netdev "${ZEEK_DIR}"/bin/capstats && \
  setcap 'CAP_NET_RAW+eip CAP_NET_ADMIN+eip' "${ZEEK_DIR}"/bin/capstats
chown root:netdev /usr/bin/tcpdump && \
  setcap 'CAP_NET_RAW+eip CAP_NET_ADMIN+eip' /usr/bin/tcpdump
chown root:netdev /usr/bin/suricata && \
  setcap 'CAP_NET_RAW+eip CAP_NET_ADMIN+eip CAP_IPC_LOCK+eip' /usr/bin/suricata
chown root:netdev /opt/arkime/bin/capture && \
  setcap 'CAP_NET_RAW+eip CAP_NET_ADMIN+eip' /opt/arkime/bin/capture

ln -s -f "${ZEEK_DIR}"/bin/zeek /usr/local/bin/
ln -s -f /usr/sbin/netsniff-ng /usr/local/bin/
ln -s -f /usr/bin/tcpdump /usr/local/bin/
ln -s -f /opt/arkime/bin/capture /usr/local/bin/
ln -s -f /opt/arkime/bin/npm /usr/local/bin
ln -s -f /opt/arkime/bin/node /usr/local/bin
ln -s -f /opt/arkime/bin/npx /usr/local/bin
###

# zeekctl won't like being run by a non-root user unless the whole stupid thing is owned by the non-root user.
# as such, we're going to reset zeek to a "clean" state after each reboot. the config files will get
# regenerated when we are about to deploy zeek itself.
# so, now make a complete backup of /opt/zeek as it's going to have its ownership changed during runtime
# TODO: check this with .deb install
rsync -a "${ZEEK_DIR}"/ "${ZEEK_DIR}".orig

# enable sensor init scripts

# run kiosk system process information daemon under systemd
if [ -f /opt/sensor/kiosk.service ]; then
  mv /opt/sensor/kiosk.service /etc/systemd/system/
  chown root:root /etc/systemd/system/kiosk.service
  chmod 644 /etc/systemd/system/kiosk.service
  systemctl daemon-reload
  systemctl enable kiosk
fi

# shutdown sensor cleanly on reboot/halt
if [ -f /etc/systemd/system/sensor-shutdown.service ]; then
  chown root:root /etc/systemd/system/sensor-shutdown.service
  chmod 644 /etc/systemd/system/sensor-shutdown.service
  systemctl daemon-reload
  systemctl enable sensor-shutdown
fi

# run resource logging services as unprivileged user.
# supervisor is run via /etc/rc.local  (vs. here with a @reboot directive)
#   so as to work for both live-usb and installed versions
#   (see 0900-setup-rc-local.hook.chroot)
if [ -d /opt/sensor/sensor_ctl/scripts ]; then
  cat << 'EOF' >> /etc/cron.d/sensor_ctl
SHELL=/bin/bash
PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin

2 0 * * 0 root rm -f /var/lib/clamav/mirrors.dat
EOF
fi

###