#175 ごく稀にログが欠損する可能性を排除できない問題を修正

osect_sensor/Infrastructure/edge_cron/work/ot_tools/bro.sh

@@ -1,10 +1,12 @@
 #!/bin/bash
 
-merge_log () {
-    cat $(find /usr/local/zeek/logs -name ${1}) > $2
+merge_and_remove_log () {
+    files=$(find /usr/local/zeek/logs -name "$1")
+    cat ${files} > $2
     sed -i '/^#/d' $2
     sed -i '1i #\n#\n#\n#\n#\n#\n#\n#' $2
     sed -i '$a #close' $2
+    rm ${files}
 }
 
 reformat_log () {
@@ -14,40 +16,39 @@ reformat_log () {
 
 cd $1/$2
 # conn_long.logと重複するためconn.logに出力されるduration>60を除外
-awk '$9<60{print}' $(find /usr/local/zeek/logs -name "conn.*.log") > "/usr/local/zeek/logs/conn_replace.log"
-rm $(find /usr/local/zeek/logs -name "conn.*.log")
+tmp_files=$(find /usr/local/zeek/logs -name "conn.*.log")
+awk '$9<60{print}' ${tmp_files} > "/usr/local/zeek/logs/conn_replace.log"
+rm ${tmp_files}
 # conn.logとconn_long.logの両方を回収
-merge_log "conn*.log" "conn.log"
-merge_log "arp.*.log" "arp.log"
-merge_log "ns.*.log" "ns.log"
-merge_log "dns.*.log" "dns.log"
-merge_log "http.*.log" "http.log"
-merge_log "cifs.*.log" "mswin-browser.log"
+merge_and_remove_log "conn*.log" "conn.log"
+merge_and_remove_log "arp.*.log" "arp.log"
+merge_and_remove_log "ns.*.log" "ns.log"
+merge_and_remove_log "dns.*.log" "dns.log"
+merge_and_remove_log "http.*.log" "http.log"
+merge_and_remove_log "cifs.*.log" "mswin-browser.log"
 reformat_log "mswin-browser.log"
-merge_log "mydhcp.*.log" "dhcp2.log"
+merge_and_remove_log "mydhcp.*.log" "dhcp2.log"
 reformat_log "dhcp2.log"
-merge_log "dhcpv6.*.log" "dhcpv6.log"
+merge_and_remove_log "dhcpv6.*.log" "dhcpv6.log"
 reformat_log "dhcpv6.log"
-merge_log "nbns.*.log" "netbios-ns.log"
+merge_and_remove_log "nbns.*.log" "netbios-ns.log"
 reformat_log "netbios-ns.log"
-merge_log "ssdp.*.log" "ssdp.log"
+merge_and_remove_log "ssdp.*.log" "ssdp.log"
 reformat_log "ssdp.log"
 # OTプロトコル: CC-Link
-merge_log "cclink-ief-basic.*.log" "cclink-ief-basic.log"
-merge_log "cclink-ie.*.log" "cclink-ie.log"
-merge_log "cclink-ie-tsn.*.log" "cclink-ie-tsn.log"
-merge_log "cclink-ie-tsn-slmp.*.log" "cclink-ie-tsn-slmp.log"
-merge_log "cclink-ie-tsn-ptp.*.log" "cclink-ie-tsn-ptp.log"
+merge_and_remove_log "cclink-ief-basic.*.log" "cclink-ief-basic.log"
+merge_and_remove_log "cclink-ie.*.log" "cclink-ie.log"
+merge_and_remove_log "cclink-ie-tsn.*.log" "cclink-ie-tsn.log"
+merge_and_remove_log "cclink-ie-tsn-slmp.*.log" "cclink-ie-tsn-slmp.log"
+merge_and_remove_log "cclink-ie-tsn-ptp.*.log" "cclink-ie-tsn-ptp.log"
 
 if [ $4 = "True" ]; then
     # tsharkでの出力と同じにするため
-    merge_log "bacnet_service.*.log" "bacnet_service.log"
+    merge_and_remove_log "bacnet_service.*.log" "bacnet_service.log"
     sed -i '/^#/d' bacnet_service.log
     sed -i '1i #' bacnet_service.log
 fi
 
 if [ $5 = "True" ]; then
-    merge_log "modbus_detailed.*.log" "modbus_detailed.log"
+    merge_and_remove_log "modbus_detailed.*.log" "modbus_detailed.log"
 fi
-
-find /usr/local/zeek/logs -name "*.log" -print0 | xargs -0 rm

osect_sensor/Infrastructure/edge_cron/work/ot_tools/p0f.sh

@@ -1,7 +1,7 @@
 #!/bin/bash
 
 cd $1/$2
-cat /var/log/p0f-k.log.* > p0f-k.log
+tmp_files=$(ls /var/log/p0f-k.log.*)
+cat ${tmp_files} > p0f-k.log
 chmod 644 p0f-k.log
-rm /var/log/p0f-k.log.*
-
+rm ${tmp_files}

osect_sensor/Infrastructure/edge_cron/work/ot_tools/suricata.sh

@@ -2,8 +2,9 @@
 
 # suricata -c $1 -r $2$3 -l $4
 
-cat /var/log/suricata/eve?*.json > $4/eve.json
-rm /var/log/suricata/eve?*.json
+tmp_files=$(ls /var/log/suricata/eve?*.json)
+cat ${tmp_files} > $4/eve.json
+rm ${tmp_files}
 
 # rm $4/fast.log
 # rm $4/stats.log

osect_sensor/Infrastructure/edge_cron/work/ot_tools/yaf.sh

@@ -1,9 +1,11 @@
 #!/bin/bash
 
-merge_log () {
-    cat $1 > $2
+merge_and_remove_log () {
+    files=$(ls $1)
+    cat ${files} > $2
     sed -i '/^#/d' $2
     sed -i '1i #ts     start-time      end-time        duration        rtt     proto   sip     sp      dip     dp srcMacAddress    destMacAddress  iflags  uflags  riflags ruflags isn     risn    tag     rtag    pktoct      rpkt    roct    end-reason' $2
+    rm ${files}
 }
 
 cd $1/$2 || exit
@@ -15,5 +17,4 @@ for flowfile in $flow; do
     rm "$flowfile" flow.csv
 done
 
-merge_log "/var/log/yaf/flow*.log" "yaf_flow.log"
-rm /var/log/yaf/flow*.log
+merge_and_remove_log "/var/log/yaf/flow*.log" "yaf_flow.log"

osect_sensor/docker-compose.sh_test.yml

@@ -0,0 +1,13 @@
+version: '3'
+services:
+  sh_test_env:
+    container_name: osect_sensor_sh_test_env
+    image: cron:revxxx
+    build: "./Infrastructure/edge_cron/"
+    volumes:
+      - ./Application/edge_cron:/opt/edge_cron
+      - ./sh_tests:/home/work/sh_tests
+      - ./Infrastructure/edge_cron/work/ot_tools/:/opt/ot_tools/:ro
+    environment:
+      - DEBUG=False
+    command: bash -c 'if [[ "$DEBUG" == "True" ]]; then sleep infinity; else bash /home/work/sh_tests/test_all.sh; fi'

osect_sensor/run_sh_test.sh

@@ -0,0 +1,7 @@
+#!/bin/bash
+
+compose_file=docker-compose.sh_test.yml
+container_name=$(cat ${compose_file} | grep -E '^\s+container_name:\s*.+' | sed -r 's/^\s+container_name:\s*//g')
+docker compose -f ${compose_file} down
+docker compose -f ${compose_file} up
+exit $(docker inspect --format='{{.State.ExitCode}}' ${container_name})

osect_sensor/sh_tests/README.md

@@ -0,0 +1,39 @@
+# sh_tests/ot_tools
+`Infrastructure/edge_cron/work/ot_tools`配下のスクリプトをテストするためのコードを格納するディレクトリ。
+
+本ディレクトリ内のファイルは、全てDocker container内で実行する前提。
+
+## ディレクトリの基本構成
+```
+OsecT/osect_sensor/sh_tests$ tree .
+.
+├── common.sh                       # テスト用の関数や変数を記述
+├── ot_tools
+│   ├── bro
+│   │   ├── bro_unfixed.sh        # 修正前のコード
+│   │   ├── data                  # テスト用データの生成、保管先ディレクトリ
+│   │   │   ├── expected
+│   │   │   └── init
+│   │   └── test.sh　　　　　　　 # 実際のテストコードを記述したスクリプト
+│   ├── p0f
+│   │   ├── data
+│   │   │   ├── expected
+│   │   │   └── init
+│   │   ├── p0f_unfixed.sh
+│   │   └── test.sh
+│   ├── suricata
+│   │   ├── data
+│   │   │   ├── expected
+│   │   │   └── init
+│   │   ├── suricata_unfixed.sh
+│   │   └── test.sh
+│   └── yaf
+│       ├── data
+│       │   ├── expected
+│       │   ├── gen_pcap.sh       # テスト用のデータ(pcap)を生成するスクリプト
+│       │   └── init
+│       │       └── udplite.pcap  # テスト用のオリジナルデータ(IPアドレスを書き換えて利用)
+│       ├── test.sh
+│       └── yaf_unfixed.sh
+└── test_all.sh                     # 各テストコードを呼び出すスクリプト
+```

osect_sensor/sh_tests/common.sh

@@ -0,0 +1,107 @@
+#!/bin/bash
+
+if [ ! -e /.dockerenv ]; then
+  echo '[ERROR] This script must be run in a Docker container'
+  exit 1
+fi
+
+DATA_DIR="data/"
+INIT_DIR="${DATA_DIR}init/"
+EXPECTED_DIR="${DATA_DIR}expected/"
+WORK_DIR="work/"
+ACTUAL_DIR="${WORK_DIR}actual/"
+test_init() {
+    local name=$1
+    echo ""
+    echo "[TEST_NAME] ${name}"
+    rm -rf ${WORK_DIR}
+    mkdir -p ${WORK_DIR}
+    cp -rp ${INIT_DIR} ${ACTUAL_DIR}
+}
+
+test_line_num() {
+    local cmd="test \$(cat ${1} | wc -l) -eq ${2}"
+    echo "[TEST_CMD] ${cmd}"
+    bash -c "${cmd}"
+    sts="$?"
+    if [ $sts -eq 0 ]; then
+        echo "<<< OK >>> ${FUNCNAME[0]}"
+        echo "----------"
+    else
+        echo "!!! NG !!! ${FUNCNAME[0]}"
+        echo "actual: $(cat ${1} | wc -l)"
+        exit 1
+    fi
+}
+
+test_fullmatch_dir() {
+    local cmd="diff -r ${ACTUAL_DIR} ${EXPECTED_DIR} $*"
+    echo "[TEST_CMD] ${cmd}"
+    bash -c "${cmd}"
+    sts="$?"
+    if [ $sts -eq 0 ]; then
+        echo "<<< OK >>> ${FUNCNAME[0]}"
+        echo "----------"
+    else
+        echo "!!! NG !!! ${FUNCNAME[0]}"
+        exit 1
+    fi
+}
+
+test_fullmatch_file() {
+    local cmd="diff -r $*"
+    echo "[TEST_CMD] ${cmd}"
+    bash -c "${cmd}"
+    sts="$?"
+    if [ $sts -eq 0 ]; then
+        echo "<<< OK >>> ${FUNCNAME[0]}"
+        echo "----------"
+    else
+        echo "!!! NG !!! ${FUNCNAME[0]}"
+        exit 1
+    fi
+}
+
+test_unmatch_file() {
+    local cmd="diff $* > /dev/null"
+    echo "[TEST_CMD] ${cmd}"
+    bash -c "${cmd}"
+    sts="$?"
+    if [ $sts -ne 0 ]; then
+        echo "<<< OK >>> ${FUNCNAME[0]}"
+        echo "----------"
+        return 0
+    fi
+    echo "!!! NG !!! ${FUNCNAME[0]}"
+    exit 1
+}
+
+test_no_empty_files() {
+    local cmd="ls $* | xargs -I{} bash -c 'cat {} | grep -cE ^.+$'"
+    echo "[TEST_CMD] ${cmd}"
+    for c in $(bash -c "${cmd}"); do
+        if [ $c -eq 0 ]; then
+            echo "<<< NG >>> ${FUNCNAME[0]}"
+            ls $* | xargs -I{} bash -c 'if [ $(cat {} | grep -cE ^.+$) -eq 0 ]; then echo "> empty file: {}"; fi'
+            exit 1
+        fi
+    done
+    echo "<<< OK >>> ${FUNCNAME[0]}"
+    echo "----------"
+    return 0
+}
+
+test_files_contain_digits() {
+    local cmd="ls $* | xargs -I{} bash -c 'cat {} | grep -cE ^[0-9]+$'"
+    echo "[TEST_CMD] ${cmd}"
+    for c in $(bash -c "${cmd}"); do
+        if [ $c -eq 0 ]; then
+            echo "<<< NG >>> ${FUNCNAME[0]}"
+            ls $* | xargs -I{} bash -c 'if [ $(cat {} | grep -cE ^[0-9]+$) -eq 0 ]; then echo "> empty file: {}"; fi'
+            exit 1
+        fi
+    done
+    echo "<<< OK >>> ${FUNCNAME[0]}"
+    echo "----------"
+    return 0
+}

osect_sensor/sh_tests/ot_tools/.gitignore

@@ -0,0 +1,3 @@
+work
+*.pcap
+!yaf/data/init/udplite.pcap

osect_sensor/sh_tests/ot_tools/bro/bro_unfixed.sh

@@ -0,0 +1,57 @@
+#!/bin/bash
+
+# ログファイルの欠損が生じる可能性があるスクリプト
+# Commit ID: 71baeab531d8170456bb20920caa62be51be8be3
+
+merge_log () {
+    # NOTE: テスト環境の都合上${1}をダブルクォートで囲んだ(Commit ID: 71baeab531d8170456bb20920caa62be51be8be3 との差分)
+    cat $(find /usr/local/zeek/logs -name "${1}") > $2
+    sed -i '/^#/d' $2
+    sed -i '1i #\n#\n#\n#\n#\n#\n#\n#' $2
+    sed -i '$a #close' $2
+}
+
+reformat_log () {
+    sed -i '/^#/d' $1
+    sed -i '1i #' $1
+}
+
+cd $1/$2
+# conn_long.logと重複するためconn.logに出力されるduration>60を除外
+awk '$9<60{print}' $(find /usr/local/zeek/logs -name "conn.*.log") > "/usr/local/zeek/logs/conn_replace.log"
+rm $(find /usr/local/zeek/logs -name "conn.*.log")
+# conn.logとconn_long.logの両方を回収
+merge_log "conn*.log" "conn.log"
+merge_log "arp.*.log" "arp.log"
+merge_log "ns.*.log" "ns.log"
+merge_log "dns.*.log" "dns.log"
+merge_log "http.*.log" "http.log"
+merge_log "cifs.*.log" "mswin-browser.log"
+reformat_log "mswin-browser.log"
+merge_log "mydhcp.*.log" "dhcp2.log"
+reformat_log "dhcp2.log"
+merge_log "dhcpv6.*.log" "dhcpv6.log"
+reformat_log "dhcpv6.log"
+merge_log "nbns.*.log" "netbios-ns.log"
+reformat_log "netbios-ns.log"
+merge_log "ssdp.*.log" "ssdp.log"
+reformat_log "ssdp.log"
+# OTプロトコル: CC-Link
+merge_log "cclink-ief-basic.*.log" "cclink-ief-basic.log"
+merge_log "cclink-ie.*.log" "cclink-ie.log"
+merge_log "cclink-ie-tsn.*.log" "cclink-ie-tsn.log"
+merge_log "cclink-ie-tsn-slmp.*.log" "cclink-ie-tsn-slmp.log"
+merge_log "cclink-ie-tsn-ptp.*.log" "cclink-ie-tsn-ptp.log"
+
+if [ $4 = "True" ]; then
+    # tsharkでの出力と同じにするため
+    merge_log "bacnet_service.*.log" "bacnet_service.log"
+    sed -i '/^#/d' bacnet_service.log
+    sed -i '1i #' bacnet_service.log
+fi
+
+if [ $5 = "True" ]; then
+    merge_log "modbus_detailed.*.log" "modbus_detailed.log"
+fi
+
+find /usr/local/zeek/logs -name "*.log" -print0 | xargs -0 rm