add: test

osect_sensor/Infrastructure/edge_cron/work/ot_tools/bro.sh

@@ -1,7 +1,7 @@
 #!/bin/bash
 
 merge_and_remove_log () {
-    files=$(find /usr/local/zeek/logs -name $1)
+    files=$(find /usr/local/zeek/logs -name "$1")
     cat ${files} > $2
     sed -i '/^#/d' $2
     sed -i '1i #\n#\n#\n#\n#\n#\n#\n#' $2

osect_sensor/docker-compose.sh_test.yml

@@ -0,0 +1,14 @@
+version: '3'
+services:
+  sh_test_env:
+    container_name: osect_sensor_sh_test_env
+    image: cron:rev2.9.0
+    build: "./Infrastructure/edge_cron/"
+    volumes:
+      - ./Application/edge_cron:/opt/edge_cron
+      - ./sh_tests:/home/work/sh_tests
+      - ./Infrastructure/edge_cron/work/ot_tools/:/opt/ot_tools/:ro
+    environment:
+      - DEBUG=False
+    network_mode: none
+    command: bash -c 'if [[ "$DEBUG" == "True" ]]; then sleep infinity; else bash /home/work/sh_tests/test_all.sh; fi'

osect_sensor/run_sh_test.sh

@@ -0,0 +1,7 @@
+#!/bin/bash
+
+compose_file=docker-compose.sh_test.yml
+container_name=$(cat ${compose_file} | grep -E '^\s+container_name:\s*.+' | sed -r 's/^\s+container_name:\s*//g')
+docker compose -f ${compose_file} down
+docker compose -f ${compose_file} up
+exit $(docker inspect --format='{{.State.ExitCode}}' ${container_name})

osect_sensor/sh_tests/common.sh

@@ -0,0 +1,107 @@
+#!/bin/bash
+
+if [ ! -e /.dockerenv ]; then
+  echo '[ERROR] This script must be run in a Docker container'
+  exit 1
+fi
+
+DATA_DIR="data/"
+INIT_DIR="${DATA_DIR}init/"
+EXPECTED_DIR="${DATA_DIR}expected/"
+WORK_DIR="work/"
+ACTUAL_DIR="${WORK_DIR}actual/"
+test_init() {
+    local name=$1
+    echo ""
+    echo "[TEST_NAME] ${name}"
+    rm -rf ${WORK_DIR}
+    mkdir -p ${WORK_DIR}
+    cp -rp ${INIT_DIR} ${ACTUAL_DIR}
+}
+
+test_line_num() {
+    local cmd="test \$(cat ${1} | wc -l) -eq ${2}"
+    echo "[TEST_CMD] ${cmd}"
+    bash -c "${cmd}"
+    sts="$?"
+    if [ $sts -eq 0 ]; then
+        echo "<<< OK >>> ${FUNCNAME[0]}"
+        echo "----------"
+    else
+        echo "!!! NG !!! ${FUNCNAME[0]}"
+        echo "actual: $(cat ${1} | wc -l)"
+        exit 1
+    fi
+}
+
+test_fullmatch_dir() {
+    local cmd="diff -r ${ACTUAL_DIR} ${EXPECTED_DIR} $*"
+    echo "[TEST_CMD] ${cmd}"
+    bash -c "${cmd}"
+    sts="$?"
+    if [ $sts -eq 0 ]; then
+        echo "<<< OK >>> ${FUNCNAME[0]}"
+        echo "----------"
+    else
+        echo "!!! NG !!! ${FUNCNAME[0]}"
+        exit 1
+    fi
+}
+
+test_fullmatch_file() {
+    local cmd="diff -r $*"
+    echo "[TEST_CMD] ${cmd}"
+    bash -c "${cmd}"
+    sts="$?"
+    if [ $sts -eq 0 ]; then
+        echo "<<< OK >>> ${FUNCNAME[0]}"
+        echo "----------"
+    else
+        echo "!!! NG !!! ${FUNCNAME[0]}"
+        exit 1
+    fi
+}
+
+test_unmatch_file() {
+    local cmd="diff $* > /dev/null"
+    echo "[TEST_CMD] ${cmd}"
+    bash -c "${cmd}"
+    sts="$?"
+    if [ $sts -ne 0 ]; then
+        echo "<<< OK >>> ${FUNCNAME[0]}"
+        echo "----------"
+        return 0
+    fi
+    echo "!!! NG !!! ${FUNCNAME[0]}"
+    exit 1
+}
+
+test_no_empty_files() {
+    local cmd="ls $* | xargs -I{} bash -c 'cat {} | grep -cE ^.+$'"
+    echo "[TEST_CMD] ${cmd}"
+    for c in $(bash -c "${cmd}"); do
+        if [ $c -eq 0 ]; then
+            echo "<<< NG >>> ${FUNCNAME[0]}"
+            ls $* | xargs -I{} bash -c 'if [ $(cat {} | grep -cE ^.+$) -eq 0 ]; then echo "> empty file: {}"; fi'
+            exit 1
+        fi
+    done
+    echo "<<< OK >>> ${FUNCNAME[0]}"
+    echo "----------"
+    return 0
+}
+
+test_files_contain_digits() {
+    local cmd="ls $* | xargs -I{} bash -c 'cat {} | grep -cE ^[0-9]+$'"
+    echo "[TEST_CMD] ${cmd}"
+    for c in $(bash -c "${cmd}"); do
+        if [ $c -eq 0 ]; then
+            echo "<<< NG >>> ${FUNCNAME[0]}"
+            ls $* | xargs -I{} bash -c 'if [ $(cat {} | grep -cE ^[0-9]+$) -eq 0 ]; then echo "> empty file: {}"; fi'
+            exit 1
+        fi
+    done
+    echo "<<< OK >>> ${FUNCNAME[0]}"
+    echo "----------"
+    return 0
+}

osect_sensor/sh_tests/ot_tools/.gitignore

@@ -0,0 +1,2 @@
+work
+*.pcap

osect_sensor/sh_tests/ot_tools/bro/bro_unfixed.sh

@@ -0,0 +1,57 @@
+#!/bin/bash
+
+# ログファイルの欠損が生じる可能性があるスクリプト
+# Commit ID: 71baeab531d8170456bb20920caa62be51be8be3
+
+merge_log () {
+    # NOTE: テスト環境の都合上${1}をダブルクォートで囲んだ(Commit ID: 71baeab531d8170456bb20920caa62be51be8be3 との差分)
+    cat $(find /usr/local/zeek/logs -name "${1}") > $2
+    sed -i '/^#/d' $2
+    sed -i '1i #\n#\n#\n#\n#\n#\n#\n#' $2
+    sed -i '$a #close' $2
+}
+
+reformat_log () {
+    sed -i '/^#/d' $1
+    sed -i '1i #' $1
+}
+
+cd $1/$2
+# conn_long.logと重複するためconn.logに出力されるduration>60を除外
+awk '$9<60{print}' $(find /usr/local/zeek/logs -name "conn.*.log") > "/usr/local/zeek/logs/conn_replace.log"
+rm $(find /usr/local/zeek/logs -name "conn.*.log")
+# conn.logとconn_long.logの両方を回収
+merge_log "conn*.log" "conn.log"
+merge_log "arp.*.log" "arp.log"
+merge_log "ns.*.log" "ns.log"
+merge_log "dns.*.log" "dns.log"
+merge_log "http.*.log" "http.log"
+merge_log "cifs.*.log" "mswin-browser.log"
+reformat_log "mswin-browser.log"
+merge_log "mydhcp.*.log" "dhcp2.log"
+reformat_log "dhcp2.log"
+merge_log "dhcpv6.*.log" "dhcpv6.log"
+reformat_log "dhcpv6.log"
+merge_log "nbns.*.log" "netbios-ns.log"
+reformat_log "netbios-ns.log"
+merge_log "ssdp.*.log" "ssdp.log"
+reformat_log "ssdp.log"
+# OTプロトコル: CC-Link
+merge_log "cclink-ief-basic.*.log" "cclink-ief-basic.log"
+merge_log "cclink-ie.*.log" "cclink-ie.log"
+merge_log "cclink-ie-tsn.*.log" "cclink-ie-tsn.log"
+merge_log "cclink-ie-tsn-slmp.*.log" "cclink-ie-tsn-slmp.log"
+merge_log "cclink-ie-tsn-ptp.*.log" "cclink-ie-tsn-ptp.log"
+
+if [ $4 = "True" ]; then
+    # tsharkでの出力と同じにするため
+    merge_log "bacnet_service.*.log" "bacnet_service.log"
+    sed -i '/^#/d' bacnet_service.log
+    sed -i '1i #' bacnet_service.log
+fi
+
+if [ $5 = "True" ]; then
+    merge_log "modbus_detailed.*.log" "modbus_detailed.log"
+fi
+
+find /usr/local/zeek/logs -name "*.log" -print0 | xargs -0 rm

osect_sensor/sh_tests/ot_tools/bro/test.sh

@@ -0,0 +1,182 @@
+#!/bin/bash
+
+source ../../common.sh
+
+target_script_path="/opt/ot_tools/bro.sh"
+export raw_log_file_dir="/usr/local/zeek/logs/"
+export merged_log_file_dir="realtime-1970-01-01-00:00:00/"
+export merged_log_file_names=$( cat << EOS
+    conn.log
+    arp.log
+    ns.log
+    dns.log
+    http.log
+    mswin-browser.log
+    dhcp2.log
+    dhcpv6.log
+    netbios-ns.log
+    ssdp.log
+    cclink-ief-basic.log
+    cclink-ie.log
+    cclink-ie-tsn.log
+    cclink-ie-tsn-slmp.log
+    cclink-ie-tsn-ptp.log
+    bacnet_service.log
+    modbus_detailed.log
+EOS
+)
+
+function merged_log_to_raw_log_name_template()
+{
+    declare -A dict=(
+            ["conn.log"]="conn*.log"
+            ["arp.log"]="arp.*.log"
+            ["ns.log"]="ns.*.log"
+            ["dns.log"]="dns.*.log"
+            ["http.log"]="http.*.log"
+            ["mswin-browser.log"]="cifs.*.log"
+            ["dhcp2.log"]="mydhcp.*.log"
+            ["dhcpv6.log"]="dhcpv6.*.log"
+            ["netbios-ns.log"]="nbns.*.log"
+            ["ssdp.log"]="ssdp.*.log"
+            ["cclink-ief-basic.log"]="cclink-ief-basic.*.log"
+            ["cclink-ie.log"]="cclink-ie.*.log"
+            ["cclink-ie-tsn.log"]="cclink-ie-tsn.*.log"
+            ["cclink-ie-tsn-slmp.log"]="cclink-ie-tsn-slmp.*.log"
+            ["cclink-ie-tsn-ptp.log"]="cclink-ie-tsn-ptp.*.log"
+            ["bacnet_service.log"]="bacnet_service.*.log"
+            ["modbus_detailed.log"]="modbus_detailed.*.log"
+    )
+    if [[ "${2}" == "." && "${1}" == "conn.log" ]]; then
+        echo "conn.*.log"
+        return 0
+    fi
+    if [ -n "${dict["${1}"]}" ]; then
+        echo "${dict["${1}"]}"
+        return 0
+    fi
+    echo ""
+    return 1
+}
+export -f merged_log_to_raw_log_name_template
+
+# gen_raw_logs <value(log_num)>
+function gen_raw_logs()
+{
+    for f in ${merged_log_file_names}; do
+        f=$(merged_log_to_raw_log_name_template ${f} .)
+        echo "echo ${1} > ${raw_log_file_dir}${f}" | \
+            sed -e "s/\*/${1}/g" | \
+            bash
+    done
+    # ls -lh ${raw_log_file_dir}
+}
+export -f gen_raw_logs
+
+### 以降、ログの欠損が生じる可能性がある問題の解消を確認するテスト ###
+# テスト条件の設定
+log_num=99
+
+## 以降、正解データの作成 ##
+rm -rf ${raw_log_file_dir}*
+expected_log_dir=${EXPECTED_DIR}${merged_log_file_dir}
+rm -rf ${expected_log_dir}
+mkdir -p ${expected_log_dir}
+for f in ${merged_log_file_names}; do
+    f=$(merged_log_to_raw_log_name_template ${f} .)
+    f=${raw_log_file_dir}$(echo ${f} | sed -e 's/\*/0/g')
+    seq -w 1 ${log_num} > ${f}
+done
+# 下記、ログ生成処理とログ結合処理が同時に実行されないため、以前のスクリプトで正解データの作成しても問題ない
+bash -c "$(pwd)/bro_unfixed.sh ${EXPECTED_DIR} ${merged_log_file_dir} PADDING True True"
+## 以上、正解データの作成 ##
+
+
+# 以下、ログ生成コマンド
+log_gen_cmd="seq -w 1 ${log_num} | xargs -I{} bash -c 'sleep 0.005; gen_raw_logs {}'"
+
+## 以降、設定した条件で問題を再現できることを確認する ##
+max_try=3
+for i in $(seq -w 1 ${max_try}); do
+    test_init "bro unfixed ${i}/${max_try}"
+    actual_log_dir=${ACTUAL_DIR}${merged_log_file_dir}
+    mkdir -p ${actual_log_dir}
+
+    # Create log files in background
+    bash -c "${log_gen_cmd}" &
+    bk_pid=$!
+
+    # Execute the target script
+    function exec_target_script()
+    {
+        bash $(pwd)/bro_unfixed.sh ${ACTUAL_DIR} ${merged_log_file_dir} PADDING True True
+        for f in ${merged_log_file_names}; do
+            # とりあえずマージする（要後処理）
+            cat ${actual_log_dir}${f} >> ${actual_log_dir}tmp.${f}.swap_merged
+        done
+    }
+    while ps -p ${bk_pid} > /dev/null; do
+        sleep 0.05
+        exec_target_script 2>/dev/null
+    done
+    sleep 0.1
+    exec_target_script 2>/dev/null  # 後処理、取り込み残しログの取り込み
+    i=0
+    for log_file_name in ${merged_log_file_names}; do
+        # 後処理準備(余計なコメント行の削除を行うため)
+        f=$(merged_log_to_raw_log_name_template ${log_file_name} . | sed -e "s/\*/$((${log_num}+1))/g")
+        cat ${actual_log_dir}tmp.${log_file_name}.swap_merged | sort > ${raw_log_file_dir}${f}
+    done
+    exec_target_script v 2>/dev/null  # 後処理
+    rm -f tmp.${actual_log_dir}*.swap_merged
+
+    for log_file_name in ${merged_log_file_names}; do
+        test_no_empty_files ${actual_log_dir}${log_file_name} ${expected_log_dir}${log_file_name}
+        test_unmatch_file ${actual_log_dir}${log_file_name} ${expected_log_dir}${log_file_name}
+    done
+done
+## 以上、設定した条件で問題を再現できることを確認する ##
+
+
+## 以降、問題の解消を確認する ##
+test_init "bro current"
+mkdir -p ${actual_log_dir}
+
+actual_log_dir=${ACTUAL_DIR}${merged_log_file_dir}
+mkdir -p ${actual_log_dir}
+
+# Create log files in background
+bash -c "${log_gen_cmd}" &
+bk_pid=$!
+
+# Execute the target script
+function exec_target_script()
+{
+    bash ${target_script_path} ${ACTUAL_DIR} ${merged_log_file_dir} PADDING True True
+    for f in ${merged_log_file_names}; do
+        # とりあえずマージする（要後処理）
+        cat ${actual_log_dir}${f} >> ${actual_log_dir}tmp.${f}.swap_merged
+    done
+}
+while ps -p ${bk_pid} > /dev/null; do
+    sleep 0.05
+    exec_target_script 2>/dev/null
+done
+sleep 0.1
+exec_target_script 2>/dev/null  # 後処理、取り込み残しログの取り込み
+i=0
+for log_file_name in ${merged_log_file_names}; do
+    # 後処理準備(余計なコメント行の削除を行うため)
+    f=$(merged_log_to_raw_log_name_template ${log_file_name} . | sed -e "s/\*/$((${log_num}+1))/g")
+    cat ${actual_log_dir}tmp.${log_file_name}.swap_merged | sort > ${raw_log_file_dir}${f}
+done
+exec_target_script v 2>/dev/null  # 後処理
+rm -f tmp.${actual_log_dir}*.swap_merged
+
+for log_file_name in ${merged_log_file_names}; do
+    test_no_empty_files ${actual_log_dir}${log_file_name} ${expected_log_dir}${log_file_name}
+    test_fullmatch_file ${actual_log_dir}${log_file_name} ${expected_log_dir}${log_file_name}
+done
+## 以上、問題の解消を確認する ##
+
+### 以上、ログの欠損が生じる可能性がある問題の解消を確認するテスト ###

osect_sensor/sh_tests/ot_tools/p0f/p0f_unfixed.sh

@@ -0,0 +1,9 @@
+#!/bin/bash
+
+# ログファイルの欠損が生じる可能性があるスクリプト
+# Commit ID: e373faa93b1b2d49e670675f17af0a44099d7831
+
+cd $1/$2
+cat $(ls /var/log/p0f-k.log.*) > p0f-k.log
+chmod 644 p0f-k.log
+rm -f $(ls /var/log/p0f-k.log.*)