Merge target/master (#19)

CHANGELOG.md

@@ -1,6 +1,37 @@
 # Changelog
 Changes to the project will be tracked in this file via the date of change.
 
+## 2022-03-02
+### Changed
+- Updating build to include `exiftool` dependency. (@cameron-dunn-sublime)
+
+## 2022-01-31
+### Changed
+- Pinned and updated all `go` build dockerfiles to `1.17.6`
+- Updated all `go mod` files to match `go` requirements.
+- Updated `numpy` dependency.
+- Updated `readme` with new client application build instructions.
+
+## 2022-01-07
+### Changed
+- Fix bug with `scan_javascript` pertaining to regular expression identification. (@cawalch)
+
+## 2021-12-27
+### Changed
+- Updating `lxml` from version `4.6.3` to `4.6.5`.
+- Updating `CAPA` from version `3.0.1` to `3.0.3`.
+- Updating `exiftool` from version `12.36` to `12.38`.
+
+## 2021-12-09
+### Changed
+- Modified `mmrpc` Dockerfile to fix compilation build issues on ARM architecture. 
+
+## 2021-11-29
+### Changed
+- Modified `exiftool` repository reference to increase stability
+- Updating `backend` dependencies
+- Updating `go` dependencies
+
 ## 2021-10-12
 ### Changed
 - Fix K8S backend configmap yaml (@cameron-dunn-sublime)

README.md

@@ -37,11 +37,10 @@ $ docker-compose -f build/docker-compose.yaml up
 #### Step 2: Build [Strelka-Fileshot](https://github.com/target/strelka/blob/master/docs/README.md#strelka-fileshot) (File Submitter)
 ```
 # Terminal 2
-$ docker build -f build/go/fileshot/Dockerfile -t strelka-fileshot .
+$ go build github.com/target/strelka/src/go/cmd/strelka-fileshot
 ```
 
-
-#### Step 3: Add File Paths / Patterns to be Scanned to [fileshot.yaml](https://github.com/target/strelka/blob/master/docs/README.md#fileshot)
+#### Step 3: Add File Paths / Patterns to be Scanned to a [fileshot.yaml] file (https://github.com/target/strelka/blob/master/docs/README.md#fileshot)
 ```
   ...
   files:
@@ -54,7 +53,7 @@ $ docker build -f build/go/fileshot/Dockerfile -t strelka-fileshot .
 #### Step 4: Run Strelka-Fileshot and Review Output
 ```
 # Terminal 2
-$ strelka-fileshot -c fileshot.yaml
+$ ./strelka-fileshot -c fileshot.yaml
 $ cat strelka.log | jq .
 ```
 
@@ -76,6 +75,10 @@ More documentation about Strelka can be found in the [README](https://target.git
 ## Contribute
 Guidelines for contributing can be found [here](https://github.com/target/strelka/blob/master/CONTRIBUTING.md).
 
+## Known Issues
+There is currently a known issue with compilation on ARM based hosts (e.g., Apple M1). Attempting to compile the current version of Strelka will lead to the following issue:
+https://github.com/target/strelka/issues/188. You can bypass this compilation issue by removing `pymupdf` from the backend Python `requriements.txt` file and commenting out ScanPDF in the `backend.yml` file. Doing this will allow you to compile the current version of Strelka at the expense of being unable to scan PDF files.
+
 ## Related Projects
 * [Laika BOSS](https://github.com/lmco/laikaboss)
 * [File Scanning Framework](https://github.com/EmersonElectricCo/fsf)

build/go/fileshot/Dockerfile

@@ -1,7 +1,7 @@
 # Strelka Fileshot
 # Client is designed to one-shot upload files and retrieve their results
 # For more information, please see: https://target.github.io/strelka/#/?id=strelka-fileshot
-FROM golang AS build
+FROM golang:1.17.6 AS build
 LABEL maintainer="Target Brands, Inc. [email protected]"
 
 # Copy source files and set the working directory

build/go/filestream/Dockerfile

@@ -1,7 +1,7 @@
 # Strelka Filestream
 # Client is designed to continuously stream files and retrieves their results.
 # For more information, please see: https://target.github.io/strelka/#/?id=strelka-filestream
-FROM golang AS build
+FROM golang:1.17.6 AS build
 LABEL maintainer="Target Brands, Inc. [email protected]"
 
 # Copy source files and set the working directory

build/go/frontend/Dockerfile

@@ -1,7 +1,7 @@
 # Strelka Frontend
 # The frontend for a cluster in which clients can connect directly via Envoy.
 # For more information, please see: https://target.github.io/strelka/#/?id=strelka-frontend
-FROM golang AS build
+FROM golang:1.17.6 AS build
 LABEL maintainer="Target Brands, Inc. [email protected]"
 
 # Copy source files and set the working directory

build/go/manager/Dockerfile

@@ -1,7 +1,7 @@
 # Strelka Manager
 # Manages portions of Strelka's Redis database.
 # For more information, please see: https://target.github.io/strelka/#/?id=strelka-manager
-FROM golang AS build
+FROM golang:1.17.6 AS build
 LABEL maintainer="Target Brands, Inc. [email protected]"
 
 # Copy source files and set the working directory

build/go/oneshot/Dockerfile

@@ -1,7 +1,7 @@
 # Strelka Oneshot
 # Client is designed to be used to submit a single file from command line and receive the result for it immediately.
 # For more information, please see: https://target.github.io/strelka/#/?id=strelka-oneshot
-FROM golang AS build
+FROM golang:1.17.6 AS build
 LABEL maintainer="Target Brands, Inc. [email protected]"
 
 # Copy source files and set the working directory

build/python/backend/Dockerfile

@@ -1,10 +1,11 @@
-FROM ubuntu:20.04
+FROM ubuntu:21.10
 ARG DEBIAN_FRONTEND=noninteractive
 LABEL maintainer="Target Brands, Inc. [email protected]"
 
-ARG YARA_VERSION=4.0.5
-ARG YARA_PYTHON_VERSION=4.0.3
-ARG CAPA_VERSION=1.1.0
+ARG YARA_VERSION=4.1.3
+ARG YARA_PYTHON_VERSION=4.1.3
+ARG CAPA_VERSION=3.0.3
+ARG EXIFTOOL_VERSION=12.38
 
 # Update packages
 RUN apt-get -qq update && \
@@ -35,11 +36,19 @@ RUN apt-get -qq update && \
     unrar \
     upx \
     jq && \
+# Download and compile Archive library, needed for exiftool to work best
+    cd /tmp/ && \
+    curl -OL https://cpan.metacpan.org/authors/id/P/PH/PHRED/Archive-Zip-1.68.tar.gz && \
+    tar -xzf Archive-Zip-1.68.tar.gz && \
+    cd Archive-Zip-1.68/ && \
+    perl Makefile.PL && \
+    make && \
+    make install && \
 # Download and compile exiftool
     cd /tmp/ && \
-    curl -OL https://exiftool.org/Image-ExifTool-12.30.tar.gz && \
-    tar -zxvf Image-ExifTool-12.30.tar.gz && \
-    cd Image-ExifTool-12.30/ && \
+    curl -OL https://github.com/exiftool/exiftool/archive/refs/tags/$EXIFTOOL_VERSION.tar.gz && \
+    tar -zxvf $EXIFTOOL_VERSION.tar.gz && \
+    cd exiftool-$EXIFTOOL_VERSION/ && \
     perl Makefile.PL && \
     make && \
     make install && \

build/python/backend/requirements.txt

@@ -6,18 +6,18 @@ cryptography==3.4.7
 docker==5.0.0
 esprima==4.0.1
 git+https://github.com/jshlbrd/python-entropy.git   # v0.11 as of this freeze (package installed as 'entropy')
-grpcio-tools==1.35.0
-grpcio==1.35.0
+grpcio==1.42.0
+grpcio-tools==1.42.0
 html5lib==1.1
 inflection==0.5.1
 interruptingcow==0.8
 jsbeautifier==1.13.13
 libarchive-c==2.9
 lief==0.11.4
-lxml==4.6.3
+lxml==4.6.5
 M2Crypto==0.37.1
 nested-lookup==0.2.22
-numpy==1.20.2
+numpy==1.21.0
 olefile==0.46
 oletools==0.56.1
 opencv-python==4.5.1.48

build/python/mmrpc/Dockerfile

@@ -6,6 +6,7 @@ LABEL maintainer="Target Brands, Inc. [email protected]"
 RUN apt-get -qq update && \
     apt-get install --no-install-recommends -qq \
 # Install build packages
+    build-essential \
     git \
     python3-dev \
     python3-pip \
@@ -28,6 +29,7 @@ RUN cd /strelka/ && \
     rm -rf dist/ strelka.egg-info && \
     pip3 uninstall -y grpcio-tools && \
     apt-get autoremove -qq --purge \
+    build-essential \
     git \
     python3-dev \
     python3-pip \

build/python/mmrpc/requirements.txt

@@ -1,4 +1,4 @@
 cryptography==3.3.2
-grpcio==1.27.2
-grpcio-tools==1.27.2
+grpcio==1.42.0
+grpcio-tools==1.42.0
 git+https://github.com/egaus/MaliciousMacroBot

docs/README.md

@@ -96,12 +96,12 @@ Please file an issue or contact the project team at [[email protected]
 Strelka can be installed on any system that can run [containers](https://www.docker.com/resources/what-container). For convenience, the project ships with [docker-compse](https://docs.docker.com/compose/) configuration files for standing up a "quickstart" cluster (found under the `build/` directory). We do not recommend using and do not plan to support OS-native installations.
 
 ### Client Install
-Strelka's core client apps are written in Go and can be run natively on a host or inside of a container.
+Strelka's core client apps are written in Go and can be run natively on a host or inside of a container. The following are multiple ways to install each of the apps.
 
-#### strelka-fileshot (gettable)
-1. Install the binary
+#### strelka-fileshot (build)
+1. Build the binary directly from github
     ```sh
-    go get github.com/target/strelka/src/go/cmd/strelka-fileshot
+    go build github.com/target/strelka/src/go/cmd/strelka-fileshot
     ```
 
 #### strelka-fileshot (build)
@@ -128,10 +128,10 @@ Strelka's core client apps are written in Go and can be run natively on a host o
     docker build -f build/go/fileshot/Dockerfile -t strelka-fileshot .
     ```
 
-#### strelka-oneshot (gettable)
-1. Install the binary
+#### strelka-oneshot (Build the binary directly from github)
+1. Build the binary
     ```sh
-    go get github.com/target/strelka/src/go/cmd/strelka-oneshot
+    go build github.com/target/strelka/src/go/cmd/strelka-oneshot
     ```
 
 #### strelka-oneshot (build)
@@ -158,10 +158,10 @@ Strelka's core client apps are written in Go and can be run natively on a host o
     docker build -f build/go/oneshot/Dockerfile -t strelka-oneshot .
     ```
 
-#### strelka-filestream (gettable)
-1. Install the binary
+#### strelka-filestream (Build the binary directly from github)
+1. Build the binary
     ```sh
-    go get github.com/target/strelka/src/go/cmd/strelka-filestream
+    go build github.com/target/strelka/src/go/cmd/strelka-filestream
     ```
 
 #### strelka-filestream (build)

go.mod

@@ -1,14 +1,24 @@
 module strelka-frontend
 
-go 1.16
+go 1.17
 
 require (
-	github.com/go-redis/redis/v8 v8.8.0
-	github.com/golang/protobuf v1.4.2
-	github.com/google/uuid v1.2.0
-	github.com/target/strelka v0.0.0-20211012121236-d9086f35d709
-	google.golang.org/grpc v1.36.0
+	github.com/go-redis/redis/v8 v8.11.4
+	github.com/golang/protobuf v1.5.2
+	github.com/google/uuid v1.3.0
+	github.com/target/strelka v0.0.0-00010101000000-000000000000
+	google.golang.org/grpc v1.44.0
 	gopkg.in/yaml.v2 v2.4.0
 )
 
-replace github.com/target/strelka => ./
+require (
+	github.com/cespare/xxhash/v2 v2.1.2 // indirect
+	github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect
+	golang.org/x/net v0.0.0-20210428140749-89ef3d95e781 // indirect
+	golang.org/x/sys v0.0.0-20210423082822-04245dca01da // indirect
+	golang.org/x/text v0.3.6 // indirect
+	google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013 // indirect
+	google.golang.org/protobuf v1.26.0 // indirect
+)
+
+replace github.com/target/strelka => ./