From 383b97f95f9e49e1a925549ec6baee7ce13468d9 Mon Sep 17 00:00:00 2001 From: Cameron Dunn <88673576+cameron-dunn-sublime@users.noreply.github.com> Date: Mon, 7 Mar 2022 13:36:53 -0800 Subject: [PATCH] Merge target/master (#19) --- CHANGELOG.md | 31 +++++++ README.md | 11 ++- build/go/fileshot/Dockerfile | 2 +- build/go/filestream/Dockerfile | 2 +- build/go/frontend/Dockerfile | 2 +- build/go/manager/Dockerfile | 2 +- build/go/oneshot/Dockerfile | 2 +- build/python/backend/Dockerfile | 23 ++++-- build/python/backend/requirements.txt | 8 +- build/python/mmrpc/Dockerfile | 2 + build/python/mmrpc/requirements.txt | 4 +- docs/README.md | 20 ++--- go.mod | 24 ++++-- go.sum | 81 +++++++++++++------ .../strelka/scanners/scan_javascript.py | 2 +- 15 files changed, 153 insertions(+), 63 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7006bf72..c732b8c0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,37 @@ # Changelog Changes to the project will be tracked in this file via the date of change. +## 2022-03-02 +### Changed +- Updating build to include `exiftool` dependency. (@cameron-dunn-sublime) + +## 2022-01-31 +### Changed +- Pinned and updated all `go` build dockerfiles to `1.17.6` +- Updated all `go mod` files to match `go` requirements. +- Updated `numpy` dependency. +- Updated `readme` with new client application build instructions. + +## 2022-01-07 +### Changed +- Fix bug with `scan_javascript` pertaining to regular expression identification. (@cawalch) + +## 2021-12-27 +### Changed +- Updating `lxml` from version `4.6.3` to `4.6.5`. +- Updating `CAPA` from version `3.0.1` to `3.0.3`. +- Updating `exiftool` from version `12.36` to `12.38`. + +## 2021-12-09 +### Changed +- Modified `mmrpc` Dockerfile to fix compilation build issues on ARM architecture. + +## 2021-11-29 +### Changed +- Modified `exiftool` repository reference to increase stability +- Updating `backend` dependencies +- Updating `go` dependencies + ## 2021-10-12 ### Changed - Fix K8S backend configmap yaml (@cameron-dunn-sublime) diff --git a/README.md b/README.md index b08b405e..bc926180 100644 --- a/README.md +++ b/README.md @@ -37,11 +37,10 @@ $ docker-compose -f build/docker-compose.yaml up #### Step 2: Build [Strelka-Fileshot](https://github.com/target/strelka/blob/master/docs/README.md#strelka-fileshot) (File Submitter) ``` # Terminal 2 -$ docker build -f build/go/fileshot/Dockerfile -t strelka-fileshot . +$ go build github.com/target/strelka/src/go/cmd/strelka-fileshot ``` - -#### Step 3: Add File Paths / Patterns to be Scanned to [fileshot.yaml](https://github.com/target/strelka/blob/master/docs/README.md#fileshot) +#### Step 3: Add File Paths / Patterns to be Scanned to a [fileshot.yaml] file (https://github.com/target/strelka/blob/master/docs/README.md#fileshot) ``` ... files: @@ -54,7 +53,7 @@ $ docker build -f build/go/fileshot/Dockerfile -t strelka-fileshot . #### Step 4: Run Strelka-Fileshot and Review Output ``` # Terminal 2 -$ strelka-fileshot -c fileshot.yaml +$ ./strelka-fileshot -c fileshot.yaml $ cat strelka.log | jq . ``` @@ -76,6 +75,10 @@ More documentation about Strelka can be found in the [README](https://target.git ## Contribute Guidelines for contributing can be found [here](https://github.com/target/strelka/blob/master/CONTRIBUTING.md). +## Known Issues +There is currently a known issue with compilation on ARM based hosts (e.g., Apple M1). Attempting to compile the current version of Strelka will lead to the following issue: +https://github.com/target/strelka/issues/188. You can bypass this compilation issue by removing `pymupdf` from the backend Python `requriements.txt` file and commenting out ScanPDF in the `backend.yml` file. Doing this will allow you to compile the current version of Strelka at the expense of being unable to scan PDF files. + ## Related Projects * [Laika BOSS](https://github.com/lmco/laikaboss) * [File Scanning Framework](https://github.com/EmersonElectricCo/fsf) diff --git a/build/go/fileshot/Dockerfile b/build/go/fileshot/Dockerfile index 423a2e20..96e6e20c 100644 --- a/build/go/fileshot/Dockerfile +++ b/build/go/fileshot/Dockerfile @@ -1,7 +1,7 @@ # Strelka Fileshot # Client is designed to one-shot upload files and retrieve their results # For more information, please see: https://target.github.io/strelka/#/?id=strelka-fileshot -FROM golang AS build +FROM golang:1.17.6 AS build LABEL maintainer="Target Brands, Inc. TTS-CFC-OpenSource@target.com" # Copy source files and set the working directory diff --git a/build/go/filestream/Dockerfile b/build/go/filestream/Dockerfile index 31fd6386..70116ab9 100644 --- a/build/go/filestream/Dockerfile +++ b/build/go/filestream/Dockerfile @@ -1,7 +1,7 @@ # Strelka Filestream # Client is designed to continuously stream files and retrieves their results. # For more information, please see: https://target.github.io/strelka/#/?id=strelka-filestream -FROM golang AS build +FROM golang:1.17.6 AS build LABEL maintainer="Target Brands, Inc. TTS-CFC-OpenSource@target.com" # Copy source files and set the working directory diff --git a/build/go/frontend/Dockerfile b/build/go/frontend/Dockerfile index d4157076..fef8e0f7 100644 --- a/build/go/frontend/Dockerfile +++ b/build/go/frontend/Dockerfile @@ -1,7 +1,7 @@ # Strelka Frontend # The frontend for a cluster in which clients can connect directly via Envoy. # For more information, please see: https://target.github.io/strelka/#/?id=strelka-frontend -FROM golang AS build +FROM golang:1.17.6 AS build LABEL maintainer="Target Brands, Inc. TTS-CFC-OpenSource@target.com" # Copy source files and set the working directory diff --git a/build/go/manager/Dockerfile b/build/go/manager/Dockerfile index 95a96cb7..e63623ab 100644 --- a/build/go/manager/Dockerfile +++ b/build/go/manager/Dockerfile @@ -1,7 +1,7 @@ # Strelka Manager # Manages portions of Strelka's Redis database. # For more information, please see: https://target.github.io/strelka/#/?id=strelka-manager -FROM golang AS build +FROM golang:1.17.6 AS build LABEL maintainer="Target Brands, Inc. TTS-CFC-OpenSource@target.com" # Copy source files and set the working directory diff --git a/build/go/oneshot/Dockerfile b/build/go/oneshot/Dockerfile index 5bf19567..abfacac7 100644 --- a/build/go/oneshot/Dockerfile +++ b/build/go/oneshot/Dockerfile @@ -1,7 +1,7 @@ # Strelka Oneshot # Client is designed to be used to submit a single file from command line and receive the result for it immediately. # For more information, please see: https://target.github.io/strelka/#/?id=strelka-oneshot -FROM golang AS build +FROM golang:1.17.6 AS build LABEL maintainer="Target Brands, Inc. TTS-CFC-OpenSource@target.com" # Copy source files and set the working directory diff --git a/build/python/backend/Dockerfile b/build/python/backend/Dockerfile index dfb5d374..93dbb834 100644 --- a/build/python/backend/Dockerfile +++ b/build/python/backend/Dockerfile @@ -1,10 +1,11 @@ -FROM ubuntu:20.04 +FROM ubuntu:21.10 ARG DEBIAN_FRONTEND=noninteractive LABEL maintainer="Target Brands, Inc. TTS-CFC-OpenSource@target.com" -ARG YARA_VERSION=4.0.5 -ARG YARA_PYTHON_VERSION=4.0.3 -ARG CAPA_VERSION=1.1.0 +ARG YARA_VERSION=4.1.3 +ARG YARA_PYTHON_VERSION=4.1.3 +ARG CAPA_VERSION=3.0.3 +ARG EXIFTOOL_VERSION=12.38 # Update packages RUN apt-get -qq update && \ @@ -35,11 +36,19 @@ RUN apt-get -qq update && \ unrar \ upx \ jq && \ +# Download and compile Archive library, needed for exiftool to work best + cd /tmp/ && \ + curl -OL https://cpan.metacpan.org/authors/id/P/PH/PHRED/Archive-Zip-1.68.tar.gz && \ + tar -xzf Archive-Zip-1.68.tar.gz && \ + cd Archive-Zip-1.68/ && \ + perl Makefile.PL && \ + make && \ + make install && \ # Download and compile exiftool cd /tmp/ && \ - curl -OL https://exiftool.org/Image-ExifTool-12.30.tar.gz && \ - tar -zxvf Image-ExifTool-12.30.tar.gz && \ - cd Image-ExifTool-12.30/ && \ + curl -OL https://github.com/exiftool/exiftool/archive/refs/tags/$EXIFTOOL_VERSION.tar.gz && \ + tar -zxvf $EXIFTOOL_VERSION.tar.gz && \ + cd exiftool-$EXIFTOOL_VERSION/ && \ perl Makefile.PL && \ make && \ make install && \ diff --git a/build/python/backend/requirements.txt b/build/python/backend/requirements.txt index 58d3fb77..1fc116f8 100644 --- a/build/python/backend/requirements.txt +++ b/build/python/backend/requirements.txt @@ -6,18 +6,18 @@ cryptography==3.4.7 docker==5.0.0 esprima==4.0.1 git+https://github.com/jshlbrd/python-entropy.git # v0.11 as of this freeze (package installed as 'entropy') -grpcio-tools==1.35.0 -grpcio==1.35.0 +grpcio==1.42.0 +grpcio-tools==1.42.0 html5lib==1.1 inflection==0.5.1 interruptingcow==0.8 jsbeautifier==1.13.13 libarchive-c==2.9 lief==0.11.4 -lxml==4.6.3 +lxml==4.6.5 M2Crypto==0.37.1 nested-lookup==0.2.22 -numpy==1.20.2 +numpy==1.21.0 olefile==0.46 oletools==0.56.1 opencv-python==4.5.1.48 diff --git a/build/python/mmrpc/Dockerfile b/build/python/mmrpc/Dockerfile index db039760..90084ddb 100644 --- a/build/python/mmrpc/Dockerfile +++ b/build/python/mmrpc/Dockerfile @@ -6,6 +6,7 @@ LABEL maintainer="Target Brands, Inc. TTS-CFC-OpenSource@target.com" RUN apt-get -qq update && \ apt-get install --no-install-recommends -qq \ # Install build packages + build-essential \ git \ python3-dev \ python3-pip \ @@ -28,6 +29,7 @@ RUN cd /strelka/ && \ rm -rf dist/ strelka.egg-info && \ pip3 uninstall -y grpcio-tools && \ apt-get autoremove -qq --purge \ + build-essential \ git \ python3-dev \ python3-pip \ diff --git a/build/python/mmrpc/requirements.txt b/build/python/mmrpc/requirements.txt index bcf245b1..243a35da 100644 --- a/build/python/mmrpc/requirements.txt +++ b/build/python/mmrpc/requirements.txt @@ -1,4 +1,4 @@ cryptography==3.3.2 -grpcio==1.27.2 -grpcio-tools==1.27.2 +grpcio==1.42.0 +grpcio-tools==1.42.0 git+https://github.com/egaus/MaliciousMacroBot diff --git a/docs/README.md b/docs/README.md index 1dee420b..992486b9 100644 --- a/docs/README.md +++ b/docs/README.md @@ -96,12 +96,12 @@ Please file an issue or contact the project team at [TTS-CFC-OpenSource@target.c Strelka can be installed on any system that can run [containers](https://www.docker.com/resources/what-container). For convenience, the project ships with [docker-compse](https://docs.docker.com/compose/) configuration files for standing up a "quickstart" cluster (found under the `build/` directory). We do not recommend using and do not plan to support OS-native installations. ### Client Install -Strelka's core client apps are written in Go and can be run natively on a host or inside of a container. +Strelka's core client apps are written in Go and can be run natively on a host or inside of a container. The following are multiple ways to install each of the apps. -#### strelka-fileshot (gettable) -1. Install the binary +#### strelka-fileshot (build) +1. Build the binary directly from github ```sh - go get github.com/target/strelka/src/go/cmd/strelka-fileshot + go build github.com/target/strelka/src/go/cmd/strelka-fileshot ``` #### strelka-fileshot (build) @@ -128,10 +128,10 @@ Strelka's core client apps are written in Go and can be run natively on a host o docker build -f build/go/fileshot/Dockerfile -t strelka-fileshot . ``` -#### strelka-oneshot (gettable) -1. Install the binary +#### strelka-oneshot (Build the binary directly from github) +1. Build the binary ```sh - go get github.com/target/strelka/src/go/cmd/strelka-oneshot + go build github.com/target/strelka/src/go/cmd/strelka-oneshot ``` #### strelka-oneshot (build) @@ -158,10 +158,10 @@ Strelka's core client apps are written in Go and can be run natively on a host o docker build -f build/go/oneshot/Dockerfile -t strelka-oneshot . ``` -#### strelka-filestream (gettable) -1. Install the binary +#### strelka-filestream (Build the binary directly from github) +1. Build the binary ```sh - go get github.com/target/strelka/src/go/cmd/strelka-filestream + go build github.com/target/strelka/src/go/cmd/strelka-filestream ``` #### strelka-filestream (build) diff --git a/go.mod b/go.mod index 8a266a0d..f724a2ad 100644 --- a/go.mod +++ b/go.mod @@ -1,14 +1,24 @@ module strelka-frontend -go 1.16 +go 1.17 require ( - github.com/go-redis/redis/v8 v8.8.0 - github.com/golang/protobuf v1.4.2 - github.com/google/uuid v1.2.0 - github.com/target/strelka v0.0.0-20211012121236-d9086f35d709 - google.golang.org/grpc v1.36.0 + github.com/go-redis/redis/v8 v8.11.4 + github.com/golang/protobuf v1.5.2 + github.com/google/uuid v1.3.0 + github.com/target/strelka v0.0.0-00010101000000-000000000000 + google.golang.org/grpc v1.44.0 gopkg.in/yaml.v2 v2.4.0 ) -replace github.com/target/strelka => ./ \ No newline at end of file +require ( + github.com/cespare/xxhash/v2 v2.1.2 // indirect + github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect + golang.org/x/net v0.0.0-20210428140749-89ef3d95e781 // indirect + golang.org/x/sys v0.0.0-20210423082822-04245dca01da // indirect + golang.org/x/text v0.3.6 // indirect + google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013 // indirect + google.golang.org/protobuf v1.26.0 // indirect +) + +replace github.com/target/strelka => ./ diff --git a/go.sum b/go.sum index 4f521f2f..44429d4c 100644 --- a/go.sum +++ b/go.sum @@ -1,65 +1,83 @@ cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= +cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= +github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= -github.com/cespare/xxhash/v2 v2.1.1 h1:6MnRN8NT7+YBpUIWxHtefFZOKTAPgGjpQSxqLNn0+qY= github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/cespare/xxhash/v2 v2.1.2 h1:YRXhKfTDauu4ajMg1TPgFO5jnlC2HCbmLXMcTG5cbYE= +github.com/cespare/xxhash/v2 v2.1.2/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= +github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= +github.com/cncf/udpa/go v0.0.0-20210930031921-04548b0d99d4/go.mod h1:6pvJx4me5XPnfI9Z40ddWsdw2W/uZgQLFXToKeRcDiI= +github.com/cncf/xds/go v0.0.0-20210805033703-aa0b78936158/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= +github.com/cncf/xds/go v0.0.0-20210922020428-25de7278fc84/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= +github.com/cncf/xds/go v0.0.0-20211011173535-cb28da3451f1/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f h1:lO4WD4F/rVNCu3HqELle0jiPLLBs70cWOduZpkS1E78= github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f/go.mod h1:cuUVRXasLTGF7a8hSLbxyZXjz+1KgoB3wDUb6vlszIc= github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= +github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= github.com/envoyproxy/go-control-plane v0.9.9-0.20201210154907-fd9021fe5dad/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk= +github.com/envoyproxy/go-control-plane v0.9.10-0.20210907150352-cf90f659a021/go.mod h1:AFq3mo9L8Lqqiid3OhADV3RfLJnjiw63cSpi+fDTRC0= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= +github.com/fsnotify/fsnotify v1.4.9 h1:hsms1Qyu0jgnwNXIxa+/V/PDsU6CfLf6CNO8H7IWoS4= github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= -github.com/go-redis/redis/v8 v8.8.0 h1:fDZP58UN/1RD3DjtTXP/fFZ04TFohSYhjZDkcDe2dnw= -github.com/go-redis/redis/v8 v8.8.0/go.mod h1:F7resOH5Kdug49Otu24RjHWwgK7u9AmtqWMnCV1iP5Y= +github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= +github.com/go-redis/redis/v8 v8.11.4 h1:kHoYkfZP6+pe04aFTnhDH6GDROa5yJdHJVNxV3F46Tg= +github.com/go-redis/redis/v8 v8.11.4/go.mod h1:2Z2wHZXdQpCDXEGzqMockDpNyYvi2l4Pxt6RJr792+w= +github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0/go.mod h1:fyg7847qk6SyHyPtNmDHnmrv/HOrqktSC+C9fM+CJOE= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA= github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs= github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w= github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0= github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8= -github.com/golang/protobuf v1.4.2 h1:+Z5KGCizgyZCbGh1KZqA0fcLLkwbsjIzS4aV2v7wJX0= github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= +github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= +github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= +github.com/golang/protobuf v1.5.2 h1:ROPKBNFfQgOUMifHyP+KYbvpjbdoFNs+aK7DXlji0Tw= +github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.6 h1:BKbKCqvP6I+rmFHt06ZmyQtvB8xAkWdhFyr0ZUNZcxQ= +github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/google/uuid v1.2.0 h1:qJYtXnJRWmpe7m/3XlyhrsLrEURqHRM2kxzoxXqyUDs= -github.com/google/uuid v1.2.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= +github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw= github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A= +github.com/nxadm/tail v1.4.8 h1:nPr65rt6Y5JFSKQO7qToXr7pePgD6Gwiw05lkbyAQTE= +github.com/nxadm/tail v1.4.8/go.mod h1:+ncqLTQzXmGhMZNUePPaPqPvBxHAIsmXswZKocGu+AU= github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk= -github.com/onsi/ginkgo v1.15.0/go.mod h1:hF8qUzuuC8DJGygJH3726JnCZX4MYbRB8yFfISqnKUg= +github.com/onsi/ginkgo v1.16.4 h1:29JGrr5oVBm5ulCWet69zQkzWipVXIol6ygQUe/EzNc= +github.com/onsi/ginkgo v1.16.4/go.mod h1:dX+/inL/fNMqNlz0e9LfyB9TswhZpCVdJM/Z6Vvnwo0= github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY= github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo= -github.com/onsi/gomega v1.10.5/go.mod h1:gza4q3jKQJijlu05nKWRCW/GavJumGt8aNRxWg7mt48= +github.com/onsi/gomega v1.16.0 h1:6gjqkI8iiRHMvdccRJM8rVKjCWk6ZIm6FTm3ddIe4/c= +github.com/onsi/gomega v1.16.0/go.mod h1:HnhC7FXeEQY45zxNK3PPoIUhzk/80Xly9PcubAlGdZY= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= +github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= -github.com/target/strelka v0.0.0-20211012121236-d9086f35d709 h1:SNK4oLaZsNoDd8cH4Jfk4xz8I8OxzxENc0BWKOB3X+Q= -github.com/target/strelka v0.0.0-20211012121236-d9086f35d709/go.mod h1:TrUTi8JWPlUs67z/Wess97t6vyoCzvv3JyDOH1wd0Co= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= -go.opentelemetry.io/otel v0.19.0 h1:Lenfy7QHRXPZVsw/12CWpxX6d/JkrX8wrx2vO8G80Ng= -go.opentelemetry.io/otel v0.19.0/go.mod h1:j9bF567N9EfomkSidSfmMwIwIBuP37AMAIzVW85OxSg= -go.opentelemetry.io/otel/metric v0.19.0 h1:dtZ1Ju44gkJkYvo+3qGqVXmf88tc+a42edOywypengg= -go.opentelemetry.io/otel/metric v0.19.0/go.mod h1:8f9fglJPRnXuskQmKpnad31lcLJ2VmNNqIsx/uIwBSc= -go.opentelemetry.io/otel/oteltest v0.19.0/go.mod h1:tI4yxwh8U21v7JD6R3BcA/2+RBoTKFexE/PJ/nSO7IA= -go.opentelemetry.io/otel/trace v0.19.0 h1:1ucYlenXIDA1OlHVLDZKX0ObXV5RLaq06DtUKz5e5zc= -go.opentelemetry.io/otel/trace v0.19.0/go.mod h1:4IXiNextNOpPnRlI4ryK69mn5iC84bjBWZQA5DXz/qg= +go.opentelemetry.io/proto/otlp v0.7.0/go.mod h1:PqfVotwruBrMGOCsRd/89rSnXhoiJIqeYNgFYFoEGnI= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= @@ -71,17 +89,21 @@ golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200520004742-59133d7f0dd7/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= -golang.org/x/net v0.0.0-20201202161906-c7110b5ffcbb h1:eBmm0M9fYhWpKZLjQUUKka/LtIxf46G4fxeEz5KJr9U= -golang.org/x/net v0.0.0-20201202161906-c7110b5ffcbb/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.0.0-20210428140749-89ef3d95e781 h1:DzZ89McO9/gWPsQXS/FVKAlG02ZjaQ6AlZRBimEYOd0= +golang.org/x/net v0.0.0-20210428140749-89ef3d95e781/go.mod h1:OJAsFXCWl8Ukc7SiCT/9KSuxbyM7479/AVlXFRxuMCk= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= +golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -93,11 +115,15 @@ golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210112080510-489259a85091 h1:DMyOG0U+gKfu8JZzg2UQe9MeaC1X+xQWlAKcRnjxjCw= +golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210112080510-489259a85091/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210423082822-04245dca01da h1:b3NXsE2LusjYGGjL5bxEVZZORm/YEFFrWFjR8eFrw/c= +golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= -golang.org/x/text v0.3.3 h1:cokOdA+Jmi5PJGXLlLllQSgYigAEfHXJAERHVMaCc2k= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.6 h1:aRYxNxv6iGQlyVaZmk6ZgYEDa+Jg18DxebPSrd6bg1M= +golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= @@ -108,19 +134,23 @@ golang.org/x/tools v0.0.0-20201224043029-2b0845dc783e/go.mod h1:emZCQorbCU4vsT4f golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= +google.golang.org/genproto v0.0.0-20200513103714-09dca8ec2884/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013 h1:+kGHl1aib/qcwaRi1CbqBZ1rk19r85MNUf8HaBghugY= google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY= google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= -google.golang.org/grpc v1.36.0 h1:o1bcQ6imQMIOpdrO3SWf2z5RV72WbDwdXuK0MDlc8As= +google.golang.org/grpc v1.33.1/go.mod h1:fr5YgcSWrqhRRxogOsw7RzIpsmvOZ6IcH4kBYTpR3n0= google.golang.org/grpc v1.36.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= +google.golang.org/grpc v1.44.0 h1:weqSxi/TMs1SqFRMHCtBgXRs8k3X39QIDEZ0pRcttUg= +google.golang.org/grpc v1.44.0/go.mod h1:k+4IHHFw41K8+bbowsex27ge2rCb65oeWqe4jJ590SU= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= @@ -129,12 +159,17 @@ google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzi google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= -google.golang.org/protobuf v1.25.0 h1:Ejskq+SyPohKW+1uil0JJMtmHCgJPJ/qWTxr8qp+R4c= google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= +google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= +google.golang.org/protobuf v1.26.0 h1:bxAC2xTBsZGibn2RTntX0oH50xLsqy1OxA9tTL3p/lk= +google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= +gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.3/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= diff --git a/src/python/strelka/scanners/scan_javascript.py b/src/python/strelka/scanners/scan_javascript.py index 6b7fb088..e4d29f29 100644 --- a/src/python/strelka/scanners/scan_javascript.py +++ b/src/python/strelka/scanners/scan_javascript.py @@ -54,6 +54,6 @@ def scan(self, data, file, options, expire_at): if t.type == 'Identifier': if t.value not in self.event['identifiers']: self.event['identifiers'].append(t.value) - if type == 'RegularExpression': + if t.type == 'RegularExpression': if t.value not in self.event['regular_expressions']: self.event['regular_expressions'].append(t.value)