From e02a6b7714ae41fa906358c59e73d9eb93db1dc2 Mon Sep 17 00:00:00 2001 From: Wes Lambert Date: Wed, 24 Feb 2021 18:14:24 +0000 Subject: [PATCH 1/4] Add processedDir option for moving files to directory after submitting for scanning --- src/go/cmd/strelka-filestream/main.go | 2 ++ src/go/pkg/rpc/rpc.go | 12 +++++++++++- src/go/pkg/structs/structs.go | 2 ++ 3 files changed, 15 insertions(+), 1 deletion(-) diff --git a/src/go/cmd/strelka-filestream/main.go b/src/go/cmd/strelka-filestream/main.go index 259df4f8..23baed54 100644 --- a/src/go/cmd/strelka-filestream/main.go +++ b/src/go/cmd/strelka-filestream/main.go @@ -141,6 +141,7 @@ func main() { Chunk: conf.Throughput.Chunk, Delay: conf.Throughput.Delay, Delete: conf.Files.Delete, + ProcessedDir: conf.Files.ProcessedDir, } sem <- 1 @@ -197,6 +198,7 @@ func main() { Chunk: conf.Throughput.Chunk, Delay: conf.Throughput.Delay, Delete: conf.Files.Delete, + ProcessedDir: conf.Files.ProcessedDir, } sem <- 1 diff --git a/src/go/pkg/rpc/rpc.go b/src/go/pkg/rpc/rpc.go index 59a1c72d..be173777 100644 --- a/src/go/pkg/rpc/rpc.go +++ b/src/go/pkg/rpc/rpc.go @@ -8,6 +8,7 @@ import ( "os" "runtime" "time" + "path/filepath" "google.golang.org/grpc" "google.golang.org/grpc/credentials" @@ -127,7 +128,16 @@ func ScanFile(client strelka.FrontendClient, timeout time.Duration, req structs. } if req.Delete { defer os.Remove(req.Attributes.Filename) - } + } else if req.ProcessedDir != "" { + defer func() { + _, name := filepath.Split(req.Attributes.Filename) + m := filepath.Join(req.ProcessedDir, name) + err := os.Rename(req.Attributes.Filename, m) + if err != nil { + log.Printf("failed to move file %s to directory %s: %v", name, req.ProcessedDir, err) + } + }() + } defer file.Close() scanFile, err := client.ScanFile(ctx, grpc.WaitForReady(true)) diff --git a/src/go/pkg/structs/structs.go b/src/go/pkg/structs/structs.go index fdce0245..8411903f 100644 --- a/src/go/pkg/structs/structs.go +++ b/src/go/pkg/structs/structs.go @@ -30,6 +30,7 @@ type ConfFiles struct { Patterns []string // required Delete bool // optional Gatekeeper bool // required + ProcessedDir string // optional } type ConfCoordinator struct { @@ -96,4 +97,5 @@ type ScanFileRequest struct { Chunk int // required Delay time.Duration // optional Delete bool // optional, only use if files must be deleted! + ProcessedDir string // optional } From 723ff8d5f519ac4b6e88667f8fe9c4ad325f0140 Mon Sep 17 00:00:00 2001 From: Wes Lambert Date: Wed, 24 Feb 2021 19:01:57 +0000 Subject: [PATCH 2/4] Rename ProcessedDir to Processed --- build/go/filestream/Dockerfile | 5 ++++- src/go/cmd/strelka-filestream/main.go | 4 ++-- src/go/pkg/rpc/rpc.go | 6 +++--- src/go/pkg/structs/structs.go | 4 ++-- 4 files changed, 11 insertions(+), 8 deletions(-) diff --git a/build/go/filestream/Dockerfile b/build/go/filestream/Dockerfile index d81574ab..8bbd520f 100644 --- a/build/go/filestream/Dockerfile +++ b/build/go/filestream/Dockerfile @@ -14,5 +14,8 @@ RUN go mod init && \ # Move build, install additional packages, and initialize non-root user FROM alpine COPY --from=build /tmp/strelka-filestream /usr/local/bin/strelka-filestream +RUN addgroup -g 939 strelka && \ + adduser -u 939 -G strelka strelka --disabled-password \ + -h /etc/strelka --no-create-home strelka RUN apk add --no-cache jq -USER 1001 +USER strelka diff --git a/src/go/cmd/strelka-filestream/main.go b/src/go/cmd/strelka-filestream/main.go index 23baed54..9a75b937 100644 --- a/src/go/cmd/strelka-filestream/main.go +++ b/src/go/cmd/strelka-filestream/main.go @@ -141,7 +141,7 @@ func main() { Chunk: conf.Throughput.Chunk, Delay: conf.Throughput.Delay, Delete: conf.Files.Delete, - ProcessedDir: conf.Files.ProcessedDir, + Processed: conf.Files.Processed, } sem <- 1 @@ -198,7 +198,7 @@ func main() { Chunk: conf.Throughput.Chunk, Delay: conf.Throughput.Delay, Delete: conf.Files.Delete, - ProcessedDir: conf.Files.ProcessedDir, + Processed: conf.Files.Processed, } sem <- 1 diff --git a/src/go/pkg/rpc/rpc.go b/src/go/pkg/rpc/rpc.go index be173777..67a81d66 100644 --- a/src/go/pkg/rpc/rpc.go +++ b/src/go/pkg/rpc/rpc.go @@ -128,13 +128,13 @@ func ScanFile(client strelka.FrontendClient, timeout time.Duration, req structs. } if req.Delete { defer os.Remove(req.Attributes.Filename) - } else if req.ProcessedDir != "" { + } else if req.Processed != "" { defer func() { _, name := filepath.Split(req.Attributes.Filename) - m := filepath.Join(req.ProcessedDir, name) + m := filepath.Join(req.Processed, name) err := os.Rename(req.Attributes.Filename, m) if err != nil { - log.Printf("failed to move file %s to directory %s: %v", name, req.ProcessedDir, err) + log.Printf("failed to move file %s to directory %s: %v", name, req.Processed, err) } }() } diff --git a/src/go/pkg/structs/structs.go b/src/go/pkg/structs/structs.go index 8411903f..9175aee9 100644 --- a/src/go/pkg/structs/structs.go +++ b/src/go/pkg/structs/structs.go @@ -30,7 +30,7 @@ type ConfFiles struct { Patterns []string // required Delete bool // optional Gatekeeper bool // required - ProcessedDir string // optional + Processed string // optional } type ConfCoordinator struct { @@ -97,5 +97,5 @@ type ScanFileRequest struct { Chunk int // required Delay time.Duration // optional Delete bool // optional, only use if files must be deleted! - ProcessedDir string // optional + Processed string // optional } From d333c6bda2013545690023f927583de0bef4d444 Mon Sep 17 00:00:00 2001 From: Wes Lambert Date: Wed, 24 Feb 2021 19:03:55 +0000 Subject: [PATCH 3/4] Remove Dockerfile mods --- build/go/filestream/Dockerfile | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/build/go/filestream/Dockerfile b/build/go/filestream/Dockerfile index 8bbd520f..d81574ab 100644 --- a/build/go/filestream/Dockerfile +++ b/build/go/filestream/Dockerfile @@ -14,8 +14,5 @@ RUN go mod init && \ # Move build, install additional packages, and initialize non-root user FROM alpine COPY --from=build /tmp/strelka-filestream /usr/local/bin/strelka-filestream -RUN addgroup -g 939 strelka && \ - adduser -u 939 -G strelka strelka --disabled-password \ - -h /etc/strelka --no-create-home strelka RUN apk add --no-cache jq -USER strelka +USER 1001 From c841715ca83afc2ffe42c78edf10b569aa267850 Mon Sep 17 00:00:00 2001 From: weslambert Date: Wed, 24 Feb 2021 14:14:55 -0500 Subject: [PATCH 4/4] Add files.processed --- docs/README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/README.md b/docs/README.md index c62825b5..3c2fda1f 100644 --- a/docs/README.md +++ b/docs/README.md @@ -281,6 +281,7 @@ For the options below, only one response setting may be configured. * "files.chunk": size of file chunks that will be sent to the frontend server (defaults to 32768b / 32kb) * "files.patterns": list of glob patterns that determine which files will be sent for scanning (defaults to example glob pattern) * "files.delete": boolean that determines if files should be deleted after being sent for scanning (defaults to false -- does not delete files) +* "files.processed": directory where files will be moved after being submitted for scanning (defaults to "", and files stay in staging directory) * "response.log": location where worker scan results are logged to (defaults to /var/log/strelka/strelka.log) * "response.report": frequency at which the frontend reports the number of files processed (no default) * "delta": time value that determines how much time must pass since a file was last modified before it is sent for scanning (defaults to 5 seconds)