diff --git a/Source/santad/BUILD b/Source/santad/BUILD index 04ace098b..e73c3e5cb 100644 --- a/Source/santad/BUILD +++ b/Source/santad/BUILD @@ -1124,6 +1124,7 @@ test_suite( ":SNTExecutionControllerTest", ":SNTRuleTableTest", ":SantadTest", + "//Source/santad/Logs/EndpointSecurity/Writers/FSSpool:fsspool_test" ], visibility = ["//:santa_package_group"], ) diff --git a/Source/santad/Logs/EndpointSecurity/Writers/FSSpool/BUILD b/Source/santad/Logs/EndpointSecurity/Writers/FSSpool/BUILD new file mode 100644 index 000000000..a5f20ff11 --- /dev/null +++ b/Source/santad/Logs/EndpointSecurity/Writers/FSSpool/BUILD @@ -0,0 +1,67 @@ +load("@rules_cc//cc:defs.bzl", "cc_proto_library") +load("//:helper.bzl", "santa_unit_test") + +package( + default_visibility = ["//:santa_package_group"], +) + +proto_library( + name = "binaryproto_proto", + srcs = ["binaryproto.proto"], + deps = [ + "@com_google_protobuf//:any_proto", + ], +) + +cc_proto_library( + name = "binaryproto_cc_proto", + visibility = ["//visibility:public"], + deps = [ + ":binaryproto_proto", + ], +) + +cc_library( + name = "fsspool", + srcs = [ + "fsspool.cc", + "fsspool_nowindows.cc", + ], + hdrs = [ + "fsspool.h", + "fsspool_platform_specific.h", + ], + deps = [ + "@com_google_absl//absl/cleanup", + "@com_google_absl//absl/container:flat_hash_set", + "@com_google_absl//absl/random", + "@com_google_absl//absl/status", + "@com_google_absl//absl/status:statusor", + "@com_google_absl//absl/strings", + "@com_google_absl//absl/strings:str_format", + "@com_google_absl//absl/time", + ], +) + +cc_library( + name = "fsspool_log_batch_writer", + srcs = ["fsspool_log_batch_writer.cc"], + hdrs = ["fsspool_log_batch_writer.h"], + deps = [ + ":binaryproto_cc_proto", + ":fsspool", + "@com_google_absl//absl/base:core_headers", + "@com_google_absl//absl/status", + "@com_google_absl//absl/synchronization", + ], +) + +santa_unit_test( + name = "fsspool_test", + srcs = ["fsspool_test.mm"], + deps = [ + ":fsspool", + ":fsspool_log_batch_writer", + "@OCMock", + ], +) diff --git a/Source/santad/Logs/EndpointSecurity/Writers/FSSpool/binaryproto.proto b/Source/santad/Logs/EndpointSecurity/Writers/FSSpool/binaryproto.proto new file mode 100644 index 000000000..3052dfe6b --- /dev/null +++ b/Source/santad/Logs/EndpointSecurity/Writers/FSSpool/binaryproto.proto @@ -0,0 +1,12 @@ +syntax = "proto3"; + +package binaryproto; + +import "google/protobuf/any.proto"; + +option objc_class_prefix = "FSS"; + +// A LogBatch is a simple array of protos. +message LogBatch { + repeated google.protobuf.Any records = 1; +} diff --git a/Source/santad/Logs/EndpointSecurity/Writers/FSSpool/fsspool.cc b/Source/santad/Logs/EndpointSecurity/Writers/FSSpool/fsspool.cc new file mode 100644 index 000000000..19a73761a --- /dev/null +++ b/Source/santad/Logs/EndpointSecurity/Writers/FSSpool/fsspool.cc @@ -0,0 +1,299 @@ +/// Copyright 2022 Google LLC +/// +/// Licensed under the Apache License, Version 2.0 (the "License"); +/// you may not use this file except in compliance with the License. +/// You may obtain a copy of the License at +/// +/// https://www.apache.org/licenses/LICENSE-2.0 +/// +/// Unless required by applicable law or agreed to in writing, software +/// distributed under the License is distributed on an "AS IS" BASIS, +/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +/// See the License for the specific language governing permissions and +/// limitations under the License. + +#include "Source/santad/Logs/EndpointSecurity/Writers/FSSpool/fsspool.h" + +#include +#include + +#include +#include +#include + +#include "absl/random/random.h" +#include "absl/status/status.h" +#include "absl/status/statusor.h" +#include "absl/strings/str_cat.h" +#include "absl/strings/str_format.h" +#include "absl/strings/string_view.h" +#include "absl/strings/substitute.h" +#include "absl/time/time.h" +#include "Source/santad/Logs/EndpointSecurity/Writers/FSSpool/fsspool_platform_specific.h" + +namespace fsspool { + +// Returns whether the given path exists and is a directory. +bool IsDirectory(const std::string& d) { + struct stat stats; + if (stat(d.c_str(), &stats) < 0) { + return false; + } + return StatIsDir(stats.st_mode); +} + +namespace { + +constexpr absl::string_view kSpoolDirName = "new"; +constexpr absl::string_view kTmpDirName = "tmp"; + +// Estimates the disk occupation of a file of the given size, +// with the following heuristic: A typical disk cluster is 4KiB; files +// usually get written to disk in multiples of this unit. +size_t EstimateDiskOccupation(size_t fileSize) { + // kDiskClusterSize defines the typical size of a disk cluster (4KiB). + static constexpr size_t kDiskClusterSize = 4096; + size_t n_clusters = (fileSize + kDiskClusterSize - 1) / kDiskClusterSize; + // Empty files still occupy some space. + if (n_clusters == 0) { + n_clusters = 1; + } + return n_clusters * kDiskClusterSize; +} + +// Creates a directory if it doesn't exist. +// It only accepts absolute paths. +absl::Status MkDir(const std::string& path) { + if (!IsAbsolutePath(path)) { + return absl::InvalidArgumentError( + absl::StrCat(path, " is not an absolute path.")); + } + if (fsspool::MkDir(path.c_str(), 0700) < 0) { + if (errno == EEXIST && IsDirectory(path)) { + return absl::OkStatus(); + } + return absl::ErrnoToStatus(errno, absl::StrCat("failed to create ", path)); + } + return absl::OkStatus(); +} + +// Writes a buffer to the given file descriptor. +// Calls to write can result in a partially written file. Very rare cases in +// which this could happen (since we're writing to a regular file) include +// if we receive a signal during write or if the disk is full. +// Retry writing until we've flushed everything, return an error if any write +// fails. +absl::Status WriteBuffer(int fd, absl::string_view msg) { + while (!msg.empty()) { + const int n_written = Write(fd, msg); + if (n_written < 0) { + return absl::ErrnoToStatus(errno, "write() failed"); + } + msg.remove_prefix(n_written); + } + return absl::OkStatus(); +} + +// Writes the given data to the given file, with permissions set to 0400. +// Roughly equivalent to file::SetContents. +absl::Status WriteTmpFile(const std::string& path, absl::string_view msg) { + const int fd = Open(path.c_str(), O_WRONLY | O_CREAT | O_TRUNC, 0400); + if (fd < 0) { + return absl::ErrnoToStatus(errno, "open() failed"); + } + absl::Status write_status = WriteBuffer(fd, msg); + Close(fd); + if (!write_status.ok()) { + // Delete the file so we don't leave garbage behind us. + if (Unlink(path.c_str()) < 0) { + // This is very unlikely (e.g. somehow permissions for the file changed + // since creation?), still worth logging the error. + return absl::ErrnoToStatus(errno, absl::StrCat("Writing to ", path, " failed (and deleting failed too)")); + } + return write_status; + } + return absl::OkStatus(); +} + +// Renames src to dest. Equivalent to file::Rename. +absl::Status RenameFile(const std::string& src, const std::string& dst) { + if (rename(src.c_str(), dst.c_str()) < 0) { + return absl::ErrnoToStatus( + errno, absl::StrCat("failed to rename ", src, " to ", dst)); + } + return absl::OkStatus(); +} + +absl::StatusOr EstimateDirSize(const std::string& dir) { + size_t estimate = 0; + absl::Status status = + IterateDirectory(dir, [&dir, &estimate](const std::string& file_name) { + /// NOMUTANTS--We could skip this condition altogether, as S_ISREG on + /// the directory would be false anyway. + if (file_name == std::string(".") || file_name == std::string("..")) { + return; + } + std::string file_path = absl::StrCat(dir, PathSeparator(), file_name); + struct stat stats; + if (stat(file_path.c_str(), &stats) < 0) { + return; + } + if (!StatIsReg(stats.st_mode)) { + return; + } + // Use st_size, as st_blocks is not available on Windows. + estimate += EstimateDiskOccupation(stats.st_size); + }); + if (status.ok()) { + return estimate; + } + return status; +} + +std::string SpoolDirectory(absl::string_view base_dir) { + return absl::StrCat(base_dir, PathSeparator(), kSpoolDirName); +} + +} // namespace + +FsSpoolWriter::FsSpoolWriter(absl::string_view base_dir, size_t max_spool_size) + : base_dir_(base_dir), + spool_dir_(SpoolDirectory(base_dir)), + tmp_dir_(absl::StrCat(base_dir, PathSeparator(), kTmpDirName)), + max_spool_size_(max_spool_size), + id_(absl::StrFormat("%016x", absl::Uniform( + absl::BitGen(), 0, + std::numeric_limits::max()))), + // Guess that the spool is full during construction, so we will recompute + // the actual spool size on the first write. + spool_size_estimate_(max_spool_size + 1) {} + +absl::Status FsSpoolWriter::BuildDirectoryStructureIfNeeded() { + if (!IsDirectory(spool_dir_)) { + if (!IsDirectory(base_dir_)) { + if (absl::Status status = MkDir(base_dir_); !status.ok()) { + return status; // failed to create base directory + } + } + + if (absl::Status status = MkDir(spool_dir_); !status.ok()) { + return status; // failed to create spool directory; + } + } + if (!IsDirectory(tmp_dir_)) { + // No need to check the base directory too, since spool_dir_ exists. + if (absl::Status status = MkDir(tmp_dir_); !status.ok()) { + return status; // failed to create tmp directory + } + } + return absl::OkStatus(); +} + +std::string FsSpoolWriter::UniqueFilename() { + std::string result = absl::StrFormat("%s_%020d", id_, sequence_number_); + sequence_number_++; + return result; +} + +absl::Status FsSpoolWriter::WriteMessage(absl::string_view msg) { + if (absl::Status status = BuildDirectoryStructureIfNeeded(); !status.ok()) { + return status; // << "can't create directory structure for writer"; + } + // Flush messages to a file in the temporary directory. + const std::string fname = UniqueFilename(); + const std::string tmp_file = absl::StrCat(tmp_dir_, PathSeparator(), fname); + const std::string spool_file = + absl::StrCat(spool_dir_, PathSeparator(), fname); + // Recompute the spool size if we think we are + // over the limit. + if (spool_size_estimate_ > max_spool_size_) { + absl::StatusOr estimate = EstimateDirSize(spool_dir_); + if (!estimate.ok()) { + return estimate.status(); // failed to recompute spool size + } + spool_size_estimate_ = *estimate; + if (spool_size_estimate_ > max_spool_size_) { + // Still over the limit: avoid writing. + return absl::UnavailableError("Spool size estimate greater than max allowed"); + } + } + spool_size_estimate_ += EstimateDiskOccupation(msg.size()); + + if (absl::Status status = WriteTmpFile(tmp_file, msg); !status.ok()) { + return status; // writing to temporary file + } + + if (absl::Status status = RenameFile(tmp_file, spool_file); !status.ok()) { + return status; // "moving tmp_file to the spooling area + } + + return absl::OkStatus(); +} + +FsSpoolReader::FsSpoolReader(absl::string_view base_directory) + : base_dir_(base_directory), spool_dir_(SpoolDirectory(base_directory)) {} + +int FsSpoolReader::NumberOfUnackedMessages() const { + return unacked_messages_.size(); +} + +absl::Status FsSpoolReader::AckMessage(const std::string& message_path) { + int remove_status = remove(message_path.c_str()); + if ((remove_status != 0) && (errno != ENOENT)) { + return absl::ErrnoToStatus( + errno, + absl::Substitute("Failed to remove $0: $1", message_path, errno)); + } + unacked_messages_.erase(message_path); + return absl::OkStatus(); +} + +absl::StatusOr FsSpoolReader::NextMessagePath() { + absl::StatusOr file_path = OldestSpooledFile(); + if (!file_path.ok()) { + return file_path; + } + unacked_messages_.insert(*file_path); + return file_path; +} + +absl::StatusOr FsSpoolReader::OldestSpooledFile() { + if (!IsDirectory(spool_dir_)) { + return absl::NotFoundError( + "Spool directory is not a directory or it doesn't exist."); + } + absl::Time oldest_file_mtime; + std::string oldest_file_path; + absl::Status status = IterateDirectory( + spool_dir_, [this, &oldest_file_path, + &oldest_file_mtime](const std::string& file_name) { + std::string file_path = + absl::StrCat(spool_dir_, PathSeparator(), file_name); + struct stat stats; + if (stat(file_path.c_str(), &stats) < 0) { + return; + } + if (!StatIsReg(stats.st_mode)) { + return; + } + if (unacked_messages_.contains(file_path)) { + return; + } + absl::Time file_mtime = absl::FromTimeT(stats.st_mtime); + if (!oldest_file_path.empty() && oldest_file_mtime < file_mtime) { + return; + } + oldest_file_path = file_path; + oldest_file_mtime = file_mtime; + }); + if (!status.ok()) { + return status; + } + + if (oldest_file_path.empty()) { + return absl::NotFoundError("Empty FsSpool directory."); + } + return oldest_file_path; +} + +} // namespace fsspool diff --git a/Source/santad/Logs/EndpointSecurity/Writers/FSSpool/fsspool.h b/Source/santad/Logs/EndpointSecurity/Writers/FSSpool/fsspool.h new file mode 100644 index 000000000..8616d9fb3 --- /dev/null +++ b/Source/santad/Logs/EndpointSecurity/Writers/FSSpool/fsspool.h @@ -0,0 +1,105 @@ +/// Copyright 2022 Google LLC +/// +/// Licensed under the Apache License, Version 2.0 (the "License"); +/// you may not use this file except in compliance with the License. +/// You may obtain a copy of the License at +/// +/// https://www.apache.org/licenses/LICENSE-2.0 +/// +/// Unless required by applicable law or agreed to in writing, software +/// distributed under the License is distributed on an "AS IS" BASIS, +/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +/// See the License for the specific language governing permissions and +/// limitations under the License. + +#ifndef SANTA__SANTAD__LOGS_ENDPOINTSECURITY_WRITERS_FSSPOOL_FSSPOOL_H_ +#define SANTA__SANTAD__LOGS_ENDPOINTSECURITY_WRITERS_FSSPOOL_FSSPOOL_H_ + +// Namespace ::fsspool::fsspool implements a filesystem-backed message spool, to +// use as a lock-free IPC mechanism. + +#include + +#include "absl/container/flat_hash_set.h" +#include "absl/status/status.h" +#include "absl/status/statusor.h" +#include "absl/strings/string_view.h" + +namespace fsspool { + +// Enqueues messages into the spool. Multiple concurrent writers can +// write to the same directory. (Note that this class is only thread-compatible +// and not thread-safe though!) +class FsSpoolWriter { + public: + // The base, spool, and temporary directory will be created as needed on the + // first call to Write() - however the base directory can be created into an + // existing path (i.e. this class will not do an `mkdir -p`). + FsSpoolWriter(absl::string_view base_dir, size_t max_spool_size); + + // Pushes the given byte array to the spool. The given maximum + // spool size will be enforced. Returns an error code. If the spool gets full, + // returns the UNAVAILABLE canonical code (which is retryable). + absl::Status WriteMessage(absl::string_view msg); + + private: + const std::string base_dir_; + const std::string spool_dir_; + const std::string tmp_dir_; + + // Approximate maximum size of the spooling area, in bytes. If a message is + // being written to a spooling area which already contains more than + // maxSpoolSize bytes, the write will not be executed. This is an approximate + // estimate: no care is taken to make an exact estimate (for example, if a + // file gets deleted from the spool while the estimate is being computed, the + // final estimate is likely to still include the size of that file). + const size_t max_spool_size_; + + // 64bit hex ID for this writer. Used in combination with the sequence + // number to generate unique names for files. This is generated through + // util::random::NewGlobalID(), hence has only 52 bits of randomness. + const std::string id_; + + // Sequence number of the next message to be written. This + // counter will be incremented at every Write call, so that the produced + // spooled files have different names. + uint64_t sequence_number_ = 0; + + // Last estimate for the spool size. The estimate will grow every time we + // write messages (basically, we compute it as if there was no reader + // consuming messages). It will get updated with the actual value whenever we + // think we've passed the size limit. The new estimate will be the sum of the + // approximate disk space occupied by each message written (in multiples of + // 4KiB, i.e. a typical disk cluster size). + size_t spool_size_estimate_; + + // Makes sure that all the required + // directories needed for correct operation of this Writer are present in the + // filesystem. + absl::Status BuildDirectoryStructureIfNeeded(); + + // Generates a unique filename by combining the random ID of + // this writer with a sequence number. + std::string UniqueFilename(); +}; + +// This class is thread-unsafe. +class FsSpoolReader { + public: + explicit FsSpoolReader(absl::string_view base_directory); + absl::Status AckMessage(const std::string& message_path); + // Returns absl::NotFoundError in case the FsSpool is empty. + absl::StatusOr NextMessagePath(); + int NumberOfUnackedMessages() const; + + private: + const std::string base_dir_; + const std::string spool_dir_; + absl::flat_hash_set unacked_messages_; + + absl::StatusOr OldestSpooledFile(); +}; + +} // namespace fsspool + +#endif // SANTA__SANTAD__LOGS_ENDPOINTSECURITY_WRITERS_FSSPOOL_FSSPOOL_H_ diff --git a/Source/santad/Logs/EndpointSecurity/Writers/FSSpool/fsspool_log_batch_writer.cc b/Source/santad/Logs/EndpointSecurity/Writers/FSSpool/fsspool_log_batch_writer.cc new file mode 100644 index 000000000..5c632c9e6 --- /dev/null +++ b/Source/santad/Logs/EndpointSecurity/Writers/FSSpool/fsspool_log_batch_writer.cc @@ -0,0 +1,75 @@ +/// Copyright 2022 Google LLC +/// +/// Licensed under the Apache License, Version 2.0 (the "License"); +/// you may not use this file except in compliance with the License. +/// You may obtain a copy of the License at +/// +/// https://www.apache.org/licenses/LICENSE-2.0 +/// +/// Unless required by applicable law or agreed to in writing, software +/// distributed under the License is distributed on an "AS IS" BASIS, +/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +/// See the License for the specific language governing permissions and +/// limitations under the License. + +#include "Source/santad/Logs/EndpointSecurity/Writers/FSSpool/fsspool_log_batch_writer.h" + +#include + +#include + +#include "absl/status/status.h" + +namespace fsspool { + +FsSpoolLogBatchWriter::FsSpoolLogBatchWriter(FsSpoolWriter* fs_spool_writer, + size_t max_batch_size) + : writer_(fs_spool_writer), max_batch_size_(max_batch_size) { + cache_.mutable_records()->Reserve(max_batch_size_); +} + +FsSpoolLogBatchWriter::~FsSpoolLogBatchWriter() { + absl::Status s = FlushNoLock(); + if (!s.ok()) { + os_log(OS_LOG_DEFAULT, "Flush() failed with %s", s.ToString(absl::StatusToStringMode::kWithEverything).c_str()); + // LOG(WARNING) << "Flush() failed with " << s; + } +} + +absl::Status FsSpoolLogBatchWriter::Flush() { + absl::MutexLock lock(&cache_mutex_); + return FlushNoLock(); +} + +absl::Status FsSpoolLogBatchWriter::FlushNoLock() { + if (cache_.mutable_records()->empty()) { + return absl::OkStatus(); + } + std::string msg; + if (!cache_.SerializeToString(&msg)) { + return absl::InternalError("Failed to serialize internal LogBatch cache."); + } + { + absl::MutexLock lock(&writer_mutex_); + if (absl::Status status = writer_->WriteMessage(msg); !status.ok()) { + return status; + } + } + cache_.mutable_records()->Clear(); + cache_.mutable_records()->Reserve(max_batch_size_); + return absl::OkStatus(); +} + +absl::Status FsSpoolLogBatchWriter::WriteMessage( + const ::google::protobuf::Any& msg) { + absl::MutexLock lock(&cache_mutex_); + if (cache_.records_size() >= max_batch_size_) { + if (absl::Status status = FlushNoLock(); !status.ok()) { + return status; + } + } + *cache_.mutable_records()->Add() = msg; + return absl::OkStatus(); +} + +} // namespace fsspool diff --git a/Source/santad/Logs/EndpointSecurity/Writers/FSSpool/fsspool_log_batch_writer.h b/Source/santad/Logs/EndpointSecurity/Writers/FSSpool/fsspool_log_batch_writer.h new file mode 100644 index 000000000..337aeca9a --- /dev/null +++ b/Source/santad/Logs/EndpointSecurity/Writers/FSSpool/fsspool_log_batch_writer.h @@ -0,0 +1,70 @@ +/// Copyright 2022 Google LLC +/// +/// Licensed under the Apache License, Version 2.0 (the "License"); +/// you may not use this file except in compliance with the License. +/// You may obtain a copy of the License at +/// +/// https://www.apache.org/licenses/LICENSE-2.0 +/// +/// Unless required by applicable law or agreed to in writing, software +/// distributed under the License is distributed on an "AS IS" BASIS, +/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +/// See the License for the specific language governing permissions and +/// limitations under the License. + +#ifndef SANTA__SANTAD__LOGS_ENDPOINTSECURITY_WRITERS_FSSPOOL_FSSPOOLLOGBATCHWRITER_H +#define SANTA__SANTAD__LOGS_ENDPOINTSECURITY_WRITERS_FSSPOOL_FSSPOOLLOGBATCHWRITER_H + +#include +#include + +#include "Source/santad/Logs/EndpointSecurity/Writers/FSSpool/binaryproto.pb.h" +#include "Source/santad/Logs/EndpointSecurity/Writers/FSSpool/fsspool.h" +#include "absl/base/thread_annotations.h" +#include "absl/synchronization/mutex.h" +#include "google/protobuf/any.pb.h" + +namespace fsspool { + +// Provides FsSpool batching mechanism in the form of LogBatch proto messages. +// +// Example: +// FsSpoolWriter fsspool_writer(...); +// FsSpoolLogBatchWriter batch_writer(&fsspool_writer, 10); +// ASSERT_OK(batch_writer.WriteMessage(any_proto); +// +// Automatic flush happens in the event of the object destruction. +// +// Flush() method is provided, so the users of this class can implement periodic +// flushes. It is not necessary to call Flush() manually otherwise. +// +// The class is thread-safe. +class FsSpoolLogBatchWriter { + public: + FsSpoolLogBatchWriter(FsSpoolWriter* fs_spool_writer, size_t max_batch_size); + ~FsSpoolLogBatchWriter(); + + // Writes Any proto message to the FsSpool. The write is cached according to + // the object configuration. + // + // This may return an error if flushing is unsuccessful. + absl::Status WriteMessage(const ::google::protobuf::Any& msg); + + // Flush internal FsSpoolLogBatchWriter cache to disk. Calling this method is + // not necessary as the cache is flushed after max_batch_size limit is reached + // or when the objects is destroyed. + absl::Status Flush(); + + private: + absl::Mutex writer_mutex_ ABSL_ACQUIRED_AFTER(cache_mutex_); + FsSpoolWriter* writer_ ABSL_PT_GUARDED_BY(writer_mutex_); + size_t max_batch_size_; + absl::Mutex cache_mutex_; + binaryproto::LogBatch cache_ ABSL_GUARDED_BY(cache_mutex_); + + absl::Status FlushNoLock() ABSL_SHARED_LOCKS_REQUIRED(cache_mutex_); +}; + +} // namespace fsspool + +#endif // SANTA__SANTAD__LOGS_ENDPOINTSECURITY_WRITERS_FSSPOOL_FSSPOOLLOGBATCHWRITER_H diff --git a/Source/santad/Logs/EndpointSecurity/Writers/FSSpool/fsspool_nowindows.cc b/Source/santad/Logs/EndpointSecurity/Writers/FSSpool/fsspool_nowindows.cc new file mode 100644 index 000000000..5e8f94362 --- /dev/null +++ b/Source/santad/Logs/EndpointSecurity/Writers/FSSpool/fsspool_nowindows.cc @@ -0,0 +1,72 @@ +/// Copyright 2022 Google LLC +/// +/// Licensed under the Apache License, Version 2.0 (the "License"); +/// you may not use this file except in compliance with the License. +/// You may obtain a copy of the License at +/// +/// https://www.apache.org/licenses/LICENSE-2.0 +/// +/// Unless required by applicable law or agreed to in writing, software +/// distributed under the License is distributed on an "AS IS" BASIS, +/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +/// See the License for the specific language governing permissions and +/// limitations under the License. + +#include +#include +#include +#include + +#include +#include + +#include "absl/strings/match.h" +#include "absl/strings/str_format.h" +#include "Source/santad/Logs/EndpointSecurity/Writers/FSSpool/fsspool.h" +#include "Source/santad/Logs/EndpointSecurity/Writers/FSSpool/fsspool_platform_specific.h" + +namespace fsspool { + +absl::string_view PathSeparator() { return "/"; } + +bool IsAbsolutePath(absl::string_view path) { + return absl::StartsWith(path, "/"); +} + +int Write(int fd, absl::string_view buf) { + return ::write(fd, buf.data(), buf.size()); +} + +int Unlink(const char* pathname) { return unlink(pathname); } + +int MkDir(const char* path, mode_t mode) { return mkdir(path, mode); } + +bool StatIsDir(mode_t mode) { return S_ISDIR(mode); } + +bool StatIsReg(mode_t mode) { return S_ISREG(mode); } + +int Open(const char* filename, int flags, mode_t mode) { + return open(filename, flags, mode); +} + +int Close(int fd) { return close(fd); } + +absl::Status IterateDirectory( + const std::string& dir, std::function callback) { + if (!IsDirectory(dir)) { + return absl::InvalidArgumentError( + absl::StrFormat("%s is not a directory", dir)); + } + DIR* dp = opendir(dir.c_str()); + if (dp == nullptr) { + return absl::ErrnoToStatus(errno, absl::StrCat("failed to open ", dir)); + } + struct dirent* ep; + while ((ep = readdir(dp)) != nullptr) { + callback(ep->d_name); + } + closedir(dp); + return absl::OkStatus(); +} + +} // namespace fsspool diff --git a/Source/santad/Logs/EndpointSecurity/Writers/FSSpool/fsspool_platform_specific.h b/Source/santad/Logs/EndpointSecurity/Writers/FSSpool/fsspool_platform_specific.h new file mode 100644 index 000000000..9acdd5f29 --- /dev/null +++ b/Source/santad/Logs/EndpointSecurity/Writers/FSSpool/fsspool_platform_specific.h @@ -0,0 +1,41 @@ +/// Copyright 2022 Google LLC +/// +/// Licensed under the Apache License, Version 2.0 (the "License"); +/// you may not use this file except in compliance with the License. +/// You may obtain a copy of the License at +/// +/// https://www.apache.org/licenses/LICENSE-2.0 +/// +/// Unless required by applicable law or agreed to in writing, software +/// distributed under the License is distributed on an "AS IS" BASIS, +/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +/// See the License for the specific language governing permissions and +/// limitations under the License. + +#ifndef SANTA__SANTAD__LOGS_ENDPOINTSECURITY_WRITERS_FSSPOOL_FSSPOOLPLATFORMSPECIFIC_H +#define SANTA__SANTAD__LOGS_ENDPOINTSECURITY_WRITERS_FSSPOOL_FSSPOOLPLATFORMSPECIFIC_H + +#include +#include + +#include "absl/strings/string_view.h" + +namespace fsspool { + +absl::string_view PathSeparator(); +bool IsAbsolutePath(absl::string_view path); +bool IsDirectory(const std::string& d); +int Close(int fd); +int Open(const char* filename, int flags, mode_t mode); +int MkDir(const char* path, mode_t mode); +bool StatIsDir(mode_t mode); +bool StatIsReg(mode_t mode); +int Unlink(const char* pathname); +int Write(int fd, absl::string_view buf); + +absl::Status IterateDirectory(const std::string& dir, + std::function callback); + +} // namespace fsspool + +#endif // SANTA__SANTAD__LOGS_ENDPOINTSECURITY_WRITERS_FSSPOOL_FSSPOOLPLATFORMSPECIFIC_H diff --git a/Source/santad/Logs/EndpointSecurity/Writers/FSSpool/fsspool_test.mm b/Source/santad/Logs/EndpointSecurity/Writers/FSSpool/fsspool_test.mm new file mode 100644 index 000000000..b4400773b --- /dev/null +++ b/Source/santad/Logs/EndpointSecurity/Writers/FSSpool/fsspool_test.mm @@ -0,0 +1,202 @@ +/// Copyright 2022 Google LLC +/// +/// Licensed under the Apache License, Version 2.0 (the "License"); +/// you may not use this file except in compliance with the License. +/// You may obtain a copy of the License at +/// +/// https://www.apache.org/licenses/LICENSE-2.0 +/// +/// Unless required by applicable law or agreed to in writing, software +/// distributed under the License is distributed on an "AS IS" BASIS, +/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +/// See the License for the specific language governing permissions and +/// limitations under the License. + +#import +#import +#import + +#include + +#include "Source/santad/Logs/EndpointSecurity/Writers/FSSpool/fsspool.h" +#include "Source/santad/Logs/EndpointSecurity/Writers/FSSpool/fsspool_log_batch_writer.h" +#include "google/protobuf/any.pb.h" +#include "google/protobuf/timestamp.pb.h" + +using fsspool::FsSpoolLogBatchWriter; +using fsspool::FsSpoolWriter; + +static constexpr size_t kSpoolSize = 1048576; + +#define XCTAssertStatusOk(s) XCTAssertTrue((s).ok()) +#define XCTAssertStatusNotOk(s) XCTAssertFalse((s).ok()) + +google::protobuf::Any TestAnyTimestamp(int64_t s, int32_t n) { + google::protobuf::Timestamp v; + v.set_seconds(s); + v.set_nanos(n); + google::protobuf::Any any; + any.PackFrom(v); + return any; +} + +@interface FSSpoolTest : XCTestCase +@property NSString *testDir; +@property NSString *baseDir; +@property NSString *spoolDir; +@property NSString *tmpDir; +@property NSFileManager *fileMgr; +@end + +@implementation FSSpoolTest + +- (void)setUp { + self.testDir = [NSString stringWithFormat:@"%@fsspool-%d", NSTemporaryDirectory(), getpid()]; + self.baseDir = [NSString stringWithFormat:@"%@/base", self.testDir]; + self.spoolDir = [NSString stringWithFormat:@"%@/new", self.baseDir]; + self.tmpDir = [NSString stringWithFormat:@"%@/tmp", self.baseDir]; + + self.fileMgr = [NSFileManager defaultManager]; + + XCTAssertFalse([self.fileMgr fileExistsAtPath:self.baseDir]); + XCTAssertFalse([self.fileMgr fileExistsAtPath:self.spoolDir]); + XCTAssertFalse([self.fileMgr fileExistsAtPath:self.tmpDir]); + + XCTAssertTrue([self.fileMgr createDirectoryAtPath:self.testDir + withIntermediateDirectories:YES + attributes:nil + error:nil]); + + // NSLog(@"testDir: %@", self.testDir); + // NSLog(@"baseDir: %@", self.baseDir); +} + +- (void)tearDown { + XCTAssertTrue([self.fileMgr removeItemAtPath:self.testDir error:nil]); +} + +- (void)testSimpleWrite { + auto writer = std::make_unique([self.baseDir UTF8String], kSpoolSize); + + XCTAssertFalse([self.fileMgr fileExistsAtPath:self.baseDir]); + XCTAssertFalse([self.fileMgr fileExistsAtPath:self.spoolDir]); + XCTAssertFalse([self.fileMgr fileExistsAtPath:self.tmpDir]); + + std::string testData = "Good morning. This is some nice test data."; + XCTAssertStatusOk(writer->WriteMessage(testData)); + + NSError *err = nil; + XCTAssertEqual([[self.fileMgr contentsOfDirectoryAtPath:self.tmpDir error:&err] count], 0); + XCTAssertNil(err); + XCTAssertEqual([[self.fileMgr contentsOfDirectoryAtPath:self.spoolDir error:&err] count], 1); + XCTAssertNil(err); +} + +- (void)testSpoolFull { + auto writer = std::make_unique([self.baseDir UTF8String], kSpoolSize); + const std::string largeMessage(kSpoolSize + 1, '\x42'); + + XCTAssertFalse([self.fileMgr fileExistsAtPath:self.baseDir]); + XCTAssertFalse([self.fileMgr fileExistsAtPath:self.spoolDir]); + XCTAssertFalse([self.fileMgr fileExistsAtPath:self.tmpDir]); + + // Write the first message. This will make the spool directory larger than the max. + XCTAssertStatusOk(writer->WriteMessage(largeMessage)); + + // Ensure the files are created + XCTAssertTrue([self.fileMgr fileExistsAtPath:self.baseDir]); + XCTAssertTrue([self.fileMgr fileExistsAtPath:self.spoolDir]); + XCTAssertTrue([self.fileMgr fileExistsAtPath:self.tmpDir]); + + NSError *err = nil; + XCTAssertEqual([[self.fileMgr contentsOfDirectoryAtPath:self.tmpDir error:&err] count], 0); + XCTAssertNil(err); + XCTAssertEqual([[self.fileMgr contentsOfDirectoryAtPath:self.spoolDir error:&err] count], 1); + XCTAssertNil(err); + + // Try to write again, but expect failure. File counts shouldn't change. + XCTAssertStatusNotOk(writer->WriteMessage(largeMessage)); + + XCTAssertEqual([[self.fileMgr contentsOfDirectoryAtPath:self.tmpDir error:&err] count], 0); + XCTAssertNil(err); + XCTAssertEqual([[self.fileMgr contentsOfDirectoryAtPath:self.spoolDir error:&err] count], 1); + XCTAssertNil(err); +} + +- (void)testWriteMessageNoFlush { + auto writer = std::make_unique([self.baseDir UTF8String], kSpoolSize); + FsSpoolLogBatchWriter batch_writer(writer.get(), 10); + + // Ensure that writing in batch mode doesn't flsuh on individual writes. + XCTAssertStatusOk(batch_writer.WriteMessage(TestAnyTimestamp(123, 456))); + + XCTAssertFalse([self.fileMgr fileExistsAtPath:self.baseDir]); + XCTAssertFalse([self.fileMgr fileExistsAtPath:self.spoolDir]); + XCTAssertFalse([self.fileMgr fileExistsAtPath:self.tmpDir]); +} + +- (void)testWriteMessageFlushAtCapacity { + static const int kCapacity = 5; + auto writer = std::make_unique([self.baseDir UTF8String], kSpoolSize); + FsSpoolLogBatchWriter batch_writer(writer.get(), kCapacity); + + // Ensure batch flushed once capacity exceeded + for (int i = 0; i < kCapacity + 1; i++) { + XCTAssertStatusOk(batch_writer.WriteMessage(TestAnyTimestamp(123, 456))); + } + + NSError *err = nil; + XCTAssertEqual([[self.fileMgr contentsOfDirectoryAtPath:self.tmpDir error:&err] count], 0); + XCTAssertNil(err); + XCTAssertEqual([[self.fileMgr contentsOfDirectoryAtPath:self.spoolDir error:&err] count], 1); + XCTAssertNil(err); +} + +- (void)testWriteMessageMultipleFlush { + static const int kCapacity = 5; + static const int kExpectedFlushes = 3; + + auto writer = std::make_unique([self.baseDir UTF8String], kSpoolSize); + FsSpoolLogBatchWriter batch_writer(writer.get(), kCapacity); + + // Ensure batch flushed expected number of times + for (int i = 0; i < kExpectedFlushes * kCapacity + 1; i++) { + XCTAssertStatusOk(batch_writer.WriteMessage(TestAnyTimestamp(123, 456))); + } + + NSError *err = nil; + XCTAssertEqual([[self.fileMgr contentsOfDirectoryAtPath:self.tmpDir error:&err] count], 0); + XCTAssertNil(err); + XCTAssertEqual([[self.fileMgr contentsOfDirectoryAtPath:self.spoolDir error:&err] count], + kExpectedFlushes); + XCTAssertNil(err); +} + +- (void)testWriteMessageFlushOnDestroy { + static const int kCapacity = 10; + static const int kNumberOfWrites = 7; + + auto writer = std::make_unique([self.baseDir UTF8String], kSpoolSize); + + { + // Extra scope to enforce early destroy of batch_writer. + FsSpoolLogBatchWriter batch_writer(writer.get(), kCapacity); + for (int i = 0; i < kNumberOfWrites; i++) { + XCTAssertStatusOk(batch_writer.WriteMessage(TestAnyTimestamp(123, 456))); + } + + // Ensure nothing was written yet + XCTAssertFalse([self.fileMgr fileExistsAtPath:self.baseDir]); + XCTAssertFalse([self.fileMgr fileExistsAtPath:self.spoolDir]); + XCTAssertFalse([self.fileMgr fileExistsAtPath:self.tmpDir]); + } + + // Ensure the write happens when FsSpoolLogBatchWriter destructed + NSError *err = nil; + XCTAssertEqual([[self.fileMgr contentsOfDirectoryAtPath:self.tmpDir error:&err] count], 0); + XCTAssertNil(err); + XCTAssertEqual([[self.fileMgr contentsOfDirectoryAtPath:self.spoolDir error:&err] count], 1); + XCTAssertNil(err); +} + +@end diff --git a/WORKSPACE b/WORKSPACE index 904be7396..feac679b5 100644 --- a/WORKSPACE +++ b/WORKSPACE @@ -13,19 +13,6 @@ http_archive( urls = ["https://github.com/bazelbuild/rules_apple/releases/download/1.1.0/rules_apple.1.1.0.tar.gz"], ) -http_archive( - name = "com_google_protobuf", - patch_args = ["-p1"], - patches = ["//external_patches/com_google_protobuf:10120.patch"], - sha256 = "73c95c7b0c13f597a6a1fec7121b07e90fd12b4ed7ff5a781253b3afe07fc077", - strip_prefix = "protobuf-3.21.6", - urls = ["https://github.com/protocolbuffers/protobuf/archive/v3.21.6.tar.gz"], -) - -load("@com_google_protobuf//:protobuf_deps.bzl", "protobuf_deps") - -protobuf_deps() - load("@build_bazel_rules_apple//apple:repositories.bzl", "apple_rules_dependencies") apple_rules_dependencies() @@ -56,6 +43,29 @@ http_archive( urls = ["https://github.com/google/googletest/archive/58d77fa8070e8cec2dc1ed015d66b454c8d78850.zip"], ) +# Abseil - Abseil LTS branch, June 2022, Patch 1 +http_archive( + name = "com_google_absl", + sha256 = "b9f490fae1c0d89a19073a081c3c588452461e5586e4ae31bc50a8f36339135e", + strip_prefix = "abseil-cpp-8c0b94e793a66495e0b1f34a5eb26bd7dc672db0", + urls = ["https://github.com/abseil/abseil-cpp/archive/8c0b94e793a66495e0b1f34a5eb26bd7dc672db0.zip"], +) + +http_archive( + name = "com_google_protobuf", + patch_args = ["-p1"], + patches = ["//external_patches/com_google_protobuf:10120.patch"], + sha256 = "73c95c7b0c13f597a6a1fec7121b07e90fd12b4ed7ff5a781253b3afe07fc077", + strip_prefix = "protobuf-3.21.6", + urls = ["https://github.com/protocolbuffers/protobuf/archive/v3.21.6.tar.gz"], +) + +# Note: Protobuf deps must be loaded after defining the ABSL archive since +# protobuf repo would pull an in earlier version of ABSL. +load("@com_google_protobuf//:protobuf_deps.bzl", "protobuf_deps") + +protobuf_deps() + # Macops MOL* dependencies git_repository(