diff --git a/CMakeLists.txt b/CMakeLists.txt index 41247d988e..e90051d7bf 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -339,6 +339,9 @@ endfunction() SET(TEST_DIR ${CMAKE_SOURCE_DIR}/libminifi/test) include(Extensions) +add_subdirectory(core) +add_subdirectory(utils) +add_subdirectory(extension-utils) add_subdirectory(libminifi) if (ENABLE_ALL OR ENABLE_AZURE) diff --git a/controller/MiNiFiController.cpp b/controller/MiNiFiController.cpp index 673680b734..33be0ed715 100644 --- a/controller/MiNiFiController.cpp +++ b/controller/MiNiFiController.cpp @@ -39,7 +39,7 @@ std::shared_ptr getControllerServic const std::string &service_name) { std::string nifi_configuration_class_name = "adaptiveconfiguration"; - minifi::core::extension::ExtensionManager::get().initialize(configuration); + minifi::core::extension::ExtensionManagerImpl::get().initialize(configuration); configuration->get(minifi::Configure::nifi_configuration_class_name, nifi_configuration_class_name); auto flow_configuration = minifi::core::createFlowConfiguration( @@ -78,7 +78,7 @@ std::shared_ptr getSSLContextService(con if (nullptr == secure_context) { std::string secureStr; if (configuration->get(minifi::Configure::nifi_remote_input_secure, secureStr) && minifi::utils::string::toBool(secureStr).value_or(false)) { - secure_context = std::make_shared("ControllerSocketProtocolSSL", configuration); + secure_context = std::make_shared("ControllerSocketProtocolSSL", configuration); secure_context->onEnable(); } } else { @@ -96,7 +96,7 @@ int main(int argc, char **argv) { return -1; } - const auto configuration = std::make_shared(); + const auto configuration = std::make_shared(); configuration->setHome(minifi_home); configuration->loadConfigureFile(DEFAULT_NIFI_PROPERTIES_FILE); diff --git a/controller/tests/ControllerTests.cpp b/controller/tests/ControllerTests.cpp index 08ae1a4bfa..adbe80ca69 100644 --- a/controller/tests/ControllerTests.cpp +++ b/controller/tests/ControllerTests.cpp @@ -34,6 +34,7 @@ #include "controllers/SSLContextService.h" #include "utils/StringUtils.h" #include "state/UpdateController.h" +#include "core/state/nodes/ResponseNodeLoader.h" using namespace std::literals::chrono_literals; @@ -201,10 +202,10 @@ class TestControllerSocketReporter : public c2::ControllerSocketReporter { } }; -class TestControllerServiceProvider : public core::controller::ControllerServiceProvider { +class TestControllerServiceProvider : public core::controller::ControllerServiceProviderImpl { public: explicit TestControllerServiceProvider(std::shared_ptr ssl_context_service) - : core::controller::ControllerServiceProvider("TestControllerServiceProvider"), + : core::controller::ControllerServiceProviderImpl("TestControllerServiceProvider"), ssl_context_service_(std::move(ssl_context_service)) { } std::shared_ptr getControllerService(const std::string&) const override { @@ -239,7 +240,7 @@ class ControllerTestFixture { }; ControllerTestFixture() - : configuration_(std::make_shared()), + : configuration_(std::make_shared()), controller_(std::make_shared()), update_sink_(std::make_unique(controller_)) { configuration_->set(minifi::Configure::controller_socket_host, "localhost"); @@ -249,7 +250,7 @@ class ControllerTestFixture { configuration_->set(minifi::Configure::nifi_security_client_pass_phrase, "abcdefgh"); configuration_->set(minifi::Configure::nifi_security_client_ca_certificate, (minifi::utils::file::FileUtils::get_executable_dir() / "resources" / "root-ca.pem").string()); configuration_->set(minifi::Configure::controller_ssl_context_service, "SSLContextService"); - ssl_context_service_ = std::make_shared("SSLContextService", configuration_); + ssl_context_service_ = std::make_shared("SSLContextService", configuration_); ssl_context_service_->onEnable(); controller_service_provider_ = std::make_unique(ssl_context_service_); controller_socket_data_.host = "localhost"; @@ -478,7 +479,7 @@ TEST_CASE_METHOD(ControllerTestFixture, "Test manifest getter", "[controllerTest } auto reporter = std::make_shared("ControllerSocketMetricsPublisher"); - auto response_node_loader = std::make_shared(configuration_, std::vector>{}, nullptr); + auto response_node_loader = std::make_shared(configuration_, std::vector>{}, nullptr); reporter->initialize(configuration_, response_node_loader); initalizeControllerSocket(reporter); @@ -501,7 +502,7 @@ TEST_CASE_METHOD(ControllerTestFixture, "Test jstack getter", "[controllerTests] } auto reporter = std::make_shared("ControllerSocketMetricsPublisher"); - auto response_node_loader = std::make_shared(configuration_, std::vector>{}, nullptr); + auto response_node_loader = std::make_shared(configuration_, std::vector>{}, nullptr); reporter->initialize(configuration_, response_node_loader); initalizeControllerSocket(reporter); @@ -529,7 +530,7 @@ TEST_CASE_METHOD(ControllerTestFixture, "Test debug bundle getter", "[controller } auto reporter = std::make_shared("ControllerSocketMetricsPublisher"); - auto response_node_loader = std::make_shared(configuration_, std::vector>{}, nullptr); + auto response_node_loader = std::make_shared(configuration_, std::vector>{}, nullptr); reporter->initialize(configuration_, response_node_loader); initalizeControllerSocket(reporter); @@ -543,7 +544,7 @@ TEST_CASE_METHOD(ControllerTestFixture, "Test debug bundle is created to non-exi setConnectionType(ControllerTestFixture::ConnectionType::UNSECURE); auto reporter = std::make_shared("ControllerSocketMetricsPublisher"); - auto response_node_loader = std::make_shared(configuration_, std::vector>{}, nullptr); + auto response_node_loader = std::make_shared(configuration_, std::vector>{}, nullptr); reporter->initialize(configuration_, response_node_loader); initalizeControllerSocket(reporter); @@ -557,7 +558,7 @@ TEST_CASE_METHOD(ControllerTestFixture, "Debug bundle retrieval fails if target setConnectionType(ControllerTestFixture::ConnectionType::UNSECURE); auto reporter = std::make_shared("ControllerSocketMetricsPublisher"); - auto response_node_loader = std::make_shared(configuration_, std::vector>{}, nullptr); + auto response_node_loader = std::make_shared(configuration_, std::vector>{}, nullptr); reporter->initialize(configuration_, response_node_loader); initalizeControllerSocket(reporter); diff --git a/core/CMakeLists.txt b/core/CMakeLists.txt new file mode 100644 index 0000000000..6b2554a3d4 --- /dev/null +++ b/core/CMakeLists.txt @@ -0,0 +1,3 @@ +add_library(minifi-core INTERFACE) +target_include_directories(minifi-core INTERFACE include) +target_link_libraries(minifi-core INTERFACE gsl-lite) diff --git a/core/include/minifi-cpp/Connection.h b/core/include/minifi-cpp/Connection.h new file mode 100644 index 0000000000..3c2e2190a6 --- /dev/null +++ b/core/include/minifi-cpp/Connection.h @@ -0,0 +1,75 @@ +/** + * @file Connection.h + * Connection class declaration + * + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#pragma once + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "core/Core.h" +#include "core/Connectable.h" +#include "core/logging/Logger.h" +#include "core/Relationship.h" +#include "core/FlowFile.h" +#include "core/Repository.h" +#include "minifi-cpp/utils/Literals.h" + +namespace org::apache::nifi::minifi { + +class Connection : public virtual core::Connectable { + public: + ~Connection() override = default; + + static constexpr uint64_t DEFAULT_BACKPRESSURE_THRESHOLD_COUNT = 2000; + static constexpr uint64_t DEFAULT_BACKPRESSURE_THRESHOLD_DATA_SIZE = 100_MB; + + virtual void setSourceUUID(const utils::Identifier &uuid) = 0; + virtual void setDestinationUUID(const utils::Identifier &uuid) = 0; + virtual utils::Identifier getSourceUUID() const = 0; + virtual utils::Identifier getDestinationUUID() const = 0; + virtual void setSource(core::Connectable* source) = 0; + virtual core::Connectable* getSource() const = 0; + virtual void setDestination(core::Connectable* dest) = 0; + virtual core::Connectable* getDestination() const = 0; + virtual void addRelationship(core::Relationship relationship) = 0; + virtual const std::set &getRelationships() const = 0; + virtual void setBackpressureThresholdCount(uint64_t size) = 0; + virtual uint64_t getBackpressureThresholdCount() const = 0; + virtual void setBackpressureThresholdDataSize(uint64_t size) = 0; + virtual uint64_t getBackpressureThresholdDataSize() const = 0; + virtual void setSwapThreshold(uint64_t size) = 0; + virtual void setFlowExpirationDuration(std::chrono::milliseconds duration) = 0; + virtual std::chrono::milliseconds getFlowExpirationDuration() const = 0; + virtual void setDropEmptyFlowFiles(bool drop) = 0; + virtual bool getDropEmptyFlowFiles() const = 0; + virtual bool isEmpty() const = 0; + virtual bool backpressureThresholdReached() const = 0; + virtual uint64_t getQueueSize() const = 0; + virtual uint64_t getQueueDataSize() = 0; + virtual void multiPut(std::vector>& flows) = 0; + virtual std::shared_ptr poll(std::set> &expiredFlowRecords) = 0; + virtual void drain(bool delete_permanently) = 0; +}; +} // namespace org::apache::nifi::minifi diff --git a/libminifi/include/Exception.h b/core/include/minifi-cpp/Exception.h similarity index 100% rename from libminifi/include/Exception.h rename to core/include/minifi-cpp/Exception.h diff --git a/core/include/minifi-cpp/FlowFileRecord.h b/core/include/minifi-cpp/FlowFileRecord.h new file mode 100644 index 0000000000..87cbbbfcfa --- /dev/null +++ b/core/include/minifi-cpp/FlowFileRecord.h @@ -0,0 +1,55 @@ +/** + * @file FlowFileRecord.h + * Flow file record class declaration + * + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#pragma once + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "minifi-cpp/core/ContentRepository.h" +#include "minifi-cpp/core/FlowFile.h" +#include "minifi-cpp/core/Repository.h" +#include "io/OutputStream.h" + +namespace org::apache::nifi::minifi { + +class FlowFileRecord : public virtual core::FlowFile { + public: + virtual bool Serialize(io::OutputStream &outStream) = 0; + + //! Serialize and Persistent to the repository + virtual bool Persist(const std::shared_ptr& flowRepository) = 0; + + static std::shared_ptr DeSerialize(std::span buffer, const std::shared_ptr &content_repo, utils::Identifier &container); + static std::shared_ptr DeSerialize(io::InputStream &stream, const std::shared_ptr &content_repo, utils::Identifier &container); + static std::shared_ptr DeSerialize(const std::string& key, const std::shared_ptr& flowRepository, + const std::shared_ptr &content_repo, utils::Identifier &container); + + virtual std::string getContentFullPath() const = 0; +}; + +} // namespace org::apache::nifi::minifi diff --git a/core/include/minifi-cpp/ResourceClaim.h b/core/include/minifi-cpp/ResourceClaim.h new file mode 100644 index 0000000000..e0b47797e1 --- /dev/null +++ b/core/include/minifi-cpp/ResourceClaim.h @@ -0,0 +1,61 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#pragma once + +#include +#include +#include +#include +#include +#include +#include +#include "core/Core.h" +#include "core/StreamManager.h" +#include "properties/Configure.h" +#include "utils/Id.h" + +namespace org::apache::nifi::minifi { + +namespace core { +class ContentRepository; +} // namespace core + +class ResourceClaim { + public: + using Path = std::string; + + virtual ~ResourceClaim() = default; + virtual void increaseFlowFileRecordOwnedCount() = 0; + virtual void decreaseFlowFileRecordOwnedCount() = 0; + virtual uint64_t getFlowFileRecordOwnedCount() = 0; + virtual Path getContentFullPath() const = 0; + virtual bool exists() = 0; + + static std::shared_ptr create(std::shared_ptr repository); + + virtual std::ostream& write(std::ostream& stream) const = 0; + + friend std::ostream& operator<<(std::ostream& stream, const ResourceClaim& claim) { + return claim.write(stream); + } + + friend std::ostream& operator<<(std::ostream& stream, const std::shared_ptr& claim) { + return claim->write(stream); + } +}; + +} // namespace org::apache::nifi::minifi diff --git a/libminifi/include/SwapManager.h b/core/include/minifi-cpp/SwapManager.h similarity index 97% rename from libminifi/include/SwapManager.h rename to core/include/minifi-cpp/SwapManager.h index ed3535dbc0..2137d4d53f 100644 --- a/libminifi/include/SwapManager.h +++ b/core/include/minifi-cpp/SwapManager.h @@ -21,7 +21,7 @@ #include #include -#include "core/FlowFile.h" +#include "minifi-cpp/core/FlowFile.h" #include "utils/Id.h" namespace org::apache::nifi::minifi { diff --git a/core/include/minifi-cpp/agent/agent_docs.h b/core/include/minifi-cpp/agent/agent_docs.h new file mode 100644 index 0000000000..3f961d8eca --- /dev/null +++ b/core/include/minifi-cpp/agent/agent_docs.h @@ -0,0 +1,71 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#pragma once + +#include +#include +#include +#include + +#include "minifi-cpp/core/Annotation.h" +#include "minifi-cpp/core/DynamicProperty.h" +#include "minifi-cpp/core/OutputAttributeDefinition.h" +#include "minifi-cpp/core/Property.h" +#include "minifi-cpp/core/Relationship.h" +#include "minifi-cpp/core/RelationshipDefinition.h" + +namespace org::apache::nifi::minifi { + +enum class ResourceType { + Processor, ControllerService, InternalResource, DescriptionOnly +}; + +struct ClassDescription { + ResourceType type_ = ResourceType::Processor; + std::string short_name_{}; + std::string full_name_{}; + std::string description_{}; + std::vector class_properties_{}; + std::span dynamic_properties_{}; + std::vector class_relationships_{}; + std::span output_attributes_{}; + bool supports_dynamic_properties_ = false; + bool supports_dynamic_relationships_ = false; + std::string inputRequirement_{}; + bool isSingleThreaded_ = false; +}; + +struct Components { + std::vector processors_; + std::vector controller_services_; + std::vector other_components_; + + [[nodiscard]] bool empty() const noexcept { + return processors_.empty() && controller_services_.empty() && other_components_.empty(); + } +}; + +class AgentDocs { + public: + static const std::map& getClassDescriptions(); + static std::map& getMutableClassDescriptions(); + + template + static void createClassDescription(const std::string& group, const std::string& name); +}; + +} // namespace org::apache::nifi::minifi diff --git a/libminifi/include/agent/agent_version.h b/core/include/minifi-cpp/agent/agent_version.h similarity index 83% rename from libminifi/include/agent/agent_version.h rename to core/include/minifi-cpp/agent/agent_version.h index f1aac9f100..0562a5b382 100644 --- a/libminifi/include/agent/agent_version.h +++ b/core/include/minifi-cpp/agent/agent_version.h @@ -15,8 +15,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -#ifndef LIBMINIFI_INCLUDE_AGENT_AGENT_VERSION_H_ -#define LIBMINIFI_INCLUDE_AGENT_AGENT_VERSION_H_ +#pragma once // The Windows version of pyconfig.h (https://github.com/python/cpython/blob/3.10/PC/pyconfig.h, also on main as of 2022-08-18) // rather unhelpfully #define's COMPILER as the detected compiler version. Since we have a COMPILER variable below, we need to #undef it @@ -25,12 +24,9 @@ #include #include -#include "utils/Export.h" +#include "minifi-cpp/utils/Export.h" -namespace org { -namespace apache { -namespace nifi { -namespace minifi { +namespace org::apache::nifi::minifi { class AgentBuild { public: @@ -44,9 +40,4 @@ class AgentBuild { static std::vector getExtensions(); }; -} // namespace minifi -} // namespace nifi -} // namespace apache -} // namespace org - -#endif // LIBMINIFI_INCLUDE_AGENT_AGENT_VERSION_H_ +} // namespace namespace org::apache::nifi::minifi diff --git a/core/include/minifi-cpp/agent/build_description.h b/core/include/minifi-cpp/agent/build_description.h new file mode 100644 index 0000000000..e85f895cbd --- /dev/null +++ b/core/include/minifi-cpp/agent/build_description.h @@ -0,0 +1,49 @@ +/** + * + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#pragma once + +#include +#include +#include +#include +#include +#include "agent_docs.h" + +namespace org::apache::nifi::minifi { + +struct BundleDetails { + std::string artifact; + std::string group; + std::string version; +}; + +class ExternalBuildDescription { + private: + static std::vector &getExternal(); + + static std::map &getExternalMappings(); + + public: + static void addExternalComponent(const BundleDetails& details, const ClassDescription& description); + + static Components getClassDescriptions(const std::string &group); + + static std::vector getExternalGroups(); +}; + +} // namespace org::apache::nifi::minifi diff --git a/core/include/minifi-cpp/controllers/AttributeProviderService.h b/core/include/minifi-cpp/controllers/AttributeProviderService.h new file mode 100644 index 0000000000..b366ddf20f --- /dev/null +++ b/core/include/minifi-cpp/controllers/AttributeProviderService.h @@ -0,0 +1,35 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#pragma once + +#include +#include +#include +#include + +#include "minifi-cpp/core/controller/ControllerService.h" + +namespace org::apache::nifi::minifi::controllers { + +class AttributeProviderService : public virtual core::controller::ControllerService { + public: + using AttributeMap = std::unordered_map; + virtual std::optional> getAttributes() = 0; + virtual std::string_view name() const = 0; +}; + +} // namespace org::apache::nifi::minifi::controllers diff --git a/libminifi/include/controllers/RecordSetReader.h b/core/include/minifi-cpp/controllers/RecordSetReader.h similarity index 86% rename from libminifi/include/controllers/RecordSetReader.h rename to core/include/minifi-cpp/controllers/RecordSetReader.h index 15f76fe6fe..1c28b4f8b9 100644 --- a/libminifi/include/controllers/RecordSetReader.h +++ b/core/include/minifi-cpp/controllers/RecordSetReader.h @@ -19,19 +19,17 @@ #include "core/controller/ControllerService.h" -#include "core/FlowFile.h" -#include "core/ProcessSession.h" -#include "Record.h" +#include "minifi-cpp/core/FlowFile.h" +#include "minifi-cpp/core/ProcessSession.h" +#include "minifi-cpp/core/Record.h" #include "utils/Enum.h" #include "utils/ProcessorConfigUtils.h" namespace org::apache::nifi::minifi::core { -class RecordSetReader : public controller::ControllerService { +class RecordSetReader : public virtual controller::ControllerService { public: - using ControllerService::ControllerService; - virtual nonstd::expected read(const std::shared_ptr& flow_file, ProcessSession& session) = 0; }; diff --git a/libminifi/include/controllers/RecordSetWriter.h b/core/include/minifi-cpp/controllers/RecordSetWriter.h similarity index 80% rename from libminifi/include/controllers/RecordSetWriter.h rename to core/include/minifi-cpp/controllers/RecordSetWriter.h index 34ee5d87f2..65eecb5b53 100644 --- a/libminifi/include/controllers/RecordSetWriter.h +++ b/core/include/minifi-cpp/controllers/RecordSetWriter.h @@ -16,18 +16,16 @@ */ #pragma once -#include "core/controller/ControllerService.h" +#include "minifi-cpp/core/controller/ControllerService.h" -#include "core/FlowFile.h" -#include "core/ProcessSession.h" -#include "Record.h" +#include "minifi-cpp/core/FlowFile.h" +#include "minifi-cpp/core/ProcessSession.h" +#include "minifi-cpp/core/Record.h" namespace org::apache::nifi::minifi::core { -class RecordSetWriter : public controller::ControllerService { +class RecordSetWriter : public virtual controller::ControllerService { public: - using ControllerService::ControllerService; - virtual void write(const RecordSet& record_set, const std::shared_ptr& flow_file, ProcessSession& session) = 0; }; diff --git a/core/include/minifi-cpp/controllers/SSLContextService.h b/core/include/minifi-cpp/controllers/SSLContextService.h new file mode 100644 index 0000000000..f16ef935e1 --- /dev/null +++ b/core/include/minifi-cpp/controllers/SSLContextService.h @@ -0,0 +1,50 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#pragma once + +#include +#include +#include +#include + +#include "minifi-cpp/core/controller/ControllerService.h" + +namespace org::apache::nifi::minifi::controllers { + +/** + * SSLContextService provides a configurable controller service from + * which we can provide an SSL Context or component parts that go + * into creating one. + * + * Justification: Abstracts SSL support out of processors into a + * configurable controller service. + */ +class SSLContextService : public virtual core::controller::ControllerService { + public: + virtual const std::filesystem::path& getCertificateFile() const = 0; + virtual const std::string& getPassphrase() const = 0; + virtual const std::filesystem::path& getPrivateKeyFile() const = 0; + virtual const std::filesystem::path& getCACertificate() const = 0; + + + virtual void setMinTlsVersion(long min_version) = 0; // NOLINT(runtime/int) long due to SSL lib API + + virtual void setMaxTlsVersion(long max_version) = 0; // NOLINT(runtime/int) long due to SSL lib API + virtual bool configure_ssl_context(void* ssl_ctx) = 0; +}; + +} // namespace org::apache::nifi::minifi::controllers diff --git a/core/include/minifi-cpp/controllers/ThreadManagementService.h b/core/include/minifi-cpp/controllers/ThreadManagementService.h new file mode 100644 index 0000000000..139e39c051 --- /dev/null +++ b/core/include/minifi-cpp/controllers/ThreadManagementService.h @@ -0,0 +1,75 @@ +/** + * + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include +#include +#include +#include + +#include "utils/StringUtils.h" +#include "io/validation.h" +#include "minifi-cpp/core/controller/ControllerService.h" + +namespace org::apache::nifi::minifi::controllers { + +/** + * Purpose: Thread management service provides a contextual awareness across + * thread pools that enables us to deliver QOS to an agent. + */ +class ThreadManagementService : public virtual core::controller::ControllerService { + public: + /** + * Helps to determine if the number of tasks will increase the pools above their threshold. + * @param new_tasks tasks to be added. + * @return true if above max, false otherwise. + */ + virtual bool isAboveMax(const int new_tasks) = 0; + + /** + * Returns the max number of threads allowed by all pools + * @return max threads. + */ + virtual uint16_t getMaxThreads() = 0; + + /** + * Function based on cooperative multitasking that will tell a caller whether or not the number of threads should be reduced. + * @return true if threading impacts QOS. + */ + virtual bool shouldReduce() = 0; + + /** + * Function to indicate to this controller service that we've reduced threads in a threadpool + */ + virtual void reduce() = 0; + + /** + * Registration function to tabulate total threads. + * @param threads threads from a thread pool. + */ + virtual void registerThreadCount(const int threads) = 0; + + /** + * Function to help callers identify if they can increase threads. + * @return true if QOS won't be breached. + */ + virtual bool canIncrease() = 0; +}; + +} // namespace org::apache::nifi::minifi::controllers diff --git a/core/include/minifi-cpp/controllers/keyvalue/KeyValueStateStorage.h b/core/include/minifi-cpp/controllers/keyvalue/KeyValueStateStorage.h new file mode 100644 index 0000000000..e3da4e928b --- /dev/null +++ b/core/include/minifi-cpp/controllers/keyvalue/KeyValueStateStorage.h @@ -0,0 +1,25 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +namespace org::apache::nifi::minifi::controllers { + +constexpr const char *ALWAYS_PERSIST_PROPERTY_NAME = "Always Persist"; +constexpr const char *AUTO_PERSISTENCE_INTERVAL_PROPERTY_NAME = "Auto Persistence Interval"; + +} // namespace org::apache::nifi::minifi::controllers diff --git a/libminifi/include/core/AgentIdentificationProvider.h b/core/include/minifi-cpp/core/AgentIdentificationProvider.h similarity index 100% rename from libminifi/include/core/AgentIdentificationProvider.h rename to core/include/minifi-cpp/core/AgentIdentificationProvider.h diff --git a/libminifi/include/core/Annotation.h b/core/include/minifi-cpp/core/Annotation.h similarity index 100% rename from libminifi/include/core/Annotation.h rename to core/include/minifi-cpp/core/Annotation.h diff --git a/libminifi/include/core/CachedValueValidator.h b/core/include/minifi-cpp/core/CachedValueValidator.h similarity index 93% rename from libminifi/include/core/CachedValueValidator.h rename to core/include/minifi-cpp/core/CachedValueValidator.h index b6f261fa0a..f0e2fd58e7 100644 --- a/libminifi/include/core/CachedValueValidator.h +++ b/core/include/minifi-cpp/core/CachedValueValidator.h @@ -23,16 +23,20 @@ #include "state/Value.h" #include "ValidationResult.h" +#include "minifi-cpp/utils/gsl.h" namespace org::apache::nifi::minifi::core { class PropertyValue; class PropertyValidator; +template struct Converter; + namespace internal { class CachedValueValidator { friend class core::PropertyValue; + template friend struct core::Converter; public: enum class Result { diff --git a/core/include/minifi-cpp/core/ClassLoader.h b/core/include/minifi-cpp/core/ClassLoader.h new file mode 100644 index 0000000000..d1fcac4a0b --- /dev/null +++ b/core/include/minifi-cpp/core/ClassLoader.h @@ -0,0 +1,103 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include +#include +#include +#include +#include +#include + +#include "Core.h" +#include "ObjectFactory.h" + +namespace org::apache::nifi::minifi::core { + +#define RESOURCE_FAILURE -1 + +#define RESOURCE_SUCCESS 1 + +#ifdef WIN32 +#define RTLD_LAZY 0 +#define RTLD_NOW 0 + +#define RTLD_GLOBAL (1 << 1) +#define RTLD_LOCAL (1 << 2) +#endif + +/** + * Processor class loader that accepts + * a variety of mechanisms to load in shared + * objects. + */ +class ClassLoader { + public: + static ClassLoader &getDefaultClassLoader(); + + /** + * Retrieves a class loader + * @param name name of class loader + * @return class loader reference + */ + virtual ClassLoader& getClassLoader(const std::string& child_name) = 0; + + /** + * Register a class with the give ProcessorFactory + */ + virtual void registerClass(const std::string &clazz, std::unique_ptr factory) = 0; + + virtual void unregisterClass(const std::string& clazz) = 0; + + virtual std::optional getGroupForClass(const std::string &class_name) const = 0; + + virtual std::unique_ptr instantiate(const std::string &class_name, const std::string &name, std::function filter) = 0; + + virtual std::unique_ptr instantiate(const std::string &class_name, const utils::Identifier &uuid, std::function filter) = 0; + + virtual CoreComponent* instantiateRaw(const std::string &class_name, const std::string &name, std::function filter) = 0; + + /** + * Instantiate object based on class_name + * @param class_name class to create + * @param uuid uuid of object + * @return nullptr or object created from class_name definition. + */ + template + std::unique_ptr instantiate(const std::string &class_name, const std::string &name); + + /** + * Instantiate object based on class_name + * @param class_name class to create + * @param uuid uuid of object + * @return nullptr or object created from class_name definition. + */ + template + std::unique_ptr instantiate(const std::string &class_name, const utils::Identifier &uuid); + + /** + * Instantiate object based on class_name + * @param class_name class to create + * @param uuid uuid of object + * @return nullptr or object created from class_name definition. + */ + template + T *instantiateRaw(const std::string &class_name, const std::string &name); +}; + +} // namespace org::apache::nifi::minifi::core diff --git a/core/include/minifi-cpp/core/ConfigurableComponent.h b/core/include/minifi-cpp/core/ConfigurableComponent.h new file mode 100644 index 0000000000..4c38e2296d --- /dev/null +++ b/core/include/minifi-cpp/core/ConfigurableComponent.h @@ -0,0 +1,89 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include +#include + +#include "Core.h" +#include +#include +#include +#include + +#include "logging/Logger.h" +#include "Property.h" +#include "utils/gsl.h" + +namespace org::apache::nifi::minifi::core { + +/** + * Represents a configurable component + * Purpose: Extracts configuration items for all components and localized them + */ +class ConfigurableComponent { + public: + virtual bool getProperty(const std::string& name, uint64_t& value) const = 0; + virtual bool getProperty(const std::string& name, int64_t& value) const = 0; + virtual bool getProperty(const std::string& name, uint32_t& value) const = 0; + virtual bool getProperty(const std::string& name, int& value) const = 0; + virtual bool getProperty(const std::string& name, bool& value) const = 0; + virtual bool getProperty(const std::string& name, double& value) const = 0; + virtual bool getProperty(const std::string& name, std::string& value) const = 0; + + + virtual bool getProperty(const std::string &name, Property &prop) const = 0; + virtual bool setProperty(const std::string& name, const std::string& value) = 0; + virtual bool updateProperty(const std::string &name, const std::string &value) = 0; + virtual bool updateProperty(const PropertyReference& property, std::string_view value) = 0; + virtual bool setProperty(const Property& prop, const std::string& value) = 0; + virtual bool setProperty(const PropertyReference& property, std::string_view value) = 0; + virtual bool setProperty(const Property& prop, PropertyValue &value) = 0; + virtual bool supportsDynamicProperties() const = 0; + virtual bool supportsDynamicRelationships() const = 0; + virtual bool getDynamicProperty(const std::string& name, std::string &value) const = 0; + virtual bool getDynamicProperty(const std::string& name, core::Property &item) const = 0; + virtual bool setDynamicProperty(const std::string& name, const std::string& value) = 0; + virtual bool updateDynamicProperty(const std::string &name, const std::string &value) = 0; + virtual void onPropertyModified(const Property& /*old_property*/, const Property& /*new_property*/) = 0; + virtual void onDynamicPropertyModified(const Property& /*old_property*/, const Property& /*new_property*/) = 0; + virtual std::vector getDynamicPropertyKeys() const = 0; + virtual std::map getProperties() const = 0; + virtual bool isPropertyExplicitlySet(const Property&) const = 0; + virtual bool isPropertyExplicitlySet(const PropertyReference&) const = 0; + virtual ~ConfigurableComponent() = default; + virtual void initialize() = 0; + virtual bool canEdit() = 0; + + template + bool getProperty(const std::string& name, T& value) const; + + template + bool getProperty(const std::string& name, std::optional& value) const; + + template + bool getProperty(const core::PropertyReference& property, T& value) const; + + template requires(std::is_default_constructible_v) + std::optional getProperty(const std::string& property_name) const; + + template requires(std::is_default_constructible_v) + std::optional getProperty(const core::PropertyReference& property) const; +}; + +} // namespace org::apache::nifi::minifi::core diff --git a/core/include/minifi-cpp/core/Connectable.h b/core/include/minifi-cpp/core/Connectable.h new file mode 100644 index 0000000000..fb116532b0 --- /dev/null +++ b/core/include/minifi-cpp/core/Connectable.h @@ -0,0 +1,71 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include +#include +#include +#include +#include +#include +#include +#include + +#include "Core.h" +#include +#include "Relationship.h" +#include "Scheduling.h" +#include "minifi-cpp/core/state/FlowIdentifier.h" +#include "utils/gsl.h" + +namespace org::apache::nifi::minifi::core { + +class FlowFile; + +/** + * Represents the base connectable component + * Purpose: As in NiFi, this represents a connection point and allows the derived + * object to be connected to other connectables. + */ +class Connectable : public virtual CoreComponent { + public: + virtual bool isSupportedRelationship(const Relationship &relationship) = 0; + virtual std::vector getSupportedRelationships() const = 0; + virtual void addAutoTerminatedRelationship(const core::Relationship& relationship) = 0; + virtual void setAutoTerminatedRelationships(std::span relationships) = 0; + virtual bool isAutoTerminated(const Relationship &relationship) = 0; + virtual std::chrono::milliseconds getPenalizationPeriod() const = 0; + virtual std::set getOutGoingConnections(const std::string &relationship) = 0; + virtual void put(const std::shared_ptr& /*flow*/) = 0; + virtual void restore(const std::shared_ptr& file) = 0; + virtual Connectable* getNextIncomingConnection() = 0; + virtual Connectable* pickIncomingConnection() = 0; + virtual bool hasIncomingConnections() const = 0; + virtual uint8_t getMaxConcurrentTasks() const = 0; + virtual void setMaxConcurrentTasks(uint8_t tasks) = 0; + virtual void yield() = 0; + ~Connectable() override = default; + virtual bool isRunning() const = 0; + virtual void waitForWork(std::chrono::milliseconds timeout) = 0; + virtual void notifyWork() = 0; + virtual bool isWorkAvailable() = 0; + virtual void setFlowIdentifier(const std::shared_ptr &version) = 0; + virtual std::shared_ptr getFlowIdentifier() const = 0; +}; + +} // namespace org::apache::nifi::minifi::core diff --git a/core/include/minifi-cpp/core/ContentRepository.h b/core/include/minifi-cpp/core/ContentRepository.h new file mode 100644 index 0000000000..ce8b889549 --- /dev/null +++ b/core/include/minifi-cpp/core/ContentRepository.h @@ -0,0 +1,54 @@ +/** + * + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#pragma once + +#include +#include +#include +#include +#include +#include +#include + +#include "minifi-cpp/properties/Configure.h" +#include "minifi-cpp/ResourceClaim.h" +#include "StreamManager.h" +#include "ContentSession.h" +#include "minifi-cpp/core/RepositoryMetricsSource.h" +#include "core/Core.h" +#include "utils/GeneralUtils.h" + +namespace org::apache::nifi::minifi::core { + +/** + * Content repository definition that extends StreamManager. + */ +class ContentRepository : public virtual core::CoreComponent, public virtual StreamManager, public virtual utils::EnableSharedFromThis, public virtual core::RepositoryMetricsSource { + public: + ~ContentRepository() override = default; + + virtual bool initialize(const std::shared_ptr &configure) = 0; + virtual std::shared_ptr createSession() = 0; + virtual void reset() = 0; + virtual void clearOrphans() = 0; + + virtual void start() = 0; + virtual void stop() = 0; +}; + +} // namespace org::apache::nifi::minifi::core diff --git a/libminifi/include/core/ContentSession.h b/core/include/minifi-cpp/core/ContentSession.h similarity index 70% rename from libminifi/include/core/ContentSession.h rename to core/include/minifi-cpp/core/ContentSession.h index 75bf29faf4..a48fb020ea 100644 --- a/libminifi/include/core/ContentSession.h +++ b/core/include/minifi-cpp/core/ContentSession.h @@ -21,7 +21,7 @@ #include #include #include -#include "ResourceClaim.h" +#include "minifi-cpp/ResourceClaim.h" #include "io/BaseStream.h" namespace org::apache::nifi::minifi::core { @@ -30,20 +30,12 @@ class StreamAppendLock; class ContentRepository; class ContentSession { - struct AppendState { - std::shared_ptr stream; - size_t base_size; - std::unique_ptr lock; - }; - public: - explicit ContentSession(std::shared_ptr repository): repository_(std::move(repository)) {} - virtual std::shared_ptr create() = 0; virtual std::shared_ptr write(const std::shared_ptr& resource_id) = 0; - virtual std::shared_ptr append(const std::shared_ptr& resource_id, size_t offset, const std::function&)>& on_copy); + virtual std::shared_ptr append(const std::shared_ptr& resource_id, size_t offset, const std::function&)>& on_copy) = 0; virtual std::shared_ptr read(const std::shared_ptr& resource_id) = 0; @@ -52,13 +44,6 @@ class ContentSession { virtual void rollback() = 0; virtual ~ContentSession() = default; - - protected: - virtual std::shared_ptr append(const std::shared_ptr& resource_id) = 0; - - // contains aux data on resources that have been appended to - std::map, AppendState> append_state_; - std::shared_ptr repository_; }; } // namespace org::apache::nifi::minifi::core diff --git a/core/include/minifi-cpp/core/Core.h b/core/include/minifi-cpp/core/Core.h new file mode 100644 index 0000000000..75d3175369 --- /dev/null +++ b/core/include/minifi-cpp/core/Core.h @@ -0,0 +1,56 @@ +/** + * + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#pragma once + +#ifdef WIN32 +// ignore the warning about inheriting via dominance from CoreComponent +#pragma warning(disable : 4250) +#endif + +#include +#include +#include +#include +#include +#include + +#include "utils/ArrayUtils.h" +#include "minifi-cpp/utils/Id.h" +#include "minifi-cpp/properties/Configure.h" +#include "utils/StringUtils.h" + +namespace org::apache::nifi::minifi::core { + +/** + * Base component within MiNiFi + * Purpose: Many objects store a name and UUID, therefore + * the functionality is localized here to avoid duplication + */ +class CoreComponent { + public: + virtual ~CoreComponent() = default; + + [[nodiscard]] virtual std::string getName() const = 0; + virtual void setName(std::string name) = 0; + virtual void setUUID(const utils::Identifier& uuid) = 0; + [[nodiscard]] virtual utils::Identifier getUUID() const = 0; + [[nodiscard]] virtual utils::SmallString<36> getUUIDStr() const = 0; + virtual void configure(const std::shared_ptr& /*configuration*/) = 0; +}; + +} // namespace org::apache::nifi::minifi::core diff --git a/libminifi/include/core/DynamicProperty.h b/core/include/minifi-cpp/core/DynamicProperty.h similarity index 100% rename from libminifi/include/core/DynamicProperty.h rename to core/include/minifi-cpp/core/DynamicProperty.h diff --git a/core/include/minifi-cpp/core/FlowFile.h b/core/include/minifi-cpp/core/FlowFile.h new file mode 100644 index 0000000000..9bdb5e7343 --- /dev/null +++ b/core/include/minifi-cpp/core/FlowFile.h @@ -0,0 +1,112 @@ +/** + * + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#pragma once + +#include +#include +#include +#include +#include +#include +#include +#include + +#include "minifi-cpp/ResourceClaim.h" +#include "Connectable.h" +#include "WeakReference.h" +#include "minifi-cpp/utils/FlatMap.h" + +namespace org::apache::nifi::minifi::core { + +class Connectable; + +class FlowFile : public virtual CoreComponent, public virtual ReferenceContainer { + public: + using AttributeMap = utils::FlatMap; + + virtual void copy(const FlowFile&) = 0; + + [[nodiscard]] virtual std::shared_ptr getResourceClaim() const = 0; + virtual void setResourceClaim(const std::shared_ptr& claim) = 0; + virtual void clearResourceClaim() = 0; + virtual std::shared_ptr getStashClaim(const std::string& key) = 0; + virtual void setStashClaim(const std::string& key, const std::shared_ptr& claim) = 0; + virtual void clearStashClaim(const std::string& key) = 0; + virtual bool hasStashClaim(const std::string& key) = 0; + virtual const std::vector& getlineageIdentifiers() const = 0; + virtual std::vector& getlineageIdentifiers() = 0; + [[nodiscard]] virtual bool isDeleted() const = 0; + virtual void setDeleted(bool deleted) = 0; + [[nodiscard]] virtual std::chrono::system_clock::time_point getEntryDate() const = 0; + [[nodiscard]] virtual std::chrono::system_clock::time_point getEventTime() const = 0; + [[nodiscard]] virtual std::chrono::system_clock::time_point getlineageStartDate() const = 0; + virtual void setLineageStartDate(std::chrono::system_clock::time_point date) = 0; + virtual void setLineageIdentifiers(const std::vector& lineage_Identifiers) = 0; + virtual bool getAttribute(std::string_view key, std::string& value) const = 0; + [[nodiscard]] virtual std::optional getAttribute(std::string_view key) const = 0; + virtual bool updateAttribute(std::string_view key, const std::string& value) = 0; + virtual bool removeAttribute(std::string_view key) = 0; + virtual bool setAttribute(std::string_view key, std::string value) = 0; + [[nodiscard]] virtual std::map getAttributes() const = 0; + virtual AttributeMap *getAttributesPtr() = 0; + virtual bool addAttribute(std::string_view key, const std::string& value) = 0; + virtual void setSize(const uint64_t size) = 0; + [[nodiscard]] virtual uint64_t getSize() const = 0; + virtual void setOffset(const uint64_t offset) = 0; + [[nodiscard]] virtual std::chrono::steady_clock::time_point getPenaltyExpiration() const = 0; + virtual void setPenaltyExpiration(std::chrono::time_point to_be_processed_after) = 0; + [[nodiscard]] virtual uint64_t getOffset() const = 0; + [[nodiscard]] virtual bool isPenalized() const = 0; + [[nodiscard]] virtual uint64_t getId() const = 0; + virtual void setConnection(core::Connectable* connection) = 0; + [[nodiscard]] virtual Connectable* getConnection() const = 0; + virtual void setStoredToRepository(bool storedInRepository) = 0; + [[nodiscard]] virtual bool isStored() const = 0; + + static std::shared_ptr create(); +}; + +// FlowFile Attribute +struct SpecialFlowAttribute { + // The flowfile's path indicates the relative directory to which a FlowFile belongs and does not contain the filename + MINIFIAPI static constexpr std::string_view PATH = "path"; + // The flowfile's absolute path indicates the absolute directory to which a FlowFile belongs and does not contain the filename + MINIFIAPI static constexpr std::string_view ABSOLUTE_PATH = "absolute.path"; + // The filename of the FlowFile. The filename should not contain any directory structure. + MINIFIAPI static constexpr std::string_view FILENAME = "filename"; + // A unique UUID assigned to this FlowFile. + MINIFIAPI static constexpr std::string_view UUID = "uuid"; + // A numeric value indicating the FlowFile priority + MINIFIAPI static constexpr std::string_view priority = "priority"; + // The MIME Type of this FlowFile + MINIFIAPI static constexpr std::string_view MIME_TYPE = "mime.type"; + // Specifies the reason that a FlowFile is being discarded + MINIFIAPI static constexpr std::string_view DISCARD_REASON = "discard.reason"; + // Indicates an identifier other than the FlowFile's UUID that is known to refer to this FlowFile. + MINIFIAPI static constexpr std::string_view ALTERNATE_IDENTIFIER = "alternate.identifier"; + // Flow identifier + MINIFIAPI static constexpr std::string_view FLOW_ID = "flow.id"; + + static constexpr std::array getSpecialFlowAttributes() { + return { + PATH, ABSOLUTE_PATH, FILENAME, UUID, priority, MIME_TYPE, DISCARD_REASON, ALTERNATE_IDENTIFIER, FLOW_ID + }; + } +}; + +} // namespace org::apache::nifi::minifi::core diff --git a/core/include/minifi-cpp/core/ObjectFactory.h b/core/include/minifi-cpp/core/ObjectFactory.h new file mode 100644 index 0000000000..9230a50c53 --- /dev/null +++ b/core/include/minifi-cpp/core/ObjectFactory.h @@ -0,0 +1,51 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include +#include +#include +#include "Core.h" + +namespace org { +namespace apache { +namespace nifi { +namespace minifi { +namespace core { + +/** + * Factory that is used as an interface for + * creating processors from shared objects. + */ +class ObjectFactory { + public: + virtual std::unique_ptr create(const std::string& /*name*/) = 0; + virtual CoreComponent *createRaw(const std::string& /*name*/) = 0; + virtual std::unique_ptr create(const std::string& /*name*/, const utils::Identifier& /*uuid*/) = 0; + virtual CoreComponent* createRaw(const std::string& /*name*/, const utils::Identifier& /*uuid*/) = 0; + virtual std::string getGroupName() const = 0; + virtual std::string getClassName() = 0; + + virtual ~ObjectFactory() = default; +}; + +} // namespace core +} // namespace minifi +} // namespace nifi +} // namespace apache +} // namespace org diff --git a/libminifi/include/core/OutputAttributeDefinition.h b/core/include/minifi-cpp/core/OutputAttributeDefinition.h similarity index 100% rename from libminifi/include/core/OutputAttributeDefinition.h rename to core/include/minifi-cpp/core/OutputAttributeDefinition.h diff --git a/core/include/minifi-cpp/core/ParameterContext.h b/core/include/minifi-cpp/core/ParameterContext.h new file mode 100644 index 0000000000..1b795414cf --- /dev/null +++ b/core/include/minifi-cpp/core/ParameterContext.h @@ -0,0 +1,66 @@ +/** + * + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#pragma once + +#include +#include +#include + +#include "Core.h" +#include "Exception.h" + +namespace org::apache::nifi::minifi::core { + +class ParameterException : public Exception { + public: + explicit ParameterException(const std::string& message) : Exception(ExceptionType::PARAMETER_EXCEPTION, message) {} + explicit ParameterException(const char* message) : Exception(ExceptionType::PARAMETER_EXCEPTION, message) {} +}; + +struct Parameter { + std::string name; + std::string description; + bool sensitive = false; + std::string value; +}; + +class ParameterContext : public CoreComponent { + public: + using CoreComponent::CoreComponent; + + void setDescription(const std::string &description) { + description_ = description; + } + + std::string getDescription() const { + return description_; + } + + void addParameter(const Parameter ¶meter); + std::optional getParameter(const std::string &name) const; + const std::unordered_map& getParameters() const { + return parameters_; + } + + private: + std::string description_; + std::unordered_map parameters_; +}; + +} // namespace org::apache::nifi::minifi::core + diff --git a/core/include/minifi-cpp/core/ProcessContext.h b/core/include/minifi-cpp/core/ProcessContext.h new file mode 100644 index 0000000000..70207fcbbe --- /dev/null +++ b/core/include/minifi-cpp/core/ProcessContext.h @@ -0,0 +1,91 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#pragma once + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "minifi-cpp/core/Core.h" +#include "minifi-cpp/core/ContentRepository.h" +#include "minifi-cpp/core/controller/ControllerServiceLookup.h" +#include "minifi-cpp/core/ProcessorNode.h" +#include "minifi-cpp/core/Property.h" +#include "minifi-cpp/core/Repository.h" +#include "minifi-cpp/core/FlowFile.h" +#include "minifi-cpp/core/StateStorage.h" +#include "minifi-cpp/core/VariableRegistry.h" + +namespace org::apache::nifi::minifi::core { + +namespace detail { +template +concept NotAFlowFile = !std::convertible_to && !std::convertible_to &>; +} // namespace detail + +class ProcessContext : public virtual core::VariableRegistry, public virtual utils::EnableSharedFromThis { + public: + virtual std::shared_ptr getProcessorNode() const = 0; + virtual std::optional getProperty(const Property&, const FlowFile* const) = 0; + virtual std::optional getProperty(const PropertyReference&, const FlowFile* const) = 0; + virtual bool getProperty(const Property &property, std::string &value, const FlowFile* const) = 0; + virtual bool getProperty(const PropertyReference& property, std::string &value, const FlowFile* const) = 0; + virtual bool getDynamicProperty(const std::string &name, std::string &value) const = 0; + virtual bool getDynamicProperty(const Property &property, std::string &value, const FlowFile* const) = 0; + virtual std::vector getDynamicPropertyKeys() const = 0; + virtual bool setProperty(const std::string &name, std::string value) = 0; + virtual bool setDynamicProperty(const std::string &name, std::string value) = 0; + virtual bool setProperty(const Property& property, std::string value) = 0; + virtual bool setProperty(const PropertyReference& property, std::string_view value) = 0; + virtual bool isAutoTerminated(Relationship relationship) const = 0; + virtual uint8_t getMaxConcurrentTasks() const = 0; + virtual void yield() = 0; + virtual std::shared_ptr getProvenanceRepository() = 0; + virtual std::shared_ptr getContentRepository() const = 0; + virtual std::shared_ptr getFlowFileRepository() const = 0; + virtual void initializeContentRepository(const std::string& home) = 0; + virtual bool isInitialized() const = 0; + + virtual std::shared_ptr getControllerService(const std::string &identifier, const utils::Identifier &processor_uuid) const = 0; + static constexpr char const* DefaultStateStorageName = "defaultstatestorage"; + + virtual StateManager* getStateManager() = 0; + virtual bool hasStateManager() const = 0; + virtual std::shared_ptr getConfiguration() const = 0; + + + + template + std::optional getProperty(const Property& property) const; + + template + std::optional getProperty(const PropertyReference& property) const; + + bool getProperty(std::string_view name, detail::NotAFlowFile auto& value) const; + + bool getProperty(const PropertyReference& property, detail::NotAFlowFile auto& value) const; +}; + +} // namespace org::apache::nifi::minifi::core diff --git a/core/include/minifi-cpp/core/ProcessContextBuilder.h b/core/include/minifi-cpp/core/ProcessContextBuilder.h new file mode 100644 index 0000000000..dee31da514 --- /dev/null +++ b/core/include/minifi-cpp/core/ProcessContextBuilder.h @@ -0,0 +1,57 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#pragma once + +#include +#include +#include +#include +#include +#include +#include +#include +#include "Property.h" +#include "minifi-cpp/core/Core.h" +#include "utils/Id.h" +#include "core/ContentRepository.h" +#include "minifi-cpp/properties/Configure.h" +#include "minifi-cpp/core/controller/ControllerServiceProvider.h" +#include "ProcessContext.h" +#include "ProcessorNode.h" +#include "minifi-cpp/core/Repository.h" +#include "VariableRegistry.h" + +namespace org::apache::nifi::minifi::core { +/** + * Could use instantiate from core, which uses a simple compile time check to figure out if a destructor is defined + * and thus that will allow us to know if the context instance exists, but I like using the build because it allows us + * to eventually share the builder across different contexts and shares up the construction ever so slightly. + * + * While this incurs a tiny cost to look up, it allows us to have a replaceable builder that erases the type we are + * constructing. + */ +class ProcessContextBuilder : public virtual core::CoreComponent, public virtual utils::EnableSharedFromThis { + public: + virtual std::shared_ptr withProvider(core::controller::ControllerServiceProvider* controller_service_provider) = 0; + virtual std::shared_ptr withProvenanceRepository(const std::shared_ptr &repo) = 0; + virtual std::shared_ptr withFlowFileRepository(const std::shared_ptr &repo) = 0; + virtual std::shared_ptr withContentRepository(const std::shared_ptr &repo) = 0; + virtual std::shared_ptr withConfiguration(const std::shared_ptr &configuration) = 0; + virtual std::shared_ptr build(const std::shared_ptr &processor) = 0; +}; + +} // namespace org::apache::nifi::minifi::core diff --git a/core/include/minifi-cpp/core/ProcessSession.h b/core/include/minifi-cpp/core/ProcessSession.h new file mode 100644 index 0000000000..cbc74f7323 --- /dev/null +++ b/core/include/minifi-cpp/core/ProcessSession.h @@ -0,0 +1,137 @@ +/** + * + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#pragma once + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "ProcessContext.h" +#include "core/logging/LoggerFactory.h" +#include "core/Deprecated.h" +#include "FlowFile.h" +#include "WeakReference.h" +#include "minifi-cpp/provenance/Provenance.h" +#include "utils/gsl.h" +#include "ProcessorMetrics.h" +#include "minifi-cpp/io/StreamCallback.h" + +namespace org::apache::nifi::minifi::core { + +namespace detail { + +struct ReadBufferResult { + int64_t status; + std::vector buffer; +}; + +} // namespace detail + +// ProcessSession Class +class ProcessSession : public virtual ReferenceContainer { + public: + ~ProcessSession() override = default; + + virtual void commit() = 0; + virtual void rollback() = 0; + + virtual nonstd::expected rollbackNoThrow() noexcept = 0; + virtual std::shared_ptr getProvenanceReporter() = 0; + virtual void flushContent() = 0; + virtual std::shared_ptr get() = 0; + virtual std::shared_ptr create(const core::FlowFile* const parent = nullptr) = 0; + virtual void add(const std::shared_ptr &record) = 0; + virtual std::shared_ptr clone(const core::FlowFile& parent) = 0; + virtual std::shared_ptr clone(const core::FlowFile& parent, int64_t offset, int64_t size) = 0; + virtual void transfer(const std::shared_ptr& flow, const Relationship& relationship) = 0; + virtual void transferToCustomRelationship(const std::shared_ptr& flow, const std::string& relationship_name) = 0; + + virtual void putAttribute(core::FlowFile& flow, std::string_view key, const std::string& value) = 0; + virtual void removeAttribute(core::FlowFile& flow, std::string_view key) = 0; + + virtual void remove(const std::shared_ptr &flow) = 0; + // Access the contents of the flow file as an input stream; returns null if the flow file has no content claim + virtual std::shared_ptr getFlowFileContentStream(const core::FlowFile& flow_file) = 0; + // Execute the given read callback against the content + virtual int64_t read(const std::shared_ptr& flow_file, const io::InputStreamCallback& callback) = 0; + + virtual int64_t read(const core::FlowFile& flow_file, const io::InputStreamCallback& callback) = 0; + // Read content into buffer + virtual detail::ReadBufferResult readBuffer(const std::shared_ptr& flow) = 0; + // Execute the given write callback against the content + virtual void write(const std::shared_ptr &flow, const io::OutputStreamCallback& callback) = 0; + + virtual void write(core::FlowFile& flow, const io::OutputStreamCallback& callback) = 0; + // Read and write the flow file at the same time (eg. for processing it line by line) + virtual int64_t readWrite(const std::shared_ptr &flow, const io::InputOutputStreamCallback& callback) = 0; + // Replace content with buffer + virtual void writeBuffer(const std::shared_ptr& flow_file, std::span buffer) = 0; + virtual void writeBuffer(const std::shared_ptr& flow_file, std::span buffer) = 0; + // Execute the given write/append callback against the content + virtual void append(const std::shared_ptr &flow, const io::OutputStreamCallback& callback) = 0; + // Append buffer to content + virtual void appendBuffer(const std::shared_ptr& flow, std::span buffer) = 0; + virtual void appendBuffer(const std::shared_ptr& flow, std::span buffer) = 0; + // Penalize the flow + virtual void penalize(const std::shared_ptr &flow) = 0; + + virtual bool outgoingConnectionsFull(const std::string& relationship) = 0; + + /** + * Imports a file from the data stream + * @param stream incoming data stream that contains the data to store into a file + * @param flow flow file + */ + virtual void importFrom(io::InputStream &stream, const std::shared_ptr &flow) = 0; + virtual void importFrom(io::InputStream&& stream, const std::shared_ptr &flow) = 0; + + // import from the data source. + virtual void import(const std::string& source, const std::shared_ptr &flow, bool keepSource = true, uint64_t offset = 0) = 0; + + /** + * Exports the data stream to a file + * @param string file to export stream to + * @param flow flow file + * @param bool whether or not to keep the content in the flow file + */ + virtual bool exportContent(const std::string &destination, const std::shared_ptr &flow, bool keepContent) = 0; + + virtual bool exportContent(const std::string &destination, const std::string &tmpFileName, const std::shared_ptr &flow, bool keepContent) = 0; + + // Stash the content to a key + virtual void stash(const std::string &key, const std::shared_ptr &flow) = 0; + // Restore content previously stashed to a key + virtual void restore(const std::string &key, const std::shared_ptr &flow) = 0; + + virtual bool existsFlowFileInRelationship(const Relationship &relationship) = 0; + + virtual void setMetrics(const std::shared_ptr& metrics) = 0; + + virtual bool hasBeenTransferred(const core::FlowFile &flow) const = 0; +}; + +} // namespace org::apache::nifi::minifi::core diff --git a/core/include/minifi-cpp/core/ProcessSessionFactory.h b/core/include/minifi-cpp/core/ProcessSessionFactory.h new file mode 100644 index 0000000000..d2ede7ceb5 --- /dev/null +++ b/core/include/minifi-cpp/core/ProcessSessionFactory.h @@ -0,0 +1,45 @@ +/** + * @file ProcessSessionFactory.h + * ProcessSessionFactory class declaration + * + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include + +#include "ProcessContext.h" +#include "ProcessSession.h" + +namespace org { +namespace apache { +namespace nifi { +namespace minifi { +namespace core { + +// ProcessSessionFactory Class +class ProcessSessionFactory { + public: + virtual std::shared_ptr createSession() = 0; + virtual ~ProcessSessionFactory() = default; +}; + +} // namespace core +} // namespace minifi +} // namespace nifi +} // namespace apache +} // namespace org diff --git a/core/include/minifi-cpp/core/ProcessSessionReadCallback.h b/core/include/minifi-cpp/core/ProcessSessionReadCallback.h new file mode 100644 index 0000000000..55389ac455 --- /dev/null +++ b/core/include/minifi-cpp/core/ProcessSessionReadCallback.h @@ -0,0 +1,44 @@ +/** + * @file ProcessSessionReadCallback.h + * ProcessSessionReadCallback class declaration + * + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#pragma once + +#include +#include + +#include "core/logging/LoggerFactory.h" +#include "FlowFileRecord.h" +#include "io/InputStream.h" + +namespace org::apache::nifi::minifi::core { +class ProcessSessionReadCallback { + public: + ProcessSessionReadCallback(std::filesystem::path temp_file, std::filesystem::path dest_file, std::shared_ptr logger); + ~ProcessSessionReadCallback(); + int64_t operator()(const std::shared_ptr& stream); + bool commit(); + + private: + std::shared_ptr logger_; + std::ofstream tmp_file_os_; + bool write_succeeded_ = false; + std::filesystem::path tmp_file_; + std::filesystem::path dest_file_; +}; +} // namespace org::apache::nifi::minifi::core diff --git a/core/include/minifi-cpp/core/Processor.h b/core/include/minifi-cpp/core/Processor.h new file mode 100644 index 0000000000..54bb8c0a38 --- /dev/null +++ b/core/include/minifi-cpp/core/Processor.h @@ -0,0 +1,104 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#pragma once + +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "ConfigurableComponent.h" +#include "Connectable.h" +#include "Core.h" +#include "minifi-cpp/core/Annotation.h" +#include "DynamicProperty.h" +#include "Scheduling.h" +#include "utils/TimeUtil.h" +#include "minifi-cpp/core/state/nodes/MetricsBase.h" +#include "ProcessorMetrics.h" +#include "utils/gsl.h" + +namespace org::apache::nifi::minifi { + +class Connection; + +namespace core { + +class ProcessContext; +class ProcessSession; +class ProcessSessionFactory; + +class Processor : public virtual Connectable, public virtual ConfigurableComponent, public virtual state::response::ResponseNodeSource { + public: + ~Processor() override = default; + + virtual void setScheduledState(ScheduledState state) = 0; + virtual ScheduledState getScheduledState() const = 0; + virtual void setSchedulingStrategy(SchedulingStrategy strategy) = 0; + virtual SchedulingStrategy getSchedulingStrategy() const = 0; + virtual void setSchedulingPeriod(std::chrono::steady_clock::duration period) = 0; + virtual std::chrono::steady_clock::duration getSchedulingPeriod() const = 0; + virtual void setCronPeriod(const std::string &period) = 0; + virtual std::string getCronPeriod() const = 0; + virtual void setRunDurationNano(std::chrono::steady_clock::duration period) = 0; + virtual std::chrono::steady_clock::duration getRunDurationNano() const = 0; + virtual void setYieldPeriodMsec(std::chrono::milliseconds period) = 0; + virtual std::chrono::steady_clock::duration getYieldPeriod() const = 0; + virtual void setPenalizationPeriod(std::chrono::milliseconds period) = 0; + virtual bool isSingleThreaded() const = 0; + virtual std::string getProcessorType() const = 0; + virtual void setTriggerWhenEmpty(bool value) = 0; + virtual bool getTriggerWhenEmpty() const = 0; + virtual uint8_t getActiveTasks() const = 0; + virtual void incrementActiveTasks() = 0; + virtual void decrementActiveTask() = 0; + virtual void clearActiveTask() = 0; + using Connectable::yield; + virtual void yield(std::chrono::steady_clock::duration delta_time) = 0; + virtual bool isYield() = 0; + virtual void clearYield() = 0; + virtual std::chrono::steady_clock::time_point getYieldExpirationTime() const = 0; + virtual std::chrono::steady_clock::duration getYieldTime() const = 0; + virtual bool flowFilesOutGoingFull() const = 0; + virtual bool addConnection(Connectable* connection) = 0; + virtual void triggerAndCommit(const std::shared_ptr& context, const std::shared_ptr& session_factory) = 0; + virtual void trigger(const std::shared_ptr& context, const std::shared_ptr& process_session) = 0; + virtual void onTrigger(ProcessContext&, ProcessSession&) = 0; + virtual void onSchedule(ProcessContext&, ProcessSessionFactory&) = 0; + virtual void onUnSchedule() = 0; + virtual bool isThrottledByBackpressure() const = 0; + virtual void validateAnnotations() const = 0; + virtual annotation::Input getInputRequirement() const = 0; + virtual gsl::not_null> getMetrics() const = 0; + + virtual void updateReachability(const std::lock_guard& graph_lock, bool force = false) = 0; + virtual const std::unordered_map>& reachable_processors() const = 0; +}; + +} // namespace core +} // namespace org::apache::nifi::minifi diff --git a/core/include/minifi-cpp/core/ProcessorConfig.h b/core/include/minifi-cpp/core/ProcessorConfig.h new file mode 100644 index 0000000000..04d3761d45 --- /dev/null +++ b/core/include/minifi-cpp/core/ProcessorConfig.h @@ -0,0 +1,62 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef LIBMINIFI_INCLUDE_CORE_PROCESSORCONFIG_H_ +#define LIBMINIFI_INCLUDE_CORE_PROCESSORCONFIG_H_ + +#include +#include + +#include "Core.h" +#include "Property.h" + +namespace org { +namespace apache { +namespace nifi { +namespace minifi { +namespace core { + + +constexpr const char* DEFAULT_SCHEDULING_STRATEGY{"TIMER_DRIVEN"}; +constexpr const char* DEFAULT_SCHEDULING_PERIOD_STR{"1 sec"}; +constexpr std::chrono::milliseconds DEFAULT_SCHEDULING_PERIOD_MILLIS{1000}; +constexpr std::chrono::nanoseconds DEFAULT_RUN_DURATION{0}; +constexpr int DEFAULT_MAX_CONCURRENT_TASKS{1}; +constexpr std::chrono::seconds DEFAULT_YIELD_PERIOD_SECONDS{1}; +constexpr std::chrono::seconds DEFAULT_PENALIZATION_PERIOD{30}; + +struct ProcessorConfig { + std::string id; + std::string name; + std::string javaClass; + std::string maxConcurrentTasks; + std::string schedulingStrategy; + std::string schedulingPeriod; + std::string penalizationPeriod; + std::string yieldPeriod; + std::string runDurationNanos; + std::vector autoTerminatedRelationships; + std::vector properties; + std::string parameterContextName; +}; + +} // namespace core +} // namespace minifi +} // namespace nifi +} // namespace apache +} // namespace org + +#endif // LIBMINIFI_INCLUDE_CORE_PROCESSORCONFIG_H_ diff --git a/core/include/minifi-cpp/core/ProcessorMetrics.h b/core/include/minifi-cpp/core/ProcessorMetrics.h new file mode 100644 index 0000000000..de2bdc97fd --- /dev/null +++ b/core/include/minifi-cpp/core/ProcessorMetrics.h @@ -0,0 +1,51 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#pragma once + +#include +#include +#include +#include +#include +#include + +#include "minifi-cpp/core/state/nodes/MetricsBase.h" +#include "minifi-cpp/core/state/PublishedMetricProvider.h" + +namespace org::apache::nifi::minifi::core { + +class ProcessorMetrics : public virtual state::response::ResponseNode { + public: + virtual void increaseRelationshipTransferCount(const std::string& relationship, size_t count = 1) = 0; + virtual std::chrono::milliseconds getAverageOnTriggerRuntime() const = 0; + virtual std::chrono::milliseconds getLastOnTriggerRuntime() const = 0; + virtual void addLastOnTriggerRuntime(std::chrono::milliseconds runtime) = 0; + + virtual std::chrono::milliseconds getAverageSessionCommitRuntime() const = 0; + virtual std::chrono::milliseconds getLastSessionCommitRuntime() const = 0; + virtual void addLastSessionCommitRuntime(std::chrono::milliseconds runtime) = 0; + + virtual std::atomic& iterations() = 0; + virtual std::atomic& transferred_flow_files() = 0; + virtual std::atomic& transferred_bytes() = 0; + + virtual const std::atomic& iterations() const = 0; + virtual const std::atomic& transferred_flow_files() const = 0; + virtual const std::atomic& transferred_bytes() const = 0; +}; + +} // namespace org::apache::nifi::minifi::core diff --git a/core/include/minifi-cpp/core/ProcessorNode.h b/core/include/minifi-cpp/core/ProcessorNode.h new file mode 100644 index 0000000000..f0a16d5608 --- /dev/null +++ b/core/include/minifi-cpp/core/ProcessorNode.h @@ -0,0 +1,52 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include +#include +#include +#include +#include + +#include "ConfigurableComponent.h" +#include "Connectable.h" +#include "Property.h" + +namespace org::apache::nifi::minifi::core { + +/** + * Processor node functions as a pass through to the implementing Connectables + */ +class ProcessorNode : public virtual ConfigurableComponent, public virtual Connectable { + public: + virtual Connectable* getProcessor() const = 0; + + using ConfigurableComponent::getProperty; + + template + bool getProperty(const std::string &name, T &value) { + const auto processor_cast = dynamic_cast(getProcessor()); + if (nullptr != processor_cast) { + return processor_cast->getProperty(name, value); + } else { + return ConfigurableComponent::getProperty(name, value); + } + } +}; + +} // namespace org::apache::nifi::minifi::core diff --git a/libminifi/include/core/Property.h b/core/include/minifi-cpp/core/Property.h similarity index 77% rename from libminifi/include/core/Property.h rename to core/include/minifi-cpp/core/Property.h index ba30016552..b840cb72a2 100644 --- a/libminifi/include/core/Property.h +++ b/core/include/minifi-cpp/core/Property.h @@ -23,8 +23,9 @@ #include #include -#include "core/PropertyDefinition.h" -#include "core/PropertyValue.h" +#include "PropertyDefinition.h" +#include "PropertyValue.h" +#include "PropertyType.h" #include "range/v3/view/transform.hpp" #include "range/v3/range/conversion.hpp" #include "utils/gsl.h" @@ -40,47 +41,19 @@ class Property { * further overwrites to inherit the bool validator. */ Property(std::string name, std::string description, const std::string& value, bool is_required, std::vector dependent_properties, - std::vector> exclusive_of_properties) - : name_(std::move(name)), - description_(std::move(description)), - is_required_(is_required), - dependent_properties_(std::move(dependent_properties)), - exclusive_of_properties_(std::move(exclusive_of_properties)), - is_collection_(false), - supports_el_(false), - is_transient_(false) { - default_value_ = coerceDefaultValue(value); - } - - Property(std::string name, std::string description, const std::string& value) - : name_(std::move(name)), - description_(std::move(description)), - is_required_(false), - is_collection_(false), - supports_el_(false), - is_transient_(false) { - default_value_ = coerceDefaultValue(value); - } + std::vector> exclusive_of_properties); - Property(std::string name, std::string description) - : name_(std::move(name)), - description_(std::move(description)), - is_required_(false), - is_collection_(true), - supports_el_(false), - is_transient_(false) {} + Property(std::string name, std::string description, const std::string& value); - explicit Property(const PropertyReference& compile_time_property); + Property(std::string name, std::string description); Property(Property &&other) = default; Property(const Property &other) = default; - Property() - : is_required_(false), - is_collection_(false), - supports_el_(false), - is_transient_(false) {} + Property(); + + Property(const PropertyReference&); virtual ~Property() = default; @@ -147,8 +120,10 @@ class Property { void setAllowedValues(gsl::span allowed_values, const core::PropertyParser& property_parser); + /** + * Add value to the collection of values. + */ void addValue(const std::string &value); - Property &operator=(const Property &other) = default; Property &operator=(Property &&other) = default; // Compare @@ -196,9 +171,7 @@ class Property { // Convert String to Integer template - static bool StringToInt(std::string input, T &output) { - return DataSizeValue::StringToInt(input, output); - } + static bool StringToInt(std::string input, T &output); static bool StringToInt(std::string input, int64_t &output) { return StringToInt(input, output); @@ -222,19 +195,7 @@ class Property { /** * Coerce default values at construction. */ - PropertyValue coerceDefaultValue(const std::string &value) { - PropertyValue ret; - if (value == "false" || value == "true") { - bool val; - std::istringstream(value) >> std::boolalpha >> val; - ret = val; - validator_ = StandardPropertyTypes::getValidator(ret.getValue()); - } else { - ret = value; - validator_ = gsl::make_not_null(&StandardPropertyTypes::VALID_TYPE); - } - return ret; - } + PropertyValue coerceDefaultValue(const std::string &value); std::string name_; std::string description_; @@ -243,7 +204,7 @@ class Property { std::vector dependent_properties_; std::vector> exclusive_of_properties_; bool is_collection_; - gsl::not_null validator_{&StandardPropertyTypes::VALID_TYPE}; + gsl::not_null validator_; PropertyValue default_value_; std::vector values_; std::string display_name_; diff --git a/libminifi/include/core/PropertyDefinition.h b/core/include/minifi-cpp/core/PropertyDefinition.h similarity index 88% rename from libminifi/include/core/PropertyDefinition.h rename to core/include/minifi-cpp/core/PropertyDefinition.h index 7d82e5bb11..57aab5aba9 100644 --- a/libminifi/include/core/PropertyDefinition.h +++ b/core/include/minifi-cpp/core/PropertyDefinition.h @@ -22,8 +22,7 @@ #include #include -#include "core/Core.h" -#include "core/PropertyType.h" +#include "PropertyType.h" #include "utils/gsl.h" namespace org::apache::nifi::minifi::core { @@ -33,17 +32,17 @@ struct PropertyDefinition { std::string_view name; std::string_view display_name; std::string_view description; - bool is_required = false; - bool is_sensitive = false; + bool is_required; + bool is_sensitive; std::array allowed_values; std::span allowed_types; std::array dependent_properties; std::array, NumExclusiveOfProperties> exclusive_of_properties; std::optional default_value; - gsl::not_null type{gsl::make_not_null(&StandardPropertyTypes::VALID_TYPE)}; - bool supports_expression_language = false; + gsl::not_null type; + bool supports_expression_language; - uint8_t version = 1; + uint8_t version; }; struct PropertyReference { @@ -57,11 +56,9 @@ struct PropertyReference { std::span dependent_properties; std::span> exclusive_of_properties; std::optional default_value; - gsl::not_null type{gsl::make_not_null(&StandardPropertyTypes::VALID_TYPE)}; + gsl::not_null type; bool supports_expression_language = false; - constexpr PropertyReference() = default; - template constexpr PropertyReference(const PropertyDefinition& property_definition) // NOLINT: non-explicit on purpose : name{property_definition.name}, diff --git a/core/include/minifi-cpp/core/PropertyType.h b/core/include/minifi-cpp/core/PropertyType.h new file mode 100644 index 0000000000..2f963491c5 --- /dev/null +++ b/core/include/minifi-cpp/core/PropertyType.h @@ -0,0 +1,54 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#pragma once + +#include +#include +#include +#include + +#include "minifi-cpp/core/Core.h" +#include "PropertyValue.h" +#include "minifi-cpp/core/state/Value.h" +#include "ValidationResult.h" + +namespace org::apache::nifi::minifi::core { + +class PropertyParser { + public: + virtual constexpr ~PropertyParser() {} // NOLINT can't use = default because of gcc bug: https://gcc.gnu.org/bugzilla/show_bug.cgi?id=93413 + + [[nodiscard]] virtual PropertyValue parse(std::string_view input) const = 0; +}; + +class PropertyValidator { + public: + virtual constexpr ~PropertyValidator() {} // NOLINT can't use = default because of gcc bug: https://gcc.gnu.org/bugzilla/show_bug.cgi?id=93413 + + [[nodiscard]] virtual std::string_view getValidatorName() const = 0; + + [[nodiscard]] virtual ValidationResult validate(const std::string &subject, const std::shared_ptr &input) const = 0; + + [[nodiscard]] virtual ValidationResult validate(const std::string &subject, const std::string &input) const = 0; +}; + +class PropertyType : public PropertyParser, public PropertyValidator { + public: + virtual constexpr ~PropertyType() {} // NOLINT can't use = default because of gcc bug: https://gcc.gnu.org/bugzilla/show_bug.cgi?id=93413 +}; + +} // namespace org::apache::nifi::minifi::core diff --git a/core/include/minifi-cpp/core/PropertyValue.h b/core/include/minifi-cpp/core/PropertyValue.h new file mode 100644 index 0000000000..f757ed71a4 --- /dev/null +++ b/core/include/minifi-cpp/core/PropertyValue.h @@ -0,0 +1,139 @@ +/** + * + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#pragma once + +#include +#include +#include +#include + +#include "CachedValueValidator.h" +#include "ValidationResult.h" +#include "state/Value.h" +#include "minifi-cpp/utils/PropertyErrors.h" + +namespace org::apache::nifi::minifi::core { + +class PropertyValidator; + +/** + * Purpose and Design: PropertyValue extends ValueNode, bringing with it value_. + * The reason for this is that there are additional features to support validation + * and value translation. + */ +class PropertyValue : public state::response::ValueNode { + using CachedValueValidator = internal::CachedValueValidator; + template friend struct Converter; + + public: + PropertyValue() + : type_id(std::type_index(typeid(std::string))) {} + + PropertyValue(const PropertyValue &o) = default; + PropertyValue(PropertyValue &&o) noexcept = default; + + template + static PropertyValue parse(std::string_view input, const PropertyValidator& validator) { + PropertyValue property_value; + property_value.value_ = std::make_shared(std::string{input}); + property_value.type_id = property_value.value_->getTypeIndex(); + property_value.cached_value_validator_ = validator; + return property_value; + } + + void setValidator(const PropertyValidator& validator) { + cached_value_validator_ = validator; + } + + ValidationResult validate(const std::string &subject) const { + return cached_value_validator_.validate(subject, getValue()); + } + + operator uint64_t() const { + return convertImpl("uint64_t"); + } + + operator int64_t() const { + return convertImpl("int64_t"); + } + + operator uint32_t() const { + return convertImpl("uint32_t"); + } + + operator int() const { + return convertImpl("int"); + } + + operator bool() const { + return convertImpl("bool"); + } + + operator double() const { + return convertImpl("double"); + } + + const char* c_str() const { + if (!isValueUsable()) { + throw utils::internal::InvalidValueException("Cannot convert invalid value"); + } + return value_ ? value_->c_str() : ""; + } + + operator std::string() const { + if (!isValueUsable()) { + throw utils::internal::InvalidValueException("Cannot convert invalid value"); + } + return to_string(); + } + + PropertyValue &operator=(PropertyValue &&o) = default; + PropertyValue &operator=(const PropertyValue &o) = default; + + std::type_index getTypeInfo() const { + return type_id; + } + /** + * Define the representations and eventual storage relationships through + * createValue + */ + template + auto operator=(const T& ref) -> std::enable_if_t, PropertyValue>, PropertyValue&>; + + private: + template + T convertImpl(const char* const type_name) const; + + bool isValueUsable() const { + if (!value_) return false; + return validate("__unknown__").valid; + } + + template + auto WithAssignmentGuard(const std::string& ref, Fn&& functor) -> decltype(std::forward(functor)()); + + protected: + std::type_index type_id; + CachedValueValidator cached_value_validator_; +}; + +inline std::string conditional_conversion(const PropertyValue &v) { + return v.getValue()->getStringValue(); +} + +} // namespace org::apache::nifi::minifi::core diff --git a/libminifi/include/core/Record.h b/core/include/minifi-cpp/core/Record.h similarity index 100% rename from libminifi/include/core/Record.h rename to core/include/minifi-cpp/core/Record.h diff --git a/libminifi/include/core/RecordField.h b/core/include/minifi-cpp/core/RecordField.h similarity index 100% rename from libminifi/include/core/RecordField.h rename to core/include/minifi-cpp/core/RecordField.h diff --git a/libminifi/include/core/Relationship.h b/core/include/minifi-cpp/core/Relationship.h similarity index 88% rename from libminifi/include/core/Relationship.h rename to core/include/minifi-cpp/core/Relationship.h index b78276d4b0..15880bed47 100644 --- a/libminifi/include/core/Relationship.h +++ b/core/include/minifi-cpp/core/Relationship.h @@ -21,7 +21,6 @@ #include #include - #include "RelationshipDefinition.h" namespace org::apache::nifi::minifi::core { @@ -64,10 +63,3 @@ class Relationship { std::string description_; }; } // namespace org::apache::nifi::minifi::core - -template<> -struct std::hash { - size_t operator()(const org::apache::nifi::minifi::core::Relationship& relationship) const noexcept { - return std::hash{}(relationship.getName()); - } -}; diff --git a/libminifi/include/core/RelationshipDefinition.h b/core/include/minifi-cpp/core/RelationshipDefinition.h similarity index 100% rename from libminifi/include/core/RelationshipDefinition.h rename to core/include/minifi-cpp/core/RelationshipDefinition.h diff --git a/core/include/minifi-cpp/core/Repository.h b/core/include/minifi-cpp/core/Repository.h new file mode 100644 index 0000000000..763a0b4a0e --- /dev/null +++ b/core/include/minifi-cpp/core/Repository.h @@ -0,0 +1,80 @@ +/** + * @file Repository + * Repository class declaration + * + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#pragma once + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "minifi-cpp/ResourceClaim.h" +#include "Connectable.h" +#include "ContentRepository.h" +#include "Property.h" +#include "SerializableComponent.h" +#include "core/logging/LoggerFactory.h" +#include "RepositoryMetricsSource.h" +#include "minifi-cpp/properties/Configure.h" +#include "utils/BackTrace.h" +#include "minifi-cpp/SwapManager.h" +#include "core/Core.h" + +#ifndef WIN32 +#include +#endif + +namespace org::apache::nifi::minifi::core { + +class Repository : public virtual core::CoreComponent, public virtual core::RepositoryMetricsSource { + public: + virtual bool initialize(const std::shared_ptr &configure) = 0; + virtual bool start() = 0; + virtual bool stop() = 0; + virtual bool isNoop() const = 0; + virtual void flush() = 0; + + virtual bool Put(const std::string& /*key*/, const uint8_t* /*buf*/, size_t /*bufLen*/) = 0; + virtual bool MultiPut(const std::vector>>& /*data*/) = 0; + virtual bool Delete(const std::string& /*key*/) = 0; + virtual bool Delete(const std::shared_ptr& item) = 0; + virtual bool Delete(std::vector> &storedValues) = 0; + + virtual void setConnectionMap(std::map connectionMap) = 0; + + virtual void setContainers(std::map containers) = 0; + + virtual bool Get(const std::string& /*key*/, std::string& /*value*/) = 0; + + virtual bool getElements(std::vector>& /*store*/, size_t& /*max_size*/) = 0; + + virtual bool storeElement(const std::shared_ptr& element) = 0; + + virtual void loadComponent(const std::shared_ptr& /*content_repo*/) = 0; + + virtual std::string getDirectory() const = 0; +}; + +} // namespace org::apache::nifi::minifi::core diff --git a/libminifi/include/core/RepositoryMetricsSource.h b/core/include/minifi-cpp/core/RepositoryMetricsSource.h similarity index 82% rename from libminifi/include/core/RepositoryMetricsSource.h rename to core/include/minifi-cpp/core/RepositoryMetricsSource.h index 6c933b7eb8..ca55e461c6 100644 --- a/libminifi/include/core/RepositoryMetricsSource.h +++ b/core/include/minifi-cpp/core/RepositoryMetricsSource.h @@ -34,22 +34,10 @@ class RepositoryMetricsSource { virtual uint64_t getRepositorySize() const = 0; virtual uint64_t getRepositoryEntryCount() const = 0; virtual std::string getRepositoryName() const = 0; - - virtual uint64_t getMaxRepositorySize() const { - return 0; - } - - virtual bool isFull() const { - return false; - } - - virtual bool isRunning() const { - return true; - } - - virtual std::optional getRocksDbStats() const { - return std::nullopt; - } + virtual uint64_t getMaxRepositorySize() const = 0; + virtual bool isFull() const = 0; + virtual bool isRunning() const = 0; + virtual std::optional getRocksDbStats() const = 0; }; } // namespace org::apache::nifi::minifi::core diff --git a/core/include/minifi-cpp/core/Scheduling.h b/core/include/minifi-cpp/core/Scheduling.h new file mode 100644 index 0000000000..9a46b6cb7b --- /dev/null +++ b/core/include/minifi-cpp/core/Scheduling.h @@ -0,0 +1,57 @@ +/** + * + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef LIBMINIFI_INCLUDE_CORE_SCHEDULING_H_ +#define LIBMINIFI_INCLUDE_CORE_SCHEDULING_H_ + +namespace org { +namespace apache { +namespace nifi { +namespace minifi { +namespace core { + +/* + * Indicates the valid values for the state of a entity + * with respect to scheduling the entity to run. + */ +enum ScheduledState { + /** + * Entity cannot be scheduled to run + */ + DISABLED, + /** + * Entity can be scheduled to run but currently is not + */ + STOPPED, + /** + * Entity is currently scheduled to run + */ + RUNNING +}; + +enum SchedulingStrategy { + EVENT_DRIVEN, + TIMER_DRIVEN, + CRON_DRIVEN +}; + +} // namespace core +} // namespace minifi +} // namespace nifi +} // namespace apache +} // namespace org +#endif // LIBMINIFI_INCLUDE_CORE_SCHEDULING_H_ diff --git a/core/include/minifi-cpp/core/SerializableComponent.h b/core/include/minifi-cpp/core/SerializableComponent.h new file mode 100644 index 0000000000..0e2652f50a --- /dev/null +++ b/core/include/minifi-cpp/core/SerializableComponent.h @@ -0,0 +1,38 @@ +/** + * + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#pragma once + +#include +#include + +#include "core/Core.h" +#include "utils/gsl.h" +#include "io/InputStream.h" +#include "io/OutputStream.h" + +namespace org::apache::nifi::minifi::core { + +class SerializableComponent : public virtual core::CoreComponent { + public: + ~SerializableComponent() override = default; + + virtual bool serialize(io::OutputStream& output_stream) = 0; + virtual bool deserialize(io::InputStream &input_stream) = 0; +}; + +} // namespace org::apache::nifi::minifi::core diff --git a/libminifi/include/core/StateManager.h b/core/include/minifi-cpp/core/StateManager.h similarity index 87% rename from libminifi/include/core/StateManager.h rename to core/include/minifi-cpp/core/StateManager.h index 89ef8b2cdd..1c30b5cc46 100644 --- a/libminifi/include/core/StateManager.h +++ b/core/include/minifi-cpp/core/StateManager.h @@ -35,21 +35,10 @@ class StateManager { public: using State = std::unordered_map; - explicit StateManager(const utils::Identifier& id) - : id_(id) { - } - virtual ~StateManager() = default; - virtual bool set(const State& kvs) = 0; virtual bool get(State& kvs) = 0; - - std::optional get() { - if (State out; get(out)) { - return out; - } - return std::nullopt; - } + virtual std::optional get() = 0; virtual bool clear() = 0; virtual bool persist() = 0; @@ -59,9 +48,6 @@ class StateManager { virtual bool beginTransaction() = 0; virtual bool commit() = 0; virtual bool rollback() = 0; - - protected: - utils::Identifier id_; }; } // namespace org::apache::nifi::minifi::core diff --git a/libminifi/include/core/StateStorage.h b/core/include/minifi-cpp/core/StateStorage.h similarity index 91% rename from libminifi/include/core/StateStorage.h rename to core/include/minifi-cpp/core/StateStorage.h index 6870b47b92..2ea2c96932 100644 --- a/libminifi/include/core/StateStorage.h +++ b/core/include/minifi-cpp/core/StateStorage.h @@ -35,9 +35,7 @@ class StateStorage { virtual std::unique_ptr getStateManager(const utils::Identifier& uuid) = 0; - std::unique_ptr getStateManager(const CoreComponent& component) { - return getStateManager(component.getUUID()); - } + virtual std::unique_ptr getStateManager(const CoreComponent& component) = 0; virtual std::unordered_map getAllStates() = 0; }; diff --git a/libminifi/include/core/StreamManager.h b/core/include/minifi-cpp/core/StreamManager.h similarity index 93% rename from libminifi/include/core/StreamManager.h rename to core/include/minifi-cpp/core/StreamManager.h index 52ff59e0bb..bedb96d2b8 100644 --- a/libminifi/include/core/StreamManager.h +++ b/core/include/minifi-cpp/core/StreamManager.h @@ -20,8 +20,8 @@ #include #include -#include "properties/Configure.h" -#include "ResourceClaim.h" +#include "minifi-cpp/properties/Configure.h" +#include "minifi-cpp/ResourceClaim.h" #include "io/BufferStream.h" #include "io/BaseStream.h" @@ -67,12 +67,7 @@ class StreamManager { */ virtual std::shared_ptr read(const T &streamId) = 0; - virtual size_t size(const T &streamId) { - auto stream = read(streamId); - if (!stream) - return 0; - return stream->size(); - } + virtual size_t size(const T &streamId) = 0; /** * Closes the stream diff --git a/core/include/minifi-cpp/core/ThreadedRepository.h b/core/include/minifi-cpp/core/ThreadedRepository.h new file mode 100644 index 0000000000..da47ac732d --- /dev/null +++ b/core/include/minifi-cpp/core/ThreadedRepository.h @@ -0,0 +1,33 @@ +/** + * + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include +#include + +#include "Repository.h" +#include "TraceableResource.h" + +namespace org::apache::nifi::minifi::core { + +class ThreadedRepository : public virtual core::Repository, public virtual core::TraceableResource { + public: +}; + +} // namespace org::apache::nifi::minifi::core diff --git a/libminifi/include/core/TraceableResource.h b/core/include/minifi-cpp/core/TraceableResource.h similarity index 90% rename from libminifi/include/core/TraceableResource.h rename to core/include/minifi-cpp/core/TraceableResource.h index c52b209401..805ee51a96 100644 --- a/libminifi/include/core/TraceableResource.h +++ b/core/include/minifi-cpp/core/TraceableResource.h @@ -15,8 +15,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -#ifndef LIBMINIFI_INCLUDE_CORE_TRACEABLERESOURCE_H_ -#define LIBMINIFI_INCLUDE_CORE_TRACEABLERESOURCE_H_ +#pragma once #include "utils/BackTrace.h" @@ -48,5 +47,3 @@ class TraceableResource { } // namespace nifi } // namespace apache } // namespace org - -#endif // LIBMINIFI_INCLUDE_CORE_TRACEABLERESOURCE_H_ diff --git a/core/include/minifi-cpp/core/ValidationResult.h b/core/include/minifi-cpp/core/ValidationResult.h new file mode 100644 index 0000000000..9a6378147c --- /dev/null +++ b/core/include/minifi-cpp/core/ValidationResult.h @@ -0,0 +1,29 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#pragma once + +#include + +namespace org::apache::nifi::minifi::core { + +struct ValidationResult { + bool valid; + std::string subject; + std::string input; +}; + +} // namespace org::apache::nifi::minifi::core diff --git a/core/include/minifi-cpp/core/VariableRegistry.h b/core/include/minifi-cpp/core/VariableRegistry.h new file mode 100644 index 0000000000..1e05af505b --- /dev/null +++ b/core/include/minifi-cpp/core/VariableRegistry.h @@ -0,0 +1,37 @@ +/** + * + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#pragma once + +#include +#include +#include +#include "minifi-cpp/properties/Configure.h" +#include "utils/StringUtils.h" + +namespace org::apache::nifi::minifi::core { + +/** + * Defines a base variable registry for minifi agents. + */ +class VariableRegistry { + public: + virtual ~VariableRegistry() = default; + virtual bool getConfigurationProperty(const std::string &property, std::string &value) const = 0; +}; + +} // namespace org::apache::nifi::minifi::core diff --git a/core/include/minifi-cpp/core/WeakReference.h b/core/include/minifi-cpp/core/WeakReference.h new file mode 100644 index 0000000000..3b5ecb402d --- /dev/null +++ b/core/include/minifi-cpp/core/WeakReference.h @@ -0,0 +1,62 @@ +/** + * + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include + +#include +#include + +namespace org { +namespace apache { +namespace nifi { +namespace minifi { +namespace core { + +/* + * An homage to weak references in java, this acts as a class + * which can be used to remove referenced classes when needed. + */ +class WeakReference { + public: + virtual ~WeakReference() = default; + virtual void remove() = 0; +}; + +/** + * Reference container is a vector of weak references that enables + * controllers to remove referenced objects as needed. + * + * There is no need to use weak ptrs here, as we do actually want + * the WeakReferences to be referenced counts. The "weak" aspect + * originates from and is defined by the corresponding object. + */ +class ReferenceContainer { + public: + virtual ~ReferenceContainer() = default; + virtual void addReference(std::shared_ptr ref) = 0; + virtual size_t getReferenceCount() = 0; + virtual void removeReferences() = 0; +}; + +} // namespace core +} // namespace minifi +} // namespace nifi +} // namespace apache +} // namespace org diff --git a/core/include/minifi-cpp/core/controller/ControllerService.h b/core/include/minifi-cpp/core/controller/ControllerService.h new file mode 100644 index 0000000000..471eaa3f9c --- /dev/null +++ b/core/include/minifi-cpp/core/controller/ControllerService.h @@ -0,0 +1,67 @@ +/** + * + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#pragma once + +#include +#include +#include +#include + +#include "minifi-cpp/properties/Configure.h" +#include "minifi-cpp/core/Core.h" +#include "minifi-cpp/core/ConfigurableComponent.h" +#include "minifi-cpp/core/Connectable.h" + +namespace org::apache::nifi::minifi::core::controller { + +enum ControllerServiceState { + /** + * Controller Service is disabled and cannot be used. + */ + DISABLED, + /** + * Controller Service is in the process of being disabled. + */ + DISABLING, + /** + * Controller Service is being enabled. + */ + ENABLING, + /** + * Controller Service is enabled. + */ + ENABLED +}; + +/** + * Controller Service base class that contains some pure virtual methods. + * + * Design: OnEnable is executed when the controller service is being enabled. + * Note that keeping state here must be protected in this function. + */ +class ControllerService : public virtual ConfigurableComponent, public virtual Connectable { + public: + virtual void setConfiguration(const std::shared_ptr &configuration) = 0; + virtual ControllerServiceState getState() const = 0; + virtual void onEnable() = 0; + virtual void notifyStop() = 0; + virtual void setState(ControllerServiceState state) = 0; + virtual void setLinkedControllerServices(const std::vector> &services) = 0; +}; + +} // namespace org::apache::nifi::minifi::core::controller diff --git a/core/include/minifi-cpp/core/controller/ControllerServiceLookup.h b/core/include/minifi-cpp/core/controller/ControllerServiceLookup.h new file mode 100644 index 0000000000..64ea9525ee --- /dev/null +++ b/core/include/minifi-cpp/core/controller/ControllerServiceLookup.h @@ -0,0 +1,92 @@ +/** + * + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef LIBMINIFI_INCLUDE_CORE_CONTROLLER_CONTROLLERSERVICELOOKUP_H_ +#define LIBMINIFI_INCLUDE_CORE_CONTROLLER_CONTROLLERSERVICELOOKUP_H_ + +#include +#include +#include +#include "minifi-cpp/core/Core.h" +#include "minifi-cpp/core/ConfigurableComponent.h" +#include "ControllerService.h" + +namespace org { +namespace apache { +namespace nifi { +namespace minifi { +namespace core { +namespace controller { + +/** + * Controller Service Lookup pure virtual class. + * + * Purpose: Provide a mechanism that controllers can lookup information about + * controller services. + * + */ +class ControllerServiceLookup { + public: + ControllerServiceLookup() = default; + + virtual ~ControllerServiceLookup() = default; + + /** + * Gets the controller service via the provided identifier. This overload returns the controller service in a global scope from all + * available controller services in the flow. + * @param identifier reference string for controller service. + * @return controller service reference. + */ + virtual std::shared_ptr getControllerService(const std::string &identifier) const = 0; + + /** + * Gets the controller service in the scope of the processor via the provided identifier. + * @param identifier reference string for controller service. + * @param processor_uuid uuid of the processor + * @return controller service reference. + */ + virtual std::shared_ptr getControllerService(const std::string &identifier, const utils::Identifier &processor_uuid) const = 0; + + /** + * Detects if controller service is enabled. + * @param identifier reference string for controller service. + * @return true if controller service is enabled. + */ + virtual bool isControllerServiceEnabled(const std::string &identifier) = 0; + + /** + * Detects if controller service is being enabled. + * @param identifier reference string for controller service. + * @return true if controller service is enabled. + */ + virtual bool isControllerServiceEnabling(const std::string &identifier) = 0; + + /** + * Gets the controller service name for the provided reference identifier + * @param identifier reference string for the controller service. + */ + virtual const std::string getControllerServiceName(const std::string &identifier) const = 0; +}; + +} // namespace controller +} // namespace core +} // namespace minifi +} // namespace nifi +} // namespace apache +} // namespace org + +#endif // LIBMINIFI_INCLUDE_CORE_CONTROLLER_CONTROLLERSERVICELOOKUP_H_ diff --git a/core/include/minifi-cpp/core/controller/ControllerServiceNode.h b/core/include/minifi-cpp/core/controller/ControllerServiceNode.h new file mode 100644 index 0000000000..daabbe38f8 --- /dev/null +++ b/core/include/minifi-cpp/core/controller/ControllerServiceNode.h @@ -0,0 +1,46 @@ +/** + * + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#pragma once + +#include +#include +#include +#include + +#include "minifi-cpp/core/Core.h" +#include "minifi-cpp/core/ConfigurableComponent.h" +#include "minifi-cpp/core/logging/Logger.h" +#include "minifi-cpp/properties/Configure.h" +#include "ControllerService.h" +#include "io/validation.h" +#include "Exception.h" + +namespace org::apache::nifi::minifi::core::controller { + +class ControllerServiceNode : public virtual CoreComponent, public virtual ConfigurableComponent { + public: + virtual std::shared_ptr getControllerServiceImplementation() = 0; + virtual const ControllerService* getControllerServiceImplementation() const = 0; + virtual const std::vector& getLinkedControllerServices() const = 0; + virtual bool canEnable() = 0; + virtual bool enabled() = 0; + virtual bool enable() = 0; + virtual bool disable() = 0; +}; + +} // namespace org::apache::nifi::minifi::core::controller diff --git a/core/include/minifi-cpp/core/controller/ControllerServiceProvider.h b/core/include/minifi-cpp/core/controller/ControllerServiceProvider.h new file mode 100644 index 0000000000..4d7a1dd351 --- /dev/null +++ b/core/include/minifi-cpp/core/controller/ControllerServiceProvider.h @@ -0,0 +1,54 @@ +/** + * + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include +#include +#include +#include +#include +#include + +#include "minifi-cpp/core/Core.h" +#include "ControllerServiceLookup.h" +#include "minifi-cpp/core/ConfigurableComponent.h" +#include "ControllerServiceNode.h" + +namespace org::apache::nifi::minifi::core { +class ProcessGroup; +} + +namespace org::apache::nifi::minifi::core::controller { + +class ControllerServiceProvider : public virtual CoreComponent, public virtual ConfigurableComponent, public virtual ControllerServiceLookup, public utils::EnableSharedFromThis { + public: + ~ControllerServiceProvider() override = default; + + virtual std::shared_ptr createControllerService(const std::string &type, const std::string &longType, const std::string &id, bool firstTimeAdded) = 0; + virtual ControllerServiceNode* getControllerServiceNode(const std::string &id) const = 0; + virtual ControllerServiceNode* getControllerServiceNode(const std::string &id, const utils::Identifier &processor_or_controller_uuid) const = 0; + virtual void putControllerServiceNode(const std::string& identifier, const std::shared_ptr& controller_service_node, ProcessGroup* process_group) = 0; + + virtual void clearControllerServices() = 0; + virtual std::vector> getAllControllerServices() = 0; + virtual void enableAllControllerServices() = 0; + virtual void disableAllControllerServices() = 0; +}; + +} // namespace org::apache::nifi::minifi::core::controller diff --git a/core/include/minifi-cpp/core/extension/Extension.h b/core/include/minifi-cpp/core/extension/Extension.h new file mode 100644 index 0000000000..e4ff35f597 --- /dev/null +++ b/core/include/minifi-cpp/core/extension/Extension.h @@ -0,0 +1,47 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include +#include +#include + +#include "minifi-cpp/properties/Configure.h" + +namespace org::apache::nifi::minifi::core::extension { + +using ExtensionConfig = std::shared_ptr; + +class Extension { + public: + virtual ~Extension() = default; + + /** + * Ensures that the extension is initialized at most once, and schedules + * an automatic deinitialization on extension unloading. This init/deinit + * is backed by a local static object and sequenced relative to other static + * variable init/deinit (specifically the registration of classes into ClassLoader) + * according to the usual rules. + * @param config + * @return True if the initialization succeeded + */ + virtual bool initialize(const ExtensionConfig& config) = 0; + virtual const std::string& getName() const = 0; +}; + +} // namespace org::apache::nifi::minifi::core::extension diff --git a/core/include/minifi-cpp/core/extension/ExtensionManager.h b/core/include/minifi-cpp/core/extension/ExtensionManager.h new file mode 100644 index 0000000000..38a6e94ff1 --- /dev/null +++ b/core/include/minifi-cpp/core/extension/ExtensionManager.h @@ -0,0 +1,36 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include +#include +#include + +#include "Extension.h" + +namespace org::apache::nifi::minifi::core::extension { + +class ExtensionManager { + public: + static ExtensionManager& get(); + + virtual void registerExtension(Extension& extension) = 0; + virtual void unregisterExtension(Extension& extension) = 0; +}; + +} // namespace org::apache::nifi::minifi::core::extension diff --git a/libminifi/include/core/logging/Logger.h b/core/include/minifi-cpp/core/logging/Logger.h similarity index 51% rename from libminifi/include/core/logging/Logger.h rename to core/include/minifi-cpp/core/logging/Logger.h index d3ddac807a..f94a467582 100644 --- a/libminifi/include/core/logging/Logger.h +++ b/core/include/minifi-cpp/core/logging/Logger.h @@ -27,8 +27,6 @@ #include #include -#include "spdlog/common.h" -#include "spdlog/logger.h" #include "utils/gsl.h" #include "utils/Enum.h" #include "utils/GeneralUtils.h" @@ -38,20 +36,6 @@ namespace org::apache::nifi::minifi::core::logging { -inline constexpr size_t LOG_BUFFER_SIZE = 4096; - -class LoggerControl { - public: - LoggerControl(); - - [[nodiscard]] bool is_enabled() const; - - void setEnabled(bool status); - - protected: - std::atomic is_enabled_; -}; - enum LOG_LEVEL { trace = 0, debug = 1, @@ -62,42 +46,6 @@ enum LOG_LEVEL { off = 6 }; -inline spdlog::level::level_enum mapToSpdLogLevel(LOG_LEVEL level) { - switch (level) { - case trace: return spdlog::level::trace; - case debug: return spdlog::level::debug; - case info: return spdlog::level::info; - case warn: return spdlog::level::warn; - case err: return spdlog::level::err; - case critical: return spdlog::level::critical; - case off: return spdlog::level::off; - } - throw std::invalid_argument(fmt::format("Invalid LOG_LEVEL {}", magic_enum::enum_underlying(level))); -} - -inline LOG_LEVEL mapFromSpdLogLevel(spdlog::level::level_enum level) { - switch (level) { - case spdlog::level::trace: return LOG_LEVEL::trace; - case spdlog::level::debug: return LOG_LEVEL::debug; - case spdlog::level::info: return LOG_LEVEL::info; - case spdlog::level::warn: return LOG_LEVEL::warn; - case spdlog::level::err: return LOG_LEVEL::err; - case spdlog::level::critical: return LOG_LEVEL::critical; - case spdlog::level::off: return LOG_LEVEL::off; - case spdlog::level::n_levels: break; - } - throw std::invalid_argument(fmt::format("Invalid spdlog::level::level_enum {}", magic_enum::enum_underlying(level))); -} - -class BaseLogger { - public: - virtual ~BaseLogger(); - - virtual void log_string(LOG_LEVEL level, std::string str) = 0; - virtual bool should_log(LOG_LEVEL level) = 0; - [[nodiscard]] virtual LOG_LEVEL level() const = 0; -}; - inline constexpr auto map_args = utils::overloaded { [](auto&& f) requires(std::is_invocable_v) { return std::invoke(std::forward(f)); }, [](auto&& value) { return std::forward(value); } @@ -106,69 +54,57 @@ inline constexpr auto map_args = utils::overloaded { template using log_format_string = fmt::format_string...>; -class Logger : public BaseLogger { +class Logger { public: - Logger(Logger const&) = delete; - Logger& operator=(Logger const&) = delete; - template void log_with_level(LOG_LEVEL log_level, log_format_string fmt, Args&& ...args) { - return log(mapToSpdLogLevel(log_level), std::move(fmt), std::forward(args)...); + return log(log_level, std::move(fmt), std::forward(args)...); } template void log_critical(log_format_string fmt, Args&& ...args) { - log(spdlog::level::critical, std::move(fmt), std::forward(args)...); + log(LOG_LEVEL::critical, std::move(fmt), std::forward(args)...); } template void log_error(log_format_string fmt, Args&& ...args) { - log(spdlog::level::err, std::move(fmt), std::forward(args)...); + log(LOG_LEVEL::err, std::move(fmt), std::forward(args)...); } template void log_warn(log_format_string fmt, Args&& ...args) { - log(spdlog::level::warn, std::move(fmt), std::forward(args)...); + log(LOG_LEVEL::warn, std::move(fmt), std::forward(args)...); } template void log_info(log_format_string fmt, Args&& ...args) { - log(spdlog::level::info, std::move(fmt), std::forward(args)...); + log(LOG_LEVEL::info, std::move(fmt), std::forward(args)...); } template void log_debug(log_format_string fmt, Args&& ...args) { - log(spdlog::level::debug, std::move(fmt), std::forward(args)...); + log(LOG_LEVEL::debug, std::move(fmt), std::forward(args)...); } template void log_trace(log_format_string fmt, Args&& ...args) { - log(spdlog::level::trace, std::move(fmt), std::forward(args)...); - } - - void set_max_log_size(int size) { - max_log_size_ = size; + log(LOG_LEVEL::trace, std::move(fmt), std::forward(args)...); } - bool should_log(LOG_LEVEL level) override; - void log_string(LOG_LEVEL level, std::string str) override; - LOG_LEVEL level() const override; - + virtual void set_max_log_size(int size) = 0; virtual std::optional get_id() = 0; + virtual void log_string(LOG_LEVEL level, std::string str) = 0; + virtual bool should_log(LOG_LEVEL level) = 0; + [[nodiscard]] virtual LOG_LEVEL level() const = 0; - protected: - Logger(std::shared_ptr delegate, std::shared_ptr controller); - - Logger(std::shared_ptr delegate); // NOLINT - - std::shared_ptr delegate_; - std::shared_ptr controller_; + virtual ~Logger() = default; - std::mutex mutex_; + protected: + virtual int getMaxLogSize() = 0; private: std::string trimToMaxSizeAndAddId(std::string my_string) { - auto max_log_size = max_log_size_.load(); + auto max_log_size = getMaxLogSize(); if (max_log_size >= 0 && my_string.size() > gsl::narrow(max_log_size)) my_string = my_string.substr(0, max_log_size); if (auto id = get_id()) { @@ -184,17 +120,12 @@ class Logger : public BaseLogger { } template - inline void log(spdlog::level::level_enum level, log_format_string fmt, Args&& ...args) { - if (controller_ && !controller_->is_enabled()) - return; - std::lock_guard lock(mutex_); - if (!delegate_->should_log(level)) { + inline void log(LOG_LEVEL level, log_format_string fmt, Args&& ...args) { + if (!should_log(level)) { return; } - delegate_->log(level, stringify(std::move(fmt), map_args(std::forward(args))...)); + log_string(level, stringify(std::move(fmt), map_args(std::forward(args))...)); } - - std::atomic max_log_size_{LOG_BUFFER_SIZE}; }; } // namespace org::apache::nifi::minifi::core::logging diff --git a/core/include/minifi-cpp/core/logging/LoggerFactory.h b/core/include/minifi-cpp/core/logging/LoggerFactory.h new file mode 100644 index 0000000000..acab5ce103 --- /dev/null +++ b/core/include/minifi-cpp/core/logging/LoggerFactory.h @@ -0,0 +1,34 @@ +/** + * + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include +#include + +#include "Logger.h" +#include "minifi-cpp/core/Core.h" + +namespace org::apache::nifi::minifi::core::logging { + +class LoggerFactoryBase { + public: + static std::shared_ptr getAliasedLogger(std::string_view name, const std::optional& id = {}); +}; + +} // namespace org::apache::nifi::minifi::core::logging diff --git a/core/include/minifi-cpp/core/repository/FileSystemRepository.h b/core/include/minifi-cpp/core/repository/FileSystemRepository.h new file mode 100644 index 0000000000..7be4c9e838 --- /dev/null +++ b/core/include/minifi-cpp/core/repository/FileSystemRepository.h @@ -0,0 +1,27 @@ +/** + * + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include "minifi-cpp/core/ContentRepository.h" + +namespace org::apache::nifi::minifi::core::repository { + +std::shared_ptr createFileSystemRepository(); + +} // namespace org::apache::nifi::minifi::core::repository diff --git a/core/include/minifi-cpp/core/state/FlowIdentifier.h b/core/include/minifi-cpp/core/state/FlowIdentifier.h new file mode 100644 index 0000000000..6e61f789f4 --- /dev/null +++ b/core/include/minifi-cpp/core/state/FlowIdentifier.h @@ -0,0 +1,48 @@ +/** + * + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include + + +namespace org { +namespace apache { +namespace nifi { +namespace minifi { +namespace state { + +/** + * Purpose: Represents a flow identifier for a given flow update or instance. + * + * Design: Immutable collection of strings for the component parts. + */ +class FlowIdentifier { + public: + virtual std::string getRegistryUrl() const = 0 ; + virtual std::string getBucketId() const = 0; + virtual std::string getFlowId() const = 0; + virtual ~FlowIdentifier() = default; +}; + + +} // namespace state +} // namespace minifi +} // namespace nifi +} // namespace apache +} // namespace org diff --git a/core/include/minifi-cpp/core/state/MetricsPublisher.h b/core/include/minifi-cpp/core/state/MetricsPublisher.h new file mode 100644 index 0000000000..6b6cd1ddc1 --- /dev/null +++ b/core/include/minifi-cpp/core/state/MetricsPublisher.h @@ -0,0 +1,37 @@ +/** + * + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#pragma once + +#include + +#include "minifi-cpp/core/Core.h" +#include "minifi-cpp/core/state/nodes/ResponseNodeLoader.h" +#include "minifi-cpp/properties/Configure.h" + +namespace org::apache::nifi::minifi::state { + +class MetricsPublisher : public virtual core::CoreComponent { + public: + using CoreComponent::CoreComponent; + virtual void initialize(const std::shared_ptr& configuration, const std::shared_ptr& response_node_loader) = 0; + virtual void clearMetricNodes() = 0; + virtual void loadMetricNodes() = 0; + virtual ~MetricsPublisher() = default; +}; + +} // namespace org::apache::nifi::minifi::state diff --git a/libminifi/include/core/state/PublishedMetricProvider.h b/core/include/minifi-cpp/core/state/PublishedMetricProvider.h similarity index 93% rename from libminifi/include/core/state/PublishedMetricProvider.h rename to core/include/minifi-cpp/core/state/PublishedMetricProvider.h index c70b2c055f..80369c9b36 100644 --- a/libminifi/include/core/state/PublishedMetricProvider.h +++ b/core/include/minifi-cpp/core/state/PublishedMetricProvider.h @@ -31,9 +31,7 @@ struct PublishedMetric { class PublishedMetricProvider { public: - virtual std::vector calculateMetrics() { - return {}; - } + virtual std::vector calculateMetrics() = 0; virtual ~PublishedMetricProvider() = default; }; diff --git a/core/include/minifi-cpp/core/state/Value.h b/core/include/minifi-cpp/core/state/Value.h new file mode 100644 index 0000000000..ac17811c1d --- /dev/null +++ b/core/include/minifi-cpp/core/state/Value.h @@ -0,0 +1,136 @@ +/** + * + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#pragma once + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "minifi-cpp/utils/Export.h" +#include "utils/meta/type_list.h" + +namespace org::apache::nifi::minifi::state::response { + +/** + * Purpose: Represents an AST value + * Contains an embedded string representation to be used for a toString analog. + * + * Extensions can be more strongly typed and can be used anywhere where an abstract + * representation is needed. + */ +class Value { + public: + virtual ~Value() = default; + + [[nodiscard]] virtual std::string getStringValue() const = 0; + + [[nodiscard]] virtual const char* c_str() const = 0; + + [[nodiscard]] virtual bool empty() const noexcept = 0; + + virtual std::type_index getTypeIndex() = 0; + + virtual bool getValue(uint32_t &ref) = 0; + virtual bool getValue(int &ref) = 0; + virtual bool getValue(int64_t &ref) = 0; + virtual bool getValue(uint64_t &ref) = 0; + virtual bool getValue(bool &ref) = 0; + virtual bool getValue(double &ref) = 0; + + MINIFIAPI static const std::type_index UINT64_TYPE; + MINIFIAPI static const std::type_index INT64_TYPE; + MINIFIAPI static const std::type_index UINT32_TYPE; + MINIFIAPI static const std::type_index INT_TYPE; + MINIFIAPI static const std::type_index BOOL_TYPE; + MINIFIAPI static const std::type_index DOUBLE_TYPE; + MINIFIAPI static const std::type_index STRING_TYPE; +}; + +/** + * Purpose: ValueNode is the AST container for a value + */ +class ValueNode { + using supported_types = utils::meta::type_list; + + public: + ValueNode() = default; + + template + requires (ValueNode::supported_types::contains()) // NOLINT + /* implicit, because it doesn't change the meaning, and it simplifies construction of maps */ + ValueNode(const T value); // NOLINT + + /** + * Define the representations and eventual storage relationships through + * createValue + */ + template + requires (ValueNode::supported_types::contains()) // NOLINT + ValueNode& operator=(const T ref); + + inline bool operator==(const ValueNode &rhs) const { + return to_string() == rhs.to_string(); + } + + inline bool operator==(const char* rhs) const { + return to_string() == rhs; + } + + friend bool operator==(const char* lhs, const ValueNode& rhs) { + return lhs == rhs.to_string(); + } + + [[nodiscard]] std::string to_string() const { + return value_ ? value_->getStringValue() : ""; + } + + [[nodiscard]] std::shared_ptr getValue() const { + return value_; + } + + [[nodiscard]] bool empty() const noexcept { + return value_ == nullptr || value_->empty(); + } + + protected: + std::shared_ptr value_; +}; + +struct SerializedResponseNode { + std::string name; + ValueNode value{}; + bool array = false; + bool collapsible = true; + bool keep_empty = false; + std::vector children{}; + + [[nodiscard]] bool empty() const noexcept { + return value.empty() && children.empty(); + } + + [[nodiscard]] std::string to_string() const; + + [[nodiscard]] std::string to_pretty_string() const; +}; + +} // namespace org::apache::nifi::minifi::state::response diff --git a/core/include/minifi-cpp/core/state/nodes/MetricsBase.h b/core/include/minifi-cpp/core/state/nodes/MetricsBase.h new file mode 100644 index 0000000000..21f20e8505 --- /dev/null +++ b/core/include/minifi-cpp/core/state/nodes/MetricsBase.h @@ -0,0 +1,93 @@ +/** + * + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#pragma once + +#include +#include +#include +#include +#include +#include + +#include "../Value.h" +#include "../PublishedMetricProvider.h" +#include "core/Core.h" +#include "core/Connectable.h" + +namespace org::apache::nifi::minifi::state::response { + +class ResponseNode; +using SharedResponseNode = gsl::not_null>; + +/** + * Purpose: Defines a metric. Serialization is intended to be thread safe. + */ +class ResponseNode : public virtual core::Connectable, public virtual PublishedMetricProvider { + public: + ~ResponseNode() override = default; + + static std::vector serializeAndMergeResponseNodes(const std::vector& nodes); + + virtual std::vector serialize() = 0; + virtual bool isArray() const = 0; + virtual bool isEmpty() = 0; +}; + +/** + * Purpose: Retrieves Metrics from the defined class. The current Metric, which is a consumable for any reader of Metrics must have the ability to set metrics. + * + */ +class ResponseNodeSource { + public: + virtual ~ResponseNodeSource() = default; + virtual SharedResponseNode getResponseNode() = 0; +}; + +class NodeReporter { + public: + struct ReportedNode { + std::string name; + bool is_array; + std::vector serialized_nodes; + }; + + NodeReporter() = default; + + virtual ~NodeReporter() = default; + + /** + * Retrieves metrics node + * @return metrics response node + */ + virtual std::optional getMetricsNode(const std::string& metricsClass) const = 0; + + /** + * Retrieves root nodes configured to be included in heartbeat + * @param includeManifest -- determines if manifest is to be included + * @return a list of response nodes + */ + virtual std::vector getHeartbeatNodes(bool includeManifest) const = 0; + + /** + * Retrieves the agent manifest to be sent as a response to C2 DESCRIBE manifest + * @return the agent manifest response node + */ + virtual ReportedNode getAgentManifest() = 0; +}; + +} // namespace org::apache::nifi::minifi::state::response diff --git a/core/include/minifi-cpp/core/state/nodes/ResponseNodeLoader.h b/core/include/minifi-cpp/core/state/nodes/ResponseNodeLoader.h new file mode 100644 index 0000000000..ee35b10b1c --- /dev/null +++ b/core/include/minifi-cpp/core/state/nodes/ResponseNodeLoader.h @@ -0,0 +1,56 @@ +/** + * + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#pragma once + +#include +#include +#include +#include +#include +#include +#include + +#include "MetricsBase.h" + +namespace org::apache::nifi::minifi::state { +class StateMonitor; +} // namespace org::apache::nifi::minifi::state + +namespace org::apache::nifi::minifi::core { +class ProcessGroup; +} // namespace org::apache::nifi::minifi::core + +namespace org::apache::nifi::minifi::core::controller { +class ControllerServiceProvider; +} // namespace org::apache::nifi::minifi::core::controller + +namespace org::apache::nifi::minifi::state::response { + +class ResponseNodeLoader { + public: + virtual void setNewConfigRoot(core::ProcessGroup* root) = 0; + virtual void clearConfigRoot() = 0; + virtual void setControllerServiceProvider(core::controller::ControllerServiceProvider* controller) = 0; + virtual void setStateMonitor(state::StateMonitor* update_sink) = 0; + virtual std::vector loadResponseNodes(const std::string& clazz) = 0; + virtual state::response::NodeReporter::ReportedNode getAgentManifest() const = 0; + + virtual ~ResponseNodeLoader() = default; +}; + +} // namespace org::apache::nifi::minifi::state::response diff --git a/libminifi/include/io/ArchiveStream.h b/core/include/minifi-cpp/io/ArchiveStream.h similarity index 90% rename from libminifi/include/io/ArchiveStream.h rename to core/include/minifi-cpp/io/ArchiveStream.h index 2f04f3ea27..df8e6e5445 100644 --- a/libminifi/include/io/ArchiveStream.h +++ b/core/include/minifi-cpp/io/ArchiveStream.h @@ -33,18 +33,18 @@ struct EntryInfo { size_t size; }; -class WriteArchiveStream : public OutputStream { +class WriteArchiveStream : public virtual OutputStream { public: virtual bool newEntry(const EntryInfo& info) = 0; virtual bool finish() = 0; }; -class ReadArchiveStream : public InputStream { +class ReadArchiveStream : public virtual InputStream { public: virtual std::optional nextEntry() = 0; }; -class ArchiveStreamProvider : public core::CoreComponent { +class ArchiveStreamProvider : public virtual core::CoreComponent { public: using CoreComponent::CoreComponent; virtual std::unique_ptr createWriteStream(int compress_level, const std::string& compress_format, diff --git a/libminifi/include/io/BaseStream.h b/core/include/minifi-cpp/io/BaseStream.h similarity index 94% rename from libminifi/include/io/BaseStream.h rename to core/include/minifi-cpp/io/BaseStream.h index 0c7b344a9b..f6a5595c87 100644 --- a/libminifi/include/io/BaseStream.h +++ b/core/include/minifi-cpp/io/BaseStream.h @@ -36,6 +36,6 @@ namespace org::apache::nifi::minifi::io { * * Extensions may be thread safe and thus shareable, but that is up to the implementation. */ -class BaseStream : public InputStream, public OutputStream {}; +class BaseStream : public virtual InputStream, public virtual OutputStream {}; } // namespace org::apache::nifi::minifi::io diff --git a/libminifi/include/io/InputStream.h b/core/include/minifi-cpp/io/InputStream.h similarity index 91% rename from libminifi/include/io/InputStream.h rename to core/include/minifi-cpp/io/InputStream.h index ad24d6908d..4d615c8a51 100644 --- a/libminifi/include/io/InputStream.h +++ b/core/include/minifi-cpp/io/InputStream.h @@ -24,15 +24,13 @@ #include #include #include "Stream.h" -#include "utils/Id.h" +#include "minifi-cpp/utils/Id.h" namespace org::apache::nifi::minifi::io { class InputStream : public virtual Stream { public: - [[nodiscard]] virtual size_t size() const { - throw std::runtime_error("Querying size is not supported"); - } + [[nodiscard]] virtual size_t size() const = 0; /** * Reads a byte array from the stream. Use isError (Stream.h) to check for errors. * @param out_buffer reference in which will set the result @@ -82,13 +80,7 @@ class InputStream : public virtual Stream { return sizeof(Integral); } - std::optional readByte() { - std::array buf{}; - if (read(buf) != 1) { - return std::nullopt; - } - return buf[0]; - } + std::optional readByte(); }; } // namespace org::apache::nifi::minifi::io diff --git a/libminifi/include/io/OutputStream.h b/core/include/minifi-cpp/io/OutputStream.h similarity index 100% rename from libminifi/include/io/OutputStream.h rename to core/include/minifi-cpp/io/OutputStream.h diff --git a/libminifi/include/io/Stream.h b/core/include/minifi-cpp/io/Stream.h similarity index 80% rename from libminifi/include/io/Stream.h rename to core/include/minifi-cpp/io/Stream.h index 048a475ca5..e69b646a06 100644 --- a/libminifi/include/io/Stream.h +++ b/core/include/minifi-cpp/io/Stream.h @@ -18,6 +18,11 @@ #pragma once +#ifdef WIN32 +// ignore the warning about inheriting via dominance from Stream +#pragma warning(disable : 4250) +#endif + #include #include "utils/gsl.h" @@ -35,23 +40,11 @@ inline bool isError(const size_t read_write_return) noexcept { */ class Stream { public: - virtual void close() {} - - virtual void seek(size_t /*offset*/) { - throw std::runtime_error("Seek is not supported"); - } - - [[nodiscard]] virtual size_t tell() const { - throw std::runtime_error("Tell is not supported"); - } - - virtual int initialize() { - return 1; - } - - [[nodiscard]] virtual std::span getBuffer() const { - throw std::runtime_error("Not a buffered stream"); - } + virtual void close() = 0; + virtual void seek(size_t /*offset*/) = 0; + [[nodiscard]] virtual size_t tell() const = 0; + virtual int initialize() = 0; + [[nodiscard]] virtual std::span getBuffer() const = 0; virtual ~Stream() = default; }; diff --git a/libminifi/include/io/StreamCallback.h b/core/include/minifi-cpp/io/StreamCallback.h similarity index 100% rename from libminifi/include/io/StreamCallback.h rename to core/include/minifi-cpp/io/StreamCallback.h diff --git a/core/include/minifi-cpp/properties/Configuration.h b/core/include/minifi-cpp/properties/Configuration.h new file mode 100644 index 0000000000..280771ca54 --- /dev/null +++ b/core/include/minifi-cpp/properties/Configuration.h @@ -0,0 +1,212 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include +#include +#include +#include +#include + +#include "Properties.h" +#include "minifi-cpp/utils/Export.h" +#include "minifi-cpp/utils/gsl.h" + +namespace org::apache::nifi::minifi { + +namespace core { +class PropertyValidator; +} + +class Configuration : public virtual Properties { + public: + static constexpr const char *nifi_volatile_repository_options = "nifi.volatile.repository.options."; + static constexpr const char *nifi_global_rocksdb_options = "nifi.global.rocksdb.options."; + static constexpr const char *nifi_flowfile_repository_rocksdb_options = "nifi.flowfile.repository.rocksdb.options."; + static constexpr const char *nifi_content_repository_rocksdb_options = "nifi.content.repository.rocksdb.options."; + static constexpr const char *nifi_provenance_repository_rocksdb_options = "nifi.provenance.repository.rocksdb.options."; + static constexpr const char *nifi_state_storage_rocksdb_options = "nifi.state.storage.rocksdb.options."; + + // nifi.flow.configuration.file + static constexpr const char *nifi_default_directory = "nifi.default.directory"; + static constexpr const char *nifi_flow_configuration_file = "nifi.flow.configuration.file"; + static constexpr const char *nifi_flow_configuration_encrypt = "nifi.flow.configuration.encrypt"; + static constexpr const char *nifi_flow_configuration_file_backup_update = "nifi.flow.configuration.backup.on.update"; + static constexpr const char *nifi_flow_engine_threads = "nifi.flow.engine.threads"; + static constexpr const char *nifi_flow_engine_alert_period = "nifi.flow.engine.alert.period"; + static constexpr const char *nifi_flow_engine_event_driven_time_slice = "nifi.flow.engine.event.driven.time.slice"; + static constexpr const char *nifi_administrative_yield_duration = "nifi.administrative.yield.duration"; + static constexpr const char *nifi_bored_yield_duration = "nifi.bored.yield.duration"; + static constexpr const char *nifi_graceful_shutdown_seconds = "nifi.flowcontroller.graceful.shutdown.period"; + static constexpr const char *nifi_flowcontroller_drain_timeout = "nifi.flowcontroller.drain.timeout"; + static constexpr const char *nifi_configuration_class_name = "nifi.flow.configuration.class.name"; + static constexpr const char *nifi_flow_repository_class_name = "nifi.flowfile.repository.class.name"; + static constexpr const char *nifi_flow_repository_rocksdb_compression = "nifi.flowfile.repository.rocksdb.compression"; + static constexpr const char *nifi_content_repository_class_name = "nifi.content.repository.class.name"; + static constexpr const char *nifi_content_repository_rocksdb_compression = "nifi.content.repository.rocksdb.compression"; + static constexpr const char *nifi_provenance_repository_class_name = "nifi.provenance.repository.class.name"; + static constexpr const char *nifi_volatile_repository_options_flowfile_max_count = "nifi.volatile.repository.options.flowfile.max.count"; + static constexpr const char *nifi_volatile_repository_options_flowfile_max_bytes = "nifi.volatile.repository.options.flowfile.max.bytes"; + static constexpr const char *nifi_volatile_repository_options_provenance_max_count = "nifi.volatile.repository.options.provenance.max.count"; + static constexpr const char *nifi_volatile_repository_options_provenance_max_bytes = "nifi.volatile.repository.options.provenance.max.bytes"; + static constexpr const char *nifi_volatile_repository_options_content_max_count = "nifi.volatile.repository.options.content.max.count"; + static constexpr const char *nifi_volatile_repository_options_content_max_bytes = "nifi.volatile.repository.options.content.max.bytes"; + static constexpr const char *nifi_volatile_repository_options_content_minimal_locking = "nifi.volatile.repository.options.content.minimal.locking"; + static constexpr const char *nifi_provenance_repository_max_storage_size = "nifi.provenance.repository.max.storage.size"; + static constexpr const char *nifi_provenance_repository_max_storage_time = "nifi.provenance.repository.max.storage.time"; + static constexpr const char *nifi_provenance_repository_directory_default = "nifi.provenance.repository.directory.default"; + static constexpr const char *nifi_flowfile_repository_directory_default = "nifi.flowfile.repository.directory.default"; + static constexpr const char *nifi_dbcontent_repository_directory_default = "nifi.database.content.repository.directory.default"; + + // these are internal properties related to the rocksdb backend + static constexpr const char *nifi_flowfile_repository_rocksdb_compaction_period = "nifi.flowfile.repository.rocksdb.compaction.period"; + static constexpr const char *nifi_dbcontent_repository_rocksdb_compaction_period = "nifi.database.content.repository.rocksdb.compaction.period"; + static constexpr const char *nifi_dbcontent_repository_purge_period = "nifi.database.content.repository.purge.period"; + static constexpr const char *nifi_content_repository_rocksdb_use_synchronous_writes = "nifi.content.repository.rocksdb.use.synchronous.writes"; + static constexpr const char *nifi_content_repository_rocksdb_read_verify_checksums = "nifi.content.repository.rocksdb.read.verify.checksums"; + static constexpr const char *nifi_flowfile_repository_rocksdb_read_verify_checksums = "nifi.flowfile.repository.rocksdb.read.verify.checksums"; + static constexpr const char *nifi_provenance_repository_rocksdb_read_verify_checksums = "nifi.provenance.repository.rocksdb.read.verify.checksums"; + static constexpr const char *nifi_rocksdb_state_storage_read_verify_checksums = "nifi.rocksdb.state.storage.read.verify.checksums"; + + static constexpr const char *nifi_remote_input_secure = "nifi.remote.input.secure"; + static constexpr const char *nifi_security_need_ClientAuth = "nifi.security.need.ClientAuth"; + static constexpr const char *nifi_sensitive_props_additional_keys = "nifi.sensitive.props.additional.keys"; + static constexpr const char *nifi_python_processor_dir = "nifi.python.processor.dir"; + static constexpr const char *nifi_extension_path = "nifi.extension.path"; + + // site2site security config + static constexpr const char *nifi_security_client_certificate = "nifi.security.client.certificate"; + static constexpr const char *nifi_security_client_private_key = "nifi.security.client.private.key"; + static constexpr const char *nifi_security_client_pass_phrase = "nifi.security.client.pass.phrase"; + static constexpr const char *nifi_security_client_ca_certificate = "nifi.security.client.ca.certificate"; + static constexpr const char *nifi_security_use_system_cert_store = "nifi.security.use.system.cert.store"; + static constexpr const char *nifi_security_windows_cert_store_location = "nifi.security.windows.cert.store.location"; + static constexpr const char *nifi_security_windows_server_cert_store = "nifi.security.windows.server.cert.store"; + static constexpr const char *nifi_security_windows_client_cert_store = "nifi.security.windows.client.cert.store"; + static constexpr const char *nifi_security_windows_client_cert_cn = "nifi.security.windows.client.cert.cn"; + static constexpr const char *nifi_security_windows_client_cert_key_usage = "nifi.security.windows.client.cert.key.usage"; + + // nifi rest api user name and password + static constexpr const char *nifi_rest_api_user_name = "nifi.rest.api.user.name"; + static constexpr const char *nifi_rest_api_password = "nifi.rest.api.password"; + + // c2 options + static constexpr const char *nifi_c2_enable = "nifi.c2.enable"; + static constexpr const char *nifi_c2_file_watch = "nifi.c2.file.watch"; + static constexpr const char *nifi_c2_flow_id = "nifi.c2.flow.id"; + static constexpr const char *nifi_c2_flow_url = "nifi.c2.flow.url"; + static constexpr const char *nifi_c2_flow_base_url = "nifi.c2.flow.base.url"; + static constexpr const char *nifi_c2_full_heartbeat = "nifi.c2.full.heartbeat"; + static constexpr const char *nifi_c2_agent_heartbeat_period = "nifi.c2.agent.heartbeat.period"; + static constexpr const char *nifi_c2_agent_class = "nifi.c2.agent.class"; + static constexpr const char *nifi_c2_agent_heartbeat_reporter_classes = "nifi.c2.agent.heartbeat.reporter.classes"; + static constexpr const char *nifi_c2_agent_identifier = "nifi.c2.agent.identifier"; + static constexpr const char *nifi_c2_agent_identifier_fallback = "nifi.c2.agent.identifier.fallback"; + static constexpr const char *nifi_c2_agent_trigger_classes = "nifi.c2.agent.trigger.classes"; + static constexpr const char *nifi_c2_root_classes = "nifi.c2.root.classes"; + static constexpr const char *nifi_c2_root_class_definitions = "nifi.c2.root.class.definitions"; + static constexpr const char *nifi_c2_rest_listener_port = "nifi.c2.rest.listener.port"; + static constexpr const char *nifi_c2_rest_listener_cacert = "nifi.c2.rest.listener.cacert"; + static constexpr const char *nifi_c2_rest_path_base = "nifi.c2.rest.path.base"; + static constexpr const char *nifi_c2_rest_url = "nifi.c2.rest.url"; + static constexpr const char *nifi_c2_rest_url_ack = "nifi.c2.rest.url.ack"; + static constexpr const char *nifi_c2_rest_ssl_context_service = "nifi.c2.rest.ssl.context.service"; + static constexpr const char *nifi_c2_rest_heartbeat_minimize_updates = "nifi.c2.rest.heartbeat.minimize.updates"; + static constexpr const char *nifi_c2_rest_request_encoding = "nifi.c2.rest.request.encoding"; + + // state management options + static constexpr const char *nifi_state_storage_local = "nifi.state.storage.local"; + static constexpr const char *nifi_state_storage_local_old = "nifi.state.management.provider.local"; + static constexpr const char *nifi_state_storage_local_class_name = "nifi.state.storage.local.class.name"; + static constexpr const char *nifi_state_storage_local_class_name_old = "nifi.state.management.provider.local.class.name"; + static constexpr const char *nifi_state_storage_local_always_persist = "nifi.state.storage.local.always.persist"; + static constexpr const char *nifi_state_storage_local_always_persist_old = "nifi.state.management.provider.local.always.persist"; + static constexpr const char *nifi_state_storage_local_auto_persistence_interval = "nifi.state.storage.local.auto.persistence.interval"; + static constexpr const char *nifi_state_storage_local_auto_persistence_interval_old = "nifi.state.management.provider.local.auto.persistence.interval"; + static constexpr const char *nifi_state_storage_local_path = "nifi.state.storage.local.path"; + static constexpr const char *nifi_state_storage_local_path_old = "nifi.state.management.provider.local.path"; + + // disk space watchdog options + static constexpr const char *minifi_disk_space_watchdog_enable = "minifi.disk.space.watchdog.enable"; + static constexpr const char *minifi_disk_space_watchdog_interval = "minifi.disk.space.watchdog.interval"; + static constexpr const char *minifi_disk_space_watchdog_stop_threshold = "minifi.disk.space.watchdog.stop.threshold"; + static constexpr const char *minifi_disk_space_watchdog_restart_threshold = "minifi.disk.space.watchdog.restart.threshold"; + + // Log options + static constexpr const char *nifi_log_spdlog_pattern = "nifi.log.spdlog.pattern"; + static constexpr const char *nifi_log_spdlog_shorten_names = "nifi.log.spdlog.shorten_names"; + static constexpr const char *nifi_log_appender_rolling = "nifi.log.appender.rolling"; + static constexpr const char *nifi_log_appender_rolling_directory = "nifi.log.appender.rolling.directory"; + static constexpr const char *nifi_log_appender_rolling_file_name = "nifi.log.appender.rolling.file_name"; + static constexpr const char *nifi_log_appender_rolling_max_files = "nifi.log.appender.rolling.max_files"; + static constexpr const char *nifi_log_appender_rolling_max_file_size = "nifi.log.appender.rolling.max_file_size"; + static constexpr const char *nifi_log_appender_stdout = "nifi.log.appender.stdout"; + static constexpr const char *nifi_log_appender_stderr = "nifi.log.appender.stderr"; + static constexpr const char *nifi_log_appender_null = "nifi.log.appender.null"; + static constexpr const char *nifi_log_appender_syslog = "nifi.log.appender.syslog"; + static constexpr const char *nifi_log_logger_root = "nifi.log.logger.root"; + static constexpr const char *nifi_log_compression_cached_log_max_size = "nifi.log.compression.cached.log.max.size"; + static constexpr const char *nifi_log_compression_compressed_log_max_size = "nifi.log.compression.compressed.log.max.size"; + static constexpr const char *nifi_log_max_log_entry_length = "nifi.log.max.log.entry.length"; + + // alert options + static constexpr const char *nifi_log_alert_url = "nifi.log.alert.url"; + static constexpr const char *nifi_log_alert_ssl_context_service = "nifi.log.alert.ssl.context.service"; + static constexpr const char *nifi_log_alert_batch_size = "nifi.log.alert.batch.size"; + static constexpr const char *nifi_log_alert_flush_period = "nifi.log.alert.flush.period"; + static constexpr const char *nifi_log_alert_filter = "nifi.log.alert.filter"; + static constexpr const char *nifi_log_alert_rate_limit = "nifi.log.alert.rate.limit"; + static constexpr const char *nifi_log_alert_buffer_limit = "nifi.log.alert.buffer.limit"; + static constexpr const char *nifi_log_alert_level = "nifi.log.alert.level"; + + static constexpr const char *nifi_asset_directory = "nifi.asset.directory"; + + // Metrics publisher options + static constexpr const char *nifi_metrics_publisher_agent_identifier = "nifi.metrics.publisher.agent.identifier"; + static constexpr const char *nifi_metrics_publisher_class = "nifi.metrics.publisher.class"; + static constexpr const char *nifi_metrics_publisher_prometheus_metrics_publisher_port = "nifi.metrics.publisher.PrometheusMetricsPublisher.port"; + static constexpr const char *nifi_metrics_publisher_prometheus_metrics_publisher_metrics = "nifi.metrics.publisher.PrometheusMetricsPublisher.metrics"; + static constexpr const char *nifi_metrics_publisher_log_metrics_publisher_metrics = "nifi.metrics.publisher.LogMetricsPublisher.metrics"; + static constexpr const char *nifi_metrics_publisher_log_metrics_logging_interval = "nifi.metrics.publisher.LogMetricsPublisher.logging.interval"; + static constexpr const char *nifi_metrics_publisher_log_metrics_log_level = "nifi.metrics.publisher.LogMetricsPublisher.log.level"; + static constexpr const char *nifi_metrics_publisher_metrics = "nifi.metrics.publisher.metrics"; + static constexpr const char *nifi_metrics_publisher_prometheus_metrics_publisher_certificate = "nifi.metrics.publisher.PrometheusMetricsPublisher.certificate"; + static constexpr const char *nifi_metrics_publisher_prometheus_metrics_publisher_ca_certificate = "nifi.metrics.publisher.PrometheusMetricsPublisher.ca.certificate"; + + // Controller socket options + static constexpr const char *controller_socket_enable = "controller.socket.enable"; + static constexpr const char *controller_socket_local_any_interface = "controller.socket.local.any.interface"; + static constexpr const char *controller_socket_host = "controller.socket.host"; + static constexpr const char *controller_socket_port = "controller.socket.port"; + static constexpr const char *controller_ssl_context_service = "controller.ssl.context.service"; + + static constexpr const char *nifi_flow_file_repository_check_health = "nifi.flowfile.repository.check.health"; + static constexpr const char *nifi_python_virtualenv_directory = "nifi.python.virtualenv.directory"; + static constexpr const char *nifi_python_env_setup_binary = "nifi.python.env.setup.binary"; + static constexpr const char *nifi_python_install_packages_automatically = "nifi.python.install.packages.automatically"; + + MINIFIAPI static const std::unordered_map> CONFIGURATION_PROPERTIES; + MINIFIAPI static const std::array DEFAULT_SENSITIVE_PROPERTIES; + + static std::vector mergeProperties(std::vector properties, + const std::vector& additional_properties); + static std::vector getSensitiveProperties(const std::function(const std::string&)>& reader); + static bool validatePropertyValue(const std::string& property_name, const std::string& property_value); +}; + +} // namespace org::apache::nifi::minifi diff --git a/core/include/minifi-cpp/properties/Configure.h b/core/include/minifi-cpp/properties/Configure.h new file mode 100644 index 0000000000..e72dec51b7 --- /dev/null +++ b/core/include/minifi-cpp/properties/Configure.h @@ -0,0 +1,49 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#pragma once + +#include +#include +#include +#include + +#include "Configuration.h" +#include "minifi-cpp/core/AgentIdentificationProvider.h" + +struct ConfigTestAccessor; + +namespace org::apache::nifi::minifi { + +class Configure : public virtual Configuration, public virtual core::AgentIdentificationProvider { + friend struct ::ConfigTestAccessor; + public: + virtual bool get(const std::string& key, std::string& value) const = 0; + virtual bool get(const std::string& key, const std::string& alternate_key, std::string& value) const = 0; + virtual std::optional get(const std::string& key) const = 0; + virtual std::optional getWithFallback(const std::string& key, const std::string& alternate_key) const = 0; + virtual std::optional getRawValue(const std::string& key) const = 0; + + virtual void setFallbackAgentIdentifier(const std::string& id) = 0; + + using Configuration::set; + virtual void set(const std::string& key, const std::string& value, PropertyChangeLifetime lifetime) override = 0; + virtual bool commitChanges() override = 0; + + static std::shared_ptr create(); +}; + +} // namespace org::apache::nifi::minifi diff --git a/core/include/minifi-cpp/properties/Properties.h b/core/include/minifi-cpp/properties/Properties.h new file mode 100644 index 0000000000..9f684bdcda --- /dev/null +++ b/core/include/minifi-cpp/properties/Properties.h @@ -0,0 +1,69 @@ +/** + * @file Configure.h + * Configure class declaration + * + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#pragma once + +#ifdef WIN32 +// ignore the warning about inheriting via dominance from Properties +#pragma warning(disable : 4250) +#endif + +#include +#include +#include +#include +#include +#include +#include + +namespace org::apache::nifi::minifi { + +namespace utils { +class ChecksumCalculator; +} // namespace utils + +enum class PropertyChangeLifetime { + TRANSIENT, // the changed value will not be committed to disk + PERSISTENT // the changed value will be written to the source file +}; + +class Properties { + public: + virtual ~Properties() = default; + virtual const std::string& getName() const = 0; + virtual void clear() = 0; + virtual void set(const std::string& key, const std::string& value) = 0; + virtual void set(const std::string &key, const std::string &value, PropertyChangeLifetime lifetime) = 0; + virtual bool has(const std::string& key) const = 0; + virtual bool getString(const std::string &key, std::string &value) const = 0; + virtual int getInt(const std::string &key, int default_value) const = 0; + virtual std::optional getString(const std::string& key) const = 0; + virtual void loadConfigureFile(const std::filesystem::path& configuration_file, std::string_view prefix = "") = 0; + virtual void setHome(std::filesystem::path minifiHome) = 0; + virtual std::vector getConfiguredKeys() const = 0; + virtual std::filesystem::path getHome() const = 0; + virtual bool commitChanges() = 0; + virtual utils::ChecksumCalculator& getChecksumCalculator() = 0; + virtual std::filesystem::path getFilePath() const = 0; + virtual std::map getProperties() const = 0; + + static std::shared_ptr create(); +}; + +} // namespace org::apache::nifi::minifi diff --git a/core/include/minifi-cpp/provenance/Provenance.h b/core/include/minifi-cpp/provenance/Provenance.h new file mode 100644 index 0000000000..6dafba625c --- /dev/null +++ b/core/include/minifi-cpp/provenance/Provenance.h @@ -0,0 +1,218 @@ +/** + * + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#pragma once + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "minifi-cpp/core/Core.h" +#include "minifi-cpp/core/SerializableComponent.h" +#include "minifi-cpp/core/Repository.h" +#include "minifi-cpp/core/Property.h" +#include "minifi-cpp/properties/Configure.h" +#include "minifi-cpp/Connection.h" +#include "minifi-cpp/ResourceClaim.h" +#include "utils/gsl.h" +#include "utils/Id.h" + +namespace org::apache::nifi::minifi::provenance { + +class ProvenanceEventRecord : public virtual core::SerializableComponent { + public: + enum ProvenanceEventType { + /** + * A CREATE event is used when a FlowFile is generated from data that was + * not received from a remote system or external process + */ + CREATE, + + /** + * Indicates a provenance event for receiving data from an external process. This Event Type + * is expected to be the first event for a FlowFile. As such, a Processor that receives data + * from an external source and uses that data to replace the content of an existing FlowFile + * should use the {@link #FETCH} event type, rather than the RECEIVE event type. + */ + RECEIVE, + + /** + * Indicates that the contents of a FlowFile were overwritten using the contents of some + * external resource. This is similar to the {@link #RECEIVE} event but varies in that + * RECEIVE events are intended to be used as the event that introduces the FlowFile into + * the system, whereas FETCH is used to indicate that the contents of an existing FlowFile + * were overwritten. + */ + FETCH, + + /** + * Indicates a provenance event for sending data to an external process + */ + SEND, + + /** + * Indicates that the contents of a FlowFile were downloaded by a user or external entity. + */ + DOWNLOAD, + + /** + * Indicates a provenance event for the conclusion of an object's life for + * some reason other than object expiration + */ + DROP, + + /** + * Indicates a provenance event for the conclusion of an object's life due + * to the fact that the object could not be processed in a timely manner + */ + EXPIRE, + + /** + * FORK is used to indicate that one or more FlowFile was derived from a + * parent FlowFile. + */ + FORK, + + /** + * JOIN is used to indicate that a single FlowFile is derived from joining + * together multiple parent FlowFiles. + */ + JOIN, + + /** + * CLONE is used to indicate that a FlowFile is an exact duplicate of its + * parent FlowFile. + */ + CLONE, + + /** + * CONTENT_MODIFIED is used to indicate that a FlowFile's content was + * modified in some way. When using this Event Type, it is advisable to + * provide details about how the content is modified. + */ + CONTENT_MODIFIED, + + /** + * ATTRIBUTES_MODIFIED is used to indicate that a FlowFile's attributes were + * modified in some way. This event is not needed when another event is + * reported at the same time, as the other event will already contain all + * FlowFile attributes. + */ + ATTRIBUTES_MODIFIED, + + /** + * ROUTE is used to show that a FlowFile was routed to a specified + * {@link org.apache.nifi.processor.Relationship Relationship} and should provide + * information about why the FlowFile was routed to this relationship. + */ + ROUTE, + + /** + * Indicates a provenance event for adding additional information such as a + * new linkage to a new URI or UUID + */ + ADDINFO, + + /** + * Indicates a provenance event for replaying a FlowFile. The UUID of the + * event will indicate the UUID of the original FlowFile that is being + * replayed. The event will contain exactly one Parent UUID that is also the + * UUID of the FlowFile that is being replayed and exactly one Child UUID + * that is the UUID of the a newly created FlowFile that will be re-queued + * for processing. + */ + REPLAY + }; + static const char *ProvenanceEventTypeStr[REPLAY + 1]; + + ~ProvenanceEventRecord() override = default; + + virtual utils::Identifier getEventId() const = 0; + virtual void setEventId(const utils::Identifier &id) = 0; + virtual std::map getAttributes() const = 0; + virtual uint64_t getFileSize() const = 0; + virtual uint64_t getFileOffset() const = 0; + virtual std::chrono::system_clock::time_point getFlowFileEntryDate() const = 0; + virtual std::chrono::system_clock::time_point getlineageStartDate() const = 0; + virtual std::chrono::system_clock::time_point getEventTime() const = 0; + virtual std::chrono::milliseconds getEventDuration() const = 0; + virtual void setEventDuration(std::chrono::milliseconds duration) = 0; + virtual ProvenanceEventType getEventType() const = 0; + virtual std::string getComponentId() const = 0; + virtual std::string getComponentType() const = 0; + virtual utils::Identifier getFlowFileUuid() const = 0; + virtual std::string getContentFullPath() const = 0; + virtual std::vector getLineageIdentifiers() const = 0; + virtual std::string getDetails() const = 0; + virtual void setDetails(const std::string& details) = 0; + virtual std::string getTransitUri() = 0; + virtual void setTransitUri(const std::string& uri) = 0; + virtual std::string getSourceSystemFlowFileIdentifier() const = 0; + virtual void setSourceSystemFlowFileIdentifier(const std::string& identifier) = 0; + virtual std::vector getParentUuids() const = 0; + virtual void addParentUuid(const utils::Identifier& uuid) = 0; + virtual void addParentFlowFile(const core::FlowFile& flow_file) = 0; + virtual void removeParentUuid(const utils::Identifier& uuid) = 0; + virtual void removeParentFlowFile(const core::FlowFile& flow_file) = 0; + virtual std::vector getChildrenUuids() const = 0; + virtual void addChildUuid(const utils::Identifier& uuid) = 0; + virtual void addChildFlowFile(const core::FlowFile& flow_file) = 0; + virtual void removeChildUuid(const utils::Identifier& uuid) = 0; + virtual void removeChildFlowFile(const core::FlowFile& flow_file) = 0; + virtual std::string getAlternateIdentifierUri() const = 0; + virtual void setAlternateIdentifierUri(const std::string& uri) = 0; + virtual std::string getRelationship() const = 0; + virtual void setRelationship(const std::string& relation) = 0; + virtual std::string getSourceQueueIdentifier() const = 0; + virtual void setSourceQueueIdentifier(const std::string& identifier) = 0; + virtual void fromFlowFile(const core::FlowFile& flow_file) = 0; + virtual bool loadFromRepository(const std::shared_ptr &repo) = 0; + + static std::shared_ptr create(); +}; + +class ProvenanceReporter { + public: + virtual ~ProvenanceReporter() = default; + + virtual std::set> getEvents() const = 0; + virtual void add(const std::shared_ptr &event) = 0 ; + virtual void remove(const std::shared_ptr &event) = 0; + virtual void clear() = 0; + + virtual void commit() = 0; + virtual void create(const core::FlowFile& flow_file, const std::string& detail) = 0; + virtual void route(const core::FlowFile& flow_file, const core::Relationship& relation, const std::string& detail, std::chrono::milliseconds processingDuration) = 0; + virtual void modifyAttributes(const core::FlowFile& flow_file, const std::string& detail) = 0; + virtual void modifyContent(const core::FlowFile& flow_file, const std::string& detail, std::chrono::milliseconds processingDuration) = 0; + virtual void clone(const core::FlowFile& parent, const core::FlowFile& child) = 0; + virtual void expire(const core::FlowFile& flow_file, const std::string& detail) = 0; + virtual void drop(const core::FlowFile& flow_file, const std::string& reason) = 0; + virtual void send(const core::FlowFile& flow_file, const std::string& transitUri, const std::string& detail, std::chrono::milliseconds processingDuration, bool force) = 0; + virtual void fetch(const core::FlowFile& flow_file, const std::string& transitUri, const std::string& detail, std::chrono::milliseconds processingDuration) = 0; + virtual void receive(const core::FlowFile& flow_file, const std::string& transitUri, + const std::string& sourceSystemFlowFileIdentifier, const std::string& detail, std::chrono::milliseconds processingDuration) = 0; +}; + +} // namespace org::apache::nifi::minifi::provenance diff --git a/core/include/minifi-cpp/utils/AnyRef.h b/core/include/minifi-cpp/utils/AnyRef.h new file mode 100644 index 0000000000..05b06ffb45 --- /dev/null +++ b/core/include/minifi-cpp/utils/AnyRef.h @@ -0,0 +1,29 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#pragma once + +namespace org::apache::nifi::minifi::utils { + +class AnyRef { + class Base { + + }; + public: + +}; + +} // namespace org::apache::nifi::minifi::utils diff --git a/core/include/minifi-cpp/utils/Export.h b/core/include/minifi-cpp/utils/Export.h new file mode 100644 index 0000000000..175821d93c --- /dev/null +++ b/core/include/minifi-cpp/utils/Export.h @@ -0,0 +1,35 @@ +/** + * + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#ifdef WIN32 + #ifdef LIBMINIFI + #define MINIFIAPI __declspec(dllexport) + #else + #define MINIFIAPI __declspec(dllimport) + #endif + #ifdef MODULE_NAME + #define EXTENSIONAPI __declspec(dllexport) + #else + #define EXTENSIONAPI __declspec(dllimport) + #endif +#else + #define MINIFIAPI + #define EXTENSIONAPI +#endif diff --git a/libminifi/include/utils/FlatMap.h b/core/include/minifi-cpp/utils/FlatMap.h similarity index 100% rename from libminifi/include/utils/FlatMap.h rename to core/include/minifi-cpp/utils/FlatMap.h diff --git a/core/include/minifi-cpp/utils/Id.h b/core/include/minifi-cpp/utils/Id.h new file mode 100644 index 0000000000..5c435dcf92 --- /dev/null +++ b/core/include/minifi-cpp/utils/Id.h @@ -0,0 +1,75 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#pragma once + +#include +#include +#include +#include +#include +#include +#include +#include + +#include "SmallString.h" + +namespace org::apache::nifi::minifi::utils { + +class Identifier { + friend struct IdentifierTestAccessor; + static constexpr const char* hex_lut = "0123456789abcdef"; + + public: + using Data = std::array; + + Identifier() = default; + explicit Identifier(const Data& data); + Identifier &operator=(const Data& data); + + Identifier &operator=(const std::string& idStr); + + explicit operator bool() const { + return !isNil(); + } + + bool operator!=(const Identifier& other) const; + bool operator==(const Identifier& other) const; + bool operator<(const Identifier& other) const; + + bool isNil() const; + + // Numerous places query the string representation + // just to then forward the temporary to build logs, + // streams, or others. Dynamically allocating in these + // instances is wasteful as we immediately discard + // the result. The difference on the test machine is 8x, + // building the representation itself takes 10ns, while + // subsequently turning it into a std::string would take + // 70ns more. + SmallString<36> to_string() const; + + static std::optional parse(const std::string& str); + + private: + friend struct ::std::hash; + + static bool parseByte(Data& data, const uint8_t* input, int& charIdx, int& byteIdx); + + Data data_{}; +}; + +} // namespace org::apache::nifi::minifi::utils diff --git a/libminifi/include/utils/Literals.h b/core/include/minifi-cpp/utils/Literals.h similarity index 100% rename from libminifi/include/utils/Literals.h rename to core/include/minifi-cpp/utils/Literals.h diff --git a/libminifi/include/utils/PropertyErrors.h b/core/include/minifi-cpp/utils/PropertyErrors.h similarity index 94% rename from libminifi/include/utils/PropertyErrors.h rename to core/include/minifi-cpp/utils/PropertyErrors.h index 85474a55dd..4fafed8ae6 100644 --- a/libminifi/include/utils/PropertyErrors.h +++ b/core/include/minifi-cpp/utils/PropertyErrors.h @@ -16,8 +16,7 @@ * limitations under the License. */ -#ifndef LIBMINIFI_INCLUDE_UTILS_PROPERTYERRORS_H_ -#define LIBMINIFI_INCLUDE_UTILS_PROPERTYERRORS_H_ +#pragma once #include @@ -89,5 +88,3 @@ class RequiredPropertyMissingException : public PropertyException { } /* namespace nifi */ } /* namespace apache */ } /* namespace org */ - -#endif // LIBMINIFI_INCLUDE_UTILS_PROPERTYERRORS_H_ diff --git a/libminifi/include/utils/SmallString.h b/core/include/minifi-cpp/utils/SmallString.h similarity index 84% rename from libminifi/include/utils/SmallString.h rename to core/include/minifi-cpp/utils/SmallString.h index 2ff5dd9b60..a92d13b5a3 100644 --- a/libminifi/include/utils/SmallString.h +++ b/core/include/minifi-cpp/utils/SmallString.h @@ -89,18 +89,3 @@ class SmallString : public std::array { }; } // namespace org::apache::nifi::minifi::utils - -template -struct fmt::formatter> { - formatter string_view_formatter; - - template - constexpr auto parse(ParseContext& ctx) { - return string_view_formatter.parse(ctx); - } - - template - auto format(const org::apache::nifi::minifi::utils::SmallString& small_string, FormatContext& ctx) { - return string_view_formatter.format(small_string.view(), ctx); - } -}; diff --git a/core/include/minifi-cpp/utils/TimeUtil.h b/core/include/minifi-cpp/utils/TimeUtil.h new file mode 100644 index 0000000000..aa16657118 --- /dev/null +++ b/core/include/minifi-cpp/utils/TimeUtil.h @@ -0,0 +1,58 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include +#include + +namespace org::apache::nifi::minifi::utils::timeutils { + +/** + * Mockable clock classes + */ +class Clock { + public: + virtual ~Clock() = default; + virtual std::chrono::milliseconds timeSinceEpoch() const = 0; + virtual bool wait_until(std::condition_variable& cv, std::unique_lock& lck, std::chrono::milliseconds time, const std::function& pred) { + return cv.wait_for(lck, time - timeSinceEpoch(), pred); + } +}; + +class SteadyClock : public Clock { + public: + std::chrono::milliseconds timeSinceEpoch() const override { + return std::chrono::duration_cast(std::chrono::steady_clock::now().time_since_epoch()); + } + + virtual std::chrono::time_point now() const { + return std::chrono::steady_clock::now(); + } +}; + +std::shared_ptr getClock(); + +// test-only utility to specify what clock to use +void setClock(std::shared_ptr clock); + +#ifdef WIN32 +void dateSetGlobalInstall(const std::string& install); +#endif + +} // namespace org::apache::nifi::minifi::utils::timeutils + diff --git a/libminifi/include/utils/gsl.h b/core/include/minifi-cpp/utils/gsl.h similarity index 100% rename from libminifi/include/utils/gsl.h rename to core/include/minifi-cpp/utils/gsl.h diff --git a/encrypt-config/CMakeLists.txt b/encrypt-config/CMakeLists.txt index cfa4365642..05146bdb74 100644 --- a/encrypt-config/CMakeLists.txt +++ b/encrypt-config/CMakeLists.txt @@ -26,7 +26,7 @@ endif() add_minifi_executable(encrypt-config "${ENCRYPT_CONFIG_FILES}") target_include_directories(encrypt-config PRIVATE ../libminifi/include) include(ArgParse) -target_link_libraries(encrypt-config libsodium argparse ${LIBMINIFI}) +target_link_libraries(encrypt-config libsodium argparse core-minifi) set_target_properties(encrypt-config PROPERTIES OUTPUT_NAME encrypt-config) set_target_properties(encrypt-config PROPERTIES RUNTIME_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}/bin") diff --git a/encrypt-config/FlowConfigEncryptor.cpp b/encrypt-config/FlowConfigEncryptor.cpp index 9c28499dd0..e5cb75cc0a 100644 --- a/encrypt-config/FlowConfigEncryptor.cpp +++ b/encrypt-config/FlowConfigEncryptor.cpp @@ -179,7 +179,7 @@ void encryptSensitiveValuesInFlowConfig(const EncryptionKeys& keys, const std::f throw std::runtime_error("Error: cannot re-encrypt without an .old key!"); } - const auto configure = std::make_shared(); + const auto configure = std::make_shared(); configure->setHome(minifi_home); configure->loadConfigureFile(DEFAULT_NIFI_PROPERTIES_FILE); @@ -191,7 +191,7 @@ void encryptSensitiveValuesInFlowConfig(const EncryptionKeys& keys, const std::f utils::crypto::EncryptionProvider{utils::crypto::XSalsa20Cipher{*keys.old_key}} : utils::crypto::EncryptionProvider{utils::crypto::XSalsa20Cipher{keys.encryption_key}}; - core::extension::ExtensionManager::get().initialize(configure); + core::extension::ExtensionManagerImpl::get().initialize(configure); core::flow::AdaptiveConfiguration adaptive_configuration{core::ConfigurationContext{ .flow_file_repo = nullptr, diff --git a/encrypt-config/tests/CMakeLists.txt b/encrypt-config/tests/CMakeLists.txt index 175e70cff2..f45d82ba06 100644 --- a/encrypt-config/tests/CMakeLists.txt +++ b/encrypt-config/tests/CMakeLists.txt @@ -31,7 +31,7 @@ foreach(testfile ${ENCRYPT_CONFIG_TESTS}) createTests(${testfilename}) - target_link_libraries(${testfilename} Catch2WithMain ${LIBMINIFI} libminifi-unittest) + target_link_libraries(${testfilename} Catch2WithMain core-minifi libminifi-unittest) add_test(NAME ${testfilename} COMMAND ${testfilename} WORKING_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}") math(EXPR ENCRYPT_CONFIG_TEST_COUNT "${ENCRYPT_CONFIG_TEST_COUNT}+1") diff --git a/extension-utils/CMakeLists.txt b/extension-utils/CMakeLists.txt new file mode 100644 index 0000000000..3ba365b2c2 --- /dev/null +++ b/extension-utils/CMakeLists.txt @@ -0,0 +1,35 @@ +file(GLOB SOURCES + src/*.cpp + src/core/*.cpp + src/controllers/keyvalue/*.cpp + src/controllers/*.cpp + src/io/*.cpp + src/serialization/*.cpp + src/utils/file/*.cpp + src/utils/net/*.cpp + src/utils/tls/*.cpp + src/utils/*.cpp) +add_minifi_library(minifi-extension-utils STATIC ${SOURCES}) +target_include_directories(minifi-extension-utils PUBLIC include) +target_link_libraries(minifi-extension-utils PUBLIC minifi-core) + +include(RangeV3) +include(Asio) +include(MagicEnum) +list(APPEND CORE_LIBRARIES ZLIB::ZLIB concurrentqueue RapidJSON spdlog Threads::Threads gsl-lite range-v3 expected-lite asio magic_enum OpenSSL::Crypto OpenSSL::SSL CURL::libcurl RapidJSON) +if(NOT WIN32) + list(APPEND CORE_LIBRARIES OSSP::libuuid++) +endif() +if ("${CMAKE_CXX_COMPILER_ID}" STREQUAL "GNU" AND CMAKE_CXX_COMPILER_VERSION VERSION_LESS 9) + list(APPEND CORE_LIBRARIES stdc++fs) +endif() +target_link_libraries(minifi-extension-utils PUBLIC ${CMAKE_DL_LIBS} ${CORE_LIBRARIES}) +if (APPLE) + target_link_libraries(minifi-extension-utils PUBLIC "-framework CoreFoundation -framework SystemConfiguration") +endif() +if (UNIX AND (CMAKE_SYSTEM_PROCESSOR MATCHES "armv7")) + target_link_libraries(minifi-extension-utils PUBLIC "-latomic") +endif() + +target_link_libraries(minifi-extension-utils PRIVATE minifi-core) +target_link_libraries(minifi-extension-utils PUBLIC minifi-utils) \ No newline at end of file diff --git a/extension-utils/include/FlowFileRecord.h b/extension-utils/include/FlowFileRecord.h new file mode 100644 index 0000000000..8a1658e333 --- /dev/null +++ b/extension-utils/include/FlowFileRecord.h @@ -0,0 +1,21 @@ +/** + * + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include "minifi-cpp/FlowFileRecord.h" diff --git a/extension-utils/include/ResourceClaim.h b/extension-utils/include/ResourceClaim.h new file mode 100644 index 0000000000..ff9ada8cc5 --- /dev/null +++ b/extension-utils/include/ResourceClaim.h @@ -0,0 +1,21 @@ +/** + * + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include "minifi-cpp/ResourceClaim.h" \ No newline at end of file diff --git a/libminifi/include/controllers/AttributeProviderService.h b/extension-utils/include/controllers/AttributeProviderService.h similarity index 80% rename from libminifi/include/controllers/AttributeProviderService.h rename to extension-utils/include/controllers/AttributeProviderService.h index b3a59be874..7e3836f531 100644 --- a/libminifi/include/controllers/AttributeProviderService.h +++ b/extension-utils/include/controllers/AttributeProviderService.h @@ -22,20 +22,18 @@ #include #include "core/controller/ControllerService.h" +#include "minifi-cpp/controllers/AttributeProviderService.h" namespace org::apache::nifi::minifi::controllers { -class AttributeProviderService : public core::controller::ControllerService { +class AttributeProviderServiceImpl : public core::controller::ControllerServiceImpl, public virtual AttributeProviderService { public: - using ControllerService::ControllerService; + using ControllerServiceImpl::ControllerServiceImpl; void yield() override {} bool isRunning() const override { return getState() == core::controller::ControllerServiceState::ENABLED; } bool isWorkAvailable() override { return false; } - - using AttributeMap = std::unordered_map; - virtual std::optional> getAttributes() = 0; - virtual std::string_view name() const = 0; }; } // namespace org::apache::nifi::minifi::controllers + diff --git a/extension-utils/include/controllers/RecordSetReader.h b/extension-utils/include/controllers/RecordSetReader.h new file mode 100644 index 0000000000..ead6e019f1 --- /dev/null +++ b/extension-utils/include/controllers/RecordSetReader.h @@ -0,0 +1,30 @@ +/** +* Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#pragma once + +#include "minifi-cpp/controllers/RecordSetReader.h" +#include "core/controller/ControllerService.h" + + +namespace org::apache::nifi::minifi::core { + +class RecordSetReaderImpl : public virtual controller::ControllerServiceImpl, public virtual RecordSetReader { + public: + using ControllerServiceImpl::ControllerServiceImpl; +}; + +} // namespace org::apache::nifi::minifi::core diff --git a/extension-utils/include/controllers/RecordSetWriter.h b/extension-utils/include/controllers/RecordSetWriter.h new file mode 100644 index 0000000000..8ba127b3d0 --- /dev/null +++ b/extension-utils/include/controllers/RecordSetWriter.h @@ -0,0 +1,29 @@ +/** +* Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#pragma once + +#include "minifi-cpp/controllers/RecordSetWriter.h" +#include "core/controller/ControllerService.h" + +namespace org::apache::nifi::minifi::core { + +class RecordSetWriterImpl : public virtual controller::ControllerServiceImpl, public virtual RecordSetWriter { + public: + using ControllerServiceImpl::ControllerServiceImpl; +}; + +} // namespace org::apache::nifi::minifi::core diff --git a/extension-utils/include/controllers/SSLContextService.h b/extension-utils/include/controllers/SSLContextService.h new file mode 100644 index 0000000000..e399a734aa --- /dev/null +++ b/extension-utils/include/controllers/SSLContextService.h @@ -0,0 +1,19 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#pragma once + +#include "minifi-cpp/controllers/SSLContextService.h" \ No newline at end of file diff --git a/extension-utils/include/controllers/keyvalue/AutoPersistor.h b/extension-utils/include/controllers/keyvalue/AutoPersistor.h new file mode 100644 index 0000000000..3cefe8c514 --- /dev/null +++ b/extension-utils/include/controllers/keyvalue/AutoPersistor.h @@ -0,0 +1,61 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#pragma once + +#include +#include +#include +#include +#include +#include + +#include "core/ConfigurableComponent.h" +#include "core/Core.h" +#include "core/logging/LoggerFactory.h" +#include "utils/Export.h" + +namespace org::apache::nifi::minifi::controllers { + +/** + * Persists in given intervals. + * Has an own thread, so stop() must be called before destruction of data used by persist_. + */ +class AutoPersistor { + public: + ~AutoPersistor(); + + void start(bool always_persist, std::chrono::milliseconds auto_persistence_interval, std::function persist); + void stop(); + + [[nodiscard]] bool isAlwaysPersisting() const { + return always_persist_; + } + + private: + void persistingThreadFunc(); + + bool always_persist_ = false; + std::chrono::milliseconds auto_persistence_interval_{0}; + std::thread persisting_thread_; + bool running_ = false; + std::mutex persisting_mutex_; + std::condition_variable persisting_cv_; + std::function persist_; + std::shared_ptr logger_ = core::logging::LoggerFactory::getLogger(); +}; + +} // namespace org::apache::nifi::minifi::controllers diff --git a/libminifi/include/controllers/keyvalue/KeyValueStateManager.h b/extension-utils/include/controllers/keyvalue/KeyValueStateManager.h similarity index 94% rename from libminifi/include/controllers/keyvalue/KeyValueStateManager.h rename to extension-utils/include/controllers/keyvalue/KeyValueStateManager.h index 19df889935..604a113aec 100644 --- a/libminifi/include/controllers/keyvalue/KeyValueStateManager.h +++ b/extension-utils/include/controllers/keyvalue/KeyValueStateManager.h @@ -29,10 +29,11 @@ namespace org::apache::nifi::minifi::controllers { class KeyValueStateStorage; -class KeyValueStateManager final : public core::StateManager { +class KeyValueStateManager final : public core::StateManagerImpl { public: KeyValueStateManager(const utils::Identifier& id, gsl::not_null storage); + using StateManagerImpl::get; bool set(const core::StateManager::State& kvs) override; bool get(core::StateManager::State& kvs) override; bool clear() override; diff --git a/libminifi/include/controllers/keyvalue/KeyValueStateStorage.h b/extension-utils/include/controllers/keyvalue/KeyValueStateStorage.h similarity index 89% rename from libminifi/include/controllers/keyvalue/KeyValueStateStorage.h rename to extension-utils/include/controllers/keyvalue/KeyValueStateStorage.h index 9fa3fca20b..149c795395 100644 --- a/libminifi/include/controllers/keyvalue/KeyValueStateStorage.h +++ b/extension-utils/include/controllers/keyvalue/KeyValueStateStorage.h @@ -24,21 +24,20 @@ #include "core/controller/ControllerService.h" #include "core/Core.h" #include "core/logging/LoggerFactory.h" -#include "core/StateManager.h" +#include "minifi-cpp/core/StateManager.h" #include "core/StateStorage.h" +#include "minifi-cpp/controllers/keyvalue/KeyValueStateStorage.h" namespace org::apache::nifi::minifi::controllers { -constexpr const char* ALWAYS_PERSIST_PROPERTY_NAME = "Always Persist"; -constexpr const char* AUTO_PERSISTENCE_INTERVAL_PROPERTY_NAME = "Auto Persistence Interval"; - -class KeyValueStateStorage : public core::StateStorage, public core::controller::ControllerService { +class KeyValueStateStorage : public core::StateStorageImpl, public core::controller::ControllerServiceImpl { public: explicit KeyValueStateStorage(const std::string& name, const utils::Identifier& uuid = {}); static core::StateManager::State deserialize(const std::string& serialized); static std::string serialize(const core::StateManager::State& kvs); + using core::StateStorageImpl::getStateManager; std::unique_ptr getStateManager(const utils::Identifier& uuid) override; std::unordered_map getAllStates() override; diff --git a/libminifi/include/core/AbstractProcessor.h b/extension-utils/include/core/AbstractProcessor.h similarity index 93% rename from libminifi/include/core/AbstractProcessor.h rename to extension-utils/include/core/AbstractProcessor.h index 2862bbda23..113361f033 100644 --- a/libminifi/include/core/AbstractProcessor.h +++ b/extension-utils/include/core/AbstractProcessor.h @@ -20,17 +20,17 @@ #include #include #include "range/v3/view/transform.hpp" -#include "core/Annotation.h" +#include "minifi-cpp/core/Annotation.h" #include "core/Core.h" #include "core/Processor.h" #include "core/PropertyDefinition.h" -#include "core/RelationshipDefinition.h" +#include "minifi-cpp/core/RelationshipDefinition.h" namespace org::apache::nifi::minifi::core { template -class AbstractProcessor : public Processor { +class AbstractProcessor : public ProcessorImpl { public: - using Processor::Processor; + using ProcessorImpl::ProcessorImpl; void initialize() final { static_assert(std::is_same_v); diff --git a/libminifi/include/core/Deprecated.h b/extension-utils/include/core/Deprecated.h similarity index 100% rename from libminifi/include/core/Deprecated.h rename to extension-utils/include/core/Deprecated.h diff --git a/extension-utils/include/core/FlowFile.h b/extension-utils/include/core/FlowFile.h new file mode 100644 index 0000000000..4a64a2d078 --- /dev/null +++ b/extension-utils/include/core/FlowFile.h @@ -0,0 +1,20 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include "minifi-cpp/core/FlowFile.h" diff --git a/libminifi/include/core/FlowFileStore.h b/extension-utils/include/core/FlowFileStore.h similarity index 100% rename from libminifi/include/core/FlowFileStore.h rename to extension-utils/include/core/FlowFileStore.h diff --git a/extension-utils/include/core/ProcessSession.h b/extension-utils/include/core/ProcessSession.h new file mode 100644 index 0000000000..26678bedb8 --- /dev/null +++ b/extension-utils/include/core/ProcessSession.h @@ -0,0 +1,27 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include "minifi-cpp/core/ProcessSession.h" + +namespace org::apache::nifi::minifi::core::detail { + +std::string to_string(const ReadBufferResult& read_buffer_result); + +} // namespace org::apache::nifi::minifi::core::detail + diff --git a/extension-utils/include/core/StateManager.h b/extension-utils/include/core/StateManager.h new file mode 100644 index 0000000000..bf5a30e665 --- /dev/null +++ b/extension-utils/include/core/StateManager.h @@ -0,0 +1,55 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include "core/Core.h" + +#include +#include +#include +#include +#include "minifi-cpp/core/StateManager.h" + +namespace org::apache::nifi::minifi::core { + +/** + * Stores state for one component. + * Supported operations: get(), set(), clear(), persist(). + * Behavior can be transactional. Use beginTransaction() to enter a transaction and commit() or rollback() to conclude it. + */ +class StateManagerImpl : public virtual StateManager { + public: + explicit StateManagerImpl(const utils::Identifier& id) + : id_(id) { + } + + ~StateManagerImpl() override = default; + + using StateManager::get; + std::optional get() override { + if (State out; get(out)) { + return out; + } + return std::nullopt; + } + + protected: + utils::Identifier id_; +}; + +} // namespace org::apache::nifi::minifi::core diff --git a/extension-utils/include/core/StateStorage.h b/extension-utils/include/core/StateStorage.h new file mode 100644 index 0000000000..89822310fb --- /dev/null +++ b/extension-utils/include/core/StateStorage.h @@ -0,0 +1,43 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include "core/Core.h" +#include "StateManager.h" + +#include +#include +#include +#include "minifi-cpp/core/StateStorage.h" + +namespace org::apache::nifi::minifi::core { + +/** + * Serves as a state storage background for the entire application, all StateManagers are created by it and use it. + */ +class StateStorageImpl : public virtual StateStorage { + public: + ~StateStorageImpl() override = default; + + using StateStorage::getStateManager; + std::unique_ptr getStateManager(const CoreComponent& component) override { + return getStateManager(component.getUUID()); + } +}; + +} // namespace org::apache::nifi::minifi::core diff --git a/extension-utils/include/properties/Configure.h b/extension-utils/include/properties/Configure.h new file mode 100644 index 0000000000..d3d229d21d --- /dev/null +++ b/extension-utils/include/properties/Configure.h @@ -0,0 +1,19 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#pragma once + +#include "minifi-cpp/properties/Configure.h" diff --git a/extension-utils/include/properties/Properties.h b/extension-utils/include/properties/Properties.h new file mode 100644 index 0000000000..c1e3eca4c9 --- /dev/null +++ b/extension-utils/include/properties/Properties.h @@ -0,0 +1,19 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#pragma once + +#include "minifi-cpp/properties/Properties.h" diff --git a/libminifi/include/serialization/FlowFileSerializer.h b/extension-utils/include/serialization/FlowFileSerializer.h similarity index 100% rename from libminifi/include/serialization/FlowFileSerializer.h rename to extension-utils/include/serialization/FlowFileSerializer.h diff --git a/libminifi/include/serialization/FlowFileV3Serializer.h b/extension-utils/include/serialization/FlowFileV3Serializer.h similarity index 100% rename from libminifi/include/serialization/FlowFileV3Serializer.h rename to extension-utils/include/serialization/FlowFileV3Serializer.h diff --git a/libminifi/include/serialization/PayloadSerializer.h b/extension-utils/include/serialization/PayloadSerializer.h similarity index 100% rename from libminifi/include/serialization/PayloadSerializer.h rename to extension-utils/include/serialization/PayloadSerializer.h diff --git a/extension-utils/include/utils/CallBackTimer.h b/extension-utils/include/utils/CallBackTimer.h new file mode 100644 index 0000000000..c5fe96b2c6 --- /dev/null +++ b/extension-utils/include/utils/CallBackTimer.h @@ -0,0 +1,62 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef LIBMINIFI_INCLUDE_UTILS_CALLBACKTIMER_H_ +#define LIBMINIFI_INCLUDE_UTILS_CALLBACKTIMER_H_ + +#include +#include +#include +#include +#include + +namespace org { +namespace apache { +namespace nifi { +namespace minifi { +namespace utils { + +class CallBackTimer { + public: + CallBackTimer(std::chrono::milliseconds interval, const std::function& func); + ~CallBackTimer(); + + void stop(); + + void start(); + + bool is_running() const; + + private: + bool execute_; + std::function func_; + std::thread thd_; + mutable std::mutex mtx_; + mutable std::mutex cv_mtx_; + std::condition_variable cv_; + + const std::chrono::milliseconds interval_; +}; + +} // namespace utils +} // namespace minifi +} // namespace nifi +} // namespace apache +} // namespace org + +#endif // LIBMINIFI_INCLUDE_UTILS_CALLBACKTIMER_H_ + diff --git a/extension-utils/include/utils/ClassUtils.h b/extension-utils/include/utils/ClassUtils.h new file mode 100644 index 0000000000..3726651669 --- /dev/null +++ b/extension-utils/include/utils/ClassUtils.h @@ -0,0 +1,32 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#pragma once + +#include +#include + +namespace org::apache::nifi::minifi::utils::ClassUtils { + +/** + * Shortens class names via the canonical representation ( package with name ) + * @param class_name input class name + * @param out output class name that is shortened. + * @return true if out has been updated, false otherwise + */ +bool shortenClassName(std::string_view class_name, std::string &out); + +} // namespace org::apache::nifi::minifi::utils::ClassUtils diff --git a/extension-utils/include/utils/CollectionUtils.h b/extension-utils/include/utils/CollectionUtils.h new file mode 100644 index 0000000000..46b3f4f832 --- /dev/null +++ b/extension-utils/include/utils/CollectionUtils.h @@ -0,0 +1,59 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include +#include + +namespace org { +namespace apache { +namespace nifi { +namespace minifi { +namespace utils { + +namespace internal { + +template +struct find_in_range { + static auto call(const T& range, const Arg& arg) -> decltype(std::find(range.begin(), range.end(), arg)) { + return std::find(range.begin(), range.end(), arg); + } +}; + +template +struct find_in_range().find(std::declval()), void())> { + static auto call(const T& range, const Arg& arg) -> decltype(range.find(arg)) { + return range.find(arg); + } +}; + +} // namespace internal + +template +bool haveCommonItem(const T& a, const U& b) { + using Item = typename T::value_type; + return std::any_of(a.begin(), a.end(), [&] (const Item& item) { + return internal::find_in_range::call(b, item) != b.end(); + }); +} + +} // namespace utils +} // namespace minifi +} // namespace nifi +} // namespace apache +} // namespace org diff --git a/extension-utils/include/utils/Cron.h b/extension-utils/include/utils/Cron.h new file mode 100644 index 0000000000..97b78e3d99 --- /dev/null +++ b/extension-utils/include/utils/Cron.h @@ -0,0 +1,57 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#pragma once + +#include +#include +#include +#include +#include +#include +#include "date/tz.h" +#include "Exception.h" + +namespace org::apache::nifi::minifi::utils { +class BadCronExpression : public minifi::Exception { + public: + explicit BadCronExpression(const std::string& errmsg) : minifi::Exception(errmsg) {} +}; + +class CronField { + public: + virtual ~CronField() = default; + + [[nodiscard]] virtual bool matches(date::local_seconds time_point) const = 0; + virtual bool operator==(const CronField&) const { throw std::runtime_error("not implemented"); } +}; + +class Cron { + public: + explicit Cron(const std::string& expression); + + [[nodiscard]] std::optional calculateNextTrigger(date::local_seconds start) const; + + std::unique_ptr second_; + std::unique_ptr minute_; + std::unique_ptr hour_; + std::unique_ptr day_; + std::unique_ptr month_; + std::unique_ptr day_of_week_; + std::unique_ptr year_; +}; + +} // namespace org::apache::nifi::minifi::utils diff --git a/extension-utils/include/utils/Error.h b/extension-utils/include/utils/Error.h new file mode 100644 index 0000000000..77ef4a6aca --- /dev/null +++ b/extension-utils/include/utils/Error.h @@ -0,0 +1,26 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include + +namespace org::apache::nifi::minifi::utils { + +std::error_code getLastError(); + +} // namespace org::apache::nifi::minifi::utils diff --git a/extension-utils/include/utils/Export.h b/extension-utils/include/utils/Export.h new file mode 100644 index 0000000000..175821d93c --- /dev/null +++ b/extension-utils/include/utils/Export.h @@ -0,0 +1,35 @@ +/** + * + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#ifdef WIN32 + #ifdef LIBMINIFI + #define MINIFIAPI __declspec(dllexport) + #else + #define MINIFIAPI __declspec(dllimport) + #endif + #ifdef MODULE_NAME + #define EXTENSIONAPI __declspec(dllexport) + #else + #define EXTENSIONAPI __declspec(dllimport) + #endif +#else + #define MINIFIAPI + #define EXTENSIONAPI +#endif diff --git a/extension-utils/include/utils/FifoExecutor.h b/extension-utils/include/utils/FifoExecutor.h new file mode 100644 index 0000000000..1aca59d560 --- /dev/null +++ b/extension-utils/include/utils/FifoExecutor.h @@ -0,0 +1,67 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include +#include +#include + +#include "utils/MinifiConcurrentQueue.h" + +namespace org::apache::nifi::minifi::utils { + +namespace detail { +class WorkerThread final { + public: + WorkerThread(); + + WorkerThread(const WorkerThread&) = delete; + WorkerThread(WorkerThread&&) = delete; + WorkerThread& operator=(WorkerThread) = delete; + + ~WorkerThread(); + + template + void enqueue(Args&&... args) { task_queue_.enqueue(std::forward(args)...); } + + private: + void run() noexcept; + + utils::ConditionConcurrentQueue> task_queue_; + std::thread thread_; +}; +} // namespace detail + +/** + * Executes arbitrary functions with no parameters asynchronously on an internal thread, returning a future to the result. + */ +class FifoExecutor final { + public: + template + auto enqueue(Func func) -> std::future { + using result_type = decltype(func()); + std::packaged_task task{std::move(func)}; + auto future = task.get_future(); + worker_thread_.enqueue(std::move(task)); + return future; + } + private: + detail::WorkerThread worker_thread_; +}; + +} // namespace org::apache::nifi::minifi::utils diff --git a/extension-utils/include/utils/FileMutex.h b/extension-utils/include/utils/FileMutex.h new file mode 100644 index 0000000000..54596ead2e --- /dev/null +++ b/extension-utils/include/utils/FileMutex.h @@ -0,0 +1,59 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include +#include +#include +#include +#include "utils/gsl.h" + +#ifdef WIN32 +#include +#endif + +namespace org::apache::nifi::minifi::utils { + +// Warning: this will write the pid of the current process into the file +class FileMutex { + public: + explicit FileMutex(std::filesystem::path path); + ~FileMutex() { + gsl_Expects(!file_handle_.has_value()); + } + + FileMutex(const FileMutex&) = delete; + FileMutex(FileMutex&&) = delete; + FileMutex& operator=(const FileMutex&) = delete; + FileMutex& operator=(FileMutex&&) = delete; + + void lock(); + void unlock(); + + private: + std::filesystem::path path_; + + std::mutex mtx_; +#ifdef WIN32 + std::optional file_handle_; +#else + std::optional file_handle_; +#endif +}; + +} // namespace org::apache::nifi::minifi::utils diff --git a/extension-utils/include/utils/IntervalSwitch.h b/extension-utils/include/utils/IntervalSwitch.h new file mode 100644 index 0000000000..0052b4e006 --- /dev/null +++ b/extension-utils/include/utils/IntervalSwitch.h @@ -0,0 +1,74 @@ +/** + * + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include +#include + +#include "utils/gsl.h" + +namespace org { +namespace apache { +namespace nifi { +namespace minifi { +namespace utils { + +enum class IntervalSwitchState { + LOWER, + UPPER, +}; + +namespace detail { +struct SwitchReturn { + IntervalSwitchState state; + bool switched; +}; +} // namespace detail + +template> +class IntervalSwitch { + public: + IntervalSwitch(T lower_threshold, T upper_threshold, const IntervalSwitchState initial_state = IntervalSwitchState::UPPER) + :lower_threshold_{std::move(lower_threshold)}, upper_threshold_{std::move(upper_threshold)}, state_{initial_state} { + gsl_Expects(!less_(upper_threshold_, lower_threshold_)); + } + + detail::SwitchReturn operator()(const T& value) { + const auto old_state = state_; + if (less_(value, lower_threshold_)) { + state_ = IntervalSwitchState::LOWER; + } else if (!less_(value, upper_threshold_)) { + state_ = IntervalSwitchState::UPPER; + } + return {state_, state_ != old_state}; + } + + private: + T lower_threshold_; + T upper_threshold_; + Comp less_; + + IntervalSwitchState state_; +}; + +} // namespace utils +} // namespace minifi +} // namespace nifi +} // namespace apache +} // namespace org diff --git a/extension-utils/include/utils/JsonCallback.h b/extension-utils/include/utils/JsonCallback.h new file mode 100644 index 0000000000..c455e6c348 --- /dev/null +++ b/extension-utils/include/utils/JsonCallback.h @@ -0,0 +1,100 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#pragma once + +#include +#include +#include +#include + +#include "rapidjson/stream.h" +#include "rapidjson/writer.h" +#include "rapidjson/prettywriter.h" + +#include "io/StreamPipe.h" + +namespace org { +namespace apache { +namespace nifi { +namespace minifi { +namespace utils { + +class JsonInputCallback { + public: + explicit JsonInputCallback(rapidjson::Document& document) : document_(document) {} + int64_t operator()(const std::shared_ptr& stream) { + std::string content; + content.resize(stream->size()); + const auto read_ret = stream->read(as_writable_bytes(std::span(content))); + if (io::isError(read_ret)) { + return -1; + } + rapidjson::ParseResult parse_result = document_.Parse(content.data()); + if (parse_result.IsError()) + return -1; + + return read_ret; + } + private: + rapidjson::Document& document_; +}; + +class JsonOutputCallback { + public: + explicit JsonOutputCallback(rapidjson::Document&& root, std::optional decimal_places) + : root_(std::move(root)), decimal_places_(decimal_places) {} + + int64_t operator()(const std::shared_ptr& stream) const { + rapidjson::StringBuffer buffer; + rapidjson::Writer writer(buffer); + if (decimal_places_.has_value()) + writer.SetMaxDecimalPlaces(decimal_places_.value()); + root_.Accept(writer); + const auto write_return = stream->write(reinterpret_cast(buffer.GetString()), buffer.GetSize()); + return !io::isError(write_return) ? gsl::narrow(write_return) : -1; + } + + protected: + rapidjson::Document root_; + std::optional decimal_places_; +}; + +class PrettyJsonOutputCallback { + public: + explicit PrettyJsonOutputCallback(rapidjson::Document&& root, std::optional decimal_places) + : root_(std::move(root)), decimal_places_(decimal_places) {} + + int64_t operator()(const std::shared_ptr& stream) const { + rapidjson::StringBuffer buffer; + rapidjson::PrettyWriter writer(buffer); + if (decimal_places_.has_value()) + writer.SetMaxDecimalPlaces(decimal_places_.value()); + root_.Accept(writer); + const auto write_return = stream->write(reinterpret_cast(buffer.GetString()), buffer.GetSize()); + return !io::isError(write_return) ? gsl::narrow(write_return) : -1; + } + + protected: + rapidjson::Document root_; + std::optional decimal_places_; +}; + +} // namespace utils +} // namespace minifi +} // namespace nifi +} // namespace apache +} // namespace org diff --git a/extension-utils/include/utils/LineByLineInputOutputStreamCallback.h b/extension-utils/include/utils/LineByLineInputOutputStreamCallback.h new file mode 100644 index 0000000000..0d68fe5680 --- /dev/null +++ b/extension-utils/include/utils/LineByLineInputOutputStreamCallback.h @@ -0,0 +1,50 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#pragma once + +#include +#include +#include +#include +#include + +#include "core/logging/Logger.h" +#include "io/InputStream.h" +#include "io/OutputStream.h" +#include "io/StreamPipe.h" + +namespace org::apache::nifi::minifi::utils { + +class LineByLineInputOutputStreamCallback { + public: + using CallbackType = std::function; + explicit LineByLineInputOutputStreamCallback(CallbackType callback); + int64_t operator()(const std::shared_ptr& input, const std::shared_ptr& output); + + private: + int64_t readInput(io::InputStream& stream); + void readLine(); + [[nodiscard]] bool isLastLine() const { return !next_line_.has_value(); } + + CallbackType callback_; + std::vector input_; + std::vector::iterator current_pos_{}; + std::optional current_line_; + std::optional next_line_; +}; + +} // namespace org::apache::nifi::minifi::utils diff --git a/libminifi/include/utils/ListingStateManager.h b/extension-utils/include/utils/ListingStateManager.h similarity index 95% rename from libminifi/include/utils/ListingStateManager.h rename to extension-utils/include/utils/ListingStateManager.h index 7fa9501a9f..0b4a0b7ee5 100644 --- a/libminifi/include/utils/ListingStateManager.h +++ b/extension-utils/include/utils/ListingStateManager.h @@ -57,9 +57,6 @@ class ListingStateManager { void storeState(const ListingState &latest_listing_state); private: - static const std::string LATEST_LISTED_OBJECT_PREFIX; - static const std::string LATEST_LISTED_OBJECT_TIMESTAMP; - [[nodiscard]] static uint64_t getLatestListedKeyTimestampInMilliseconds(const std::unordered_map &state); [[nodiscard]] static std::unordered_set getLatestListedKeys(const std::unordered_map &state); diff --git a/extension-utils/include/utils/LogUtils.h b/extension-utils/include/utils/LogUtils.h new file mode 100644 index 0000000000..c8d852c9b7 --- /dev/null +++ b/extension-utils/include/utils/LogUtils.h @@ -0,0 +1,87 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#pragma once + +#include +#include + +#include "utils/Enum.h" + +namespace org::apache::nifi::minifi::utils::LogUtils { + +enum class LogLevelOption { + LOGGING_TRACE, + LOGGING_DEBUG, + LOGGING_INFO, + LOGGING_WARN, + LOGGING_ERROR, + LOGGING_CRITICAL, + LOGGING_OFF +}; + +inline LogLevelOption mapToLogLevelOption(core::logging::LOG_LEVEL level) { + switch (level) { + case core::logging::trace: return LogLevelOption::LOGGING_TRACE; + case core::logging::debug: return LogLevelOption::LOGGING_DEBUG; + case core::logging::info: return LogLevelOption::LOGGING_INFO; + case core::logging::warn: return LogLevelOption::LOGGING_WARN; + case core::logging::err: return LogLevelOption::LOGGING_ERROR; + case core::logging::critical: return LogLevelOption::LOGGING_CRITICAL; + case core::logging::off: return LogLevelOption::LOGGING_OFF; + } + throw std::invalid_argument(fmt::format("Invalid LOG_LEVEL {}", magic_enum::enum_underlying(level))); +} + +inline core::logging::LOG_LEVEL mapToLogLevel(LogLevelOption option) { + switch (option) { + case LogLevelOption::LOGGING_TRACE: return core::logging::trace; + case LogLevelOption::LOGGING_DEBUG: return core::logging::debug; + case LogLevelOption::LOGGING_INFO: return core::logging::info; + case LogLevelOption::LOGGING_WARN: return core::logging::warn; + case LogLevelOption::LOGGING_ERROR: return core::logging::err; + case LogLevelOption::LOGGING_CRITICAL: return core::logging::critical; + case LogLevelOption::LOGGING_OFF: return core::logging::off; + } + throw std::invalid_argument(fmt::format("Invalid LogLevelOption {}", magic_enum::enum_underlying(option))); +} + +} // namespace org::apache::nifi::minifi::utils::LogUtils + +namespace magic_enum::customize { +using LogLevelOption = org::apache::nifi::minifi::utils::LogUtils::LogLevelOption; + +template <> +constexpr customize_t enum_name(LogLevelOption value) noexcept { + switch (value) { + case LogLevelOption::LOGGING_TRACE: + return "TRACE"; + case LogLevelOption::LOGGING_DEBUG: + return "DEBUG"; + case LogLevelOption::LOGGING_INFO: + return "INFO"; + case LogLevelOption::LOGGING_WARN: + return "WARN"; + case LogLevelOption::LOGGING_ERROR: + return "ERROR"; + case LogLevelOption::LOGGING_CRITICAL: + return "CRITICAL"; + case LogLevelOption::LOGGING_OFF: + return "OFF"; + } + return invalid_tag; +} +} // namespace magic_enum::customize diff --git a/extension-utils/include/utils/MapUtils.h b/extension-utils/include/utils/MapUtils.h new file mode 100644 index 0000000000..55011d78df --- /dev/null +++ b/extension-utils/include/utils/MapUtils.h @@ -0,0 +1,45 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#pragma once + +#include + +namespace org { +namespace apache { +namespace nifi { +namespace minifi { +namespace utils { +namespace MapUtils { + +/** + * Return a set of keys from a map + */ +template