MINIFICPP-2765 Move GCP Extension to stable C API

behave_framework/src/minifi_behave/steps/checking_steps.py

@@ -119,7 +119,7 @@ def verify_minifi_logs_match_regex(context, regex, duration):
 @step('no errors were generated on the http-proxy regarding "{url}"')
 def verify_no_errors_on_http_proxy(context: MinifiTestContext, url: str):
     http_proxy_container = next(container for container in context.containers.values() if isinstance(container, HttpProxy))
-    assert http_proxy_container.check_http_proxy_access(url) or http_proxy_container.log_app_output()
+    assert http_proxy_container.check_http_proxy_access(url) or log_due_to_failure(context)
 
 
 @then('in the "{container}" container no files are placed in the "{directory}" directory in {duration} of running time')

core-framework/common/include/core/PropertyDefinitionBuilder.h

@@ -27,7 +27,21 @@ namespace org::apache::nifi::minifi::core {
 namespace detail {
 template<typename... Types>
 inline constexpr auto TypeNames = std::array<std::string_view, sizeof...(Types)>{core::className<Types>()...};
-}
+
+template <size_t N>
+struct StringLiteral {
+  char value[N];
+  constexpr StringLiteral(const char (&str)[N]) {  // NOLINT(runtime/explicit)
+    for (size_t i = 0; i < N; ++i) {
+      value[i] = str[i];
+    }
+  }
+};
+
+// A variable template that creates permanent static memory for the span to point to
+template <StringLiteral str>
+inline constexpr auto StaticAllowedType = std::array<std::string_view, 1>{std::string_view{str.value, sizeof(str.value) - 1}};
+}  // namespace detail
 
 template<size_t NumAllowedValues = 0>
 struct PropertyDefinitionBuilder {
@@ -81,6 +95,12 @@ struct PropertyDefinitionBuilder {
     return *this;
   }
 
+  template <detail::StringLiteral TypeName>
+  constexpr PropertyDefinitionBuilder<NumAllowedValues> withAllowedType() {
+    property.allowed_types = detail::StaticAllowedType<TypeName>;
+    return *this;
+  }
+
   constexpr PropertyDefinitionBuilder<NumAllowedValues> withValidator(const PropertyValidator& property_validator) {
     property.validator = gsl::make_not_null(&property_validator);
     return *this;

extension-framework/cpp-extension-lib/include/api/core/ProcessContext.h

@@ -17,10 +17,11 @@
 
 #pragma once
 
-#include <string>
 #include <expected>
+#include <string>
 
 #include "api/core/FlowFile.h"
+#include "api/utils/Proxy.h"
 #include "api/utils/Ssl.h"
 #include "minifi-c.h"
 #include "minifi-cpp/core/PropertyDefinition.h"
@@ -39,12 +40,12 @@ class ProcessContext {
 
   [[nodiscard]] virtual std::expected<std::string, std::error_code> getProperty(const minifi::core::PropertyReference& prop,
       const FlowFile* ff) const = 0;
-  [[nodiscard]] virtual std::expected<MinifiControllerService*, std::error_code> getControllerService(std::string_view name,
-      std::string_view type) const = 0;
+  [[nodiscard]] virtual std::expected<MinifiControllerService*, std::error_code> getControllerService(const minifi::core::PropertyReference& prop) const = 0;
   [[nodiscard]] virtual bool hasNonEmptyProperty(std::string_view name) const = 0;
   [[nodiscard]] virtual std::map<std::string, std::string> getDynamicProperties(const FlowFile* flow_file) const = 0;
 
-  [[nodiscard]] virtual std::expected<utils::net::SslData, std::error_code> getSslData(std::string_view name) const = 0;
+  [[nodiscard]] virtual std::expected<std::optional<utils::net::SslData>, std::error_code> getSslData(const minifi::core::PropertyReference& prop) const = 0;
+  [[nodiscard]] virtual std::expected<std::optional<utils::ProxyData>, std::error_code> getProxyData(const minifi::core::PropertyReference& prop) const = 0;
 };
 
 class CffiProcessContext : public ProcessContext {
@@ -53,12 +54,12 @@ class CffiProcessContext : public ProcessContext {
 
   [[nodiscard]] std::expected<std::string, std::error_code> getProperty(const minifi::core::PropertyReference& property_reference,
       const FlowFile* flow_file) const override;
-  [[nodiscard]] std::expected<MinifiControllerService*, std::error_code> getControllerService(std::string_view name,
-      std::string_view type) const override;
+  [[nodiscard]] std::expected<MinifiControllerService*, std::error_code> getControllerService(const minifi::core::PropertyReference& prop) const override;
   [[nodiscard]] std::map<std::string, std::string> getDynamicProperties(const FlowFile* flow_file) const override;
   [[nodiscard]] bool hasNonEmptyProperty(std::string_view name) const override;
 
-  [[nodiscard]] std::expected<utils::net::SslData, std::error_code> getSslData(std::string_view name) const override;
+  [[nodiscard]] std::expected<std::optional<utils::net::SslData>, std::error_code> getSslData(const minifi::core::PropertyReference& prop) const override;
+  [[nodiscard]] std::expected<std::optional<utils::ProxyData>, std::error_code> getProxyData(const minifi::core::PropertyReference& prop) const override;
 
  private:
   [[nodiscard]] std::expected<std::string, std::error_code> getProperty(std::string_view name, const FlowFile* flow_file) const;

extension-framework/cpp-extension-lib/include/api/utils/ProcessorConfigUtils.h

@@ -169,12 +169,8 @@ std::optional<T> parseOptionalEnumProperty(const core::ProcessContext& context,
 
 template<typename ControllerServiceType>
 ControllerServiceType* parseOptionalControllerService(const core::ProcessContext& context, const minifi::core::PropertyReference& prop) {
-  const auto controller_service_name = context.getProperty(prop, nullptr);
-  if (!controller_service_name || controller_service_name->empty()) {
-    return nullptr;
-  }
+  auto service = context.getControllerService(prop);
 
-  auto service = context.getControllerService(*controller_service_name, minifi::core::className<ControllerServiceType>());
   if (!service) {
     return nullptr;
   }

extension-framework/cpp-extension-lib/include/api/utils/Proxy.h

@@ -0,0 +1,42 @@
+/**
+* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+#pragma once
+
+#include <filesystem>
+#include <optional>
+#include <string>
+
+namespace org::apache::nifi::minifi::api::utils {
+
+enum class ProxyType {
+  DIRECT,
+  HTTP
+};
+
+struct BasicAuthCredentials {
+  std::string username;
+  std::string password;
+};
+
+struct ProxyData {
+  std::string host;
+  uint16_t port;
+  std::optional<BasicAuthCredentials> proxy_credentials;
+  ProxyType proxy_type;
+};
+
+}  // namespace org::apache::nifi::minifi::api::utils

extension-framework/cpp-extension-lib/include/api/utils/Ssl.h

@@ -17,12 +17,8 @@
 #pragma once
 
 #include <string>
-#include <memory>
-#include <optional>
 #include <filesystem>
 
-#include "utils/Enum.h"
-
 namespace org::apache::nifi::minifi::api::utils::net {
 
 enum class ClientAuthOption {

extension-framework/cpp-extension-lib/mocklib/include/MockProcessContext.h

@@ -29,12 +29,12 @@ class MockProcessContext : public api::core::ProcessContext {
 
   [[nodiscard]] std::expected<std::string, std::error_code> getProperty(const core::PropertyReference& property_reference,
       const api::core::FlowFile* flow_file) const override;
-  [[nodiscard]] std::expected<MinifiControllerService*, std::error_code> getControllerService(std::string_view controller_service_name,
-      std::string_view controller_service_class) const override;
+  [[nodiscard]] std::expected<MinifiControllerService*, std::error_code> getControllerService(const minifi::core::PropertyReference& prop) const override;
   [[nodiscard]] std::map<std::string, std::string> getDynamicProperties(const api::core::FlowFile* flow_file) const override;
   [[nodiscard]] bool hasNonEmptyProperty(std::string_view name) const override;
 
-  [[nodiscard]] std::expected<api::utils::net::SslData, std::error_code> getSslData(std::string_view name) const override;
+  [[nodiscard]] std::expected<std::optional<api::utils::net::SslData>, std::error_code> getSslData(const minifi::core::PropertyReference& prop) const override;
+  [[nodiscard]] std::expected<std::optional<api::utils::ProxyData>, std::error_code> getProxyData(const minifi::core::PropertyReference& prop) const override;
 
   std::map<std::string, std::string, std::less<>> properties_;
 

extension-framework/cpp-extension-lib/mocklib/src/MockProcessContext.cpp

@@ -32,7 +32,7 @@ std::expected<std::string, std::error_code> MockProcessContext::getProperty(cons
   return std::unexpected{make_error_code(core::PropertyErrorCode::PropertyNotSet)};
 }
 
-std::expected<MinifiControllerService*, std::error_code> MockProcessContext::getControllerService(std::string_view, std::string_view) const {
+std::expected<MinifiControllerService*, std::error_code> MockProcessContext::getControllerService(const minifi::core::PropertyReference&) const {
   return nullptr;
 }
 
@@ -44,7 +44,11 @@ bool MockProcessContext::hasNonEmptyProperty(const std::string_view name) const
   return properties_.contains(name);
 }
 
-std::expected<api::utils::net::SslData, std::error_code> MockProcessContext::getSslData(std::string_view) const {
-  return api::utils::net::SslData{};
+std::expected<std::optional<api::utils::net::SslData>, std::error_code> MockProcessContext::getSslData(const minifi::core::PropertyReference&) const {
+  return std::nullopt;
+}
+
+[[nodiscard]] std::expected<std::optional<api::utils::ProxyData>, std::error_code> MockProcessContext::getProxyData(const minifi::core::PropertyReference&) const {
+  return std::nullopt;
 }
 }  // namespace org::apache::nifi::minifi::mock

extension-framework/cpp-extension-lib/mocklib/src/mock-minifi-c.cpp

@@ -43,7 +43,7 @@ MinifiBool MinifiProcessContextHasNonEmptyProperty(MinifiProcessContext*, Minifi
   throw std::runtime_error("Not implemented");
 }
 
-MinifiStatus MinifiProcessContextGetControllerService(MinifiProcessContext*, MinifiStringView, MinifiStringView, MinifiControllerService**) {
+MinifiStatus MinifiProcessContextGetControllerServiceFromProperty(MinifiProcessContext*, MinifiStringView, MinifiStringView, MinifiControllerService**) {
   throw std::runtime_error("Not implemented");
 }
 void MinifiProcessContextGetDynamicProperties(MinifiProcessContext*, MinifiFlowFile*,

extension-framework/cpp-extension-lib/src/core/ProcessContext.cpp

@@ -44,14 +44,17 @@ bool CffiProcessContext::hasNonEmptyProperty(std::string_view name) const {
   return MinifiProcessContextHasNonEmptyProperty(impl_, utils::minifiStringView(name));
 }
 
-std::expected<MinifiControllerService*, std::error_code> CffiProcessContext::getControllerService(const std::string_view name,
-    const std::string_view type) const {
+std::expected<MinifiControllerService*, std::error_code> CffiProcessContext::getControllerService(const minifi::core::PropertyReference& prop) const {
   MinifiControllerService* controller_service = nullptr;
-  if (const MinifiStatus status = MinifiProcessContextGetControllerService(impl_,
-          utils::minifiStringView(name),
-          utils::minifiStringView(type),
+  gsl_Assert(prop.allowed_types.size() == 1);
+  const MinifiStatus status = MinifiProcessContextGetControllerServiceFromProperty(impl_,
+          utils::minifiStringView(prop.name),
+          utils::minifiStringView(prop.allowed_types[0]),
           &controller_service);
-      status != MINIFI_STATUS_SUCCESS) {
+  if (status == MINIFI_STATUS_PROPERTY_NOT_SET) {
+    return nullptr;
+  }
+  if (status != MINIFI_STATUS_SUCCESS) {
     return std::unexpected{utils::make_error_code(status)};
   }
   return controller_service;
@@ -69,10 +72,13 @@ std::map<std::string, std::string> CffiProcessContext::getDynamicProperties(cons
   return result;
 }
 
-std::expected<utils::net::SslData, std::error_code> CffiProcessContext::getSslData(const std::string_view name) const {
+std::expected<std::optional<utils::net::SslData>, std::error_code> CffiProcessContext::getSslData(const minifi::core::PropertyReference& prop) const {
+  const auto controller_name = getProperty(prop, nullptr);
+  if (!controller_name) { return std::nullopt; }
+
   auto ssl_data = utils::net::SslData{};
 
-  if (const auto status = MinifiProcessContextGetSslData(impl_, utils::minifiStringView(name), [](void* data, const MinifiSslData* minifi_ssl_data) {
+  if (const auto status = MinifiProcessContextGetSslData(impl_, utils::minifiStringView(*controller_name), [](void* data, const MinifiSslData* minifi_ssl_data) {
       auto* my_ssl_data = static_cast<utils::net::SslData*>(data);
       my_ssl_data->ca_loc = utils::toString(minifi_ssl_data->ca_certificate_file);
       my_ssl_data->cert_loc = utils::toString(minifi_ssl_data->certificate_file);
@@ -86,4 +92,31 @@ std::expected<utils::net::SslData, std::error_code> CffiProcessContext::getSslDa
   return ssl_data;
 }
 
+std::expected<std::optional<utils::ProxyData>, std::error_code> CffiProcessContext::getProxyData(const minifi::core::PropertyReference& prop) const {
+  auto proxy_data = utils::ProxyData{};
+  const auto status = MinifiProcessContextGetProxyDataFromProperty(
+      impl_,
+      utils::minifiStringView(prop.name),
+      [](void* data, const MinifiProxyData* minifi_proxy_data) {
+        auto* proxy = static_cast<utils::ProxyData*>(data);
+        proxy->host = utils::toString(minifi_proxy_data->hostname);
+        proxy->port = minifi_proxy_data->port;
+        if (minifi_proxy_data->password && minifi_proxy_data->username) {
+          proxy->proxy_credentials = utils::BasicAuthCredentials{.username = utils::toString(*minifi_proxy_data->username),
+              .password = utils::toString(*minifi_proxy_data->password)};
+        } else {
+          proxy->proxy_credentials = std::nullopt;
+        }
+        if (minifi_proxy_data->proxy_type == MINIFI_PROXY_TYPE_HTTP) {
+          proxy->proxy_type = utils::ProxyType::HTTP;
+        } else {
+          proxy->proxy_type = utils::ProxyType::DIRECT;
+        }
+      },
+      &proxy_data);
+  if (status == MINIFI_STATUS_PROPERTY_NOT_SET) { return std::nullopt; }
+  if (status == MINIFI_STATUS_SUCCESS) { return proxy_data; }
+  return std::unexpected{utils::make_error_code(status)};
+}
+
 }  // namespace org::apache::nifi::minifi::api::core

extensions/gcp/CMakeLists.txt

@@ -30,8 +30,8 @@ add_minifi_library(minifi-gcp SHARED ${SOURCES})
 if (NOT WIN32)
     target_compile_options(minifi-gcp PRIVATE -Wno-error=deprecated-declarations)  # Suppress deprecation warnings for std::rel_ops usage
 endif()
-target_link_libraries(minifi-gcp ${LIBMINIFI} google-cloud-cpp::storage)
-target_include_directories(minifi-gcp SYSTEM PUBLIC ${google-cloud-cpp_INCLUDE_DIRS})
+target_link_libraries(minifi-gcp minifi-cpp-extension-lib google-cloud-cpp::storage)
 
-register_extension(minifi-gcp "GCP EXTENSIONS" GCP-EXTENSIONS "This enables Google Cloud Platform support" "extensions/gcp/tests")
+target_include_directories(minifi-gcp SYSTEM PUBLIC ${google-cloud-cpp_INCLUDE_DIRS})
 
+register_c_api_extension(minifi-gcp "GCP EXTENSIONS" GCP-EXTENSIONS "This enables Google Cloud Platform support" "extensions/gcp/tests")

extensions/gcp/ExtensionInitializer.cpp

@@ -0,0 +1,46 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "../../extension-framework/cpp-extension-lib/include/api/core/Resource.h"
+#include "api/core/Resource.h"
+#include "api/utils/minifi-c-utils.h"
+#include "processors/DeleteGCSObject.h"
+#include "processors/FetchGCSObject.h"
+#include "processors/ListGCSBucket.h"
+#include "processors/PutGCSObject.h"
+
+#define MKSOC(x) #x
+#define MAKESTRING(x) MKSOC(x)  // NOLINT(cppcoreguidelines-macro-usage)
+
+namespace minifi = org::apache::nifi::minifi;
+
+CEXTENSIONAPI const uint32_t MinifiApiVersion = MINIFI_API_VERSION;
+
+CEXTENSIONAPI void MinifiInitExtension(MinifiExtensionContext* extension_context) {
+  const MinifiExtensionDefinition extension_definition{
+    .name = minifi::api::utils::minifiStringView(MAKESTRING(EXTENSION_NAME)),
+    .version = minifi::api::utils::minifiStringView(MAKESTRING(EXTENSION_VERSION)),
+    .deinit = nullptr,
+    .user_data = nullptr
+  };
+  auto* extension = MinifiRegisterExtension(extension_context, &extension_definition);
+  minifi::api::core::registerProcessors<minifi::extensions::gcp::DeleteGCSObject,
+      minifi::extensions::gcp::FetchGCSObject,
+      minifi::extensions::gcp::ListGCSBucket,
+      minifi::extensions::gcp::PutGCSObject>(extension);
+  minifi::api::core::registerControllerServices<minifi::extensions::gcp::GCPCredentialsControllerService>(extension);
+}