From 35e78e3f50b84cb50b95106b630f793f6dac770f Mon Sep 17 00:00:00 2001 From: Martin Zink Date: Wed, 13 May 2026 13:40:08 +0200 Subject: [PATCH 01/26] MINIFICPP-2740 Rust bindings based on stable C API --- .github/workflows/ci.yml | 62 +- .../workflows/create-release-artifacts.yml | 10 + CMakeLists.txt | 2 + cmake/MiNiFiOptions.cmake | 1 + docker/rockylinux/Dockerfile | 3 +- minifi_rust/.cargo/config.toml | 19 + minifi_rust/.dockerignore | 12 + minifi_rust/.gitignore | 7 + minifi_rust/CMakeLists.txt | 42 ++ minifi_rust/Cargo.toml | 9 + minifi_rust/LICENSE | 202 +++++++ minifi_rust/README.md | 75 +++ .../minifi_rs_playground/.cargo/config.toml | 5 + .../minifi_rs_playground/.gitignore | 7 + .../minifi_rs_playground/Cargo.toml | 20 + .../features/basic.feature | 84 +++ .../features/environment.py | 74 +++ .../features/error-handling.feature | 70 +++ .../features/lazy-logging.feature | 13 + .../features/metrics.feature | 23 + .../features/steps/steps.py | 45 ++ .../features/streaming.feature | 34 ++ .../minifi_rs_playground.md | 260 ++++++++ .../dummy_controller_service.rs | 22 + .../lorem_ipsum_controller_service.rs | 33 ++ .../properties.rs | 13 + .../src/controller_services/mod.rs | 2 + .../minifi_rs_playground/src/lib.rs | 37 ++ .../src/processors/asciify_german.rs | 67 +++ .../asciify_german/processor_definition.rs | 15 + .../asciify_german/relationships.rs | 11 + .../src/processors/asciify_german/tests.rs | 54 ++ .../src/processors/count_actual_logging.rs | 57 ++ .../src/processors/duplicate_text.rs | 54 ++ .../src/processors/generate_flow_file.rs | 168 ++++++ .../processor_definition.rs | 21 + .../generate_flow_file/properties.rs | 61 ++ .../generate_flow_file/relationships.rs | 6 + .../processors/generate_flow_file/tests.rs | 138 +++++ .../src/processors/get_file.rs | 303 ++++++++++ .../processors/get_file/output_attributes.rs | 13 + .../get_file/processor_definition.rs | 30 + .../src/processors/get_file/properties.rs | 121 ++++ .../src/processors/get_file/relationships.rs | 6 + .../src/processors/get_file/tests.rs | 172 ++++++ .../src/processors/kamikaze_processor.rs | 103 ++++ .../processor_definition.rs | 17 + .../kamikaze_processor/properties.rs | 39 ++ .../kamikaze_processor/relationships.rs | 6 + .../processors/kamikaze_processor/tests.rs | 83 +++ .../src/processors/log_attribute.rs | 160 +++++ .../log_attribute/processor_definition.rs | 24 + .../processors/log_attribute/properties.rs | 86 +++ .../processors/log_attribute/relationships.rs | 6 + .../src/processors/log_attribute/tests.rs | 114 ++++ .../src/processors/lorem_ipsum_cs_user.rs | 75 +++ .../processor_definition.rs | 16 + .../lorem_ipsum_cs_user/properties.rs | 28 + .../lorem_ipsum_cs_user/relationships.rs | 6 + .../processors/lorem_ipsum_cs_user/tests.rs | 19 + .../src/processors/mod.rs | 9 + .../src/processors/put_file.rs | 239 ++++++++ .../put_file/processor_definition.rs | 36 ++ .../src/processors/put_file/properties.rs | 51 ++ .../src/processors/put_file/relationships.rs | 11 + .../src/processors/put_file/tests.rs | 143 +++++ .../put_file/unix_only_properties.rs | 25 + .../generate_docs/generate_docs.dockerfile | 29 + minifi_rust/generate_docs/generate_docs.sh | 29 + minifi_rust/minifi_native/.gitignore | 2 + minifi_rust/minifi_native/Cargo.toml | 15 + minifi_rust/minifi_native/src/api.rs | 25 + .../minifi_native/src/api/attribute.rs | 11 + .../src/api/component_definition_traits.rs | 22 + .../src/api/controller_service.rs | 62 ++ minifi_rust/minifi_native/src/api/errors.rs | 128 ++++ .../minifi_native/src/api/flow_file.rs | 1 + minifi_rust/minifi_native/src/api/logger.rs | 95 +++ .../minifi_native/src/api/process_context.rs | 126 ++++ .../minifi_native/src/api/process_session.rs | 60 ++ .../minifi_native/src/api/processor.rs | 60 ++ .../src/api/processor_wrappers.rs | 5 + .../processor_wrappers/complex_processor.rs | 89 +++ .../processor_wrappers/flow_file_source.rs | 129 ++++ .../flow_file_stream_transform.rs | 174 ++++++ .../processor_wrappers/flow_file_transform.rs | 184 ++++++ .../src/api/processor_wrappers/utils.rs | 2 + .../utils/context_session_flowfile_bundle.rs | 70 +++ .../utils/flow_file_content.rs | 27 + minifi_rust/minifi_native/src/api/property.rs | 98 ++++ .../src/api/raw_controller_service.rs | 11 + .../minifi_native/src/api/raw_processor.rs | 70 +++ .../minifi_native/src/api/relationship.rs | 5 + minifi_rust/minifi_native/src/c_ffi.rs | 26 + .../c_ffi/c_ffi_controller_service_context.rs | 70 +++ .../c_ffi_controller_service_definition.rs | 162 +++++ .../c_ffi/c_ffi_controller_service_list.rs | 53 ++ .../src/c_ffi/c_ffi_flow_file.rs | 22 + .../minifi_native/src/c_ffi/c_ffi_logger.rs | 52 ++ .../src/c_ffi/c_ffi_output_attribute.rs | 41 ++ .../src/c_ffi/c_ffi_primitives.rs | 82 +++ .../src/c_ffi/c_ffi_process_context.rs | 146 +++++ .../src/c_ffi/c_ffi_process_session.rs | 554 ++++++++++++++++++ .../src/c_ffi/c_ffi_processor_definition.rs | 266 +++++++++ .../src/c_ffi/c_ffi_processor_list.rs | 49 ++ .../minifi_native/src/c_ffi/c_ffi_property.rs | 132 +++++ .../src/c_ffi/c_ffi_relationship.rs | 15 + .../minifi_native/src/c_ffi/c_ffi_streams.rs | 114 ++++ minifi_rust/minifi_native/src/lib.rs | 116 ++++ minifi_rust/minifi_native/src/mock.rs | 12 + .../mock/mock_controller_service_context.rs | 20 + .../minifi_native/src/mock/mock_flow_file.rs | 45 ++ .../minifi_native/src/mock/mock_logger.rs | 83 +++ .../src/mock/mock_process_context.rs | 121 ++++ .../src/mock/mock_process_session.rs | 181 ++++++ minifi_rust/minifi_native_macros/Cargo.toml | 11 + minifi_rust/minifi_native_macros/src/lib.rs | 20 + minifi_rust/minifi_native_sys/.gitignore | 2 + minifi_rust/minifi_native_sys/Cargo.toml | 18 + minifi_rust/minifi_native_sys/build.rs | 195 ++++++ minifi_rust/minifi_native_sys/src/lib.rs | 11 + minifi_rust/minifi_rs_behave/Cargo.toml | 12 + .../minifi_rs_behave/alpine.dockerfile | 27 + minifi_rust/minifi_rs_behave/build.rs | 6 + .../minifi_rs_behave/debian.dockerfile | 26 + minifi_rust/minifi_rs_behave/linux_build.sh | 94 +++ minifi_rust/minifi_rs_behave/src/main.rs | 155 +++++ minifi_rust/rustfmt.toml | 0 128 files changed, 8243 insertions(+), 4 deletions(-) create mode 100644 minifi_rust/.cargo/config.toml create mode 100644 minifi_rust/.dockerignore create mode 100644 minifi_rust/.gitignore create mode 100644 minifi_rust/CMakeLists.txt create mode 100644 minifi_rust/Cargo.toml create mode 100644 minifi_rust/LICENSE create mode 100644 minifi_rust/README.md create mode 100644 minifi_rust/extensions/minifi_rs_playground/.cargo/config.toml create mode 100644 minifi_rust/extensions/minifi_rs_playground/.gitignore create mode 100644 minifi_rust/extensions/minifi_rs_playground/Cargo.toml create mode 100644 minifi_rust/extensions/minifi_rs_playground/features/basic.feature create mode 100644 minifi_rust/extensions/minifi_rs_playground/features/environment.py create mode 100644 minifi_rust/extensions/minifi_rs_playground/features/error-handling.feature create mode 100644 minifi_rust/extensions/minifi_rs_playground/features/lazy-logging.feature create mode 100644 minifi_rust/extensions/minifi_rs_playground/features/metrics.feature create mode 100644 minifi_rust/extensions/minifi_rs_playground/features/steps/steps.py create mode 100644 minifi_rust/extensions/minifi_rs_playground/features/streaming.feature create mode 100644 minifi_rust/extensions/minifi_rs_playground/minifi_rs_playground.md create mode 100644 minifi_rust/extensions/minifi_rs_playground/src/controller_services/dummy_controller_service.rs create mode 100644 minifi_rust/extensions/minifi_rs_playground/src/controller_services/lorem_ipsum_controller_service.rs create mode 100644 minifi_rust/extensions/minifi_rs_playground/src/controller_services/lorem_ipsum_controller_service/properties.rs create mode 100644 minifi_rust/extensions/minifi_rs_playground/src/controller_services/mod.rs create mode 100644 minifi_rust/extensions/minifi_rs_playground/src/lib.rs create mode 100644 minifi_rust/extensions/minifi_rs_playground/src/processors/asciify_german.rs create mode 100644 minifi_rust/extensions/minifi_rs_playground/src/processors/asciify_german/processor_definition.rs create mode 100644 minifi_rust/extensions/minifi_rs_playground/src/processors/asciify_german/relationships.rs create mode 100644 minifi_rust/extensions/minifi_rs_playground/src/processors/asciify_german/tests.rs create mode 100644 minifi_rust/extensions/minifi_rs_playground/src/processors/count_actual_logging.rs create mode 100644 minifi_rust/extensions/minifi_rs_playground/src/processors/duplicate_text.rs create mode 100644 minifi_rust/extensions/minifi_rs_playground/src/processors/generate_flow_file.rs create mode 100644 minifi_rust/extensions/minifi_rs_playground/src/processors/generate_flow_file/processor_definition.rs create mode 100644 minifi_rust/extensions/minifi_rs_playground/src/processors/generate_flow_file/properties.rs create mode 100644 minifi_rust/extensions/minifi_rs_playground/src/processors/generate_flow_file/relationships.rs create mode 100644 minifi_rust/extensions/minifi_rs_playground/src/processors/generate_flow_file/tests.rs create mode 100644 minifi_rust/extensions/minifi_rs_playground/src/processors/get_file.rs create mode 100644 minifi_rust/extensions/minifi_rs_playground/src/processors/get_file/output_attributes.rs create mode 100644 minifi_rust/extensions/minifi_rs_playground/src/processors/get_file/processor_definition.rs create mode 100644 minifi_rust/extensions/minifi_rs_playground/src/processors/get_file/properties.rs create mode 100644 minifi_rust/extensions/minifi_rs_playground/src/processors/get_file/relationships.rs create mode 100644 minifi_rust/extensions/minifi_rs_playground/src/processors/get_file/tests.rs create mode 100644 minifi_rust/extensions/minifi_rs_playground/src/processors/kamikaze_processor.rs create mode 100644 minifi_rust/extensions/minifi_rs_playground/src/processors/kamikaze_processor/processor_definition.rs create mode 100644 minifi_rust/extensions/minifi_rs_playground/src/processors/kamikaze_processor/properties.rs create mode 100644 minifi_rust/extensions/minifi_rs_playground/src/processors/kamikaze_processor/relationships.rs create mode 100644 minifi_rust/extensions/minifi_rs_playground/src/processors/kamikaze_processor/tests.rs create mode 100644 minifi_rust/extensions/minifi_rs_playground/src/processors/log_attribute.rs create mode 100644 minifi_rust/extensions/minifi_rs_playground/src/processors/log_attribute/processor_definition.rs create mode 100644 minifi_rust/extensions/minifi_rs_playground/src/processors/log_attribute/properties.rs create mode 100644 minifi_rust/extensions/minifi_rs_playground/src/processors/log_attribute/relationships.rs create mode 100644 minifi_rust/extensions/minifi_rs_playground/src/processors/log_attribute/tests.rs create mode 100644 minifi_rust/extensions/minifi_rs_playground/src/processors/lorem_ipsum_cs_user.rs create mode 100644 minifi_rust/extensions/minifi_rs_playground/src/processors/lorem_ipsum_cs_user/processor_definition.rs create mode 100644 minifi_rust/extensions/minifi_rs_playground/src/processors/lorem_ipsum_cs_user/properties.rs create mode 100644 minifi_rust/extensions/minifi_rs_playground/src/processors/lorem_ipsum_cs_user/relationships.rs create mode 100644 minifi_rust/extensions/minifi_rs_playground/src/processors/lorem_ipsum_cs_user/tests.rs create mode 100644 minifi_rust/extensions/minifi_rs_playground/src/processors/mod.rs create mode 100644 minifi_rust/extensions/minifi_rs_playground/src/processors/put_file.rs create mode 100644 minifi_rust/extensions/minifi_rs_playground/src/processors/put_file/processor_definition.rs create mode 100644 minifi_rust/extensions/minifi_rs_playground/src/processors/put_file/properties.rs create mode 100644 minifi_rust/extensions/minifi_rs_playground/src/processors/put_file/relationships.rs create mode 100644 minifi_rust/extensions/minifi_rs_playground/src/processors/put_file/tests.rs create mode 100644 minifi_rust/extensions/minifi_rs_playground/src/processors/put_file/unix_only_properties.rs create mode 100644 minifi_rust/generate_docs/generate_docs.dockerfile create mode 100755 minifi_rust/generate_docs/generate_docs.sh create mode 100644 minifi_rust/minifi_native/.gitignore create mode 100644 minifi_rust/minifi_native/Cargo.toml create mode 100644 minifi_rust/minifi_native/src/api.rs create mode 100644 minifi_rust/minifi_native/src/api/attribute.rs create mode 100644 minifi_rust/minifi_native/src/api/component_definition_traits.rs create mode 100644 minifi_rust/minifi_native/src/api/controller_service.rs create mode 100644 minifi_rust/minifi_native/src/api/errors.rs create mode 100644 minifi_rust/minifi_native/src/api/flow_file.rs create mode 100644 minifi_rust/minifi_native/src/api/logger.rs create mode 100644 minifi_rust/minifi_native/src/api/process_context.rs create mode 100644 minifi_rust/minifi_native/src/api/process_session.rs create mode 100644 minifi_rust/minifi_native/src/api/processor.rs create mode 100644 minifi_rust/minifi_native/src/api/processor_wrappers.rs create mode 100644 minifi_rust/minifi_native/src/api/processor_wrappers/complex_processor.rs create mode 100644 minifi_rust/minifi_native/src/api/processor_wrappers/flow_file_source.rs create mode 100644 minifi_rust/minifi_native/src/api/processor_wrappers/flow_file_stream_transform.rs create mode 100644 minifi_rust/minifi_native/src/api/processor_wrappers/flow_file_transform.rs create mode 100644 minifi_rust/minifi_native/src/api/processor_wrappers/utils.rs create mode 100644 minifi_rust/minifi_native/src/api/processor_wrappers/utils/context_session_flowfile_bundle.rs create mode 100644 minifi_rust/minifi_native/src/api/processor_wrappers/utils/flow_file_content.rs create mode 100644 minifi_rust/minifi_native/src/api/property.rs create mode 100644 minifi_rust/minifi_native/src/api/raw_controller_service.rs create mode 100644 minifi_rust/minifi_native/src/api/raw_processor.rs create mode 100644 minifi_rust/minifi_native/src/api/relationship.rs create mode 100644 minifi_rust/minifi_native/src/c_ffi.rs create mode 100644 minifi_rust/minifi_native/src/c_ffi/c_ffi_controller_service_context.rs create mode 100644 minifi_rust/minifi_native/src/c_ffi/c_ffi_controller_service_definition.rs create mode 100644 minifi_rust/minifi_native/src/c_ffi/c_ffi_controller_service_list.rs create mode 100644 minifi_rust/minifi_native/src/c_ffi/c_ffi_flow_file.rs create mode 100644 minifi_rust/minifi_native/src/c_ffi/c_ffi_logger.rs create mode 100644 minifi_rust/minifi_native/src/c_ffi/c_ffi_output_attribute.rs create mode 100644 minifi_rust/minifi_native/src/c_ffi/c_ffi_primitives.rs create mode 100644 minifi_rust/minifi_native/src/c_ffi/c_ffi_process_context.rs create mode 100644 minifi_rust/minifi_native/src/c_ffi/c_ffi_process_session.rs create mode 100644 minifi_rust/minifi_native/src/c_ffi/c_ffi_processor_definition.rs create mode 100644 minifi_rust/minifi_native/src/c_ffi/c_ffi_processor_list.rs create mode 100644 minifi_rust/minifi_native/src/c_ffi/c_ffi_property.rs create mode 100644 minifi_rust/minifi_native/src/c_ffi/c_ffi_relationship.rs create mode 100644 minifi_rust/minifi_native/src/c_ffi/c_ffi_streams.rs create mode 100644 minifi_rust/minifi_native/src/lib.rs create mode 100644 minifi_rust/minifi_native/src/mock.rs create mode 100644 minifi_rust/minifi_native/src/mock/mock_controller_service_context.rs create mode 100644 minifi_rust/minifi_native/src/mock/mock_flow_file.rs create mode 100644 minifi_rust/minifi_native/src/mock/mock_logger.rs create mode 100644 minifi_rust/minifi_native/src/mock/mock_process_context.rs create mode 100644 minifi_rust/minifi_native/src/mock/mock_process_session.rs create mode 100644 minifi_rust/minifi_native_macros/Cargo.toml create mode 100644 minifi_rust/minifi_native_macros/src/lib.rs create mode 100644 minifi_rust/minifi_native_sys/.gitignore create mode 100644 minifi_rust/minifi_native_sys/Cargo.toml create mode 100644 minifi_rust/minifi_native_sys/build.rs create mode 100644 minifi_rust/minifi_native_sys/src/lib.rs create mode 100644 minifi_rust/minifi_rs_behave/Cargo.toml create mode 100644 minifi_rust/minifi_rs_behave/alpine.dockerfile create mode 100644 minifi_rust/minifi_rs_behave/build.rs create mode 100644 minifi_rust/minifi_rs_behave/debian.dockerfile create mode 100755 minifi_rust/minifi_rs_behave/linux_build.sh create mode 100644 minifi_rust/minifi_rs_behave/src/main.rs create mode 100644 minifi_rust/rustfmt.toml diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 221aa9fe28..e6e2cf729a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -11,7 +11,7 @@ concurrency: env: DOCKER_CMAKE_FLAGS: -DDOCKER_VERIFY_THREAD=3 -DUSE_SHARED_LIBS= -DSTRICT_GSL_CHECKS=AUDIT -DCI_BUILD=ON -DENABLE_AWS=ON -DENABLE_KAFKA=ON -DENABLE_MQTT=ON -DENABLE_AZURE=ON -DENABLE_SQL=ON \ -DENABLE_SPLUNK=ON -DENABLE_GCP=ON -DENABLE_OPC=ON -DENABLE_PYTHON_SCRIPTING=ON -DENABLE_LUA_SCRIPTING=ON -DENABLE_KUBERNETES=ON -DENABLE_TEST_PROCESSORS=ON -DENABLE_PROMETHEUS=ON \ - -DENABLE_ELASTICSEARCH=ON -DENABLE_GRAFANA_LOKI=ON -DENABLE_COUCHBASE=ON -DENABLE_LLAMACPP=ON -DDOCKER_BUILD_ONLY=ON -DMINIFI_PERFORMANCE_TESTS=ON + -DENABLE_ELASTICSEARCH=ON -DENABLE_GRAFANA_LOKI=ON -DENABLE_COUCHBASE=ON -DENABLE_LLAMACPP=ON -DDOCKER_BUILD_ONLY=ON -DMINIFI_PERFORMANCE_TESTS=ON -DMINIFI_RUST=OFF CCACHE_DIR: ${{ GITHUB.WORKSPACE }}/.ccache jobs: macos_xcode: @@ -52,6 +52,7 @@ jobs: -DENABLE_SFTP=ON -DENABLE_SPLUNK=ON -DENABLE_SQL=ON + -DMINIFI_RUST=ON -DENABLE_TEST_PROCESSORS=ON -DFORCE_COLORED_OUTPUT=OFF -DLIBC_STATIC=OFF @@ -161,6 +162,7 @@ jobs: -DENABLE_SQL=ON -DENABLE_TEST_PROCESSORS=ON -DENABLE_WEL=ON + -DMINIFI_RUST=ON -DFORCE_COLORED_OUTPUT=ON -DINSTALLER_MERGE_MODULES=OFF -DMINIFI_FAIL_ON_WARNINGS=ON @@ -261,6 +263,7 @@ jobs: -DENABLE_SPLUNK=ON -DENABLE_SQL=ON -DENABLE_SYSTEMD=ON + -DMINIFI_RUST=ON -DENABLE_TEST_PROCESSORS=ON -DFORCE_COLORED_OUTPUT=ON -DMINIFI_FAIL_ON_WARNINGS=ON @@ -552,8 +555,55 @@ jobs: with: name: behavex_output_modular path: build/behavex_output_modular + rusty_docker_tests: + name: "Rusty Docker integration tests (x86_64)" + needs: docker_build + runs-on: ubuntu-24.04 + timeout-minutes: 20 + steps: + - id: checkout + uses: actions/checkout@v6 + - id: run_cmake + name: Run CMake + run: | + mkdir build + cd build + cmake ${DOCKER_CMAKE_FLAGS} .. + - name: Download artifact + uses: actions/download-artifact@v8 + with: + name: minifi_docker + path: build + - name: Load Docker image + run: | + docker load --input ./build/minifi_docker.tar && docker tag apacheminificpp:1.0.0 apacheminificpp:behave + - id: install_deps + name: Install dependencies for Docker Verify + run: | + sudo apt update + sudo apt install -y python3-virtualenv + - id: test + name: Docker Verify + working-directory: ./minifi_rust + run: cargo behave-alpine + - name: Test Reporter + if: always() + uses: dorny/test-reporter@a43b3a5f7366b97d083190328d2c652e1a8b6aa2 # v3 + with: + name: Docker integration tests + path: minifi_rust/minifi_rs_behave/output/behave/*.xml + reporter: java-junit + only-summary: 'true' + list-tests: 'failed' + list-suites: 'failed' + - name: Upload artifact + if: failure() + uses: actions/upload-artifact@v7 + with: + name: minifi_rs_behave + path: minifi_rust/minifi_rs_behave/output linters: - name: "C++ lint + Shellcheck + Flake8" + name: "C++ lint + Shellcheck + Flake8 + Cargo check" runs-on: ubuntu-22.04-arm timeout-minutes: 15 steps: @@ -578,8 +628,14 @@ jobs: continue-on-error: true run: ./run_flake8.sh . + - id: cargo_check + name: Cargo check + continue-on-error: true + run: cargo fmt --check + working-directory: minifi_rust + - name: Check Linter Statuses - if: steps.cpp_lint.outcome == 'failure' || steps.shellcheck.outcome == 'failure' || steps.flake8_check.outcome == 'failure' + if: steps.cpp_lint.outcome == 'failure' || steps.shellcheck.outcome == 'failure' || steps.flake8_check.outcome == 'failure' || steps.cargo_check.outcome == 'failure' run: | echo "One or more linters failed. Failing the workflow." exit 1 diff --git a/.github/workflows/create-release-artifacts.yml b/.github/workflows/create-release-artifacts.yml index 53c91e34d5..cbf6ac7f3a 100644 --- a/.github/workflows/create-release-artifacts.yml +++ b/.github/workflows/create-release-artifacts.yml @@ -43,6 +43,16 @@ jobs: with: name: ${{ matrix.platform.rpm-artifact }} path: build/nifi-minifi-cpp-*.rpm + + - id: cargo + working-directory: minifi_rust + run: | + cargo build --release + + - uses: actions/upload-artifact@v7 + with: + name: minifi_rust_extensions + path: minifi_rust/target/release/*.so windows_VS2022: name: "Windows Server 2025 x86_64" runs-on: windows-2025 diff --git a/CMakeLists.txt b/CMakeLists.txt index 5af9c071c7..47993663f1 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -849,3 +849,5 @@ if (MINIFI_ADVANCED_CODE_COVERAGE) endif() add_subdirectory(packaging) + +add_subdirectory(minifi_rust) diff --git a/cmake/MiNiFiOptions.cmake b/cmake/MiNiFiOptions.cmake index 5c9820d419..59993fe6d4 100644 --- a/cmake/MiNiFiOptions.cmake +++ b/cmake/MiNiFiOptions.cmake @@ -116,6 +116,7 @@ add_minifi_option(ENABLE_EXECUTE_PROCESS "Enable ExecuteProcess processor" OFF) add_minifi_option(ENABLE_CONTROLLER "Enables the build of MiNiFi controller binary." ON) add_minifi_option(ENABLE_LLAMACPP "Enables llama.cpp support." ON) add_minifi_option(ENABLE_OPC "Instructs the build system to enable the OPC extension" ON) +add_minifi_option(MINIFI_RUST "Enables the build of rust based extensions." OFF) set_minifi_cache_variable(CUSTOM_MALLOC OFF "Overwrite malloc implementation.") set_property(CACHE CUSTOM_MALLOC PROPERTY STRINGS "jemalloc" "mimalloc" "rpmalloc" OFF) diff --git a/docker/rockylinux/Dockerfile b/docker/rockylinux/Dockerfile index 7af9a3de23..019bfe4653 100644 --- a/docker/rockylinux/Dockerfile +++ b/docker/rockylinux/Dockerfile @@ -42,11 +42,12 @@ COPY . ${MINIFI_BASE_DIR} # Install the system dependencies needed for a build # ccache is in EPEL RUN dnf -y install epel-release && dnf -y install gcc-toolset-14 gcc-toolset-14-libatomic-devel sudo git which make libarchive ccache ca-certificates perl patch bison flex libtool cmake rpmdevtools && \ + if echo "$MINIFI_OPTIONS" | grep -q "MINIFI_RUST=ON"; then dnf -y install rust cargo clang; fi && \ if echo "$MINIFI_OPTIONS" | grep -q "ENABLE_ALL=ON\|ENABLE_PYTHON_SCRIPTING=ON\|ENABLE_OPC=ON"; then dnf -y --enablerepo=devel install python3-devel; fi && \ if echo "$MINIFI_OPTIONS" | grep -q "ENABLE_SFTP=ON" && [ "${DOCKER_SKIP_TESTS}" == "OFF" ]; then dnf -y install java-1.8.0-openjdk maven; fi RUN cd $MINIFI_BASE_DIR && \ - ln -s /usr/bin/ccache /usr/lib64/ccache/c++ + ln -sfn /usr/bin/ccache /usr/lib64/ccache/c++ # Setup minificpp user RUN groupadd -g ${GID} ${USER} && useradd -g ${GID} ${USER} && \ diff --git a/minifi_rust/.cargo/config.toml b/minifi_rust/.cargo/config.toml new file mode 100644 index 0000000000..b871040ed5 --- /dev/null +++ b/minifi_rust/.cargo/config.toml @@ -0,0 +1,19 @@ +[target.aarch64-unknown-linux-musl] +rustflags = ["-C", "target-feature=-crt-static"] + +[target.x86_64-unknown-linux-musl] +rustflags = ["-C", "target-feature=-crt-static"] + +[target.aarch64-apple-darwin] +rustflags = ["-C", "link-arg=-undefined", "-C", "link-arg=dynamic_lookup"] + +[target.x86_64-apple-darwin] +rustflags = ["-C", "link-arg=-undefined", "-C", "link-arg=dynamic_lookup"] + +[alias] +behave = ["run", "--release", "--bin", "minifi_rs_behave"] +behave-alpine = ["run", "--release", "--bin", "minifi_rs_behave", "--", "--alpine"] +behave-debian = ["run", "--release", "--bin", "minifi_rs_behave", "--", "--debian"] + +[env] +MINIFI_SDK_PATH="../" diff --git a/minifi_rust/.dockerignore b/minifi_rust/.dockerignore new file mode 100644 index 0000000000..aeb3a1bf07 --- /dev/null +++ b/minifi_rust/.dockerignore @@ -0,0 +1,12 @@ +.git +.vscode +.gitmodules +.idea +output +features +target/debug +target/release/build +target/release/deps +target/release/.fingerprint +target/release/examples +target/release/incremental diff --git a/minifi_rust/.gitignore b/minifi_rust/.gitignore new file mode 100644 index 0000000000..eb4d6b29b9 --- /dev/null +++ b/minifi_rust/.gitignore @@ -0,0 +1,7 @@ +.idea +output +venv +.venv +target +.DS_Store +Cargo.lock diff --git a/minifi_rust/CMakeLists.txt b/minifi_rust/CMakeLists.txt new file mode 100644 index 0000000000..c73f684d89 --- /dev/null +++ b/minifi_rust/CMakeLists.txt @@ -0,0 +1,42 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +if (NOT MINIFI_RUST) + return() +endif() + +include(FetchContent) + +# Corrosion is a tool for integrating Rust into an existing CMake project +FetchContent_Declare(Corrosion + URL https://github.com/corrosion-rs/corrosion/archive/refs/tags/v0.6.1.tar.gz + URL_HASH SHA256=e9e95b1ee2bad52681f347993fb1a5af5cce458c5ce8a2636c9e476e4babf8e3 + SYSTEM) + +FetchContent_MakeAvailable(Corrosion) + +corrosion_import_crate(MANIFEST_PATH ${CMAKE_CURRENT_SOURCE_DIR}/Cargo.toml) + +include(CTest) + +add_test( + NAME cargo_tests + COMMAND cargo test + WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR} +) diff --git a/minifi_rust/Cargo.toml b/minifi_rust/Cargo.toml new file mode 100644 index 0000000000..6dc306f11f --- /dev/null +++ b/minifi_rust/Cargo.toml @@ -0,0 +1,9 @@ +[workspace] +resolver = "3" +members = ["minifi_native_sys", "minifi_native", "minifi_native_macros", "minifi_rs_behave", "extensions/*"] + +[profile.release] +panic = "abort" + +[profile.dev] +panic = "abort" diff --git a/minifi_rust/LICENSE b/minifi_rust/LICENSE new file mode 100644 index 0000000000..7968421e21 --- /dev/null +++ b/minifi_rust/LICENSE @@ -0,0 +1,202 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright 2026 The Apache Software Foundation + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/minifi_rust/README.md b/minifi_rust/README.md new file mode 100644 index 0000000000..27c5b96868 --- /dev/null +++ b/minifi_rust/README.md @@ -0,0 +1,75 @@ +# MiNiFi Native Rust + +This repository provides a safe, idiomatic, and high-performance Rust framework for building native extensions (processors) for [Apache NiFi MiNiFi C++](https://github.com/apache/nifi-minifi-cpp) + +It is designed to offer a robust developer experience, allowing you to write powerful and reliable data processing components in safe Rust. + +The framework completely encapsulates the unsafe C FFI (Foreign Function Interface) boundary, providing a pure Rust API that is fully mockable for unit testing. + +## Project Philosophy + - **Safety First**: Leverage Rust's compile-time guarantees to prevent common bugs like null pointers, buffer overflows, and data races. + - **Zero-Cost Abstraction**: The safe API wrapper is designed to compile down with zero runtime overhead compared to writing raw C code. + - **Ergonomics**: Provide a clean, idiomatic Rust API that is a pleasure to use. Developers should not need to think about unsafe code or C++ interoperability. + - **Testability**: Every component of a processor's logic should be unit-testable in a pure Rust environment, without needing a C++ host. + - **Cross Platform**: The library should work on all platforms that are supported by [Apache NiFi MiNiFi C++](https://github.com/apache/nifi-minifi-cpp) + - macOS (aarch64) + - Linux (x86_64, aarch64) + - Windows (x86_64) + + +The project is structured as a Cargo workspace with a clear, layered architecture: +### [minifi-native-sys](minifi_native_sys) +Contains the raw, unsafe FFI bindings to the minifi-c.h C API. +### [minifi-native](minifi_native) +Provides the public, safe, and idiomatic Rust API. This is the crate that developers will use to build their processors. +#### API Traits +Pure Rust traits (Processor, ProcessSession, Logger, etc.) that define the abstract behavior of the MiNiFi environment. +#### Higher level API +Pure rust traits that simplify the requirements for a working processor + - FlowFileTransform + - FlowFileSource +#### FFI Wrappers +Concrete structs (CffiSession, CffiLogger, etc.) that implement the API traits by calling the unsafe functions from minifi-native-sys. +#### Thread safety +The trait system differentiates between thread-safe (&self) and single-threaded (&mut self) processors at compile time. +#### Comprehensive Mocking: +A full suite of mock objects allows for fast and reliable unit testing of all processor logic. +### [minifi_native_macros](minifi_native_macros) +Helper crate that includes the procedural macros +### [minifi_rs_behave](minifi_rs_behave) +Run the behave integration tests using Minifi's docker framework. This will test the release artifacts against the latest released [MiNiFi native docker container](https://hub.docker.com/r/apache/nifi-minifi-cpp). +There is a handy alias to initiate all behave tests. + +`cargo behave` + +## Creating an Extension +Building an extension is straightforward. The framework provides a declare_minifi_extension! macro that automatically generates the C-compatible entry points and registers your components. + +```rust +declare_minifi_extension!( + processors: [ + (FlowFileSourceProcessorType, Concurrent, MyFlowFileSource), + (FlowFileTransformProcessorType, Exclusive, MyDataTransformer), + ], + controllers: [ + MyCustomControllerService, + ] +); +``` + + +## Deployment +Build your extension as a dynamic library (cd extensions/your_extension && cargo build --release). + +Locate the output artifact in target/release/ (it will be a .so on Linux, .dll on Windows, or .dylib on macOS). + +Copy the library file into the MiNiFi C++ application's extensions/ directory. + +Restart the MiNiFi Native agent to automatically discover and load the new processors. + +## Included Extensions +### [minifi_rs_playground](extensions/minifi_rs_playground) +A concrete example and testing ground for extensions built using the minifi-native crate. +- Demonstrates how to implement the Processor traits, define processor properties, and route to relationships. +- Includes comprehensive unit tests using the pure-Rust mocking framework. +- Includes integration testing that verifies the processor works as expected in a real, containerized MiNiFi environment. diff --git a/minifi_rust/extensions/minifi_rs_playground/.cargo/config.toml b/minifi_rust/extensions/minifi_rs_playground/.cargo/config.toml new file mode 100644 index 0000000000..cb8c02ddc4 --- /dev/null +++ b/minifi_rust/extensions/minifi_rs_playground/.cargo/config.toml @@ -0,0 +1,5 @@ +[target.aarch64-apple-darwin] +rustflags = ["-C", "link-arg=-undefined", "-C", "link-arg=dynamic_lookup"] + +[target.x86_64-apple-darwin] +rustflags = ["-C", "link-arg=-undefined", "-C", "link-arg=dynamic_lookup"] diff --git a/minifi_rust/extensions/minifi_rs_playground/.gitignore b/minifi_rust/extensions/minifi_rs_playground/.gitignore new file mode 100644 index 0000000000..32c9b263bd --- /dev/null +++ b/minifi_rust/extensions/minifi_rs_playground/.gitignore @@ -0,0 +1,7 @@ +target +output +features/.venv +features/output +integration-test/features/.venv +integration-test/features/linux_so +integration-test/.venv diff --git a/minifi_rust/extensions/minifi_rs_playground/Cargo.toml b/minifi_rust/extensions/minifi_rs_playground/Cargo.toml new file mode 100644 index 0000000000..0100847daa --- /dev/null +++ b/minifi_rust/extensions/minifi_rs_playground/Cargo.toml @@ -0,0 +1,20 @@ +[package] +name = "minifi_rs_playground" +version = "0.1.0" +edition = "2024" + +[lib] +crate-type = ["cdylib"] + +[dependencies] +minifi_native = { path = "../../minifi_native" } +strum = "0.28.0" +walkdir = "2.5.0" +rand = "0.10.0" +hex = "0.4.3" +strum_macros = "0.28.0" +lipsum = "0.9.1" + +[dev-dependencies] +tempfile = "3.22.0" +filetime = "0.2.26" diff --git a/minifi_rust/extensions/minifi_rs_playground/features/basic.feature b/minifi_rust/extensions/minifi_rs_playground/features/basic.feature new file mode 100644 index 0000000000..4c642f8530 --- /dev/null +++ b/minifi_rust/extensions/minifi_rs_playground/features/basic.feature @@ -0,0 +1,84 @@ +@SUPPORTS_WINDOWS +Feature: Basic scenarios + + Scenario: The rust library is loaded into minifi + Given log property "logger.org::apache::nifi::minifi::core::extension::ExtensionManager" is set to "TRACE,stderr" + And log property "logger.org::apache::nifi::minifi::core::ClassLoader" is set to "TRACE,stderr" + + When the MiNiFi instance starts up + + Then the Minifi logs contain the following message: "Registering class 'GenerateFlowFileRs' at '/minifi_rs_playground'" in less than 10 seconds + And the Minifi logs contain the following message: "Registering class 'GetFileRs' at '/minifi_rs_playground'" in less than 1 seconds + And the Minifi logs contain the following message: "Registering class 'KamikazeProcessorRs' at '/minifi_rs_playground'" in less than 1 seconds + And the Minifi logs contain the following message: "Registering class 'LogAttributeRs' at '/minifi_rs_playground'" in less than 1 seconds + And the Minifi logs contain the following message: "Registering class 'PutFileRs' at '/minifi_rs_playground'" in less than 1 seconds + And the Minifi logs do not contain errors + And the Minifi logs do not contain warnings + + Scenario: Simple GenerateFlowFileRs -> PutFileRs + Given a GenerateFlowFileRs processor with the "Custom Text" property set to "Ferris the crab" + And the "Data Format" property of the GenerateFlowFileRs processor is set to "Text" + And the "Unique FlowFiles" property of the GenerateFlowFileRs processor is set to "false" + And a PutFileRs processor with the "Directory" property set to "/tmp/output" + And the "success" relationship of the GenerateFlowFileRs processor is connected to the PutFileRs + And PutFileRs's success relationship is auto-terminated + + When the MiNiFi instance starts up + + Then at least one file with the content "Ferris the crab" is placed in the "/tmp/output" directory in less than 10 seconds + And the Minifi logs do not contain errors + And the Minifi logs do not contain warnings + + Scenario: Simple GetFileRs -> PutFileRs + Given a GetFileRs processor with the "Input Directory" property set to "/tmp/input" + And a PutFileRs processor with the "Directory" property set to "/tmp/output" + And the "success" relationship of the GetFileRs processor is connected to the PutFileRs + And PutFileRs's success relationship is auto-terminated + And PutFileRs's failure relationship is auto-terminated + And a directory at "/tmp/input" has a file "test_file.log" with the content "test content" + + When the MiNiFi instance starts up + + Then at least one file with the content "test content" is placed in the "/tmp/output" directory in less than 10 seconds + And the Minifi logs do not contain errors + And the Minifi logs do not contain warnings + + Scenario Outline: The LogAttributeRs can read and log FlowFile content + Given a GenerateFlowFileRs processor with the "Custom Text" property set to "" + And the "Data Format" property of the GenerateFlowFileRs processor is set to "Text" + And the "Unique FlowFiles" property of the GenerateFlowFileRs processor is set to "false" + And a LogAttributeRs processor with the "Log Level" property set to "" + And the "Log Payload" property of the LogAttributeRs processor is set to "true" + And the "success" relationship of the GenerateFlowFileRs processor is connected to the LogAttributeRs + And LogAttributeRs's success relationship is auto-terminated + And log property "logger.minifi_rs_playground::processors::log_attribute::LogAttributeRs" is set to "TRACE,stderr" + + When the MiNiFi instance starts up + + Then the Minifi logs contain the following message: "" in less than 20 seconds + And the Minifi logs contain the following message: "" in less than 1 seconds + And the Minifi logs do not contain errors + And the Minifi logs do not contain warnings + Examples: + | custom_text | log_level | expected_log_1 | expected_log_2 | + | Elephant | Critical | [critical] Logging for flow file | Elephant | + | Lynx | Info | [info] Logging for flow file | Lynx | + | Ant | Trace | [trace] Logging for flow file | Ant | + + Scenario Outline: Controller services work + Given a LoremIpsumCSUser processor with the "Lorem Ipsum Controller Service" property set to "LoremIpsumControllerService" + And the "Write Method" property of the LoremIpsumCSUser processor is set to "" + And a LoremIpsumControllerService controller service is set up and the "Length" property set to "200000" + And a PutFileRs processor with the "Directory" property set to "/tmp/output" + And the "success" relationship of the LoremIpsumCSUser processor is connected to the PutFileRs + And PutFileRs's success relationship is auto-terminated + + When the MiNiFi instance starts up + + Then at least one file with minimum size of "1 MB" is placed in the "/tmp/output" directory in less than 10 seconds + And the Minifi logs do not contain errors + And the Minifi logs do not contain warnings + Examples: + | write_method | + | Buffer | + | Stream | diff --git a/minifi_rust/extensions/minifi_rs_playground/features/environment.py b/minifi_rust/extensions/minifi_rs_playground/features/environment.py new file mode 100644 index 0000000000..c5d67ba23d --- /dev/null +++ b/minifi_rust/extensions/minifi_rs_playground/features/environment.py @@ -0,0 +1,74 @@ +import os +from typing import List + +from minifi_behave.containers.docker_image_builder import DockerImageBuilder +from minifi_behave.core.hooks import common_after_scenario +from minifi_behave.core.hooks import common_before_scenario, get_minifi_container_image +from minifi_behave.core.minifi_test_context import MinifiTestContext + + +def add_extension_to_minifi_container( + extension_name: str, possible_paths: List[str], context: MinifiTestContext +): + new_container_name = f"apacheminificpp:{extension_name}" + is_windows = os.name == "nt" + if is_windows: + lib_filename = f"{extension_name}.dll" + container_extension_dir = ( + "C:/Program Files/ApacheNiFiMiNiFi/nifi-minifi-cpp/extensions" + ) + else: + lib_filename = f"lib{extension_name}.so" + container_extension_dir = "/opt/minifi/minifi-current/extensions/" + + host_path = None + for path in possible_paths: + if os.path.exists(os.path.join(path, lib_filename)): + host_path = os.path.join(path, lib_filename) + break + + assert host_path is not None, ( + f"Could not find {lib_filename} in {[p for p in possible_paths]}" + ) + + with open(host_path, "rb") as f: + lib_content = f.read() + + base_img = get_minifi_container_image() + + if is_windows: + dockerfile = f""" +FROM {base_img} +COPY ["{lib_filename}", "{container_extension_dir}/{lib_filename}"] +""" + else: + dockerfile = f""" +FROM {base_img} +COPY --chown=minificpp:minificpp {lib_filename} {container_extension_dir} +RUN chmod 755 {container_extension_dir}{lib_filename} +""" + + builder = DockerImageBuilder( + image_tag=new_container_name, + dockerfile_content=dockerfile, + files_on_context={lib_filename: lib_content}, + ) + + builder.build() + return new_container_name + + +def before_all(context): + dir_path = os.path.dirname(os.path.realpath(__file__)) + build_path = os.path.normpath(os.path.join(dir_path, "../../../target/release/")) + deps_build_path = os.path.normpath(os.path.join(dir_path, "../../../target/release/deps/")) + add_extension_to_minifi_container("minifi_rs_playground", [build_path, deps_build_path], context) + + +def before_scenario(context, scenario): + context.minifi_container_image = "apacheminificpp:minifi_rs_playground" + common_before_scenario(context, scenario) + + +def after_scenario(context, scenario): + common_after_scenario(context, scenario) diff --git a/minifi_rust/extensions/minifi_rs_playground/features/error-handling.feature b/minifi_rust/extensions/minifi_rs_playground/features/error-handling.feature new file mode 100644 index 0000000000..23dbf43b44 --- /dev/null +++ b/minifi_rust/extensions/minifi_rs_playground/features/error-handling.feature @@ -0,0 +1,70 @@ +@SUPPORTS_WINDOWS +Feature: API error handling and logging + + Scenario: The Api handles empty flow-files + Given a GenerateFlowFileRs processor with the "Custom Text" property set to "${invalid_attribute}" + And the "Data Format" property of the GenerateFlowFileRs processor is set to "Text" + And the "Unique FlowFiles" property of the GenerateFlowFileRs processor is set to "false" + And a LogAttributeRs processor with the "Log Level" property set to "Critical" + And the "success" relationship of the GenerateFlowFileRs processor is connected to the LogAttributeRs + And LogAttributeRs's success relationship is auto-terminated + + When the MiNiFi instance starts up + + Then after 3 sec have passed + And the Minifi logs do not contain errors + And the Minifi logs do not contain warnings + + Scenario: Minifi handles errors from on_schedule + Given a KamikazeProcessorRs processor with the "On Schedule Behaviour" property set to "ReturnErr" + And KamikazeProcessorRs's success relationship is auto-terminated + + When the MiNiFi instance starts up + + Then the Minifi logs contain the following message: "KamikazeProcessorRs] [error] Error during on_schedule: ScheduleError("it was designed to fail during schedule")" in less than 10 seconds + And the Minifi logs contain the following message: "(KamikazeProcessorRs): Process Schedule Operation: Error while scheduling processor" in less than 10 seconds + + Scenario: Minifi handles errors from on_trigger + Given a KamikazeProcessorRs processor with the "On Schedule Behaviour" property set to "ReturnOk" + And the "On Trigger Behaviour" property of the KamikazeProcessorRs processor is set to "ReturnErr" + And KamikazeProcessorRs's success relationship is auto-terminated + + When the MiNiFi instance starts up + + Then the Minifi logs contain the following message: "KamikazeProcessorRs] [error] Error during on_trigger TriggerError("it was designed to fail in trigger")" in less than 10 seconds + And the Minifi logs contain the following message: "Trigger and commit failed for processor KamikazeProcessorRs" in less than 10 seconds + + Scenario: Panic in extension's on_schedule crashes the agent aswell + Given a KamikazeProcessorRs processor with the "On Schedule Behaviour" property set to "Panic" + And KamikazeProcessorRs's success relationship is auto-terminated + + When the MiNiFi instance is started without assertions + Then Minifi crashes with the following "KamikazeProcessor::on_schedule panic" in less than 10 seconds + + Scenario: Panic in extension's on_trigger crashes the agent aswell + Given a KamikazeProcessorRs processor with the "On Schedule Behaviour" property set to "ReturnOk" + And the "On Trigger Behaviour" property of the KamikazeProcessorRs processor is set to "Panic" + And KamikazeProcessorRs's success relationship is auto-terminated + + When the MiNiFi instance is started without assertions + Then Minifi crashes with the following "KamikazeProcessor::on_trigger panic" in less than 10 seconds + + Scenario: Get not supported property + Given a KamikazeProcessorRs processor with the "On Schedule Behaviour" property set to "ReturnOk" + And the "On Trigger Behaviour" property of the KamikazeProcessorRs processor is set to "GetNotRegisteredProperty" + And KamikazeProcessorRs's success relationship is auto-terminated + + When the MiNiFi instance starts up + + Then the Minifi logs contain the following message: "MinifiProcessContextGetProperty("Kamikaze Processor Property"), not supported property" in less than 10 seconds + And the Minifi logs contain the following message: "Trigger and commit failed for processor KamikazeProcessorRs" in less than 10 seconds + + Scenario: Get wrong typed Controller Service + Given a LoremIpsumCSUser processor with the "Lorem Ipsum Controller Service" property set to "My Controller Service" + And a DummyControllerService controller service is set up + And LoremIpsumCSUser's success relationship is auto-terminated + + When the MiNiFi instance starts up + + Then the Minifi logs contain the following message: "MinifiProcessContextGetControllerService::<"minifi_rs_playground::controller_services::lorem_ipsum_controller_service::LoremIpsumControllerService">("My Controller Service"), validation failed" in less than 10 seconds + And the Minifi logs contain the following message: "Trigger and commit failed for processor LoremIpsumCSUser" in less than 10 seconds diff --git a/minifi_rust/extensions/minifi_rs_playground/features/lazy-logging.feature b/minifi_rust/extensions/minifi_rs_playground/features/lazy-logging.feature new file mode 100644 index 0000000000..89e11197ba --- /dev/null +++ b/minifi_rust/extensions/minifi_rs_playground/features/lazy-logging.feature @@ -0,0 +1,13 @@ +@SUPPORTS_WINDOWS +Feature: Logs should be lazily evaluated + + Scenario: CountActualLogging only increments self when actually logging + Given a CountActualLogging processor + And log property "logger.minifi_rs_playground::processors::count_actual_logging::CountActualLogging" is set to "INFO,stderr" + And CountActualLogging is TIMER_DRIVEN with 1 min scheduling period + + When the MiNiFi instance starts up + + Then the Minifi logs contain the following message: "[minifi_rs_playground::processors::count_actual_logging::CountActualLogging] [info] info 1" in less than 10 seconds + And the Minifi logs do not contain errors + And the Minifi logs do not contain warnings diff --git a/minifi_rust/extensions/minifi_rs_playground/features/metrics.feature b/minifi_rust/extensions/minifi_rs_playground/features/metrics.feature new file mode 100644 index 0000000000..7ebf844e2a --- /dev/null +++ b/minifi_rust/extensions/minifi_rs_playground/features/metrics.feature @@ -0,0 +1,23 @@ +@SUPPORTS_WINDOWS +Feature: Testing custom and default metrics + + Scenario: CustomMetrics(GetFileRs), DefaultMetrics from streaming(DuplicateStreamText) API and DefaultMetrics from buffer(PutFileRs) API + Given a GetFileRs processor with the "Input Directory" property set to "/tmp/input" + And a DuplicateStreamText processor + And a PutFileRs processor with the "Directory" property set to "/tmp/output" + And the "success" relationship of the GetFileRs processor is connected to the DuplicateStreamText + And the "success" relationship of the DuplicateStreamText processor is connected to the PutFileRs + And PutFileRs's success relationship is auto-terminated + And PutFileRs's failure relationship is auto-terminated + And a directory at "/tmp/input" has a file "hello.txt" with the content "hello" + And MiNiFi logs processor metrics + + When the MiNiFi instance starts up + + Then at least one file with the content "hheelllloo" is placed in the "/tmp/output" directory in less than 10 seconds + And the Minifi logs match the following regex: "GetFileRsMetrics": {\n[ ]+\"[0-9a-z-]+\": \{\n[ a-zA-Z0-9":,\n]*"BytesRead": "0",[\n ]*"BytesWritten": "5"[ a-zA-Z0-9":,\n]*"InputBytes": "5"[ a-zA-Z0-9":,\n]*}" in less than 10 seconds + And the Minifi logs match the following regex: "DuplicateStreamTextMetrics": {\n[ ]+\"[0-9a-z-]+\": \{\n[ a-zA-Z0-9":,\n]*"BytesRead": "5",[\n ]*"BytesWritten": "10"[ a-zA-Z0-9":,\n]*}" in less than 10 seconds + And the Minifi logs match the following regex: "PutFileRsMetrics": {\n[ ]+\"[0-9a-z-]+\": \{\n[ a-zA-Z0-9":,\n]*"BytesRead": "10"[ a-zA-Z0-9":,\n]*}" in less than 10 seconds + + And the Minifi logs do not contain errors + And the Minifi logs do not contain warnings diff --git a/minifi_rust/extensions/minifi_rs_playground/features/steps/steps.py b/minifi_rust/extensions/minifi_rs_playground/features/steps/steps.py new file mode 100644 index 0000000000..181105116e --- /dev/null +++ b/minifi_rust/extensions/minifi_rs_playground/features/steps/steps.py @@ -0,0 +1,45 @@ +from behave import then, when, given +import humanfriendly + +from minifi_behave.steps import checking_steps # noqa: F401 +from minifi_behave.steps import configuration_steps # noqa: F401 +from minifi_behave.steps import core_steps # noqa: F401 +from minifi_behave.steps import flow_building_steps # noqa: F401 +from minifi_behave.core.helpers import wait_for_condition +from minifi_behave.core.minifi_test_context import MinifiTestContext + + +@when("the MiNiFi instance is started without assertions") +def minifi_starts_wo_assertions(context: MinifiTestContext): + context.get_or_create_default_minifi_container().deploy(context) + + +@then('Minifi crashes with the following "{crash_msg}" in less than {duration}') +def minifi_crashes(context: MinifiTestContext, crash_msg: str, duration: str): + duration_seconds = humanfriendly.parse_timespan(duration) + assert wait_for_condition( + condition=lambda: ( + context.get_or_create_default_minifi_container().exited + and crash_msg in context.get_or_create_default_minifi_container().get_logs() + ), + timeout_seconds=duration_seconds, + bail_condition=lambda: False, + context=context, + ) + + +@given("MiNiFi logs processor metrics") +def minifi_logs_processor_metrics(context: MinifiTestContext): + context.get_or_create_default_minifi_container().set_property( + "nifi.metrics.publisher.LogMetricsPublisher.metrics", + "GetFileRsMetrics,DuplicateStreamTextMetrics,PutFileRsMetrics", + ) + context.get_or_create_default_minifi_container().set_property( + "nifi.metrics.publisher.LogMetricsPublisher.logging.interval", "1s" + ) + context.get_or_create_default_minifi_container().set_property( + "nifi.metrics.publisher.class", "LogMetricsPublisher" + ) + context.get_or_create_default_minifi_container().set_property( + "nifi.metrics.publisher.agent.identifier", "Agent1" + ) diff --git a/minifi_rust/extensions/minifi_rs_playground/features/streaming.feature b/minifi_rust/extensions/minifi_rs_playground/features/streaming.feature new file mode 100644 index 0000000000..d6fea8933d --- /dev/null +++ b/minifi_rust/extensions/minifi_rs_playground/features/streaming.feature @@ -0,0 +1,34 @@ +@SUPPORTS_WINDOWS +Feature: Testing streaming reads and writes + + Scenario: Streaming Transforms work + Given a GetFileRs processor with the "Input Directory" property set to "/tmp/input" + And a AsciifyGerman processor + And a PutFileRs processor with the "Directory" property set to "/tmp/output" + And the "success" relationship of the GetFileRs processor is connected to the AsciifyGerman + And the "success" relationship of the AsciifyGerman processor is connected to the PutFileRs + And PutFileRs's success relationship is auto-terminated + And PutFileRs's failure relationship is auto-terminated + And a directory at "/tmp/input" has a file "german.txt" with the content "Üben von Xylophon und Querflöte ist ja zweckmäßig." + + When the MiNiFi instance starts up + + Then at least one file with the content "Ueben von Xylophon und Querfloete ist ja zweckmaessig." is placed in the "/tmp/output" directory in less than 10 seconds + And the Minifi logs do not contain errors + And the Minifi logs do not contain warnings + + Scenario: Streaming can be cancelled + Given a GetFileRs processor with the "Input Directory" property set to "/tmp/input" + And a AsciifyGerman processor + And a PutFileRs processor with the "Directory" property set to "/tmp/output" + And the "success" relationship of the GetFileRs processor is connected to the AsciifyGerman + And the "failure" relationship of the AsciifyGerman processor is connected to the PutFileRs + And PutFileRs's success relationship is auto-terminated + And PutFileRs's failure relationship is auto-terminated + And a directory at "/tmp/input" has a file "french.txt" with the content "Voix ambiguë d'un cœur qui, au zéphyr, préfère les jattes de kiwis." + + When the MiNiFi instance starts up + + Then at least one file with the content "Voix ambiguë d'un cœur qui, au zéphyr, préfère les jattes de kiwis." is placed in the "/tmp/output" directory in less than 10 seconds + And the Minifi logs do not contain errors + And the Minifi logs do not contain warnings diff --git a/minifi_rust/extensions/minifi_rs_playground/minifi_rs_playground.md b/minifi_rust/extensions/minifi_rs_playground/minifi_rs_playground.md new file mode 100644 index 0000000000..9b2397fcd5 --- /dev/null +++ b/minifi_rust/extensions/minifi_rs_playground/minifi_rs_playground.md @@ -0,0 +1,260 @@ + + +## Table of Contents + +### Processors + +- [AsciifyGerman](#AsciifyGerman) +- [CountActualLogging](#CountActualLogging) +- [GenerateFlowFileRs](#GenerateFlowFileRs) +- [GetFileRs](#GetFileRs) +- [KamikazeProcessorRs](#KamikazeProcessorRs) +- [LogAttributeRs](#LogAttributeRs) +- [LoremIpsumCSUser](#LoremIpsumCSUser) +- [PutFileRs](#PutFileRs) +### Controller Services + +- [DummyControllerService](#DummyControllerService) +- [LoremIpsumControllerService](#LoremIpsumControllerService) + + +## AsciifyGerman + +### Description + +This processor switches German characters with their ascii counterparts. (to test stream API) + +### Properties + +In the list below, the names of required properties appear in bold. Any other properties (not in bold) are considered optional. The table also indicates any default values, and whether a property supports the NiFi Expression Language. + +| Name | Default Value | Allowable Values | Description | +|------|---------------|------------------|-------------| + +### Relationships + +| Name | Description | +|---------|-----------------------------------------| +| success | All asciified flowfiles are routed here | +| failure | Non-german flowfiles are routed here | + + +## CountActualLogging + +### Description + +For testing lazy logging + +### Properties + +In the list below, the names of required properties appear in bold. Any other properties (not in bold) are considered optional. The table also indicates any default values, and whether a property supports the NiFi Expression Language. + +| Name | Default Value | Allowable Values | Description | +|------|---------------|------------------|-------------| + +### Relationships + +| Name | Description | +|------|-------------| + + +## GenerateFlowFileRs + +### Description + +This processor creates FlowFiles with random data or custom content. GenerateFlowFile is useful for load testing, configuration, and simulation. + +### Properties + +In the list below, the names of required properties appear in bold. Any other properties (not in bold) are considered optional. The table also indicates any default values, and whether a property supports the NiFi Expression Language. + +| Name | Default Value | Allowable Values | Description | +|----------------------|---------------|------------------|----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| **File Size** | 1 kB | | The size of the file that will be used
**Supports Expression Language: true** | +| **Batch Size** | 1 | | The number of FlowFiles to be transferred in each invocation | +| **Data Format** | Binary | Text
Binary | Specifies whether the data should be Text or Binary | +| **Unique FlowFiles** | true | true
false | If true, each FlowFile that is generated will be unique. If false, a random value will be generated and all FlowFiles will get the same content but this offers much higher throughput (but see the description of Custom Text for special non-random use cases) | +| Custom Text | | | If Data Format is text and if Unique FlowFiles is false, then this custom text will be used as content of the generated FlowFiles and the File Size will be ignored. Finally, if Expression Language is used, evaluation will be performed only once per batch of generated FlowFiles
**Supports Expression Language: true** | + +### Relationships + +| Name | Description | +|---------|----------------------------------------| +| success | success operational on the flow record | + + +## GetFileRs + +### Description + +Creates FlowFiles from files in a directory. MiNiFi will ignore files for which it doesn't have read permissions. + +### Properties + +In the list below, the names of required properties appear in bold. Any other properties (not in bold) are considered optional. The table also indicates any default values, and whether a property supports the NiFi Expression Language. + +| Name | Default Value | Allowable Values | Description | +|-------------------------|---------------|------------------|------------------------------------------------------------------------------------------------------------------------------------------------------------| +| **Input Directory** | | | The input directory from which to pull files
**Supports Expression Language: true** | +| Polling Interval | | | Indicates how long to wait before performing a directory listing | +| Recurse Subdirectories | true | true
false | Indicates whether or not to pull files from subdirectories | +| Keep Source File | false | true
false | If true, the file is not deleted after it has been copied to the Content Repository | +| Minimum File Age | | | The minimum age that a file must be in order to be pulled; any file younger than this amount of time (according to last modification date) will be ignored | +| Maximum File Age | | | The maximum age that a file must be in order to be pulled; any file older than this amount of time (according to last modification date) will be ignored | +| Minimum File Size | | | The minimum size that a file can be in order to be pulled | +| Maximum File Size | | | The maximum size that a file can be in order to be pulled | +| **Ignore Hidden Files** | true | true
false | Indicates whether or not hidden files should be ignored | +| **Batch Size** | 10 | | The maximum number of files to pull in each iteration | + +### Relationships + +| Name | Description | +|---------|----------------------------------------------| +| success | FlowFiles are transferred here after logging | + +### Output Attributes + +| Attribute | Relationship | Description | +|---------------|--------------|-------------------------------------------------------------------------------------------------------------------------------------| +| absolute.path | success | The full/absolute path from where a file was picked up. The current 'path' attribute is still populated, but may be a relative path | +| filename | success | The filename is set to the name of the file on disk | + + +## KamikazeProcessorRs + +### Description + +This processor can fail or panic in on_trigger and on_schedule calls based on configuration. Only for testing purposes. + +### Properties + +In the list below, the names of required properties appear in bold. Any other properties (not in bold) are considered optional. The table also indicates any default values, and whether a property supports the NiFi Expression Language. + +| Name | Default Value | Allowable Values | Description | +|---------------------------|---------------|-----------------------------------------------------------------------------------------------|------------------------------------------| +| **On Schedule Behaviour** | ReturnOk | ReturnErr
ReturnOk
GetNotRegisteredProperty
GetInvalidControllerService
Panic | What to do during the on_schedule method | +| **On Trigger Behaviour** | ReturnOk | ReturnErr
ReturnOk
GetNotRegisteredProperty
GetInvalidControllerService
Panic | What to do during the on_trigger method | + +### Relationships + +| Name | Description | +|---------|----------------------| +| success | success relationship | + + +## LogAttributeRs + +### Description + +Logs attributes of flow files in the MiNiFi application log. + +### Properties + +In the list below, the names of required properties appear in bold. Any other properties (not in bold) are considered optional. The table also indicates any default values, and whether a property supports the NiFi Expression Language. + +| Name | Default Value | Allowable Values | Description | +|-----------------------|---------------|------------------------------------------------------------------|----------------------------------------------------------------------------------------------------------------------------------------------------------------| +| **Log Level** | Info | Trace
Debug
Info
Warn
Error
Critical
Off | The Log Level to use when logging the Attributes | +| Attributes to Log | | | A comma-separated list of Attributes to Log. If not specified, all attributes will be logged. | +| Attributes to Ignore | | | A comma-separated list of Attributes to ignore. If not specified, no attributes will be ignored. | +| **Log Payload** | false | true
false | If true, the FlowFile's payload will be logged, in addition to its attributes. Otherwise, just the Attributes will be logged. | +| Log Prefix | | | Log prefix appended to the log lines. It helps to distinguish the output of multiple LogAttribute processors. | +| **FlowFiles To Log** | 1 | | Number of flow files to log. If set to zero all flow files will be logged. Please note that this may block other threads from running if not used judiciously. | +| **Hexencode Payload** | false | true
false | If true, the FlowFile's payload will be logged in a hexencoded format | + +### Relationships + +| Name | Description | +|---------|----------------------------------------------| +| success | FlowFiles are transferred here after logging | + + +## LoremIpsumCSUser + +### Description + +Processor to test Controller Service API + +### Properties + +In the list below, the names of required properties appear in bold. Any other properties (not in bold) are considered optional. The table also indicates any default values, and whether a property supports the NiFi Expression Language. + +| Name | Default Value | Allowable Values | Description | +|------------------------------------|---------------|-------------------|--------------------------------------------| +| **Lorem Ipsum Controller Service** | | | Name of the lorem ipsum controller service | +| **Write Method** | Buffer | Buffer
Stream | Which API to test | + +### Relationships + +| Name | Description | +|---------|------------------------------| +| success | All flowfile are routed here | + + +## PutFileRs + +### Description + +Writes the contents of a FlowFile to the local file system. + +### Properties + +In the list below, the names of required properties appear in bold. Any other properties (not in bold) are considered optional. The table also indicates any default values, and whether a property supports the NiFi Expression Language. + +| Name | Default Value | Allowable Values | Description | +|----------------------------------|---------------|-----------------------------|--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| **Directory** | . | | The output directory to which to put files
**Supports Expression Language: true** | +| **Conflict Resolution Strategy** | fail | fail
replace
ignore | Indicates what should happen when a file with the same name already exists in the output directory | +| **Create Missing Directories** | true | true
false | If true, then missing destination directories will be created. If false, flowfiles are penalized and sent to failure. | +| Maximum File Count | | | Specifies the maximum number of files that can exist in the output directory | +| Permissions | | | Sets the permissions on the output file to the value of this attribute. Must be an octal number (e.g. 644 or 0755). Not supported on Windows systems. | +| Directory Permissions | | | Sets the permissions on the directories being created if 'Create Missing Directories' property is set. Must be an octal number (e.g. 644 or 0755). Not supported on Windows systems. | + +### Relationships + +| Name | Description | +|---------|-----------------------------------------------------------------------------------| +| success | Flowfiles that are successfully written to a file are routed to this relationship | +| failure | Failed files (conflict, write failure, etc.) are transferred to failure | + + +## DummyControllerService + +### Description + +Dummy Controller Service + +### Properties + +In the list below, the names of required properties appear in bold. Any other properties (not in bold) are considered optional. The table also indicates any default values, and whether a property supports the NiFi Expression Language. + +| Name | Default Value | Allowable Values | Description | +|------|---------------|------------------|-------------| + + +## LoremIpsumControllerService + +### Description + +Simple Rusty Controller Service to test API + +### Properties + +In the list below, the names of required properties appear in bold. Any other properties (not in bold) are considered optional. The table also indicates any default values, and whether a property supports the NiFi Expression Language. + +| Name | Default Value | Allowable Values | Description | +|------------|---------------|------------------|-------------------------------------------------------------| +| **Length** | 25 | | How many words to generate
**Sensitive Property: true** | diff --git a/minifi_rust/extensions/minifi_rs_playground/src/controller_services/dummy_controller_service.rs b/minifi_rust/extensions/minifi_rs_playground/src/controller_services/dummy_controller_service.rs new file mode 100644 index 0000000000..ccd098ab41 --- /dev/null +++ b/minifi_rust/extensions/minifi_rs_playground/src/controller_services/dummy_controller_service.rs @@ -0,0 +1,22 @@ +use minifi_native::macros::ComponentIdentifier; +use minifi_native::{ + ControllerServiceDefinition, EnableControllerService, GetProperty, Logger, MinifiError, + Property, +}; + +#[derive(Debug, ComponentIdentifier)] +pub(crate) struct DummyControllerService {} + +impl EnableControllerService for DummyControllerService { + fn enable(_context: &Ctx, _logger: &L) -> Result + where + Self: Sized, + { + Ok(Self {}) + } +} + +impl ControllerServiceDefinition for DummyControllerService { + const DESCRIPTION: &'static str = "Dummy Controller Service"; + const PROPERTIES: &'static [Property] = &[]; +} diff --git a/minifi_rust/extensions/minifi_rs_playground/src/controller_services/lorem_ipsum_controller_service.rs b/minifi_rust/extensions/minifi_rs_playground/src/controller_services/lorem_ipsum_controller_service.rs new file mode 100644 index 0000000000..eeb9ab5213 --- /dev/null +++ b/minifi_rust/extensions/minifi_rs_playground/src/controller_services/lorem_ipsum_controller_service.rs @@ -0,0 +1,33 @@ +mod properties; + +use crate::controller_services::lorem_ipsum_controller_service::properties::LENGTH; +use lipsum::lipsum; +use minifi_native::macros::ComponentIdentifier; +use minifi_native::{ + ControllerServiceDefinition, EnableControllerService, GetProperty, Logger, MinifiError, + Property, +}; + +#[derive(Debug, ComponentIdentifier)] +pub(crate) struct LoremIpsumControllerService { + pub data: String, +} + +impl EnableControllerService for LoremIpsumControllerService { + fn enable(context: &P, _logger: &L) -> Result + where + Self: Sized, + { + let length = context + .get_u64_property(&LENGTH)? + .ok_or(MinifiError::missing_required_property("Length is required"))?; + + let data = lipsum(length as usize); + Ok(Self { data }) + } +} + +impl ControllerServiceDefinition for LoremIpsumControllerService { + const DESCRIPTION: &'static str = "Simple Rusty Controller Service to test API"; + const PROPERTIES: &'static [Property] = &[LENGTH]; +} diff --git a/minifi_rust/extensions/minifi_rs_playground/src/controller_services/lorem_ipsum_controller_service/properties.rs b/minifi_rust/extensions/minifi_rs_playground/src/controller_services/lorem_ipsum_controller_service/properties.rs new file mode 100644 index 0000000000..b895647a7c --- /dev/null +++ b/minifi_rust/extensions/minifi_rs_playground/src/controller_services/lorem_ipsum_controller_service/properties.rs @@ -0,0 +1,13 @@ +use minifi_native::{Property, StandardPropertyValidator}; + +pub(crate) const LENGTH: Property = Property { + name: "Length", + description: "How many words to generate", + is_required: true, + is_sensitive: false, + supports_expr_lang: false, + default_value: Some("25"), + validator: StandardPropertyValidator::U64Validator, + allowed_values: &[], + allowed_type: "", +}; diff --git a/minifi_rust/extensions/minifi_rs_playground/src/controller_services/mod.rs b/minifi_rust/extensions/minifi_rs_playground/src/controller_services/mod.rs new file mode 100644 index 0000000000..7935987e7f --- /dev/null +++ b/minifi_rust/extensions/minifi_rs_playground/src/controller_services/mod.rs @@ -0,0 +1,2 @@ +pub(crate) mod dummy_controller_service; +pub(crate) mod lorem_ipsum_controller_service; diff --git a/minifi_rust/extensions/minifi_rs_playground/src/lib.rs b/minifi_rust/extensions/minifi_rs_playground/src/lib.rs new file mode 100644 index 0000000000..ad83792f28 --- /dev/null +++ b/minifi_rust/extensions/minifi_rs_playground/src/lib.rs @@ -0,0 +1,37 @@ +mod controller_services; +mod processors; + +use crate::controller_services::dummy_controller_service::DummyControllerService; +use crate::controller_services::lorem_ipsum_controller_service::LoremIpsumControllerService; +use crate::processors::asciify_german::AsciifyGerman; +use crate::processors::count_actual_logging::CountActualLogging; +use crate::processors::duplicate_text::DuplicateStreamText; +use crate::processors::generate_flow_file::GenerateFlowFileRs; +use crate::processors::get_file::GetFileRs; +use crate::processors::kamikaze_processor::KamikazeProcessorRs; +use crate::processors::log_attribute::LogAttributeRs; +use crate::processors::lorem_ipsum_cs_user::LoremIpsumCSUser; +use crate::processors::put_file::PutFileRs; + +use minifi_native::{ + ComplexProcessorType, Concurrent, Exclusive, FlowFileSourceProcessorType, + FlowFileStreamTransformProcessorType, FlowFileTransformProcessorType, +}; + +minifi_native::declare_minifi_extension!( +processors: [ + (ComplexProcessorType, Concurrent, GenerateFlowFileRs), + (ComplexProcessorType, Concurrent, LogAttributeRs), + (ComplexProcessorType, Concurrent, GetFileRs), + (ComplexProcessorType, Concurrent, KamikazeProcessorRs), + (ComplexProcessorType, Exclusive, CountActualLogging), + (FlowFileSourceProcessorType, Concurrent, LoremIpsumCSUser), + (FlowFileTransformProcessorType, Concurrent, PutFileRs), + (FlowFileStreamTransformProcessorType, Concurrent, AsciifyGerman), + (FlowFileStreamTransformProcessorType, Exclusive, DuplicateStreamText), +], +controllers: [ + LoremIpsumControllerService, + DummyControllerService, +] +); diff --git a/minifi_rust/extensions/minifi_rs_playground/src/processors/asciify_german.rs b/minifi_rust/extensions/minifi_rs_playground/src/processors/asciify_german.rs new file mode 100644 index 0000000000..e85949a07f --- /dev/null +++ b/minifi_rust/extensions/minifi_rs_playground/src/processors/asciify_german.rs @@ -0,0 +1,67 @@ +use crate::processors::asciify_german::relationships::FAILURE; +use minifi_native::macros::{ComponentIdentifier}; +use minifi_native::{ + FlowFileStreamTransform, GetProperty, InputStream, Logger, MinifiError, OutputStream, Schedule, + TransformStreamResult, +}; +use std::collections::HashMap; + +mod relationships; + +#[derive(Debug, ComponentIdentifier)] +pub(crate) struct AsciifyGerman {} + +impl Schedule for AsciifyGerman { + fn schedule(_context: &P, _logger: &L) -> Result + where + Self: Sized, + { + Ok(Self {}) + } +} + +impl FlowFileStreamTransform for AsciifyGerman { + fn transform( + &self, + _context: &Ctx, + input_stream: &mut dyn InputStream, + output_stream: &mut dyn OutputStream, + _logger: &LoggerImpl, + ) -> Result { + let mut byte = [0u8; 1]; + + while input_stream.read(&mut byte)? > 0 { + match byte[0] { + 0..=127 => { + output_stream.write_all(&byte)?; + } + 0xC3 => { + let mut next = [0u8; 1]; + if input_stream.read(&mut next)? > 0 { + match next[0] { + 0xA4 => output_stream.write_all(b"ae")?, // ä + 0xB6 => output_stream.write_all(b"oe")?, // ö + 0xBC => output_stream.write_all(b"ue")?, // ü + 0x84 => output_stream.write_all(b"Ae")?, // Ä + 0x96 => output_stream.write_all(b"Oe")?, // Ö + 0x9C => output_stream.write_all(b"Ue")?, // Ü + 0x9F => output_stream.write_all(b"ss")?, // ß + _ => return Ok(TransformStreamResult::route_without_changes(&FAILURE)), + } + } + } + _ => return Ok(TransformStreamResult::route_without_changes(&FAILURE)), + } + } + + output_stream.flush()?; + Ok(TransformStreamResult::new( + &relationships::SUCCESS, + HashMap::new(), + )) + } +} + +mod processor_definition; +#[cfg(test)] +mod tests; diff --git a/minifi_rust/extensions/minifi_rs_playground/src/processors/asciify_german/processor_definition.rs b/minifi_rust/extensions/minifi_rs_playground/src/processors/asciify_german/processor_definition.rs new file mode 100644 index 0000000000..62f135741d --- /dev/null +++ b/minifi_rust/extensions/minifi_rs_playground/src/processors/asciify_german/processor_definition.rs @@ -0,0 +1,15 @@ +use crate::processors::asciify_german::AsciifyGerman; +use minifi_native::{ + OutputAttribute, ProcessorDefinition, ProcessorInputRequirement, Property, Relationship, +}; + +impl ProcessorDefinition for AsciifyGerman { + const DESCRIPTION: &'static str = "This processor switches German characters with their ascii counterparts. (to test stream API)"; + const INPUT_REQUIREMENT: ProcessorInputRequirement = ProcessorInputRequirement::Required; + const SUPPORTS_DYNAMIC_PROPERTIES: bool = false; + const SUPPORTS_DYNAMIC_RELATIONSHIPS: bool = false; + const OUTPUT_ATTRIBUTES: &'static [OutputAttribute] = &[]; + const RELATIONSHIPS: &'static [Relationship] = + &[super::relationships::SUCCESS, super::relationships::FAILURE]; + const PROPERTIES: &'static [Property] = &[]; +} diff --git a/minifi_rust/extensions/minifi_rs_playground/src/processors/asciify_german/relationships.rs b/minifi_rust/extensions/minifi_rs_playground/src/processors/asciify_german/relationships.rs new file mode 100644 index 0000000000..ccb190d200 --- /dev/null +++ b/minifi_rust/extensions/minifi_rs_playground/src/processors/asciify_german/relationships.rs @@ -0,0 +1,11 @@ +use minifi_native::Relationship; + +pub(crate) const SUCCESS: Relationship = Relationship { + name: "success", + description: "All asciified flowfiles are routed here", +}; + +pub(crate) const FAILURE: Relationship = Relationship { + name: "failure", + description: "Non-german flowfiles are routed here", +}; diff --git a/minifi_rust/extensions/minifi_rs_playground/src/processors/asciify_german/tests.rs b/minifi_rust/extensions/minifi_rs_playground/src/processors/asciify_german/tests.rs new file mode 100644 index 0000000000..268a95ff92 --- /dev/null +++ b/minifi_rust/extensions/minifi_rs_playground/src/processors/asciify_german/tests.rs @@ -0,0 +1,54 @@ +use super::*; +use crate::processors::asciify_german::relationships::SUCCESS; +use minifi_native::{IoState, MockLogger, MockProcessContext}; +use std::io::BufReader; + +#[test] +fn schedule_succeeds_with_default_values() { + assert!(AsciifyGerman::schedule(&MockProcessContext::new(), &MockLogger::new()).is_ok()); +} + +#[test] +fn simple_test() { + let process_context = MockProcessContext::new(); + let context = MockProcessContext::new(); + let logger = MockLogger::new(); + + let asciify_german = + AsciifyGerman::schedule(&process_context, &logger).expect("Should succeed"); + let input_str = "Falsches Üben von Xylophonmusik quält jeden größeren Zwerg."; + let mut input_stream = BufReader::new(input_str.as_bytes()); + let mut output_vec: Vec = Vec::new(); + { + let result = asciify_german + .transform(&context, &mut input_stream, &mut output_vec, &logger) + .expect("Should succeed"); + assert_eq!(result.write_status(), IoState::Ok); + assert_eq!(result.target_relationship_name(), SUCCESS.name); + } + assert_eq!( + output_vec, + "Falsches Ueben von Xylophonmusik quaelt jeden groesseren Zwerg.".as_bytes() + ); +} + +#[test] +fn simple_failure_test() { + let process_context = MockProcessContext::new(); + let context = MockProcessContext::new(); + let logger = MockLogger::new(); + + let asciify_german = + AsciifyGerman::schedule(&process_context, &logger).expect("Should succeed"); + let input_str = "Üldögélő műújságíró"; + let mut input_stream = BufReader::new(input_str.as_bytes()); + let mut output_vec: Vec = Vec::new(); + { + let result = asciify_german + .transform(&context, &mut input_stream, &mut output_vec, &logger) + .expect("Should succeed"); + assert_eq!(result.write_status(), IoState::Cancel); + assert_eq!(result.target_relationship_name(), FAILURE.name); + } + assert_eq!(output_vec, "Ueldoeg".as_bytes()); +} diff --git a/minifi_rust/extensions/minifi_rs_playground/src/processors/count_actual_logging.rs b/minifi_rust/extensions/minifi_rs_playground/src/processors/count_actual_logging.rs new file mode 100644 index 0000000000..f712f37c9a --- /dev/null +++ b/minifi_rust/extensions/minifi_rs_playground/src/processors/count_actual_logging.rs @@ -0,0 +1,57 @@ +use minifi_native::macros::{ComponentIdentifier}; +use minifi_native::{ + GetProperty, Logger, MinifiError, MutTrigger, OnTriggerResult, OutputAttribute, ProcessContext, + ProcessSession, ProcessorDefinition, ProcessorInputRequirement, Property, Relationship, + Schedule, debug, info, trace, +}; + +#[derive(Debug, ComponentIdentifier)] +pub(crate) struct CountActualLogging { + log_count: usize, +} + +impl CountActualLogging { + fn get_incremented_log_count(&mut self) -> usize { + self.log_count += 1; + self.log_count + } +} + +impl Schedule for CountActualLogging { + fn schedule(_context: &P, _logger: &L) -> Result + where + Self: Sized, + { + Ok(Self { log_count: 0 }) + } +} + +impl MutTrigger for CountActualLogging { + fn trigger( + &mut self, + _context: &mut PC, + _session: &mut PS, + logger: &L, + ) -> Result + where + PC: ProcessContext, + PS: ProcessSession, + L: Logger, + { + trace!(logger, "trace {}", self.get_incremented_log_count()); + debug!(logger, "debug {}", self.get_incremented_log_count()); + info!(logger, "info {}", self.get_incremented_log_count()); + + Ok(OnTriggerResult::Ok) + } +} + +impl ProcessorDefinition for CountActualLogging { + const DESCRIPTION: &'static str = "For testing lazy logging"; + const INPUT_REQUIREMENT: ProcessorInputRequirement = ProcessorInputRequirement::Forbidden; + const SUPPORTS_DYNAMIC_PROPERTIES: bool = false; + const SUPPORTS_DYNAMIC_RELATIONSHIPS: bool = false; + const OUTPUT_ATTRIBUTES: &'static [OutputAttribute] = &[]; + const RELATIONSHIPS: &'static [Relationship] = &[]; + const PROPERTIES: &'static [Property] = &[]; +} diff --git a/minifi_rust/extensions/minifi_rs_playground/src/processors/duplicate_text.rs b/minifi_rust/extensions/minifi_rs_playground/src/processors/duplicate_text.rs new file mode 100644 index 0000000000..38ff066e7b --- /dev/null +++ b/minifi_rust/extensions/minifi_rs_playground/src/processors/duplicate_text.rs @@ -0,0 +1,54 @@ +use minifi_native::macros::{ComponentIdentifier}; +use minifi_native::{ + GetAttribute, GetControllerService, GetProperty, InputStream, Logger, MinifiError, + MutFlowFileStreamTransform, OutputAttribute, OutputStream, ProcessorDefinition, + ProcessorInputRequirement, Property, Relationship, Schedule, TransformStreamResult, +}; +use std::collections::HashMap; + +#[derive(Debug, ComponentIdentifier)] +pub(crate) struct DuplicateStreamText {} + +pub(crate) const SUCCESS: Relationship = Relationship { + name: "success", + description: "", +}; + +impl Schedule for DuplicateStreamText { + fn schedule( + _context: &Ctx, + _logger: &L, + ) -> Result + where + Self: Sized, + { + Ok(Self {}) + } +} + +impl MutFlowFileStreamTransform for DuplicateStreamText { + fn transform( + &mut self, + _context: &Ctx, + input_stream: &mut dyn InputStream, + output_stream: &mut dyn OutputStream, + _logger: &LoggerImpl, + ) -> Result { + let mut byte = [0u8; 1]; + while input_stream.read(&mut byte)? > 0 { + output_stream.write(&byte)?; + output_stream.write(&byte)?; + } + Ok(TransformStreamResult::new(&SUCCESS, HashMap::new())) + } +} + +impl ProcessorDefinition for DuplicateStreamText { + const DESCRIPTION: &'static str = "Duplicate text"; + const INPUT_REQUIREMENT: ProcessorInputRequirement = ProcessorInputRequirement::Required; + const SUPPORTS_DYNAMIC_PROPERTIES: bool = false; + const SUPPORTS_DYNAMIC_RELATIONSHIPS: bool = false; + const OUTPUT_ATTRIBUTES: &'static [OutputAttribute] = &[]; + const RELATIONSHIPS: &'static [Relationship] = &[SUCCESS]; + const PROPERTIES: &'static [Property] = &[]; +} diff --git a/minifi_rust/extensions/minifi_rs_playground/src/processors/generate_flow_file.rs b/minifi_rust/extensions/minifi_rs_playground/src/processors/generate_flow_file.rs new file mode 100644 index 0000000000..1e9800ce5f --- /dev/null +++ b/minifi_rust/extensions/minifi_rs_playground/src/processors/generate_flow_file.rs @@ -0,0 +1,168 @@ +use minifi_native::macros::{ComponentIdentifier}; +use minifi_native::{ + GetProperty, Logger, MinifiError, OnTriggerResult, ProcessContext, + ProcessSession, Schedule, Trigger, +}; +use rand::RngExt; +use rand::distr::Alphanumeric; +use std::cmp::PartialEq; + +mod properties; +mod relationships; + +#[derive(Debug, PartialEq)] +enum Mode { + UniqueBytes, + UniqueText, + NotUniqueBytes, + NotUniqueText, + CustomText, + Empty, +} + +#[derive(Debug, ComponentIdentifier)] +pub(crate) struct GenerateFlowFileRs { + mode: Mode, + batch_size: u64, + file_size: u64, + data_generated_during_on_schedule: Vec, +} + +impl Schedule for GenerateFlowFileRs { + fn schedule(context: &P, _logger: &L) -> Result + where + Self: Sized, + { + let is_unique = context + .get_bool_property(&properties::UNIQUE_FLOW_FILES)? + .expect("Required property"); + let is_text = context + .get_property(&properties::DATA_FORMAT)? + .expect("Required property") + .as_str() + == "Text"; + let has_custom_text = context.get_property(&properties::CUSTOM_TEXT)?.is_some(); + + let file_size = context + .get_size_property(&properties::FILE_SIZE)? + .expect("Required property"); + let batch_size = context + .get_u64_property(&properties::BATCH_SIZE)? + .expect("Required property"); + + let mode = Self::get_mode(is_unique, is_text, has_custom_text, file_size); + let data_generated_during_on_schedule = + if mode == Mode::NotUniqueText || mode == Mode::NotUniqueBytes { + let mut data = vec![0; file_size as usize]; + Self::generate_data(&mut data, is_text); + data + } else { + vec![] + }; + + Ok(Self { + mode, + batch_size, + file_size, + data_generated_during_on_schedule, + }) + } +} + +impl GenerateFlowFileRs { + fn is_unique(&self) -> bool { + match self.mode { + Mode::UniqueBytes => true, + Mode::UniqueText => true, + Mode::NotUniqueBytes => false, + Mode::NotUniqueText => false, + Mode::CustomText => false, + Mode::Empty => false, + } + } + + fn is_text(&self) -> bool { + match self.mode { + Mode::UniqueBytes => false, + Mode::UniqueText => true, + Mode::NotUniqueBytes => false, + Mode::NotUniqueText => true, + Mode::CustomText => true, + Mode::Empty => false, + } + } + + fn get_mode(is_unique: bool, is_text: bool, has_custom_text: bool, file_size: u64) -> Mode { + if is_text && !is_unique && has_custom_text { + return Mode::CustomText; + } + + if file_size == 0 { + return Mode::Empty; + } + + match (is_unique, is_text) { + (true, true) => Mode::UniqueText, + (true, false) => Mode::UniqueBytes, + (false, true) => Mode::NotUniqueText, + (false, false) => Mode::NotUniqueBytes, + } + } + + fn generate_data(data: &mut [u8], text_data: bool) { + let mut rng = rand::rng(); + + if text_data { + for byte in data.iter_mut() { + *byte = rng.sample(Alphanumeric); + } + } else { + rng.fill(data); + } + } +} + +impl Trigger for GenerateFlowFileRs { + fn trigger( + &self, + context: &mut PC, + session: &mut PS, + _logger: &L, + ) -> Result + where + PC: ProcessContext, + PS: ProcessSession, + { + let non_unique_data_buffer: &[u8]; + let custom_text_for_batch: Option; + + if self.mode == Mode::CustomText + && let Some(custom_text) = context.get_property(&properties::CUSTOM_TEXT, None)? + { + custom_text_for_batch = Some(custom_text); + non_unique_data_buffer = custom_text_for_batch.as_ref().unwrap().as_bytes(); + } else { + non_unique_data_buffer = self.data_generated_during_on_schedule.as_slice(); + } + + for _ in 0..self.batch_size { + let mut ff = session.create()?; + if self.mode != Mode::Empty { + if self.is_unique() { + let mut unique_data: Vec = vec![0; self.file_size as usize]; + Self::generate_data(&mut unique_data, self.is_text()); + session.write(&mut ff, unique_data.as_slice())?; + } else { + session.write(&mut ff, non_unique_data_buffer)?; + } + } + session.transfer(ff, relationships::SUCCESS.name)?; + } + Ok(OnTriggerResult::Ok) + } +} + +pub(crate) mod processor_definition; + +#[cfg(test)] +mod tests; diff --git a/minifi_rust/extensions/minifi_rs_playground/src/processors/generate_flow_file/processor_definition.rs b/minifi_rust/extensions/minifi_rs_playground/src/processors/generate_flow_file/processor_definition.rs new file mode 100644 index 0000000000..7e0c21b520 --- /dev/null +++ b/minifi_rust/extensions/minifi_rs_playground/src/processors/generate_flow_file/processor_definition.rs @@ -0,0 +1,21 @@ +use super::properties::*; +use super::{GenerateFlowFileRs, relationships}; +use minifi_native::{ + OutputAttribute, ProcessorDefinition, ProcessorInputRequirement, Property, Relationship, +}; + +impl ProcessorDefinition for GenerateFlowFileRs { + const DESCRIPTION: &'static str = "This processor creates FlowFiles with random data or custom content. GenerateFlowFile is useful for load testing, configuration, and simulation."; + const INPUT_REQUIREMENT: ProcessorInputRequirement = ProcessorInputRequirement::Forbidden; + const SUPPORTS_DYNAMIC_PROPERTIES: bool = false; + const SUPPORTS_DYNAMIC_RELATIONSHIPS: bool = false; + const OUTPUT_ATTRIBUTES: &'static [OutputAttribute] = &[]; + const RELATIONSHIPS: &'static [Relationship] = &[relationships::SUCCESS]; + const PROPERTIES: &'static [Property] = &[ + FILE_SIZE, + BATCH_SIZE, + DATA_FORMAT, + UNIQUE_FLOW_FILES, + CUSTOM_TEXT, + ]; +} diff --git a/minifi_rust/extensions/minifi_rs_playground/src/processors/generate_flow_file/properties.rs b/minifi_rust/extensions/minifi_rs_playground/src/processors/generate_flow_file/properties.rs new file mode 100644 index 0000000000..19d81836c9 --- /dev/null +++ b/minifi_rust/extensions/minifi_rs_playground/src/processors/generate_flow_file/properties.rs @@ -0,0 +1,61 @@ +use minifi_native::{Property, StandardPropertyValidator}; + +pub(crate) const FILE_SIZE: Property = Property { + name: "File Size", + description: "The size of the file that will be used", + is_required: true, + is_sensitive: false, + supports_expr_lang: true, + default_value: Some("1 kB"), + validator: StandardPropertyValidator::DataSizeValidator, + allowed_values: &[], + allowed_type: "", +}; + +pub(crate) const BATCH_SIZE: Property = Property { + name: "Batch Size", + description: "The number of FlowFiles to be transferred in each invocation", + is_required: true, + is_sensitive: false, + supports_expr_lang: false, + default_value: Some("1"), + validator: StandardPropertyValidator::U64Validator, + allowed_values: &[], + allowed_type: "", +}; + +pub(crate) const DATA_FORMAT: Property = Property { + name: "Data Format", + description: "Specifies whether the data should be Text or Binary", + is_required: true, + is_sensitive: false, + supports_expr_lang: false, + default_value: Some("Binary"), + validator: StandardPropertyValidator::AlwaysValidValidator, + allowed_values: &["Text", "Binary"], + allowed_type: "", +}; + +pub(crate) const UNIQUE_FLOW_FILES: Property = Property { + name: "Unique FlowFiles", + description: "If true, each FlowFile that is generated will be unique. If false, a random value will be generated and all FlowFiles will get the same content but this offers much higher throughput (but see the description of Custom Text for special non-random use cases)", + is_required: true, + is_sensitive: false, + supports_expr_lang: false, + default_value: Some("true"), + validator: StandardPropertyValidator::BoolValidator, + allowed_values: &[], + allowed_type: "", +}; + +pub(crate) const CUSTOM_TEXT: Property = Property { + name: "Custom Text", + description: "If Data Format is text and if Unique FlowFiles is false, then this custom text will be used as content of the generated FlowFiles and the File Size will be ignored. Finally, if Expression Language is used, evaluation will be performed only once per batch of generated FlowFiles", + is_required: false, + is_sensitive: false, + supports_expr_lang: true, + default_value: None, + validator: StandardPropertyValidator::AlwaysValidValidator, + allowed_values: &[], + allowed_type: "", +}; diff --git a/minifi_rust/extensions/minifi_rs_playground/src/processors/generate_flow_file/relationships.rs b/minifi_rust/extensions/minifi_rs_playground/src/processors/generate_flow_file/relationships.rs new file mode 100644 index 0000000000..3fb72c7114 --- /dev/null +++ b/minifi_rust/extensions/minifi_rs_playground/src/processors/generate_flow_file/relationships.rs @@ -0,0 +1,6 @@ +use minifi_native::Relationship; + +pub(crate) const SUCCESS: Relationship = Relationship { + name: "success", + description: "success operational on the flow record", // TODO(wtf... but copy paste) +}; diff --git a/minifi_rust/extensions/minifi_rs_playground/src/processors/generate_flow_file/tests.rs b/minifi_rust/extensions/minifi_rs_playground/src/processors/generate_flow_file/tests.rs new file mode 100644 index 0000000000..2a1f4108f8 --- /dev/null +++ b/minifi_rust/extensions/minifi_rs_playground/src/processors/generate_flow_file/tests.rs @@ -0,0 +1,138 @@ +use super::*; +use crate::processors::generate_flow_file::properties::{ + BATCH_SIZE, CUSTOM_TEXT, DATA_FORMAT, UNIQUE_FLOW_FILES, +}; +use minifi_native::{MockLogger, MockProcessContext, MockProcessSession}; + +#[test] +fn schedule_succeeds_with_default_values() { + assert!(GenerateFlowFileRs::schedule(&MockProcessContext::new(), &MockLogger::new()).is_ok()); +} + +#[test] +fn generate_flow_file_empty_test() { + let logger = MockLogger::new(); + let mut context = MockProcessContext::new(); + context + .properties + .insert(properties::FILE_SIZE.name.to_string(), "0".to_string()); + context + .properties + .insert(UNIQUE_FLOW_FILES.name.to_string(), "false".to_string()); + context + .properties + .insert(DATA_FORMAT.name.to_string(), "Text".to_string()); + + let processor = GenerateFlowFileRs::schedule(&context, &logger).unwrap(); + let mut session = MockProcessSession::new(); + assert_eq!( + processor + .trigger(&mut context, &mut session, &logger) + .unwrap(), + OnTriggerResult::Ok + ); + let result_flow_files = session.transferred_flow_files.borrow(); + assert_eq!(result_flow_files.len(), 1); + assert_eq!(result_flow_files[0].flow_file.content_len(), 0); +} + +#[test] +fn generate_custom_text() { + let mut context = MockProcessContext::new(); + context + .properties + .insert(properties::FILE_SIZE.name.to_string(), "0".to_string()); + context + .properties + .insert(UNIQUE_FLOW_FILES.name.to_string(), "false".to_string()); + context + .properties + .insert(DATA_FORMAT.name.to_string(), "Text".to_string()); + context + .properties + .insert(CUSTOM_TEXT.name.to_string(), "foo bar baz".to_string()); + + let logger = MockLogger::new(); + let processor = GenerateFlowFileRs::schedule(&context, &logger).unwrap(); + + let mut session = MockProcessSession::new(); + assert_eq!( + processor + .trigger(&mut context, &mut session, &logger) + .expect("Should trigger successfully"), + OnTriggerResult::Ok + ); + let result_flow_files = session.transferred_flow_files.borrow(); + assert_eq!(result_flow_files.len(), 1); + assert!(result_flow_files[0].flow_file.content_eq("foo bar baz"),); +} + +#[test] +fn random_bytes_unique() { + let mut context = MockProcessContext::new(); + context + .properties + .insert(properties::FILE_SIZE.name.to_string(), "40 B".to_string()); + context + .properties + .insert(UNIQUE_FLOW_FILES.name.to_string(), "true".to_string()); + context + .properties + .insert(DATA_FORMAT.name.to_string(), "Bytes".to_string()); + context + .properties + .insert(BATCH_SIZE.name.to_string(), "2".to_string()); + + let logger = MockLogger::new(); + let processor = GenerateFlowFileRs::schedule(&context, &logger).unwrap(); + let mut session = MockProcessSession::new(); + assert_eq!( + processor + .trigger(&mut context, &mut session, &logger) + .expect("Should trigger successfully"), + OnTriggerResult::Ok + ); + let result_flow_files = session.transferred_flow_files.borrow(); + assert_eq!(result_flow_files.len(), 2); + assert_eq!(result_flow_files[0].flow_file.content_len(), 40); + assert_eq!(result_flow_files[1].flow_file.content_len(), 40); + assert_ne!( + *result_flow_files[0].flow_file.content.borrow(), + *result_flow_files[1].flow_file.content.borrow() + ); +} + +#[test] +fn random_bytes_non_unique() { + let mut context = MockProcessContext::new(); + context + .properties + .insert(properties::FILE_SIZE.name.to_string(), "40 B".to_string()); + context + .properties + .insert(UNIQUE_FLOW_FILES.name.to_string(), "false".to_string()); + context + .properties + .insert(DATA_FORMAT.name.to_string(), "Bytes".to_string()); + context + .properties + .insert(BATCH_SIZE.name.to_string(), "2".to_string()); + + let logger = MockLogger::new(); + let processor = GenerateFlowFileRs::schedule(&context, &logger).unwrap(); + let mut session = MockProcessSession::new(); + assert_eq!( + processor + .trigger(&mut context, &mut session, &logger) + .expect("Should trigger successfully"), + OnTriggerResult::Ok + ); + let result_flow_files = session.transferred_flow_files.borrow(); + assert_eq!(result_flow_files.len(), 2); + assert_eq!(result_flow_files[0].flow_file.content_len(), 40); + assert_eq!(result_flow_files[1].flow_file.content_len(), 40); + assert_eq!( + *result_flow_files[0].flow_file.content.borrow(), + *result_flow_files[1].flow_file.content.borrow() + ); +} diff --git a/minifi_rust/extensions/minifi_rs_playground/src/processors/get_file.rs b/minifi_rust/extensions/minifi_rs_playground/src/processors/get_file.rs new file mode 100644 index 0000000000..1b05b030af --- /dev/null +++ b/minifi_rust/extensions/minifi_rs_playground/src/processors/get_file.rs @@ -0,0 +1,303 @@ +use crate::processors::get_file::output_attributes::{ + ABSOLUTE_PATH_OUTPUT_ATTRIBUTE, FILENAME_OUTPUT_ATTRIBUTE, +}; +use crate::processors::get_file::properties::{ + BATCH_SIZE, DIRECTORY, IGNORE_HIDDEN_FILES, KEEP_SOURCE_FILE, MAX_AGE, MAX_SIZE, MIN_AGE, + MIN_SIZE, RECURSE, +}; +use minifi_native::macros::{ComponentIdentifier}; +use minifi_native::{ + GetProperty, IoState, Logger, MinifiError, OnTriggerResult, ProcessContext, + ProcessSession, Schedule, Trigger, debug, info, trace, warn, +}; +use std::collections::VecDeque; +use std::error; +use std::fs::File; +use std::path::PathBuf; +use std::sync::Mutex; +use std::time::{Duration, Instant, SystemTime}; +use walkdir::{DirEntry, WalkDir}; + +mod properties; +mod relationships; + +#[derive(Debug)] +struct GetFileMetrics { + accepted_files: u32, + input_bytes: u64, +} + +#[derive(Debug)] +struct DirectoryListing { + paths: VecDeque, + last_polling_time: Option, +} + +impl DirectoryListing { + fn new() -> Self { + Self { + paths: VecDeque::new(), + last_polling_time: None, + } + } +} + +#[derive(Debug, ComponentIdentifier)] +pub(crate) struct GetFileRs { + recursive: bool, + keep_source_file: bool, + input_directory: PathBuf, + poll_interval: Option, + directory_listing: Mutex, + batch_size: u64, + min_size: Option, + max_size: Option, + min_age: Option, + max_age: Option, + ignore_hidden_files: bool, + metrics: Mutex, +} + +impl GetFileRs { + fn is_listing_empty(&self) -> bool { + let directory_listing = self.directory_listing.lock().unwrap(); + directory_listing.paths.is_empty() + } + + fn poll_listing(&self, batch_size: u64) -> VecDeque { + let mut directory_listings = self.directory_listing.lock().unwrap(); + + let mut res = VecDeque::new(); + for _ in 0..batch_size { + if let Some(path) = directory_listings.paths.pop_back() { + res.push_back(path); + } else { + break; + } + } + + res + } + + fn should_poll(&self) -> bool { + if self.poll_interval.is_none() { + return true; + } + let directory_listings = self.directory_listing.lock().unwrap(); + + if directory_listings.last_polling_time.is_none() { + return true; + } + Instant::now() - directory_listings.last_polling_time.unwrap() > self.poll_interval.unwrap() + } + + fn perform_listing(&self) { + let mut directory_listings = self.directory_listing.lock().unwrap(); + let mut walker = WalkDir::new(&self.input_directory); + + if !self.recursive { + walker = walker.max_depth(1); + } + + let mut files_added = 0u32; + let mut bytes_added = 0u64; + for entry in walker.into_iter().filter_map(Result::ok) { + if self.entry_matches_criteria(&entry).unwrap_or(false) { + let file_size = entry.metadata().map(|m| m.len()).unwrap_or(0); + directory_listings.paths.push_back(entry.into_path()); + files_added += 1; + bytes_added += file_size; + } + } + let mut metrics = self.metrics.lock().unwrap(); + metrics.accepted_files += files_added; + metrics.input_bytes += bytes_added; + directory_listings.last_polling_time = Some(Instant::now()); + } + + fn entry_matches_criteria(&self, dir_entry: &DirEntry) -> Result> { + let metadata = dir_entry.metadata()?; + if !metadata.is_file() { + return Ok(false); + } + let age = SystemTime::now().duration_since(metadata.modified()?)?; + let size = metadata.len(); + + if self.min_age.is_some() && age < self.min_age.unwrap() { + return Ok(false); + } + if self.max_age.is_some() && age > self.max_age.unwrap() { + return Ok(false); + } + if self.min_size.is_some() && size < self.min_size.unwrap() { + return Ok(false); + } + if self.max_size.is_some() && size > self.max_size.unwrap() { + return Ok(false); + } + + fn is_hidden(path: PathBuf) -> bool { + path.file_name() + .and_then(|f| f.to_str()) + .is_some_and(|f| f.starts_with('.')) + } + + if self.ignore_hidden_files && is_hidden(dir_entry.path().to_path_buf()) { + return Ok(false); + } + + Ok(true) + } + + fn get_single_file( + &self, + session: &mut PS, + logger: &L, + path: PathBuf, + ) -> Result<(), MinifiError> { + info!(logger, "GetFile process {:?}", &path); + let mut ff = session + .create() + .expect("Successful FlowFile creation is expected"); + + if let Some(file_name) = path.file_name().and_then(|f| f.to_str()) { + session.set_attribute(&mut ff, FILENAME_OUTPUT_ATTRIBUTE.name, file_name)?; + } else { + warn!(logger, "Couldnt get filename of {:?}", path); + } + session.set_attribute( + &mut ff, + ABSOLUTE_PATH_OUTPUT_ATTRIBUTE.name, + path.to_string_lossy().trim(), + )?; + + session.write_stream(&ff, |output_stream| { + let mut file = File::open(&path)?; + std::io::copy(&mut file, output_stream)?; + Ok(((), IoState::Ok)) + })?; + if !self.keep_source_file { + match std::fs::remove_file(&path) { + Ok(_) => {} + Err(err) => { + warn!(logger, "Failed to remove source file {:?}", err); + } + } + } + session.transfer(ff, relationships::SUCCESS.name)?; + Ok(()) + } + + fn calculate_metrics(&self) -> Vec<(String, f64)> { + let metrics = self.metrics.lock().unwrap(); + vec![ + ("accepted_files".to_string(), metrics.accepted_files as f64), + ("input_bytes".to_string(), metrics.input_bytes as f64), + ] + } +} + +impl Schedule for GetFileRs { + fn schedule(context: &P, _logger: &L) -> Result + where + Self: Sized, + { + let input_directory: PathBuf = context + .get_property(&DIRECTORY)? + .expect("Required property") + .into(); + if !input_directory.is_dir() { + return Err(MinifiError::schedule_err(format!( + "{:?} is not a valid directory", + input_directory + ))); + } + + let recursive = context + .get_bool_property(&RECURSE)? + .expect("Required property"); + + let keep_source_file = context + .get_bool_property(&KEEP_SOURCE_FILE)? + .expect("Required property"); + + let poll_interval = context.get_duration_property(&properties::POLLING_INTERVAL)?; + let min_size = context.get_size_property(&MIN_SIZE)?; + let max_size = context.get_size_property(&MAX_SIZE)?; + let min_age = context.get_duration_property(&MIN_AGE)?; + let max_age = context.get_duration_property(&MAX_AGE)?; + let batch_size = context + .get_u64_property(&BATCH_SIZE)? + .expect("required property"); + let ignore_hidden_files = context + .get_bool_property(&IGNORE_HIDDEN_FILES)? + .expect("required property"); + + Ok(GetFileRs { + recursive, + keep_source_file, + input_directory, + poll_interval, + directory_listing: Mutex::new(DirectoryListing::new()), + batch_size, + min_size, + max_size, + min_age, + max_age, + ignore_hidden_files, + metrics: Mutex::new(GetFileMetrics { + accepted_files: 0, + input_bytes: 0, + }), + }) + } +} + +impl Trigger for GetFileRs { + fn trigger( + &self, + context: &mut PC, + session: &mut PS, + logger: &L, + ) -> Result + where + PC: ProcessContext, + PS: ProcessSession, + L: Logger, + { + trace!(logger, "on_trigger: {:?}", self); + { + let is_dir_empty_before_poll = self.is_listing_empty(); + debug!( + logger, + "Listing is {} before polling directory", is_dir_empty_before_poll + ); + if is_dir_empty_before_poll && self.should_poll() { + self.perform_listing(); + } + } + { + let is_dir_empty_after_poll = self.is_listing_empty(); + debug!( + logger, + "Listing is {} after polling directory", is_dir_empty_after_poll + ); + if is_dir_empty_after_poll { + return Ok(OnTriggerResult::Yield); + } + } + + let files = self.poll_listing(self.batch_size); + for file in files { + self.get_single_file(session, logger, file)?; + } + context.report_metrics(self.calculate_metrics())?; + Ok(OnTriggerResult::Ok) + } +} + +pub(crate) mod processor_definition; + +mod output_attributes; +#[cfg(test)] +mod tests; diff --git a/minifi_rust/extensions/minifi_rs_playground/src/processors/get_file/output_attributes.rs b/minifi_rust/extensions/minifi_rs_playground/src/processors/get_file/output_attributes.rs new file mode 100644 index 0000000000..62b1fbc11f --- /dev/null +++ b/minifi_rust/extensions/minifi_rs_playground/src/processors/get_file/output_attributes.rs @@ -0,0 +1,13 @@ +use minifi_native::OutputAttribute; + +pub(crate) const FILENAME_OUTPUT_ATTRIBUTE: OutputAttribute = OutputAttribute { + name: "filename", + relationships: &["success"], + description: "The filename is set to the name of the file on disk", +}; + +pub(crate) const ABSOLUTE_PATH_OUTPUT_ATTRIBUTE: OutputAttribute = OutputAttribute { + name: "absolute.path", + relationships: &["success"], + description: "The full/absolute path from where a file was picked up. The current 'path' attribute is still populated, but may be a relative path", +}; diff --git a/minifi_rust/extensions/minifi_rs_playground/src/processors/get_file/processor_definition.rs b/minifi_rust/extensions/minifi_rs_playground/src/processors/get_file/processor_definition.rs new file mode 100644 index 0000000000..b3bd89146d --- /dev/null +++ b/minifi_rust/extensions/minifi_rs_playground/src/processors/get_file/processor_definition.rs @@ -0,0 +1,30 @@ +use crate::processors::get_file::output_attributes::{ + ABSOLUTE_PATH_OUTPUT_ATTRIBUTE, FILENAME_OUTPUT_ATTRIBUTE, +}; +use crate::processors::get_file::properties::*; +use crate::processors::get_file::{GetFileRs, relationships}; +use minifi_native::{ + OutputAttribute, ProcessorDefinition, ProcessorInputRequirement, Property, Relationship, +}; + +impl ProcessorDefinition for GetFileRs { + const DESCRIPTION: &'static str = "Creates FlowFiles from files in a directory. MiNiFi will ignore files for which it doesn't have read permissions."; + const INPUT_REQUIREMENT: ProcessorInputRequirement = ProcessorInputRequirement::Forbidden; + const SUPPORTS_DYNAMIC_PROPERTIES: bool = false; + const SUPPORTS_DYNAMIC_RELATIONSHIPS: bool = false; + const OUTPUT_ATTRIBUTES: &'static [OutputAttribute] = + &[ABSOLUTE_PATH_OUTPUT_ATTRIBUTE, FILENAME_OUTPUT_ATTRIBUTE]; + const RELATIONSHIPS: &'static [Relationship] = &[relationships::SUCCESS]; + const PROPERTIES: &'static [Property] = &[ + DIRECTORY, + POLLING_INTERVAL, + RECURSE, + KEEP_SOURCE_FILE, + MIN_AGE, + MAX_AGE, + MIN_SIZE, + MAX_SIZE, + IGNORE_HIDDEN_FILES, + BATCH_SIZE, + ]; +} diff --git a/minifi_rust/extensions/minifi_rs_playground/src/processors/get_file/properties.rs b/minifi_rust/extensions/minifi_rs_playground/src/processors/get_file/properties.rs new file mode 100644 index 0000000000..44f81fa717 --- /dev/null +++ b/minifi_rust/extensions/minifi_rs_playground/src/processors/get_file/properties.rs @@ -0,0 +1,121 @@ +use minifi_native::{Property, StandardPropertyValidator}; + +pub(crate) const DIRECTORY: Property = Property { + name: "Input Directory", + description: "The input directory from which to pull files", + is_required: true, + is_sensitive: false, + supports_expr_lang: true, + default_value: None, + validator: StandardPropertyValidator::NonBlankValidator, + allowed_values: &[], + allowed_type: "", +}; + +pub(crate) const RECURSE: Property = Property { + name: "Recurse Subdirectories", + description: "Indicates whether or not to pull files from subdirectories", + is_required: false, + is_sensitive: false, + supports_expr_lang: false, + default_value: Some("true"), + validator: StandardPropertyValidator::BoolValidator, + allowed_values: &[], + allowed_type: "", +}; + +pub(crate) const KEEP_SOURCE_FILE: Property = Property { + name: "Keep Source File", + description: "If true, the file is not deleted after it has been copied to the Content Repository", + is_required: false, + is_sensitive: false, + supports_expr_lang: false, + default_value: Some("false"), + validator: StandardPropertyValidator::BoolValidator, + allowed_values: &[], + allowed_type: "", +}; + +pub(crate) const MIN_AGE: Property = Property { + name: "Minimum File Age", + description: "The minimum age that a file must be in order to be pulled; any file younger than this amount of time (according to last modification date) will be ignored", + is_required: false, + is_sensitive: false, + supports_expr_lang: false, + default_value: None, + validator: StandardPropertyValidator::TimePeriodValidator, + allowed_values: &[], + allowed_type: "", +}; + +pub(crate) const MAX_AGE: Property = Property { + name: "Maximum File Age", + description: "The maximum age that a file must be in order to be pulled; any file older than this amount of time (according to last modification date) will be ignored", + is_required: false, + is_sensitive: false, + supports_expr_lang: false, + default_value: None, + validator: StandardPropertyValidator::TimePeriodValidator, + allowed_values: &[], + allowed_type: "", +}; + +pub(crate) const MIN_SIZE: Property = Property { + name: "Minimum File Size", + description: "The minimum size that a file can be in order to be pulled", + is_required: false, + is_sensitive: false, + supports_expr_lang: false, + default_value: None, + validator: StandardPropertyValidator::DataSizeValidator, + allowed_values: &[], + allowed_type: "", +}; + +pub(crate) const MAX_SIZE: Property = Property { + name: "Maximum File Size", + description: "The maximum size that a file can be in order to be pulled", + is_required: false, + is_sensitive: false, + supports_expr_lang: false, + default_value: None, + validator: StandardPropertyValidator::DataSizeValidator, + allowed_values: &[], + allowed_type: "", +}; + +pub(crate) const IGNORE_HIDDEN_FILES: Property = Property { + name: "Ignore Hidden Files", + description: "Indicates whether or not hidden files should be ignored", + is_required: true, + is_sensitive: false, + supports_expr_lang: false, + default_value: Some("true"), + validator: StandardPropertyValidator::BoolValidator, + allowed_values: &[], + allowed_type: "", +}; + +pub(crate) const POLLING_INTERVAL: Property = Property { + name: "Polling Interval", + description: "Indicates how long to wait before performing a directory listing", + is_required: false, + is_sensitive: false, + supports_expr_lang: false, + default_value: None, + validator: StandardPropertyValidator::TimePeriodValidator, + allowed_values: &[], + allowed_type: "", +}; + +pub(crate) const BATCH_SIZE: Property = Property { + name: "Batch Size", + description: "The maximum number of files to pull in each iteration", + is_required: true, + is_sensitive: false, + supports_expr_lang: false, + default_value: Some("10"), + validator: StandardPropertyValidator::U64Validator, + allowed_values: &[], + allowed_type: "", +}; diff --git a/minifi_rust/extensions/minifi_rs_playground/src/processors/get_file/relationships.rs b/minifi_rust/extensions/minifi_rs_playground/src/processors/get_file/relationships.rs new file mode 100644 index 0000000000..5f3c286715 --- /dev/null +++ b/minifi_rust/extensions/minifi_rs_playground/src/processors/get_file/relationships.rs @@ -0,0 +1,6 @@ +use minifi_native::Relationship; + +pub(crate) const SUCCESS: Relationship = Relationship { + name: "success", + description: "FlowFiles are transferred here after logging", +}; diff --git a/minifi_rust/extensions/minifi_rs_playground/src/processors/get_file/tests.rs b/minifi_rust/extensions/minifi_rs_playground/src/processors/get_file/tests.rs new file mode 100644 index 0000000000..143fadaf7f --- /dev/null +++ b/minifi_rust/extensions/minifi_rs_playground/src/processors/get_file/tests.rs @@ -0,0 +1,172 @@ +use super::*; +use crate::processors::get_file::relationships::SUCCESS; +use filetime::FileTime; +use minifi_native::{MockLogger, MockProcessContext, MockProcessSession}; +use tempfile::TempDir; + +#[test] +fn schedule_fails_without_input_dir() { + assert!(matches!( + GetFileRs::schedule(&MockProcessContext::new(), &MockLogger::new()) + .err() + .unwrap(), + MinifiError::MissingRequiredProperty(_) + )); +} + +#[test] +fn schedule_fails_with_invalid_input_dir() { + let mut context = MockProcessContext::new(); + context.properties.insert( + "Input Directory".to_string(), + "/invalid_directory".to_string(), + ); + assert!(matches!( + GetFileRs::schedule(&context, &MockLogger::new()), + Err(MinifiError::ScheduleError(_)) + )); +} + +#[test] +fn simple_get_file_test() { + let temp_dir = tempfile::tempdir().expect("temp dir is required for testing GetFile"); + let file_path = temp_dir.path().join("input_file"); + std::fs::write(&file_path, "test").unwrap(); + + let mut context = MockProcessContext::new(); + context.properties.insert( + "Input Directory".to_string(), + temp_dir.path().to_str().unwrap().to_string(), + ); + + let get_file = GetFileRs::schedule(&context, &MockLogger::new()).unwrap(); + + let mut session = MockProcessSession::new(); + get_file + .trigger(&mut context, &mut session, &MockLogger::new()) + .expect("Should succeed"); + assert_eq!(session.num_of_transferred_flow_files(), 1); +} + +fn make_file(temp_dir: &TempDir, file_name: &str, size: usize, age: Duration) { + let path = temp_dir.path().join(file_name); + std::fs::write(&path, "a".repeat(size)).unwrap(); + let file_time = FileTime::from_system_time(SystemTime::now() - age); + filetime::set_file_mtime(path, file_time).expect("Cannot set file time"); +} + +fn create_test_directory() -> TempDir { + let temp_dir = tempfile::tempdir().expect("temp dir is required for testing GetFile"); + make_file(&temp_dir, "small_new", 10, Duration::from_secs(10)); + make_file(&temp_dir, "small_old", 11, Duration::from_secs(3600)); + + make_file(&temp_dir, "large_new", 1000, Duration::from_secs(0)); + make_file(&temp_dir, "large_old", 2000, Duration::from_secs(3600)); + make_file(&temp_dir, ".small_hidden", 10, Duration::from_secs(0)); + temp_dir +} + +#[test] +fn complex_dir_without_filters() { + let test_directory = create_test_directory(); + + let mut context = MockProcessContext::new(); + context.properties.insert( + "Input Directory".to_string(), + test_directory.path().to_str().unwrap().to_string(), + ); + context + .properties + .insert("Batch Size".to_string(), "10".to_string()); + + let mut session = MockProcessSession::new(); + let get_file = GetFileRs::schedule(&context, &MockLogger::new()).unwrap(); + get_file + .trigger(&mut context, &mut session, &MockLogger::new()) + .expect("Should succeed"); + assert_eq!(session.num_of_transferred_flow_files(), 4); +} + +fn test_complex_dir_with_filter( + property_name: &str, + property_vale: &str, + expected_filename_part: &str, +) { + let test_directory = create_test_directory(); + + let mut context = MockProcessContext::new(); + context.properties.insert( + DIRECTORY.name.to_string(), + test_directory.path().to_str().unwrap().to_string(), + ); + context + .properties + .insert(BATCH_SIZE.name.to_string(), "10".to_string()); + + context + .properties + .insert(property_name.to_string(), property_vale.to_string()); + + let mut session = MockProcessSession::new(); + let get_file = GetFileRs::schedule(&context, &MockLogger::new()).unwrap(); + get_file + .trigger(&mut context, &mut session, &MockLogger::new()) + .expect("Should succeed"); + assert_eq!(session.num_of_transferred_flow_files(), 2); + let transferred_flow_files = session.transferred_flow_files.borrow(); + assert!(transferred_flow_files.iter().all(|transfer| { + transfer.relationship == SUCCESS.name + && transfer + .flow_file + .attributes + .get("filename") + .and_then(|filename| Some(filename.contains(expected_filename_part))) + .unwrap_or(false) + })); + let sum_file_len = transferred_flow_files + .iter() + .fold(0, |acc, transfer| acc + transfer.flow_file.content_len()); + + let metrics = get_file.calculate_metrics(); + assert_eq!(metrics.len(), 2); + assert_eq!(metrics[0].0, "accepted_files".to_string()); + assert_eq!(metrics[0].1, 2.0); + assert_eq!(metrics[1].0, "input_bytes".to_string()); + assert_eq!(metrics[1].1, sum_file_len as f64); +} + +#[test] +fn complex_dir_with_filters() { + test_complex_dir_with_filter(MIN_AGE.name, "5 min", "old"); + test_complex_dir_with_filter(MAX_AGE.name, "5 min", "new"); + test_complex_dir_with_filter(MIN_SIZE.name, "50 B", "large"); + test_complex_dir_with_filter(MAX_SIZE.name, "50 B", "small"); +} + +#[test] +fn test_hidden_files_and_batch_size() { + let temp_dir = tempfile::tempdir().expect("temp dir is required for testing GetFile"); + make_file(&temp_dir, ".one", 10, Duration::from_secs(0)); + make_file(&temp_dir, ".two", 10, Duration::from_secs(0)); + make_file(&temp_dir, ".three", 10, Duration::from_secs(0)); + + let mut context = MockProcessContext::new(); + context.properties.insert( + DIRECTORY.name.to_string(), + temp_dir.path().to_str().unwrap().to_string(), + ); + context + .properties + .insert(BATCH_SIZE.name.to_string(), "2".to_string()); + + context + .properties + .insert(IGNORE_HIDDEN_FILES.name.to_string(), "false".to_string()); + + let mut session = MockProcessSession::new(); + let get_file = GetFileRs::schedule(&context, &MockLogger::new()).unwrap(); + get_file + .trigger(&mut context, &mut session, &MockLogger::new()) + .expect("Should succeed"); + assert_eq!(session.num_of_transferred_flow_files(), 2); +} diff --git a/minifi_rust/extensions/minifi_rs_playground/src/processors/kamikaze_processor.rs b/minifi_rust/extensions/minifi_rs_playground/src/processors/kamikaze_processor.rs new file mode 100644 index 0000000000..ad65040ff1 --- /dev/null +++ b/minifi_rust/extensions/minifi_rs_playground/src/processors/kamikaze_processor.rs @@ -0,0 +1,103 @@ +mod properties; +mod relationships; + +use crate::controller_services::lorem_ipsum_controller_service::LoremIpsumControllerService; +use crate::processors::kamikaze_processor::properties::NOT_REGISTERED_PROPERTY; +use minifi_native::macros::{ComponentIdentifier}; +use minifi_native::{ + GetProperty, Logger, MinifiError, OnTriggerResult, ProcessContext, + ProcessSession, Schedule, Trigger, +}; +use strum_macros::{Display, EnumString, IntoStaticStr, VariantNames}; + +#[derive(Debug, Clone, Copy, PartialEq, Display, EnumString, VariantNames, IntoStaticStr)] +#[strum(serialize_all = "PascalCase", const_into_str)] +enum KamikazeBehaviour { + ReturnErr, + ReturnOk, + GetNotRegisteredProperty, + GetInvalidControllerService, + Panic, +} + +#[derive(Debug, ComponentIdentifier)] +pub(crate) struct KamikazeProcessorRs { + on_trigger_behaviour: KamikazeBehaviour, +} + +impl Schedule for KamikazeProcessorRs { + fn schedule(context: &P, _logger: &L) -> Result + where + Self: Sized, + { + let on_trigger_behaviour = context + .get_property(&properties::ON_TRIGGER_BEHAVIOUR)? + .expect("required property") + .parse::()?; + + let on_schedule_behaviour = context + .get_property(&properties::ON_SCHEDULE_BEHAVIOUR)? + .expect("required property") + .parse::()?; + + match on_schedule_behaviour { + KamikazeBehaviour::ReturnErr => Err(MinifiError::schedule_err( + "it was designed to fail during schedule", + )), + KamikazeBehaviour::ReturnOk => Ok(KamikazeProcessorRs { + on_trigger_behaviour, + }), + KamikazeBehaviour::GetNotRegisteredProperty => { + let _ = context.get_property(&NOT_REGISTERED_PROPERTY)?; + Ok(KamikazeProcessorRs { + on_trigger_behaviour, + }) + } + KamikazeBehaviour::Panic => { + panic!("KamikazeProcessor::on_schedule panic") + } + KamikazeBehaviour::GetInvalidControllerService => { + unimplemented!("KamikazeProcessor::get_invalid_controller_service"); + } + } + } +} + +impl Trigger for KamikazeProcessorRs { + fn trigger( + &self, + context: &mut PC, + _session: &mut PS, + _logger: &L, + ) -> Result + where + PC: ProcessContext, + PS: ProcessSession, + L: Logger, + { + match self.on_trigger_behaviour { + KamikazeBehaviour::ReturnErr => Err(MinifiError::trigger_err( + "it was designed to fail in trigger", + )), + KamikazeBehaviour::ReturnOk => Ok(OnTriggerResult::Ok), + KamikazeBehaviour::Panic => { + panic!("KamikazeProcessor::on_trigger panic") + } + KamikazeBehaviour::GetNotRegisteredProperty => { + let _ = context.get_property(&NOT_REGISTERED_PROPERTY, None)?; + Ok(OnTriggerResult::Ok) + } + KamikazeBehaviour::GetInvalidControllerService => { + let _ = context.get_controller_service::( + &NOT_REGISTERED_PROPERTY, + )?; + Ok(OnTriggerResult::Ok) + } + } + } +} + +pub(crate) mod processor_definition; + +#[cfg(test)] +mod tests; diff --git a/minifi_rust/extensions/minifi_rs_playground/src/processors/kamikaze_processor/processor_definition.rs b/minifi_rust/extensions/minifi_rs_playground/src/processors/kamikaze_processor/processor_definition.rs new file mode 100644 index 0000000000..1cbc76634c --- /dev/null +++ b/minifi_rust/extensions/minifi_rs_playground/src/processors/kamikaze_processor/processor_definition.rs @@ -0,0 +1,17 @@ +use super::*; +use minifi_native::{ + OutputAttribute, ProcessorDefinition, ProcessorInputRequirement, Property, Relationship, +}; + +impl ProcessorDefinition for KamikazeProcessorRs { + const DESCRIPTION: &'static str = "This processor can fail or panic in on_trigger and on_schedule calls based on configuration. Only for testing purposes."; + const INPUT_REQUIREMENT: ProcessorInputRequirement = ProcessorInputRequirement::Allowed; + const SUPPORTS_DYNAMIC_PROPERTIES: bool = false; + const SUPPORTS_DYNAMIC_RELATIONSHIPS: bool = false; + const OUTPUT_ATTRIBUTES: &'static [OutputAttribute] = &[]; + const RELATIONSHIPS: &'static [Relationship] = &[relationships::SUCCESS]; + const PROPERTIES: &'static [Property] = &[ + properties::ON_SCHEDULE_BEHAVIOUR, + properties::ON_TRIGGER_BEHAVIOUR, + ]; +} diff --git a/minifi_rust/extensions/minifi_rs_playground/src/processors/kamikaze_processor/properties.rs b/minifi_rust/extensions/minifi_rs_playground/src/processors/kamikaze_processor/properties.rs new file mode 100644 index 0000000000..0f2e550e7e --- /dev/null +++ b/minifi_rust/extensions/minifi_rs_playground/src/processors/kamikaze_processor/properties.rs @@ -0,0 +1,39 @@ +use crate::processors::kamikaze_processor::KamikazeBehaviour; +use minifi_native::{Property, StandardPropertyValidator}; +use strum::VariantNames; + +pub(crate) const ON_SCHEDULE_BEHAVIOUR: Property = Property { + name: "On Schedule Behaviour", + description: "What to do during the on_schedule method", + is_required: true, + is_sensitive: false, + supports_expr_lang: false, + default_value: Some(KamikazeBehaviour::ReturnOk.into_str()), + validator: StandardPropertyValidator::AlwaysValidValidator, + allowed_values: &KamikazeBehaviour::VARIANTS, + allowed_type: "", +}; + +pub(crate) const ON_TRIGGER_BEHAVIOUR: Property = Property { + name: "On Trigger Behaviour", + description: "What to do during the on_trigger method", + is_required: true, + is_sensitive: false, + supports_expr_lang: false, + default_value: Some(KamikazeBehaviour::ReturnOk.into_str()), + validator: StandardPropertyValidator::AlwaysValidValidator, + allowed_values: &KamikazeBehaviour::VARIANTS, + allowed_type: "", +}; + +pub(crate) const NOT_REGISTERED_PROPERTY: Property = Property { + name: "Kamikaze Processor Property", + description: "Property purposely left out of Processor description", + is_required: false, + is_sensitive: false, + supports_expr_lang: false, + default_value: None, + validator: StandardPropertyValidator::AlwaysValidValidator, + allowed_values: &[], + allowed_type: "", +}; diff --git a/minifi_rust/extensions/minifi_rs_playground/src/processors/kamikaze_processor/relationships.rs b/minifi_rust/extensions/minifi_rs_playground/src/processors/kamikaze_processor/relationships.rs new file mode 100644 index 0000000000..1bdfca63f5 --- /dev/null +++ b/minifi_rust/extensions/minifi_rs_playground/src/processors/kamikaze_processor/relationships.rs @@ -0,0 +1,6 @@ +use minifi_native::Relationship; + +pub(crate) const SUCCESS: Relationship = Relationship { + name: "success", + description: "success relationship", +}; diff --git a/minifi_rust/extensions/minifi_rs_playground/src/processors/kamikaze_processor/tests.rs b/minifi_rust/extensions/minifi_rs_playground/src/processors/kamikaze_processor/tests.rs new file mode 100644 index 0000000000..76acbb3e5f --- /dev/null +++ b/minifi_rust/extensions/minifi_rs_playground/src/processors/kamikaze_processor/tests.rs @@ -0,0 +1,83 @@ +use super::*; +use crate::processors::kamikaze_processor::properties::{ + ON_SCHEDULE_BEHAVIOUR, ON_TRIGGER_BEHAVIOUR, +}; +use minifi_native::MinifiError::{ScheduleError, TriggerError}; +use minifi_native::{MockLogger, MockProcessContext, MockProcessSession}; +use std::panic::AssertUnwindSafe; + +#[test] +fn on_schedule_ok() { + let context = MockProcessContext::new(); + let processor = KamikazeProcessorRs::schedule(&context, &MockLogger::new()); + assert!(processor.is_ok()); +} + +#[test] +fn on_schedule_err() { + let mut context = MockProcessContext::new(); + context.properties.insert( + ON_SCHEDULE_BEHAVIOUR.name.to_string(), + "ReturnErr".to_string(), + ); + let processor = KamikazeProcessorRs::schedule(&context, &MockLogger::new()); + assert!(matches!(processor, Err(ScheduleError(_)))); +} + +#[test] +fn on_schedule_panic() { + let mut context = MockProcessContext::new(); + context + .properties + .insert(ON_SCHEDULE_BEHAVIOUR.name.to_string(), "Panic".to_string()); + + let result = std::panic::catch_unwind(AssertUnwindSafe(|| { + KamikazeProcessorRs::schedule(&context, &MockLogger::new()) + })); + assert!(result.is_err()); +} + +#[test] +fn on_trigger_ok() { + let mut context = MockProcessContext::new(); + let processor = KamikazeProcessorRs::schedule(&context, &MockLogger::new()).unwrap(); + + let mut session = MockProcessSession::new(); + assert_eq!( + processor + .trigger(&mut context, &mut session, &MockLogger::new()) + .expect("Should trigger successfully"), + OnTriggerResult::Ok + ); +} + +#[test] +fn on_trigger_err() { + let mut context = MockProcessContext::new(); + context.properties.insert( + ON_TRIGGER_BEHAVIOUR.name.to_string(), + "ReturnErr".to_string(), + ); + let processor = KamikazeProcessorRs::schedule(&context, &MockLogger::new()).unwrap(); + + let mut session = MockProcessSession::new(); + assert!(matches!( + processor.trigger(&mut context, &mut session, &MockLogger::new()), + Err(TriggerError(_)) + )); +} + +#[test] +fn on_trigger_panic() { + let mut context = MockProcessContext::new(); + context + .properties + .insert(ON_TRIGGER_BEHAVIOUR.name.to_string(), "Panic".to_string()); + let processor = KamikazeProcessorRs::schedule(&context, &MockLogger::new()).unwrap(); + + let mut session = MockProcessSession::new(); + let result = std::panic::catch_unwind(AssertUnwindSafe(|| { + processor.trigger(&mut context, &mut session, &MockLogger::new()) + })); + assert!(result.is_err()); +} diff --git a/minifi_rust/extensions/minifi_rs_playground/src/processors/log_attribute.rs b/minifi_rust/extensions/minifi_rs_playground/src/processors/log_attribute.rs new file mode 100644 index 0000000000..b49f7dd442 --- /dev/null +++ b/minifi_rust/extensions/minifi_rs_playground/src/processors/log_attribute.rs @@ -0,0 +1,160 @@ +use crate::processors::log_attribute::properties::{FLOW_FILES_TO_LOG, LOG_LEVEL, LOG_PAYLOAD}; +use minifi_native::macros::{ComponentIdentifier}; +use minifi_native::{ + GetProperty, LogLevel, Logger, MinifiError, OnTriggerResult, ProcessContext, ProcessSession, + Property, Schedule, Trigger, debug, log, trace, +}; + +mod properties; +mod relationships; + +#[derive(Debug, ComponentIdentifier)] +pub(crate) struct LogAttributeRs { + log_level: LogLevel, + attributes_to_log: Option>, + attributes_to_ignore: Option>, + log_payload: bool, + flow_files_to_log: usize, + dash_line: String, + hex_encode_payload: bool, +} + +impl LogAttributeRs { + fn generate_log_message(&self, session: &mut PS, flow_file: &PS::FlowFile) -> String + where + PS: ProcessSession, + { + let mut log_msg = String::with_capacity(1024); + log_msg.push_str("Logging for flow file\n"); + log_msg.push_str(self.dash_line.as_str()); + + log_msg.push_str("\nFlowFile Attributes Map Content"); + session.on_attributes(flow_file, |key, value| { + if let Some(attributes_to_ignore) = &self.attributes_to_ignore { + if attributes_to_ignore.iter().any(|ign| ign == key) { + return; + } + } + if let Some(attributes_to_log) = &self.attributes_to_log { + if !attributes_to_log.iter().any(|ign| ign == key) { + return; + } + } + log_msg.push_str(format!("\nkey:{} value:{}", &key, &value).as_str()); + }); + if self.log_payload { + log_msg.push_str("\nPayload:\n"); + if let Some(flow_file_payload) = session.read(flow_file) { + if self.hex_encode_payload { + log_msg.push_str(&hex::encode(flow_file_payload)); + } else { + log_msg.push_str( + String::from_utf8(flow_file_payload) + .unwrap_or(String::new()) + .as_str(), + ); + } + } + } + log_msg.push_str("\n"); + log_msg.push_str(self.dash_line.as_str()); + log_msg + } +} + +impl Trigger for LogAttributeRs { + fn trigger( + &self, + _context: &mut PC, + session: &mut PS, + logger: &L, + ) -> Result + where + PC: ProcessContext, + PS: ProcessSession, + L: Logger, + { + trace!( + logger, + "enter log attribute, attempting to retrieve {} flow files", self.flow_files_to_log + ); + let max_flow_files_to_process = if self.flow_files_to_log == 0 { + usize::MAX + } else { + self.flow_files_to_log + }; + let mut flow_files_processed = 0usize; + for _ in 0..max_flow_files_to_process { + if let Some(mut flow_file) = session.get() { + let log_msg = self.generate_log_message(session, &mut flow_file); + log!(logger, self.log_level, "{}", log_msg); + session.transfer(flow_file, relationships::SUCCESS.name)?; + flow_files_processed += 1; + } else { + break; + } + } + debug!(logger, "Logged {} flow files", flow_files_processed); + + Ok(OnTriggerResult::Ok) + } +} + +impl Schedule for LogAttributeRs { + fn schedule(context: &P, _logger: &L) -> Result + where + Self: Sized, + { + let log_level = context + .get_property(&LOG_LEVEL)? + .expect("required property") + .parse::()?; + + let log_payload = context + .get_bool_property(&LOG_PAYLOAD)? + .expect("required property"); + + let flow_files_to_log = context + .get_property(&FLOW_FILES_TO_LOG)? + .expect("required property") + .parse::()?; + + fn get_csv_property( + context: &P, + property: &Property, + ) -> Result>, MinifiError> { + Ok(context + .get_property(property)? + .and_then(|s| Some(s.split(",").map(|s| s.to_string()).collect::>()))) + } + + let attributes_to_log = get_csv_property(context, &properties::ATTRIBUTES_TO_LOG)?; + let attributes_to_ignore = get_csv_property(context, &properties::ATTRIBUTES_TO_IGNORE)?; + + let dash_line = format!( + "{:-^50}", + context + .get_property(&properties::LOG_PREFIX)? + .unwrap_or(String::new()) + ); + + let hex_encode_payload = context + .get_bool_property(&properties::HEX_ENCODE_PAYLOAD)? + .expect("required property"); + + Ok(LogAttributeRs { + log_level, + attributes_to_log, + attributes_to_ignore, + log_payload, + flow_files_to_log, + dash_line, + hex_encode_payload, + }) + } +} + +pub(crate) mod processor_definition; + +#[cfg(test)] +mod tests; diff --git a/minifi_rust/extensions/minifi_rs_playground/src/processors/log_attribute/processor_definition.rs b/minifi_rust/extensions/minifi_rs_playground/src/processors/log_attribute/processor_definition.rs new file mode 100644 index 0000000000..0d4c08fdb1 --- /dev/null +++ b/minifi_rust/extensions/minifi_rs_playground/src/processors/log_attribute/processor_definition.rs @@ -0,0 +1,24 @@ +use crate::processors::log_attribute::properties::*; +use crate::processors::log_attribute::{LogAttributeRs, relationships}; +use minifi_native::{ + OutputAttribute, ProcessorDefinition, ProcessorInputRequirement, Property, Relationship, +}; + +impl ProcessorDefinition for LogAttributeRs { + const DESCRIPTION: &'static str = + "Logs attributes of flow files in the MiNiFi application log."; + const INPUT_REQUIREMENT: ProcessorInputRequirement = ProcessorInputRequirement::Required; + const SUPPORTS_DYNAMIC_PROPERTIES: bool = false; + const SUPPORTS_DYNAMIC_RELATIONSHIPS: bool = false; + const OUTPUT_ATTRIBUTES: &'static [OutputAttribute] = &[]; + const RELATIONSHIPS: &'static [Relationship] = &[relationships::SUCCESS]; + const PROPERTIES: &'static [Property] = &[ + LOG_LEVEL, + ATTRIBUTES_TO_LOG, + ATTRIBUTES_TO_IGNORE, + LOG_PAYLOAD, + LOG_PREFIX, + FLOW_FILES_TO_LOG, + HEX_ENCODE_PAYLOAD, + ]; +} diff --git a/minifi_rust/extensions/minifi_rs_playground/src/processors/log_attribute/properties.rs b/minifi_rust/extensions/minifi_rs_playground/src/processors/log_attribute/properties.rs new file mode 100644 index 0000000000..5ec30f72ea --- /dev/null +++ b/minifi_rust/extensions/minifi_rs_playground/src/processors/log_attribute/properties.rs @@ -0,0 +1,86 @@ +use minifi_native::{LogLevel, Property, StandardPropertyValidator}; +use strum::VariantNames; + +pub(crate) const LOG_LEVEL: Property = Property { + name: "Log Level", + description: "The Log Level to use when logging the Attributes", + is_required: true, + is_sensitive: false, + supports_expr_lang: false, + default_value: Some("Info"), // todo! it would be nicer to come from enum value but wasnt able to use into_const_str from another crate + validator: StandardPropertyValidator::AlwaysValidValidator, + allowed_values: &LogLevel::VARIANTS, + allowed_type: "", +}; + +pub(crate) const ATTRIBUTES_TO_LOG: Property = Property { + name: "Attributes to Log", + description: "A comma-separated list of Attributes to Log. If not specified, all attributes will be logged.", + is_required: false, + is_sensitive: false, + supports_expr_lang: false, + default_value: None, + validator: StandardPropertyValidator::AlwaysValidValidator, + allowed_values: &[], + allowed_type: "", +}; + +pub(crate) const ATTRIBUTES_TO_IGNORE: Property = Property { + name: "Attributes to Ignore", + description: "A comma-separated list of Attributes to ignore. If not specified, no attributes will be ignored.", + is_required: false, + is_sensitive: false, + supports_expr_lang: false, + default_value: None, + validator: StandardPropertyValidator::AlwaysValidValidator, + allowed_values: &[], + allowed_type: "", +}; + +pub(crate) const LOG_PAYLOAD: Property = Property { + name: "Log Payload", + description: "If true, the FlowFile's payload will be logged, in addition to its attributes. Otherwise, just the Attributes will be logged.", + is_required: true, + is_sensitive: false, + supports_expr_lang: false, + default_value: Some("false"), + validator: StandardPropertyValidator::BoolValidator, + allowed_values: &[], + allowed_type: "", +}; + +pub(crate) const LOG_PREFIX: Property = Property { + name: "Log Prefix", + description: "Log prefix appended to the log lines. It helps to distinguish the output of multiple LogAttribute processors.", + is_required: false, + is_sensitive: false, + supports_expr_lang: false, + default_value: None, + validator: StandardPropertyValidator::AlwaysValidValidator, + allowed_values: &[], + allowed_type: "", +}; + +pub(crate) const FLOW_FILES_TO_LOG: Property = Property { + name: "FlowFiles To Log", + description: "Number of flow files to log. If set to zero all flow files will be logged. Please note that this may block other threads from running if not used judiciously.", + is_required: true, + is_sensitive: false, + supports_expr_lang: false, + default_value: Some("1"), + validator: StandardPropertyValidator::AlwaysValidValidator, + allowed_values: &[], + allowed_type: "", +}; + +pub(crate) const HEX_ENCODE_PAYLOAD: Property = Property { + name: "Hexencode Payload", + description: "If true, the FlowFile's payload will be logged in a hexencoded format", + is_required: true, + is_sensitive: false, + supports_expr_lang: false, + default_value: Some("false"), + validator: StandardPropertyValidator::BoolValidator, + allowed_values: &[], + allowed_type: "", +}; diff --git a/minifi_rust/extensions/minifi_rs_playground/src/processors/log_attribute/relationships.rs b/minifi_rust/extensions/minifi_rs_playground/src/processors/log_attribute/relationships.rs new file mode 100644 index 0000000000..5f3c286715 --- /dev/null +++ b/minifi_rust/extensions/minifi_rs_playground/src/processors/log_attribute/relationships.rs @@ -0,0 +1,6 @@ +use minifi_native::Relationship; + +pub(crate) const SUCCESS: Relationship = Relationship { + name: "success", + description: "FlowFiles are transferred here after logging", +}; diff --git a/minifi_rust/extensions/minifi_rs_playground/src/processors/log_attribute/tests.rs b/minifi_rust/extensions/minifi_rs_playground/src/processors/log_attribute/tests.rs new file mode 100644 index 0000000000..c87070b9f3 --- /dev/null +++ b/minifi_rust/extensions/minifi_rs_playground/src/processors/log_attribute/tests.rs @@ -0,0 +1,114 @@ +use super::*; +use minifi_native::{ + ComponentIdentifier, MockFlowFile, MockLogger, MockProcessContext, MockProcessSession, +}; + +#[test] +fn test_component_id() { + assert_eq!( + LogAttributeRs::CLASS_NAME, + "minifi_rs_playground::processors::log_attribute::LogAttributeRs" + ); + assert_eq!(LogAttributeRs::GROUP_NAME, "minifi_rs_playground"); + assert_eq!(LogAttributeRs::VERSION, "0.1.0"); +} +#[test] +fn schedule_succeeds_with_default_values() { + assert!(LogAttributeRs::schedule(&MockProcessContext::new(), &MockLogger::new()).is_ok()); +} + +fn tester( + input_flow_files: Vec, + log_level: LogLevel, + properties: Box<[(&str, &str)]>, + expected_log_msg: String, +) { + let logger = MockLogger::new(); + let mut context = MockProcessContext::new(); + for (k, v) in properties { + context.properties.insert(k.to_string(), v.to_string()); + } + + let processor = LogAttributeRs::schedule(&context, &logger).unwrap(); + + let mut session = MockProcessSession::new(); + for flow_file in input_flow_files { + session.input_flow_files.push(flow_file); + } + processor + .trigger(&mut context, &mut session, &logger) + .expect("The on_trigger should succeed"); + + let logs = logger.logs.lock().unwrap(); + assert_eq!(logs[1], (log_level, expected_log_msg)); +} + +#[test] +fn warn_single_log_payload() { + let mut flow_file = MockFlowFile::with_content("Lorem ipsum dolor sit amet, consectetur adipiscing elit. Integer facilisis diam sit amet nisl interdum, vitae interdum arcu viverra. Nam placerat mi in erat pellentesque, at ultrices orci faucibus. Cras sollicitudin iaculis posuere. Sed tempus, dolor nec lacinia suscipit, tellus odio venenatis odio, nec sollicitudin dolor augue non urna. Aliquam tincidunt viverra ipsum eget hendrerit. Suspendisse varius, augue vel fermentum varius, velit elit euismod lacus, a placerat purus est a lacus. Aenean nibh neque, consectetur hendrerit egestas vitae, commodo non quam. Nullam luctus tempor ante, sed tempus quam imperdiet in. Maecenas gravida erat orci, in consequat urna pretium nec. In sodales iaculis magna at vehicula.".as_bytes()); + flow_file + .attributes + .insert(String::from("apple"), String::from("apfel")); + let vec = vec![flow_file]; + + let expected = + "Logging for flow file +-------------------------------------------------- +FlowFile Attributes Map Content +key:apple value:apfel +Payload: +Lorem ipsum dolor sit amet, consectetur adipiscing elit. Integer facilisis diam sit amet nisl interdum, vitae interdum arcu viverra. Nam placerat mi in erat pellentesque, at ultrices orci faucibus. Cras sollicitudin iaculis posuere. Sed tempus, dolor nec lacinia suscipit, tellus odio venenatis odio, nec sollicitudin dolor augue non urna. Aliquam tincidunt viverra ipsum eget hendrerit. Suspendisse varius, augue vel fermentum varius, velit elit euismod lacus, a placerat purus est a lacus. Aenean nibh neque, consectetur hendrerit egestas vitae, commodo non quam. Nullam luctus tempor ante, sed tempus quam imperdiet in. Maecenas gravida erat orci, in consequat urna pretium nec. In sodales iaculis magna at vehicula. +--------------------------------------------------".to_string(); + let properties_set = [ + ("Log Payload", "true"), + ("Hexencode Payload", "false"), + ("Log Level", "Warn"), + ]; + tester(vec, LogLevel::Warn, Box::new(properties_set), expected); +} + +#[test] +fn critical_single_hexencode_payload() { + let mut flow_file = MockFlowFile::with_content("Lorem ipsum dolor sit amet, consectetur adipiscing elit. Integer facilisis diam sit amet nisl interdum, vitae interdum arcu viverra. Nam placerat mi in erat pellentesque, at ultrices orci faucibus. Cras sollicitudin iaculis posuere. Sed tempus, dolor nec lacinia suscipit, tellus odio venenatis odio, nec sollicitudin dolor augue non urna. Aliquam tincidunt viverra ipsum eget hendrerit. Suspendisse varius, augue vel fermentum varius, velit elit euismod lacus, a placerat purus est a lacus. Aenean nibh neque, consectetur hendrerit egestas vitae, commodo non quam. Nullam luctus tempor ante, sed tempus quam imperdiet in. Maecenas gravida erat orci, in consequat urna pretium nec. In sodales iaculis magna at vehicula.".as_bytes()); + flow_file + .attributes + .insert(String::from("apple"), String::from("apfel")); + let vec = vec![flow_file]; + + let expected = + "Logging for flow file +-------------------------------------------------- +FlowFile Attributes Map Content +key:apple value:apfel +Payload: +4c6f72656d20697073756d20646f6c6f722073697420616d65742c20636f6e73656374657475722061646970697363696e6720656c69742e20496e746567657220666163696c69736973206469616d2073697420616d6574206e69736c20696e74657264756d2c20766974616520696e74657264756d206172637520766976657272612e204e616d20706c616365726174206d6920696e20657261742070656c6c656e7465737175652c20617420756c747269636573206f7263692066617563696275732e204372617320736f6c6c696369747564696e20696163756c697320706f73756572652e205365642074656d7075732c20646f6c6f72206e6563206c6163696e69612073757363697069742c2074656c6c7573206f64696f2076656e656e61746973206f64696f2c206e656320736f6c6c696369747564696e20646f6c6f72206175677565206e6f6e2075726e612e20416c697175616d2074696e636964756e74207669766572726120697073756d20656765742068656e6472657269742e2053757370656e6469737365207661726975732c2061756775652076656c206665726d656e74756d207661726975732c2076656c697420656c697420657569736d6f64206c616375732c206120706c616365726174207075727573206573742061206c616375732e2041656e65616e206e696268206e657175652c20636f6e73656374657475722068656e64726572697420656765737461732076697461652c20636f6d6d6f646f206e6f6e207175616d2e204e756c6c616d206c75637475732074656d706f7220616e74652c207365642074656d707573207175616d20696d7065726469657420696e2e204d616563656e617320677261766964612065726174206f7263692c20696e20636f6e7365717561742075726e61207072657469756d206e65632e20496e20736f64616c657320696163756c6973206d61676e61206174207665686963756c612e +--------------------------------------------------".to_string(); + let properties_set = [ + ("Log Payload", "true"), + ("Hexencode Payload", "true"), + ("Log Level", "Critical"), + ]; + tester(vec, LogLevel::Critical, Box::new(properties_set), expected); +} + +#[test] +fn default_level_multiple_attributes() { + let mut flow_file = MockFlowFile::with_content("Lorem ipsum dolor sit amet, consectetur adipiscing elit. Integer facilisis diam sit amet nisl interdum, vitae interdum arcu viverra. Nam placerat mi in erat pellentesque, at ultrices orci faucibus. Cras sollicitudin iaculis posuere. Sed tempus, dolor nec lacinia suscipit, tellus odio venenatis odio, nec sollicitudin dolor augue non urna. Aliquam tincidunt viverra ipsum eget hendrerit. Suspendisse varius, augue vel fermentum varius, velit elit euismod lacus, a placerat purus est a lacus. Aenean nibh neque, consectetur hendrerit egestas vitae, commodo non quam. Nullam luctus tempor ante, sed tempus quam imperdiet in. Maecenas gravida erat orci, in consequat urna pretium nec. In sodales iaculis magna at vehicula.".as_bytes()); + flow_file + .attributes + .insert(String::from("apple"), String::from("apfel")); + flow_file + .attributes + .insert(String::from("pear"), String::from("birne")); + let vec = vec![flow_file]; + + let expected = "Logging for flow file +-------------------------------------------------- +FlowFile Attributes Map Content +key:apple value:apfel +key:pear value:birne +--------------------------------------------------" + .to_string(); + let properties_set = [("Log Payload", "false")]; + tester(vec, LogLevel::Info, Box::new(properties_set), expected); +} diff --git a/minifi_rust/extensions/minifi_rs_playground/src/processors/lorem_ipsum_cs_user.rs b/minifi_rust/extensions/minifi_rs_playground/src/processors/lorem_ipsum_cs_user.rs new file mode 100644 index 0000000000..50e8d9c60e --- /dev/null +++ b/minifi_rust/extensions/minifi_rs_playground/src/processors/lorem_ipsum_cs_user.rs @@ -0,0 +1,75 @@ +mod properties; + +use crate::controller_services::lorem_ipsum_controller_service::LoremIpsumControllerService; +use crate::processors::lorem_ipsum_cs_user::properties::CONTROLLER_SERVICE; +use crate::processors::lorem_ipsum_cs_user::relationships::SUCCESS; +use minifi_native::macros::{ComponentIdentifier}; +use minifi_native::{ + Content, FlowFileSource, GeneratedFlowFile, GetControllerService, GetProperty, Logger, + MinifiError, Schedule, trace, +}; +use std::collections::HashMap; +use strum_macros::{Display, EnumString, IntoStaticStr, VariantNames}; + +#[derive(Debug, Clone, Copy, PartialEq, Display, EnumString, VariantNames, IntoStaticStr)] +#[strum(serialize_all = "PascalCase", const_into_str)] +enum WriteMethod { + Buffer, + Stream, +} + +#[derive(Debug, ComponentIdentifier)] +pub(crate) struct LoremIpsumCSUser { + write_method: WriteMethod, +} + +impl Schedule for LoremIpsumCSUser { + fn schedule(context: &P, _logger: &L) -> Result + where + Self: Sized, + { + let write_method = context + .get_property(&properties::WRITE_METHOD)? + .expect("required property") + .parse::()?; + Ok(Self { write_method }) + } +} + +impl FlowFileSource for LoremIpsumCSUser { + fn generate<'a, Context: GetProperty + GetControllerService, LoggerImpl: Logger>( + &self, + context: &'a mut Context, + logger: &LoggerImpl, + ) -> Result>, MinifiError> { + trace!(logger, "generate call {:?}", self); + let controller_service = context + .get_controller_service::(&CONTROLLER_SERVICE)? + .ok_or(MinifiError::missing_required_property( + "A valid usable controller service is required", + ))?; + match self.write_method { + WriteMethod::Buffer => { + let generated_flow_file = GeneratedFlowFile::new( + &SUCCESS, + Some(Content::from(controller_service.data.clone())), + HashMap::new(), + ); + Ok(vec![generated_flow_file]) + } + WriteMethod::Stream => { + let reader = controller_service.data.as_bytes(); + let content = Content::Stream(Box::new(reader)); + let generated_flow_file = + GeneratedFlowFile::new(&SUCCESS, Some(content), HashMap::new()); + Ok(vec![generated_flow_file]) + } + } + } +} + +pub(crate) mod processor_definition; + +mod relationships; +#[cfg(test)] +mod tests; diff --git a/minifi_rust/extensions/minifi_rs_playground/src/processors/lorem_ipsum_cs_user/processor_definition.rs b/minifi_rust/extensions/minifi_rs_playground/src/processors/lorem_ipsum_cs_user/processor_definition.rs new file mode 100644 index 0000000000..2ba155a80f --- /dev/null +++ b/minifi_rust/extensions/minifi_rs_playground/src/processors/lorem_ipsum_cs_user/processor_definition.rs @@ -0,0 +1,16 @@ +use super::LoremIpsumCSUser; +use super::properties::*; +use crate::processors::lorem_ipsum_cs_user::relationships::SUCCESS; +use minifi_native::{ + OutputAttribute, ProcessorDefinition, ProcessorInputRequirement, Property, Relationship, +}; + +impl ProcessorDefinition for LoremIpsumCSUser { + const DESCRIPTION: &'static str = "Processor to test Controller Service API"; + const INPUT_REQUIREMENT: ProcessorInputRequirement = ProcessorInputRequirement::Forbidden; + const SUPPORTS_DYNAMIC_PROPERTIES: bool = false; + const SUPPORTS_DYNAMIC_RELATIONSHIPS: bool = false; + const OUTPUT_ATTRIBUTES: &'static [OutputAttribute] = &[]; + const RELATIONSHIPS: &'static [Relationship] = &[SUCCESS]; + const PROPERTIES: &'static [Property] = &[CONTROLLER_SERVICE, WRITE_METHOD]; +} diff --git a/minifi_rust/extensions/minifi_rs_playground/src/processors/lorem_ipsum_cs_user/properties.rs b/minifi_rust/extensions/minifi_rs_playground/src/processors/lorem_ipsum_cs_user/properties.rs new file mode 100644 index 0000000000..c18e1a7d01 --- /dev/null +++ b/minifi_rust/extensions/minifi_rs_playground/src/processors/lorem_ipsum_cs_user/properties.rs @@ -0,0 +1,28 @@ +use crate::controller_services::lorem_ipsum_controller_service::LoremIpsumControllerService; +use minifi_native::ComponentIdentifier; +use minifi_native::{Property, StandardPropertyValidator}; +use strum::VariantNames; + +pub(crate) const CONTROLLER_SERVICE: Property = Property { + name: "Lorem Ipsum Controller Service", + description: "Name of the lorem ipsum controller service", + is_required: true, + is_sensitive: false, + supports_expr_lang: false, + default_value: None, + validator: StandardPropertyValidator::AlwaysValidValidator, + allowed_values: &[], + allowed_type: LoremIpsumControllerService::CLASS_NAME, +}; + +pub(crate) const WRITE_METHOD: Property = Property { + name: "Write Method", + description: "Which API to test", + is_required: true, + is_sensitive: false, + supports_expr_lang: false, + default_value: Some(super::WriteMethod::Buffer.into_str()), + validator: StandardPropertyValidator::AlwaysValidValidator, + allowed_values: super::WriteMethod::VARIANTS, + allowed_type: "", +}; diff --git a/minifi_rust/extensions/minifi_rs_playground/src/processors/lorem_ipsum_cs_user/relationships.rs b/minifi_rust/extensions/minifi_rs_playground/src/processors/lorem_ipsum_cs_user/relationships.rs new file mode 100644 index 0000000000..97d0d98286 --- /dev/null +++ b/minifi_rust/extensions/minifi_rs_playground/src/processors/lorem_ipsum_cs_user/relationships.rs @@ -0,0 +1,6 @@ +use minifi_native::Relationship; + +pub(crate) const SUCCESS: Relationship = Relationship { + name: "success", + description: "All flowfile are routed here", +}; diff --git a/minifi_rust/extensions/minifi_rs_playground/src/processors/lorem_ipsum_cs_user/tests.rs b/minifi_rust/extensions/minifi_rs_playground/src/processors/lorem_ipsum_cs_user/tests.rs new file mode 100644 index 0000000000..e1eb6b54dc --- /dev/null +++ b/minifi_rust/extensions/minifi_rs_playground/src/processors/lorem_ipsum_cs_user/tests.rs @@ -0,0 +1,19 @@ +use crate::processors::lorem_ipsum_cs_user::LoremIpsumCSUser; +use minifi_native::{ComponentIdentifier, MockLogger, MockProcessContext, Schedule}; + +#[test] +fn test_ids() { + assert_eq!( + LoremIpsumCSUser::CLASS_NAME, + "minifi_rs_playground::processors::lorem_ipsum_cs_user::LoremIpsumCSUser" + ); + assert_eq!(LoremIpsumCSUser::GROUP_NAME, "minifi_rs_playground"); + assert_eq!(LoremIpsumCSUser::VERSION, "0.1.0"); +} + +#[test] +fn schedules_with_controller() { + let context = MockProcessContext::new(); + let schedule_result = LoremIpsumCSUser::schedule(&context, &MockLogger::new()); + assert!(schedule_result.is_ok()); +} diff --git a/minifi_rust/extensions/minifi_rs_playground/src/processors/mod.rs b/minifi_rust/extensions/minifi_rs_playground/src/processors/mod.rs new file mode 100644 index 0000000000..0a6e3c7271 --- /dev/null +++ b/minifi_rust/extensions/minifi_rs_playground/src/processors/mod.rs @@ -0,0 +1,9 @@ +pub(crate) mod asciify_german; +pub(crate) mod count_actual_logging; +pub(crate) mod duplicate_text; +pub(crate) mod generate_flow_file; +pub(crate) mod get_file; +pub(crate) mod kamikaze_processor; +pub(crate) mod log_attribute; +pub(crate) mod lorem_ipsum_cs_user; +pub(crate) mod put_file; diff --git a/minifi_rust/extensions/minifi_rs_playground/src/processors/put_file.rs b/minifi_rust/extensions/minifi_rs_playground/src/processors/put_file.rs new file mode 100644 index 0000000000..973e790e32 --- /dev/null +++ b/minifi_rust/extensions/minifi_rs_playground/src/processors/put_file.rs @@ -0,0 +1,239 @@ +use crate::processors::put_file::relationships::{FAILURE, SUCCESS}; +use minifi_native::macros::{ComponentIdentifier}; +use minifi_native::{ + FlowFileTransform, GetAttribute, GetControllerService, GetProperty, InputStream, Logger, + MinifiError, Schedule, TransformedFlowFile, trace, warn, +}; +use std::path::{Path, PathBuf}; +use strum_macros::{Display, EnumString, IntoStaticStr, VariantNames}; +use walkdir::WalkDir; + +mod properties; +mod relationships; +#[cfg(unix)] +mod unix_only_properties; + +#[derive(Debug, Clone, Copy, PartialEq, Display, EnumString, VariantNames, IntoStaticStr)] +#[strum(serialize_all = "camelCase", const_into_str)] +enum ConflictResolutionStrategy { + Fail, + Replace, + Ignore, +} + +#[cfg(unix)] +#[derive(Debug)] +struct PutFileUnixPermissions { + file_permissions: Option, + directory_permissions: Option, +} + +#[cfg(unix)] +impl PutFileUnixPermissions { + fn set_directory_permissions(&self, path: &Path) -> std::io::Result<()> { + if let Some(permissions) = self.directory_permissions.as_ref().map(|p| p.clone()) { + return std::fs::set_permissions(path, permissions); + } + Ok(()) + } + + fn set_file_permissions(&self, file: &Path) -> std::io::Result<()> { + if let Some(permissions) = self.file_permissions.as_ref().map(|p| p.clone()) { + return std::fs::set_permissions(file, permissions); + } + Ok(()) + } +} + +#[cfg(windows)] +#[derive(Debug)] +struct PutFileUnixPermissions {} + +#[cfg(windows)] +impl PutFileUnixPermissions { + fn set_directory_permissions(&self, _path: &Path) -> std::io::Result<()> { + Ok(()) + } + + fn set_file_permissions(&self, _file: &Path) -> std::io::Result<()> { + Ok(()) + } +} + +#[derive(Debug, ComponentIdentifier)] +pub(crate) struct PutFileRs { + conflict_resolution_strategy: ConflictResolutionStrategy, + try_make_dirs: bool, + maximum_file_count: Option, + unix_permissions: PutFileUnixPermissions, +} + +impl PutFileRs { + pub(crate) fn directory_is_full(&self, p0: &Path) -> bool { + if let Some(max_file_count) = self.maximum_file_count + && let Some(parent) = p0.parent() + { + parent.exists() + && WalkDir::new(parent) + .into_iter() + .filter_map(Result::ok) + .filter(|e| e.file_type().is_file()) + .count() + >= max_file_count as usize + } else { + false + } + } + + fn get_destination_path(context: &Ctx) -> Result + where + Ctx: GetProperty + GetAttribute, + { + let directory = context + .get_property(&properties::DIRECTORY)? + .expect("required property"); + + let file_name = context + .get_attribute("filename")? + .unwrap_or("foo.txt".to_string()); // TODO fallback to UUID + Ok(PathBuf::from(directory + "/" + file_name.as_str())) + } + + fn prepare_destination(&self, destination: &Path) -> std::io::Result<()> { + if let Some(parent) = destination.parent() { + if self.try_make_dirs { + std::fs::create_dir_all(parent)?; + self.unix_permissions.set_directory_permissions(parent)?; + } + } + Ok(()) + } + + fn put_file( + &self, + input_stream: &mut dyn InputStream, + logger: &L, + destination: &Path, + ) -> Result<(), MinifiError> + where + L: Logger, + { + match self.prepare_destination(destination) { + Ok(_) => {} + Err(err) => { + warn!(logger, "Failed to prepare destination due to {:?}", err); + } + } + let mut file = std::fs::File::create(destination)?; + std::io::copy(input_stream, &mut file)?; + match self.unix_permissions.set_file_permissions(destination) { + Ok(_) => {} + Err(err) => { + warn!(logger, "Failed to set file permissions due to {:?}", err); + } + } + Ok(()) + } + + #[cfg(unix)] + fn parse_unix_permissions( + context: &P, + ) -> Result { + use std::os::unix::fs::PermissionsExt; + let parse_permission = + |property: &minifi_native::Property| -> Result, MinifiError> { + Ok(context + .get_property(&property)? + .map(|perm_str| u32::from_str_radix(&perm_str, 8)) + .transpose()? + .map(|perm| std::fs::Permissions::from_mode(perm))) + }; + let file_permissions = parse_permission(&unix_only_properties::PERMISSIONS)?; + let directory_permissions = parse_permission(&unix_only_properties::DIRECTORY_PERMISSIONS)?; + + Ok(PutFileUnixPermissions { + file_permissions, + directory_permissions, + }) + } + + #[cfg(windows)] + fn parse_unix_permissions( + _context: &P, + ) -> Result { + Ok(PutFileUnixPermissions {}) + } +} + +impl Schedule for PutFileRs { + fn schedule(context: &P, _logger: &L) -> Result { + let conflict_resolution_strategy = context + .get_property(&properties::CONFLICT_RESOLUTION)? + .expect("required property") + .parse::()?; + + let try_make_dirs = context + .get_bool_property(&properties::CREATE_DIRS)? + .expect("required property"); + + let maximum_file_count = context.get_u64_property(&properties::MAX_FILE_COUNT)?; + + let unix_permissions = Self::parse_unix_permissions(context)?; + + Ok(PutFileRs { + conflict_resolution_strategy, + try_make_dirs, + maximum_file_count, + unix_permissions, + }) + } +} + +impl FlowFileTransform for PutFileRs { + fn transform< + 'a, + Context: GetProperty + GetControllerService + GetAttribute, + LoggerImpl: Logger, + >( + &self, + context: &Context, + input_stream: &'a mut dyn InputStream, + logger: &LoggerImpl, + ) -> Result, MinifiError> { + trace!(logger, "on_trigger: {:?}", self); + + let Ok(destination_path) = Self::get_destination_path(context) else { + warn!(logger, "Invalid destination path"); + return Ok(TransformedFlowFile::route_without_changes(&FAILURE)); + }; + + if self.directory_is_full(&destination_path) { + warn!(logger, "Directory is full"); + return Ok(TransformedFlowFile::route_without_changes(&FAILURE)); + } + + if destination_path.exists() { + match self.conflict_resolution_strategy { + ConflictResolutionStrategy::Fail => { + return Ok(TransformedFlowFile::route_without_changes(&FAILURE)); + } + ConflictResolutionStrategy::Replace => { + // continue with PutFile operation + } + ConflictResolutionStrategy::Ignore => { + return Ok(TransformedFlowFile::route_without_changes(&SUCCESS)); + } + } + } + + match self.put_file(input_stream, logger, &destination_path) { + Ok(_) => Ok(TransformedFlowFile::route_without_changes(&SUCCESS)), + Err(_e) => Ok(TransformedFlowFile::route_without_changes(&FAILURE)), + } + } +} + +pub(crate) mod processor_definition; + +#[cfg(test)] +mod tests; diff --git a/minifi_rust/extensions/minifi_rs_playground/src/processors/put_file/processor_definition.rs b/minifi_rust/extensions/minifi_rs_playground/src/processors/put_file/processor_definition.rs new file mode 100644 index 0000000000..245782a870 --- /dev/null +++ b/minifi_rust/extensions/minifi_rs_playground/src/processors/put_file/processor_definition.rs @@ -0,0 +1,36 @@ +use super::*; +use minifi_native::{ + OutputAttribute, ProcessorDefinition, ProcessorInputRequirement, Property, Relationship, +}; + +#[cfg(windows)] +const fn get_properties() -> &'static [Property] { + &[ + properties::DIRECTORY, + properties::CONFLICT_RESOLUTION, + properties::CREATE_DIRS, + properties::MAX_FILE_COUNT, + ] +} + +#[cfg(unix)] +const fn get_properties() -> &'static [Property] { + &[ + properties::DIRECTORY, + properties::CONFLICT_RESOLUTION, + properties::CREATE_DIRS, + properties::MAX_FILE_COUNT, + unix_only_properties::PERMISSIONS, + unix_only_properties::DIRECTORY_PERMISSIONS, + ] +} + +impl ProcessorDefinition for PutFileRs { + const DESCRIPTION: &'static str = "Writes the contents of a FlowFile to the local file system."; + const INPUT_REQUIREMENT: ProcessorInputRequirement = ProcessorInputRequirement::Required; + const SUPPORTS_DYNAMIC_PROPERTIES: bool = false; + const SUPPORTS_DYNAMIC_RELATIONSHIPS: bool = false; + const OUTPUT_ATTRIBUTES: &'static [OutputAttribute] = &[]; + const RELATIONSHIPS: &'static [Relationship] = &[SUCCESS, FAILURE]; + const PROPERTIES: &'static [Property] = get_properties(); +} diff --git a/minifi_rust/extensions/minifi_rs_playground/src/processors/put_file/properties.rs b/minifi_rust/extensions/minifi_rs_playground/src/processors/put_file/properties.rs new file mode 100644 index 0000000000..91953868c9 --- /dev/null +++ b/minifi_rust/extensions/minifi_rs_playground/src/processors/put_file/properties.rs @@ -0,0 +1,51 @@ +use minifi_native::{Property, StandardPropertyValidator}; +use strum::VariantNames; + +use super::ConflictResolutionStrategy; +pub(crate) const DIRECTORY: Property = Property { + name: "Directory", + description: "The output directory to which to put files", + is_required: true, + is_sensitive: false, + supports_expr_lang: true, + default_value: Some("."), + validator: StandardPropertyValidator::NonBlankValidator, + allowed_values: &[], + allowed_type: "", +}; + +pub(crate) const CONFLICT_RESOLUTION: Property = Property { + name: "Conflict Resolution Strategy", + description: "Indicates what should happen when a file with the same name already exists in the output directory", + is_required: true, + is_sensitive: false, + supports_expr_lang: false, + default_value: Some(ConflictResolutionStrategy::Fail.into_str()), + validator: StandardPropertyValidator::AlwaysValidValidator, + allowed_values: &ConflictResolutionStrategy::VARIANTS, + allowed_type: "", +}; + +pub(crate) const CREATE_DIRS: Property = Property { + name: "Create Missing Directories", + description: "If true, then missing destination directories will be created. If false, flowfiles are penalized and sent to failure.", + is_required: true, + is_sensitive: false, + supports_expr_lang: false, + default_value: Some("true"), + validator: StandardPropertyValidator::BoolValidator, + allowed_values: &[], + allowed_type: "", +}; + +pub(crate) const MAX_FILE_COUNT: Property = Property { + name: "Maximum File Count", + description: "Specifies the maximum number of files that can exist in the output directory", + is_required: false, + is_sensitive: false, + supports_expr_lang: false, + default_value: None, // Diverged from the original implementation, u64 with no default describes the behavior better + validator: StandardPropertyValidator::U64Validator, + allowed_values: &[], + allowed_type: "", +}; diff --git a/minifi_rust/extensions/minifi_rs_playground/src/processors/put_file/relationships.rs b/minifi_rust/extensions/minifi_rs_playground/src/processors/put_file/relationships.rs new file mode 100644 index 0000000000..a3387b65b1 --- /dev/null +++ b/minifi_rust/extensions/minifi_rs_playground/src/processors/put_file/relationships.rs @@ -0,0 +1,11 @@ +use minifi_native::Relationship; + +pub(crate) const SUCCESS: Relationship = Relationship { + name: "success", + description: "Flowfiles that are successfully written to a file are routed to this relationship", +}; + +pub(crate) const FAILURE: Relationship = Relationship { + name: "failure", + description: "Failed files (conflict, write failure, etc.) are transferred to failure", +}; diff --git a/minifi_rust/extensions/minifi_rs_playground/src/processors/put_file/tests.rs b/minifi_rust/extensions/minifi_rs_playground/src/processors/put_file/tests.rs new file mode 100644 index 0000000000..076d8d270d --- /dev/null +++ b/minifi_rust/extensions/minifi_rs_playground/src/processors/put_file/tests.rs @@ -0,0 +1,143 @@ +use super::*; +use crate::processors::put_file::relationships::{FAILURE, SUCCESS}; +use minifi_native::{MockLogger, MockProcessContext}; + +#[test] +fn schedule_succeeds_with_default_values() { + assert!(PutFileRs::schedule(&MockProcessContext::new(), &MockLogger::new()).is_ok()); +} + +#[test] +fn simple_put_file_test() { + let mut context = MockProcessContext::new(); + let temp_dir = tempfile::tempdir().expect("temp dir is required for testing PutFile"); + let put_file_dir = temp_dir.path().join("subdir"); + + context.properties.insert( + "Directory".to_string(), + put_file_dir.to_str().unwrap().to_string(), + ); + let put_file = PutFileRs::schedule(&context, &MockLogger::new()).expect("Should succeed"); + + let mut input_stream = std::io::Cursor::new("test".as_bytes()); + context + .attributes + .insert("filename".to_string(), "test.txt".to_string()); + let result = put_file + .transform(&context, &mut input_stream, &MockLogger::new()) + .expect("Should succeed"); + + assert_eq!(result.target_relationship(), SUCCESS.name); + + let expected_path = temp_dir.path().join("subdir/test.txt"); + assert!(expected_path.exists()); + assert_eq!(std::fs::read_to_string(expected_path).unwrap(), "test"); +} + +#[test] +fn put_file_without_create_dirs() { + let mut context = MockProcessContext::new(); + let temp_dir = tempfile::tempdir().expect("temp dir is required for testing PutFile"); + + let put_file_dir = temp_dir.path().join("subdir"); + + context.properties.insert( + "Directory".to_string(), + put_file_dir.to_str().unwrap().to_string(), + ); + + context.properties.insert( + "Create Missing Directories".to_string(), + "false".to_string(), + ); + + let put_file = PutFileRs::schedule(&context, &MockLogger::new()).expect("Should succeed"); + + let mut input_stream = std::io::Cursor::new("test".as_bytes()); + context + .attributes + .insert("filename".to_string(), "test.txt".to_string()); + let result = put_file + .transform(&context, &mut input_stream, &MockLogger::new()) + .expect("Should succeed"); + + assert_eq!(result.target_relationship(), FAILURE.name); + + let expected_path = temp_dir.path().join("subdir/test.txt"); + assert!(!expected_path.exists()); +} + +#[test] +fn directory_is_full_counts_only_files() { + let mut context = MockProcessContext::new(); + let temp_dir = tempfile::tempdir().expect("temp dir is required for testing PutFile"); + + context.properties.insert( + "Directory".to_string(), + temp_dir.path().to_str().unwrap().to_string(), + ); + context + .properties + .insert("Maximum File Count".to_string(), "2".to_string()); + + let put_file = PutFileRs::schedule(&context, &MockLogger::new()).expect("Should succeed"); + + let destination = temp_dir.path().join("test.txt"); + + // No files yet → not full + assert!(!put_file.directory_is_full(&destination)); + + // Create a subdirectory; it must not be counted as a file + std::fs::create_dir(temp_dir.path().join("subdir")).unwrap(); + assert!(!put_file.directory_is_full(&destination)); + + // Add two files → full + std::fs::write(temp_dir.path().join("a.txt"), b"a").unwrap(); + std::fs::write(temp_dir.path().join("b.txt"), b"b").unwrap(); + assert!(put_file.directory_is_full(&destination)); + + // Remove one file → not full again + std::fs::remove_file(temp_dir.path().join("a.txt")).unwrap(); + assert!(!put_file.directory_is_full(&destination)); +} + +#[cfg(unix)] +#[test] +fn put_file_test_permissions() { + use std::os::unix::fs::PermissionsExt; + let mut context = MockProcessContext::new(); + let temp_dir = tempfile::tempdir().expect("temp dir is required for testing PutFile"); + let put_file_dir = temp_dir.path().join("subdir"); + + context.properties.insert( + "Directory".to_string(), + put_file_dir.to_str().unwrap().to_string(), + ); + + context + .properties + .insert("Directory Permissions".to_string(), "0777".to_string()); + + context + .properties + .insert("Permissions".to_string(), "0777".to_string()); + let put_file = PutFileRs::schedule(&context, &MockLogger::new()).expect("Should succeed"); + + let mut input_stream = std::io::Cursor::new("test".as_bytes()); + context + .attributes + .insert("filename".to_string(), "test.txt".to_string()); + let result = put_file + .transform(&context, &mut input_stream, &MockLogger::new()) + .expect("Should succeed"); + + assert_eq!(result.target_relationship(), SUCCESS.name); + + let expected_path = temp_dir.path().join("subdir/test.txt"); + assert!(expected_path.exists()); + assert_eq!(std::fs::read_to_string(&expected_path).unwrap(), "test"); + let parent_permissions = std::fs::metadata(put_file_dir).unwrap().permissions(); + let permissions = expected_path.metadata().unwrap().permissions(); + assert_eq!(permissions.mode(), 0o100777); + assert_eq!(parent_permissions.mode(), 0o40777); +} diff --git a/minifi_rust/extensions/minifi_rs_playground/src/processors/put_file/unix_only_properties.rs b/minifi_rust/extensions/minifi_rs_playground/src/processors/put_file/unix_only_properties.rs new file mode 100644 index 0000000000..8e4ba1cc6e --- /dev/null +++ b/minifi_rust/extensions/minifi_rs_playground/src/processors/put_file/unix_only_properties.rs @@ -0,0 +1,25 @@ +use minifi_native::{Property, StandardPropertyValidator}; + +pub(crate) const PERMISSIONS: Property = Property { + name: "Permissions", + description: "Sets the permissions on the output file to the value of this attribute. Must be an octal number (e.g. 644 or 0755). Not supported on Windows systems.", + is_required: false, + is_sensitive: false, + supports_expr_lang: false, + default_value: None, + validator: StandardPropertyValidator::AlwaysValidValidator, + allowed_values: &[], + allowed_type: "", +}; + +pub(crate) const DIRECTORY_PERMISSIONS: Property = Property { + name: "Directory Permissions", + description: "Sets the permissions on the directories being created if 'Create Missing Directories' property is set. Must be an octal number (e.g. 644 or 0755). Not supported on Windows systems.", + is_required: false, + is_sensitive: false, + supports_expr_lang: false, + default_value: None, + validator: StandardPropertyValidator::AlwaysValidValidator, + allowed_values: &[], + allowed_type: "", +}; diff --git a/minifi_rust/generate_docs/generate_docs.dockerfile b/minifi_rust/generate_docs/generate_docs.dockerfile new file mode 100644 index 0000000000..3d07516176 --- /dev/null +++ b/minifi_rust/generate_docs/generate_docs.dockerfile @@ -0,0 +1,29 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +ARG BASE_IMAGE="apacheminificpp:behave" + +FROM ${BASE_IMAGE} AS builder +LABEL maintainer="Martin Zink " + +COPY ./target/release/libminifi_rs_playground.so /opt/minifi/minifi-current/extensions + +RUN /opt/minifi/minifi-current/bin/minifi --docs DOCS_OUTPUT + +FROM scratch AS docs-export + +COPY --from=builder /opt/minifi/minifi-current/DOCS_OUTPUT / diff --git a/minifi_rust/generate_docs/generate_docs.sh b/minifi_rust/generate_docs/generate_docs.sh new file mode 100755 index 0000000000..a6fd08cc2a --- /dev/null +++ b/minifi_rust/generate_docs/generate_docs.sh @@ -0,0 +1,29 @@ +#!/usr/bin/env bash +set -e + +PROJECT_ROOT="$(cd "$(dirname "$0")/.." && pwd)" +cd "$PROJECT_ROOT" + +echo "$PROJECT_ROOT" + +DOCKERFILE="generate_docs/generate_docs.dockerfile" +TARGET_DIR="target/docs" + +mkdir -p "$TARGET_DIR" + +docker buildx build \ + -f "$DOCKERFILE" \ + --target docs-export \ + --output type=local,dest="$TARGET_DIR" \ + . + +echo "Build complete. Artifacts updated in $TARGET_DIR" + +for filepath in target/docs/modules/*.md; do + filename=$(basename "$filepath") + dirname="${filename%.md}" + if [ -d "extensions/$dirname" ]; then + echo "Copying $filename to extensions/$dirname/" + cp "$filepath" "extensions/$dirname/" + fi +done diff --git a/minifi_rust/minifi_native/.gitignore b/minifi_rust/minifi_native/.gitignore new file mode 100644 index 0000000000..a9d37c560c --- /dev/null +++ b/minifi_rust/minifi_native/.gitignore @@ -0,0 +1,2 @@ +target +Cargo.lock diff --git a/minifi_rust/minifi_native/Cargo.toml b/minifi_rust/minifi_native/Cargo.toml new file mode 100644 index 0000000000..5998c4fd32 --- /dev/null +++ b/minifi_rust/minifi_native/Cargo.toml @@ -0,0 +1,15 @@ +[package] +name = "minifi_native" +version = "0.1.0" +edition = "2024" +description = "A safe, idiomatic Rust wrapper for the Apache MiNiFi Native C API" +license = "Apache-2.0" + +[dependencies] +minifi_native_sys = { path = "../minifi_native_sys" } +minifi_native_macros = { path = "../minifi_native_macros" } +strum = "0.28.0" +strum_macros = "0.28.0" +humantime = "2.3.0" +byte-unit = "5.1.6" +itertools = "0.14.0" diff --git a/minifi_rust/minifi_native/src/api.rs b/minifi_rust/minifi_native/src/api.rs new file mode 100644 index 0000000000..7413ddddeb --- /dev/null +++ b/minifi_rust/minifi_native/src/api.rs @@ -0,0 +1,25 @@ +pub(crate) mod attribute; +pub(crate) mod component_definition_traits; +pub(crate) mod controller_service; +pub(crate) mod errors; +mod flow_file; +pub(crate) mod logger; +mod process_context; +pub(crate) mod process_session; +pub(crate) mod processor; +pub(crate) mod processor_wrappers; +pub(crate) mod property; +pub(crate) mod raw_controller_service; +pub(crate) mod raw_processor; +mod relationship; + +pub use flow_file::FlowFile; +pub use logger::{LogLevel, Logger}; +pub use process_context::ProcessContext; +pub use process_session::{InputStream, OutputStream, ProcessSession}; +pub use raw_controller_service::RawControllerService; +pub use raw_processor::{OnTriggerResult, ProcessorInputRequirement, RawProcessor, ThreadingModel}; + +pub use property::StandardPropertyValidator; + +pub use relationship::Relationship; diff --git a/minifi_rust/minifi_native/src/api/attribute.rs b/minifi_rust/minifi_native/src/api/attribute.rs new file mode 100644 index 0000000000..f80b9eff42 --- /dev/null +++ b/minifi_rust/minifi_native/src/api/attribute.rs @@ -0,0 +1,11 @@ +use crate::MinifiError; + +pub struct OutputAttribute { + pub name: &'static str, + pub relationships: &'static [&'static str], + pub description: &'static str, +} + +pub trait GetAttribute { + fn get_attribute(&self, name: &str) -> Result, MinifiError>; +} diff --git a/minifi_rust/minifi_native/src/api/component_definition_traits.rs b/minifi_rust/minifi_native/src/api/component_definition_traits.rs new file mode 100644 index 0000000000..671997e5fa --- /dev/null +++ b/minifi_rust/minifi_native/src/api/component_definition_traits.rs @@ -0,0 +1,22 @@ +use crate::{OutputAttribute, ProcessorInputRequirement, Property, Relationship}; + +pub trait ComponentIdentifier { + const CLASS_NAME: &'static str; + const GROUP_NAME: &'static str; + const VERSION: &'static str; +} + +pub trait ProcessorDefinition { + const DESCRIPTION: &'static str; + const INPUT_REQUIREMENT: ProcessorInputRequirement; + const SUPPORTS_DYNAMIC_PROPERTIES: bool; + const SUPPORTS_DYNAMIC_RELATIONSHIPS: bool; + const OUTPUT_ATTRIBUTES: &'static [OutputAttribute]; + const RELATIONSHIPS: &'static [Relationship]; + const PROPERTIES: &'static [Property]; +} + +pub trait ControllerServiceDefinition { + const DESCRIPTION: &'static str; + const PROPERTIES: &'static [Property]; +} diff --git a/minifi_rust/minifi_native/src/api/controller_service.rs b/minifi_rust/minifi_native/src/api/controller_service.rs new file mode 100644 index 0000000000..875dc02766 --- /dev/null +++ b/minifi_rust/minifi_native/src/api/controller_service.rs @@ -0,0 +1,62 @@ +use crate::api::RawControllerService; +use crate::{ComponentIdentifier, GetProperty, LogLevel, Logger, MinifiError}; + +pub trait EnableControllerService { + fn enable(context: &Ctx, logger: &L) -> Result + where + Self: Sized; +} + +#[derive(Debug)] +pub struct ControllerService +where + Implementation: EnableControllerService + ComponentIdentifier, + L: Logger, +{ + logger: L, + enabled_impl: Option, +} + +impl ControllerService +where + Implementation: EnableControllerService + ComponentIdentifier, + L: Logger, +{ + pub fn get_implementation(&self) -> Option<&Implementation> { + self.enabled_impl.as_ref() + } +} + +impl RawControllerService for ControllerService +where + Implementation: EnableControllerService + ComponentIdentifier, + L: Logger, +{ + type LoggerType = L; + + fn new(logger: Self::LoggerType) -> Self { + Self { + logger, + enabled_impl: None, + } + } + + fn log(&self, log_level: LogLevel, args: std::fmt::Arguments) { + self.logger.log(log_level, args); + } + + fn enable(&mut self, context: &P) -> Result<(), MinifiError> { + self.enabled_impl = Some(Implementation::enable(context, &self.logger)?); + Ok(()) + } +} + +impl ComponentIdentifier for ControllerService +where + Implementation: EnableControllerService + ComponentIdentifier, + L: Logger, +{ + const CLASS_NAME: &'static str = Implementation::CLASS_NAME; + const GROUP_NAME: &'static str = Implementation::GROUP_NAME; + const VERSION: &'static str = Implementation::VERSION; +} diff --git a/minifi_rust/minifi_native/src/api/errors.rs b/minifi_rust/minifi_native/src/api/errors.rs new file mode 100644 index 0000000000..dbe68ea31c --- /dev/null +++ b/minifi_rust/minifi_native/src/api/errors.rs @@ -0,0 +1,128 @@ +use minifi_native_sys::minifi_status; +use std::borrow::Cow; +use std::ffi::NulError; +use std::fmt; +use std::num::{NonZeroU32, ParseIntError}; +use std::str::ParseBoolError; + +#[derive(Debug, Clone)] +pub enum ParseError { + Strum(strum::ParseError), + Bool(ParseBoolError), + Int(ParseIntError), + Duration(humantime::DurationError), + Size(byte_unit::ParseError), + Nul(NulError), + Other, +} + +#[derive(Debug)] +pub enum MinifiError { + UnknownError, + StatusError((Cow<'static, str>, NonZeroU32)), + MissingRequiredProperty(Cow<'static, str>), + ControllerServiceError(Cow<'static, str>), + ValidationError(Cow<'static, str>), + ScheduleError(Cow<'static, str>), + TriggerError(Cow<'static, str>), + Parse(ParseError), + IoError(std::io::Error), +} + +impl From for MinifiError { + fn from(error: std::io::Error) -> Self { + MinifiError::IoError(error) + } +} + +impl From for MinifiError { + fn from(err: strum::ParseError) -> Self { + MinifiError::Parse(ParseError::Strum(err)) + } +} + +impl From for MinifiError { + fn from(err: ParseBoolError) -> Self { + MinifiError::Parse(ParseError::Bool(err)) + } +} + +impl From for MinifiError { + fn from(err: ParseIntError) -> Self { + MinifiError::Parse(ParseError::Int(err)) + } +} + +impl From for MinifiError { + fn from(err: humantime::DurationError) -> Self { + MinifiError::Parse(ParseError::Duration(err)) + } +} + +impl From for MinifiError { + fn from(err: byte_unit::ParseError) -> Self { + MinifiError::Parse(ParseError::Size(err)) + } +} + +impl From for MinifiError { + fn from(err: NulError) -> Self { + MinifiError::Parse(ParseError::Nul(err)) + } +} + +impl MinifiError { + pub(crate) fn to_status(&self) -> minifi_status { + // TODO expand this + minifi_native_sys::minifi_status_MINIFI_STATUS_UNKNOWN_ERROR + } + + pub fn validation_err>>(msg: S) -> Self { + MinifiError::ValidationError(msg.into()) + } + + pub fn schedule_err>>(msg: S) -> Self { + MinifiError::ScheduleError(msg.into()) + } + + pub fn trigger_err>>(msg: S) -> Self { + MinifiError::TriggerError(msg.into()) + } + + pub fn missing_required_property>>(msg: S) -> Self { + MinifiError::MissingRequiredProperty(msg.into()) + } + + pub fn controller_service_err>>(msg: S) -> Self { + MinifiError::ControllerServiceError(msg.into()) + } +} + +impl fmt::Display for MinifiError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + MinifiError::StatusError((context, code)) => match code.get() { + minifi_native_sys::minifi_status_MINIFI_STATUS_UNKNOWN_ERROR => { + write!(f, "{}, unknown error", context) + } + minifi_native_sys::minifi_status_MINIFI_STATUS_NOT_SUPPORTED_PROPERTY => { + write!(f, "{}, not supported property", context) + } + minifi_native_sys::minifi_status_MINIFI_STATUS_DYNAMIC_PROPERTIES_NOT_SUPPORTED => { + write!(f, "{}, dynamic properties not supported", context) + } + minifi_native_sys::minifi_status_MINIFI_STATUS_PROPERTY_NOT_SET => { + write!(f, "{}, property not set", context) + } + minifi_native_sys::minifi_status_MINIFI_STATUS_VALIDATION_FAILED => { + write!(f, "{}, validation failed", context) + } + minifi_native_sys::minifi_status_MINIFI_STATUS_PROCESSOR_YIELD => { + write!(f, "{}, processor yield", context) + } + _ => write!(f, "{} (Unknown Status Code: {})", context, code), + }, + _ => write!(f, "{:?}", self), + } + } +} diff --git a/minifi_rust/minifi_native/src/api/flow_file.rs b/minifi_rust/minifi_native/src/api/flow_file.rs new file mode 100644 index 0000000000..24c441e1df --- /dev/null +++ b/minifi_rust/minifi_native/src/api/flow_file.rs @@ -0,0 +1 @@ +pub trait FlowFile {} diff --git a/minifi_rust/minifi_native/src/api/logger.rs b/minifi_rust/minifi_native/src/api/logger.rs new file mode 100644 index 0000000000..a7c3fbfacb --- /dev/null +++ b/minifi_rust/minifi_native/src/api/logger.rs @@ -0,0 +1,95 @@ +use std::fmt; +use std::fmt::Debug; + +use strum_macros::{Display, EnumString, VariantNames}; + +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Display, EnumString, VariantNames)] +#[strum(serialize_all = "PascalCase", const_into_str)] +pub enum LogLevel { + Trace, + Debug, + Info, + Warn, + Error, + Critical, + Off, +} + +pub trait Logger: Debug { + fn log(&self, level: LogLevel, args: fmt::Arguments); + fn should_log(&self, level: LogLevel) -> bool; +} + +/// The "Master" macro that handles the core logic. +/// It takes the logger instance, the level, and the format string/args. +#[macro_export] +macro_rules! log { + ($logger:expr, $level:expr, $($arg:tt)+) => { + if $logger.should_log($level) { + $logger.log($level, format_args!($($arg)+)) + } + }; +} + +/// Log at the Trace level +#[macro_export] +macro_rules! trace { + ($logger:expr, $($arg:tt)+) => { + $crate::log!($logger, $crate::LogLevel::Trace, $($arg)+) + }; +} + +/// Log at the Debug level +#[macro_export] +macro_rules! debug { + ($logger:expr, $($arg:tt)+) => { + $crate::log!($logger, $crate::LogLevel::Debug, $($arg)+) + }; +} + +/// Log at the Info level +#[macro_export] +macro_rules! info { + ($logger:expr, $($arg:tt)+) => { + $crate::log!($logger, $crate::LogLevel::Info, $($arg)+) + }; +} + +/// Log at the Warn level +#[macro_export] +macro_rules! warn { + ($logger:expr, $($arg:tt)+) => { + $crate::log!($logger, $crate::LogLevel::Warn, $($arg)+) + }; +} + +/// Log at the Error level +#[macro_export] +macro_rules! error { + ($logger:expr, $($arg:tt)+) => { + $crate::log!($logger, $crate::LogLevel::Error, $($arg)+) + }; +} + +/// Log at the Critical level +#[macro_export] +macro_rules! critical { + ($logger:expr, $($arg:tt)+) => { + $crate::log!($logger, $crate::LogLevel::Critical, $($arg)+) + }; +} + +#[cfg(test)] +mod tests { + use crate::LogLevel; + + #[test] + fn order_test() { + assert!(LogLevel::Debug > LogLevel::Trace); + assert!(LogLevel::Info > LogLevel::Debug); + assert!(LogLevel::Warn > LogLevel::Info); + assert!(LogLevel::Error > LogLevel::Warn); + assert!(LogLevel::Critical > LogLevel::Error); + assert!(LogLevel::Off > LogLevel::Critical); + } +} diff --git a/minifi_rust/minifi_native/src/api/process_context.rs b/minifi_rust/minifi_native/src/api/process_context.rs new file mode 100644 index 0000000000..fc60ca502f --- /dev/null +++ b/minifi_rust/minifi_native/src/api/process_context.rs @@ -0,0 +1,126 @@ +use crate::StandardPropertyValidator::*; +use crate::api::RawControllerService; +use crate::api::component_definition_traits::ComponentIdentifier; +use crate::api::flow_file::FlowFile; +use crate::api::property::GetControllerService; +use crate::{EnableControllerService, GetProperty, MinifiError, Property}; +use std::str::FromStr; +use std::time::Duration; + +pub trait ProcessContext { + type FlowFile: FlowFile; + + fn get_property( + &self, + property: &Property, + flow_file: Option<&Self::FlowFile>, + ) -> Result, MinifiError>; + + fn get_bool_property( + &self, + property: &Property, + flow_file: Option<&Self::FlowFile>, + ) -> Result, MinifiError> { + if property.validator != BoolValidator { + return Err(MinifiError::validation_err(format!( + "to use get_bool_property {:?} must have BoolValidator", + property + ))); + } + + if let Some(property_val) = self.get_property(property, flow_file)? { + Ok(Some(bool::from_str(&property_val)?)) + } else { + Ok(None) + } + } + + fn get_duration_property( + &self, + property: &Property, + flow_file: Option<&Self::FlowFile>, + ) -> Result, MinifiError> { + if property.validator != TimePeriodValidator { + return Err(MinifiError::validation_err(format!( + "to use get_duration_property {:?} must have TimePeriodValidator", + property + ))); + } + + if let Some(property_val) = self.get_property(property, flow_file)? { + Ok(Some(humantime::parse_duration(property_val.as_str())?)) + } else { + Ok(None) + } + } + + fn get_size_property( + &self, + property: &Property, + flow_file: Option<&Self::FlowFile>, + ) -> Result, MinifiError> { + if property.validator != DataSizeValidator { + return Err(MinifiError::validation_err(format!( + "to use get_size_property {:?} must have DataSizeValidator", + property + ))); + } + if let Some(property_val) = self.get_property(property, flow_file)? { + Ok(Some(byte_unit::Byte::from_str(&property_val)?.as_u64())) + } else { + Ok(None) + } + } + + fn get_u64_property( + &self, + property: &Property, + flow_file: Option<&Self::FlowFile>, + ) -> Result, MinifiError> { + if property.validator != U64Validator { + return Err(MinifiError::validation_err(format!( + "to use get_u64_property {:?} must have U64Validator", + property + ))); + } + if let Some(property_val) = self.get_property(property, flow_file)? { + Ok(Some(u64::from_str(&property_val)?)) + } else { + Ok(None) + } + } + + fn get_raw_controller_service( + &self, + property: &Property, + ) -> Result, MinifiError> + where + Cs: RawControllerService + ComponentIdentifier + 'static; + + fn get_controller_service(&self, property: &Property) -> Result, MinifiError> + where + Cs: EnableControllerService + ComponentIdentifier + 'static; + + fn report_metrics(&self, metrics: Vec<(String, f64)>) -> Result<(), MinifiError>; +} + +impl GetProperty for S +where + S: ProcessContext, +{ + fn get_property(&self, property: &Property) -> Result, MinifiError> { + self.get_property(property, None) + } +} + +impl GetControllerService for S +where + S: ProcessContext, +{ + fn get_controller_service(&self, property: &Property) -> Result, MinifiError> + where + Cs: EnableControllerService + ComponentIdentifier + 'static, + { + ProcessContext::get_controller_service::(self, property) + } +} diff --git a/minifi_rust/minifi_native/src/api/process_session.rs b/minifi_rust/minifi_native/src/api/process_session.rs new file mode 100644 index 0000000000..8fb78dfd18 --- /dev/null +++ b/minifi_rust/minifi_native/src/api/process_session.rs @@ -0,0 +1,60 @@ +use crate::MinifiError; +use crate::api::flow_file::FlowFile; +pub trait InputStream: std::io::BufRead + Send + std::fmt::Debug {} +pub trait OutputStream: std::io::Write + Send + std::fmt::Debug {} +impl OutputStream for T {} +impl InputStream for T {} + +#[derive(Debug, Clone, Copy, Eq, PartialEq)] +pub enum IoState { + Ok, + Cancel, +} + +pub trait ProcessSession { + type FlowFile: FlowFile; + + fn create(&mut self) -> Result; + fn get(&mut self) -> Option; + fn transfer(&self, flow_file: Self::FlowFile, relationship: &str) -> Result<(), MinifiError>; + fn remove(&mut self, flow_file: Self::FlowFile) -> Result<(), MinifiError>; + + fn set_attribute( + &self, + flow_file: &mut Self::FlowFile, + attr_key: &str, + attr_value: &str, + ) -> Result<(), MinifiError>; + fn get_attribute(&self, flow_file: &Self::FlowFile, attr_key: &str) -> Option; + fn on_attributes( + &self, + flow_file: &Self::FlowFile, + process_attr: F, + ) -> bool; + + fn write(&self, flow_file: &Self::FlowFile, data: &[u8]) -> Result<(), MinifiError>; + fn write_lazy<'a>( + &self, + flow_file: &Self::FlowFile, + stream: Box, + ) -> Result<(), MinifiError>; + + fn write_stream(&self, flow_file: &Self::FlowFile, callback: F) -> Result + where + F: FnOnce(&mut dyn OutputStream) -> Result<(R, IoState), MinifiError>; + + fn read(&self, flow_file: &Self::FlowFile) -> Option>; + fn read_stream(&self, flow_file: &Self::FlowFile, callback: F) -> Result + where + F: FnOnce(&mut dyn InputStream) -> Result; + fn read_in_batches( + &self, + flow_file: &Self::FlowFile, + batch_size: usize, + process_batch: F, + ) -> Result<(), MinifiError> + where + F: FnMut(&[u8]) -> Result<(), MinifiError>; + + fn get_flow_file_id(&self, flow_file: &Self::FlowFile) -> Result; +} diff --git a/minifi_rust/minifi_native/src/api/processor.rs b/minifi_rust/minifi_native/src/api/processor.rs new file mode 100644 index 0000000000..60b15ceacf --- /dev/null +++ b/minifi_rust/minifi_native/src/api/processor.rs @@ -0,0 +1,60 @@ +use crate::api::{RawProcessor, ThreadingModel}; +use crate::{GetProperty, LogLevel, Logger, MinifiError, ProcessContext}; +use std::marker::PhantomData; + +pub trait Schedule { + fn schedule( + context: &Ctx, + logger: &L, + ) -> Result + where + Self: Sized; + + fn unschedule(&mut self) {} +} + +pub struct Processor +where + Impl: Schedule, + T: ThreadingModel, + L: Logger, +{ + pub(crate) logger: L, + pub(crate) scheduled_impl: Option, + threading_model: PhantomData, + flow_file_type: PhantomData, +} + +impl RawProcessor for Processor +where + Impl: Schedule, + T: ThreadingModel, + L: Logger, +{ + type Threading = T; + type LoggerType = L; + + fn new(logger: Self::LoggerType) -> Self { + Self { + logger, + scheduled_impl: None, + threading_model: PhantomData, + flow_file_type: PhantomData, + } + } + + fn log(&self, log_level: LogLevel, args: std::fmt::Arguments) { + self.logger.log(log_level, args); + } + + fn on_schedule(&mut self, context: &P) -> Result<(), MinifiError> { + self.scheduled_impl = Some(Impl::schedule(context, &self.logger)?); + Ok(()) + } + + fn on_unschedule(&mut self) { + if let Some(ref mut scheduled_impl) = self.scheduled_impl { + scheduled_impl.unschedule() + } + } +} diff --git a/minifi_rust/minifi_native/src/api/processor_wrappers.rs b/minifi_rust/minifi_native/src/api/processor_wrappers.rs new file mode 100644 index 0000000000..ee5827f785 --- /dev/null +++ b/minifi_rust/minifi_native/src/api/processor_wrappers.rs @@ -0,0 +1,5 @@ +pub(crate) mod complex_processor; +pub(crate) mod flow_file_source; +pub(crate) mod flow_file_stream_transform; +pub(crate) mod flow_file_transform; +pub(crate) mod utils; diff --git a/minifi_rust/minifi_native/src/api/processor_wrappers/complex_processor.rs b/minifi_rust/minifi_native/src/api/processor_wrappers/complex_processor.rs new file mode 100644 index 0000000000..4abebc0fcf --- /dev/null +++ b/minifi_rust/minifi_native/src/api/processor_wrappers/complex_processor.rs @@ -0,0 +1,89 @@ +use crate::api::raw_processor::{MultiThreadedTrigger, SingleThreadedTrigger}; +use crate::{ + ComponentIdentifier, Concurrent, Exclusive, Logger, MinifiError, OnTriggerResult, + ProcessContext, ProcessSession, Processor, ProcessorDefinition, Schedule, +}; + +pub trait MutTrigger { + fn trigger( + &mut self, + context: &mut Ctx, + session: &mut Session, + logger: &Lggr, + ) -> Result + where + Ctx: ProcessContext, + Session: ProcessSession, + Lggr: Logger; +} + +pub trait Trigger { + fn trigger( + &self, + context: &mut Context, + session: &mut Session, + logger: &Lggr, + ) -> Result + where + Context: ProcessContext, + Session: ProcessSession, + Lggr: Logger; +} + +pub struct ComplexProcessorType {} + +impl SingleThreadedTrigger + for Processor +where + Implementation: Schedule + + MutTrigger + + ComponentIdentifier + + ProcessorDefinition, + L: Logger, +{ + fn on_trigger( + &mut self, + context: &mut PC, + session: &mut PS, + ) -> Result + where + PC: ProcessContext, + PS: ProcessSession, + { + if let Some(ref mut scheduled_impl) = self.scheduled_impl { + scheduled_impl.trigger(context, session, &self.logger) + } else { + Err(MinifiError::trigger_err( + "The processor hasn't been scheduled yet", + )) + } + } +} + +impl MultiThreadedTrigger + for Processor +where + Implementation: Schedule + + Trigger + + ComponentIdentifier + + ProcessorDefinition, + L: Logger, +{ + fn on_trigger( + &self, + context: &mut PC, + session: &mut PS, + ) -> Result + where + PC: ProcessContext, + PS: ProcessSession, + { + if let Some(ref scheduled_impl) = self.scheduled_impl { + scheduled_impl.trigger(context, session, &self.logger) + } else { + Err(MinifiError::trigger_err( + "The processor hasn't been scheduled yet", + )) + } + } +} diff --git a/minifi_rust/minifi_native/src/api/processor_wrappers/flow_file_source.rs b/minifi_rust/minifi_native/src/api/processor_wrappers/flow_file_source.rs new file mode 100644 index 0000000000..5bf2f33d10 --- /dev/null +++ b/minifi_rust/minifi_native/src/api/processor_wrappers/flow_file_source.rs @@ -0,0 +1,129 @@ +use crate::api::processor_wrappers::utils::flow_file_content::Content; +use crate::api::raw_processor::{MultiThreadedTrigger, SingleThreadedTrigger}; +use crate::{ + Concurrent, Exclusive, GetControllerService, GetProperty, Logger, + MinifiError, OnTriggerResult, ProcessContext, ProcessSession, Processor, Relationship, + Schedule, +}; +use std::collections::HashMap; + +pub struct GeneratedFlowFile<'a> { + target_relationship_name: &'static str, + new_content: Option>, + attributes_to_add: HashMap, +} + +impl<'a> GeneratedFlowFile<'a> { + pub fn new( + target_relationship: &'a Relationship, + new_content: Option>, + attributes_to_add: HashMap, + ) -> Self { + Self { + target_relationship_name: target_relationship.name, + new_content, + attributes_to_add, + } + } + + pub fn target_relationship_name(&self) -> &'static str { + self.target_relationship_name + } +} + +pub trait FlowFileSource { + fn generate<'a, Context: GetProperty + GetControllerService, LoggerImpl: Logger>( + &self, + context: &'a mut Context, + logger: &LoggerImpl, + ) -> Result>, MinifiError>; +} + +pub trait MutFlowFileSource { + fn generate<'a, Context: GetProperty + GetControllerService, LoggerImpl: Logger>( + &mut self, + context: &'a mut Context, + logger: &LoggerImpl, + ) -> Result>, MinifiError>; +} + +fn handle_generated_flow_files( + session: &mut PS, + generated_flow_files: Vec, +) -> Result +where + PC: ProcessContext, + PS: ProcessSession, +{ + if generated_flow_files.is_empty() { + return Ok(OnTriggerResult::Yield); + } + + for new_flow_file_data in generated_flow_files { + let mut ff = session.create()?; + match new_flow_file_data.new_content { + None => {} + Some(Content::Buffer(buffer)) => session.write(&mut ff, &buffer)?, + Some(Content::Stream(stream)) => session.write_lazy(&mut ff, stream)?, + } + for (k, v) in &new_flow_file_data.attributes_to_add { + session.set_attribute(&mut ff, k, v)?; + } + session.transfer(ff, new_flow_file_data.target_relationship_name)?; + } + Ok(OnTriggerResult::Ok) +} + +pub struct FlowFileSourceProcessorType {} + +impl<'a, Implementation, L> MultiThreadedTrigger + for Processor +where + Implementation: Schedule + FlowFileSource, + L: Logger, +{ + fn on_trigger( + &self, + context: &mut PC, + session: &mut PS, + ) -> Result + where + PC: ProcessContext, + PS: ProcessSession, + { + if let Some(ref scheduled_impl) = self.scheduled_impl { + let files = scheduled_impl.generate(context, &self.logger)?; + handle_generated_flow_files::(session, files) + } else { + Err(MinifiError::trigger_err( + "The processor hasn't been scheduled yet", + )) + } + } +} + +impl<'a, Implementation, L> SingleThreadedTrigger + for Processor +where + Implementation: Schedule + MutFlowFileSource, + L: Logger, +{ + fn on_trigger( + &mut self, + context: &mut PC, + session: &mut PS, + ) -> Result + where + PC: ProcessContext, + PS: ProcessSession, + { + if let Some(ref mut scheduled_impl) = self.scheduled_impl { + let files = scheduled_impl.generate(context, &self.logger)?; + handle_generated_flow_files::(session, files) + } else { + Err(MinifiError::trigger_err( + "The processor hasn't been scheduled yet", + )) + } + } +} diff --git a/minifi_rust/minifi_native/src/api/processor_wrappers/flow_file_stream_transform.rs b/minifi_rust/minifi_native/src/api/processor_wrappers/flow_file_stream_transform.rs new file mode 100644 index 0000000000..71dae71a4c --- /dev/null +++ b/minifi_rust/minifi_native/src/api/processor_wrappers/flow_file_stream_transform.rs @@ -0,0 +1,174 @@ +use crate::api::process_session::IoState; +use crate::api::processor_wrappers::utils::context_session_flowfile_bundle::ContextSessionFlowFileBundle; +use crate::api::raw_processor::{MultiThreadedTrigger, SingleThreadedTrigger}; +use crate::{ + Concurrent, Exclusive, GetAttribute, GetControllerService, GetProperty, + InputStream, LogLevel, Logger, MinifiError, OnTriggerResult, OutputStream, ProcessContext, + ProcessSession, Processor, Relationship, Schedule, +}; +use std::collections::HashMap; + +pub struct TransformStreamResult { + target_relationship_name: &'static str, + attributes_to_add: HashMap, + write_status: IoState, +} + +impl TransformStreamResult { + pub fn new( + target_relationship: &Relationship, + attributes_to_add: HashMap, + ) -> Self { + Self { + target_relationship_name: target_relationship.name, + attributes_to_add, + write_status: IoState::Ok, + } + } + + pub fn route_without_changes(target_relationship: &Relationship) -> Self { + Self { + target_relationship_name: target_relationship.name, + attributes_to_add: HashMap::new(), + write_status: IoState::Cancel, + } + } + + pub fn target_relationship_name(&self) -> &'static str { + self.target_relationship_name + } + + pub fn get_attribute(&self, name: &str) -> Option { + self.attributes_to_add.get(name).cloned() + } + + pub fn write_status(&self) -> IoState { + self.write_status + } +} + +pub trait FlowFileStreamTransform { + fn transform( + &self, + context: &Ctx, + input_stream: &mut dyn InputStream, + output_stream: &mut dyn OutputStream, + logger: &LoggerImpl, + ) -> Result; +} + +pub trait MutFlowFileStreamTransform { + fn transform( + &mut self, + context: &Ctx, + input_stream: &mut dyn InputStream, + output_stream: &mut dyn OutputStream, + logger: &LoggerImpl, + ) -> Result; +} + +pub struct FlowFileStreamTransformProcessorType {} + +fn handle_stream_transform( + context: &mut PC, + session: &mut PS, + logger: &L, + mut transform_fn: F, +) -> Result +where + PC: ProcessContext, + PS: ProcessSession, + L: Logger, + F: FnMut( + &ContextSessionFlowFileBundle, + &mut dyn InputStream, + &mut dyn OutputStream, + ) -> Result, +{ + if let Some(mut flow_file) = session.get() { + let simple_context = ContextSessionFlowFileBundle::new(context, session, Some(&flow_file)); + + let (relationship, attrs) = session.read_stream(&flow_file, |input_stream| { + session.write_stream(&flow_file, |output_stream| { + let transformed = transform_fn(&simple_context, input_stream, output_stream)?; + + Ok(( + ( + transformed.target_relationship_name, + transformed.attributes_to_add, + ), + transformed.write_status, + )) + }) + })?; + + for (k, v) in attrs { + session.set_attribute(&mut flow_file, &k, &v)?; + } + + session.transfer(flow_file, relationship)?; + + Ok(OnTriggerResult::Ok) + } else { + logger.log(LogLevel::Trace, format_args!("No flowfile to transform")); + Ok(OnTriggerResult::Yield) + } +} + +// Concurrent Implementation (Multi-Threaded) +impl<'a, Implementation, L> MultiThreadedTrigger + for Processor +where + Implementation: + Schedule + FlowFileStreamTransform, + L: Logger, +{ + fn on_trigger( + &self, + context: &mut PC, + session: &mut PS, + ) -> Result + where + PC: ProcessContext, + PS: ProcessSession, + { + if let Some(ref scheduled_impl) = self.scheduled_impl { + handle_stream_transform(context, session, &self.logger, |ctx, input, output| { + scheduled_impl.transform(ctx, input, output, &self.logger) + }) + } else { + Err(MinifiError::trigger_err( + "The processor hasn't been scheduled yet", + )) + } + } +} + +// Exclusive Implementation (Single-Threaded) +impl<'a, Implementation, L> SingleThreadedTrigger + for Processor +where + Implementation: + Schedule + MutFlowFileStreamTransform, + L: Logger, +{ + fn on_trigger( + &mut self, + context: &mut PC, + session: &mut PS, + ) -> Result + where + PC: ProcessContext, + PS: ProcessSession, + { + if let Some(ref mut scheduled_impl) = self.scheduled_impl { + handle_stream_transform(context, session, &self.logger, |ctx, input, output| { + scheduled_impl.transform(ctx, input, output, &self.logger) + }) + } else { + Err(MinifiError::trigger_err( + "The processor hasn't been scheduled yet", + )) + } + } +} diff --git a/minifi_rust/minifi_native/src/api/processor_wrappers/flow_file_transform.rs b/minifi_rust/minifi_native/src/api/processor_wrappers/flow_file_transform.rs new file mode 100644 index 0000000000..3ddd19029e --- /dev/null +++ b/minifi_rust/minifi_native/src/api/processor_wrappers/flow_file_transform.rs @@ -0,0 +1,184 @@ +use crate::api::InputStream; +use crate::api::processor::{Processor}; +use crate::api::processor_wrappers::utils::context_session_flowfile_bundle::ContextSessionFlowFileBundle; +use crate::api::processor_wrappers::utils::flow_file_content::Content; +use crate::api::property::{GetControllerService, GetProperty}; +use crate::api::raw_processor::{MultiThreadedTrigger, SingleThreadedTrigger}; +use crate::{ + Concurrent, Exclusive, GetAttribute, LogLevel, Logger, MinifiError, + OnTriggerResult, ProcessContext, ProcessSession, Relationship, Schedule, info, +}; +use std::collections::HashMap; + +#[derive(Debug)] +pub struct TransformedFlowFile<'a> { + target_relationship_name: &'static str, + new_content: Option>, + attributes_to_add: HashMap, +} + +impl<'a> TransformedFlowFile<'a> { + pub fn route_without_changes(target_relationship: &Relationship) -> Self { + Self { + target_relationship_name: target_relationship.name, + new_content: None, + attributes_to_add: HashMap::new(), + } + } + + pub fn new( + target_relationship: &Relationship, + new_content: Option>, + attributes_to_add: HashMap, + ) -> Self { + Self { + target_relationship_name: target_relationship.name, + new_content: new_content.map(|b| Content::Buffer(b)), + attributes_to_add, + } + } + + pub fn new_content(&'_ self) -> Option<&'_ Content<'_>> { + self.new_content.as_ref() + } + + pub fn target_relationship(&self) -> &'static str { + self.target_relationship_name + } + + pub fn attributes_to_add(&self) -> &HashMap { + &self.attributes_to_add + } +} + +pub trait FlowFileTransform { + fn transform< + 'a, + Context: GetProperty + GetControllerService + GetAttribute, + LoggerImpl: Logger, + >( + &self, + context: &Context, + input_stream: &'a mut dyn InputStream, + logger: &LoggerImpl, + ) -> Result, MinifiError>; +} + +pub trait MutFlowFileTransform { + fn transform< + 'a, + Context: GetProperty + GetControllerService + GetAttribute, + LoggerImpl: Logger, + >( + &mut self, + context: &Context, + input_stream: &'a mut dyn InputStream, + logger: &LoggerImpl, + ) -> Result, MinifiError>; +} + +pub struct FlowFileTransformProcessorType {} + +fn handle_transform( + context: &mut PC, + session: &mut PS, + logger: &L, + mut transform_fn: F, +) -> Result +where + PC: ProcessContext, + PS: ProcessSession, + L: Logger, + F: for<'stream> FnMut( + &ContextSessionFlowFileBundle<'_, PC, PS>, + &'stream mut dyn InputStream, + ) -> Result, MinifiError>, +{ + if let Some(mut flow_file) = session.get() { + let simple_context = ContextSessionFlowFileBundle::new(context, session, Some(&flow_file)); + + let (attrs_to_add, relationship) = session.read_stream(&flow_file, |input_stream| { + let transformed = transform_fn(&simple_context, input_stream)?; + + info!(logger, "{:?}", transformed); + match transformed.new_content { + None => {} + Some(Content::Buffer(buffer)) => { + session.write(&flow_file, &buffer)?; + } + Some(Content::Stream(stream)) => { + session.write_lazy(&flow_file, stream)?; + } + }; + + Ok(( + transformed.attributes_to_add, + transformed.target_relationship_name, + )) + })?; + + for (k, v) in attrs_to_add { + session.set_attribute(&mut flow_file, &k, &v)?; + } + + session.transfer(flow_file, relationship)?; + Ok(OnTriggerResult::Ok) + } else { + logger.log(LogLevel::Trace, format_args!("No flowfile to transform")); + Ok(OnTriggerResult::Yield) + } +} + +impl MultiThreadedTrigger + for Processor +where + Implementation: Schedule + FlowFileTransform, + L: Logger, +{ + fn on_trigger( + &self, + context: &mut PC, + session: &mut PS, + ) -> Result + where + PC: ProcessContext, + PS: ProcessSession, + { + if let Some(ref scheduled_impl) = self.scheduled_impl { + handle_transform(context, session, &self.logger, |ctx, input| { + scheduled_impl.transform(ctx, input, &self.logger) + }) + } else { + Err(MinifiError::trigger_err( + "The processor hasn't been scheduled yet", + )) + } + } +} + +impl SingleThreadedTrigger + for Processor +where + Implementation: Schedule + MutFlowFileTransform, + L: Logger, +{ + fn on_trigger( + &mut self, + context: &mut PC, + session: &mut PS, + ) -> Result + where + PC: ProcessContext, + PS: ProcessSession, + { + if let Some(ref mut scheduled_impl) = self.scheduled_impl { + handle_transform(context, session, &self.logger, |ctx, input| { + scheduled_impl.transform(ctx, input, &self.logger) + }) + } else { + Err(MinifiError::trigger_err( + "The processor hasn't been scheduled yet", + )) + } + } +} diff --git a/minifi_rust/minifi_native/src/api/processor_wrappers/utils.rs b/minifi_rust/minifi_native/src/api/processor_wrappers/utils.rs new file mode 100644 index 0000000000..e10592e96c --- /dev/null +++ b/minifi_rust/minifi_native/src/api/processor_wrappers/utils.rs @@ -0,0 +1,2 @@ +pub(crate) mod context_session_flowfile_bundle; +pub(crate) mod flow_file_content; diff --git a/minifi_rust/minifi_native/src/api/processor_wrappers/utils/context_session_flowfile_bundle.rs b/minifi_rust/minifi_native/src/api/processor_wrappers/utils/context_session_flowfile_bundle.rs new file mode 100644 index 0000000000..8af603c9ff --- /dev/null +++ b/minifi_rust/minifi_native/src/api/processor_wrappers/utils/context_session_flowfile_bundle.rs @@ -0,0 +1,70 @@ +use crate::api::attribute::GetAttribute; +use crate::api::property::{GetControllerService, GetProperty}; +use crate::{ + ComponentIdentifier, EnableControllerService, MinifiError, ProcessContext, ProcessSession, + Property, +}; + +pub struct ContextSessionFlowFileBundle<'a, PC, PS> +where + PC: ProcessContext, + PS: ProcessSession, +{ + context: &'a PC, + session: &'a PS, + flow_file: Option<&'a PC::FlowFile>, +} + +impl<'a, PC, PS> ContextSessionFlowFileBundle<'a, PC, PS> +where + PC: ProcessContext, + PS: ProcessSession, +{ + pub(crate) fn new( + context: &'a PC, + session: &'a PS, + flow_file: Option<&'a PC::FlowFile>, + ) -> Self { + Self { + context, + session, + flow_file, + } + } +} +impl<'a, PC, PS> GetProperty for ContextSessionFlowFileBundle<'a, PC, PS> +where + PC: ProcessContext, + PS: ProcessSession, +{ + fn get_property(&self, property: &Property) -> Result, MinifiError> { + self.context.get_property(property, self.flow_file) + } +} + +impl<'a, PC, PS> GetControllerService for ContextSessionFlowFileBundle<'a, PC, PS> +where + PC: ProcessContext, + PS: ProcessSession, +{ + fn get_controller_service(&self, property: &Property) -> Result, MinifiError> + where + Cs: EnableControllerService + ComponentIdentifier + 'static, + { + self.context.get_controller_service(property) + } +} + +impl<'a, PC, PS> GetAttribute for ContextSessionFlowFileBundle<'a, PC, PS> +where + PC: ProcessContext, + PS: ProcessSession, +{ + fn get_attribute(&self, name: &str) -> Result, MinifiError> { + if let Some(ff) = &self.flow_file { + Ok(self.session.get_attribute(ff, name)) + } else { + Ok(None) + } + } +} diff --git a/minifi_rust/minifi_native/src/api/processor_wrappers/utils/flow_file_content.rs b/minifi_rust/minifi_native/src/api/processor_wrappers/utils/flow_file_content.rs new file mode 100644 index 0000000000..6bd5492605 --- /dev/null +++ b/minifi_rust/minifi_native/src/api/processor_wrappers/utils/flow_file_content.rs @@ -0,0 +1,27 @@ +use std::fmt::Formatter; + +pub enum Content<'a> { + Buffer(Vec), + Stream(Box), +} + +impl std::fmt::Debug for Content<'_> { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + match self { + Content::Buffer(b) => f.debug_struct("Content").field("Buffer", &b).finish(), + Content::Stream(_s) => f.debug_struct("Content::Stream").finish(), + } + } +} + +impl From> for Content<'_> { + fn from(v: Vec) -> Self { + Content::Buffer(v) + } +} + +impl From for Content<'_> { + fn from(s: String) -> Self { + Content::Buffer(s.into_bytes()) + } +} diff --git a/minifi_rust/minifi_native/src/api/property.rs b/minifi_rust/minifi_native/src/api/property.rs new file mode 100644 index 0000000000..3f87b5615c --- /dev/null +++ b/minifi_rust/minifi_native/src/api/property.rs @@ -0,0 +1,98 @@ +use crate::StandardPropertyValidator::{ + BoolValidator, DataSizeValidator, TimePeriodValidator, U64Validator, +}; +use crate::{ComponentIdentifier, EnableControllerService, MinifiError}; +use std::str::FromStr; +use std::time::Duration; + +#[derive(Debug, Eq, PartialEq)] +pub enum StandardPropertyValidator { + AlwaysValidValidator, + NonBlankValidator, + TimePeriodValidator, + BoolValidator, + I64Validator, + U64Validator, + DataSizeValidator, + PortValidator, +} + +#[derive(Debug)] +pub struct Property { + pub name: &'static str, + pub description: &'static str, + pub is_required: bool, + pub is_sensitive: bool, + pub supports_expr_lang: bool, + pub default_value: Option<&'static str>, + pub validator: StandardPropertyValidator, + pub allowed_values: &'static [&'static str], + pub allowed_type: &'static str, +} + +pub trait GetProperty { + fn get_property(&self, property: &Property) -> Result, MinifiError>; + fn get_bool_property(&self, property: &Property) -> Result, MinifiError> { + if property.validator != BoolValidator { + return Err(MinifiError::validation_err(format!( + "to use get_bool_property {:?} must have BoolValidator", + property + ))); + } + + if let Some(property_val) = self.get_property(property)? { + Ok(Some(bool::from_str(&property_val)?)) + } else { + Ok(None) + } + } + + fn get_duration_property(&self, property: &Property) -> Result, MinifiError> { + if property.validator != TimePeriodValidator { + return Err(MinifiError::validation_err(format!( + "to use get_duration_property {:?} must have TimePeriodValidator", + property + ))); + } + + if let Some(property_val) = self.get_property(property)? { + Ok(Some(humantime::parse_duration(property_val.as_str())?)) + } else { + Ok(None) + } + } + + fn get_size_property(&self, property: &Property) -> Result, MinifiError> { + if property.validator != DataSizeValidator { + return Err(MinifiError::validation_err(format!( + "to use get_size_property {:?} must have DataSizeValidator", + property + ))); + } + if let Some(property_val) = self.get_property(property)? { + Ok(Some(byte_unit::Byte::from_str(&property_val)?.as_u64())) + } else { + Ok(None) + } + } + + fn get_u64_property(&self, property: &Property) -> Result, MinifiError> { + if property.validator != U64Validator { + return Err(MinifiError::validation_err(format!( + "to use get_u64_property {:?} must have U64Validator", + property + ))); + } + if let Some(property_val) = self.get_property(property)? { + Ok(Some(u64::from_str(&property_val)?)) + } else { + Ok(None) + } + } +} + +pub trait GetControllerService { + fn get_controller_service(&self, property: &Property) -> Result, MinifiError> + where + Cs: EnableControllerService + ComponentIdentifier + 'static; +} diff --git a/minifi_rust/minifi_native/src/api/raw_controller_service.rs b/minifi_rust/minifi_native/src/api/raw_controller_service.rs new file mode 100644 index 0000000000..3348a398e0 --- /dev/null +++ b/minifi_rust/minifi_native/src/api/raw_controller_service.rs @@ -0,0 +1,11 @@ +use crate::{GetProperty, LogLevel, Logger, MinifiError}; + +/// This RawControllerService will be instantiated, and called on by the agent +pub trait RawControllerService: Sized { + type LoggerType: Logger; + + fn new(logger: Self::LoggerType) -> Self; + fn log(&self, log_level: LogLevel, args: std::fmt::Arguments); + fn enable(&mut self, context: &P) -> Result<(), MinifiError>; + fn disable(&mut self) {} +} diff --git a/minifi_rust/minifi_native/src/api/raw_processor.rs b/minifi_rust/minifi_native/src/api/raw_processor.rs new file mode 100644 index 0000000000..6926db17a8 --- /dev/null +++ b/minifi_rust/minifi_native/src/api/raw_processor.rs @@ -0,0 +1,70 @@ +use crate::api::errors::MinifiError; +use crate::{LogLevel, Logger, ProcessContext, ProcessSession}; + +pub enum ProcessorInputRequirement { + Required, + Allowed, + Forbidden, +} + +#[derive(Debug, PartialEq, Eq)] +pub enum OnTriggerResult { + Ok, + Yield, +} + +/// This RawProcessor will be instantiated, and called on by the agent +pub trait RawProcessor: Sized { + type Threading: ThreadingModel; + type LoggerType: Logger; + + fn new(logger: Self::LoggerType) -> Self; + fn log(&self, log_level: LogLevel, args: std::fmt::Arguments); + fn on_schedule(&mut self, context: &P) -> Result<(), MinifiError>; + fn on_unschedule(&mut self); +} + +/// To differentiate between single and multithreaded processors +pub trait ThreadingModel: sealed::Sealed { + const IS_EXCLUSIVE: bool; +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct Concurrent; +impl ThreadingModel for Concurrent { + const IS_EXCLUSIVE: bool = false; +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct Exclusive; +impl ThreadingModel for Exclusive { + const IS_EXCLUSIVE: bool = true; +} + +mod sealed { + pub trait Sealed {} + impl Sealed for super::Concurrent {} + impl Sealed for super::Exclusive {} +} + +pub trait SingleThreadedTrigger: RawProcessor { + fn on_trigger( + &mut self, + context: &mut PC, + session: &mut PS, + ) -> Result + where + PC: ProcessContext, + PS: ProcessSession; +} + +pub trait MultiThreadedTrigger: RawProcessor { + fn on_trigger( + &self, + context: &mut PC, + session: &mut PS, + ) -> Result + where + PC: ProcessContext, + PS: ProcessSession; +} diff --git a/minifi_rust/minifi_native/src/api/relationship.rs b/minifi_rust/minifi_native/src/api/relationship.rs new file mode 100644 index 0000000000..dc80477fa8 --- /dev/null +++ b/minifi_rust/minifi_native/src/api/relationship.rs @@ -0,0 +1,5 @@ +#[derive(Debug, Eq, PartialEq)] +pub struct Relationship { + pub name: &'static str, + pub description: &'static str, +} diff --git a/minifi_rust/minifi_native/src/c_ffi.rs b/minifi_rust/minifi_native/src/c_ffi.rs new file mode 100644 index 0000000000..d426cda447 --- /dev/null +++ b/minifi_rust/minifi_native/src/c_ffi.rs @@ -0,0 +1,26 @@ +mod c_ffi_controller_service_context; +mod c_ffi_controller_service_definition; +mod c_ffi_controller_service_list; +mod c_ffi_flow_file; +mod c_ffi_logger; +mod c_ffi_output_attribute; +mod c_ffi_primitives; +mod c_ffi_process_context; +mod c_ffi_process_session; +mod c_ffi_processor_definition; +mod c_ffi_processor_list; +mod c_ffi_property; +mod c_ffi_relationship; +mod c_ffi_streams; + +pub use c_ffi_controller_service_definition::CffiControllerServiceDefinition; +pub use c_ffi_controller_service_definition::DynRawControllerServiceDefinition; +pub use c_ffi_controller_service_definition::RegisterableControllerService; +pub use c_ffi_controller_service_list::CffiControllerServiceList; +pub use c_ffi_logger::CffiLogger; +pub use c_ffi_primitives::StaticStrAsMinifiCStr; +pub use c_ffi_processor_definition::DispatchOnTrigger; +pub use c_ffi_processor_definition::DynRawProcessorDefinition; +pub use c_ffi_processor_definition::RawProcessorDefinition; +pub use c_ffi_processor_definition::RawRegisterableProcessor; +pub use c_ffi_processor_list::CffiProcessorList; diff --git a/minifi_rust/minifi_native/src/c_ffi/c_ffi_controller_service_context.rs b/minifi_rust/minifi_native/src/c_ffi/c_ffi_controller_service_context.rs new file mode 100644 index 0000000000..52ae73cbb8 --- /dev/null +++ b/minifi_rust/minifi_native/src/c_ffi/c_ffi_controller_service_context.rs @@ -0,0 +1,70 @@ +use crate::c_ffi::c_ffi_primitives::StringView; +use crate::{GetProperty, MinifiError, Property}; +use minifi_native_sys::{ + minifi_controller_service_context, + minifi_controller_service_context_get_property, + minifi_status_MINIFI_STATUS_SUCCESS, minifi_string_view, +}; +use std::borrow::Cow; +use std::ffi::c_void; + +pub struct CffiControllerServiceContext<'a> { + ptr: *mut minifi_controller_service_context, + _lifetime: std::marker::PhantomData<&'a ()>, +} + +impl<'a> CffiControllerServiceContext<'a> { + pub fn new(ptr: *mut minifi_controller_service_context) -> Self { + Self { + ptr, + _lifetime: std::marker::PhantomData, + } + } +} + +unsafe extern "C" fn property_callback( + output_option: *mut c_void, + property_c_value: minifi_string_view, +) { + unsafe { + let result_target = &mut *(output_option as *mut Option); + + if property_c_value.data.is_null() || property_c_value.length == 0 { + *result_target = None; + return; + } + + let value_slice = + std::slice::from_raw_parts(property_c_value.data as *const u8, property_c_value.length); + if let Ok(string_value) = String::from_utf8(value_slice.to_vec()) { + *result_target = Some(string_value); + } + } +} + +impl<'a> GetProperty for CffiControllerServiceContext<'a> { + fn get_property(&self, property: &Property) -> Result, MinifiError> { + let mut result: Option = None; + let property_name: StringView = StringView::new(property.name); + + let status = unsafe { + minifi_controller_service_context_get_property( + self.ptr, + property_name.as_raw(), + Some(property_callback), + &mut result as *mut _ as *mut c_void, + ) + }; + + #[allow(non_upper_case_globals)] + match status { + minifi_status_MINIFI_STATUS_SUCCESS => Ok(result), + _ => match property.is_required { + true => Err(MinifiError::MissingRequiredProperty(Cow::from( + property.name, + ))), + false => Ok(None), + }, + } + } +} diff --git a/minifi_rust/minifi_native/src/c_ffi/c_ffi_controller_service_definition.rs b/minifi_rust/minifi_native/src/c_ffi/c_ffi_controller_service_definition.rs new file mode 100644 index 0000000000..6923889a7e --- /dev/null +++ b/minifi_rust/minifi_native/src/c_ffi/c_ffi_controller_service_definition.rs @@ -0,0 +1,162 @@ +use crate::api::RawControllerService; +use crate::c_ffi::c_ffi_controller_service_context::CffiControllerServiceContext; +use crate::c_ffi::c_ffi_property::CProperties; +use crate::c_ffi::{CffiLogger, StaticStrAsMinifiCStr}; +use crate::{ + ComponentIdentifier, ControllerService, ControllerServiceDefinition, EnableControllerService, + LogLevel, Property, +}; +use minifi_native_sys::{ + minifi_controller_service_callbacks, minifi_controller_service_class_definition, + minifi_controller_service_context, minifi_controller_service_metadata, minifi_status, +}; +use std::ffi::c_void; + +#[derive(Debug)] +pub struct ControllerServiceClassDefinition<'a> { + inner: minifi_controller_service_class_definition, + _lifetime: std::marker::PhantomData<&'a ()>, +} + +impl<'a> ControllerServiceClassDefinition<'a> { + pub(crate) fn new(inner: minifi_controller_service_class_definition) -> Self { + Self { + inner, + _lifetime: std::marker::PhantomData, + } + } + + pub unsafe fn as_raw(&self) -> minifi_controller_service_class_definition { + self.inner + } +} + +pub struct CffiControllerServiceDefinition +where + T: RawControllerService + ComponentIdentifier, +{ + name: &'static str, + description_text: &'static str, + + c_properties: CProperties, + + _phantom: std::marker::PhantomData, +} + +impl CffiControllerServiceDefinition +where + T: RawControllerService + ComponentIdentifier, +{ + pub fn new(description_text: &'static str, properties: &'static [Property]) -> Self { + let c_properties = Property::create_c_properties(properties); + + Self { + name: T::CLASS_NAME, + description_text, + c_properties, + _phantom: std::marker::PhantomData, + } + } + + unsafe extern "C" fn create_controller_service( + metadata: minifi_controller_service_metadata, + ) -> *mut c_void { + let logger = CffiLogger::new(metadata.logger); + let controller_service = Box::new(T::new(logger)); + Box::into_raw(controller_service) as *mut c_void + } + + unsafe extern "C" fn destroy_controller_service(controller_service_ptr: *mut c_void) { + unsafe { + if !controller_service_ptr.is_null() { + let _ = Box::from_raw(controller_service_ptr as *mut T); + } + } + } + + unsafe extern "C" fn enable_controller_service( + controller_service_ptr: *mut c_void, + context_ptr: *mut minifi_controller_service_context, + ) -> minifi_status { + unsafe { + let controller_service = &mut *(controller_service_ptr as *mut T); + let context = CffiControllerServiceContext::new(context_ptr); + match controller_service.enable(&context) { + Ok(_) => 0, + Err(err) => { + controller_service.log(LogLevel::Error, format_args!("{:?}", err)); + err.to_status() + } + } + } + } + + unsafe extern "C" fn disable_controller_service(controller_service_ptr: *mut c_void) { + unsafe { + let controller_service = &mut *(controller_service_ptr as *mut T); + controller_service.disable() + } + } +} + +pub trait DynRawControllerServiceDefinition { + fn class_description(&'_ self) -> ControllerServiceClassDefinition<'_>; +} + +impl DynRawControllerServiceDefinition for CffiControllerServiceDefinition +where + T: RawControllerService + ComponentIdentifier, +{ + fn class_description(&'_ self) -> ControllerServiceClassDefinition<'_> { + unsafe { + ControllerServiceClassDefinition::new(minifi_controller_service_class_definition { + full_name: self.name.as_minifi_c_type(), + description: self.description_text.as_minifi_c_type(), + properties_count: self.c_properties.len(), + properties_ptr: self.c_properties.get_ptr(), + callbacks: minifi_controller_service_callbacks { + create: Some(Self::create_controller_service), + destroy: Some(Self::destroy_controller_service), + enable: Some(Self::enable_controller_service), + disable: Some(Self::disable_controller_service), + get_interface: None, // TODO(mzink) + }, + provided_interfaces_count: 0, + provided_interfaces_ptr: std::ptr::null_mut() // TODO(mzink) + }) + } + } +} + +pub trait RegisterableControllerService { + fn get_definition() -> Box; +} + +impl RegisterableControllerService for T +where + T: ComponentIdentifier + + ControllerServiceDefinition + + RawControllerService + + 'static, +{ + fn get_definition() -> Box { + Box::new(CffiControllerServiceDefinition::::new( + T::DESCRIPTION, + T::PROPERTIES, + )) + } +} + +impl RegisterableControllerService for ControllerService +where + Implementation: + EnableControllerService + ComponentIdentifier + ControllerServiceDefinition + 'static, +{ + fn get_definition() -> Box { + Box::new(CffiControllerServiceDefinition::< + ControllerService, + >::new( + Implementation::DESCRIPTION, Implementation::PROPERTIES + )) + } +} diff --git a/minifi_rust/minifi_native/src/c_ffi/c_ffi_controller_service_list.rs b/minifi_rust/minifi_native/src/c_ffi/c_ffi_controller_service_list.rs new file mode 100644 index 0000000000..d527490890 --- /dev/null +++ b/minifi_rust/minifi_native/src/c_ffi/c_ffi_controller_service_list.rs @@ -0,0 +1,53 @@ +use crate::c_ffi::RegisterableControllerService; +use crate::c_ffi::c_ffi_controller_service_definition::DynRawControllerServiceDefinition; +use minifi_native_sys::minifi_controller_service_class_definition; + +pub struct CffiControllerServiceList { + controller_service_definitions: Vec>, + minifi_controller_service_class_description_list: Vec, +} + +impl CffiControllerServiceList { + pub fn new() -> Self { + Self { + controller_service_definitions: Vec::new(), + minifi_controller_service_class_description_list: Vec::new(), + } + } + + pub fn add(&mut self) { + self.add_controller_service_definition(T::get_definition()) + } + + pub fn add_controller_service_definition( + &mut self, + processor_definition: Box, + ) { + unsafe { + self.controller_service_definitions + .push(processor_definition); + self.minifi_controller_service_class_description_list.push( + self.controller_service_definitions + .last() + .unwrap() + .class_description() + .as_raw(), + ); + } + } + + pub fn get_controller_service_count(&self) -> usize { + assert_eq!( + self.controller_service_definitions.len(), + self.minifi_controller_service_class_description_list.len() + ); + self.minifi_controller_service_class_description_list.len() + } + + pub unsafe fn get_controller_service_ptr( + &self, + ) -> *const minifi_controller_service_class_definition { + self.minifi_controller_service_class_description_list + .as_ptr() + } +} diff --git a/minifi_rust/minifi_native/src/c_ffi/c_ffi_flow_file.rs b/minifi_rust/minifi_native/src/c_ffi/c_ffi_flow_file.rs new file mode 100644 index 0000000000..0fdeac4b50 --- /dev/null +++ b/minifi_rust/minifi_native/src/c_ffi/c_ffi_flow_file.rs @@ -0,0 +1,22 @@ +use crate::api::FlowFile; +use minifi_native_sys::minifi_flow_file; + +pub struct CffiFlowFile<'a> { + ptr: *mut minifi_flow_file, + _lifetime: std::marker::PhantomData<&'a ()>, +} + +impl CffiFlowFile<'_> { + pub(crate) fn new(ptr: *mut minifi_flow_file) -> Self { + Self { + ptr, + _lifetime: std::marker::PhantomData, + } + } + + pub(crate) fn get_ptr(&self) -> *mut minifi_flow_file { + self.ptr + } +} + +impl FlowFile for CffiFlowFile<'_> {} diff --git a/minifi_rust/minifi_native/src/c_ffi/c_ffi_logger.rs b/minifi_rust/minifi_native/src/c_ffi/c_ffi_logger.rs new file mode 100644 index 0000000000..2ffb893c8e --- /dev/null +++ b/minifi_rust/minifi_native/src/c_ffi/c_ffi_logger.rs @@ -0,0 +1,52 @@ +use crate::api::LogLevel; +use crate::api::Logger; +use minifi_native_sys::{minifi_log_level, minifi_logger, minifi_logger_log_string, minifi_logger_should_log, minifi_string_view}; +use std::ffi::CString; +use std::fmt; + +impl From for minifi_log_level { + fn from(level: LogLevel) -> Self { + match level { + LogLevel::Trace => minifi_native_sys::minifi_log_level_MINIFI_LOG_LEVEL_TRACE, + LogLevel::Debug => minifi_native_sys::minifi_log_level_MINIFI_LOG_LEVEL_DEBUG, + LogLevel::Info => minifi_native_sys::minifi_log_level_MINIFI_LOG_LEVEL_INFO, + LogLevel::Warn => minifi_native_sys::minifi_log_level_MINIFI_LOG_LEVEL_WARNING, + LogLevel::Error => minifi_native_sys::minifi_log_level_MINIFI_LOG_LEVEL_ERROR, + LogLevel::Critical => minifi_native_sys::minifi_log_level_MINIFI_LOG_LEVEL_CRITICAL, + LogLevel::Off => minifi_native_sys::minifi_log_level_MINIFI_LOG_LEVEL_OFF, + } + } +} + +#[derive(Clone, Copy, Debug)] +pub struct CffiLogger { + ptr: *mut minifi_logger, +} + +impl CffiLogger { + pub fn new(logger: *mut minifi_logger) -> Self { + Self { ptr: logger } + } +} + +impl Logger for CffiLogger { + fn log(&self, level: LogLevel, args: fmt::Arguments) { + unsafe { + let message = fmt::format(args); + if let Ok(c_message) = CString::new(message) { + minifi_logger_log_string( + self.ptr, + level.into(), + minifi_string_view { + data: c_message.as_ptr(), + length: c_message.as_bytes().len(), + }, + ); + } + } + } + + fn should_log(&self, level: LogLevel) -> bool { + unsafe { minifi_logger_should_log(self.ptr, level.into()) } + } +} diff --git a/minifi_rust/minifi_native/src/c_ffi/c_ffi_output_attribute.rs b/minifi_rust/minifi_native/src/c_ffi/c_ffi_output_attribute.rs new file mode 100644 index 0000000000..3485b156a2 --- /dev/null +++ b/minifi_rust/minifi_native/src/c_ffi/c_ffi_output_attribute.rs @@ -0,0 +1,41 @@ +use crate::OutputAttribute; +use crate::c_ffi::StaticStrAsMinifiCStr; +use minifi_native_sys::{minifi_string_view, minifi_output_attribute_definition}; + +#[allow(dead_code)] // the c_ vecs are holding the values referenced from the output attributes +pub struct COutputAttributes { + c_relationship_names: Vec>, + c_output_attributes: Vec, +} + +impl COutputAttributes { + pub(crate) fn new(output_attributes: &[OutputAttribute]) -> Self { + let mut c_relationship_names = Vec::new(); + let mut c_output_attributes = Vec::new(); + for output_attribute in output_attributes { + let mut output_attribute_relationships = Vec::new(); + for relationship in output_attribute.relationships { + output_attribute_relationships.push(relationship.as_minifi_c_type()); + } + c_output_attributes.push(minifi_output_attribute_definition { + name: output_attribute.name.as_minifi_c_type(), + relationships_count: output_attribute_relationships.len(), + relationships_ptr: output_attribute_relationships.as_ptr(), + description: output_attribute.description.as_minifi_c_type(), + }); + c_relationship_names.push(output_attribute_relationships); + } + COutputAttributes { + c_relationship_names, + c_output_attributes, + } + } + + pub(crate) fn len(&self) -> usize { + self.c_output_attributes.len() + } + + pub(crate) unsafe fn get_ptr(&self) -> *const minifi_output_attribute_definition { + self.c_output_attributes.as_ptr() + } +} diff --git a/minifi_rust/minifi_native/src/c_ffi/c_ffi_primitives.rs b/minifi_rust/minifi_native/src/c_ffi/c_ffi_primitives.rs new file mode 100644 index 0000000000..687fbd4382 --- /dev/null +++ b/minifi_rust/minifi_native/src/c_ffi/c_ffi_primitives.rs @@ -0,0 +1,82 @@ +use crate::ProcessorInputRequirement; +use minifi_native_sys::{minifi_string_view, minifi_input_requirement_MINIFI_INPUT_REQUIRED, minifi_input_requirement_MINIFI_INPUT_ALLOWED, minifi_input_requirement_MINIFI_INPUT_FORBIDDEN, minifi_input_requirement}; +use std::os::raw::c_char; + +#[derive(Debug)] +pub enum FfiConversionError { + NullPointer, + InvalidUtf8, +} + +#[repr(transparent)] // This allows us to pass Vec as a pointer to minifi_string_view array +#[derive(Debug)] +pub(crate) struct StringView<'a> { + inner: minifi_string_view, + _marker: std::marker::PhantomData<&'a ()>, +} + +impl<'a> StringView<'a> { + pub(crate) fn new(str: &'a str) -> Self { + Self { + inner: minifi_string_view { + data: str.as_ptr() as *const c_char, + length: str.len(), + }, + _marker: std::marker::PhantomData, + } + } + + pub unsafe fn as_raw(&self) -> minifi_string_view { + self.inner + } +} + +pub trait StaticStrAsMinifiCStr { + fn as_minifi_c_type(&self) -> minifi_string_view; +} + +impl StaticStrAsMinifiCStr for &'static str { + fn as_minifi_c_type(&self) -> minifi_string_view { + minifi_string_view { + data: self.as_ptr() as *const c_char, + length: self.len(), + } + } +} + +pub trait ConvertMinifiStringView { + unsafe fn as_string(&self) -> Result; + unsafe fn as_str(&self) -> Result<&str, FfiConversionError>; +} + +impl ConvertMinifiStringView for minifi_string_view { + unsafe fn as_string(&self) -> Result { + if self.data.is_null() { + return Err(FfiConversionError::NullPointer); + } + unsafe { + let slice = std::slice::from_raw_parts(self.data as *const u8, self.length); + String::from_utf8(slice.to_vec()).map_err(|_| FfiConversionError::InvalidUtf8) + } + } + + unsafe fn as_str(&self) -> Result<&str, FfiConversionError> { + if self.data.is_null() { + return Err(FfiConversionError::NullPointer); + } + unsafe { + let slice = std::slice::from_raw_parts(self.data as *const u8, self.length); + str::from_utf8(slice).map_err(|_| FfiConversionError::InvalidUtf8) + } + } +} + +impl ProcessorInputRequirement { + pub fn as_minifi_c_type(&self) -> minifi_input_requirement { + match self { + ProcessorInputRequirement::Required => minifi_input_requirement_MINIFI_INPUT_REQUIRED, + ProcessorInputRequirement::Allowed => minifi_input_requirement_MINIFI_INPUT_ALLOWED, + ProcessorInputRequirement::Forbidden => minifi_input_requirement_MINIFI_INPUT_FORBIDDEN, + } + } +} diff --git a/minifi_rust/minifi_native/src/c_ffi/c_ffi_process_context.rs b/minifi_rust/minifi_native/src/c_ffi/c_ffi_process_context.rs new file mode 100644 index 0000000000..e84f7fa108 --- /dev/null +++ b/minifi_rust/minifi_native/src/c_ffi/c_ffi_process_context.rs @@ -0,0 +1,146 @@ +use super::c_ffi_flow_file::CffiFlowFile; +use super::c_ffi_primitives::StringView; +use crate::api::ProcessContext; +use crate::api::controller_service::ControllerService; +use crate::c_ffi::{CffiLogger, StaticStrAsMinifiCStr}; +use crate::{ComponentIdentifier, EnableControllerService, MinifiError, Property}; +use minifi_native_sys::*; +use std::borrow::Cow; +use std::ffi::c_void; +use std::num::NonZeroU32; + +/// A safe wrapper around a `minifi_process_context` pointer. +pub struct CffiProcessContext<'a> { + ptr: *mut minifi_process_context, + _lifetime: std::marker::PhantomData<&'a ()>, +} + +impl<'a> CffiProcessContext<'a> { + pub fn new(ptr: *mut minifi_process_context) -> Self { + Self { + ptr, + _lifetime: std::marker::PhantomData, + } + } +} + +unsafe extern "C" fn get_property_callback( + output_option: *mut c_void, + property_c_value: minifi_string_view, +) { + unsafe { + let result_target = &mut *(output_option as *mut Option); + + if property_c_value.data.is_null() || property_c_value.length == 0 { + *result_target = None; + return; + } + + let value_slice = + std::slice::from_raw_parts(property_c_value.data as *const u8, property_c_value.length); + if let Ok(string_value) = String::from_utf8(value_slice.to_vec()) { + *result_target = Some(string_value); + } + } +} + +impl<'a> ProcessContext for CffiProcessContext<'a> { + type FlowFile = CffiFlowFile<'a>; // FlowFile shouldn't outlive the ProcessContext + fn get_property( + &self, + property: &Property, + flow_file: Option<&Self::FlowFile>, + ) -> Result, MinifiError> { + let ff_ptr = flow_file.map_or(std::ptr::null_mut(), |ff| ff.get_ptr()); + + let mut result: Option = None; + let property_name: StringView = StringView::new(property.name); + + #[allow(non_upper_case_globals)] + unsafe { + match minifi_process_context_get_property( + self.ptr, + property_name.as_raw(), + ff_ptr, + Some(get_property_callback), + &mut result as *mut _ as *mut c_void, + ) { + minifi_status_MINIFI_STATUS_SUCCESS => Ok(result), + minifi_status_MINIFI_STATUS_PROPERTY_NOT_SET => match property.is_required { + true => Err(MinifiError::MissingRequiredProperty(Cow::from( + property.name, + ))), + false => Ok(None), + }, + err_code => Err(MinifiError::StatusError(( + format!("minifi_process_context_get_property({:?})", property.name).into(), + NonZeroU32::new_unchecked(err_code), + ))), + } + } + } + + fn get_raw_controller_service( + &self, + property: &Property, + ) -> Result, MinifiError> + where + Cs: ComponentIdentifier + 'static, + { + let str_view = StringView::new(property.name); + + let mut controller_service_ptr: *mut minifi_controller_service = std::ptr::null_mut(); + + unsafe { + let get_cs_status = minifi_process_context_get_controller_service_from_property( + self.ptr, + str_view.as_raw(), + Cs::CLASS_NAME.as_minifi_c_type(), + &mut controller_service_ptr, + ); + if get_cs_status != minifi_status_MINIFI_STATUS_SUCCESS { + return Err(MinifiError::StatusError(( + format!( + "minifi_process_context_get_controller_service_from_property::<{:?}>({:?})", + Cs::CLASS_NAME, + property + ) + .into(), + NonZeroU32::new_unchecked(get_cs_status), + ))); + } + let cs_ref: &Cs = { + (controller_service_ptr as *const Cs) + .as_ref() + .expect("C returned a null pointer") + }; + Ok(Some(cs_ref)) + } + } + + fn get_controller_service(&self, property: &Property) -> Result, MinifiError> + where + Cs: EnableControllerService + ComponentIdentifier + 'static, + { + match self.get_raw_controller_service::>(property)? { + None => Ok(None), + Some(f) => Ok(f.get_implementation()), + } + } + + fn report_metrics(&self, metrics: Vec<(String, f64)>) -> Result<(), MinifiError>{ + let mut keys = Vec::new(); + let mut values = Vec::new(); + for (key, value) in &metrics { + keys.push(StringView::new(key)); + values.push(*value); + } + unsafe { + let err_code = minifi_process_context_report_metrics(self.ptr, keys.len(), keys.as_ptr() as *const minifi_string_view, values.as_ptr()); + if err_code != minifi_status_MINIFI_STATUS_SUCCESS { + return Err(MinifiError::StatusError(("report_metrics".into(), NonZeroU32::new_unchecked(err_code)))); + } + } + Ok(()) + } +} diff --git a/minifi_rust/minifi_native/src/c_ffi/c_ffi_process_session.rs b/minifi_rust/minifi_native/src/c_ffi/c_ffi_process_session.rs new file mode 100644 index 0000000000..3ec6a71237 --- /dev/null +++ b/minifi_rust/minifi_native/src/c_ffi/c_ffi_process_session.rs @@ -0,0 +1,554 @@ +use super::c_ffi_flow_file::CffiFlowFile; +use crate::MinifiError; +use crate::api::process_session::{IoState, OutputStream}; +use crate::api::{InputStream, ProcessSession}; +use crate::c_ffi::c_ffi_primitives::{ConvertMinifiStringView, StringView}; +use crate::c_ffi::c_ffi_streams::{CffiInputStream, CffiOutputStream}; +use minifi_native_sys::{minifi_status_MINIFI_STATUS_SUCCESS, minifi_string_view, minifi_process_session, minifi_output_stream, minifi_output_stream_write, minifi_io_status_MINIFI_IO_ERROR, minifi_process_session_write, minifi_process_session_create, minifi_process_session_get, minifi_process_session_transfer, minifi_process_session_remove, minifi_process_session_set_flow_file_attribute, minifi_process_session_get_flow_file_attribute, minifi_process_session_get_flow_file_attributes, minifi_io_status_MINIFI_IO_CANCEL, minifi_input_stream, minifi_input_stream_size, minifi_input_stream_read, minifi_process_session_read, minifi_process_session_get_flow_file_id}; +use std::ffi::{CString, c_void}; +use std::io::Read; +use std::num::NonZeroU32; +use std::os::raw::c_char; + +const _: () = { + if minifi_status_MINIFI_STATUS_SUCCESS != 0 { + panic!("minifi_status_MINIFI_STATUS_SUCCESS expected to be 0"); + } +}; + +pub struct CffiProcessSession<'a> { + ptr: *mut minifi_process_session, + // The lifetime ensures the session cannot outlive the `on_trigger` call. + _lifetime: std::marker::PhantomData<&'a ()>, +} + +impl<'a> CffiProcessSession<'a> { + pub fn new(ptr: *mut minifi_process_session) -> Self { + Self { + ptr, + _lifetime: std::marker::PhantomData, + } + } + + fn write_in_batches( + &self, + flow_file: &CffiFlowFile, + produce_batch: F, + ) -> Result<(), MinifiError> + where + F: FnMut(&mut [u8]) -> Option, + { + unsafe { + struct State<'b, F: FnMut(&mut [u8]) -> Option> { + callback: F, + buffer: &'b mut [u8], + } + + let mut buffer = [0u8; 8192]; + let mut state = State { + callback: produce_batch, + buffer: &mut buffer, + }; + + unsafe extern "C" fn cb Option>( + user_ctx: *mut c_void, + output_stream: *mut minifi_output_stream, + ) -> i64 { + unsafe { + let state = &mut *(user_ctx as *mut State); + let mut overall_writes = 0; + + // The user fills our provided buffer + while let Some(n) = (state.callback)(state.buffer) { + let written = minifi_output_stream_write( + output_stream, + state.buffer.as_ptr() as *const c_char, + n, + ); + + if written < 0 { + return minifi_io_status_MINIFI_IO_ERROR; + } + overall_writes += written; + } + overall_writes + } + } + + match minifi_process_session_write( + self.ptr, + flow_file.get_ptr(), + Some(cb::), + &mut state as *mut _ as *mut c_void, + ) { + #[allow(non_upper_case_globals)] + minifi_status_MINIFI_STATUS_SUCCESS => Ok(()), + error_code => Err(MinifiError::StatusError(( + "MinifiProcessSessionWrite".into(), + NonZeroU32::new_unchecked(error_code), + ))), + } + } + } +} + +impl<'a> ProcessSession for CffiProcessSession<'a> { + type FlowFile = CffiFlowFile<'a>; // FlowFile shouldn't outlive the Session + + fn create(&mut self) -> Result { + let ff_ptr = unsafe { minifi_process_session_create(self.ptr, std::ptr::null_mut()) }; + if ff_ptr.is_null() { + Err(MinifiError::UnknownError) + } else { + Ok(CffiFlowFile::new(ff_ptr)) + } + } + + fn get(&mut self) -> Option { + let ff_ptr = unsafe { minifi_process_session_get(self.ptr) }; + if ff_ptr.is_null() { + None + } else { + Some(CffiFlowFile::new(ff_ptr)) + } + } + + fn transfer(&self, flow_file: Self::FlowFile, relationship: &str) -> Result<(), MinifiError> { + let c_relationship = CString::new(relationship)?; + unsafe { + match minifi_process_session_transfer( + self.ptr, + flow_file.get_ptr(), + minifi_string_view { + data: c_relationship.as_ptr(), + length: c_relationship.as_bytes().len(), + }, + ) { + #[allow(non_upper_case_globals)] + minifi_status_MINIFI_STATUS_SUCCESS => Ok(()), + err_code => Err(MinifiError::StatusError(( + "MinifiProcessSessionTransfer".into(), + NonZeroU32::new_unchecked(err_code), + ))), + } + } + } + + fn remove(&mut self, flow_file: Self::FlowFile) -> Result<(), MinifiError> { + unsafe { + match minifi_process_session_remove(self.ptr, flow_file.get_ptr()) { + #[allow(non_upper_case_globals)] + minifi_status_MINIFI_STATUS_SUCCESS => Ok(()), + err_code => Err(MinifiError::StatusError(( + "MinifiProcessSessionRemove".into(), + NonZeroU32::new_unchecked(err_code), + ))), + } + } + } + + fn set_attribute( + &self, + flow_file: &mut Self::FlowFile, + attr_key: &str, + attr_value: &str, + ) -> Result<(), MinifiError> { + unsafe { + let attr_key_string_view = StringView::new(attr_key); + let attr_value_string_view = StringView::new(attr_value); + match minifi_process_session_set_flow_file_attribute( + self.ptr, + flow_file.get_ptr(), + attr_key_string_view.as_raw(), + &attr_value_string_view.as_raw(), + ) { + #[allow(non_upper_case_globals)] + minifi_status_MINIFI_STATUS_SUCCESS => Ok(()), + err_code => Err(MinifiError::StatusError(( + format!( + "MinifiProcessSessionFlowFileSetAttribute({}, {})", + attr_key, attr_value + ) + .into(), + NonZeroU32::new_unchecked(err_code), + ))), + } + } + } + + fn get_attribute(&self, flow_file: &Self::FlowFile, attr_key: &str) -> Option { + let mut attr_value: Option = None; + unsafe { + unsafe extern "C" fn cb( + rs_attr_value: *mut c_void, + minifi_attr_value: minifi_string_view, + ) { + unsafe { + let result_target = &mut *(rs_attr_value as *mut Option); + *result_target = minifi_attr_value.as_string().ok() + } + } + + let attr_key_string_view = StringView::new(attr_key); + minifi_process_session_get_flow_file_attribute( + self.ptr, + flow_file.get_ptr(), + attr_key_string_view.as_raw(), + Some(cb), + &mut attr_value as *mut _ as *mut c_void, + ); + } + attr_value + } + + fn on_attributes( + &self, + flow_file: &Self::FlowFile, + process_attr: F, + ) -> bool { + struct OnAttrHelper { + result: bool, + process_attr: F, + } + + let mut on_attr_helper = OnAttrHelper { + result: true, + process_attr, + }; + + unsafe extern "C" fn get_attributes_callback<'b, F: FnMut(&str, &str)>( + user_ctx: *mut c_void, + minifi_attr_key: minifi_string_view, + minifi_attr_val: minifi_string_view, + ) { + unsafe { + let helper = &mut *(user_ctx as *mut OnAttrHelper); + let attr_key = minifi_attr_key.as_str(); + let attr_value = minifi_attr_val.as_str(); + if attr_key.is_err() || attr_value.is_err() { + helper.result = false; + return; + } + (helper.process_attr)(attr_key.unwrap(), attr_value.unwrap()); + } + } + + unsafe { + minifi_process_session_get_flow_file_attributes( + self.ptr, + flow_file.get_ptr(), + Some(get_attributes_callback::), + &mut on_attr_helper as *mut _ as *mut c_void, + ) + } + on_attr_helper.result + } + + fn write(&self, flow_file: &Self::FlowFile, data: &[u8]) -> Result<(), MinifiError> { + let mut dt: Option<&[u8]> = Some(data); + unsafe { + unsafe extern "C" fn cb( + user_ctx: *mut c_void, + output_stream: *mut minifi_output_stream, + ) -> i64 { + unsafe { + let result_target = &mut *(user_ctx as *mut Option<&[u8]>); + if result_target.is_none() { + return minifi_io_status_MINIFI_IO_ERROR; + } + + minifi_output_stream_write( + output_stream, + result_target.unwrap().as_ptr() as *const c_char, + result_target.unwrap().len(), + ) + } + } + + match minifi_process_session_write( + self.ptr, + flow_file.get_ptr(), + Some(cb), + &mut dt as *mut _ as *mut c_void, + ) { + #[allow(non_upper_case_globals)] + minifi_status_MINIFI_STATUS_SUCCESS => Ok(()), + err_code => Err(MinifiError::StatusError(( + "MinifiProcessSessionWrite".into(), + NonZeroU32::new_unchecked(err_code), + ))), + } + } + } + + fn write_lazy<'b>( + &self, + flow_file: &Self::FlowFile, + mut stream: Box, + ) -> Result<(), MinifiError> { + self.write_in_batches(flow_file, |buffer| { + match stream.read(buffer) { + Ok(0) => None, // EOF + Ok(n) => Some(n), + Err(_e) => None, // Signal failure/EOF + } + }) + } + + fn write_stream(&self, flow_file: &Self::FlowFile, callback: F) -> Result + where + F: FnOnce(&mut dyn OutputStream) -> Result<(R, IoState), MinifiError>, + { + struct WriteCallbackCtx { + callback: Option, + result: Option>, + } + + let mut ctx = WriteCallbackCtx { + callback: Some(callback), + result: None, + }; + + unsafe extern "C" fn write_cb( + user_data: *mut c_void, + stream_ptr: *mut minifi_output_stream, + ) -> i64 + where + F: FnOnce(&mut dyn OutputStream) -> Result<(R, IoState), MinifiError>, + { + unsafe { + let ctx = &mut *(user_data as *mut WriteCallbackCtx); + + let mut writer = CffiOutputStream::new(stream_ptr); + + if let Some(cb) = ctx.callback.take() { + let (cb_result, state) = match cb(&mut writer) { + Ok((f, b)) => (Ok(f), b), + Err(e) => (Err(e), IoState::Cancel), + }; + let is_err = cb_result.is_err(); + ctx.result = Some(cb_result); + if is_err { + return minifi_io_status_MINIFI_IO_ERROR; + } + if state == IoState::Cancel { + return minifi_io_status_MINIFI_IO_CANCEL; + } + } + + writer.written_bytes() as i64 + } + } + + unsafe { + let session_status = minifi_process_session_write( + self.ptr, + flow_file.get_ptr(), + Some(write_cb::), + &mut ctx as *mut _ as *mut c_void, + ); + if session_status != minifi_status_MINIFI_STATUS_SUCCESS { + return Err(MinifiError::StatusError(( + "MinifiProcessSessionWrite".into(), + NonZeroU32::new_unchecked(session_status), + ))); + } + } + + ctx.result + .expect("Agent returned with success, so ctx.result should be set") + } + + fn read(&self, flow_file: &Self::FlowFile) -> Option> { + let mut output: Option> = None; + unsafe { + unsafe extern "C" fn cb( + output_option: *mut c_void, + input_stream: *mut minifi_input_stream, + ) -> i64 { + unsafe { + let result_target = &mut *(output_option as *mut Option>); + + let stream_size = minifi_input_stream_size(input_stream); + if stream_size == 0 { + *result_target = None; + return 0; + } + let mut buffer: Vec = Vec::with_capacity(stream_size); + + let bytes_read = minifi_input_stream_read( + input_stream, + buffer.as_mut_ptr() as *mut c_char, + stream_size, + ); + + if bytes_read < 0 { + *result_target = None; + return bytes_read; + } + + buffer.set_len(bytes_read as usize); + + *result_target = Some(buffer); + bytes_read + } + } + + minifi_process_session_read( + self.ptr, + flow_file.get_ptr(), + Some(cb), + &mut output as *mut _ as *mut c_void, + ); + } + output + } + + fn read_stream(&self, flow_file: &Self::FlowFile, callback: F) -> Result + where + F: FnOnce(&mut dyn InputStream) -> Result, + { + unsafe { + struct CallbackCtx { + callback: Option, + result: Option>, + } + let mut ctx = CallbackCtx { + callback: Some(callback), + result: None, + }; + + unsafe extern "C" fn read_cb( + user_data: *mut c_void, + stream_ptr: *mut minifi_input_stream, + ) -> i64 + where + F: FnOnce(&mut dyn InputStream) -> Result, + { + unsafe { + let ctx = &mut *(user_data as *mut CallbackCtx); + + let mut reader = CffiInputStream::new(stream_ptr); + + if let Some(cb) = ctx.callback.take() { + ctx.result = Some(cb(&mut reader)); + } + + reader.total_bytes_read() as i64 + } + } + + minifi_process_session_read( + self.ptr, + flow_file.get_ptr(), + Some(read_cb::), + &mut ctx as *mut _ as *mut c_void, + ); + + ctx.result + .expect("Agent returned with success, so ctx.result should be set") + } + } + + fn read_in_batches( + &self, + flow_file: &Self::FlowFile, + batch_size: usize, + process_batch: F, + ) -> Result<(), MinifiError> + where + F: FnMut(&[u8]) -> Result<(), MinifiError>, + { + struct BatchReadHelper + where + F: FnMut(&[u8]) -> Result<(), MinifiError>, + { + batch_size: usize, + process_batch: F, + } + + let mut batch_helper = BatchReadHelper { + batch_size, + process_batch, + }; + unsafe { + unsafe extern "C" fn cb( + output_option: *mut c_void, + input_stream: *mut minifi_input_stream, + ) -> i64 + where + F: FnMut(&[u8]) -> Result<(), MinifiError>, + { + unsafe { + let batch_helper = &mut *(output_option as *mut BatchReadHelper); + + let mut remaining_size = minifi_input_stream_size(input_stream); + let mut overall_read = 0; + while remaining_size > 0 { + let read_size = remaining_size.min(batch_helper.batch_size); + let mut buffer: Vec = Vec::with_capacity(read_size); + + let bytes_read = minifi_input_stream_read( + input_stream, + buffer.as_mut_ptr() as *mut c_char, + read_size, + ); + if bytes_read < 0 || bytes_read > read_size as i64 { + return minifi_io_status_MINIFI_IO_ERROR; + } + + buffer.set_len(bytes_read as usize); + + match (batch_helper.process_batch)(&*buffer) { + Ok(_) => {} + Err(err) => { + eprintln!("Error during read_in_batch {:?}", err); + return minifi_io_status_MINIFI_IO_ERROR; + } + } + remaining_size -= bytes_read as usize; + overall_read += bytes_read; + } + overall_read + } + } + + match minifi_process_session_read( + self.ptr, + flow_file.get_ptr(), + Some(cb::), + &mut batch_helper as *mut _ as *mut c_void, + ) { + #[allow(non_upper_case_globals)] + minifi_status_MINIFI_STATUS_SUCCESS => Ok(()), + status_code => Err(MinifiError::StatusError(( + "minifi_process_session_read".into(), + NonZeroU32::new_unchecked(status_code), + ))), + } + } + } + + fn get_flow_file_id(&self, flow_file: &Self::FlowFile) -> Result { + let mut attr_value: Option = None; + unsafe { + unsafe extern "C" fn cb( + rs_flow_file_id: *mut c_void, + minifi_flow_file_id: minifi_string_view, + ) { + unsafe { + let result_target = &mut *(rs_flow_file_id as *mut Option); + *result_target = minifi_flow_file_id.as_string().ok() + } + } + + minifi_process_session_get_flow_file_id( + self.ptr, + flow_file.get_ptr(), + Some(cb), + &mut attr_value as *mut _ as *mut c_void, + ); + } + attr_value.ok_or(MinifiError::UnknownError) + } +} diff --git a/minifi_rust/minifi_native/src/c_ffi/c_ffi_processor_definition.rs b/minifi_rust/minifi_native/src/c_ffi/c_ffi_processor_definition.rs new file mode 100644 index 0000000000..67cacd566d --- /dev/null +++ b/minifi_rust/minifi_native/src/c_ffi/c_ffi_processor_definition.rs @@ -0,0 +1,266 @@ +use std::ffi::c_void; +use std::ptr; + +use super::c_ffi_primitives::{StaticStrAsMinifiCStr}; +use super::c_ffi_process_context::CffiProcessContext; +use super::c_ffi_process_session::CffiProcessSession; +use crate::api::raw_processor::{MultiThreadedTrigger, SingleThreadedTrigger}; +use crate::api::{ProcessorInputRequirement, RawProcessor, ThreadingModel}; +use crate::c_ffi::CffiLogger; +use crate::c_ffi::c_ffi_output_attribute::COutputAttributes; +use crate::c_ffi::c_ffi_property::CProperties; +use crate::{ + ComponentIdentifier, Concurrent, Exclusive, + LogLevel, OutputAttribute, Processor, ProcessorDefinition, Property, Schedule, +}; +use crate::{OnTriggerResult, Relationship}; +use minifi_native_sys::*; + +pub trait DispatchOnTrigger { + unsafe fn dispatch_on_trigger( + processor: *mut c_void, + context: *mut minifi_process_context, + session: *mut minifi_process_session, + ) -> minifi_status; +} + +impl DispatchOnTrigger for T +where + T: MultiThreadedTrigger, +{ + unsafe fn dispatch_on_trigger( + processor_ptr: *mut c_void, + context_ptr: *mut minifi_process_context, + session_ptr: *mut minifi_process_session, + ) -> minifi_status { + unsafe { + let processor = &*(processor_ptr as *const T); + let mut context = CffiProcessContext::new(context_ptr); + let mut session = CffiProcessSession::new(session_ptr); + match processor.on_trigger(&mut context, &mut session) { + Ok(OnTriggerResult::Ok) => minifi_status_MINIFI_STATUS_SUCCESS, + Ok(OnTriggerResult::Yield) => minifi_status_MINIFI_STATUS_PROCESSOR_YIELD, + Err(minifi_error) => { + processor.log( + LogLevel::Error, + format_args!("Error during on_trigger {}", minifi_error), + ); + minifi_error.to_status() + } + } + } + } +} + +impl DispatchOnTrigger for T +where + T: SingleThreadedTrigger, +{ + unsafe fn dispatch_on_trigger( + processor_ptr: *mut c_void, + context_ptr: *mut minifi_process_context, + session_ptr: *mut minifi_process_session, + ) -> minifi_status { + unsafe { + let processor = &mut *(processor_ptr as *mut T); + let mut context = CffiProcessContext::new(context_ptr); + let mut session = CffiProcessSession::new(session_ptr); + match processor.on_trigger(&mut context, &mut session) { + Ok(OnTriggerResult::Ok) => minifi_status_MINIFI_STATUS_SUCCESS, + Ok(OnTriggerResult::Yield) => minifi_status_MINIFI_STATUS_PROCESSOR_YIELD, + Err(error_code) => error_code.to_status(), + } + } + } +} + +pub struct RawProcessorDefinition +where + T: RawProcessor + DispatchOnTrigger, +{ + name: &'static str, + description_text: &'static str, + input_requirement: ProcessorInputRequirement, + supports_dynamic_properties: bool, + supports_dynamic_relationships: bool, + + c_output_attributes: COutputAttributes, + c_relationships: Vec, + c_properties: CProperties, + + _phantom: std::marker::PhantomData, +} + +impl RawProcessorDefinition +where + T: RawProcessor + DispatchOnTrigger, +{ + pub fn new( + name: &'static str, + description_text: &'static str, + input_requirement: ProcessorInputRequirement, + supports_dynamic_properties: bool, + supports_dynamic_relationships: bool, + output_attributes: &'static [OutputAttribute], + relationships: &'static [Relationship], + properties: &'static [Property], + ) -> Self { + let c_relationships = Relationship::create_c_vec(relationships); + let c_properties = Property::create_c_properties(properties); + let c_output_attributes = COutputAttributes::new(output_attributes); + + Self { + name, + description_text, + input_requirement, + supports_dynamic_properties, + supports_dynamic_relationships, + c_output_attributes, + c_relationships, + c_properties, + _phantom: std::marker::PhantomData, + } + } + + unsafe extern "C" fn create_processor(metadata: minifi_processor_metadata) -> *mut c_void { + let logger = CffiLogger::new(metadata.logger); + let processor = Box::new(T::new(logger)); + Box::into_raw(processor) as *mut c_void + } + + unsafe extern "C" fn destroy_processor(processor_ptr: *mut c_void) { + unsafe { + if !processor_ptr.is_null() { + let _ = Box::from_raw(processor_ptr as *mut T); + } + } + } + + unsafe extern "C" fn on_trigger_processor( + processor_ptr: *mut c_void, + context_ptr: *mut minifi_process_context, + session_ptr: *mut minifi_process_session, + ) -> minifi_status { + unsafe { + >::dispatch_on_trigger( + processor_ptr, + context_ptr, + session_ptr, + ) + } + } + + unsafe extern "C" fn on_schedule_processor( + processor_ptr: *mut c_void, + context_ptr: *mut minifi_process_context, + ) -> minifi_status { + unsafe { + let processor = &mut *(processor_ptr as *mut T); + let context = CffiProcessContext::new(context_ptr); + match processor.on_schedule(&context) { + Ok(_) => 0, + Err(error_code) => { + processor.log( + LogLevel::Error, + format_args!("Error during on_schedule: {}", error_code), + ); + error_code.to_status() + } + } + } + } + + unsafe extern "C" fn on_unschedule_processor(processor_ptr: *mut c_void) { + unsafe { + let processor = &mut *(processor_ptr as *mut T); + processor.on_unschedule(); + } + } +} + +#[derive(Debug)] +pub struct ProcessorClassDefinition<'a> { + inner: minifi_processor_class_definition, + _marker: std::marker::PhantomData<&'a ()>, +} + +impl<'a> ProcessorClassDefinition<'a> { + pub(crate) fn new(inner: minifi_processor_class_definition) -> Self { + Self { + inner, + _marker: std::marker::PhantomData, + } + } + + pub unsafe fn as_raw(&self) -> minifi_processor_class_definition { + self.inner + } +} + +pub trait DynRawProcessorDefinition { + fn class_description(&'_ self) -> ProcessorClassDefinition<'_>; +} + +impl DynRawProcessorDefinition for RawProcessorDefinition +where + T: RawProcessor + DispatchOnTrigger, +{ + fn class_description(&'_ self) -> ProcessorClassDefinition<'_> { + unsafe { + ProcessorClassDefinition::new(minifi_processor_class_definition { + full_name: self.name.as_minifi_c_type(), + description: self.description_text.as_minifi_c_type(), + properties_count: self.c_properties.len(), + properties_ptr: self.c_properties.get_ptr(), + dynamic_properties_count: 0, + dynamic_properties_ptr: ptr::null(), + relationships_count: self.c_relationships.len(), + relationships_ptr: self.c_relationships.as_ptr(), + output_attributes_count: self.c_output_attributes.len(), + output_attributes_ptr: self.c_output_attributes.get_ptr(), + supports_dynamic_properties: self.supports_dynamic_properties, + supports_dynamic_relationships: self.supports_dynamic_relationships, + input_requirement: self.input_requirement.as_minifi_c_type(), + is_single_threaded: T::Threading::IS_EXCLUSIVE, + callbacks: minifi_processor_callbacks { + create: Some(Self::create_processor), + destroy: Some(Self::destroy_processor), + trigger: Some(Self::on_trigger_processor), + schedule: Some(Self::on_schedule_processor), + unschedule: Some(Self::on_unschedule_processor), + }, + }) + } + } +} + +pub trait RawRegisterableProcessor { + fn get_definition() -> Box; +} + +impl RawRegisterableProcessor + for Processor +where + Threading: ThreadingModel + 'static, + Implementation: Schedule + + ComponentIdentifier + + ProcessorDefinition + + 'static, + Processor: + RawProcessor + DispatchOnTrigger, +{ + fn get_definition() -> Box { + Box::new(RawProcessorDefinition::< + Processor, + >::new( + Implementation::CLASS_NAME, + Implementation::DESCRIPTION, + Implementation::INPUT_REQUIREMENT, + Implementation::SUPPORTS_DYNAMIC_PROPERTIES, + Implementation::SUPPORTS_DYNAMIC_RELATIONSHIPS, + Implementation::OUTPUT_ATTRIBUTES, + Implementation::RELATIONSHIPS, + Implementation::PROPERTIES, + )) + } +} diff --git a/minifi_rust/minifi_native/src/c_ffi/c_ffi_processor_list.rs b/minifi_rust/minifi_native/src/c_ffi/c_ffi_processor_list.rs new file mode 100644 index 0000000000..9f5bf153bf --- /dev/null +++ b/minifi_rust/minifi_native/src/c_ffi/c_ffi_processor_list.rs @@ -0,0 +1,49 @@ +use minifi_native_sys::minifi_processor_class_definition; +use crate::c_ffi::RawRegisterableProcessor; +use crate::c_ffi::c_ffi_processor_definition::DynRawProcessorDefinition; + +pub struct CffiProcessorList { + processor_definitions: Vec>, + minifi_processor_class_description_list: Vec, +} + +impl CffiProcessorList { + pub fn new() -> Self { + Self { + processor_definitions: Vec::new(), + minifi_processor_class_description_list: Vec::new(), + } + } + + pub fn add(&mut self) { + self.add_processor_definition(T::get_definition()) + } + + pub fn add_processor_definition( + &mut self, + processor_definition: Box, + ) { + unsafe { + self.processor_definitions.push(processor_definition); + self.minifi_processor_class_description_list.push( + self.processor_definitions + .last() + .unwrap() + .class_description() + .as_raw(), + ); + } + } + + pub fn get_processor_count(&self) -> usize { + assert_eq!( + self.processor_definitions.len(), + self.minifi_processor_class_description_list.len() + ); + self.minifi_processor_class_description_list.len() + } + + pub unsafe fn get_processor_ptr(&self) -> *const minifi_processor_class_definition { + self.minifi_processor_class_description_list.as_ptr() + } +} diff --git a/minifi_rust/minifi_native/src/c_ffi/c_ffi_property.rs b/minifi_rust/minifi_native/src/c_ffi/c_ffi_property.rs new file mode 100644 index 0000000000..11ebe4c5ca --- /dev/null +++ b/minifi_rust/minifi_native/src/c_ffi/c_ffi_property.rs @@ -0,0 +1,132 @@ +use super::c_ffi_primitives::StaticStrAsMinifiCStr; +use crate::{Property, StandardPropertyValidator}; +use minifi_native_sys::{minifi_property_definition, minifi_validator, minifi_string_view, minifi_validator_MINIFI_VALIDATOR_ALWAYS_VALID, minifi_validator_MINIFI_VALIDATOR_BOOLEAN, minifi_validator_MINIFI_VALIDATOR_DATA_SIZE, minifi_validator_MINIFI_VALIDATOR_INTEGER, minifi_validator_MINIFI_VALIDATOR_NON_BLANK, minifi_validator_MINIFI_VALIDATOR_PORT, minifi_validator_MINIFI_VALIDATOR_TIME_PERIOD, minifi_validator_MINIFI_VALIDATOR_UNSIGNED_INTEGER}; +use std::ptr; + +#[allow(dead_code)] // these c_ vecs are holding the values referenced from the properties, so they live long enough for registration +pub struct CProperties { + c_default_values: Vec, + c_allowed_values: Vec>, + c_allowed_types: Vec, + properties: Vec, +} + +impl CProperties { + pub(crate) fn new( + c_default_values: Vec, + c_allowed_values: Vec>, + c_allowed_types: Vec, + properties: Vec, + ) -> Self { + Self { + c_default_values, + c_allowed_values, + c_allowed_types, + properties, + } + } + + pub(crate) fn len(&self) -> usize { + self.properties.len() + } + + pub(crate) unsafe fn get_ptr(&self) -> *const minifi_property_definition { + self.properties.as_ptr() + } +} + +impl Property { + fn create_c_default_value_holder(properties: &[Self]) -> Vec { + properties + .iter() + .map(|p| match p.default_value { + Some(dv) => dv.as_minifi_c_type(), + None => minifi_string_view { + data: ptr::null(), + length: 0, + }, + }) + .collect() + } + + fn create_c_allowed_values_vec_vec(properties: &[Self]) -> Vec> { + properties + .iter() + .map(|p| { + p.allowed_values + .iter() + .map(|av| av.as_minifi_c_type()) + .collect() + }) + .collect() + } + + fn create_c_allowed_types_vec(properties: &[Self]) -> Vec { + properties + .iter() + .map(|p| p.allowed_type.as_minifi_c_type()) + .collect() + } + + pub(crate) fn create_c_properties(properties: &[Self]) -> CProperties { + let c_default_values = Property::create_c_default_value_holder(properties); + let c_allowed_values = Property::create_c_allowed_values_vec_vec(properties); + let c_allowed_types = Property::create_c_allowed_types_vec(properties); + assert_eq!(c_default_values.len(), properties.len()); + assert_eq!(c_allowed_values.len(), properties.len()); + assert_eq!(c_allowed_types.len(), properties.len()); + + let c_properties = properties + .iter() + .zip(c_default_values.iter()) + .zip(c_allowed_values.iter()) + .zip(c_allowed_types.iter()) + .map(|(((property, def_value), allowed_values), allowed_type)| { + minifi_property_definition { + name: property.name.as_minifi_c_type(), + display_name: property.name.as_minifi_c_type(), + description: property.description.as_minifi_c_type(), + is_required: property.is_required, + is_sensitive: property.is_sensitive, + default_value: def_value, + allowed_values_count: allowed_values.len(), + allowed_values_ptr: allowed_values.as_ptr(), + validator: property.validator.as_minifi_c_type(), + allowed_type, + supports_expression_language: property.supports_expr_lang, + } + }) + .collect(); + CProperties::new( + c_default_values, + c_allowed_values, + c_allowed_types, + c_properties, + ) + } +} + +impl StandardPropertyValidator { + pub(crate) fn as_minifi_c_type(&self) -> minifi_validator { + match self { + StandardPropertyValidator::AlwaysValidValidator => { + minifi_validator_MINIFI_VALIDATOR_ALWAYS_VALID + } + StandardPropertyValidator::NonBlankValidator => { + minifi_validator_MINIFI_VALIDATOR_NON_BLANK + } + StandardPropertyValidator::TimePeriodValidator => { + minifi_validator_MINIFI_VALIDATOR_TIME_PERIOD + } + StandardPropertyValidator::BoolValidator => minifi_validator_MINIFI_VALIDATOR_BOOLEAN, + StandardPropertyValidator::I64Validator => minifi_validator_MINIFI_VALIDATOR_INTEGER, + StandardPropertyValidator::U64Validator => { + minifi_validator_MINIFI_VALIDATOR_UNSIGNED_INTEGER + } + StandardPropertyValidator::DataSizeValidator => { + minifi_validator_MINIFI_VALIDATOR_DATA_SIZE + } + StandardPropertyValidator::PortValidator => minifi_validator_MINIFI_VALIDATOR_PORT, + } + } +} diff --git a/minifi_rust/minifi_native/src/c_ffi/c_ffi_relationship.rs b/minifi_rust/minifi_native/src/c_ffi/c_ffi_relationship.rs new file mode 100644 index 0000000000..86d3ee9083 --- /dev/null +++ b/minifi_rust/minifi_native/src/c_ffi/c_ffi_relationship.rs @@ -0,0 +1,15 @@ +use super::c_ffi_primitives::StaticStrAsMinifiCStr; +use crate::Relationship; +use minifi_native_sys::minifi_relationship_definition; + +impl Relationship { + pub(crate) fn create_c_vec(relationships: &[Self]) -> Vec { + relationships + .iter() + .map(|r| minifi_relationship_definition { + name: r.name.as_minifi_c_type(), + description: r.description.as_minifi_c_type(), + }) + .collect() + } +} diff --git a/minifi_rust/minifi_native/src/c_ffi/c_ffi_streams.rs b/minifi_rust/minifi_native/src/c_ffi/c_ffi_streams.rs new file mode 100644 index 0000000000..64fbaf95f1 --- /dev/null +++ b/minifi_rust/minifi_native/src/c_ffi/c_ffi_streams.rs @@ -0,0 +1,114 @@ +use std::io::{BufRead, Error, ErrorKind, Read}; +use minifi_native_sys::{minifi_input_stream, minifi_input_stream_read, minifi_output_stream, minifi_output_stream_write}; + +#[derive(Debug)] +pub struct CffiInputStream<'a> { + ptr: *mut minifi_input_stream, + buffer: [u8; 8192], + pos: usize, + cap: usize, + total_read: usize, + _marker: std::marker::PhantomData<&'a ()>, +} + +unsafe impl<'a> Send for CffiInputStream<'a> {} + +impl<'a> CffiInputStream<'a> { + pub fn new(ptr: *mut minifi_input_stream) -> Self { + Self { + ptr, + buffer: [0u8; 8192], + pos: 0, + cap: 0, + total_read: 0, + _marker: std::marker::PhantomData, + } + } + + pub fn total_bytes_read(&self) -> usize { + self.total_read + } +} + +impl<'a> Read for CffiInputStream<'a> { + fn read(&mut self, buf: &mut [u8]) -> std::io::Result { + // Delegate to the BufRead implementation to ensure consistency + let nread = { + let mut rem = self.fill_buf()?; + rem.read(buf)? + }; + self.consume(nread); + Ok(nread) + } +} + +impl<'a> BufRead for CffiInputStream<'a> { + fn fill_buf(&mut self) -> std::io::Result<&[u8]> { + if self.pos >= self.cap { + unsafe { + let ret = minifi_input_stream_read( + self.ptr, + self.buffer.as_mut_ptr() as *mut std::ffi::c_char, + self.buffer.len(), + ); + if ret < 0 { + return Err(Error::new(ErrorKind::Other, "Minifi Read Error")); + } + self.cap = ret as usize; + self.pos = 0; + } + } + Ok(&self.buffer[self.pos..self.cap]) + } + + fn consume(&mut self, amount: usize) { + let actual_consumed = std::cmp::min(amount, self.cap - self.pos); + self.pos += actual_consumed; + self.total_read += actual_consumed; + } +} + +#[derive(Debug)] +pub struct CffiOutputStream<'a> { + ptr: *mut minifi_output_stream, + written_bytes: usize, + _marker: std::marker::PhantomData<&'a ()>, +} + +impl<'a> CffiOutputStream<'a> { + pub(crate) fn new(ptr: *mut minifi_output_stream) -> Self { + Self { + ptr, + written_bytes: 0, + _marker: std::marker::PhantomData, + } + } +} + +impl<'a> CffiOutputStream<'a> { + pub fn written_bytes(&self) -> usize { + self.written_bytes + } +} + +unsafe impl<'a> Send for CffiOutputStream<'a> {} + +impl<'a> std::io::Write for CffiOutputStream<'a> { + fn write(&mut self, buf: &[u8]) -> std::io::Result { + unsafe { + let ret = minifi_output_stream_write( + self.ptr, + buf.as_ptr() as *const std::ffi::c_char, + buf.len(), + ); + if ret < 0 { + return Err(Error::new(ErrorKind::Other, "Minifi Write Error")); + } + self.written_bytes += ret as usize; + Ok(ret as usize) + } + } + fn flush(&mut self) -> std::io::Result<()> { + Ok(()) // Handled by C++ session commit + } +} diff --git a/minifi_rust/minifi_native/src/lib.rs b/minifi_rust/minifi_native/src/lib.rs new file mode 100644 index 0000000000..97286e3337 --- /dev/null +++ b/minifi_rust/minifi_native/src/lib.rs @@ -0,0 +1,116 @@ +mod api; +pub mod c_ffi; +pub mod mock; + +pub use api::errors::MinifiError; + +pub use api::component_definition_traits::{ + ComponentIdentifier, ControllerServiceDefinition, ProcessorDefinition, +}; +pub use api::controller_service::{ControllerService, EnableControllerService}; +pub use api::processor_wrappers::complex_processor::{ComplexProcessorType, MutTrigger, Trigger}; +pub use api::processor_wrappers::flow_file_source::{ + FlowFileSource, FlowFileSourceProcessorType, GeneratedFlowFile, +}; +pub use api::processor_wrappers::flow_file_stream_transform::{ + FlowFileStreamTransform, FlowFileStreamTransformProcessorType, MutFlowFileStreamTransform, + TransformStreamResult, +}; +pub use api::processor_wrappers::flow_file_transform::{ + FlowFileTransform, FlowFileTransformProcessorType, TransformedFlowFile, +}; + +pub use api::processor_wrappers::utils::flow_file_content::Content; + +pub use api::processor::{Processor, Schedule}; + +pub use api::raw_processor::{Concurrent, Exclusive}; + +pub use api::logger::{LogLevel, Logger}; + +pub use api::property::{GetControllerService, GetProperty, Property}; + +pub use api::process_session::IoState; + +pub use api::attribute::{GetAttribute, OutputAttribute}; + +pub use api::{ + FlowFile, InputStream, OnTriggerResult, OutputStream, ProcessContext, ProcessSession, + ProcessorInputRequirement, Relationship, StandardPropertyValidator, +}; + +pub use minifi_native_macros as macros; +pub use minifi_native_sys as sys; +pub use mock::{ + MockControllerServiceContext, MockFlowFile, MockLogger, MockProcessContext, MockProcessSession, + StdLogger, +}; + +#[unsafe(no_mangle)] +#[allow(non_upper_case_globals)] +#[cfg_attr(target_os = "linux", unsafe(link_section = ".rodata"))] +#[cfg_attr(target_os = "macos", unsafe(link_section = "__DATA,__const"))] +#[cfg_attr(target_os = "windows", unsafe(link_section = ".rdata"))] +pub static MinifiApiVersion: u32 = minifi_native_sys::MINIFI_API_VERSION; + +/// Defines the required MinifiInitExtension C function to register the listed processors and controller services +#[macro_export] +macro_rules! declare_minifi_extension { + ( + // Match a tuple of three types for each processor + processors: [ $( ($kind:ty, $thread:ty, $impl:ty) ),* $(,)? ], + // Match a single type for each controller service + controllers: [ $( $ctrl:ty ),* $(,)? ] + ) => { + + #[unsafe(no_mangle)] + pub extern "C" fn minifi_init_extension( + extension_context: *mut minifi_native::sys::minifi_extension_context + ) { + + use minifi_native::c_ffi::StaticStrAsMinifiCStr; + use minifi_native::c_ffi::RawRegisterableProcessor; + use minifi_native::c_ffi::RegisterableControllerService; + unsafe { + let extension_definition = minifi_native::sys::minifi_extension_definition { + name: env!("CARGO_PKG_NAME").as_minifi_c_type(), + version: env!("CARGO_PKG_VERSION").as_minifi_c_type(), + deinit: None, + user_data: std::ptr::null_mut(), + }; + + let extension = minifi_native::sys::minifi_register_extension(extension_context, &extension_definition); + + + $( + { + let processor_def = minifi_native::Processor::< + $impl, + $kind, + $thread, + minifi_native::c_ffi::CffiLogger + >::get_definition(); + + let proc_desc = processor_def.class_description(); + + minifi_native::sys::minifi_register_processor(extension, &proc_desc.as_raw()); + } + )* + + $( + { + let controller_def = + minifi_native::ControllerService::< + $ctrl, + minifi_native::c_ffi::CffiLogger + >::get_definition(); + + let controller_desc = controller_def.class_description(); + + minifi_native::sys::minifi_register_controller_service(extension, &controller_desc.as_raw()); + } + )* + } + } + }; +} diff --git a/minifi_rust/minifi_native/src/mock.rs b/minifi_rust/minifi_native/src/mock.rs new file mode 100644 index 0000000000..e98b637468 --- /dev/null +++ b/minifi_rust/minifi_native/src/mock.rs @@ -0,0 +1,12 @@ +mod mock_controller_service_context; +mod mock_flow_file; +mod mock_logger; +mod mock_process_context; +mod mock_process_session; + +pub use mock_controller_service_context::MockControllerServiceContext; +pub use mock_flow_file::MockFlowFile; +pub use mock_logger::MockLogger; +pub use mock_logger::StdLogger; +pub use mock_process_context::MockProcessContext; +pub use mock_process_session::MockProcessSession; diff --git a/minifi_rust/minifi_native/src/mock/mock_controller_service_context.rs b/minifi_rust/minifi_native/src/mock/mock_controller_service_context.rs new file mode 100644 index 0000000000..e24c0bce6d --- /dev/null +++ b/minifi_rust/minifi_native/src/mock/mock_controller_service_context.rs @@ -0,0 +1,20 @@ +use crate::mock::mock_process_context::MockPropertyMap; +use crate::{GetProperty, MinifiError, Property}; + +pub struct MockControllerServiceContext { + pub properties: MockPropertyMap, +} + +impl GetProperty for MockControllerServiceContext { + fn get_property(&self, property: &Property) -> Result, MinifiError> { + self.properties.get_property(property, None) + } +} + +impl MockControllerServiceContext { + pub fn new() -> Self { + Self { + properties: MockPropertyMap::new(), + } + } +} diff --git a/minifi_rust/minifi_native/src/mock/mock_flow_file.rs b/minifi_rust/minifi_native/src/mock/mock_flow_file.rs new file mode 100644 index 0000000000..06b2c61966 --- /dev/null +++ b/minifi_rust/minifi_native/src/mock/mock_flow_file.rs @@ -0,0 +1,45 @@ +use crate::api::FlowFile; +use std::cell::RefCell; +use std::collections::HashMap; + +pub struct MockFlowFile { + pub content: RefCell>, + pub attributes: HashMap, + pub id: String, +} + +impl FlowFile for MockFlowFile {} + +impl MockFlowFile { + pub fn new() -> MockFlowFile { + MockFlowFile { + content: RefCell::new(Vec::new()), + attributes: HashMap::new(), + id: "67e55044-10b1-426f-9247-bb680e5fe0c8".to_string(), // TODO generate something? + } + } + + pub fn with_content(content: &[u8]) -> MockFlowFile { + Self { + content: RefCell::new(content.to_vec()), + attributes: HashMap::new(), + id: "67e55044-10b1-426f-9247-bb680e5fe0c8".to_string(), // TODO generate something? + } + } + + pub fn content_len(&self) -> usize { + self.content.borrow().len() + } + + pub fn content_eq(&self, other: S) -> bool + where + S: Into, + { + let my_content = self.content.borrow(); + *my_content == other.into().as_bytes() + } + + pub fn get_stream(&self) -> std::io::Cursor> { + std::io::Cursor::new(self.content.borrow().clone()) + } +} diff --git a/minifi_rust/minifi_native/src/mock/mock_logger.rs b/minifi_rust/minifi_native/src/mock/mock_logger.rs new file mode 100644 index 0000000000..a6451ac9b4 --- /dev/null +++ b/minifi_rust/minifi_native/src/mock/mock_logger.rs @@ -0,0 +1,83 @@ +use crate::LogLevel::Trace; +use crate::api::LogLevel; +use crate::api::Logger; +use std::fmt; +use std::sync::Mutex; + +#[derive(Debug)] +pub struct MockLogger { + pub logs: Mutex>, + pub log_level: LogLevel, +} + +impl Logger for MockLogger { + fn log(&self, level: LogLevel, args: fmt::Arguments) { + let message = fmt::format(args); + let mut logs_guard = self.logs.lock().unwrap(); + logs_guard.push((level, message.to_string())); + } + + fn should_log(&self, level: LogLevel) -> bool { + level >= self.log_level + } +} + +impl MockLogger { + pub fn new() -> Self { + MockLogger { + logs: Mutex::new(Vec::new()), + log_level: Trace, + } + } +} + +/// For easier debugging +#[derive(Debug)] +pub struct StdLogger { + pub log_level: LogLevel, +} + +impl Logger for StdLogger { + fn log(&self, level: LogLevel, args: fmt::Arguments) { + let message = fmt::format(args); + println!("[{}] {}", level, message); + } + + fn should_log(&self, level: LogLevel) -> bool { + level >= self.log_level + } +} + +#[cfg(test)] +mod tests { + use crate::api::logger::Logger; + use crate::{LogLevel, MockLogger, error, trace}; + + #[test] + fn test_macro_laziness() { + let mut mock_logger = MockLogger::new(); + mock_logger.log_level = LogLevel::Warn; + + let mut call_count = 0; + + trace!( + mock_logger, + "This is a trace message {}", + || -> u32 { + call_count += 1; + call_count + }() + ); + error!( + mock_logger, + "This is an error message {}", + || -> u32 { + call_count += 1; + call_count + }() + ); + + assert_eq!(mock_logger.logs.lock().unwrap().len(), 1); + assert_eq!(call_count, 1); + } +} diff --git a/minifi_rust/minifi_native/src/mock/mock_process_context.rs b/minifi_rust/minifi_native/src/mock/mock_process_context.rs new file mode 100644 index 0000000000..c68afd96d0 --- /dev/null +++ b/minifi_rust/minifi_native/src/mock/mock_process_context.rs @@ -0,0 +1,121 @@ +use crate::api::{ProcessContext, RawControllerService}; +use crate::{ + ComponentIdentifier, EnableControllerService, GetAttribute, MinifiError, MockFlowFile, Property, +}; +use std::any::Any; +use std::borrow::Cow; +use std::collections::HashMap; + +pub struct MockPropertyMap { + pub properties: HashMap, +} + +impl MockPropertyMap { + pub fn new() -> Self { + Self { + properties: HashMap::new(), + } + } + + pub fn insert(&mut self, key: K, value: V) + where + K: Into, + V: Into, + { + self.properties.insert(key.into(), value.into()); + } + + pub fn extend(&mut self, iter: I) + where + I: IntoIterator, + K: Into, + V: Into, + { + self.properties + .extend(iter.into_iter().map(|(k, v)| (k.into(), v.into()))) + } +} + +impl MockPropertyMap { + pub fn get_property( + &self, + property: &Property, + _flow_file: Option<&MockFlowFile>, + ) -> Result, MinifiError> { + if let Some(property) = self.properties.get(property.name) { + Ok(Some(property.clone())) + } else { + if let Some(default_val) = property.default_value { + return Ok(Some(default_val.to_string())); + } + match property.is_required { + true => Err(MinifiError::MissingRequiredProperty(Cow::from( + property.name, + ))), + false => Ok(None), + } + } + } +} + +pub struct MockProcessContext { + pub properties: MockPropertyMap, + pub controller_services: HashMap>, + pub attributes: HashMap, +} + +impl ProcessContext for MockProcessContext { + type FlowFile = MockFlowFile; + + fn get_property( + &self, + property: &Property, + _flow_file: Option<&Self::FlowFile>, + ) -> Result, MinifiError> { + self.properties.get_property(property, _flow_file) + } + + fn get_raw_controller_service( + &self, + _property: &Property, + ) -> Result, MinifiError> + where + Cs: RawControllerService + ComponentIdentifier + 'static, + { + panic!("Not implemented yet"); + } + + fn get_controller_service(&self, property: &Property) -> Result, MinifiError> + where + Cs: EnableControllerService + ComponentIdentifier + 'static, + { + if let Some(service_name) = self.get_property(property, None)? { + Ok(self + .controller_services + .get(&service_name) + .and_then(|c| c.downcast_ref::())) + } else { + Ok(None) + } + } + + fn report_metrics(&self, _metrics: Vec<(String, f64)>) -> Result<(), MinifiError> { + Ok(()) + } +} + +impl MockProcessContext { + pub fn new() -> Self { + Self { + properties: MockPropertyMap::new(), + controller_services: HashMap::new(), + attributes: HashMap::new(), + } + } +} + +impl GetAttribute for MockProcessContext { + fn get_attribute(&self, name: &str) -> Result, MinifiError> { + Ok(self.attributes.get(name).cloned()) + } +} diff --git a/minifi_rust/minifi_native/src/mock/mock_process_session.rs b/minifi_rust/minifi_native/src/mock/mock_process_session.rs new file mode 100644 index 0000000000..475eb3a8cf --- /dev/null +++ b/minifi_rust/minifi_native/src/mock/mock_process_session.rs @@ -0,0 +1,181 @@ +use crate::api::process_session::IoState; +use crate::api::{InputStream, ProcessSession}; +use crate::{MinifiError, MockFlowFile}; +use itertools::Itertools; +use std::cell::RefCell; +use std::io::Read; + +pub struct TransferredFlowFile { + pub relationship: String, + pub flow_file: MockFlowFile, +} + +pub struct MockProcessSession { + pub input_flow_files: Vec, + pub transferred_flow_files: RefCell>, +} + +impl ProcessSession for MockProcessSession { + type FlowFile = MockFlowFile; + + fn create(&mut self) -> Result { + Ok(Self::FlowFile::new()) + } + fn get(&mut self) -> Option { + self.input_flow_files.pop() + } + fn transfer(&self, flow_file: Self::FlowFile, relationship: &str) -> Result<(), MinifiError> { + self.transferred_flow_files + .borrow_mut() + .push(TransferredFlowFile { + relationship: relationship.to_string(), + flow_file, + }); + Ok(()) + } + + fn remove(&mut self, _flow_file: Self::FlowFile) -> Result<(), MinifiError> { + Ok(()) + } + + fn set_attribute( + &self, + flow_file: &mut Self::FlowFile, + attr_key: &str, + attr_value: &str, + ) -> Result<(), MinifiError> { + flow_file + .attributes + .insert(attr_key.to_string(), attr_value.to_string()); + Ok(()) + } + fn get_attribute(&self, flow_file: &Self::FlowFile, attr_key: &str) -> Option { + flow_file.attributes.get(attr_key).cloned() + } + + fn on_attributes( + &self, + flow_file: &Self::FlowFile, + mut process_attr: F, + ) -> bool { + // Sorting for deterministic tests. + for (attr_key, attr_value) in flow_file.attributes.iter().sorted_by_key(|x| x.0) { + process_attr(attr_key, attr_value); + } + true + } + + fn write(&self, flow_file: &Self::FlowFile, data: &[u8]) -> Result<(), MinifiError> { + *flow_file.content.borrow_mut() = data.to_vec(); + Ok(()) + } + + fn write_lazy<'a>( + &self, + flow_file: &Self::FlowFile, + mut stream: Box, + ) -> Result<(), MinifiError> { + stream + .read_to_end(&mut flow_file.content.borrow_mut()) + .expect("Mock data should be readable"); + Ok(()) + } + + fn write_stream(&self, flow_file: &Self::FlowFile, callback: F) -> Result + where + F: FnOnce( + &mut dyn crate::api::process_session::OutputStream, + ) -> Result<(R, IoState), MinifiError>, + { + let mut new_content: Vec = Vec::new(); + let mut cursor = std::io::Cursor::new(&mut new_content); + let (r, _state) = callback(&mut cursor)?; + *flow_file.content.borrow_mut() = new_content; + Ok(r) + } + + fn read(&self, flow_file: &Self::FlowFile) -> Option> { + Some(flow_file.content.borrow().clone()) + } + + fn read_stream(&self, _flow_file: &Self::FlowFile, _callback: F) -> Result + where + F: FnOnce(&mut dyn InputStream) -> Result, + { + unimplemented!("Not implemented yet") + } + + fn read_in_batches( + &self, + flow_file: &Self::FlowFile, + batch_size: usize, + mut process_batch: F, + ) -> Result<(), MinifiError> + where + F: FnMut(&[u8]) -> Result<(), MinifiError>, + { + for chunk in flow_file.content.borrow().chunks(batch_size) { + process_batch(chunk)?; + } + Ok(()) + } + + fn get_flow_file_id(&self, flow_file: &Self::FlowFile) -> Result { + Ok(flow_file.id.clone()) + } +} + +impl MockProcessSession { + pub fn new() -> Self { + Self { + transferred_flow_files: RefCell::new(Vec::new()), + input_flow_files: Vec::new(), + } + } + + pub fn num_of_transferred_flow_files(&self) -> usize { + self.transferred_flow_files.borrow().len() + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::api::process_session::IoState; + + #[test] + fn test_read_in_batches() { + let session = MockProcessSession::new(); + let mut flow_file = MockFlowFile::new(); + flow_file.content = RefCell::from("Hello, World!".to_string().as_bytes().to_vec()); + let mut vec: Vec = Vec::new(); + + let res = session.read_in_batches(&mut flow_file, 1, |batch| { + assert_eq!(batch.len(), 1); + vec.push(batch[0]); + Ok(()) + }); + + assert!(res.is_ok()); + + assert_eq!(vec.len(), 13); + assert_eq!(vec, b"Hello, World!"); + } + + #[test] + fn test_write_stream_replaces_content() { + let session = MockProcessSession::new(); + let flow_file = MockFlowFile::new(); + // Pre-populate the flow file with longer content + *flow_file.content.borrow_mut() = b"Hello, World!".to_vec(); + + let res: Result<(), MinifiError> = session.write_stream(&flow_file, |stream| { + stream.write_all(b"Hi")?; + Ok(((), IoState::Ok)) + }); + + assert!(res.is_ok()); + // Old trailing bytes must not survive + assert_eq!(*flow_file.content.borrow(), b"Hi"); + } +} diff --git a/minifi_rust/minifi_native_macros/Cargo.toml b/minifi_rust/minifi_native_macros/Cargo.toml new file mode 100644 index 0000000000..c5e6edf9a0 --- /dev/null +++ b/minifi_rust/minifi_native_macros/Cargo.toml @@ -0,0 +1,11 @@ +[package] +name = "minifi_native_macros" +version = "0.1.0" +edition = "2024" + +[lib] +proc-macro = true + +[dependencies] +syn = "2.0" +quote = "1.0" diff --git a/minifi_rust/minifi_native_macros/src/lib.rs b/minifi_rust/minifi_native_macros/src/lib.rs new file mode 100644 index 0000000000..9961a58664 --- /dev/null +++ b/minifi_rust/minifi_native_macros/src/lib.rs @@ -0,0 +1,20 @@ +use proc_macro::TokenStream; +use quote::quote; +use syn::{DeriveInput, parse_macro_input}; + +#[proc_macro_derive(ComponentIdentifier)] +pub fn derive_component_identifier(input: TokenStream) -> TokenStream { + let input = parse_macro_input!(input as DeriveInput); + let name = &input.ident; + let name_str = name.to_string(); + + let expanded = quote! { + impl ::minifi_native::ComponentIdentifier for #name { + const CLASS_NAME: &'static str = concat!(module_path!(), "::", #name_str); + const GROUP_NAME: &'static str = env!("CARGO_PKG_NAME"); + const VERSION: &'static str = env!("CARGO_PKG_VERSION"); + } + }; + + TokenStream::from(expanded) +} diff --git a/minifi_rust/minifi_native_sys/.gitignore b/minifi_rust/minifi_native_sys/.gitignore new file mode 100644 index 0000000000..a9d37c560c --- /dev/null +++ b/minifi_rust/minifi_native_sys/.gitignore @@ -0,0 +1,2 @@ +target +Cargo.lock diff --git a/minifi_rust/minifi_native_sys/Cargo.toml b/minifi_rust/minifi_native_sys/Cargo.toml new file mode 100644 index 0000000000..db9a249ed0 --- /dev/null +++ b/minifi_rust/minifi_native_sys/Cargo.toml @@ -0,0 +1,18 @@ +[package] +name = "minifi_native_sys" +version = "0.1.0" +edition = "2024" +description = "Raw FFI bindings to the Apache MiNiFi Native C API" +license = "Apache-2.0" +links = "minifi" + +[features] +default = ["last-release"] + +last-release = [] +local-sdk = [] +local-repo = [] + +[build-dependencies] +bindgen = "0.72.0" +cc = "1.2.41" diff --git a/minifi_rust/minifi_native_sys/build.rs b/minifi_rust/minifi_native_sys/build.rs new file mode 100644 index 0000000000..9b09df452e --- /dev/null +++ b/minifi_rust/minifi_native_sys/build.rs @@ -0,0 +1,195 @@ +use std::env; +use std::path::{Path, PathBuf}; +use std::process::Command; + +struct SDK { + header_path: PathBuf, + #[cfg_attr(not(windows), allow(dead_code))] + def_path: PathBuf, + behave_path: PathBuf, +} + +impl SDK { + fn from_local_repo(repo_root: &Path) -> Option { + let canon = std::fs::canonicalize(&repo_root).ok()?; + println!( + "cargo:info=MINIFI_SDK_PATH from from_local_repo: {:?} -> {:?}", + repo_root, canon + ); + + let header_path = canon + .to_path_buf() + .join("minifi-api/include/minifi-c/minifi-api.h"); + let def_path = canon.to_path_buf().join("minifi-api/minifi-api.def"); + let behave_path = canon.to_path_buf().join("behave_framework"); + + println!( + "cargo:info=header_path {:?}\n def_path {:?}\n behave_path {:?}", + header_path, def_path, behave_path + ); + + if !header_path.exists() || !def_path.exists() { + return None; + } + + Some(Self { + header_path, + def_path, + behave_path, + }) + } + + fn from_local_sdk_path(sdk_path: &Path) -> Option { + let base_path = if sdk_path.join("minifi-api.h").exists() { + sdk_path.to_path_buf() + } else if sdk_path.join("minifi-native-sdk/minifi-api.h").exists() { + sdk_path.join("minifi-native-sdk") + } else { + return None; + }; + + let header_path = base_path.join("minifi-api.h"); + let def_path = base_path.join("minifi-api.def"); + let behave_path = std::fs::read_dir(&base_path) + .ok()? + .filter_map(|e| e.ok()) + .find(|e| e.path().extension().unwrap_or_default() == "whl") + .map(|e| e.path())?; + + if !header_path.exists() || !def_path.exists() || !behave_path.exists() { + return None; + } + + Some(Self { + header_path, + def_path, + behave_path, + }) + } + + fn new_from_env_variable() -> Option { + let out_dir = PathBuf::from(env::var("OUT_DIR").unwrap()); + let sdk_path = env::var("MINIFI_SDK_PATH").expect( + "MINIFI_SDK_PATH must be set. Please define it in your environment or .cargo/config.toml" + ); + + println!("cargo:info=Using MINIFI_SDK_PATH: {}", sdk_path); + + if sdk_path.starts_with("https://") { + let zip_path = out_dir.join("downloaded-sdk.zip"); + let extract_dir = out_dir.join("extracted-sdk"); + if !extract_dir.exists() { + Self::download_file(&sdk_path, &zip_path); + Self::extract_zip(&zip_path, &extract_dir); + } + Self::from_local_sdk_path(&extract_dir) + } else { + let local_path = if sdk_path.starts_with(".") { + PathBuf::from(format!("../{}", sdk_path)) + } else { + PathBuf::from(&sdk_path) + }; + if local_path.is_file() && local_path.extension().unwrap_or_default() == "zip" { + let extract_dir = out_dir.join("extracted-local-sdk"); + if !extract_dir.exists() { + Self::extract_zip(&local_path, &extract_dir); + } + Self::from_local_sdk_path(&extract_dir) + } else { + Self::from_local_sdk_path(&local_path) + .or_else(|| Self::from_local_repo(&local_path)) + } + } + } + + fn download_file(url: &str, dest: &Path) { + println!("cargo:warning=Downloading SDK from {}...", url); + let status = Command::new("curl") + .args(["-fSL", "-o", dest.to_str().unwrap(), url]) + .status() + .expect("Failed to execute curl to download SDK"); + assert!(status.success(), "Failed to download SDK from {}", url); + } + + fn extract_zip(archive: &Path, dest_dir: &Path) { + println!("cargo:warning=Extracting SDK archive..."); + std::fs::create_dir_all(dest_dir).unwrap(); + let status = Command::new("tar") + .args([ + "-xf", + archive.to_str().unwrap(), + "-C", + dest_dir.to_str().unwrap(), + ]) + .status() + .expect("Failed to execute tar to extract SDK zip"); + assert!(status.success(), "Failed to extract SDK zip file"); + } +} + +#[cfg(windows)] +fn generate_minifi_c_api_lib(def_path: &Path) { + let tool = cc::Build::new() + .try_get_compiler() + .expect("Failed to find the MSVC toolchain. Is Visual Studio or the C++ Build Tools workload installed?"); + + let lib_exe_path = tool + .path() + .parent() + .expect("Compiler path is expected to be in a 'bin' directory.") + .join("lib.exe"); + + if !lib_exe_path.exists() { + panic!( + "Could not find lib.exe at the expected path: {}", + lib_exe_path.display() + ); + } + + let out_dir = PathBuf::from(env::var("OUT_DIR").unwrap()); + let lib_out_path = out_dir.join("minifi-api.lib"); + + let status = Command::new(&lib_exe_path) + .arg(format!("/def:{}", def_path.display())) + .arg(format!("/out:{}", lib_out_path.display())) + .arg(format!( + "/machine:{}", + env::var("CARGO_CFG_TARGET_ARCH").unwrap().to_uppercase() + )) + .status() + .expect("Failed to execute lib.exe."); + + if !status.success() { + panic!("lib.exe failed to create the import library from the .def file."); + } + + println!("cargo:rustc-link-lib=dylib=minifi-api"); + println!("cargo:rustc-link-search=native={}", out_dir.display()); +} + +fn main() { + println!("cargo:rerun-if-env-changed=MINIFI_SDK_PATH"); + + let sdk = SDK::new_from_env_variable() + .expect("Couldn't find valid SDK. Ensure MINIFI_SDK_PATH is set correctly."); + + #[cfg(windows)] + generate_minifi_c_api_lib(&sdk.def_path); + + println!("cargo:rerun-if-changed={}", sdk.header_path.display()); + println!( + "cargo:behave_path={}", + sdk.behave_path.canonicalize().unwrap().display() + ); + + let bindings = bindgen::Builder::default() + .header(sdk.header_path.to_str().unwrap()) + .parse_callbacks(Box::new(bindgen::CargoCallbacks::new())) + .generate() + .expect("Unable to generate bindings"); + + let out_path = PathBuf::from(env::var("OUT_DIR").unwrap()); + bindings + .write_to_file(out_path.join("bindings.rs")) + .expect("Couldn't write bindings!"); +} diff --git a/minifi_rust/minifi_native_sys/src/lib.rs b/minifi_rust/minifi_native_sys/src/lib.rs new file mode 100644 index 0000000000..1b9a6c3e45 --- /dev/null +++ b/minifi_rust/minifi_native_sys/src/lib.rs @@ -0,0 +1,11 @@ +// minifi-sys/src/lib.rs + +#![allow(non_upper_case_globals)] +#![allow(non_camel_case_types)] +#![allow(non_snake_case)] + +//! This crate provides raw, unsafe FFI bindings for the Apache NiFi MiNiFi C API. +//! +//! The bindings are generated automatically by `bindgen` from the `minifi-c.h` header. + +include!(concat!(env!("OUT_DIR"), "/bindings.rs")); diff --git a/minifi_rust/minifi_rs_behave/Cargo.toml b/minifi_rust/minifi_rs_behave/Cargo.toml new file mode 100644 index 0000000000..c13e8521e5 --- /dev/null +++ b/minifi_rust/minifi_rs_behave/Cargo.toml @@ -0,0 +1,12 @@ +[package] +name = "minifi_rs_behave" +version = "0.1.0" +edition = "2024" + +[dependencies] +glob = "0.3.3" +minifi_rs_playground = { path = "../extensions/minifi_rs_playground" } +minifi_native_sys = { path = "../minifi_native_sys" } + +[lints.rust] +unused_crate_dependencies = "allow" diff --git a/minifi_rust/minifi_rs_behave/alpine.dockerfile b/minifi_rust/minifi_rs_behave/alpine.dockerfile new file mode 100644 index 0000000000..b32cf63f02 --- /dev/null +++ b/minifi_rust/minifi_rs_behave/alpine.dockerfile @@ -0,0 +1,27 @@ +FROM rust:alpine3.22 AS chef +# Install build dependencies required for compiling C code & extensions on Alpine +RUN apk add --no-cache musl-dev gcc g++ clang-dev lld pkgconfig curl tar +RUN cargo install cargo-chef +WORKDIR /app + +FROM chef AS planner +COPY . . +RUN cargo chef prepare --recipe-path recipe.json + +FROM chef AS builder +ARG MINIFI_SDK_PATH +ENV MINIFI_SDK_PATH=${MINIFI_SDK_PATH} + +COPY --from=planner /app/recipe.json recipe.json + +# Conditionally copy the local SDK files from the target directory +COPY target/.docker_sd[k] /app/target/.docker_sdk/ +COPY target/.docker_sdk.zi[p] /app/target/ + +RUN cargo chef cook --release --recipe-path recipe.json +COPY . . +RUN cargo build --release + +# Export Stage +FROM scratch AS bin-export +COPY --from=builder /app/target/release/libminifi_rs_playground.so / diff --git a/minifi_rust/minifi_rs_behave/build.rs b/minifi_rust/minifi_rs_behave/build.rs new file mode 100644 index 0000000000..a675fa0e2b --- /dev/null +++ b/minifi_rust/minifi_rs_behave/build.rs @@ -0,0 +1,6 @@ +fn main() { + println!("cargo:rerun-if-env-changed=DEP_MINIFI_BEHAVE_PATH"); + let behave_path = + std::env::var("DEP_MINIFI_BEHAVE_PATH").expect("DEP_MINIFI_BEHAVE_PATH is required"); + println!("cargo:rustc-env=MINIFI_BEHAVE_PATH={}", behave_path); +} diff --git a/minifi_rust/minifi_rs_behave/debian.dockerfile b/minifi_rust/minifi_rs_behave/debian.dockerfile new file mode 100644 index 0000000000..643328716d --- /dev/null +++ b/minifi_rust/minifi_rs_behave/debian.dockerfile @@ -0,0 +1,26 @@ +FROM rust:slim-bullseye AS chef +RUN apt-get update && apt-get install -y clang lld pkg-config curl tar +RUN cargo install cargo-chef +WORKDIR /app + +FROM chef AS planner +COPY . . +RUN cargo chef prepare --recipe-path recipe.json + +FROM chef AS builder +ARG MINIFI_SDK_PATH +ENV MINIFI_SDK_PATH=${MINIFI_SDK_PATH} + +COPY --from=planner /app/recipe.json recipe.json + +# Conditionally copy the local SDK files from the target directory +COPY target/.docker_sd[k] /app/target/.docker_sdk/ +COPY target/.docker_sdk.zi[p] /app/target/ + +RUN cargo chef cook --release --recipe-path recipe.json +COPY . . +RUN cargo build --release + +# Export Stage +FROM scratch AS bin-export +COPY --from=builder /app/target/release/libminifi_rs_playground.so / diff --git a/minifi_rust/minifi_rs_behave/linux_build.sh b/minifi_rust/minifi_rs_behave/linux_build.sh new file mode 100755 index 0000000000..fc94e48961 --- /dev/null +++ b/minifi_rust/minifi_rs_behave/linux_build.sh @@ -0,0 +1,94 @@ +#!/usr/bin/env bash +set -e + +PROJECT_ROOT="$(cd "$(dirname "$0")/.." && pwd)" +cd "$PROJECT_ROOT" + +echo "Project root: $PROJECT_ROOT" + +# --- Argument Parsing --- +FLAVOR="debian" # Default + +while [[ "$#" -gt 0 ]]; do + case $1 in + --debian) FLAVOR="debian" ;; + --alpine) FLAVOR="alpine" ;; + *) echo "Unknown parameter passed: $1"; exit 1 ;; + esac + shift +done + +# Ensure target dir exists +mkdir -p target + +# 1. Resolve MINIFI_SDK_PATH (from env, or fallback to cargo config) +RESOLVED_SDK_PATH="${MINIFI_SDK_PATH}" +if [ -z "$RESOLVED_SDK_PATH" ] && [ -f .cargo/config.toml ]; then + RESOLVED_SDK_PATH=$(grep -m 1 'MINIFI_SDK_PATH' .cargo/config.toml | cut -d '=' -f2 | tr -d ' "') +fi + +# 2. Define target locations +DOCKER_SDK_DIR="target/.docker_sdk" +DOCKER_SDK_ZIP="target/.docker_sdk.zip" +DOCKER_SDK_ARG="$RESOLVED_SDK_PATH" + +# Clean up any leftover SDK injections +rm -rf "$DOCKER_SDK_DIR" "$DOCKER_SDK_ZIP" + +if [[ "$RESOLVED_SDK_PATH" == http* ]]; then + echo "Detected URL SDK: $RESOLVED_SDK_PATH" +elif [ -f "$RESOLVED_SDK_PATH" ] && [[ "$RESOLVED_SDK_PATH" == *.zip ]]; then + echo "Detected local ZIP SDK: $RESOLVED_SDK_PATH" + cp "$RESOLVED_SDK_PATH" "$DOCKER_SDK_ZIP" + DOCKER_SDK_ARG="/app/$DOCKER_SDK_ZIP" +elif [ -d "$RESOLVED_SDK_PATH" ]; then + echo "Detected local directory SDK: $RESOLVED_SDK_PATH" + echo "Extracting only required files to minimize Docker context..." + + mkdir -p "$DOCKER_SDK_DIR" + + # Case A: It's a flat SDK layout + if [ -f "$RESOLVED_SDK_PATH/minifi-c.h" ]; then + cp "$RESOLVED_SDK_PATH/minifi-c.h" "$DOCKER_SDK_DIR/" + cp "$RESOLVED_SDK_PATH/minifi-c-api.def" "$DOCKER_SDK_DIR/" 2>/dev/null || true + cp "$RESOLVED_SDK_PATH"/*.whl "$DOCKER_SDK_DIR/" 2>/dev/null || true + + # Case B: It's a raw repository layout + elif [ -f "$RESOLVED_SDK_PATH/minifi-api/include/minifi-c/minifi-c.h" ]; then + mkdir -p "$DOCKER_SDK_DIR/minifi-api/include/minifi-c" + + cp "$RESOLVED_SDK_PATH/minifi-api/include/minifi-c/minifi-c.h" "$DOCKER_SDK_DIR/minifi-api/include/minifi-c/" + cp "$RESOLVED_SDK_PATH/minifi-api/minifi-c-api.def" "$DOCKER_SDK_DIR/minifi-api/" 2>/dev/null || true + + if [ -d "$RESOLVED_SDK_PATH/behave_framework" ]; then + cp -R "$RESOLVED_SDK_PATH/behave_framework" "$DOCKER_SDK_DIR/" + fi + else + echo "ERROR: Could not find minifi-c.h in $RESOLVED_SDK_PATH" + exit 1 + fi + + DOCKER_SDK_ARG="/app/$DOCKER_SDK_DIR" +else + echo "WARNING: MINIFI_SDK_PATH is neither a valid URL nor a local path: $RESOLVED_SDK_PATH" +fi + +# --- Set specific build configurations based on flavor --- +echo "Building for $FLAVOR" +DOCKERFILE="minifi_rs_behave/$FLAVOR.dockerfile" +TARGET_DIR="target/release" + +mkdir -p "$TARGET_DIR" + +# 3. Build using Docker +docker buildx build \ + -f "$DOCKERFILE" \ + --target bin-export \ + --build-arg MINIFI_SDK_PATH="$DOCKER_SDK_ARG" \ + --output type=local,dest="$TARGET_DIR" \ + . + +# 4. Cleanup +rm -rf "$DOCKER_SDK_DIR" "$DOCKER_SDK_ZIP" + +echo "Build complete. Artifacts updated in $TARGET_DIR" diff --git a/minifi_rust/minifi_rs_behave/src/main.rs b/minifi_rust/minifi_rs_behave/src/main.rs new file mode 100644 index 0000000000..0dc99686ae --- /dev/null +++ b/minifi_rust/minifi_rs_behave/src/main.rs @@ -0,0 +1,155 @@ +use glob::glob; +use std::env; +use std::path::{Path, PathBuf}; +use std::process::{Command, ExitStatus}; + +#[derive(Debug, PartialEq)] +enum DockerBuildFlavor { + Debian, + Alpine, + Skip, +} + +impl DockerBuildFlavor { + fn from_args() -> Self { + let args: Vec = env::args().collect(); + if args.contains(&"--alpine".to_string()) { + DockerBuildFlavor::Alpine + } else if args.contains(&"--debian".to_string()) { + DockerBuildFlavor::Debian + } else if args.contains(&"--skip-build".to_string()) { + DockerBuildFlavor::Skip + } else { + println!("No build flag provided. Defaulting to skip build."); + DockerBuildFlavor::Skip + } + } +} + +struct BehaveRunner { + venv_path: PathBuf, + minifi_behave_path: PathBuf, +} + +impl BehaveRunner { + fn new(out_dir: &Path) -> Self { + let behave_path = std::env::var("MINIFI_BEHAVE_PATH") + .map(PathBuf::from) + .expect("MINIFI_BEHAVE_PATH is required"); + + let venv_dir = out_dir.join(".venv"); + if !venv_dir.exists() { + println!("Creating virtual environment at {:?}", venv_dir); + let status = Command::new("python3") + .arg("-m") + .arg("venv") + .arg(&venv_dir) + .status() + .expect("Failed to create venv"); + + assert!(status.success(), "Failed to initialize venv"); + } + + Self { + venv_path: venv_dir, + minifi_behave_path: behave_path, + } + } + + fn get_venv_behave(&self) -> PathBuf { + if cfg!(windows) { + self.venv_path.join("Scripts").join("behavex.exe") + } else { + self.venv_path.join("bin").join("behavex") + } + } + + fn get_venv_python(&self) -> PathBuf { + if cfg!(windows) { + self.venv_path.join("Scripts").join("python.exe") + } else { + self.venv_path.join("bin").join("python") + } + } + + fn install_minifi_behave(&self) -> ExitStatus { + Command::new(&self.get_venv_python()) + .arg("-m") + .arg("pip") + .arg("install") + .arg(self.minifi_behave_path.to_string_lossy().as_ref()) + .status() + .expect("Failed to install dependencies") + } + + fn find_features(root: &Path) -> Vec { + println!("Searching for features in {:?}", root); + let pattern = format!("{}/../**/*.feature", root.display()); + let mut feature_files = Vec::new(); + for entry in glob(&pattern).expect("Failed to read glob pattern") { + match entry { + Ok(path) => { + if !path.to_string_lossy().contains(".venv") { + println!("Found feature file: {:?}", path); + feature_files.push(path); + } + } + Err(e) => println!("{:?}", e), + } + } + feature_files + } + + fn run_tests(&self, root: &Path) -> ExitStatus { + let mut cmd = Command::new(&self.get_venv_behave()); + + cmd.args(&Self::find_features(root)); + + cmd.arg("--show-progress-bar"); + cmd.arg("--parallel-processes"); + cmd.arg("2"); + #[cfg(unix)] + cmd.env("TMPDIR", "/tmp"); + + cmd.current_dir(root) + .status() + .expect("Failed to run behave") + } +} + +fn build_linux_so(root_path: &Path, flavor: &DockerBuildFlavor) { + let script_arg = match flavor { + DockerBuildFlavor::Debian => "--debian", + DockerBuildFlavor::Alpine => "--alpine", + DockerBuildFlavor::Skip => return, + }; + + println!("Triggering linux build with flavor: {}", script_arg); + + let mut build_linux = Command::new("./linux_build.sh"); + build_linux.arg(script_arg); + + let status = build_linux + .current_dir(root_path) + .status() + .expect("Failed to execute linux_build.sh"); + + assert!(status.success(), "Linux build script failed"); +} + +fn main() { + let manifest_dir = std::env::var("CARGO_MANIFEST_DIR").expect("CARGO_MANIFEST_DIR not set"); + let root = Path::new(&manifest_dir); + + let flavor = DockerBuildFlavor::from_args(); + build_linux_so(root, &flavor); + + let out_dir = PathBuf::from(std::env::var("OUT_DIR").expect("OUT_DIR not set")); + let behave_runner = BehaveRunner::new(&out_dir); + + let install_status = behave_runner.install_minifi_behave(); + assert!(install_status.success(), "Pip install failed"); + + let status = behave_runner.run_tests(&root); + assert!(status.success(), "Behave tests failed"); +} diff --git a/minifi_rust/rustfmt.toml b/minifi_rust/rustfmt.toml new file mode 100644 index 0000000000..e69de29bb2 From 512b5ecb7e5178ba690579979689b7e0ef436c09 Mon Sep 17 00:00:00 2001 From: Martin Zink Date: Mon, 22 Jun 2026 17:40:25 +0200 Subject: [PATCH 02/26] rebase changes --- .../steps/flow_building_steps.py | 7 + minifi_rust/README.md | 2 +- .../features/controller_apis.feature | 14 ++ .../features/error-handling.feature | 36 +++--- .../minifi_rs_playground.md | 121 ++++++++++++++---- .../animal_controller_apis.rs | 12 ++ .../dog_controller_service.rs | 71 ++++++++++ .../duck_controller_service.rs | 40 ++++++ .../dummy_controller_service.rs | 6 +- .../lorem_ipsum_controller_service.rs | 6 +- .../properties.rs | 2 +- .../src/controller_services/mod.rs | 3 + .../minifi_rs_playground/src/lib.rs | 6 + .../generate_flow_file/properties.rs | 10 +- .../src/processors/get_file/properties.rs | 20 +-- .../src/processors/kamikaze_processor.rs | 22 ++-- .../processor_definition.rs | 4 +- .../kamikaze_processor/properties.rs | 16 +-- .../processors/kamikaze_processor/tests.rs | 10 +- .../src/processors/log_attribute.rs | 4 +- .../processors/log_attribute/properties.rs | 14 +- .../lorem_ipsum_cs_user/properties.rs | 4 +- .../src/processors/mod.rs | 1 + .../src/processors/put_file/properties.rs | 8 +- .../put_file/unix_only_properties.rs | 4 +- .../src/processors/zoo_processor.rs | 70 ++++++++++ minifi_rust/minifi_native/src/api.rs | 3 +- .../src/api/component_definition_traits.rs | 4 +- .../minifi_native/src/api/process_context.rs | 10 +- .../minifi_native/src/api/process_session.rs | 2 +- .../minifi_native/src/api/processor.rs | 4 +- .../processor_wrappers/complex_processor.rs | 4 +- .../processor_wrappers/flow_file_source.rs | 6 +- .../flow_file_stream_transform.rs | 4 +- .../processor_wrappers/flow_file_transform.rs | 8 +- minifi_rust/minifi_native/src/api/property.rs | 6 +- .../src/api/provided_interface.rs | 36 ++++++ .../minifi_native/src/api/raw_processor.rs | 8 +- .../c_ffi_controller_service_definition.rs | 83 +++++++----- .../src/c_ffi/c_ffi_process_context.rs | 46 ++++++- .../src/c_ffi/c_ffi_process_session.rs | 16 +-- .../src/c_ffi/c_ffi_processor_definition.rs | 24 ++-- .../minifi_native/src/c_ffi/c_ffi_property.rs | 12 +- minifi_rust/minifi_native/src/lib.rs | 4 +- .../src/mock/mock_process_context.rs | 8 +- .../src/mock/mock_process_session.rs | 2 +- minifi_rust/minifi_native_macros/src/lib.rs | 19 ++- minifi_rust/minifi_native_sys/src/lib.rs | 2 +- minifi_rust/minifi_rs_behave/linux_build.sh | 14 +- 49 files changed, 631 insertions(+), 207 deletions(-) create mode 100644 minifi_rust/extensions/minifi_rs_playground/features/controller_apis.feature create mode 100644 minifi_rust/extensions/minifi_rs_playground/src/controller_services/animal_controller_apis.rs create mode 100644 minifi_rust/extensions/minifi_rs_playground/src/controller_services/dog_controller_service.rs create mode 100644 minifi_rust/extensions/minifi_rs_playground/src/controller_services/duck_controller_service.rs create mode 100644 minifi_rust/extensions/minifi_rs_playground/src/processors/zoo_processor.rs create mode 100644 minifi_rust/minifi_native/src/api/provided_interface.rs diff --git a/behave_framework/src/minifi_behave/steps/flow_building_steps.py b/behave_framework/src/minifi_behave/steps/flow_building_steps.py index 40414a9e90..b880bbfbc0 100644 --- a/behave_framework/src/minifi_behave/steps/flow_building_steps.py +++ b/behave_framework/src/minifi_behave/steps/flow_building_steps.py @@ -301,6 +301,13 @@ def setup_controller_service(context: MinifiTestContext, service_name: str): context.get_or_create_default_minifi_container().flow_definition.controller_services.append(controller_service) +@given('a {class_name} controller service named "{service_name}" is set up and the "{property_name}" property set to "{property_value}"') +def setup_controller_service_with_property(context: MinifiTestContext, class_name: str, service_name: str, property_name: str, property_value: str): + controller_service = ControllerService(class_name=class_name, service_name=service_name) + controller_service.add_property(property_name, property_value) + context.get_or_create_default_minifi_container().flow_definition.controller_services.append(controller_service) + + @given('a {service_name} controller service is set up and the "{property_name}" property set to "{property_value}"') def setup_controller_service_with_property(context: MinifiTestContext, service_name: str, property_name: str, property_value: str): controller_service = ControllerService(class_name=service_name, service_name=service_name) diff --git a/minifi_rust/README.md b/minifi_rust/README.md index 27c5b96868..950f9153b0 100644 --- a/minifi_rust/README.md +++ b/minifi_rust/README.md @@ -19,7 +19,7 @@ The framework completely encapsulates the unsafe C FFI (Foreign Function Interfa The project is structured as a Cargo workspace with a clear, layered architecture: ### [minifi-native-sys](minifi_native_sys) -Contains the raw, unsafe FFI bindings to the minifi-c.h C API. +Contains the raw, unsafe FFI bindings to the minifi-api.h C API. ### [minifi-native](minifi_native) Provides the public, safe, and idiomatic Rust API. This is the crate that developers will use to build their processors. #### API Traits diff --git a/minifi_rust/extensions/minifi_rs_playground/features/controller_apis.feature b/minifi_rust/extensions/minifi_rs_playground/features/controller_apis.feature new file mode 100644 index 0000000000..a6bcf92252 --- /dev/null +++ b/minifi_rust/extensions/minifi_rs_playground/features/controller_apis.feature @@ -0,0 +1,14 @@ +@SUPPORTS_WINDOWS +Feature: Testing controller service api casting + + Scenario: Zoo has a jetpack dog and a normal duck + Given a ZooProcessor processor with the "Can fly service" property set to "Wolfie the magical" + And the "Number of Legs Service" property of the ZooProcessor processor is set to "Wolfie the magical" + And a DogController controller service named "Wolfie the magical" is set up and the "Has Jetpack" property set to "true" + And the "Extra information" property of the Wolfie the magical controller service is set to "The dog (Canis familiaris or Canis lupus familiaris) is a domesticated descendant of wolves." + When the MiNiFi instance starts up + + Then the Minifi logs contain the following message: "[minifi_rs_playground::processors::zoo_processor::ZooProcessor] [critical] Can DogController { has_jetpack: true, extra_info: "The dog (Canis familiaris or Canis lupus familiaris) is a domesticated descendant of wolves." } fly? true" in less than 10 seconds + And the Minifi logs do not contain errors + And the Minifi logs do not contain warnings + diff --git a/minifi_rust/extensions/minifi_rs_playground/features/error-handling.feature b/minifi_rust/extensions/minifi_rs_playground/features/error-handling.feature index 23dbf43b44..543c4a45a9 100644 --- a/minifi_rust/extensions/minifi_rs_playground/features/error-handling.feature +++ b/minifi_rust/extensions/minifi_rs_playground/features/error-handling.feature @@ -15,48 +15,48 @@ Feature: API error handling and logging And the Minifi logs do not contain errors And the Minifi logs do not contain warnings - Scenario: Minifi handles errors from on_schedule - Given a KamikazeProcessorRs processor with the "On Schedule Behaviour" property set to "ReturnErr" + Scenario: Minifi handles errors from schedule + Given a KamikazeProcessorRs processor with the "Schedule Behaviour" property set to "ReturnErr" And KamikazeProcessorRs's success relationship is auto-terminated When the MiNiFi instance starts up - Then the Minifi logs contain the following message: "KamikazeProcessorRs] [error] Error during on_schedule: ScheduleError("it was designed to fail during schedule")" in less than 10 seconds + Then the Minifi logs contain the following message: "KamikazeProcessorRs] [error] Error during schedule: ScheduleError("it was designed to fail during schedule")" in less than 10 seconds And the Minifi logs contain the following message: "(KamikazeProcessorRs): Process Schedule Operation: Error while scheduling processor" in less than 10 seconds - Scenario: Minifi handles errors from on_trigger - Given a KamikazeProcessorRs processor with the "On Schedule Behaviour" property set to "ReturnOk" - And the "On Trigger Behaviour" property of the KamikazeProcessorRs processor is set to "ReturnErr" + Scenario: Minifi handles errors from trigger + Given a KamikazeProcessorRs processor with the "Schedule Behaviour" property set to "ReturnOk" + And the "Trigger Behaviour" property of the KamikazeProcessorRs processor is set to "ReturnErr" And KamikazeProcessorRs's success relationship is auto-terminated When the MiNiFi instance starts up - Then the Minifi logs contain the following message: "KamikazeProcessorRs] [error] Error during on_trigger TriggerError("it was designed to fail in trigger")" in less than 10 seconds + Then the Minifi logs contain the following message: "KamikazeProcessorRs] [error] Error during trigger TriggerError("it was designed to fail in trigger")" in less than 10 seconds And the Minifi logs contain the following message: "Trigger and commit failed for processor KamikazeProcessorRs" in less than 10 seconds - Scenario: Panic in extension's on_schedule crashes the agent aswell - Given a KamikazeProcessorRs processor with the "On Schedule Behaviour" property set to "Panic" + Scenario: Panic in extension's schedule crashes the agent aswell + Given a KamikazeProcessorRs processor with the "Schedule Behaviour" property set to "Panic" And KamikazeProcessorRs's success relationship is auto-terminated When the MiNiFi instance is started without assertions - Then Minifi crashes with the following "KamikazeProcessor::on_schedule panic" in less than 10 seconds + Then Minifi crashes with the following "KamikazeProcessor::schedule panic" in less than 10 seconds - Scenario: Panic in extension's on_trigger crashes the agent aswell - Given a KamikazeProcessorRs processor with the "On Schedule Behaviour" property set to "ReturnOk" - And the "On Trigger Behaviour" property of the KamikazeProcessorRs processor is set to "Panic" + Scenario: Panic in extension's trigger crashes the agent aswell + Given a KamikazeProcessorRs processor with the "Schedule Behaviour" property set to "ReturnOk" + And the "Trigger Behaviour" property of the KamikazeProcessorRs processor is set to "Panic" And KamikazeProcessorRs's success relationship is auto-terminated When the MiNiFi instance is started without assertions - Then Minifi crashes with the following "KamikazeProcessor::on_trigger panic" in less than 10 seconds + Then Minifi crashes with the following "KamikazeProcessor::trigger panic" in less than 10 seconds Scenario: Get not supported property - Given a KamikazeProcessorRs processor with the "On Schedule Behaviour" property set to "ReturnOk" - And the "On Trigger Behaviour" property of the KamikazeProcessorRs processor is set to "GetNotRegisteredProperty" + Given a KamikazeProcessorRs processor with the "Schedule Behaviour" property set to "ReturnOk" + And the "Trigger Behaviour" property of the KamikazeProcessorRs processor is set to "GetNotRegisteredProperty" And KamikazeProcessorRs's success relationship is auto-terminated When the MiNiFi instance starts up - Then the Minifi logs contain the following message: "MinifiProcessContextGetProperty("Kamikaze Processor Property"), not supported property" in less than 10 seconds + Then the Minifi logs contain the following message: "minifi_process_context_get_property("Kamikaze Processor Property"), not supported property" in less than 10 seconds And the Minifi logs contain the following message: "Trigger and commit failed for processor KamikazeProcessorRs" in less than 10 seconds Scenario: Get wrong typed Controller Service @@ -66,5 +66,5 @@ Feature: API error handling and logging When the MiNiFi instance starts up - Then the Minifi logs contain the following message: "MinifiProcessContextGetControllerService::<"minifi_rs_playground::controller_services::lorem_ipsum_controller_service::LoremIpsumControllerService">("My Controller Service"), validation failed" in less than 10 seconds + Then the Minifi logs contain the following message: "Error during trigger minifi_process_context_get_controller_service_from_property::<"minifi_rs_playground::controller_services::lorem_ipsum_controller_service::LoremIpsumControllerService">" in less than 10 seconds And the Minifi logs contain the following message: "Trigger and commit failed for processor LoremIpsumCSUser" in less than 10 seconds diff --git a/minifi_rust/extensions/minifi_rs_playground/minifi_rs_playground.md b/minifi_rust/extensions/minifi_rs_playground/minifi_rs_playground.md index 9b2397fcd5..8cc65a7fbe 100644 --- a/minifi_rust/extensions/minifi_rs_playground/minifi_rs_playground.md +++ b/minifi_rust/extensions/minifi_rs_playground/minifi_rs_playground.md @@ -19,14 +19,18 @@ limitations under the License. - [AsciifyGerman](#AsciifyGerman) - [CountActualLogging](#CountActualLogging) +- [DuplicateStreamText](#DuplicateStreamText) - [GenerateFlowFileRs](#GenerateFlowFileRs) - [GetFileRs](#GetFileRs) - [KamikazeProcessorRs](#KamikazeProcessorRs) - [LogAttributeRs](#LogAttributeRs) - [LoremIpsumCSUser](#LoremIpsumCSUser) - [PutFileRs](#PutFileRs) +- [ZooProcessor](#ZooProcessor) ### Controller Services +- [DogController](#DogController) +- [DuckController](#DuckController) - [DummyControllerService](#DummyControllerService) - [LoremIpsumControllerService](#LoremIpsumControllerService) @@ -48,8 +52,8 @@ In the list below, the names of required properties appear in bold. Any other pr | Name | Description | |---------|-----------------------------------------| -| success | All asciified flowfiles are routed here | | failure | Non-german flowfiles are routed here | +| success | All asciified flowfiles are routed here | ## CountActualLogging @@ -71,6 +75,26 @@ In the list below, the names of required properties appear in bold. Any other pr |------|-------------| +## DuplicateStreamText + +### Description + +Duplicate text + +### Properties + +In the list below, the names of required properties appear in bold. Any other properties (not in bold) are considered optional. The table also indicates any default values, and whether a property supports the NiFi Expression Language. + +| Name | Default Value | Allowable Values | Description | +|------|---------------|------------------|-------------| + +### Relationships + +| Name | Description | +|---------|-------------| +| success | | + + ## GenerateFlowFileRs ### Description @@ -83,11 +107,11 @@ In the list below, the names of required properties appear in bold. Any other pr | Name | Default Value | Allowable Values | Description | |----------------------|---------------|------------------|----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| -| **File Size** | 1 kB | | The size of the file that will be used
**Supports Expression Language: true** | | **Batch Size** | 1 | | The number of FlowFiles to be transferred in each invocation | +| Custom Text | | | If Data Format is text and if Unique FlowFiles is false, then this custom text will be used as content of the generated FlowFiles and the File Size will be ignored. Finally, if Expression Language is used, evaluation will be performed only once per batch of generated FlowFiles
**Supports Expression Language: true** | | **Data Format** | Binary | Text
Binary | Specifies whether the data should be Text or Binary | +| **File Size** | 1 kB | | The size of the file that will be used
**Supports Expression Language: true** | | **Unique FlowFiles** | true | true
false | If true, each FlowFile that is generated will be unique. If false, a random value will be generated and all FlowFiles will get the same content but this offers much higher throughput (but see the description of Custom Text for special non-random use cases) | -| Custom Text | | | If Data Format is text and if Unique FlowFiles is false, then this custom text will be used as content of the generated FlowFiles and the File Size will be ignored. Finally, if Expression Language is used, evaluation will be performed only once per batch of generated FlowFiles
**Supports Expression Language: true** | ### Relationships @@ -108,16 +132,16 @@ In the list below, the names of required properties appear in bold. Any other pr | Name | Default Value | Allowable Values | Description | |-------------------------|---------------|------------------|------------------------------------------------------------------------------------------------------------------------------------------------------------| +| **Batch Size** | 10 | | The maximum number of files to pull in each iteration | +| **Ignore Hidden Files** | true | true
false | Indicates whether or not hidden files should be ignored | | **Input Directory** | | | The input directory from which to pull files
**Supports Expression Language: true** | -| Polling Interval | | | Indicates how long to wait before performing a directory listing | -| Recurse Subdirectories | true | true
false | Indicates whether or not to pull files from subdirectories | | Keep Source File | false | true
false | If true, the file is not deleted after it has been copied to the Content Repository | -| Minimum File Age | | | The minimum age that a file must be in order to be pulled; any file younger than this amount of time (according to last modification date) will be ignored | | Maximum File Age | | | The maximum age that a file must be in order to be pulled; any file older than this amount of time (according to last modification date) will be ignored | -| Minimum File Size | | | The minimum size that a file can be in order to be pulled | | Maximum File Size | | | The maximum size that a file can be in order to be pulled | -| **Ignore Hidden Files** | true | true
false | Indicates whether or not hidden files should be ignored | -| **Batch Size** | 10 | | The maximum number of files to pull in each iteration | +| Minimum File Age | | | The minimum age that a file must be in order to be pulled; any file younger than this amount of time (according to last modification date) will be ignored | +| Minimum File Size | | | The minimum size that a file can be in order to be pulled | +| Polling Interval | | | Indicates how long to wait before performing a directory listing | +| Recurse Subdirectories | true | true
false | Indicates whether or not to pull files from subdirectories | ### Relationships @@ -143,10 +167,10 @@ This processor can fail or panic in on_trigger and on_schedule calls based on co In the list below, the names of required properties appear in bold. Any other properties (not in bold) are considered optional. The table also indicates any default values, and whether a property supports the NiFi Expression Language. -| Name | Default Value | Allowable Values | Description | -|---------------------------|---------------|-----------------------------------------------------------------------------------------------|------------------------------------------| -| **On Schedule Behaviour** | ReturnOk | ReturnErr
ReturnOk
GetNotRegisteredProperty
GetInvalidControllerService
Panic | What to do during the on_schedule method | -| **On Trigger Behaviour** | ReturnOk | ReturnErr
ReturnOk
GetNotRegisteredProperty
GetInvalidControllerService
Panic | What to do during the on_trigger method | +| Name | Default Value | Allowable Values | Description | +|------------------------|---------------|-----------------------------------------------------------------------------------------------|------------------------------------------| +| **Schedule Behaviour** | ReturnOk | ReturnErr
ReturnOk
GetNotRegisteredProperty
GetInvalidControllerService
Panic | What to do during the on_schedule method | +| **Trigger Behaviour** | ReturnOk | ReturnErr
ReturnOk
GetNotRegisteredProperty
GetInvalidControllerService
Panic | What to do during the trigger method | ### Relationships @@ -167,13 +191,13 @@ In the list below, the names of required properties appear in bold. Any other pr | Name | Default Value | Allowable Values | Description | |-----------------------|---------------|------------------------------------------------------------------|----------------------------------------------------------------------------------------------------------------------------------------------------------------| -| **Log Level** | Info | Trace
Debug
Info
Warn
Error
Critical
Off | The Log Level to use when logging the Attributes | -| Attributes to Log | | | A comma-separated list of Attributes to Log. If not specified, all attributes will be logged. | | Attributes to Ignore | | | A comma-separated list of Attributes to ignore. If not specified, no attributes will be ignored. | -| **Log Payload** | false | true
false | If true, the FlowFile's payload will be logged, in addition to its attributes. Otherwise, just the Attributes will be logged. | -| Log Prefix | | | Log prefix appended to the log lines. It helps to distinguish the output of multiple LogAttribute processors. | +| Attributes to Log | | | A comma-separated list of Attributes to Log. If not specified, all attributes will be logged. | | **FlowFiles To Log** | 1 | | Number of flow files to log. If set to zero all flow files will be logged. Please note that this may block other threads from running if not used judiciously. | | **Hexencode Payload** | false | true
false | If true, the FlowFile's payload will be logged in a hexencoded format | +| **Log Level** | Info | Trace
Debug
Info
Warn
Error
Critical
Off | The Log Level to use when logging the Attributes | +| **Log Payload** | false | true
false | If true, the FlowFile's payload will be logged, in addition to its attributes. Otherwise, just the Attributes will be logged. | +| Log Prefix | | | Log prefix appended to the log lines. It helps to distinguish the output of multiple LogAttribute processors. | ### Relationships @@ -216,19 +240,70 @@ In the list below, the names of required properties appear in bold. Any other pr | Name | Default Value | Allowable Values | Description | |----------------------------------|---------------|-----------------------------|--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| -| **Directory** | . | | The output directory to which to put files
**Supports Expression Language: true** | | **Conflict Resolution Strategy** | fail | fail
replace
ignore | Indicates what should happen when a file with the same name already exists in the output directory | | **Create Missing Directories** | true | true
false | If true, then missing destination directories will be created. If false, flowfiles are penalized and sent to failure. | +| **Directory** | . | | The output directory to which to put files
**Supports Expression Language: true** | +| Directory Permissions | | | Sets the permissions on the directories being created if 'Create Missing Directories' property is set. Must be an octal number (e.g. 644 or 0755). Not supported on Windows systems. | | Maximum File Count | | | Specifies the maximum number of files that can exist in the output directory | | Permissions | | | Sets the permissions on the output file to the value of this attribute. Must be an octal number (e.g. 644 or 0755). Not supported on Windows systems. | -| Directory Permissions | | | Sets the permissions on the directories being created if 'Create Missing Directories' property is set. Must be an octal number (e.g. 644 or 0755). Not supported on Windows systems. | ### Relationships | Name | Description | |---------|-----------------------------------------------------------------------------------| -| success | Flowfiles that are successfully written to a file are routed to this relationship | | failure | Failed files (conflict, write failure, etc.) are transferred to failure | +| success | Flowfiles that are successfully written to a file are routed to this relationship | + + +## ZooProcessor + +### Description + +Test ZooProcessor + +### Properties + +In the list below, the names of required properties appear in bold. Any other properties (not in bold) are considered optional. The table also indicates any default values, and whether a property supports the NiFi Expression Language. + +| Name | Default Value | Allowable Values | Description | +|----------------------------|---------------|------------------|--------------------------| +| **Can fly service** | | | Test CanFlyService | +| **Number of Legs Service** | | | Test NumberOfLegsService | + +### Relationships + +| Name | Description | +|------|-------------| + + +## DogController + +### Description + +Test DogController + +### Properties + +In the list below, the names of required properties appear in bold. Any other properties (not in bold) are considered optional. The table also indicates any default values, and whether a property supports the NiFi Expression Language. + +| Name | Default Value | Allowable Values | Description | +|-------------------|---------------|------------------|-------------------------------------------------------| +| Extra information | | | We need this to verify the casting was done correctly | +| **Has Jetpack** | false | true
false | Whether or not the dog has a jetpack | + + +## DuckController + +### Description + +Test DuckController + +### Properties + +In the list below, the names of required properties appear in bold. Any other properties (not in bold) are considered optional. The table also indicates any default values, and whether a property supports the NiFi Expression Language. + +| Name | Default Value | Allowable Values | Description | +|------|---------------|------------------|-------------| ## DummyControllerService @@ -255,6 +330,6 @@ Simple Rusty Controller Service to test API In the list below, the names of required properties appear in bold. Any other properties (not in bold) are considered optional. The table also indicates any default values, and whether a property supports the NiFi Expression Language. -| Name | Default Value | Allowable Values | Description | -|------------|---------------|------------------|-------------------------------------------------------------| -| **Length** | 25 | | How many words to generate
**Sensitive Property: true** | +| Name | Default Value | Allowable Values | Description | +|------------|---------------|------------------|----------------------------| +| **Length** | 25 | | How many words to generate | diff --git a/minifi_rust/extensions/minifi_rs_playground/src/controller_services/animal_controller_apis.rs b/minifi_rust/extensions/minifi_rs_playground/src/controller_services/animal_controller_apis.rs new file mode 100644 index 0000000000..358963784e --- /dev/null +++ b/minifi_rust/extensions/minifi_rs_playground/src/controller_services/animal_controller_apis.rs @@ -0,0 +1,12 @@ +use std::fmt::Debug; +use minifi_native::macros::controller_service_api; + +#[controller_service_api] +pub trait NumberOfLegsControllerApi: Debug { + fn number_of_legs(&self) -> u8; +} + +#[controller_service_api] +pub trait CanFlyControllerApi: Debug { + fn can_fly(&self) -> bool; +} diff --git a/minifi_rust/extensions/minifi_rs_playground/src/controller_services/dog_controller_service.rs b/minifi_rust/extensions/minifi_rs_playground/src/controller_services/dog_controller_service.rs new file mode 100644 index 0000000000..9649b9d1f4 --- /dev/null +++ b/minifi_rust/extensions/minifi_rs_playground/src/controller_services/dog_controller_service.rs @@ -0,0 +1,71 @@ +use minifi_native::ControllerServiceApi; +use minifi_native::{create_provided_interface, ControllerServiceDefinition, EnableControllerService, GetProperty, Logger, MinifiError, Property, ProvidedInterface, StandardPropertyValidator}; +use minifi_native::macros::ComponentIdentifier; +use crate::controller_services::animal_controller_apis::{CanFlyControllerApi, NumberOfLegsControllerApi}; + +pub(crate) const HAS_JETPACK: Property = Property { + name: "Has Jetpack", + description: "Whether or not the dog has a jetpack", + is_required: true, + is_sensitive: false, + supports_expr_lang: false, + default_value: Some("false"), + validator: StandardPropertyValidator::BoolValidator, + allowed_values: &[], + allowed_type: None, +}; + +pub(crate) const EXTRA_INFO: Property = Property { + name: "Extra information", + description: "We need this to verify the casting was done correctly", + is_required: false, + is_sensitive: false, + supports_expr_lang: false, + default_value: None, + validator: StandardPropertyValidator::AlwaysValidValidator, + allowed_values: &[], + allowed_type: None, +}; + +#[allow(dead_code)] // extra_info is only used by {:?} +#[derive(Debug, ComponentIdentifier)] +pub(crate) struct DogController { + has_jetpack: bool, + extra_info: String +} + +impl NumberOfLegsControllerApi for DogController { + fn number_of_legs(&self) -> u8 { + 4 + } +} + +impl CanFlyControllerApi for DogController { + fn can_fly(&self) -> bool { + self.has_jetpack + } +} + +impl EnableControllerService for DogController { + fn enable(context: &Ctx, _logger: &L) -> Result + where + Self: Sized + { + let has_jetpack = context + .get_bool_property(&HAS_JETPACK)? + .ok_or(MinifiError::missing_required_property("Has jetpack is required"))?; + + let extra_info = context.get_property(&EXTRA_INFO)?.unwrap_or("".into()); + + Ok(Self { has_jetpack, extra_info }) + } +} + +impl ControllerServiceDefinition for DogController { + const DESCRIPTION: &'static str = "Test DogController"; + const PROPERTIES: &'static [Property] = &[HAS_JETPACK, EXTRA_INFO]; + const PROVIDED_APIS: &'static [ProvidedInterface] = &[ + create_provided_interface!(dyn CanFlyControllerApi), + create_provided_interface!(dyn NumberOfLegsControllerApi), + ]; +} \ No newline at end of file diff --git a/minifi_rust/extensions/minifi_rs_playground/src/controller_services/duck_controller_service.rs b/minifi_rust/extensions/minifi_rs_playground/src/controller_services/duck_controller_service.rs new file mode 100644 index 0000000000..a565ccbffb --- /dev/null +++ b/minifi_rust/extensions/minifi_rs_playground/src/controller_services/duck_controller_service.rs @@ -0,0 +1,40 @@ +use minifi_native::ControllerServiceApi; +use minifi_native::{create_provided_interface, ControllerServiceDefinition, EnableControllerService, GetProperty, Logger, MinifiError, Property, ProvidedInterface}; +use minifi_native::macros::ComponentIdentifier; +use crate::controller_services::animal_controller_apis::{CanFlyControllerApi, NumberOfLegsControllerApi}; + +#[derive(Debug, ComponentIdentifier)] +pub(crate) struct DuckController { + +} + + +impl NumberOfLegsControllerApi for DuckController { + fn number_of_legs(&self) -> u8 { + 2 + } +} + +impl CanFlyControllerApi for DuckController { + fn can_fly(&self) -> bool { + true + } +} + +impl EnableControllerService for DuckController { + fn enable(_context: &Ctx, _logger: &L) -> Result + where + Self: Sized + { + Ok(Self{}) + } +} + +impl ControllerServiceDefinition for DuckController { + const DESCRIPTION: &'static str = "Test DuckController"; + const PROPERTIES: &'static [Property] = &[]; + const PROVIDED_APIS: &'static [ProvidedInterface] = &[ + create_provided_interface!(dyn CanFlyControllerApi), + create_provided_interface!(dyn NumberOfLegsControllerApi), + ]; +} \ No newline at end of file diff --git a/minifi_rust/extensions/minifi_rs_playground/src/controller_services/dummy_controller_service.rs b/minifi_rust/extensions/minifi_rs_playground/src/controller_services/dummy_controller_service.rs index ccd098ab41..fb346e7444 100644 --- a/minifi_rust/extensions/minifi_rs_playground/src/controller_services/dummy_controller_service.rs +++ b/minifi_rust/extensions/minifi_rs_playground/src/controller_services/dummy_controller_service.rs @@ -1,8 +1,5 @@ use minifi_native::macros::ComponentIdentifier; -use minifi_native::{ - ControllerServiceDefinition, EnableControllerService, GetProperty, Logger, MinifiError, - Property, -}; +use minifi_native::{ControllerServiceDefinition, EnableControllerService, GetProperty, Logger, MinifiError, Property, ProvidedInterface}; #[derive(Debug, ComponentIdentifier)] pub(crate) struct DummyControllerService {} @@ -19,4 +16,5 @@ impl EnableControllerService for DummyControllerService { impl ControllerServiceDefinition for DummyControllerService { const DESCRIPTION: &'static str = "Dummy Controller Service"; const PROPERTIES: &'static [Property] = &[]; + const PROVIDED_APIS: &'static [ProvidedInterface] = &[]; } diff --git a/minifi_rust/extensions/minifi_rs_playground/src/controller_services/lorem_ipsum_controller_service.rs b/minifi_rust/extensions/minifi_rs_playground/src/controller_services/lorem_ipsum_controller_service.rs index eeb9ab5213..bafc80c5c3 100644 --- a/minifi_rust/extensions/minifi_rs_playground/src/controller_services/lorem_ipsum_controller_service.rs +++ b/minifi_rust/extensions/minifi_rs_playground/src/controller_services/lorem_ipsum_controller_service.rs @@ -3,10 +3,7 @@ mod properties; use crate::controller_services::lorem_ipsum_controller_service::properties::LENGTH; use lipsum::lipsum; use minifi_native::macros::ComponentIdentifier; -use minifi_native::{ - ControllerServiceDefinition, EnableControllerService, GetProperty, Logger, MinifiError, - Property, -}; +use minifi_native::{ControllerServiceDefinition, EnableControllerService, GetProperty, Logger, MinifiError, Property, ProvidedInterface}; #[derive(Debug, ComponentIdentifier)] pub(crate) struct LoremIpsumControllerService { @@ -30,4 +27,5 @@ impl EnableControllerService for LoremIpsumControllerService { impl ControllerServiceDefinition for LoremIpsumControllerService { const DESCRIPTION: &'static str = "Simple Rusty Controller Service to test API"; const PROPERTIES: &'static [Property] = &[LENGTH]; + const PROVIDED_APIS: &'static [ProvidedInterface] = &[]; } diff --git a/minifi_rust/extensions/minifi_rs_playground/src/controller_services/lorem_ipsum_controller_service/properties.rs b/minifi_rust/extensions/minifi_rs_playground/src/controller_services/lorem_ipsum_controller_service/properties.rs index b895647a7c..f15fd95f21 100644 --- a/minifi_rust/extensions/minifi_rs_playground/src/controller_services/lorem_ipsum_controller_service/properties.rs +++ b/minifi_rust/extensions/minifi_rs_playground/src/controller_services/lorem_ipsum_controller_service/properties.rs @@ -9,5 +9,5 @@ pub(crate) const LENGTH: Property = Property { default_value: Some("25"), validator: StandardPropertyValidator::U64Validator, allowed_values: &[], - allowed_type: "", + allowed_type: None, }; diff --git a/minifi_rust/extensions/minifi_rs_playground/src/controller_services/mod.rs b/minifi_rust/extensions/minifi_rs_playground/src/controller_services/mod.rs index 7935987e7f..0467a80410 100644 --- a/minifi_rust/extensions/minifi_rs_playground/src/controller_services/mod.rs +++ b/minifi_rust/extensions/minifi_rs_playground/src/controller_services/mod.rs @@ -1,2 +1,5 @@ pub(crate) mod dummy_controller_service; pub(crate) mod lorem_ipsum_controller_service; +pub(crate) mod dog_controller_service; +pub(crate) mod animal_controller_apis; +pub(crate) mod duck_controller_service; diff --git a/minifi_rust/extensions/minifi_rs_playground/src/lib.rs b/minifi_rust/extensions/minifi_rs_playground/src/lib.rs index ad83792f28..ed55e963f8 100644 --- a/minifi_rust/extensions/minifi_rs_playground/src/lib.rs +++ b/minifi_rust/extensions/minifi_rs_playground/src/lib.rs @@ -1,6 +1,8 @@ mod controller_services; mod processors; +use crate::controller_services::dog_controller_service::DogController; +use crate::controller_services::duck_controller_service::DuckController; use crate::controller_services::dummy_controller_service::DummyControllerService; use crate::controller_services::lorem_ipsum_controller_service::LoremIpsumControllerService; use crate::processors::asciify_german::AsciifyGerman; @@ -12,6 +14,7 @@ use crate::processors::kamikaze_processor::KamikazeProcessorRs; use crate::processors::log_attribute::LogAttributeRs; use crate::processors::lorem_ipsum_cs_user::LoremIpsumCSUser; use crate::processors::put_file::PutFileRs; +use crate::processors::zoo_processor::ZooProcessor; use minifi_native::{ ComplexProcessorType, Concurrent, Exclusive, FlowFileSourceProcessorType, @@ -29,9 +32,12 @@ processors: [ (FlowFileTransformProcessorType, Concurrent, PutFileRs), (FlowFileStreamTransformProcessorType, Concurrent, AsciifyGerman), (FlowFileStreamTransformProcessorType, Exclusive, DuplicateStreamText), + (ComplexProcessorType, Concurrent, ZooProcessor), ], controllers: [ LoremIpsumControllerService, DummyControllerService, + DogController, + DuckController ] ); diff --git a/minifi_rust/extensions/minifi_rs_playground/src/processors/generate_flow_file/properties.rs b/minifi_rust/extensions/minifi_rs_playground/src/processors/generate_flow_file/properties.rs index 19d81836c9..58840edb99 100644 --- a/minifi_rust/extensions/minifi_rs_playground/src/processors/generate_flow_file/properties.rs +++ b/minifi_rust/extensions/minifi_rs_playground/src/processors/generate_flow_file/properties.rs @@ -9,7 +9,7 @@ pub(crate) const FILE_SIZE: Property = Property { default_value: Some("1 kB"), validator: StandardPropertyValidator::DataSizeValidator, allowed_values: &[], - allowed_type: "", + allowed_type: None, }; pub(crate) const BATCH_SIZE: Property = Property { @@ -21,7 +21,7 @@ pub(crate) const BATCH_SIZE: Property = Property { default_value: Some("1"), validator: StandardPropertyValidator::U64Validator, allowed_values: &[], - allowed_type: "", + allowed_type: None, }; pub(crate) const DATA_FORMAT: Property = Property { @@ -33,7 +33,7 @@ pub(crate) const DATA_FORMAT: Property = Property { default_value: Some("Binary"), validator: StandardPropertyValidator::AlwaysValidValidator, allowed_values: &["Text", "Binary"], - allowed_type: "", + allowed_type: None, }; pub(crate) const UNIQUE_FLOW_FILES: Property = Property { @@ -45,7 +45,7 @@ pub(crate) const UNIQUE_FLOW_FILES: Property = Property { default_value: Some("true"), validator: StandardPropertyValidator::BoolValidator, allowed_values: &[], - allowed_type: "", + allowed_type: None, }; pub(crate) const CUSTOM_TEXT: Property = Property { @@ -57,5 +57,5 @@ pub(crate) const CUSTOM_TEXT: Property = Property { default_value: None, validator: StandardPropertyValidator::AlwaysValidValidator, allowed_values: &[], - allowed_type: "", + allowed_type: None, }; diff --git a/minifi_rust/extensions/minifi_rs_playground/src/processors/get_file/properties.rs b/minifi_rust/extensions/minifi_rs_playground/src/processors/get_file/properties.rs index 44f81fa717..5945ce29d5 100644 --- a/minifi_rust/extensions/minifi_rs_playground/src/processors/get_file/properties.rs +++ b/minifi_rust/extensions/minifi_rs_playground/src/processors/get_file/properties.rs @@ -9,7 +9,7 @@ pub(crate) const DIRECTORY: Property = Property { default_value: None, validator: StandardPropertyValidator::NonBlankValidator, allowed_values: &[], - allowed_type: "", + allowed_type: None, }; pub(crate) const RECURSE: Property = Property { @@ -21,7 +21,7 @@ pub(crate) const RECURSE: Property = Property { default_value: Some("true"), validator: StandardPropertyValidator::BoolValidator, allowed_values: &[], - allowed_type: "", + allowed_type: None, }; pub(crate) const KEEP_SOURCE_FILE: Property = Property { @@ -33,7 +33,7 @@ pub(crate) const KEEP_SOURCE_FILE: Property = Property { default_value: Some("false"), validator: StandardPropertyValidator::BoolValidator, allowed_values: &[], - allowed_type: "", + allowed_type: None, }; pub(crate) const MIN_AGE: Property = Property { @@ -45,7 +45,7 @@ pub(crate) const MIN_AGE: Property = Property { default_value: None, validator: StandardPropertyValidator::TimePeriodValidator, allowed_values: &[], - allowed_type: "", + allowed_type: None, }; pub(crate) const MAX_AGE: Property = Property { @@ -57,7 +57,7 @@ pub(crate) const MAX_AGE: Property = Property { default_value: None, validator: StandardPropertyValidator::TimePeriodValidator, allowed_values: &[], - allowed_type: "", + allowed_type: None, }; pub(crate) const MIN_SIZE: Property = Property { @@ -69,7 +69,7 @@ pub(crate) const MIN_SIZE: Property = Property { default_value: None, validator: StandardPropertyValidator::DataSizeValidator, allowed_values: &[], - allowed_type: "", + allowed_type: None, }; pub(crate) const MAX_SIZE: Property = Property { @@ -81,7 +81,7 @@ pub(crate) const MAX_SIZE: Property = Property { default_value: None, validator: StandardPropertyValidator::DataSizeValidator, allowed_values: &[], - allowed_type: "", + allowed_type: None, }; pub(crate) const IGNORE_HIDDEN_FILES: Property = Property { @@ -93,7 +93,7 @@ pub(crate) const IGNORE_HIDDEN_FILES: Property = Property { default_value: Some("true"), validator: StandardPropertyValidator::BoolValidator, allowed_values: &[], - allowed_type: "", + allowed_type: None, }; pub(crate) const POLLING_INTERVAL: Property = Property { @@ -105,7 +105,7 @@ pub(crate) const POLLING_INTERVAL: Property = Property { default_value: None, validator: StandardPropertyValidator::TimePeriodValidator, allowed_values: &[], - allowed_type: "", + allowed_type: None, }; pub(crate) const BATCH_SIZE: Property = Property { @@ -117,5 +117,5 @@ pub(crate) const BATCH_SIZE: Property = Property { default_value: Some("10"), validator: StandardPropertyValidator::U64Validator, allowed_values: &[], - allowed_type: "", + allowed_type: None, }; diff --git a/minifi_rust/extensions/minifi_rs_playground/src/processors/kamikaze_processor.rs b/minifi_rust/extensions/minifi_rs_playground/src/processors/kamikaze_processor.rs index ad65040ff1..bd01eab573 100644 --- a/minifi_rust/extensions/minifi_rs_playground/src/processors/kamikaze_processor.rs +++ b/minifi_rust/extensions/minifi_rs_playground/src/processors/kamikaze_processor.rs @@ -22,7 +22,7 @@ enum KamikazeBehaviour { #[derive(Debug, ComponentIdentifier)] pub(crate) struct KamikazeProcessorRs { - on_trigger_behaviour: KamikazeBehaviour, + trigger_behaviour: KamikazeBehaviour, } impl Schedule for KamikazeProcessorRs { @@ -30,31 +30,31 @@ impl Schedule for KamikazeProcessorRs { where Self: Sized, { - let on_trigger_behaviour = context - .get_property(&properties::ON_TRIGGER_BEHAVIOUR)? + let trigger_behaviour = context + .get_property(&properties::TRIGGER_BEHAVIOUR)? .expect("required property") .parse::()?; - let on_schedule_behaviour = context - .get_property(&properties::ON_SCHEDULE_BEHAVIOUR)? + let schedule_behaviour = context + .get_property(&properties::SCHEDULE_BEHAVIOUR)? .expect("required property") .parse::()?; - match on_schedule_behaviour { + match schedule_behaviour { KamikazeBehaviour::ReturnErr => Err(MinifiError::schedule_err( "it was designed to fail during schedule", )), KamikazeBehaviour::ReturnOk => Ok(KamikazeProcessorRs { - on_trigger_behaviour, + trigger_behaviour, }), KamikazeBehaviour::GetNotRegisteredProperty => { let _ = context.get_property(&NOT_REGISTERED_PROPERTY)?; Ok(KamikazeProcessorRs { - on_trigger_behaviour, + trigger_behaviour, }) } KamikazeBehaviour::Panic => { - panic!("KamikazeProcessor::on_schedule panic") + panic!("KamikazeProcessor::schedule panic") } KamikazeBehaviour::GetInvalidControllerService => { unimplemented!("KamikazeProcessor::get_invalid_controller_service"); @@ -75,13 +75,13 @@ impl Trigger for KamikazeProcessorRs { PS: ProcessSession, L: Logger, { - match self.on_trigger_behaviour { + match self.trigger_behaviour { KamikazeBehaviour::ReturnErr => Err(MinifiError::trigger_err( "it was designed to fail in trigger", )), KamikazeBehaviour::ReturnOk => Ok(OnTriggerResult::Ok), KamikazeBehaviour::Panic => { - panic!("KamikazeProcessor::on_trigger panic") + panic!("KamikazeProcessor::trigger panic") } KamikazeBehaviour::GetNotRegisteredProperty => { let _ = context.get_property(&NOT_REGISTERED_PROPERTY, None)?; diff --git a/minifi_rust/extensions/minifi_rs_playground/src/processors/kamikaze_processor/processor_definition.rs b/minifi_rust/extensions/minifi_rs_playground/src/processors/kamikaze_processor/processor_definition.rs index 1cbc76634c..5c38f209da 100644 --- a/minifi_rust/extensions/minifi_rs_playground/src/processors/kamikaze_processor/processor_definition.rs +++ b/minifi_rust/extensions/minifi_rs_playground/src/processors/kamikaze_processor/processor_definition.rs @@ -11,7 +11,7 @@ impl ProcessorDefinition for KamikazeProcessorRs { const OUTPUT_ATTRIBUTES: &'static [OutputAttribute] = &[]; const RELATIONSHIPS: &'static [Relationship] = &[relationships::SUCCESS]; const PROPERTIES: &'static [Property] = &[ - properties::ON_SCHEDULE_BEHAVIOUR, - properties::ON_TRIGGER_BEHAVIOUR, + properties::SCHEDULE_BEHAVIOUR, + properties::TRIGGER_BEHAVIOUR, ]; } diff --git a/minifi_rust/extensions/minifi_rs_playground/src/processors/kamikaze_processor/properties.rs b/minifi_rust/extensions/minifi_rs_playground/src/processors/kamikaze_processor/properties.rs index 0f2e550e7e..ba81c6b025 100644 --- a/minifi_rust/extensions/minifi_rs_playground/src/processors/kamikaze_processor/properties.rs +++ b/minifi_rust/extensions/minifi_rs_playground/src/processors/kamikaze_processor/properties.rs @@ -2,8 +2,8 @@ use crate::processors::kamikaze_processor::KamikazeBehaviour; use minifi_native::{Property, StandardPropertyValidator}; use strum::VariantNames; -pub(crate) const ON_SCHEDULE_BEHAVIOUR: Property = Property { - name: "On Schedule Behaviour", +pub(crate) const SCHEDULE_BEHAVIOUR: Property = Property { + name: "Schedule Behaviour", description: "What to do during the on_schedule method", is_required: true, is_sensitive: false, @@ -11,19 +11,19 @@ pub(crate) const ON_SCHEDULE_BEHAVIOUR: Property = Property { default_value: Some(KamikazeBehaviour::ReturnOk.into_str()), validator: StandardPropertyValidator::AlwaysValidValidator, allowed_values: &KamikazeBehaviour::VARIANTS, - allowed_type: "", + allowed_type: None, }; -pub(crate) const ON_TRIGGER_BEHAVIOUR: Property = Property { - name: "On Trigger Behaviour", - description: "What to do during the on_trigger method", +pub(crate) const TRIGGER_BEHAVIOUR: Property = Property { + name: "Trigger Behaviour", + description: "What to do during the trigger method", is_required: true, is_sensitive: false, supports_expr_lang: false, default_value: Some(KamikazeBehaviour::ReturnOk.into_str()), validator: StandardPropertyValidator::AlwaysValidValidator, allowed_values: &KamikazeBehaviour::VARIANTS, - allowed_type: "", + allowed_type: None, }; pub(crate) const NOT_REGISTERED_PROPERTY: Property = Property { @@ -35,5 +35,5 @@ pub(crate) const NOT_REGISTERED_PROPERTY: Property = Property { default_value: None, validator: StandardPropertyValidator::AlwaysValidValidator, allowed_values: &[], - allowed_type: "", + allowed_type: None, }; diff --git a/minifi_rust/extensions/minifi_rs_playground/src/processors/kamikaze_processor/tests.rs b/minifi_rust/extensions/minifi_rs_playground/src/processors/kamikaze_processor/tests.rs index 76acbb3e5f..ab959f6e91 100644 --- a/minifi_rust/extensions/minifi_rs_playground/src/processors/kamikaze_processor/tests.rs +++ b/minifi_rust/extensions/minifi_rs_playground/src/processors/kamikaze_processor/tests.rs @@ -1,6 +1,6 @@ use super::*; use crate::processors::kamikaze_processor::properties::{ - ON_SCHEDULE_BEHAVIOUR, ON_TRIGGER_BEHAVIOUR, + SCHEDULE_BEHAVIOUR, TRIGGER_BEHAVIOUR, }; use minifi_native::MinifiError::{ScheduleError, TriggerError}; use minifi_native::{MockLogger, MockProcessContext, MockProcessSession}; @@ -17,7 +17,7 @@ fn on_schedule_ok() { fn on_schedule_err() { let mut context = MockProcessContext::new(); context.properties.insert( - ON_SCHEDULE_BEHAVIOUR.name.to_string(), + SCHEDULE_BEHAVIOUR.name.to_string(), "ReturnErr".to_string(), ); let processor = KamikazeProcessorRs::schedule(&context, &MockLogger::new()); @@ -29,7 +29,7 @@ fn on_schedule_panic() { let mut context = MockProcessContext::new(); context .properties - .insert(ON_SCHEDULE_BEHAVIOUR.name.to_string(), "Panic".to_string()); + .insert(SCHEDULE_BEHAVIOUR.name.to_string(), "Panic".to_string()); let result = std::panic::catch_unwind(AssertUnwindSafe(|| { KamikazeProcessorRs::schedule(&context, &MockLogger::new()) @@ -55,7 +55,7 @@ fn on_trigger_ok() { fn on_trigger_err() { let mut context = MockProcessContext::new(); context.properties.insert( - ON_TRIGGER_BEHAVIOUR.name.to_string(), + TRIGGER_BEHAVIOUR.name.to_string(), "ReturnErr".to_string(), ); let processor = KamikazeProcessorRs::schedule(&context, &MockLogger::new()).unwrap(); @@ -72,7 +72,7 @@ fn on_trigger_panic() { let mut context = MockProcessContext::new(); context .properties - .insert(ON_TRIGGER_BEHAVIOUR.name.to_string(), "Panic".to_string()); + .insert(TRIGGER_BEHAVIOUR.name.to_string(), "Panic".to_string()); let processor = KamikazeProcessorRs::schedule(&context, &MockLogger::new()).unwrap(); let mut session = MockProcessSession::new(); diff --git a/minifi_rust/extensions/minifi_rs_playground/src/processors/log_attribute.rs b/minifi_rust/extensions/minifi_rs_playground/src/processors/log_attribute.rs index b49f7dd442..3143db9d44 100644 --- a/minifi_rust/extensions/minifi_rs_playground/src/processors/log_attribute.rs +++ b/minifi_rust/extensions/minifi_rs_playground/src/processors/log_attribute.rs @@ -101,9 +101,7 @@ impl Trigger for LogAttributeRs { } impl Schedule for LogAttributeRs { - fn schedule(context: &P, _logger: &L) -> Result - where - Self: Sized, + fn schedule(context: &P, _logger: &L) -> Result { let log_level = context .get_property(&LOG_LEVEL)? diff --git a/minifi_rust/extensions/minifi_rs_playground/src/processors/log_attribute/properties.rs b/minifi_rust/extensions/minifi_rs_playground/src/processors/log_attribute/properties.rs index 5ec30f72ea..b104224e00 100644 --- a/minifi_rust/extensions/minifi_rs_playground/src/processors/log_attribute/properties.rs +++ b/minifi_rust/extensions/minifi_rs_playground/src/processors/log_attribute/properties.rs @@ -10,7 +10,7 @@ pub(crate) const LOG_LEVEL: Property = Property { default_value: Some("Info"), // todo! it would be nicer to come from enum value but wasnt able to use into_const_str from another crate validator: StandardPropertyValidator::AlwaysValidValidator, allowed_values: &LogLevel::VARIANTS, - allowed_type: "", + allowed_type: None, }; pub(crate) const ATTRIBUTES_TO_LOG: Property = Property { @@ -22,7 +22,7 @@ pub(crate) const ATTRIBUTES_TO_LOG: Property = Property { default_value: None, validator: StandardPropertyValidator::AlwaysValidValidator, allowed_values: &[], - allowed_type: "", + allowed_type: None, }; pub(crate) const ATTRIBUTES_TO_IGNORE: Property = Property { @@ -34,7 +34,7 @@ pub(crate) const ATTRIBUTES_TO_IGNORE: Property = Property { default_value: None, validator: StandardPropertyValidator::AlwaysValidValidator, allowed_values: &[], - allowed_type: "", + allowed_type: None, }; pub(crate) const LOG_PAYLOAD: Property = Property { @@ -46,7 +46,7 @@ pub(crate) const LOG_PAYLOAD: Property = Property { default_value: Some("false"), validator: StandardPropertyValidator::BoolValidator, allowed_values: &[], - allowed_type: "", + allowed_type: None, }; pub(crate) const LOG_PREFIX: Property = Property { @@ -58,7 +58,7 @@ pub(crate) const LOG_PREFIX: Property = Property { default_value: None, validator: StandardPropertyValidator::AlwaysValidValidator, allowed_values: &[], - allowed_type: "", + allowed_type: None, }; pub(crate) const FLOW_FILES_TO_LOG: Property = Property { @@ -70,7 +70,7 @@ pub(crate) const FLOW_FILES_TO_LOG: Property = Property { default_value: Some("1"), validator: StandardPropertyValidator::AlwaysValidValidator, allowed_values: &[], - allowed_type: "", + allowed_type: None, }; pub(crate) const HEX_ENCODE_PAYLOAD: Property = Property { @@ -82,5 +82,5 @@ pub(crate) const HEX_ENCODE_PAYLOAD: Property = Property { default_value: Some("false"), validator: StandardPropertyValidator::BoolValidator, allowed_values: &[], - allowed_type: "", + allowed_type: None, }; diff --git a/minifi_rust/extensions/minifi_rs_playground/src/processors/lorem_ipsum_cs_user/properties.rs b/minifi_rust/extensions/minifi_rs_playground/src/processors/lorem_ipsum_cs_user/properties.rs index c18e1a7d01..687cbbec01 100644 --- a/minifi_rust/extensions/minifi_rs_playground/src/processors/lorem_ipsum_cs_user/properties.rs +++ b/minifi_rust/extensions/minifi_rs_playground/src/processors/lorem_ipsum_cs_user/properties.rs @@ -12,7 +12,7 @@ pub(crate) const CONTROLLER_SERVICE: Property = Property { default_value: None, validator: StandardPropertyValidator::AlwaysValidValidator, allowed_values: &[], - allowed_type: LoremIpsumControllerService::CLASS_NAME, + allowed_type: Some(LoremIpsumControllerService::CLASS_NAME), }; pub(crate) const WRITE_METHOD: Property = Property { @@ -24,5 +24,5 @@ pub(crate) const WRITE_METHOD: Property = Property { default_value: Some(super::WriteMethod::Buffer.into_str()), validator: StandardPropertyValidator::AlwaysValidValidator, allowed_values: super::WriteMethod::VARIANTS, - allowed_type: "", + allowed_type: None, }; diff --git a/minifi_rust/extensions/minifi_rs_playground/src/processors/mod.rs b/minifi_rust/extensions/minifi_rs_playground/src/processors/mod.rs index 0a6e3c7271..f1a885c728 100644 --- a/minifi_rust/extensions/minifi_rs_playground/src/processors/mod.rs +++ b/minifi_rust/extensions/minifi_rs_playground/src/processors/mod.rs @@ -7,3 +7,4 @@ pub(crate) mod kamikaze_processor; pub(crate) mod log_attribute; pub(crate) mod lorem_ipsum_cs_user; pub(crate) mod put_file; +pub(crate) mod zoo_processor; diff --git a/minifi_rust/extensions/minifi_rs_playground/src/processors/put_file/properties.rs b/minifi_rust/extensions/minifi_rs_playground/src/processors/put_file/properties.rs index 91953868c9..a6bd205c2d 100644 --- a/minifi_rust/extensions/minifi_rs_playground/src/processors/put_file/properties.rs +++ b/minifi_rust/extensions/minifi_rs_playground/src/processors/put_file/properties.rs @@ -11,7 +11,7 @@ pub(crate) const DIRECTORY: Property = Property { default_value: Some("."), validator: StandardPropertyValidator::NonBlankValidator, allowed_values: &[], - allowed_type: "", + allowed_type: None, }; pub(crate) const CONFLICT_RESOLUTION: Property = Property { @@ -23,7 +23,7 @@ pub(crate) const CONFLICT_RESOLUTION: Property = Property { default_value: Some(ConflictResolutionStrategy::Fail.into_str()), validator: StandardPropertyValidator::AlwaysValidValidator, allowed_values: &ConflictResolutionStrategy::VARIANTS, - allowed_type: "", + allowed_type: None, }; pub(crate) const CREATE_DIRS: Property = Property { @@ -35,7 +35,7 @@ pub(crate) const CREATE_DIRS: Property = Property { default_value: Some("true"), validator: StandardPropertyValidator::BoolValidator, allowed_values: &[], - allowed_type: "", + allowed_type: None, }; pub(crate) const MAX_FILE_COUNT: Property = Property { @@ -47,5 +47,5 @@ pub(crate) const MAX_FILE_COUNT: Property = Property { default_value: None, // Diverged from the original implementation, u64 with no default describes the behavior better validator: StandardPropertyValidator::U64Validator, allowed_values: &[], - allowed_type: "", + allowed_type: None, }; diff --git a/minifi_rust/extensions/minifi_rs_playground/src/processors/put_file/unix_only_properties.rs b/minifi_rust/extensions/minifi_rs_playground/src/processors/put_file/unix_only_properties.rs index 8e4ba1cc6e..8c98a861f8 100644 --- a/minifi_rust/extensions/minifi_rs_playground/src/processors/put_file/unix_only_properties.rs +++ b/minifi_rust/extensions/minifi_rs_playground/src/processors/put_file/unix_only_properties.rs @@ -9,7 +9,7 @@ pub(crate) const PERMISSIONS: Property = Property { default_value: None, validator: StandardPropertyValidator::AlwaysValidValidator, allowed_values: &[], - allowed_type: "", + allowed_type: None, }; pub(crate) const DIRECTORY_PERMISSIONS: Property = Property { @@ -21,5 +21,5 @@ pub(crate) const DIRECTORY_PERMISSIONS: Property = Property { default_value: None, validator: StandardPropertyValidator::AlwaysValidValidator, allowed_values: &[], - allowed_type: "", + allowed_type: None, }; diff --git a/minifi_rust/extensions/minifi_rs_playground/src/processors/zoo_processor.rs b/minifi_rust/extensions/minifi_rs_playground/src/processors/zoo_processor.rs new file mode 100644 index 0000000000..92ce6f89ad --- /dev/null +++ b/minifi_rust/extensions/minifi_rs_playground/src/processors/zoo_processor.rs @@ -0,0 +1,70 @@ +use crate::controller_services::animal_controller_apis::{CanFlyControllerApi, NumberOfLegsControllerApi}; +use minifi_native::{critical, info, GetProperty, Logger, MinifiError, OnTriggerResult, OutputAttribute, ProcessContext, ProcessSession, ProcessorDefinition, ProcessorInputRequirement, Property, Relationship, Schedule, StandardPropertyValidator, Trigger}; +use minifi_native::ControllerServiceApi; +use minifi_native::macros::ComponentIdentifier; + +pub(crate) const CAN_FLY_SERVICE: Property = Property { + name: "Can fly service", + description: "Test CanFlyService", + is_required: true, + is_sensitive: false, + supports_expr_lang: false, + default_value: None, + validator: StandardPropertyValidator::AlwaysValidValidator, + allowed_values: &[], + allowed_type: Some(::INTERFACE_NAME), +}; + +pub(crate) const NUMBER_OF_LEGS: Property = Property { + name: "Number of Legs Service", + description: "Test NumberOfLegsService", + is_required: true, + is_sensitive: false, + supports_expr_lang: false, + default_value: None, + validator: StandardPropertyValidator::AlwaysValidValidator, + allowed_values: &[], + allowed_type: Some(::INTERFACE_NAME), +}; + +#[derive(Debug, ComponentIdentifier)] +pub(crate) struct ZooProcessor { + +} + +impl Schedule for ZooProcessor { + fn schedule(_context: &Ctx, _logger: &L) -> Result + where + Self: Sized + { + Ok(Self{}) + } +} + +impl Trigger for ZooProcessor { + fn trigger(&self, context: &mut Context, _session: &mut Session, logger: &Lggr) -> Result + where + Context: ProcessContext, + Session: ProcessSession, + Lggr: Logger + { + info!(logger, "{:?}", self); + if let Some(maybe_flyer) = context.get_controller_service_api::(&CAN_FLY_SERVICE)? { + critical!(logger, "Can {:?} fly? {}", maybe_flyer, maybe_flyer.can_fly()); + } + if let Some(legged) = context.get_controller_service_api::(&NUMBER_OF_LEGS)? { + critical!(logger, "{:?} has {} legs", legged, legged.number_of_legs()); + } + Ok(OnTriggerResult::Ok) + } +} + +impl ProcessorDefinition for ZooProcessor { + const DESCRIPTION: &'static str = "Test ZooProcessor"; + const INPUT_REQUIREMENT: ProcessorInputRequirement = ProcessorInputRequirement::Forbidden; + const SUPPORTS_DYNAMIC_PROPERTIES: bool = false; + const SUPPORTS_DYNAMIC_RELATIONSHIPS: bool = false; + const OUTPUT_ATTRIBUTES: &'static [OutputAttribute] = &[]; + const RELATIONSHIPS: &'static [Relationship] = &[]; + const PROPERTIES: &'static [Property] = &[CAN_FLY_SERVICE, NUMBER_OF_LEGS]; +} \ No newline at end of file diff --git a/minifi_rust/minifi_native/src/api.rs b/minifi_rust/minifi_native/src/api.rs index 7413ddddeb..34d9e2e496 100644 --- a/minifi_rust/minifi_native/src/api.rs +++ b/minifi_rust/minifi_native/src/api.rs @@ -11,7 +11,8 @@ pub(crate) mod processor_wrappers; pub(crate) mod property; pub(crate) mod raw_controller_service; pub(crate) mod raw_processor; -mod relationship; +pub(crate) mod relationship; +pub(crate) mod provided_interface; pub use flow_file::FlowFile; pub use logger::{LogLevel, Logger}; diff --git a/minifi_rust/minifi_native/src/api/component_definition_traits.rs b/minifi_rust/minifi_native/src/api/component_definition_traits.rs index 671997e5fa..9afc25a193 100644 --- a/minifi_rust/minifi_native/src/api/component_definition_traits.rs +++ b/minifi_rust/minifi_native/src/api/component_definition_traits.rs @@ -1,4 +1,5 @@ use crate::{OutputAttribute, ProcessorInputRequirement, Property, Relationship}; +use crate::api::provided_interface::ProvidedInterface; pub trait ComponentIdentifier { const CLASS_NAME: &'static str; @@ -16,7 +17,8 @@ pub trait ProcessorDefinition { const PROPERTIES: &'static [Property]; } -pub trait ControllerServiceDefinition { +pub trait ControllerServiceDefinition: Sized + 'static { const DESCRIPTION: &'static str; const PROPERTIES: &'static [Property]; + const PROVIDED_APIS: &'static [ProvidedInterface]; } diff --git a/minifi_rust/minifi_native/src/api/process_context.rs b/minifi_rust/minifi_native/src/api/process_context.rs index fc60ca502f..cc8d184062 100644 --- a/minifi_rust/minifi_native/src/api/process_context.rs +++ b/minifi_rust/minifi_native/src/api/process_context.rs @@ -3,7 +3,7 @@ use crate::api::RawControllerService; use crate::api::component_definition_traits::ComponentIdentifier; use crate::api::flow_file::FlowFile; use crate::api::property::GetControllerService; -use crate::{EnableControllerService, GetProperty, MinifiError, Property}; +use crate::{ControllerServiceApi, ControllerServiceDefinition, EnableControllerService, GetProperty, MinifiError, Property}; use std::str::FromStr; use std::time::Duration; @@ -101,6 +101,12 @@ pub trait ProcessContext { where Cs: EnableControllerService + ComponentIdentifier + 'static; + fn get_controller_service_api( + &self, + property: &Property, + ) -> Result>, MinifiError>; + + fn report_metrics(&self, metrics: Vec<(String, f64)>) -> Result<(), MinifiError>; } @@ -119,7 +125,7 @@ where { fn get_controller_service(&self, property: &Property) -> Result, MinifiError> where - Cs: EnableControllerService + ComponentIdentifier + 'static, + Cs: EnableControllerService + ComponentIdentifier + ControllerServiceDefinition + 'static, { ProcessContext::get_controller_service::(self, property) } diff --git a/minifi_rust/minifi_native/src/api/process_session.rs b/minifi_rust/minifi_native/src/api/process_session.rs index 8fb78dfd18..f58dc9cc60 100644 --- a/minifi_rust/minifi_native/src/api/process_session.rs +++ b/minifi_rust/minifi_native/src/api/process_session.rs @@ -33,7 +33,7 @@ pub trait ProcessSession { ) -> bool; fn write(&self, flow_file: &Self::FlowFile, data: &[u8]) -> Result<(), MinifiError>; - fn write_lazy<'a>( + fn write_from_stream<'a>( &self, flow_file: &Self::FlowFile, stream: Box, diff --git a/minifi_rust/minifi_native/src/api/processor.rs b/minifi_rust/minifi_native/src/api/processor.rs index 60b15ceacf..c8900036f0 100644 --- a/minifi_rust/minifi_native/src/api/processor.rs +++ b/minifi_rust/minifi_native/src/api/processor.rs @@ -47,12 +47,12 @@ where self.logger.log(log_level, args); } - fn on_schedule(&mut self, context: &P) -> Result<(), MinifiError> { + fn schedule(&mut self, context: &P) -> Result<(), MinifiError> { self.scheduled_impl = Some(Impl::schedule(context, &self.logger)?); Ok(()) } - fn on_unschedule(&mut self) { + fn unschedule(&mut self) { if let Some(ref mut scheduled_impl) = self.scheduled_impl { scheduled_impl.unschedule() } diff --git a/minifi_rust/minifi_native/src/api/processor_wrappers/complex_processor.rs b/minifi_rust/minifi_native/src/api/processor_wrappers/complex_processor.rs index 4abebc0fcf..d70ea846ff 100644 --- a/minifi_rust/minifi_native/src/api/processor_wrappers/complex_processor.rs +++ b/minifi_rust/minifi_native/src/api/processor_wrappers/complex_processor.rs @@ -41,7 +41,7 @@ where + ProcessorDefinition, L: Logger, { - fn on_trigger( + fn trigger( &mut self, context: &mut PC, session: &mut PS, @@ -69,7 +69,7 @@ where + ProcessorDefinition, L: Logger, { - fn on_trigger( + fn trigger( &self, context: &mut PC, session: &mut PS, diff --git a/minifi_rust/minifi_native/src/api/processor_wrappers/flow_file_source.rs b/minifi_rust/minifi_native/src/api/processor_wrappers/flow_file_source.rs index 5bf2f33d10..27052a0273 100644 --- a/minifi_rust/minifi_native/src/api/processor_wrappers/flow_file_source.rs +++ b/minifi_rust/minifi_native/src/api/processor_wrappers/flow_file_source.rs @@ -64,7 +64,7 @@ where match new_flow_file_data.new_content { None => {} Some(Content::Buffer(buffer)) => session.write(&mut ff, &buffer)?, - Some(Content::Stream(stream)) => session.write_lazy(&mut ff, stream)?, + Some(Content::Stream(stream)) => session.write_from_stream(&mut ff, stream)?, } for (k, v) in &new_flow_file_data.attributes_to_add { session.set_attribute(&mut ff, k, v)?; @@ -82,7 +82,7 @@ where Implementation: Schedule + FlowFileSource, L: Logger, { - fn on_trigger( + fn trigger( &self, context: &mut PC, session: &mut PS, @@ -108,7 +108,7 @@ where Implementation: Schedule + MutFlowFileSource, L: Logger, { - fn on_trigger( + fn trigger( &mut self, context: &mut PC, session: &mut PS, diff --git a/minifi_rust/minifi_native/src/api/processor_wrappers/flow_file_stream_transform.rs b/minifi_rust/minifi_native/src/api/processor_wrappers/flow_file_stream_transform.rs index 71dae71a4c..0a47a48da1 100644 --- a/minifi_rust/minifi_native/src/api/processor_wrappers/flow_file_stream_transform.rs +++ b/minifi_rust/minifi_native/src/api/processor_wrappers/flow_file_stream_transform.rs @@ -123,7 +123,7 @@ where Schedule + FlowFileStreamTransform, L: Logger, { - fn on_trigger( + fn trigger( &self, context: &mut PC, session: &mut PS, @@ -152,7 +152,7 @@ where Schedule + MutFlowFileStreamTransform, L: Logger, { - fn on_trigger( + fn trigger( &mut self, context: &mut PC, session: &mut PS, diff --git a/minifi_rust/minifi_native/src/api/processor_wrappers/flow_file_transform.rs b/minifi_rust/minifi_native/src/api/processor_wrappers/flow_file_transform.rs index 3ddd19029e..620d57afba 100644 --- a/minifi_rust/minifi_native/src/api/processor_wrappers/flow_file_transform.rs +++ b/minifi_rust/minifi_native/src/api/processor_wrappers/flow_file_transform.rs @@ -13,7 +13,7 @@ use std::collections::HashMap; #[derive(Debug)] pub struct TransformedFlowFile<'a> { target_relationship_name: &'static str, - new_content: Option>, + new_content: Option>, // If None the content doesn't change attributes_to_add: HashMap, } @@ -107,7 +107,7 @@ where session.write(&flow_file, &buffer)?; } Some(Content::Stream(stream)) => { - session.write_lazy(&flow_file, stream)?; + session.write_from_stream(&flow_file, stream)?; } }; @@ -135,7 +135,7 @@ where Implementation: Schedule + FlowFileTransform, L: Logger, { - fn on_trigger( + fn trigger( &self, context: &mut PC, session: &mut PS, @@ -162,7 +162,7 @@ where Implementation: Schedule + MutFlowFileTransform, L: Logger, { - fn on_trigger( + fn trigger( &mut self, context: &mut PC, session: &mut PS, diff --git a/minifi_rust/minifi_native/src/api/property.rs b/minifi_rust/minifi_native/src/api/property.rs index 3f87b5615c..e5233571c3 100644 --- a/minifi_rust/minifi_native/src/api/property.rs +++ b/minifi_rust/minifi_native/src/api/property.rs @@ -1,7 +1,7 @@ use crate::StandardPropertyValidator::{ BoolValidator, DataSizeValidator, TimePeriodValidator, U64Validator, }; -use crate::{ComponentIdentifier, EnableControllerService, MinifiError}; +use crate::{ComponentIdentifier, ControllerServiceDefinition, EnableControllerService, MinifiError}; use std::str::FromStr; use std::time::Duration; @@ -27,7 +27,7 @@ pub struct Property { pub default_value: Option<&'static str>, pub validator: StandardPropertyValidator, pub allowed_values: &'static [&'static str], - pub allowed_type: &'static str, + pub allowed_type: Option<&'static str>, } pub trait GetProperty { @@ -94,5 +94,5 @@ pub trait GetProperty { pub trait GetControllerService { fn get_controller_service(&self, property: &Property) -> Result, MinifiError> where - Cs: EnableControllerService + ComponentIdentifier + 'static; + Cs: EnableControllerService + ComponentIdentifier + ControllerServiceDefinition + 'static; } diff --git a/minifi_rust/minifi_native/src/api/provided_interface.rs b/minifi_rust/minifi_native/src/api/provided_interface.rs new file mode 100644 index 0000000000..2ce3021aed --- /dev/null +++ b/minifi_rust/minifi_native/src/api/provided_interface.rs @@ -0,0 +1,36 @@ +pub trait ControllerServiceApi { + const INTERFACE_NAME: &'static str; +} + +#[macro_export] +macro_rules! impl_interface_fqn { + ($trait_name:ident) => { + impl ControllerServiceApi for dyn $trait_name { + const INTERFACE_NAME: &'static str = concat!( + module_path!(), + "::", + stringify!($trait_name) + ); + } + }; +} + +#[derive(Debug)] +pub struct ProvidedInterface { + pub name: &'static str, + pub cast: fn(&T) -> *mut std::ffi::c_void, +} + +#[macro_export] +macro_rules! create_provided_interface { + ($trait_type:ty) => { + ProvidedInterface { + name: <$trait_type as ControllerServiceApi>::INTERFACE_NAME, + cast: |instance| { + let trait_ref: &$trait_type = instance; + let boxed_ref: Box<&$trait_type> = Box::new(trait_ref); + Box::into_raw(boxed_ref) as *mut std::ffi::c_void + }, + } + }; +} \ No newline at end of file diff --git a/minifi_rust/minifi_native/src/api/raw_processor.rs b/minifi_rust/minifi_native/src/api/raw_processor.rs index 6926db17a8..ea1d3d7f67 100644 --- a/minifi_rust/minifi_native/src/api/raw_processor.rs +++ b/minifi_rust/minifi_native/src/api/raw_processor.rs @@ -20,8 +20,8 @@ pub trait RawProcessor: Sized { fn new(logger: Self::LoggerType) -> Self; fn log(&self, log_level: LogLevel, args: std::fmt::Arguments); - fn on_schedule(&mut self, context: &P) -> Result<(), MinifiError>; - fn on_unschedule(&mut self); + fn schedule(&mut self, context: &P) -> Result<(), MinifiError>; + fn unschedule(&mut self); } /// To differentiate between single and multithreaded processors @@ -48,7 +48,7 @@ mod sealed { } pub trait SingleThreadedTrigger: RawProcessor { - fn on_trigger( + fn trigger( &mut self, context: &mut PC, session: &mut PS, @@ -59,7 +59,7 @@ pub trait SingleThreadedTrigger: RawProcessor { } pub trait MultiThreadedTrigger: RawProcessor { - fn on_trigger( + fn trigger( &self, context: &mut PC, session: &mut PS, diff --git a/minifi_rust/minifi_native/src/c_ffi/c_ffi_controller_service_definition.rs b/minifi_rust/minifi_native/src/c_ffi/c_ffi_controller_service_definition.rs index 6923889a7e..7d8a1367f0 100644 --- a/minifi_rust/minifi_native/src/c_ffi/c_ffi_controller_service_definition.rs +++ b/minifi_rust/minifi_native/src/c_ffi/c_ffi_controller_service_definition.rs @@ -2,14 +2,8 @@ use crate::api::RawControllerService; use crate::c_ffi::c_ffi_controller_service_context::CffiControllerServiceContext; use crate::c_ffi::c_ffi_property::CProperties; use crate::c_ffi::{CffiLogger, StaticStrAsMinifiCStr}; -use crate::{ - ComponentIdentifier, ControllerService, ControllerServiceDefinition, EnableControllerService, - LogLevel, Property, -}; -use minifi_native_sys::{ - minifi_controller_service_callbacks, minifi_controller_service_class_definition, - minifi_controller_service_context, minifi_controller_service_metadata, minifi_status, -}; +use crate::{ComponentIdentifier, ControllerService, ControllerServiceDefinition, EnableControllerService, LogLevel, Property, ProvidedInterface}; +use minifi_native_sys::{minifi_controller_service_callbacks, minifi_controller_service_class_definition, minifi_controller_service_context, minifi_controller_service_metadata, minifi_status, minifi_string_view}; use std::ffi::c_void; #[derive(Debug)] @@ -31,30 +25,39 @@ impl<'a> ControllerServiceClassDefinition<'a> { } } -pub struct CffiControllerServiceDefinition +pub struct CffiControllerServiceDefinition where T: RawControllerService + ComponentIdentifier, + P: ControllerServiceDefinition { name: &'static str, description_text: &'static str, c_properties: CProperties, + c_provided_interfaces: Vec, _phantom: std::marker::PhantomData, + _phantom_2: std::marker::PhantomData

, } -impl CffiControllerServiceDefinition +impl CffiControllerServiceDefinition where T: RawControllerService + ComponentIdentifier, + P: ControllerServiceDefinition { - pub fn new(description_text: &'static str, properties: &'static [Property]) -> Self { + pub fn new(description_text: &'static str, properties: &'static [Property], provided_interfaces: &'static [ProvidedInterface

]) -> Self { let c_properties = Property::create_c_properties(properties); - + let mut c_provided_apis = Vec::new(); + for provided_interface in provided_interfaces { + c_provided_apis.push(provided_interface.name.as_minifi_c_type()); + } Self { name: T::CLASS_NAME, description_text, c_properties, + c_provided_interfaces: c_provided_apis, _phantom: std::marker::PhantomData, + _phantom_2: std::marker::PhantomData } } @@ -97,15 +100,42 @@ where controller_service.disable() } } + + extern "C" fn get_interface( + self_ptr: *mut std::ffi::c_void, + interface_name: minifi_native_sys::minifi_string_view, + ) -> *mut std::ffi::c_void { + let name = unsafe { + let slice = std::slice::from_raw_parts( + interface_name.data as *const u8, + interface_name.length + ); + std::str::from_utf8_unchecked(slice) + }; + + let wrapper = unsafe { + &*(self_ptr as *const ControllerService) + }; + + if let Some(inner_instance) = wrapper.get_implementation() { + for iface in C::PROVIDED_APIS { + if iface.name == name { + return (iface.cast)(inner_instance); + } + } + } + + std::ptr::null_mut() + } } pub trait DynRawControllerServiceDefinition { fn class_description(&'_ self) -> ControllerServiceClassDefinition<'_>; } -impl DynRawControllerServiceDefinition for CffiControllerServiceDefinition +impl DynRawControllerServiceDefinition for CffiControllerServiceDefinition, Impl> where - T: RawControllerService + ComponentIdentifier, + Impl: EnableControllerService + ComponentIdentifier + ControllerServiceDefinition + 'static, { fn class_description(&'_ self) -> ControllerServiceClassDefinition<'_> { unsafe { @@ -119,10 +149,10 @@ where destroy: Some(Self::destroy_controller_service), enable: Some(Self::enable_controller_service), disable: Some(Self::disable_controller_service), - get_interface: None, // TODO(mzink) + get_interface: Some(Self::get_interface::) }, - provided_interfaces_count: 0, - provided_interfaces_ptr: std::ptr::null_mut() // TODO(mzink) + provided_interfaces_count: self.c_provided_interfaces.len(), + provided_interfaces_ptr: self.c_provided_interfaces.as_ptr() }) } } @@ -132,21 +162,6 @@ pub trait RegisterableControllerService { fn get_definition() -> Box; } -impl RegisterableControllerService for T -where - T: ComponentIdentifier - + ControllerServiceDefinition - + RawControllerService - + 'static, -{ - fn get_definition() -> Box { - Box::new(CffiControllerServiceDefinition::::new( - T::DESCRIPTION, - T::PROPERTIES, - )) - } -} - impl RegisterableControllerService for ControllerService where Implementation: @@ -154,9 +169,9 @@ where { fn get_definition() -> Box { Box::new(CffiControllerServiceDefinition::< - ControllerService, + ControllerService, Implementation >::new( - Implementation::DESCRIPTION, Implementation::PROPERTIES + Implementation::DESCRIPTION, Implementation::PROPERTIES, Implementation::PROVIDED_APIS )) } } diff --git a/minifi_rust/minifi_native/src/c_ffi/c_ffi_process_context.rs b/minifi_rust/minifi_native/src/c_ffi/c_ffi_process_context.rs index e84f7fa108..9fc2104f1a 100644 --- a/minifi_rust/minifi_native/src/c_ffi/c_ffi_process_context.rs +++ b/minifi_rust/minifi_native/src/c_ffi/c_ffi_process_context.rs @@ -3,7 +3,7 @@ use super::c_ffi_primitives::StringView; use crate::api::ProcessContext; use crate::api::controller_service::ControllerService; use crate::c_ffi::{CffiLogger, StaticStrAsMinifiCStr}; -use crate::{ComponentIdentifier, EnableControllerService, MinifiError, Property}; +use crate::{ComponentIdentifier, ControllerServiceApi, EnableControllerService, MinifiError, Property}; use minifi_native_sys::*; use std::borrow::Cow; use std::ffi::c_void; @@ -128,6 +128,50 @@ impl<'a> ProcessContext for CffiProcessContext<'a> { } } + fn get_controller_service_api( + &self, + property: &Property, + ) -> Result>, MinifiError> { + let str_view = StringView::new(property.name); + let interface_view = StringView::new(Trait::INTERFACE_NAME); + + let mut interface_ptr: *mut minifi_controller_service = std::ptr::null_mut(); + + unsafe { + let get_cs_status = minifi_process_context_get_controller_service_from_property( + self.ptr, + str_view.as_raw(), + interface_view.as_raw(), + &mut interface_ptr, + ); + + if get_cs_status == minifi_status_MINIFI_STATUS_PROPERTY_NOT_SET { + return Ok(None); + } + + if get_cs_status != minifi_status_MINIFI_STATUS_SUCCESS { + return Err(MinifiError::StatusError(( + format!( + "Failed to get controller service interface <{}> for property {:?}", + Trait::INTERFACE_NAME, property.name + ) + .into(), + NonZeroU32::new_unchecked(get_cs_status), + ))); + } + + if interface_ptr.is_null() { + return Ok(None); + } + + // MAGIC: We receive the thin void pointer from the C API, + // cast it back to a Box of a Fat Pointer, and unbox it! + let boxed_ref = Box::from_raw(interface_ptr as *mut &Trait); + + Ok(Some(boxed_ref)) + } + } + fn report_metrics(&self, metrics: Vec<(String, f64)>) -> Result<(), MinifiError>{ let mut keys = Vec::new(); let mut values = Vec::new(); diff --git a/minifi_rust/minifi_native/src/c_ffi/c_ffi_process_session.rs b/minifi_rust/minifi_native/src/c_ffi/c_ffi_process_session.rs index 3ec6a71237..6e4364a626 100644 --- a/minifi_rust/minifi_native/src/c_ffi/c_ffi_process_session.rs +++ b/minifi_rust/minifi_native/src/c_ffi/c_ffi_process_session.rs @@ -18,7 +18,7 @@ const _: () = { pub struct CffiProcessSession<'a> { ptr: *mut minifi_process_session, - // The lifetime ensures the session cannot outlive the `on_trigger` call. + // The lifetime ensures the session cannot outlive the `trigger` call. _lifetime: std::marker::PhantomData<&'a ()>, } @@ -84,7 +84,7 @@ impl<'a> CffiProcessSession<'a> { #[allow(non_upper_case_globals)] minifi_status_MINIFI_STATUS_SUCCESS => Ok(()), error_code => Err(MinifiError::StatusError(( - "MinifiProcessSessionWrite".into(), + "minifi_process_session_write".into(), NonZeroU32::new_unchecked(error_code), ))), } @@ -127,7 +127,7 @@ impl<'a> ProcessSession for CffiProcessSession<'a> { #[allow(non_upper_case_globals)] minifi_status_MINIFI_STATUS_SUCCESS => Ok(()), err_code => Err(MinifiError::StatusError(( - "MinifiProcessSessionTransfer".into(), + "minifi_process_session_transfer".into(), NonZeroU32::new_unchecked(err_code), ))), } @@ -140,7 +140,7 @@ impl<'a> ProcessSession for CffiProcessSession<'a> { #[allow(non_upper_case_globals)] minifi_status_MINIFI_STATUS_SUCCESS => Ok(()), err_code => Err(MinifiError::StatusError(( - "MinifiProcessSessionRemove".into(), + "minifi_process_session_remove".into(), NonZeroU32::new_unchecked(err_code), ))), } @@ -166,7 +166,7 @@ impl<'a> ProcessSession for CffiProcessSession<'a> { minifi_status_MINIFI_STATUS_SUCCESS => Ok(()), err_code => Err(MinifiError::StatusError(( format!( - "MinifiProcessSessionFlowFileSetAttribute({}, {})", + "minifi_process_session_set_flow_file_attribute({}, {})", attr_key, attr_value ) .into(), @@ -274,14 +274,14 @@ impl<'a> ProcessSession for CffiProcessSession<'a> { #[allow(non_upper_case_globals)] minifi_status_MINIFI_STATUS_SUCCESS => Ok(()), err_code => Err(MinifiError::StatusError(( - "MinifiProcessSessionWrite".into(), + "minifi_process_session_write".into(), NonZeroU32::new_unchecked(err_code), ))), } } } - fn write_lazy<'b>( + fn write_from_stream<'b>( &self, flow_file: &Self::FlowFile, mut stream: Box, @@ -349,7 +349,7 @@ impl<'a> ProcessSession for CffiProcessSession<'a> { ); if session_status != minifi_status_MINIFI_STATUS_SUCCESS { return Err(MinifiError::StatusError(( - "MinifiProcessSessionWrite".into(), + "minifi_process_session_write".into(), NonZeroU32::new_unchecked(session_status), ))); } diff --git a/minifi_rust/minifi_native/src/c_ffi/c_ffi_processor_definition.rs b/minifi_rust/minifi_native/src/c_ffi/c_ffi_processor_definition.rs index 67cacd566d..a9db67e876 100644 --- a/minifi_rust/minifi_native/src/c_ffi/c_ffi_processor_definition.rs +++ b/minifi_rust/minifi_native/src/c_ffi/c_ffi_processor_definition.rs @@ -37,13 +37,13 @@ where let processor = &*(processor_ptr as *const T); let mut context = CffiProcessContext::new(context_ptr); let mut session = CffiProcessSession::new(session_ptr); - match processor.on_trigger(&mut context, &mut session) { + match processor.trigger(&mut context, &mut session) { Ok(OnTriggerResult::Ok) => minifi_status_MINIFI_STATUS_SUCCESS, Ok(OnTriggerResult::Yield) => minifi_status_MINIFI_STATUS_PROCESSOR_YIELD, Err(minifi_error) => { processor.log( LogLevel::Error, - format_args!("Error during on_trigger {}", minifi_error), + format_args!("Error during trigger {}", minifi_error), ); minifi_error.to_status() } @@ -65,7 +65,7 @@ where let processor = &mut *(processor_ptr as *mut T); let mut context = CffiProcessContext::new(context_ptr); let mut session = CffiProcessSession::new(session_ptr); - match processor.on_trigger(&mut context, &mut session) { + match processor.trigger(&mut context, &mut session) { Ok(OnTriggerResult::Ok) => minifi_status_MINIFI_STATUS_SUCCESS, Ok(OnTriggerResult::Yield) => minifi_status_MINIFI_STATUS_PROCESSOR_YIELD, Err(error_code) => error_code.to_status(), @@ -136,7 +136,7 @@ where } } - unsafe extern "C" fn on_trigger_processor( + unsafe extern "C" fn trigger_processor( processor_ptr: *mut c_void, context_ptr: *mut minifi_process_context, session_ptr: *mut minifi_process_session, @@ -150,19 +150,19 @@ where } } - unsafe extern "C" fn on_schedule_processor( + unsafe extern "C" fn schedule_processor( processor_ptr: *mut c_void, context_ptr: *mut minifi_process_context, ) -> minifi_status { unsafe { let processor = &mut *(processor_ptr as *mut T); let context = CffiProcessContext::new(context_ptr); - match processor.on_schedule(&context) { + match processor.schedule(&context) { Ok(_) => 0, Err(error_code) => { processor.log( LogLevel::Error, - format_args!("Error during on_schedule: {}", error_code), + format_args!("Error during schedule: {}", error_code), ); error_code.to_status() } @@ -170,10 +170,10 @@ where } } - unsafe extern "C" fn on_unschedule_processor(processor_ptr: *mut c_void) { + unsafe extern "C" fn unschedule_processor(processor_ptr: *mut c_void) { unsafe { let processor = &mut *(processor_ptr as *mut T); - processor.on_unschedule(); + processor.unschedule(); } } } @@ -225,9 +225,9 @@ where callbacks: minifi_processor_callbacks { create: Some(Self::create_processor), destroy: Some(Self::destroy_processor), - trigger: Some(Self::on_trigger_processor), - schedule: Some(Self::on_schedule_processor), - unschedule: Some(Self::on_unschedule_processor), + trigger: Some(Self::trigger_processor), + schedule: Some(Self::schedule_processor), + unschedule: Some(Self::unschedule_processor), }, }) } diff --git a/minifi_rust/minifi_native/src/c_ffi/c_ffi_property.rs b/minifi_rust/minifi_native/src/c_ffi/c_ffi_property.rs index 11ebe4c5ca..bf3d8f5830 100644 --- a/minifi_rust/minifi_native/src/c_ffi/c_ffi_property.rs +++ b/minifi_rust/minifi_native/src/c_ffi/c_ffi_property.rs @@ -64,7 +64,13 @@ impl Property { fn create_c_allowed_types_vec(properties: &[Self]) -> Vec { properties .iter() - .map(|p| p.allowed_type.as_minifi_c_type()) + .map(|p| match p.allowed_type { + Some(dv) => dv.as_minifi_c_type(), + None => minifi_string_view { + data: ptr::null(), + length: 0, + }, + }) .collect() } @@ -88,11 +94,11 @@ impl Property { description: property.description.as_minifi_c_type(), is_required: property.is_required, is_sensitive: property.is_sensitive, - default_value: def_value, + default_value: if def_value.data == std::ptr::null() { std::ptr::null() } else { def_value }, allowed_values_count: allowed_values.len(), allowed_values_ptr: allowed_values.as_ptr(), validator: property.validator.as_minifi_c_type(), - allowed_type, + allowed_type: if allowed_type.data == std::ptr::null() { std::ptr::null() } else { allowed_type }, supports_expression_language: property.supports_expr_lang, } }) diff --git a/minifi_rust/minifi_native/src/lib.rs b/minifi_rust/minifi_native/src/lib.rs index 97286e3337..6f50d9beeb 100644 --- a/minifi_rust/minifi_native/src/lib.rs +++ b/minifi_rust/minifi_native/src/lib.rs @@ -30,6 +30,8 @@ pub use api::logger::{LogLevel, Logger}; pub use api::property::{GetControllerService, GetProperty, Property}; +pub use api::provided_interface::{ProvidedInterface, ControllerServiceApi}; + pub use api::process_session::IoState; pub use api::attribute::{GetAttribute, OutputAttribute}; @@ -51,7 +53,7 @@ pub use mock::{ #[cfg_attr(target_os = "linux", unsafe(link_section = ".rodata"))] #[cfg_attr(target_os = "macos", unsafe(link_section = "__DATA,__const"))] #[cfg_attr(target_os = "windows", unsafe(link_section = ".rdata"))] -pub static MinifiApiVersion: u32 = minifi_native_sys::MINIFI_API_VERSION; +pub static minifi_api_version: u32 = minifi_native_sys::MINIFI_API_VERSION; /// Defines the required MinifiInitExtension C function to register the listed processors and controller services #[macro_export] diff --git a/minifi_rust/minifi_native/src/mock/mock_process_context.rs b/minifi_rust/minifi_native/src/mock/mock_process_context.rs index c68afd96d0..b939319d9c 100644 --- a/minifi_rust/minifi_native/src/mock/mock_process_context.rs +++ b/minifi_rust/minifi_native/src/mock/mock_process_context.rs @@ -1,7 +1,5 @@ use crate::api::{ProcessContext, RawControllerService}; -use crate::{ - ComponentIdentifier, EnableControllerService, GetAttribute, MinifiError, MockFlowFile, Property, -}; +use crate::{ComponentIdentifier, ControllerServiceApi, EnableControllerService, GetAttribute, MinifiError, MockFlowFile, Property}; use std::any::Any; use std::borrow::Cow; use std::collections::HashMap; @@ -99,6 +97,10 @@ impl ProcessContext for MockProcessContext { } } + fn get_controller_service_api(&self, _property: &Property) -> Result>, MinifiError> { + todo!() + } + fn report_metrics(&self, _metrics: Vec<(String, f64)>) -> Result<(), MinifiError> { Ok(()) } diff --git a/minifi_rust/minifi_native/src/mock/mock_process_session.rs b/minifi_rust/minifi_native/src/mock/mock_process_session.rs index 475eb3a8cf..d9c5864df1 100644 --- a/minifi_rust/minifi_native/src/mock/mock_process_session.rs +++ b/minifi_rust/minifi_native/src/mock/mock_process_session.rs @@ -70,7 +70,7 @@ impl ProcessSession for MockProcessSession { Ok(()) } - fn write_lazy<'a>( + fn write_from_stream<'a>( &self, flow_file: &Self::FlowFile, mut stream: Box, diff --git a/minifi_rust/minifi_native_macros/src/lib.rs b/minifi_rust/minifi_native_macros/src/lib.rs index 9961a58664..f093b82a7a 100644 --- a/minifi_rust/minifi_native_macros/src/lib.rs +++ b/minifi_rust/minifi_native_macros/src/lib.rs @@ -1,6 +1,6 @@ use proc_macro::TokenStream; use quote::quote; -use syn::{DeriveInput, parse_macro_input}; +use syn::{DeriveInput, parse_macro_input, ItemTrait}; #[proc_macro_derive(ComponentIdentifier)] pub fn derive_component_identifier(input: TokenStream) -> TokenStream { @@ -18,3 +18,20 @@ pub fn derive_component_identifier(input: TokenStream) -> TokenStream { TokenStream::from(expanded) } + +#[proc_macro_attribute] +pub fn controller_service_api(_attr: TokenStream, item: TokenStream) -> TokenStream { + let input = parse_macro_input!(item as ItemTrait); + let name = &input.ident; + let name_str = name.to_string(); + + let expanded = quote! { + #input + + impl ::minifi_native::ControllerServiceApi for dyn #name { + const INTERFACE_NAME: &'static str = concat!(module_path!(), "::", #name_str); + } + }; + + TokenStream::from(expanded) +} diff --git a/minifi_rust/minifi_native_sys/src/lib.rs b/minifi_rust/minifi_native_sys/src/lib.rs index 1b9a6c3e45..fd4dddab9a 100644 --- a/minifi_rust/minifi_native_sys/src/lib.rs +++ b/minifi_rust/minifi_native_sys/src/lib.rs @@ -6,6 +6,6 @@ //! This crate provides raw, unsafe FFI bindings for the Apache NiFi MiNiFi C API. //! -//! The bindings are generated automatically by `bindgen` from the `minifi-c.h` header. +//! The bindings are generated automatically by `bindgen` from the `minifi-api.h` header. include!(concat!(env!("OUT_DIR"), "/bindings.rs")); diff --git a/minifi_rust/minifi_rs_behave/linux_build.sh b/minifi_rust/minifi_rs_behave/linux_build.sh index fc94e48961..10b9fb2e77 100755 --- a/minifi_rust/minifi_rs_behave/linux_build.sh +++ b/minifi_rust/minifi_rs_behave/linux_build.sh @@ -48,23 +48,23 @@ elif [ -d "$RESOLVED_SDK_PATH" ]; then mkdir -p "$DOCKER_SDK_DIR" # Case A: It's a flat SDK layout - if [ -f "$RESOLVED_SDK_PATH/minifi-c.h" ]; then - cp "$RESOLVED_SDK_PATH/minifi-c.h" "$DOCKER_SDK_DIR/" - cp "$RESOLVED_SDK_PATH/minifi-c-api.def" "$DOCKER_SDK_DIR/" 2>/dev/null || true + if [ -f "$RESOLVED_SDK_PATH/minifi-api.h" ]; then + cp "$RESOLVED_SDK_PATH/minifi-api.h" "$DOCKER_SDK_DIR/" + cp "$RESOLVED_SDK_PATH/minifi-api.def" "$DOCKER_SDK_DIR/" 2>/dev/null || true cp "$RESOLVED_SDK_PATH"/*.whl "$DOCKER_SDK_DIR/" 2>/dev/null || true # Case B: It's a raw repository layout - elif [ -f "$RESOLVED_SDK_PATH/minifi-api/include/minifi-c/minifi-c.h" ]; then + elif [ -f "$RESOLVED_SDK_PATH/minifi-api/include/minifi-c/minifi-api.h" ]; then mkdir -p "$DOCKER_SDK_DIR/minifi-api/include/minifi-c" - cp "$RESOLVED_SDK_PATH/minifi-api/include/minifi-c/minifi-c.h" "$DOCKER_SDK_DIR/minifi-api/include/minifi-c/" - cp "$RESOLVED_SDK_PATH/minifi-api/minifi-c-api.def" "$DOCKER_SDK_DIR/minifi-api/" 2>/dev/null || true + cp "$RESOLVED_SDK_PATH/minifi-api/include/minifi-c/minifi-api.h" "$DOCKER_SDK_DIR/minifi-api/include/minifi-c/" + cp "$RESOLVED_SDK_PATH/minifi-api/minifi-api.def" "$DOCKER_SDK_DIR/minifi-api/" 2>/dev/null || true if [ -d "$RESOLVED_SDK_PATH/behave_framework" ]; then cp -R "$RESOLVED_SDK_PATH/behave_framework" "$DOCKER_SDK_DIR/" fi else - echo "ERROR: Could not find minifi-c.h in $RESOLVED_SDK_PATH" + echo "ERROR: Could not find minifi-api.h in $RESOLVED_SDK_PATH" exit 1 fi From 187e07c9489121f9949a0b0ec10667ca0b31fe2e Mon Sep 17 00:00:00 2001 From: Martin Zink Date: Thu, 25 Jun 2026 14:44:03 +0200 Subject: [PATCH 03/26] cargo fmt --- .../animal_controller_apis.rs | 2 +- .../dog_controller_service.rs | 28 ++++++---- .../duck_controller_service.rs | 20 +++---- .../dummy_controller_service.rs | 5 +- .../lorem_ipsum_controller_service.rs | 5 +- .../src/controller_services/mod.rs | 6 +-- .../src/processors/asciify_german.rs | 2 +- .../src/processors/count_actual_logging.rs | 2 +- .../src/processors/duplicate_text.rs | 2 +- .../src/processors/generate_flow_file.rs | 6 +-- .../src/processors/get_file.rs | 6 +-- .../src/processors/kamikaze_processor.rs | 14 ++--- .../processors/kamikaze_processor/tests.rs | 18 +++---- .../src/processors/log_attribute.rs | 5 +- .../src/processors/lorem_ipsum_cs_user.rs | 2 +- .../src/processors/put_file.rs | 2 +- .../src/processors/zoo_processor.rs | 51 ++++++++++++------ minifi_rust/minifi_native/src/api.rs | 2 +- .../src/api/component_definition_traits.rs | 2 +- .../minifi_native/src/api/process_context.rs | 6 ++- .../processor_wrappers/complex_processor.rs | 10 +--- .../processor_wrappers/flow_file_source.rs | 5 +- .../flow_file_stream_transform.rs | 12 ++--- .../processor_wrappers/flow_file_transform.rs | 8 +-- minifi_rust/minifi_native/src/api/property.rs | 4 +- .../src/api/provided_interface.rs | 9 ++-- .../c_ffi/c_ffi_controller_service_context.rs | 3 +- .../c_ffi_controller_service_definition.rs | 53 ++++++++++++------- .../c_ffi/c_ffi_controller_service_list.rs | 3 +- .../minifi_native/src/c_ffi/c_ffi_logger.rs | 5 +- .../src/c_ffi/c_ffi_output_attribute.rs | 2 +- .../src/c_ffi/c_ffi_primitives.rs | 8 ++- .../src/c_ffi/c_ffi_process_context.rs | 23 +++++--- .../src/c_ffi/c_ffi_process_session.rs | 11 +++- .../src/c_ffi/c_ffi_processor_definition.rs | 11 ++-- .../src/c_ffi/c_ffi_processor_list.rs | 2 +- .../minifi_native/src/c_ffi/c_ffi_property.rs | 21 ++++++-- .../minifi_native/src/c_ffi/c_ffi_streams.rs | 4 +- minifi_rust/minifi_native/src/lib.rs | 2 +- .../src/mock/mock_process_context.rs | 10 +++- minifi_rust/minifi_native_macros/src/lib.rs | 2 +- 41 files changed, 237 insertions(+), 157 deletions(-) diff --git a/minifi_rust/extensions/minifi_rs_playground/src/controller_services/animal_controller_apis.rs b/minifi_rust/extensions/minifi_rs_playground/src/controller_services/animal_controller_apis.rs index 358963784e..f2f1e26760 100644 --- a/minifi_rust/extensions/minifi_rs_playground/src/controller_services/animal_controller_apis.rs +++ b/minifi_rust/extensions/minifi_rs_playground/src/controller_services/animal_controller_apis.rs @@ -1,5 +1,5 @@ -use std::fmt::Debug; use minifi_native::macros::controller_service_api; +use std::fmt::Debug; #[controller_service_api] pub trait NumberOfLegsControllerApi: Debug { diff --git a/minifi_rust/extensions/minifi_rs_playground/src/controller_services/dog_controller_service.rs b/minifi_rust/extensions/minifi_rs_playground/src/controller_services/dog_controller_service.rs index 9649b9d1f4..3fc17cafc9 100644 --- a/minifi_rust/extensions/minifi_rs_playground/src/controller_services/dog_controller_service.rs +++ b/minifi_rust/extensions/minifi_rs_playground/src/controller_services/dog_controller_service.rs @@ -1,7 +1,12 @@ +use crate::controller_services::animal_controller_apis::{ + CanFlyControllerApi, NumberOfLegsControllerApi, +}; use minifi_native::ControllerServiceApi; -use minifi_native::{create_provided_interface, ControllerServiceDefinition, EnableControllerService, GetProperty, Logger, MinifiError, Property, ProvidedInterface, StandardPropertyValidator}; use minifi_native::macros::ComponentIdentifier; -use crate::controller_services::animal_controller_apis::{CanFlyControllerApi, NumberOfLegsControllerApi}; +use minifi_native::{ + ControllerServiceDefinition, EnableControllerService, GetProperty, Logger, MinifiError, + Property, ProvidedInterface, StandardPropertyValidator, create_provided_interface, +}; pub(crate) const HAS_JETPACK: Property = Property { name: "Has Jetpack", @@ -27,11 +32,11 @@ pub(crate) const EXTRA_INFO: Property = Property { allowed_type: None, }; -#[allow(dead_code)] // extra_info is only used by {:?} +#[allow(dead_code)] // extra_info is only used by {:?} #[derive(Debug, ComponentIdentifier)] pub(crate) struct DogController { has_jetpack: bool, - extra_info: String + extra_info: String, } impl NumberOfLegsControllerApi for DogController { @@ -49,15 +54,18 @@ impl CanFlyControllerApi for DogController { impl EnableControllerService for DogController { fn enable(context: &Ctx, _logger: &L) -> Result where - Self: Sized + Self: Sized, { - let has_jetpack = context - .get_bool_property(&HAS_JETPACK)? - .ok_or(MinifiError::missing_required_property("Has jetpack is required"))?; + let has_jetpack = context.get_bool_property(&HAS_JETPACK)?.ok_or( + MinifiError::missing_required_property("Has jetpack is required"), + )?; let extra_info = context.get_property(&EXTRA_INFO)?.unwrap_or("".into()); - Ok(Self { has_jetpack, extra_info }) + Ok(Self { + has_jetpack, + extra_info, + }) } } @@ -68,4 +76,4 @@ impl ControllerServiceDefinition for DogController { create_provided_interface!(dyn CanFlyControllerApi), create_provided_interface!(dyn NumberOfLegsControllerApi), ]; -} \ No newline at end of file +} diff --git a/minifi_rust/extensions/minifi_rs_playground/src/controller_services/duck_controller_service.rs b/minifi_rust/extensions/minifi_rs_playground/src/controller_services/duck_controller_service.rs index a565ccbffb..41132ce69b 100644 --- a/minifi_rust/extensions/minifi_rs_playground/src/controller_services/duck_controller_service.rs +++ b/minifi_rust/extensions/minifi_rs_playground/src/controller_services/duck_controller_service.rs @@ -1,13 +1,15 @@ +use crate::controller_services::animal_controller_apis::{ + CanFlyControllerApi, NumberOfLegsControllerApi, +}; use minifi_native::ControllerServiceApi; -use minifi_native::{create_provided_interface, ControllerServiceDefinition, EnableControllerService, GetProperty, Logger, MinifiError, Property, ProvidedInterface}; use minifi_native::macros::ComponentIdentifier; -use crate::controller_services::animal_controller_apis::{CanFlyControllerApi, NumberOfLegsControllerApi}; +use minifi_native::{ + ControllerServiceDefinition, EnableControllerService, GetProperty, Logger, MinifiError, + Property, ProvidedInterface, create_provided_interface, +}; #[derive(Debug, ComponentIdentifier)] -pub(crate) struct DuckController { - -} - +pub(crate) struct DuckController {} impl NumberOfLegsControllerApi for DuckController { fn number_of_legs(&self) -> u8 { @@ -24,9 +26,9 @@ impl CanFlyControllerApi for DuckController { impl EnableControllerService for DuckController { fn enable(_context: &Ctx, _logger: &L) -> Result where - Self: Sized + Self: Sized, { - Ok(Self{}) + Ok(Self {}) } } @@ -37,4 +39,4 @@ impl ControllerServiceDefinition for DuckController { create_provided_interface!(dyn CanFlyControllerApi), create_provided_interface!(dyn NumberOfLegsControllerApi), ]; -} \ No newline at end of file +} diff --git a/minifi_rust/extensions/minifi_rs_playground/src/controller_services/dummy_controller_service.rs b/minifi_rust/extensions/minifi_rs_playground/src/controller_services/dummy_controller_service.rs index fb346e7444..d69efb8f98 100644 --- a/minifi_rust/extensions/minifi_rs_playground/src/controller_services/dummy_controller_service.rs +++ b/minifi_rust/extensions/minifi_rs_playground/src/controller_services/dummy_controller_service.rs @@ -1,5 +1,8 @@ use minifi_native::macros::ComponentIdentifier; -use minifi_native::{ControllerServiceDefinition, EnableControllerService, GetProperty, Logger, MinifiError, Property, ProvidedInterface}; +use minifi_native::{ + ControllerServiceDefinition, EnableControllerService, GetProperty, Logger, MinifiError, + Property, ProvidedInterface, +}; #[derive(Debug, ComponentIdentifier)] pub(crate) struct DummyControllerService {} diff --git a/minifi_rust/extensions/minifi_rs_playground/src/controller_services/lorem_ipsum_controller_service.rs b/minifi_rust/extensions/minifi_rs_playground/src/controller_services/lorem_ipsum_controller_service.rs index bafc80c5c3..6b3809917d 100644 --- a/minifi_rust/extensions/minifi_rs_playground/src/controller_services/lorem_ipsum_controller_service.rs +++ b/minifi_rust/extensions/minifi_rs_playground/src/controller_services/lorem_ipsum_controller_service.rs @@ -3,7 +3,10 @@ mod properties; use crate::controller_services::lorem_ipsum_controller_service::properties::LENGTH; use lipsum::lipsum; use minifi_native::macros::ComponentIdentifier; -use minifi_native::{ControllerServiceDefinition, EnableControllerService, GetProperty, Logger, MinifiError, Property, ProvidedInterface}; +use minifi_native::{ + ControllerServiceDefinition, EnableControllerService, GetProperty, Logger, MinifiError, + Property, ProvidedInterface, +}; #[derive(Debug, ComponentIdentifier)] pub(crate) struct LoremIpsumControllerService { diff --git a/minifi_rust/extensions/minifi_rs_playground/src/controller_services/mod.rs b/minifi_rust/extensions/minifi_rs_playground/src/controller_services/mod.rs index 0467a80410..774dd42076 100644 --- a/minifi_rust/extensions/minifi_rs_playground/src/controller_services/mod.rs +++ b/minifi_rust/extensions/minifi_rs_playground/src/controller_services/mod.rs @@ -1,5 +1,5 @@ -pub(crate) mod dummy_controller_service; -pub(crate) mod lorem_ipsum_controller_service; -pub(crate) mod dog_controller_service; pub(crate) mod animal_controller_apis; +pub(crate) mod dog_controller_service; pub(crate) mod duck_controller_service; +pub(crate) mod dummy_controller_service; +pub(crate) mod lorem_ipsum_controller_service; diff --git a/minifi_rust/extensions/minifi_rs_playground/src/processors/asciify_german.rs b/minifi_rust/extensions/minifi_rs_playground/src/processors/asciify_german.rs index e85949a07f..7491fc60dd 100644 --- a/minifi_rust/extensions/minifi_rs_playground/src/processors/asciify_german.rs +++ b/minifi_rust/extensions/minifi_rs_playground/src/processors/asciify_german.rs @@ -1,5 +1,5 @@ use crate::processors::asciify_german::relationships::FAILURE; -use minifi_native::macros::{ComponentIdentifier}; +use minifi_native::macros::ComponentIdentifier; use minifi_native::{ FlowFileStreamTransform, GetProperty, InputStream, Logger, MinifiError, OutputStream, Schedule, TransformStreamResult, diff --git a/minifi_rust/extensions/minifi_rs_playground/src/processors/count_actual_logging.rs b/minifi_rust/extensions/minifi_rs_playground/src/processors/count_actual_logging.rs index f712f37c9a..2aa3b4c4a2 100644 --- a/minifi_rust/extensions/minifi_rs_playground/src/processors/count_actual_logging.rs +++ b/minifi_rust/extensions/minifi_rs_playground/src/processors/count_actual_logging.rs @@ -1,4 +1,4 @@ -use minifi_native::macros::{ComponentIdentifier}; +use minifi_native::macros::ComponentIdentifier; use minifi_native::{ GetProperty, Logger, MinifiError, MutTrigger, OnTriggerResult, OutputAttribute, ProcessContext, ProcessSession, ProcessorDefinition, ProcessorInputRequirement, Property, Relationship, diff --git a/minifi_rust/extensions/minifi_rs_playground/src/processors/duplicate_text.rs b/minifi_rust/extensions/minifi_rs_playground/src/processors/duplicate_text.rs index 38ff066e7b..708371aabf 100644 --- a/minifi_rust/extensions/minifi_rs_playground/src/processors/duplicate_text.rs +++ b/minifi_rust/extensions/minifi_rs_playground/src/processors/duplicate_text.rs @@ -1,4 +1,4 @@ -use minifi_native::macros::{ComponentIdentifier}; +use minifi_native::macros::ComponentIdentifier; use minifi_native::{ GetAttribute, GetControllerService, GetProperty, InputStream, Logger, MinifiError, MutFlowFileStreamTransform, OutputAttribute, OutputStream, ProcessorDefinition, diff --git a/minifi_rust/extensions/minifi_rs_playground/src/processors/generate_flow_file.rs b/minifi_rust/extensions/minifi_rs_playground/src/processors/generate_flow_file.rs index 1e9800ce5f..217c5d03ee 100644 --- a/minifi_rust/extensions/minifi_rs_playground/src/processors/generate_flow_file.rs +++ b/minifi_rust/extensions/minifi_rs_playground/src/processors/generate_flow_file.rs @@ -1,7 +1,7 @@ -use minifi_native::macros::{ComponentIdentifier}; +use minifi_native::macros::ComponentIdentifier; use minifi_native::{ - GetProperty, Logger, MinifiError, OnTriggerResult, ProcessContext, - ProcessSession, Schedule, Trigger, + GetProperty, Logger, MinifiError, OnTriggerResult, ProcessContext, ProcessSession, Schedule, + Trigger, }; use rand::RngExt; use rand::distr::Alphanumeric; diff --git a/minifi_rust/extensions/minifi_rs_playground/src/processors/get_file.rs b/minifi_rust/extensions/minifi_rs_playground/src/processors/get_file.rs index 1b05b030af..1366b71bb6 100644 --- a/minifi_rust/extensions/minifi_rs_playground/src/processors/get_file.rs +++ b/minifi_rust/extensions/minifi_rs_playground/src/processors/get_file.rs @@ -5,10 +5,10 @@ use crate::processors::get_file::properties::{ BATCH_SIZE, DIRECTORY, IGNORE_HIDDEN_FILES, KEEP_SOURCE_FILE, MAX_AGE, MAX_SIZE, MIN_AGE, MIN_SIZE, RECURSE, }; -use minifi_native::macros::{ComponentIdentifier}; +use minifi_native::macros::ComponentIdentifier; use minifi_native::{ - GetProperty, IoState, Logger, MinifiError, OnTriggerResult, ProcessContext, - ProcessSession, Schedule, Trigger, debug, info, trace, warn, + GetProperty, IoState, Logger, MinifiError, OnTriggerResult, ProcessContext, ProcessSession, + Schedule, Trigger, debug, info, trace, warn, }; use std::collections::VecDeque; use std::error; diff --git a/minifi_rust/extensions/minifi_rs_playground/src/processors/kamikaze_processor.rs b/minifi_rust/extensions/minifi_rs_playground/src/processors/kamikaze_processor.rs index bd01eab573..b6ed74b499 100644 --- a/minifi_rust/extensions/minifi_rs_playground/src/processors/kamikaze_processor.rs +++ b/minifi_rust/extensions/minifi_rs_playground/src/processors/kamikaze_processor.rs @@ -3,10 +3,10 @@ mod relationships; use crate::controller_services::lorem_ipsum_controller_service::LoremIpsumControllerService; use crate::processors::kamikaze_processor::properties::NOT_REGISTERED_PROPERTY; -use minifi_native::macros::{ComponentIdentifier}; +use minifi_native::macros::ComponentIdentifier; use minifi_native::{ - GetProperty, Logger, MinifiError, OnTriggerResult, ProcessContext, - ProcessSession, Schedule, Trigger, + GetProperty, Logger, MinifiError, OnTriggerResult, ProcessContext, ProcessSession, Schedule, + Trigger, }; use strum_macros::{Display, EnumString, IntoStaticStr, VariantNames}; @@ -44,14 +44,10 @@ impl Schedule for KamikazeProcessorRs { KamikazeBehaviour::ReturnErr => Err(MinifiError::schedule_err( "it was designed to fail during schedule", )), - KamikazeBehaviour::ReturnOk => Ok(KamikazeProcessorRs { - trigger_behaviour, - }), + KamikazeBehaviour::ReturnOk => Ok(KamikazeProcessorRs { trigger_behaviour }), KamikazeBehaviour::GetNotRegisteredProperty => { let _ = context.get_property(&NOT_REGISTERED_PROPERTY)?; - Ok(KamikazeProcessorRs { - trigger_behaviour, - }) + Ok(KamikazeProcessorRs { trigger_behaviour }) } KamikazeBehaviour::Panic => { panic!("KamikazeProcessor::schedule panic") diff --git a/minifi_rust/extensions/minifi_rs_playground/src/processors/kamikaze_processor/tests.rs b/minifi_rust/extensions/minifi_rs_playground/src/processors/kamikaze_processor/tests.rs index ab959f6e91..9bd44336f8 100644 --- a/minifi_rust/extensions/minifi_rs_playground/src/processors/kamikaze_processor/tests.rs +++ b/minifi_rust/extensions/minifi_rs_playground/src/processors/kamikaze_processor/tests.rs @@ -1,7 +1,5 @@ use super::*; -use crate::processors::kamikaze_processor::properties::{ - SCHEDULE_BEHAVIOUR, TRIGGER_BEHAVIOUR, -}; +use crate::processors::kamikaze_processor::properties::{SCHEDULE_BEHAVIOUR, TRIGGER_BEHAVIOUR}; use minifi_native::MinifiError::{ScheduleError, TriggerError}; use minifi_native::{MockLogger, MockProcessContext, MockProcessSession}; use std::panic::AssertUnwindSafe; @@ -16,10 +14,9 @@ fn on_schedule_ok() { #[test] fn on_schedule_err() { let mut context = MockProcessContext::new(); - context.properties.insert( - SCHEDULE_BEHAVIOUR.name.to_string(), - "ReturnErr".to_string(), - ); + context + .properties + .insert(SCHEDULE_BEHAVIOUR.name.to_string(), "ReturnErr".to_string()); let processor = KamikazeProcessorRs::schedule(&context, &MockLogger::new()); assert!(matches!(processor, Err(ScheduleError(_)))); } @@ -54,10 +51,9 @@ fn on_trigger_ok() { #[test] fn on_trigger_err() { let mut context = MockProcessContext::new(); - context.properties.insert( - TRIGGER_BEHAVIOUR.name.to_string(), - "ReturnErr".to_string(), - ); + context + .properties + .insert(TRIGGER_BEHAVIOUR.name.to_string(), "ReturnErr".to_string()); let processor = KamikazeProcessorRs::schedule(&context, &MockLogger::new()).unwrap(); let mut session = MockProcessSession::new(); diff --git a/minifi_rust/extensions/minifi_rs_playground/src/processors/log_attribute.rs b/minifi_rust/extensions/minifi_rs_playground/src/processors/log_attribute.rs index 3143db9d44..318d8e8f60 100644 --- a/minifi_rust/extensions/minifi_rs_playground/src/processors/log_attribute.rs +++ b/minifi_rust/extensions/minifi_rs_playground/src/processors/log_attribute.rs @@ -1,5 +1,5 @@ use crate::processors::log_attribute::properties::{FLOW_FILES_TO_LOG, LOG_LEVEL, LOG_PAYLOAD}; -use minifi_native::macros::{ComponentIdentifier}; +use minifi_native::macros::ComponentIdentifier; use minifi_native::{ GetProperty, LogLevel, Logger, MinifiError, OnTriggerResult, ProcessContext, ProcessSession, Property, Schedule, Trigger, debug, log, trace, @@ -101,8 +101,7 @@ impl Trigger for LogAttributeRs { } impl Schedule for LogAttributeRs { - fn schedule(context: &P, _logger: &L) -> Result - { + fn schedule(context: &P, _logger: &L) -> Result { let log_level = context .get_property(&LOG_LEVEL)? .expect("required property") diff --git a/minifi_rust/extensions/minifi_rs_playground/src/processors/lorem_ipsum_cs_user.rs b/minifi_rust/extensions/minifi_rs_playground/src/processors/lorem_ipsum_cs_user.rs index 50e8d9c60e..6361669d36 100644 --- a/minifi_rust/extensions/minifi_rs_playground/src/processors/lorem_ipsum_cs_user.rs +++ b/minifi_rust/extensions/minifi_rs_playground/src/processors/lorem_ipsum_cs_user.rs @@ -3,7 +3,7 @@ mod properties; use crate::controller_services::lorem_ipsum_controller_service::LoremIpsumControllerService; use crate::processors::lorem_ipsum_cs_user::properties::CONTROLLER_SERVICE; use crate::processors::lorem_ipsum_cs_user::relationships::SUCCESS; -use minifi_native::macros::{ComponentIdentifier}; +use minifi_native::macros::ComponentIdentifier; use minifi_native::{ Content, FlowFileSource, GeneratedFlowFile, GetControllerService, GetProperty, Logger, MinifiError, Schedule, trace, diff --git a/minifi_rust/extensions/minifi_rs_playground/src/processors/put_file.rs b/minifi_rust/extensions/minifi_rs_playground/src/processors/put_file.rs index 973e790e32..4f682229e4 100644 --- a/minifi_rust/extensions/minifi_rs_playground/src/processors/put_file.rs +++ b/minifi_rust/extensions/minifi_rs_playground/src/processors/put_file.rs @@ -1,5 +1,5 @@ use crate::processors::put_file::relationships::{FAILURE, SUCCESS}; -use minifi_native::macros::{ComponentIdentifier}; +use minifi_native::macros::ComponentIdentifier; use minifi_native::{ FlowFileTransform, GetAttribute, GetControllerService, GetProperty, InputStream, Logger, MinifiError, Schedule, TransformedFlowFile, trace, warn, diff --git a/minifi_rust/extensions/minifi_rs_playground/src/processors/zoo_processor.rs b/minifi_rust/extensions/minifi_rs_playground/src/processors/zoo_processor.rs index 92ce6f89ad..40058c1c5f 100644 --- a/minifi_rust/extensions/minifi_rs_playground/src/processors/zoo_processor.rs +++ b/minifi_rust/extensions/minifi_rs_playground/src/processors/zoo_processor.rs @@ -1,7 +1,13 @@ -use crate::controller_services::animal_controller_apis::{CanFlyControllerApi, NumberOfLegsControllerApi}; -use minifi_native::{critical, info, GetProperty, Logger, MinifiError, OnTriggerResult, OutputAttribute, ProcessContext, ProcessSession, ProcessorDefinition, ProcessorInputRequirement, Property, Relationship, Schedule, StandardPropertyValidator, Trigger}; +use crate::controller_services::animal_controller_apis::{ + CanFlyControllerApi, NumberOfLegsControllerApi, +}; use minifi_native::ControllerServiceApi; use minifi_native::macros::ComponentIdentifier; +use minifi_native::{ + GetProperty, Logger, MinifiError, OnTriggerResult, OutputAttribute, ProcessContext, + ProcessSession, ProcessorDefinition, ProcessorInputRequirement, Property, Relationship, + Schedule, StandardPropertyValidator, Trigger, critical, info, +}; pub(crate) const CAN_FLY_SERVICE: Property = Property { name: "Can fly service", @@ -28,31 +34,46 @@ pub(crate) const NUMBER_OF_LEGS: Property = Property { }; #[derive(Debug, ComponentIdentifier)] -pub(crate) struct ZooProcessor { - -} +pub(crate) struct ZooProcessor {} impl Schedule for ZooProcessor { - fn schedule(_context: &Ctx, _logger: &L) -> Result + fn schedule( + _context: &Ctx, + _logger: &L, + ) -> Result where - Self: Sized + Self: Sized, { - Ok(Self{}) + Ok(Self {}) } } impl Trigger for ZooProcessor { - fn trigger(&self, context: &mut Context, _session: &mut Session, logger: &Lggr) -> Result + fn trigger( + &self, + context: &mut Context, + _session: &mut Session, + logger: &Lggr, + ) -> Result where Context: ProcessContext, - Session: ProcessSession, - Lggr: Logger + Session: ProcessSession, + Lggr: Logger, { info!(logger, "{:?}", self); - if let Some(maybe_flyer) = context.get_controller_service_api::(&CAN_FLY_SERVICE)? { - critical!(logger, "Can {:?} fly? {}", maybe_flyer, maybe_flyer.can_fly()); + if let Some(maybe_flyer) = + context.get_controller_service_api::(&CAN_FLY_SERVICE)? + { + critical!( + logger, + "Can {:?} fly? {}", + maybe_flyer, + maybe_flyer.can_fly() + ); } - if let Some(legged) = context.get_controller_service_api::(&NUMBER_OF_LEGS)? { + if let Some(legged) = + context.get_controller_service_api::(&NUMBER_OF_LEGS)? + { critical!(logger, "{:?} has {} legs", legged, legged.number_of_legs()); } Ok(OnTriggerResult::Ok) @@ -67,4 +88,4 @@ impl ProcessorDefinition for ZooProcessor { const OUTPUT_ATTRIBUTES: &'static [OutputAttribute] = &[]; const RELATIONSHIPS: &'static [Relationship] = &[]; const PROPERTIES: &'static [Property] = &[CAN_FLY_SERVICE, NUMBER_OF_LEGS]; -} \ No newline at end of file +} diff --git a/minifi_rust/minifi_native/src/api.rs b/minifi_rust/minifi_native/src/api.rs index 34d9e2e496..e68e7aebd7 100644 --- a/minifi_rust/minifi_native/src/api.rs +++ b/minifi_rust/minifi_native/src/api.rs @@ -9,10 +9,10 @@ pub(crate) mod process_session; pub(crate) mod processor; pub(crate) mod processor_wrappers; pub(crate) mod property; +pub(crate) mod provided_interface; pub(crate) mod raw_controller_service; pub(crate) mod raw_processor; pub(crate) mod relationship; -pub(crate) mod provided_interface; pub use flow_file::FlowFile; pub use logger::{LogLevel, Logger}; diff --git a/minifi_rust/minifi_native/src/api/component_definition_traits.rs b/minifi_rust/minifi_native/src/api/component_definition_traits.rs index 9afc25a193..947c10518e 100644 --- a/minifi_rust/minifi_native/src/api/component_definition_traits.rs +++ b/minifi_rust/minifi_native/src/api/component_definition_traits.rs @@ -1,5 +1,5 @@ -use crate::{OutputAttribute, ProcessorInputRequirement, Property, Relationship}; use crate::api::provided_interface::ProvidedInterface; +use crate::{OutputAttribute, ProcessorInputRequirement, Property, Relationship}; pub trait ComponentIdentifier { const CLASS_NAME: &'static str; diff --git a/minifi_rust/minifi_native/src/api/process_context.rs b/minifi_rust/minifi_native/src/api/process_context.rs index cc8d184062..1a967ffea3 100644 --- a/minifi_rust/minifi_native/src/api/process_context.rs +++ b/minifi_rust/minifi_native/src/api/process_context.rs @@ -3,7 +3,10 @@ use crate::api::RawControllerService; use crate::api::component_definition_traits::ComponentIdentifier; use crate::api::flow_file::FlowFile; use crate::api::property::GetControllerService; -use crate::{ControllerServiceApi, ControllerServiceDefinition, EnableControllerService, GetProperty, MinifiError, Property}; +use crate::{ + ControllerServiceApi, ControllerServiceDefinition, EnableControllerService, GetProperty, + MinifiError, Property, +}; use std::str::FromStr; use std::time::Duration; @@ -106,7 +109,6 @@ pub trait ProcessContext { property: &Property, ) -> Result>, MinifiError>; - fn report_metrics(&self, metrics: Vec<(String, f64)>) -> Result<(), MinifiError>; } diff --git a/minifi_rust/minifi_native/src/api/processor_wrappers/complex_processor.rs b/minifi_rust/minifi_native/src/api/processor_wrappers/complex_processor.rs index d70ea846ff..a907f77e59 100644 --- a/minifi_rust/minifi_native/src/api/processor_wrappers/complex_processor.rs +++ b/minifi_rust/minifi_native/src/api/processor_wrappers/complex_processor.rs @@ -35,10 +35,7 @@ pub struct ComplexProcessorType {} impl SingleThreadedTrigger for Processor where - Implementation: Schedule - + MutTrigger - + ComponentIdentifier - + ProcessorDefinition, + Implementation: Schedule + MutTrigger + ComponentIdentifier + ProcessorDefinition, L: Logger, { fn trigger( @@ -63,10 +60,7 @@ where impl MultiThreadedTrigger for Processor where - Implementation: Schedule - + Trigger - + ComponentIdentifier - + ProcessorDefinition, + Implementation: Schedule + Trigger + ComponentIdentifier + ProcessorDefinition, L: Logger, { fn trigger( diff --git a/minifi_rust/minifi_native/src/api/processor_wrappers/flow_file_source.rs b/minifi_rust/minifi_native/src/api/processor_wrappers/flow_file_source.rs index 27052a0273..0776ee4782 100644 --- a/minifi_rust/minifi_native/src/api/processor_wrappers/flow_file_source.rs +++ b/minifi_rust/minifi_native/src/api/processor_wrappers/flow_file_source.rs @@ -1,9 +1,8 @@ use crate::api::processor_wrappers::utils::flow_file_content::Content; use crate::api::raw_processor::{MultiThreadedTrigger, SingleThreadedTrigger}; use crate::{ - Concurrent, Exclusive, GetControllerService, GetProperty, Logger, - MinifiError, OnTriggerResult, ProcessContext, ProcessSession, Processor, Relationship, - Schedule, + Concurrent, Exclusive, GetControllerService, GetProperty, Logger, MinifiError, OnTriggerResult, + ProcessContext, ProcessSession, Processor, Relationship, Schedule, }; use std::collections::HashMap; diff --git a/minifi_rust/minifi_native/src/api/processor_wrappers/flow_file_stream_transform.rs b/minifi_rust/minifi_native/src/api/processor_wrappers/flow_file_stream_transform.rs index 0a47a48da1..ef938c7b31 100644 --- a/minifi_rust/minifi_native/src/api/processor_wrappers/flow_file_stream_transform.rs +++ b/minifi_rust/minifi_native/src/api/processor_wrappers/flow_file_stream_transform.rs @@ -2,9 +2,9 @@ use crate::api::process_session::IoState; use crate::api::processor_wrappers::utils::context_session_flowfile_bundle::ContextSessionFlowFileBundle; use crate::api::raw_processor::{MultiThreadedTrigger, SingleThreadedTrigger}; use crate::{ - Concurrent, Exclusive, GetAttribute, GetControllerService, GetProperty, - InputStream, LogLevel, Logger, MinifiError, OnTriggerResult, OutputStream, ProcessContext, - ProcessSession, Processor, Relationship, Schedule, + Concurrent, Exclusive, GetAttribute, GetControllerService, GetProperty, InputStream, LogLevel, + Logger, MinifiError, OnTriggerResult, OutputStream, ProcessContext, ProcessSession, Processor, + Relationship, Schedule, }; use std::collections::HashMap; @@ -119,8 +119,7 @@ where impl<'a, Implementation, L> MultiThreadedTrigger for Processor where - Implementation: - Schedule + FlowFileStreamTransform, + Implementation: Schedule + FlowFileStreamTransform, L: Logger, { fn trigger( @@ -148,8 +147,7 @@ where impl<'a, Implementation, L> SingleThreadedTrigger for Processor where - Implementation: - Schedule + MutFlowFileStreamTransform, + Implementation: Schedule + MutFlowFileStreamTransform, L: Logger, { fn trigger( diff --git a/minifi_rust/minifi_native/src/api/processor_wrappers/flow_file_transform.rs b/minifi_rust/minifi_native/src/api/processor_wrappers/flow_file_transform.rs index 620d57afba..d9a2c0d5ce 100644 --- a/minifi_rust/minifi_native/src/api/processor_wrappers/flow_file_transform.rs +++ b/minifi_rust/minifi_native/src/api/processor_wrappers/flow_file_transform.rs @@ -1,19 +1,19 @@ use crate::api::InputStream; -use crate::api::processor::{Processor}; +use crate::api::processor::Processor; use crate::api::processor_wrappers::utils::context_session_flowfile_bundle::ContextSessionFlowFileBundle; use crate::api::processor_wrappers::utils::flow_file_content::Content; use crate::api::property::{GetControllerService, GetProperty}; use crate::api::raw_processor::{MultiThreadedTrigger, SingleThreadedTrigger}; use crate::{ - Concurrent, Exclusive, GetAttribute, LogLevel, Logger, MinifiError, - OnTriggerResult, ProcessContext, ProcessSession, Relationship, Schedule, info, + Concurrent, Exclusive, GetAttribute, LogLevel, Logger, MinifiError, OnTriggerResult, + ProcessContext, ProcessSession, Relationship, Schedule, info, }; use std::collections::HashMap; #[derive(Debug)] pub struct TransformedFlowFile<'a> { target_relationship_name: &'static str, - new_content: Option>, // If None the content doesn't change + new_content: Option>, // If None the content doesn't change attributes_to_add: HashMap, } diff --git a/minifi_rust/minifi_native/src/api/property.rs b/minifi_rust/minifi_native/src/api/property.rs index e5233571c3..63f8210a58 100644 --- a/minifi_rust/minifi_native/src/api/property.rs +++ b/minifi_rust/minifi_native/src/api/property.rs @@ -1,7 +1,9 @@ use crate::StandardPropertyValidator::{ BoolValidator, DataSizeValidator, TimePeriodValidator, U64Validator, }; -use crate::{ComponentIdentifier, ControllerServiceDefinition, EnableControllerService, MinifiError}; +use crate::{ + ComponentIdentifier, ControllerServiceDefinition, EnableControllerService, MinifiError, +}; use std::str::FromStr; use std::time::Duration; diff --git a/minifi_rust/minifi_native/src/api/provided_interface.rs b/minifi_rust/minifi_native/src/api/provided_interface.rs index 2ce3021aed..bee680e128 100644 --- a/minifi_rust/minifi_native/src/api/provided_interface.rs +++ b/minifi_rust/minifi_native/src/api/provided_interface.rs @@ -6,11 +6,8 @@ pub trait ControllerServiceApi { macro_rules! impl_interface_fqn { ($trait_name:ident) => { impl ControllerServiceApi for dyn $trait_name { - const INTERFACE_NAME: &'static str = concat!( - module_path!(), - "::", - stringify!($trait_name) - ); + const INTERFACE_NAME: &'static str = + concat!(module_path!(), "::", stringify!($trait_name)); } }; } @@ -33,4 +30,4 @@ macro_rules! create_provided_interface { }, } }; -} \ No newline at end of file +} diff --git a/minifi_rust/minifi_native/src/c_ffi/c_ffi_controller_service_context.rs b/minifi_rust/minifi_native/src/c_ffi/c_ffi_controller_service_context.rs index 52ae73cbb8..b39bbfd3e5 100644 --- a/minifi_rust/minifi_native/src/c_ffi/c_ffi_controller_service_context.rs +++ b/minifi_rust/minifi_native/src/c_ffi/c_ffi_controller_service_context.rs @@ -1,8 +1,7 @@ use crate::c_ffi::c_ffi_primitives::StringView; use crate::{GetProperty, MinifiError, Property}; use minifi_native_sys::{ - minifi_controller_service_context, - minifi_controller_service_context_get_property, + minifi_controller_service_context, minifi_controller_service_context_get_property, minifi_status_MINIFI_STATUS_SUCCESS, minifi_string_view, }; use std::borrow::Cow; diff --git a/minifi_rust/minifi_native/src/c_ffi/c_ffi_controller_service_definition.rs b/minifi_rust/minifi_native/src/c_ffi/c_ffi_controller_service_definition.rs index 7d8a1367f0..2fcd7ede5b 100644 --- a/minifi_rust/minifi_native/src/c_ffi/c_ffi_controller_service_definition.rs +++ b/minifi_rust/minifi_native/src/c_ffi/c_ffi_controller_service_definition.rs @@ -2,8 +2,15 @@ use crate::api::RawControllerService; use crate::c_ffi::c_ffi_controller_service_context::CffiControllerServiceContext; use crate::c_ffi::c_ffi_property::CProperties; use crate::c_ffi::{CffiLogger, StaticStrAsMinifiCStr}; -use crate::{ComponentIdentifier, ControllerService, ControllerServiceDefinition, EnableControllerService, LogLevel, Property, ProvidedInterface}; -use minifi_native_sys::{minifi_controller_service_callbacks, minifi_controller_service_class_definition, minifi_controller_service_context, minifi_controller_service_metadata, minifi_status, minifi_string_view}; +use crate::{ + ComponentIdentifier, ControllerService, ControllerServiceDefinition, EnableControllerService, + LogLevel, Property, ProvidedInterface, +}; +use minifi_native_sys::{ + minifi_controller_service_callbacks, minifi_controller_service_class_definition, + minifi_controller_service_context, minifi_controller_service_metadata, minifi_status, + minifi_string_view, +}; use std::ffi::c_void; #[derive(Debug)] @@ -28,7 +35,7 @@ impl<'a> ControllerServiceClassDefinition<'a> { pub struct CffiControllerServiceDefinition where T: RawControllerService + ComponentIdentifier, - P: ControllerServiceDefinition + P: ControllerServiceDefinition, { name: &'static str, description_text: &'static str, @@ -40,12 +47,16 @@ where _phantom_2: std::marker::PhantomData

, } -impl CffiControllerServiceDefinition +impl CffiControllerServiceDefinition where T: RawControllerService + ComponentIdentifier, - P: ControllerServiceDefinition + P: ControllerServiceDefinition, { - pub fn new(description_text: &'static str, properties: &'static [Property], provided_interfaces: &'static [ProvidedInterface

]) -> Self { + pub fn new( + description_text: &'static str, + properties: &'static [Property], + provided_interfaces: &'static [ProvidedInterface

], + ) -> Self { let c_properties = Property::create_c_properties(properties); let mut c_provided_apis = Vec::new(); for provided_interface in provided_interfaces { @@ -57,7 +68,7 @@ where c_properties, c_provided_interfaces: c_provided_apis, _phantom: std::marker::PhantomData, - _phantom_2: std::marker::PhantomData + _phantom_2: std::marker::PhantomData, } } @@ -101,21 +112,19 @@ where } } - extern "C" fn get_interface( + extern "C" fn get_interface< + C: ControllerServiceDefinition + EnableControllerService + ComponentIdentifier, + >( self_ptr: *mut std::ffi::c_void, interface_name: minifi_native_sys::minifi_string_view, ) -> *mut std::ffi::c_void { let name = unsafe { - let slice = std::slice::from_raw_parts( - interface_name.data as *const u8, - interface_name.length - ); + let slice = + std::slice::from_raw_parts(interface_name.data as *const u8, interface_name.length); std::str::from_utf8_unchecked(slice) }; - let wrapper = unsafe { - &*(self_ptr as *const ControllerService) - }; + let wrapper = unsafe { &*(self_ptr as *const ControllerService) }; if let Some(inner_instance) = wrapper.get_implementation() { for iface in C::PROVIDED_APIS { @@ -133,7 +142,8 @@ pub trait DynRawControllerServiceDefinition { fn class_description(&'_ self) -> ControllerServiceClassDefinition<'_>; } -impl DynRawControllerServiceDefinition for CffiControllerServiceDefinition, Impl> +impl DynRawControllerServiceDefinition + for CffiControllerServiceDefinition, Impl> where Impl: EnableControllerService + ComponentIdentifier + ControllerServiceDefinition + 'static, { @@ -149,10 +159,10 @@ where destroy: Some(Self::destroy_controller_service), enable: Some(Self::enable_controller_service), disable: Some(Self::disable_controller_service), - get_interface: Some(Self::get_interface::) + get_interface: Some(Self::get_interface::), }, provided_interfaces_count: self.c_provided_interfaces.len(), - provided_interfaces_ptr: self.c_provided_interfaces.as_ptr() + provided_interfaces_ptr: self.c_provided_interfaces.as_ptr(), }) } } @@ -169,9 +179,12 @@ where { fn get_definition() -> Box { Box::new(CffiControllerServiceDefinition::< - ControllerService, Implementation + ControllerService, + Implementation, >::new( - Implementation::DESCRIPTION, Implementation::PROPERTIES, Implementation::PROVIDED_APIS + Implementation::DESCRIPTION, + Implementation::PROPERTIES, + Implementation::PROVIDED_APIS, )) } } diff --git a/minifi_rust/minifi_native/src/c_ffi/c_ffi_controller_service_list.rs b/minifi_rust/minifi_native/src/c_ffi/c_ffi_controller_service_list.rs index d527490890..4bec07d636 100644 --- a/minifi_rust/minifi_native/src/c_ffi/c_ffi_controller_service_list.rs +++ b/minifi_rust/minifi_native/src/c_ffi/c_ffi_controller_service_list.rs @@ -4,7 +4,8 @@ use minifi_native_sys::minifi_controller_service_class_definition; pub struct CffiControllerServiceList { controller_service_definitions: Vec>, - minifi_controller_service_class_description_list: Vec, + minifi_controller_service_class_description_list: + Vec, } impl CffiControllerServiceList { diff --git a/minifi_rust/minifi_native/src/c_ffi/c_ffi_logger.rs b/minifi_rust/minifi_native/src/c_ffi/c_ffi_logger.rs index 2ffb893c8e..bb1ea0980e 100644 --- a/minifi_rust/minifi_native/src/c_ffi/c_ffi_logger.rs +++ b/minifi_rust/minifi_native/src/c_ffi/c_ffi_logger.rs @@ -1,6 +1,9 @@ use crate::api::LogLevel; use crate::api::Logger; -use minifi_native_sys::{minifi_log_level, minifi_logger, minifi_logger_log_string, minifi_logger_should_log, minifi_string_view}; +use minifi_native_sys::{ + minifi_log_level, minifi_logger, minifi_logger_log_string, minifi_logger_should_log, + minifi_string_view, +}; use std::ffi::CString; use std::fmt; diff --git a/minifi_rust/minifi_native/src/c_ffi/c_ffi_output_attribute.rs b/minifi_rust/minifi_native/src/c_ffi/c_ffi_output_attribute.rs index 3485b156a2..40858bdd04 100644 --- a/minifi_rust/minifi_native/src/c_ffi/c_ffi_output_attribute.rs +++ b/minifi_rust/minifi_native/src/c_ffi/c_ffi_output_attribute.rs @@ -1,6 +1,6 @@ use crate::OutputAttribute; use crate::c_ffi::StaticStrAsMinifiCStr; -use minifi_native_sys::{minifi_string_view, minifi_output_attribute_definition}; +use minifi_native_sys::{minifi_output_attribute_definition, minifi_string_view}; #[allow(dead_code)] // the c_ vecs are holding the values referenced from the output attributes pub struct COutputAttributes { diff --git a/minifi_rust/minifi_native/src/c_ffi/c_ffi_primitives.rs b/minifi_rust/minifi_native/src/c_ffi/c_ffi_primitives.rs index 687fbd4382..78b144cee0 100644 --- a/minifi_rust/minifi_native/src/c_ffi/c_ffi_primitives.rs +++ b/minifi_rust/minifi_native/src/c_ffi/c_ffi_primitives.rs @@ -1,5 +1,9 @@ use crate::ProcessorInputRequirement; -use minifi_native_sys::{minifi_string_view, minifi_input_requirement_MINIFI_INPUT_REQUIRED, minifi_input_requirement_MINIFI_INPUT_ALLOWED, minifi_input_requirement_MINIFI_INPUT_FORBIDDEN, minifi_input_requirement}; +use minifi_native_sys::{ + minifi_input_requirement, minifi_input_requirement_MINIFI_INPUT_ALLOWED, + minifi_input_requirement_MINIFI_INPUT_FORBIDDEN, + minifi_input_requirement_MINIFI_INPUT_REQUIRED, minifi_string_view, +}; use std::os::raw::c_char; #[derive(Debug)] @@ -8,7 +12,7 @@ pub enum FfiConversionError { InvalidUtf8, } -#[repr(transparent)] // This allows us to pass Vec as a pointer to minifi_string_view array +#[repr(transparent)] // This allows us to pass Vec as a pointer to minifi_string_view array #[derive(Debug)] pub(crate) struct StringView<'a> { inner: minifi_string_view, diff --git a/minifi_rust/minifi_native/src/c_ffi/c_ffi_process_context.rs b/minifi_rust/minifi_native/src/c_ffi/c_ffi_process_context.rs index 9fc2104f1a..d95e809d23 100644 --- a/minifi_rust/minifi_native/src/c_ffi/c_ffi_process_context.rs +++ b/minifi_rust/minifi_native/src/c_ffi/c_ffi_process_context.rs @@ -3,7 +3,9 @@ use super::c_ffi_primitives::StringView; use crate::api::ProcessContext; use crate::api::controller_service::ControllerService; use crate::c_ffi::{CffiLogger, StaticStrAsMinifiCStr}; -use crate::{ComponentIdentifier, ControllerServiceApi, EnableControllerService, MinifiError, Property}; +use crate::{ + ComponentIdentifier, ControllerServiceApi, EnableControllerService, MinifiError, Property, +}; use minifi_native_sys::*; use std::borrow::Cow; use std::ffi::c_void; @@ -153,9 +155,10 @@ impl<'a> ProcessContext for CffiProcessContext<'a> { return Err(MinifiError::StatusError(( format!( "Failed to get controller service interface <{}> for property {:?}", - Trait::INTERFACE_NAME, property.name + Trait::INTERFACE_NAME, + property.name ) - .into(), + .into(), NonZeroU32::new_unchecked(get_cs_status), ))); } @@ -172,7 +175,7 @@ impl<'a> ProcessContext for CffiProcessContext<'a> { } } - fn report_metrics(&self, metrics: Vec<(String, f64)>) -> Result<(), MinifiError>{ + fn report_metrics(&self, metrics: Vec<(String, f64)>) -> Result<(), MinifiError> { let mut keys = Vec::new(); let mut values = Vec::new(); for (key, value) in &metrics { @@ -180,9 +183,17 @@ impl<'a> ProcessContext for CffiProcessContext<'a> { values.push(*value); } unsafe { - let err_code = minifi_process_context_report_metrics(self.ptr, keys.len(), keys.as_ptr() as *const minifi_string_view, values.as_ptr()); + let err_code = minifi_process_context_report_metrics( + self.ptr, + keys.len(), + keys.as_ptr() as *const minifi_string_view, + values.as_ptr(), + ); if err_code != minifi_status_MINIFI_STATUS_SUCCESS { - return Err(MinifiError::StatusError(("report_metrics".into(), NonZeroU32::new_unchecked(err_code)))); + return Err(MinifiError::StatusError(( + "report_metrics".into(), + NonZeroU32::new_unchecked(err_code), + ))); } } Ok(()) diff --git a/minifi_rust/minifi_native/src/c_ffi/c_ffi_process_session.rs b/minifi_rust/minifi_native/src/c_ffi/c_ffi_process_session.rs index 6e4364a626..72cd14dccd 100644 --- a/minifi_rust/minifi_native/src/c_ffi/c_ffi_process_session.rs +++ b/minifi_rust/minifi_native/src/c_ffi/c_ffi_process_session.rs @@ -4,7 +4,16 @@ use crate::api::process_session::{IoState, OutputStream}; use crate::api::{InputStream, ProcessSession}; use crate::c_ffi::c_ffi_primitives::{ConvertMinifiStringView, StringView}; use crate::c_ffi::c_ffi_streams::{CffiInputStream, CffiOutputStream}; -use minifi_native_sys::{minifi_status_MINIFI_STATUS_SUCCESS, minifi_string_view, minifi_process_session, minifi_output_stream, minifi_output_stream_write, minifi_io_status_MINIFI_IO_ERROR, minifi_process_session_write, minifi_process_session_create, minifi_process_session_get, minifi_process_session_transfer, minifi_process_session_remove, minifi_process_session_set_flow_file_attribute, minifi_process_session_get_flow_file_attribute, minifi_process_session_get_flow_file_attributes, minifi_io_status_MINIFI_IO_CANCEL, minifi_input_stream, minifi_input_stream_size, minifi_input_stream_read, minifi_process_session_read, minifi_process_session_get_flow_file_id}; +use minifi_native_sys::{ + minifi_input_stream, minifi_input_stream_read, minifi_input_stream_size, + minifi_io_status_MINIFI_IO_CANCEL, minifi_io_status_MINIFI_IO_ERROR, minifi_output_stream, + minifi_output_stream_write, minifi_process_session, minifi_process_session_create, + minifi_process_session_get, minifi_process_session_get_flow_file_attribute, + minifi_process_session_get_flow_file_attributes, minifi_process_session_get_flow_file_id, + minifi_process_session_read, minifi_process_session_remove, + minifi_process_session_set_flow_file_attribute, minifi_process_session_transfer, + minifi_process_session_write, minifi_status_MINIFI_STATUS_SUCCESS, minifi_string_view, +}; use std::ffi::{CString, c_void}; use std::io::Read; use std::num::NonZeroU32; diff --git a/minifi_rust/minifi_native/src/c_ffi/c_ffi_processor_definition.rs b/minifi_rust/minifi_native/src/c_ffi/c_ffi_processor_definition.rs index a9db67e876..6d90eda48c 100644 --- a/minifi_rust/minifi_native/src/c_ffi/c_ffi_processor_definition.rs +++ b/minifi_rust/minifi_native/src/c_ffi/c_ffi_processor_definition.rs @@ -1,7 +1,7 @@ use std::ffi::c_void; use std::ptr; -use super::c_ffi_primitives::{StaticStrAsMinifiCStr}; +use super::c_ffi_primitives::StaticStrAsMinifiCStr; use super::c_ffi_process_context::CffiProcessContext; use super::c_ffi_process_session::CffiProcessSession; use crate::api::raw_processor::{MultiThreadedTrigger, SingleThreadedTrigger}; @@ -10,8 +10,8 @@ use crate::c_ffi::CffiLogger; use crate::c_ffi::c_ffi_output_attribute::COutputAttributes; use crate::c_ffi::c_ffi_property::CProperties; use crate::{ - ComponentIdentifier, Concurrent, Exclusive, - LogLevel, OutputAttribute, Processor, ProcessorDefinition, Property, Schedule, + ComponentIdentifier, Concurrent, Exclusive, LogLevel, OutputAttribute, Processor, + ProcessorDefinition, Property, Schedule, }; use crate::{OnTriggerResult, Relationship}; use minifi_native_sys::*; @@ -242,10 +242,7 @@ impl RawRegisterableProcessor for Processor where Threading: ThreadingModel + 'static, - Implementation: Schedule - + ComponentIdentifier - + ProcessorDefinition - + 'static, + Implementation: Schedule + ComponentIdentifier + ProcessorDefinition + 'static, Processor: RawProcessor + DispatchOnTrigger, { diff --git a/minifi_rust/minifi_native/src/c_ffi/c_ffi_processor_list.rs b/minifi_rust/minifi_native/src/c_ffi/c_ffi_processor_list.rs index 9f5bf153bf..1ac5389163 100644 --- a/minifi_rust/minifi_native/src/c_ffi/c_ffi_processor_list.rs +++ b/minifi_rust/minifi_native/src/c_ffi/c_ffi_processor_list.rs @@ -1,6 +1,6 @@ -use minifi_native_sys::minifi_processor_class_definition; use crate::c_ffi::RawRegisterableProcessor; use crate::c_ffi::c_ffi_processor_definition::DynRawProcessorDefinition; +use minifi_native_sys::minifi_processor_class_definition; pub struct CffiProcessorList { processor_definitions: Vec>, diff --git a/minifi_rust/minifi_native/src/c_ffi/c_ffi_property.rs b/minifi_rust/minifi_native/src/c_ffi/c_ffi_property.rs index bf3d8f5830..85aff0426a 100644 --- a/minifi_rust/minifi_native/src/c_ffi/c_ffi_property.rs +++ b/minifi_rust/minifi_native/src/c_ffi/c_ffi_property.rs @@ -1,6 +1,13 @@ use super::c_ffi_primitives::StaticStrAsMinifiCStr; use crate::{Property, StandardPropertyValidator}; -use minifi_native_sys::{minifi_property_definition, minifi_validator, minifi_string_view, minifi_validator_MINIFI_VALIDATOR_ALWAYS_VALID, minifi_validator_MINIFI_VALIDATOR_BOOLEAN, minifi_validator_MINIFI_VALIDATOR_DATA_SIZE, minifi_validator_MINIFI_VALIDATOR_INTEGER, minifi_validator_MINIFI_VALIDATOR_NON_BLANK, minifi_validator_MINIFI_VALIDATOR_PORT, minifi_validator_MINIFI_VALIDATOR_TIME_PERIOD, minifi_validator_MINIFI_VALIDATOR_UNSIGNED_INTEGER}; +use minifi_native_sys::{ + minifi_property_definition, minifi_string_view, minifi_validator, + minifi_validator_MINIFI_VALIDATOR_ALWAYS_VALID, minifi_validator_MINIFI_VALIDATOR_BOOLEAN, + minifi_validator_MINIFI_VALIDATOR_DATA_SIZE, minifi_validator_MINIFI_VALIDATOR_INTEGER, + minifi_validator_MINIFI_VALIDATOR_NON_BLANK, minifi_validator_MINIFI_VALIDATOR_PORT, + minifi_validator_MINIFI_VALIDATOR_TIME_PERIOD, + minifi_validator_MINIFI_VALIDATOR_UNSIGNED_INTEGER, +}; use std::ptr; #[allow(dead_code)] // these c_ vecs are holding the values referenced from the properties, so they live long enough for registration @@ -94,11 +101,19 @@ impl Property { description: property.description.as_minifi_c_type(), is_required: property.is_required, is_sensitive: property.is_sensitive, - default_value: if def_value.data == std::ptr::null() { std::ptr::null() } else { def_value }, + default_value: if def_value.data == std::ptr::null() { + std::ptr::null() + } else { + def_value + }, allowed_values_count: allowed_values.len(), allowed_values_ptr: allowed_values.as_ptr(), validator: property.validator.as_minifi_c_type(), - allowed_type: if allowed_type.data == std::ptr::null() { std::ptr::null() } else { allowed_type }, + allowed_type: if allowed_type.data == std::ptr::null() { + std::ptr::null() + } else { + allowed_type + }, supports_expression_language: property.supports_expr_lang, } }) diff --git a/minifi_rust/minifi_native/src/c_ffi/c_ffi_streams.rs b/minifi_rust/minifi_native/src/c_ffi/c_ffi_streams.rs index 64fbaf95f1..5684dbc7a9 100644 --- a/minifi_rust/minifi_native/src/c_ffi/c_ffi_streams.rs +++ b/minifi_rust/minifi_native/src/c_ffi/c_ffi_streams.rs @@ -1,5 +1,7 @@ +use minifi_native_sys::{ + minifi_input_stream, minifi_input_stream_read, minifi_output_stream, minifi_output_stream_write, +}; use std::io::{BufRead, Error, ErrorKind, Read}; -use minifi_native_sys::{minifi_input_stream, minifi_input_stream_read, minifi_output_stream, minifi_output_stream_write}; #[derive(Debug)] pub struct CffiInputStream<'a> { diff --git a/minifi_rust/minifi_native/src/lib.rs b/minifi_rust/minifi_native/src/lib.rs index 6f50d9beeb..37efc942a4 100644 --- a/minifi_rust/minifi_native/src/lib.rs +++ b/minifi_rust/minifi_native/src/lib.rs @@ -30,7 +30,7 @@ pub use api::logger::{LogLevel, Logger}; pub use api::property::{GetControllerService, GetProperty, Property}; -pub use api::provided_interface::{ProvidedInterface, ControllerServiceApi}; +pub use api::provided_interface::{ControllerServiceApi, ProvidedInterface}; pub use api::process_session::IoState; diff --git a/minifi_rust/minifi_native/src/mock/mock_process_context.rs b/minifi_rust/minifi_native/src/mock/mock_process_context.rs index b939319d9c..5c4a3ffab6 100644 --- a/minifi_rust/minifi_native/src/mock/mock_process_context.rs +++ b/minifi_rust/minifi_native/src/mock/mock_process_context.rs @@ -1,5 +1,8 @@ use crate::api::{ProcessContext, RawControllerService}; -use crate::{ComponentIdentifier, ControllerServiceApi, EnableControllerService, GetAttribute, MinifiError, MockFlowFile, Property}; +use crate::{ + ComponentIdentifier, ControllerServiceApi, EnableControllerService, GetAttribute, MinifiError, + MockFlowFile, Property, +}; use std::any::Any; use std::borrow::Cow; use std::collections::HashMap; @@ -97,7 +100,10 @@ impl ProcessContext for MockProcessContext { } } - fn get_controller_service_api(&self, _property: &Property) -> Result>, MinifiError> { + fn get_controller_service_api( + &self, + _property: &Property, + ) -> Result>, MinifiError> { todo!() } diff --git a/minifi_rust/minifi_native_macros/src/lib.rs b/minifi_rust/minifi_native_macros/src/lib.rs index f093b82a7a..4b486e84f9 100644 --- a/minifi_rust/minifi_native_macros/src/lib.rs +++ b/minifi_rust/minifi_native_macros/src/lib.rs @@ -1,6 +1,6 @@ use proc_macro::TokenStream; use quote::quote; -use syn::{DeriveInput, parse_macro_input, ItemTrait}; +use syn::{DeriveInput, ItemTrait, parse_macro_input}; #[proc_macro_derive(ComponentIdentifier)] pub fn derive_component_identifier(input: TokenStream) -> TokenStream { From ffc3b4a1b180f07083badeed141cb55ac2f37955 Mon Sep 17 00:00:00 2001 From: Martin Zink Date: Fri, 26 Jun 2026 11:09:47 +0200 Subject: [PATCH 04/26] cargo behave fix disable minifi_rs_playground without ENABLE_TEST_PROCESSORS set CMAKE_LIBRARY_OUTPUT_DIRECTORY for corrosion --- minifi_rust/CMakeLists.txt | 6 ++++++ .../features/controller_apis.feature | 8 ++++---- .../minifi_rs_playground/minifi_rs_playground.md | 8 ++++---- .../extensions/minifi_rs_playground/src/lib.rs | 4 ++-- .../src/processors/zoo_processor.rs | 12 ++++++------ 5 files changed, 22 insertions(+), 16 deletions(-) diff --git a/minifi_rust/CMakeLists.txt b/minifi_rust/CMakeLists.txt index c73f684d89..63b8a407e1 100644 --- a/minifi_rust/CMakeLists.txt +++ b/minifi_rust/CMakeLists.txt @@ -31,8 +31,14 @@ FetchContent_Declare(Corrosion FetchContent_MakeAvailable(Corrosion) +set(CMAKE_LIBRARY_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}/bin") + corrosion_import_crate(MANIFEST_PATH ${CMAKE_CURRENT_SOURCE_DIR}/Cargo.toml) +if (NOT ENABLE_TEST_PROCESSORS) + set_target_properties(cargo-build_minifi_rs_playground PROPERTIES EXCLUDE_FROM_ALL TRUE) +endif() + include(CTest) add_test( diff --git a/minifi_rust/extensions/minifi_rs_playground/features/controller_apis.feature b/minifi_rust/extensions/minifi_rs_playground/features/controller_apis.feature index a6bcf92252..c55b6584fd 100644 --- a/minifi_rust/extensions/minifi_rs_playground/features/controller_apis.feature +++ b/minifi_rust/extensions/minifi_rs_playground/features/controller_apis.feature @@ -1,14 +1,14 @@ @SUPPORTS_WINDOWS Feature: Testing controller service api casting - Scenario: Zoo has a jetpack dog and a normal duck - Given a ZooProcessor processor with the "Can fly service" property set to "Wolfie the magical" - And the "Number of Legs Service" property of the ZooProcessor processor is set to "Wolfie the magical" + Scenario: Zoo has a jetpack dog + Given a ZooProcessorRs processor with the "Can fly service" property set to "Wolfie the magical" + And the "Number of Legs service" property of the ZooProcessorRs processor is set to "Wolfie the magical" And a DogController controller service named "Wolfie the magical" is set up and the "Has Jetpack" property set to "true" And the "Extra information" property of the Wolfie the magical controller service is set to "The dog (Canis familiaris or Canis lupus familiaris) is a domesticated descendant of wolves." When the MiNiFi instance starts up - Then the Minifi logs contain the following message: "[minifi_rs_playground::processors::zoo_processor::ZooProcessor] [critical] Can DogController { has_jetpack: true, extra_info: "The dog (Canis familiaris or Canis lupus familiaris) is a domesticated descendant of wolves." } fly? true" in less than 10 seconds + Then the Minifi logs contain the following message: "[minifi_rs_playground::processors::zoo_processor::ZooProcessorRs] [critical] Can DogController { has_jetpack: true, extra_info: "The dog (Canis familiaris or Canis lupus familiaris) is a domesticated descendant of wolves." } fly? true" in less than 10 seconds And the Minifi logs do not contain errors And the Minifi logs do not contain warnings diff --git a/minifi_rust/extensions/minifi_rs_playground/minifi_rs_playground.md b/minifi_rust/extensions/minifi_rs_playground/minifi_rs_playground.md index 8cc65a7fbe..92dfaa840f 100644 --- a/minifi_rust/extensions/minifi_rs_playground/minifi_rs_playground.md +++ b/minifi_rust/extensions/minifi_rs_playground/minifi_rs_playground.md @@ -26,7 +26,7 @@ limitations under the License. - [LogAttributeRs](#LogAttributeRs) - [LoremIpsumCSUser](#LoremIpsumCSUser) - [PutFileRs](#PutFileRs) -- [ZooProcessor](#ZooProcessor) +- [ZooProcessorRs](#ZooProcessorRs) ### Controller Services - [DogController](#DogController) @@ -255,11 +255,11 @@ In the list below, the names of required properties appear in bold. Any other pr | success | Flowfiles that are successfully written to a file are routed to this relationship | -## ZooProcessor +## ZooProcessorRs ### Description -Test ZooProcessor +Test ZooProcessorRs ### Properties @@ -268,7 +268,7 @@ In the list below, the names of required properties appear in bold. Any other pr | Name | Default Value | Allowable Values | Description | |----------------------------|---------------|------------------|--------------------------| | **Can fly service** | | | Test CanFlyService | -| **Number of Legs Service** | | | Test NumberOfLegsService | +| **Number of Legs service** | | | Test NumberOfLegsService | ### Relationships diff --git a/minifi_rust/extensions/minifi_rs_playground/src/lib.rs b/minifi_rust/extensions/minifi_rs_playground/src/lib.rs index ed55e963f8..d34e99e160 100644 --- a/minifi_rust/extensions/minifi_rs_playground/src/lib.rs +++ b/minifi_rust/extensions/minifi_rs_playground/src/lib.rs @@ -14,7 +14,7 @@ use crate::processors::kamikaze_processor::KamikazeProcessorRs; use crate::processors::log_attribute::LogAttributeRs; use crate::processors::lorem_ipsum_cs_user::LoremIpsumCSUser; use crate::processors::put_file::PutFileRs; -use crate::processors::zoo_processor::ZooProcessor; +use crate::processors::zoo_processor::ZooProcessorRs; use minifi_native::{ ComplexProcessorType, Concurrent, Exclusive, FlowFileSourceProcessorType, @@ -32,7 +32,7 @@ processors: [ (FlowFileTransformProcessorType, Concurrent, PutFileRs), (FlowFileStreamTransformProcessorType, Concurrent, AsciifyGerman), (FlowFileStreamTransformProcessorType, Exclusive, DuplicateStreamText), - (ComplexProcessorType, Concurrent, ZooProcessor), + (ComplexProcessorType, Concurrent, ZooProcessorRs), ], controllers: [ LoremIpsumControllerService, diff --git a/minifi_rust/extensions/minifi_rs_playground/src/processors/zoo_processor.rs b/minifi_rust/extensions/minifi_rs_playground/src/processors/zoo_processor.rs index 40058c1c5f..52295dfe30 100644 --- a/minifi_rust/extensions/minifi_rs_playground/src/processors/zoo_processor.rs +++ b/minifi_rust/extensions/minifi_rs_playground/src/processors/zoo_processor.rs @@ -22,7 +22,7 @@ pub(crate) const CAN_FLY_SERVICE: Property = Property { }; pub(crate) const NUMBER_OF_LEGS: Property = Property { - name: "Number of Legs Service", + name: "Number of Legs service", description: "Test NumberOfLegsService", is_required: true, is_sensitive: false, @@ -34,9 +34,9 @@ pub(crate) const NUMBER_OF_LEGS: Property = Property { }; #[derive(Debug, ComponentIdentifier)] -pub(crate) struct ZooProcessor {} +pub(crate) struct ZooProcessorRs {} -impl Schedule for ZooProcessor { +impl Schedule for ZooProcessorRs { fn schedule( _context: &Ctx, _logger: &L, @@ -48,7 +48,7 @@ impl Schedule for ZooProcessor { } } -impl Trigger for ZooProcessor { +impl Trigger for ZooProcessorRs { fn trigger( &self, context: &mut Context, @@ -80,8 +80,8 @@ impl Trigger for ZooProcessor { } } -impl ProcessorDefinition for ZooProcessor { - const DESCRIPTION: &'static str = "Test ZooProcessor"; +impl ProcessorDefinition for ZooProcessorRs { + const DESCRIPTION: &'static str = "Test ZooProcessorRs"; const INPUT_REQUIREMENT: ProcessorInputRequirement = ProcessorInputRequirement::Forbidden; const SUPPORTS_DYNAMIC_PROPERTIES: bool = false; const SUPPORTS_DYNAMIC_RELATIONSHIPS: bool = false; From 67fdae73a1478b02cc12fa7b56a2b1174e1240cd Mon Sep 17 00:00:00 2001 From: Martin Zink Date: Mon, 29 Jun 2026 16:03:28 +0200 Subject: [PATCH 05/26] small improvements --- .../features/controller_apis.feature | 4 ++-- .../minifi_rs_playground/minifi_rs_playground.md | 12 ++++++------ .../controller_services/dog_controller_service.rs | 12 ++++++------ .../controller_services/duck_controller_service.rs | 12 ++++++------ .../extensions/minifi_rs_playground/src/lib.rs | 8 ++++---- .../processors/generate_flow_file/relationships.rs | 2 +- .../minifi_rs_playground/src/processors/put_file.rs | 11 ++++------- minifi_rust/minifi_native/src/api.rs | 2 +- minifi_rust/minifi_native/src/api/errors.rs | 1 + minifi_rust/minifi_native/src/api/flow_file.rs | 6 ++++++ .../api/processor_wrappers/flow_file_transform.rs | 3 ++- .../utils/context_session_flowfile_bundle.rs | 13 +++++++++++++ minifi_rust/minifi_native/src/lib.rs | 1 + .../minifi_native/src/mock/mock_process_context.rs | 8 +++++++- minifi_rust/minifi_native_sys/build.rs | 2 +- minifi_rust/minifi_rs_behave/linux_build.sh | 6 +++--- 16 files changed, 64 insertions(+), 39 deletions(-) diff --git a/minifi_rust/extensions/minifi_rs_playground/features/controller_apis.feature b/minifi_rust/extensions/minifi_rs_playground/features/controller_apis.feature index c55b6584fd..49411ce8b5 100644 --- a/minifi_rust/extensions/minifi_rs_playground/features/controller_apis.feature +++ b/minifi_rust/extensions/minifi_rs_playground/features/controller_apis.feature @@ -4,11 +4,11 @@ Feature: Testing controller service api casting Scenario: Zoo has a jetpack dog Given a ZooProcessorRs processor with the "Can fly service" property set to "Wolfie the magical" And the "Number of Legs service" property of the ZooProcessorRs processor is set to "Wolfie the magical" - And a DogController controller service named "Wolfie the magical" is set up and the "Has Jetpack" property set to "true" + And a DogControllerRs controller service named "Wolfie the magical" is set up and the "Has Jetpack" property set to "true" And the "Extra information" property of the Wolfie the magical controller service is set to "The dog (Canis familiaris or Canis lupus familiaris) is a domesticated descendant of wolves." When the MiNiFi instance starts up - Then the Minifi logs contain the following message: "[minifi_rs_playground::processors::zoo_processor::ZooProcessorRs] [critical] Can DogController { has_jetpack: true, extra_info: "The dog (Canis familiaris or Canis lupus familiaris) is a domesticated descendant of wolves." } fly? true" in less than 10 seconds + Then the Minifi logs contain the following message: "[minifi_rs_playground::processors::zoo_processor::ZooProcessorRs] [critical] Can DogControllerRs { has_jetpack: true, extra_info: "The dog (Canis familiaris or Canis lupus familiaris) is a domesticated descendant of wolves." } fly? true" in less than 10 seconds And the Minifi logs do not contain errors And the Minifi logs do not contain warnings diff --git a/minifi_rust/extensions/minifi_rs_playground/minifi_rs_playground.md b/minifi_rust/extensions/minifi_rs_playground/minifi_rs_playground.md index 92dfaa840f..f599175a87 100644 --- a/minifi_rust/extensions/minifi_rs_playground/minifi_rs_playground.md +++ b/minifi_rust/extensions/minifi_rs_playground/minifi_rs_playground.md @@ -29,8 +29,8 @@ limitations under the License. - [ZooProcessorRs](#ZooProcessorRs) ### Controller Services -- [DogController](#DogController) -- [DuckController](#DuckController) +- [DogControllerRs](#DogControllerRs) +- [DuckControllerRs](#DuckControllerRs) - [DummyControllerService](#DummyControllerService) - [LoremIpsumControllerService](#LoremIpsumControllerService) @@ -276,11 +276,11 @@ In the list below, the names of required properties appear in bold. Any other pr |------|-------------| -## DogController +## DogControllerRs ### Description -Test DogController +Test DogControllerRs ### Properties @@ -292,11 +292,11 @@ In the list below, the names of required properties appear in bold. Any other pr | **Has Jetpack** | false | true
false | Whether or not the dog has a jetpack | -## DuckController +## DuckControllerRs ### Description -Test DuckController +Test DuckControllerRs ### Properties diff --git a/minifi_rust/extensions/minifi_rs_playground/src/controller_services/dog_controller_service.rs b/minifi_rust/extensions/minifi_rs_playground/src/controller_services/dog_controller_service.rs index 3fc17cafc9..939c789fe4 100644 --- a/minifi_rust/extensions/minifi_rs_playground/src/controller_services/dog_controller_service.rs +++ b/minifi_rust/extensions/minifi_rs_playground/src/controller_services/dog_controller_service.rs @@ -34,24 +34,24 @@ pub(crate) const EXTRA_INFO: Property = Property { #[allow(dead_code)] // extra_info is only used by {:?} #[derive(Debug, ComponentIdentifier)] -pub(crate) struct DogController { +pub(crate) struct DogControllerRs { has_jetpack: bool, extra_info: String, } -impl NumberOfLegsControllerApi for DogController { +impl NumberOfLegsControllerApi for DogControllerRs { fn number_of_legs(&self) -> u8 { 4 } } -impl CanFlyControllerApi for DogController { +impl CanFlyControllerApi for DogControllerRs { fn can_fly(&self) -> bool { self.has_jetpack } } -impl EnableControllerService for DogController { +impl EnableControllerService for DogControllerRs { fn enable(context: &Ctx, _logger: &L) -> Result where Self: Sized, @@ -69,8 +69,8 @@ impl EnableControllerService for DogController { } } -impl ControllerServiceDefinition for DogController { - const DESCRIPTION: &'static str = "Test DogController"; +impl ControllerServiceDefinition for DogControllerRs { + const DESCRIPTION: &'static str = "Test DogControllerRs"; const PROPERTIES: &'static [Property] = &[HAS_JETPACK, EXTRA_INFO]; const PROVIDED_APIS: &'static [ProvidedInterface] = &[ create_provided_interface!(dyn CanFlyControllerApi), diff --git a/minifi_rust/extensions/minifi_rs_playground/src/controller_services/duck_controller_service.rs b/minifi_rust/extensions/minifi_rs_playground/src/controller_services/duck_controller_service.rs index 41132ce69b..74a123ea1e 100644 --- a/minifi_rust/extensions/minifi_rs_playground/src/controller_services/duck_controller_service.rs +++ b/minifi_rust/extensions/minifi_rs_playground/src/controller_services/duck_controller_service.rs @@ -9,21 +9,21 @@ use minifi_native::{ }; #[derive(Debug, ComponentIdentifier)] -pub(crate) struct DuckController {} +pub(crate) struct DuckControllerRs {} -impl NumberOfLegsControllerApi for DuckController { +impl NumberOfLegsControllerApi for DuckControllerRs { fn number_of_legs(&self) -> u8 { 2 } } -impl CanFlyControllerApi for DuckController { +impl CanFlyControllerApi for DuckControllerRs { fn can_fly(&self) -> bool { true } } -impl EnableControllerService for DuckController { +impl EnableControllerService for DuckControllerRs { fn enable(_context: &Ctx, _logger: &L) -> Result where Self: Sized, @@ -32,8 +32,8 @@ impl EnableControllerService for DuckController { } } -impl ControllerServiceDefinition for DuckController { - const DESCRIPTION: &'static str = "Test DuckController"; +impl ControllerServiceDefinition for DuckControllerRs { + const DESCRIPTION: &'static str = "Test DuckControllerRs"; const PROPERTIES: &'static [Property] = &[]; const PROVIDED_APIS: &'static [ProvidedInterface] = &[ create_provided_interface!(dyn CanFlyControllerApi), diff --git a/minifi_rust/extensions/minifi_rs_playground/src/lib.rs b/minifi_rust/extensions/minifi_rs_playground/src/lib.rs index d34e99e160..b73087b499 100644 --- a/minifi_rust/extensions/minifi_rs_playground/src/lib.rs +++ b/minifi_rust/extensions/minifi_rs_playground/src/lib.rs @@ -1,8 +1,8 @@ mod controller_services; mod processors; -use crate::controller_services::dog_controller_service::DogController; -use crate::controller_services::duck_controller_service::DuckController; +use crate::controller_services::dog_controller_service::DogControllerRs; +use crate::controller_services::duck_controller_service::DuckControllerRs; use crate::controller_services::dummy_controller_service::DummyControllerService; use crate::controller_services::lorem_ipsum_controller_service::LoremIpsumControllerService; use crate::processors::asciify_german::AsciifyGerman; @@ -37,7 +37,7 @@ processors: [ controllers: [ LoremIpsumControllerService, DummyControllerService, - DogController, - DuckController + DogControllerRs, + DuckControllerRs ] ); diff --git a/minifi_rust/extensions/minifi_rs_playground/src/processors/generate_flow_file/relationships.rs b/minifi_rust/extensions/minifi_rs_playground/src/processors/generate_flow_file/relationships.rs index 3fb72c7114..e108d52230 100644 --- a/minifi_rust/extensions/minifi_rs_playground/src/processors/generate_flow_file/relationships.rs +++ b/minifi_rust/extensions/minifi_rs_playground/src/processors/generate_flow_file/relationships.rs @@ -2,5 +2,5 @@ use minifi_native::Relationship; pub(crate) const SUCCESS: Relationship = Relationship { name: "success", - description: "success operational on the flow record", // TODO(wtf... but copy paste) + description: "success", }; diff --git a/minifi_rust/extensions/minifi_rs_playground/src/processors/put_file.rs b/minifi_rust/extensions/minifi_rs_playground/src/processors/put_file.rs index 4f682229e4..b745353c23 100644 --- a/minifi_rust/extensions/minifi_rs_playground/src/processors/put_file.rs +++ b/minifi_rust/extensions/minifi_rs_playground/src/processors/put_file.rs @@ -1,9 +1,6 @@ use crate::processors::put_file::relationships::{FAILURE, SUCCESS}; use minifi_native::macros::ComponentIdentifier; -use minifi_native::{ - FlowFileTransform, GetAttribute, GetControllerService, GetProperty, InputStream, Logger, - MinifiError, Schedule, TransformedFlowFile, trace, warn, -}; +use minifi_native::{FlowFileTransform, GetAttribute, GetControllerService, GetProperty, InputStream, Logger, MinifiError, Schedule, TransformedFlowFile, trace, warn, GetId}; use std::path::{Path, PathBuf}; use strum_macros::{Display, EnumString, IntoStaticStr, VariantNames}; use walkdir::WalkDir; @@ -87,7 +84,7 @@ impl PutFileRs { fn get_destination_path(context: &Ctx) -> Result where - Ctx: GetProperty + GetAttribute, + Ctx: GetProperty + GetAttribute + GetId, { let directory = context .get_property(&properties::DIRECTORY)? @@ -95,7 +92,7 @@ impl PutFileRs { let file_name = context .get_attribute("filename")? - .unwrap_or("foo.txt".to_string()); // TODO fallback to UUID + .unwrap_or(context.get_id()?); Ok(PathBuf::from(directory + "/" + file_name.as_str())) } @@ -192,7 +189,7 @@ impl Schedule for PutFileRs { impl FlowFileTransform for PutFileRs { fn transform< 'a, - Context: GetProperty + GetControllerService + GetAttribute, + Context: GetProperty + GetControllerService + GetAttribute + GetId, LoggerImpl: Logger, >( &self, diff --git a/minifi_rust/minifi_native/src/api.rs b/minifi_rust/minifi_native/src/api.rs index e68e7aebd7..494a998b63 100644 --- a/minifi_rust/minifi_native/src/api.rs +++ b/minifi_rust/minifi_native/src/api.rs @@ -14,7 +14,7 @@ pub(crate) mod raw_controller_service; pub(crate) mod raw_processor; pub(crate) mod relationship; -pub use flow_file::FlowFile; +pub use flow_file::{FlowFile, GetId}; pub use logger::{LogLevel, Logger}; pub use process_context::ProcessContext; pub use process_session::{InputStream, OutputStream, ProcessSession}; diff --git a/minifi_rust/minifi_native/src/api/errors.rs b/minifi_rust/minifi_native/src/api/errors.rs index dbe68ea31c..b0baf22d2a 100644 --- a/minifi_rust/minifi_native/src/api/errors.rs +++ b/minifi_rust/minifi_native/src/api/errors.rs @@ -26,6 +26,7 @@ pub enum MinifiError { ScheduleError(Cow<'static, str>), TriggerError(Cow<'static, str>), Parse(ParseError), + MissingFlowFileError, IoError(std::io::Error), } diff --git a/minifi_rust/minifi_native/src/api/flow_file.rs b/minifi_rust/minifi_native/src/api/flow_file.rs index 24c441e1df..81aa2bb734 100644 --- a/minifi_rust/minifi_native/src/api/flow_file.rs +++ b/minifi_rust/minifi_native/src/api/flow_file.rs @@ -1 +1,7 @@ +use crate::MinifiError; + pub trait FlowFile {} + +pub trait GetId { + fn get_id(&self) -> Result; +} diff --git a/minifi_rust/minifi_native/src/api/processor_wrappers/flow_file_transform.rs b/minifi_rust/minifi_native/src/api/processor_wrappers/flow_file_transform.rs index d9a2c0d5ce..deb91869a7 100644 --- a/minifi_rust/minifi_native/src/api/processor_wrappers/flow_file_transform.rs +++ b/minifi_rust/minifi_native/src/api/processor_wrappers/flow_file_transform.rs @@ -9,6 +9,7 @@ use crate::{ ProcessContext, ProcessSession, Relationship, Schedule, info, }; use std::collections::HashMap; +use crate::api::flow_file::GetId; #[derive(Debug)] pub struct TransformedFlowFile<'a> { @@ -54,7 +55,7 @@ impl<'a> TransformedFlowFile<'a> { pub trait FlowFileTransform { fn transform< 'a, - Context: GetProperty + GetControllerService + GetAttribute, + Context: GetProperty + GetControllerService + GetAttribute + GetId, LoggerImpl: Logger, >( &self, diff --git a/minifi_rust/minifi_native/src/api/processor_wrappers/utils/context_session_flowfile_bundle.rs b/minifi_rust/minifi_native/src/api/processor_wrappers/utils/context_session_flowfile_bundle.rs index 8af603c9ff..45753abdbb 100644 --- a/minifi_rust/minifi_native/src/api/processor_wrappers/utils/context_session_flowfile_bundle.rs +++ b/minifi_rust/minifi_native/src/api/processor_wrappers/utils/context_session_flowfile_bundle.rs @@ -4,6 +4,7 @@ use crate::{ ComponentIdentifier, EnableControllerService, MinifiError, ProcessContext, ProcessSession, Property, }; +use crate::api::flow_file::GetId; pub struct ContextSessionFlowFileBundle<'a, PC, PS> where @@ -68,3 +69,15 @@ where } } } + +impl<'a, PC, PS> GetId for ContextSessionFlowFileBundle<'a, PC, PS> +where PC: ProcessContext, + PS: ProcessSession { + fn get_id(&self) -> Result { + if let Some(ff) = self.flow_file { + self.session.get_flow_file_id(ff) + } else { + Err(MinifiError::MissingFlowFileError) + } + } +} diff --git a/minifi_rust/minifi_native/src/lib.rs b/minifi_rust/minifi_native/src/lib.rs index 37efc942a4..dc1b74e2b9 100644 --- a/minifi_rust/minifi_native/src/lib.rs +++ b/minifi_rust/minifi_native/src/lib.rs @@ -37,6 +37,7 @@ pub use api::process_session::IoState; pub use api::attribute::{GetAttribute, OutputAttribute}; pub use api::{ + GetId, FlowFile, InputStream, OnTriggerResult, OutputStream, ProcessContext, ProcessSession, ProcessorInputRequirement, Relationship, StandardPropertyValidator, }; diff --git a/minifi_rust/minifi_native/src/mock/mock_process_context.rs b/minifi_rust/minifi_native/src/mock/mock_process_context.rs index 5c4a3ffab6..0739647bbf 100644 --- a/minifi_rust/minifi_native/src/mock/mock_process_context.rs +++ b/minifi_rust/minifi_native/src/mock/mock_process_context.rs @@ -1,4 +1,4 @@ -use crate::api::{ProcessContext, RawControllerService}; +use crate::api::{GetId, ProcessContext, RawControllerService}; use crate::{ ComponentIdentifier, ControllerServiceApi, EnableControllerService, GetAttribute, MinifiError, MockFlowFile, Property, @@ -127,3 +127,9 @@ impl GetAttribute for MockProcessContext { Ok(self.attributes.get(name).cloned()) } } + +impl GetId for MockProcessContext { + fn get_id(&self) -> Result { + Ok("mock_flow_file_id".into()) + } +} diff --git a/minifi_rust/minifi_native_sys/build.rs b/minifi_rust/minifi_native_sys/build.rs index 9b09df452e..4621ca380a 100644 --- a/minifi_rust/minifi_native_sys/build.rs +++ b/minifi_rust/minifi_native_sys/build.rs @@ -19,7 +19,7 @@ impl SDK { let header_path = canon .to_path_buf() - .join("minifi-api/include/minifi-c/minifi-api.h"); + .join("minifi-api/include/minifi-api.h"); let def_path = canon.to_path_buf().join("minifi-api/minifi-api.def"); let behave_path = canon.to_path_buf().join("behave_framework"); diff --git a/minifi_rust/minifi_rs_behave/linux_build.sh b/minifi_rust/minifi_rs_behave/linux_build.sh index 10b9fb2e77..11ecc221b4 100755 --- a/minifi_rust/minifi_rs_behave/linux_build.sh +++ b/minifi_rust/minifi_rs_behave/linux_build.sh @@ -54,10 +54,10 @@ elif [ -d "$RESOLVED_SDK_PATH" ]; then cp "$RESOLVED_SDK_PATH"/*.whl "$DOCKER_SDK_DIR/" 2>/dev/null || true # Case B: It's a raw repository layout - elif [ -f "$RESOLVED_SDK_PATH/minifi-api/include/minifi-c/minifi-api.h" ]; then - mkdir -p "$DOCKER_SDK_DIR/minifi-api/include/minifi-c" + elif [ -f "$RESOLVED_SDK_PATH/minifi-api/include/minifi-api.h" ]; then + mkdir -p "$DOCKER_SDK_DIR/minifi-api/include" - cp "$RESOLVED_SDK_PATH/minifi-api/include/minifi-c/minifi-api.h" "$DOCKER_SDK_DIR/minifi-api/include/minifi-c/" + cp "$RESOLVED_SDK_PATH/minifi-api/include/minifi-api.h" "$DOCKER_SDK_DIR/minifi-api/include/" cp "$RESOLVED_SDK_PATH/minifi-api/minifi-api.def" "$DOCKER_SDK_DIR/minifi-api/" 2>/dev/null || true if [ -d "$RESOLVED_SDK_PATH/behave_framework" ]; then From 8da953e26cb36a74ca488867ca97f4c64ee4a86a Mon Sep 17 00:00:00 2001 From: Martin Zink Date: Mon, 29 Jun 2026 16:03:36 +0200 Subject: [PATCH 06/26] cargo fmt --- .../minifi_rs_playground/src/processors/put_file.rs | 5 ++++- .../src/api/processor_wrappers/flow_file_transform.rs | 2 +- .../utils/context_session_flowfile_bundle.rs | 8 +++++--- minifi_rust/minifi_native/src/lib.rs | 3 +-- minifi_rust/minifi_native_sys/build.rs | 4 +--- 5 files changed, 12 insertions(+), 10 deletions(-) diff --git a/minifi_rust/extensions/minifi_rs_playground/src/processors/put_file.rs b/minifi_rust/extensions/minifi_rs_playground/src/processors/put_file.rs index b745353c23..b58267ab97 100644 --- a/minifi_rust/extensions/minifi_rs_playground/src/processors/put_file.rs +++ b/minifi_rust/extensions/minifi_rs_playground/src/processors/put_file.rs @@ -1,6 +1,9 @@ use crate::processors::put_file::relationships::{FAILURE, SUCCESS}; use minifi_native::macros::ComponentIdentifier; -use minifi_native::{FlowFileTransform, GetAttribute, GetControllerService, GetProperty, InputStream, Logger, MinifiError, Schedule, TransformedFlowFile, trace, warn, GetId}; +use minifi_native::{ + FlowFileTransform, GetAttribute, GetControllerService, GetId, GetProperty, InputStream, Logger, + MinifiError, Schedule, TransformedFlowFile, trace, warn, +}; use std::path::{Path, PathBuf}; use strum_macros::{Display, EnumString, IntoStaticStr, VariantNames}; use walkdir::WalkDir; diff --git a/minifi_rust/minifi_native/src/api/processor_wrappers/flow_file_transform.rs b/minifi_rust/minifi_native/src/api/processor_wrappers/flow_file_transform.rs index deb91869a7..7dc8c9ebca 100644 --- a/minifi_rust/minifi_native/src/api/processor_wrappers/flow_file_transform.rs +++ b/minifi_rust/minifi_native/src/api/processor_wrappers/flow_file_transform.rs @@ -1,4 +1,5 @@ use crate::api::InputStream; +use crate::api::flow_file::GetId; use crate::api::processor::Processor; use crate::api::processor_wrappers::utils::context_session_flowfile_bundle::ContextSessionFlowFileBundle; use crate::api::processor_wrappers::utils::flow_file_content::Content; @@ -9,7 +10,6 @@ use crate::{ ProcessContext, ProcessSession, Relationship, Schedule, info, }; use std::collections::HashMap; -use crate::api::flow_file::GetId; #[derive(Debug)] pub struct TransformedFlowFile<'a> { diff --git a/minifi_rust/minifi_native/src/api/processor_wrappers/utils/context_session_flowfile_bundle.rs b/minifi_rust/minifi_native/src/api/processor_wrappers/utils/context_session_flowfile_bundle.rs index 45753abdbb..72bc5933b6 100644 --- a/minifi_rust/minifi_native/src/api/processor_wrappers/utils/context_session_flowfile_bundle.rs +++ b/minifi_rust/minifi_native/src/api/processor_wrappers/utils/context_session_flowfile_bundle.rs @@ -1,10 +1,10 @@ use crate::api::attribute::GetAttribute; +use crate::api::flow_file::GetId; use crate::api::property::{GetControllerService, GetProperty}; use crate::{ ComponentIdentifier, EnableControllerService, MinifiError, ProcessContext, ProcessSession, Property, }; -use crate::api::flow_file::GetId; pub struct ContextSessionFlowFileBundle<'a, PC, PS> where @@ -71,8 +71,10 @@ where } impl<'a, PC, PS> GetId for ContextSessionFlowFileBundle<'a, PC, PS> -where PC: ProcessContext, - PS: ProcessSession { +where + PC: ProcessContext, + PS: ProcessSession, +{ fn get_id(&self) -> Result { if let Some(ff) = self.flow_file { self.session.get_flow_file_id(ff) diff --git a/minifi_rust/minifi_native/src/lib.rs b/minifi_rust/minifi_native/src/lib.rs index dc1b74e2b9..a78834cb9f 100644 --- a/minifi_rust/minifi_native/src/lib.rs +++ b/minifi_rust/minifi_native/src/lib.rs @@ -37,8 +37,7 @@ pub use api::process_session::IoState; pub use api::attribute::{GetAttribute, OutputAttribute}; pub use api::{ - GetId, - FlowFile, InputStream, OnTriggerResult, OutputStream, ProcessContext, ProcessSession, + FlowFile, GetId, InputStream, OnTriggerResult, OutputStream, ProcessContext, ProcessSession, ProcessorInputRequirement, Relationship, StandardPropertyValidator, }; diff --git a/minifi_rust/minifi_native_sys/build.rs b/minifi_rust/minifi_native_sys/build.rs index 4621ca380a..5ed0728aa7 100644 --- a/minifi_rust/minifi_native_sys/build.rs +++ b/minifi_rust/minifi_native_sys/build.rs @@ -17,9 +17,7 @@ impl SDK { repo_root, canon ); - let header_path = canon - .to_path_buf() - .join("minifi-api/include/minifi-api.h"); + let header_path = canon.to_path_buf().join("minifi-api/include/minifi-api.h"); let def_path = canon.to_path_buf().join("minifi-api/minifi-api.def"); let behave_path = canon.to_path_buf().join("behave_framework"); From 4024b262b732f52fb7199e3a04fb83b213bdc4a6 Mon Sep 17 00:00:00 2001 From: Martin Zink Date: Wed, 1 Jul 2026 10:21:43 +0200 Subject: [PATCH 07/26] update docs --- .../extensions/minifi_rs_playground/minifi_rs_playground.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/minifi_rust/extensions/minifi_rs_playground/minifi_rs_playground.md b/minifi_rust/extensions/minifi_rs_playground/minifi_rs_playground.md index f599175a87..c7560edc9b 100644 --- a/minifi_rust/extensions/minifi_rs_playground/minifi_rs_playground.md +++ b/minifi_rust/extensions/minifi_rs_playground/minifi_rs_playground.md @@ -115,9 +115,9 @@ In the list below, the names of required properties appear in bold. Any other pr ### Relationships -| Name | Description | -|---------|----------------------------------------| -| success | success operational on the flow record | +| Name | Description | +|---------|-------------| +| success | success | ## GetFileRs From 9e306aa22b367cc4caf5737f17d6aecc716effeb Mon Sep 17 00:00:00 2001 From: Martin Zink Date: Wed, 1 Jul 2026 11:43:15 +0200 Subject: [PATCH 08/26] fix clippy --- .github/workflows/ci.yml | 8 ++++++- .../src/processors/duplicate_text.rs | 4 ++-- .../src/processors/generate_flow_file.rs | 6 +++--- .../kamikaze_processor/properties.rs | 4 ++-- .../src/processors/log_attribute.rs | 21 ++++++++----------- .../processors/log_attribute/properties.rs | 2 +- .../src/processors/put_file.rs | 13 ++++++------ .../src/processors/put_file/properties.rs | 2 +- .../processor_wrappers/flow_file_source.rs | 8 +++---- .../flow_file_stream_transform.rs | 4 ++-- .../processor_wrappers/flow_file_transform.rs | 2 +- .../c_ffi/c_ffi_controller_service_list.rs | 10 +++++++++ .../src/c_ffi/c_ffi_process_session.rs | 4 ++-- .../src/c_ffi/c_ffi_processor_definition.rs | 4 ++++ .../src/c_ffi/c_ffi_processor_list.rs | 10 +++++++++ .../minifi_native/src/c_ffi/c_ffi_property.rs | 4 ++-- .../minifi_native/src/c_ffi/c_ffi_streams.rs | 6 +++--- minifi_rust/minifi_native/src/lib.rs | 2 +- .../mock/mock_controller_service_context.rs | 6 ++++++ .../minifi_native/src/mock/mock_flow_file.rs | 6 ++++++ .../minifi_native/src/mock/mock_logger.rs | 6 ++++++ .../src/mock/mock_process_context.rs | 6 ++++++ .../src/mock/mock_process_session.rs | 6 ++++++ minifi_rust/minifi_native_sys/build.rs | 8 +++---- minifi_rust/minifi_rs_behave/src/main.rs | 8 +++---- 25 files changed, 108 insertions(+), 52 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e6e2cf729a..e3db0a3eb6 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -634,8 +634,14 @@ jobs: run: cargo fmt --check working-directory: minifi_rust + - id: clippy_check + name: Clippy check + continue-on-error: true + run: cargo clippy + working-directory: minifi_rust + - name: Check Linter Statuses - if: steps.cpp_lint.outcome == 'failure' || steps.shellcheck.outcome == 'failure' || steps.flake8_check.outcome == 'failure' || steps.cargo_check.outcome == 'failure' + if: steps.cpp_lint.outcome == 'failure' || steps.shellcheck.outcome == 'failure' || steps.flake8_check.outcome == 'failure' || steps.cargo_check.outcome == 'failure' || steps.clippy_check.outcome == 'failure' run: | echo "One or more linters failed. Failing the workflow." exit 1 diff --git a/minifi_rust/extensions/minifi_rs_playground/src/processors/duplicate_text.rs b/minifi_rust/extensions/minifi_rs_playground/src/processors/duplicate_text.rs index 708371aabf..9f81bc2990 100644 --- a/minifi_rust/extensions/minifi_rs_playground/src/processors/duplicate_text.rs +++ b/minifi_rust/extensions/minifi_rs_playground/src/processors/duplicate_text.rs @@ -36,8 +36,8 @@ impl MutFlowFileStreamTransform for DuplicateStreamText { ) -> Result { let mut byte = [0u8; 1]; while input_stream.read(&mut byte)? > 0 { - output_stream.write(&byte)?; - output_stream.write(&byte)?; + let _ = output_stream.write(&byte)?; + let _ = output_stream.write(&byte)?; } Ok(TransformStreamResult::new(&SUCCESS, HashMap::new())) } diff --git a/minifi_rust/extensions/minifi_rs_playground/src/processors/generate_flow_file.rs b/minifi_rust/extensions/minifi_rs_playground/src/processors/generate_flow_file.rs index 217c5d03ee..fa894ef599 100644 --- a/minifi_rust/extensions/minifi_rs_playground/src/processors/generate_flow_file.rs +++ b/minifi_rust/extensions/minifi_rs_playground/src/processors/generate_flow_file.rs @@ -146,14 +146,14 @@ impl Trigger for GenerateFlowFileRs { } for _ in 0..self.batch_size { - let mut ff = session.create()?; + let ff = session.create()?; if self.mode != Mode::Empty { if self.is_unique() { let mut unique_data: Vec = vec![0; self.file_size as usize]; Self::generate_data(&mut unique_data, self.is_text()); - session.write(&mut ff, unique_data.as_slice())?; + session.write(&ff, unique_data.as_slice())?; } else { - session.write(&mut ff, non_unique_data_buffer)?; + session.write(&ff, non_unique_data_buffer)?; } } session.transfer(ff, relationships::SUCCESS.name)?; diff --git a/minifi_rust/extensions/minifi_rs_playground/src/processors/kamikaze_processor/properties.rs b/minifi_rust/extensions/minifi_rs_playground/src/processors/kamikaze_processor/properties.rs index ba81c6b025..02fab5b106 100644 --- a/minifi_rust/extensions/minifi_rs_playground/src/processors/kamikaze_processor/properties.rs +++ b/minifi_rust/extensions/minifi_rs_playground/src/processors/kamikaze_processor/properties.rs @@ -10,7 +10,7 @@ pub(crate) const SCHEDULE_BEHAVIOUR: Property = Property { supports_expr_lang: false, default_value: Some(KamikazeBehaviour::ReturnOk.into_str()), validator: StandardPropertyValidator::AlwaysValidValidator, - allowed_values: &KamikazeBehaviour::VARIANTS, + allowed_values: KamikazeBehaviour::VARIANTS, allowed_type: None, }; @@ -22,7 +22,7 @@ pub(crate) const TRIGGER_BEHAVIOUR: Property = Property { supports_expr_lang: false, default_value: Some(KamikazeBehaviour::ReturnOk.into_str()), validator: StandardPropertyValidator::AlwaysValidValidator, - allowed_values: &KamikazeBehaviour::VARIANTS, + allowed_values: KamikazeBehaviour::VARIANTS, allowed_type: None, }; diff --git a/minifi_rust/extensions/minifi_rs_playground/src/processors/log_attribute.rs b/minifi_rust/extensions/minifi_rs_playground/src/processors/log_attribute.rs index 318d8e8f60..56f88f9a72 100644 --- a/minifi_rust/extensions/minifi_rs_playground/src/processors/log_attribute.rs +++ b/minifi_rust/extensions/minifi_rs_playground/src/processors/log_attribute.rs @@ -30,16 +30,14 @@ impl LogAttributeRs { log_msg.push_str("\nFlowFile Attributes Map Content"); session.on_attributes(flow_file, |key, value| { - if let Some(attributes_to_ignore) = &self.attributes_to_ignore { - if attributes_to_ignore.iter().any(|ign| ign == key) { + if let Some(attributes_to_ignore) = &self.attributes_to_ignore + && attributes_to_ignore.iter().any(|ign| ign == key) { return; } - } - if let Some(attributes_to_log) = &self.attributes_to_log { - if !attributes_to_log.iter().any(|ign| ign == key) { + if let Some(attributes_to_log) = &self.attributes_to_log + && !attributes_to_log.iter().any(|ign| ign == key) { return; } - } log_msg.push_str(format!("\nkey:{} value:{}", &key, &value).as_str()); }); if self.log_payload { @@ -50,13 +48,13 @@ impl LogAttributeRs { } else { log_msg.push_str( String::from_utf8(flow_file_payload) - .unwrap_or(String::new()) + .unwrap_or_default() .as_str(), ); } } } - log_msg.push_str("\n"); + log_msg.push('\n'); log_msg.push_str(self.dash_line.as_str()); log_msg } @@ -85,8 +83,8 @@ impl Trigger for LogAttributeRs { }; let mut flow_files_processed = 0usize; for _ in 0..max_flow_files_to_process { - if let Some(mut flow_file) = session.get() { - let log_msg = self.generate_log_message(session, &mut flow_file); + if let Some(flow_file) = session.get() { + let log_msg = self.generate_log_message(session, &flow_file); log!(logger, self.log_level, "{}", log_msg); session.transfer(flow_file, relationships::SUCCESS.name)?; flow_files_processed += 1; @@ -121,8 +119,7 @@ impl Schedule for LogAttributeRs { property: &Property, ) -> Result>, MinifiError> { Ok(context - .get_property(property)? - .and_then(|s| Some(s.split(",").map(|s| s.to_string()).collect::>()))) + .get_property(property)?.map(|s| s.split(",").map(|s| s.to_string()).collect::>())) } let attributes_to_log = get_csv_property(context, &properties::ATTRIBUTES_TO_LOG)?; diff --git a/minifi_rust/extensions/minifi_rs_playground/src/processors/log_attribute/properties.rs b/minifi_rust/extensions/minifi_rs_playground/src/processors/log_attribute/properties.rs index b104224e00..ea0de2b32d 100644 --- a/minifi_rust/extensions/minifi_rs_playground/src/processors/log_attribute/properties.rs +++ b/minifi_rust/extensions/minifi_rs_playground/src/processors/log_attribute/properties.rs @@ -9,7 +9,7 @@ pub(crate) const LOG_LEVEL: Property = Property { supports_expr_lang: false, default_value: Some("Info"), // todo! it would be nicer to come from enum value but wasnt able to use into_const_str from another crate validator: StandardPropertyValidator::AlwaysValidValidator, - allowed_values: &LogLevel::VARIANTS, + allowed_values: LogLevel::VARIANTS, allowed_type: None, }; diff --git a/minifi_rust/extensions/minifi_rs_playground/src/processors/put_file.rs b/minifi_rust/extensions/minifi_rs_playground/src/processors/put_file.rs index b58267ab97..61dfc057f8 100644 --- a/minifi_rust/extensions/minifi_rs_playground/src/processors/put_file.rs +++ b/minifi_rust/extensions/minifi_rs_playground/src/processors/put_file.rs @@ -31,14 +31,14 @@ struct PutFileUnixPermissions { #[cfg(unix)] impl PutFileUnixPermissions { fn set_directory_permissions(&self, path: &Path) -> std::io::Result<()> { - if let Some(permissions) = self.directory_permissions.as_ref().map(|p| p.clone()) { + if let Some(permissions) = self.directory_permissions.clone() { return std::fs::set_permissions(path, permissions); } Ok(()) } fn set_file_permissions(&self, file: &Path) -> std::io::Result<()> { - if let Some(permissions) = self.file_permissions.as_ref().map(|p| p.clone()) { + if let Some(permissions) = self.file_permissions.clone() { return std::fs::set_permissions(file, permissions); } Ok(()) @@ -100,12 +100,11 @@ impl PutFileRs { } fn prepare_destination(&self, destination: &Path) -> std::io::Result<()> { - if let Some(parent) = destination.parent() { - if self.try_make_dirs { + if let Some(parent) = destination.parent() + && self.try_make_dirs { std::fs::create_dir_all(parent)?; self.unix_permissions.set_directory_permissions(parent)?; } - } Ok(()) } @@ -143,10 +142,10 @@ impl PutFileRs { let parse_permission = |property: &minifi_native::Property| -> Result, MinifiError> { Ok(context - .get_property(&property)? + .get_property(property)? .map(|perm_str| u32::from_str_radix(&perm_str, 8)) .transpose()? - .map(|perm| std::fs::Permissions::from_mode(perm))) + .map(std::fs::Permissions::from_mode)) }; let file_permissions = parse_permission(&unix_only_properties::PERMISSIONS)?; let directory_permissions = parse_permission(&unix_only_properties::DIRECTORY_PERMISSIONS)?; diff --git a/minifi_rust/extensions/minifi_rs_playground/src/processors/put_file/properties.rs b/minifi_rust/extensions/minifi_rs_playground/src/processors/put_file/properties.rs index a6bd205c2d..bea299be62 100644 --- a/minifi_rust/extensions/minifi_rs_playground/src/processors/put_file/properties.rs +++ b/minifi_rust/extensions/minifi_rs_playground/src/processors/put_file/properties.rs @@ -22,7 +22,7 @@ pub(crate) const CONFLICT_RESOLUTION: Property = Property { supports_expr_lang: false, default_value: Some(ConflictResolutionStrategy::Fail.into_str()), validator: StandardPropertyValidator::AlwaysValidValidator, - allowed_values: &ConflictResolutionStrategy::VARIANTS, + allowed_values: ConflictResolutionStrategy::VARIANTS, allowed_type: None, }; diff --git a/minifi_rust/minifi_native/src/api/processor_wrappers/flow_file_source.rs b/minifi_rust/minifi_native/src/api/processor_wrappers/flow_file_source.rs index 0776ee4782..35ff73bc4e 100644 --- a/minifi_rust/minifi_native/src/api/processor_wrappers/flow_file_source.rs +++ b/minifi_rust/minifi_native/src/api/processor_wrappers/flow_file_source.rs @@ -62,8 +62,8 @@ where let mut ff = session.create()?; match new_flow_file_data.new_content { None => {} - Some(Content::Buffer(buffer)) => session.write(&mut ff, &buffer)?, - Some(Content::Stream(stream)) => session.write_from_stream(&mut ff, stream)?, + Some(Content::Buffer(buffer)) => session.write(&ff, &buffer)?, + Some(Content::Stream(stream)) => session.write_from_stream(&ff, stream)?, } for (k, v) in &new_flow_file_data.attributes_to_add { session.set_attribute(&mut ff, k, v)?; @@ -75,7 +75,7 @@ where pub struct FlowFileSourceProcessorType {} -impl<'a, Implementation, L> MultiThreadedTrigger +impl MultiThreadedTrigger for Processor where Implementation: Schedule + FlowFileSource, @@ -101,7 +101,7 @@ where } } -impl<'a, Implementation, L> SingleThreadedTrigger +impl SingleThreadedTrigger for Processor where Implementation: Schedule + MutFlowFileSource, diff --git a/minifi_rust/minifi_native/src/api/processor_wrappers/flow_file_stream_transform.rs b/minifi_rust/minifi_native/src/api/processor_wrappers/flow_file_stream_transform.rs index ef938c7b31..68a05ebdf3 100644 --- a/minifi_rust/minifi_native/src/api/processor_wrappers/flow_file_stream_transform.rs +++ b/minifi_rust/minifi_native/src/api/processor_wrappers/flow_file_stream_transform.rs @@ -116,7 +116,7 @@ where } // Concurrent Implementation (Multi-Threaded) -impl<'a, Implementation, L> MultiThreadedTrigger +impl MultiThreadedTrigger for Processor where Implementation: Schedule + FlowFileStreamTransform, @@ -144,7 +144,7 @@ where } // Exclusive Implementation (Single-Threaded) -impl<'a, Implementation, L> SingleThreadedTrigger +impl SingleThreadedTrigger for Processor where Implementation: Schedule + MutFlowFileStreamTransform, diff --git a/minifi_rust/minifi_native/src/api/processor_wrappers/flow_file_transform.rs b/minifi_rust/minifi_native/src/api/processor_wrappers/flow_file_transform.rs index 7dc8c9ebca..a594ef7368 100644 --- a/minifi_rust/minifi_native/src/api/processor_wrappers/flow_file_transform.rs +++ b/minifi_rust/minifi_native/src/api/processor_wrappers/flow_file_transform.rs @@ -34,7 +34,7 @@ impl<'a> TransformedFlowFile<'a> { ) -> Self { Self { target_relationship_name: target_relationship.name, - new_content: new_content.map(|b| Content::Buffer(b)), + new_content: new_content.map(Content::Buffer), attributes_to_add, } } diff --git a/minifi_rust/minifi_native/src/c_ffi/c_ffi_controller_service_list.rs b/minifi_rust/minifi_native/src/c_ffi/c_ffi_controller_service_list.rs index 4bec07d636..ced9461200 100644 --- a/minifi_rust/minifi_native/src/c_ffi/c_ffi_controller_service_list.rs +++ b/minifi_rust/minifi_native/src/c_ffi/c_ffi_controller_service_list.rs @@ -8,6 +8,12 @@ pub struct CffiControllerServiceList { Vec, } +impl Default for CffiControllerServiceList { + fn default() -> Self { + Self::new() + } +} + impl CffiControllerServiceList { pub fn new() -> Self { Self { @@ -45,6 +51,10 @@ impl CffiControllerServiceList { self.minifi_controller_service_class_description_list.len() } + /// # Safety + /// + /// The returned *minifi_controller_service_class_definition only valid until self lives + /// TODO(mzink) maybe some lifetimes? pub unsafe fn get_controller_service_ptr( &self, ) -> *const minifi_controller_service_class_definition { diff --git a/minifi_rust/minifi_native/src/c_ffi/c_ffi_process_session.rs b/minifi_rust/minifi_native/src/c_ffi/c_ffi_process_session.rs index 72cd14dccd..7efc353e44 100644 --- a/minifi_rust/minifi_native/src/c_ffi/c_ffi_process_session.rs +++ b/minifi_rust/minifi_native/src/c_ffi/c_ffi_process_session.rs @@ -225,7 +225,7 @@ impl<'a> ProcessSession for CffiProcessSession<'a> { process_attr, }; - unsafe extern "C" fn get_attributes_callback<'b, F: FnMut(&str, &str)>( + unsafe extern "C" fn get_attributes_callback( user_ctx: *mut c_void, minifi_attr_key: minifi_string_view, minifi_attr_val: minifi_string_view, @@ -508,7 +508,7 @@ impl<'a> ProcessSession for CffiProcessSession<'a> { buffer.set_len(bytes_read as usize); - match (batch_helper.process_batch)(&*buffer) { + match (batch_helper.process_batch)(&buffer) { Ok(_) => {} Err(err) => { eprintln!("Error during read_in_batch {:?}", err); diff --git a/minifi_rust/minifi_native/src/c_ffi/c_ffi_processor_definition.rs b/minifi_rust/minifi_native/src/c_ffi/c_ffi_processor_definition.rs index 6d90eda48c..9393944abd 100644 --- a/minifi_rust/minifi_native/src/c_ffi/c_ffi_processor_definition.rs +++ b/minifi_rust/minifi_native/src/c_ffi/c_ffi_processor_definition.rs @@ -17,6 +17,9 @@ use crate::{OnTriggerResult, Relationship}; use minifi_native_sys::*; pub trait DispatchOnTrigger { + /// # Safety + /// + /// This function calls C functions unsafe fn dispatch_on_trigger( processor: *mut c_void, context: *mut minifi_process_context, @@ -95,6 +98,7 @@ impl RawProcessorDefinition where T: RawProcessor + DispatchOnTrigger, { + #[allow(clippy::too_many_arguments)] pub fn new( name: &'static str, description_text: &'static str, diff --git a/minifi_rust/minifi_native/src/c_ffi/c_ffi_processor_list.rs b/minifi_rust/minifi_native/src/c_ffi/c_ffi_processor_list.rs index 1ac5389163..fddd9c8189 100644 --- a/minifi_rust/minifi_native/src/c_ffi/c_ffi_processor_list.rs +++ b/minifi_rust/minifi_native/src/c_ffi/c_ffi_processor_list.rs @@ -7,6 +7,12 @@ pub struct CffiProcessorList { minifi_processor_class_description_list: Vec, } +impl Default for CffiProcessorList { + fn default() -> Self { + Self::new() + } +} + impl CffiProcessorList { pub fn new() -> Self { Self { @@ -43,6 +49,10 @@ impl CffiProcessorList { self.minifi_processor_class_description_list.len() } + /// # Safety + /// + /// The returned *minifi_processor_class_definition only valid until self lives + /// TODO(mzink) maybe some lifetimes? pub unsafe fn get_processor_ptr(&self) -> *const minifi_processor_class_definition { self.minifi_processor_class_description_list.as_ptr() } diff --git a/minifi_rust/minifi_native/src/c_ffi/c_ffi_property.rs b/minifi_rust/minifi_native/src/c_ffi/c_ffi_property.rs index 85aff0426a..bfb7d61395 100644 --- a/minifi_rust/minifi_native/src/c_ffi/c_ffi_property.rs +++ b/minifi_rust/minifi_native/src/c_ffi/c_ffi_property.rs @@ -101,7 +101,7 @@ impl Property { description: property.description.as_minifi_c_type(), is_required: property.is_required, is_sensitive: property.is_sensitive, - default_value: if def_value.data == std::ptr::null() { + default_value: if def_value.data.is_null() { std::ptr::null() } else { def_value @@ -109,7 +109,7 @@ impl Property { allowed_values_count: allowed_values.len(), allowed_values_ptr: allowed_values.as_ptr(), validator: property.validator.as_minifi_c_type(), - allowed_type: if allowed_type.data == std::ptr::null() { + allowed_type: if allowed_type.data.is_null() { std::ptr::null() } else { allowed_type diff --git a/minifi_rust/minifi_native/src/c_ffi/c_ffi_streams.rs b/minifi_rust/minifi_native/src/c_ffi/c_ffi_streams.rs index 5684dbc7a9..964653e75d 100644 --- a/minifi_rust/minifi_native/src/c_ffi/c_ffi_streams.rs +++ b/minifi_rust/minifi_native/src/c_ffi/c_ffi_streams.rs @@ -1,7 +1,7 @@ use minifi_native_sys::{ minifi_input_stream, minifi_input_stream_read, minifi_output_stream, minifi_output_stream_write, }; -use std::io::{BufRead, Error, ErrorKind, Read}; +use std::io::{BufRead, Error, Read}; #[derive(Debug)] pub struct CffiInputStream<'a> { @@ -54,7 +54,7 @@ impl<'a> BufRead for CffiInputStream<'a> { self.buffer.len(), ); if ret < 0 { - return Err(Error::new(ErrorKind::Other, "Minifi Read Error")); + return Err(Error::other("Minifi Read Error")); } self.cap = ret as usize; self.pos = 0; @@ -104,7 +104,7 @@ impl<'a> std::io::Write for CffiOutputStream<'a> { buf.len(), ); if ret < 0 { - return Err(Error::new(ErrorKind::Other, "Minifi Write Error")); + return Err(Error::other("Minifi Write Error")); } self.written_bytes += ret as usize; Ok(ret as usize) diff --git a/minifi_rust/minifi_native/src/lib.rs b/minifi_rust/minifi_native/src/lib.rs index a78834cb9f..027efa55c0 100644 --- a/minifi_rust/minifi_native/src/lib.rs +++ b/minifi_rust/minifi_native/src/lib.rs @@ -66,7 +66,7 @@ macro_rules! declare_minifi_extension { ) => { #[unsafe(no_mangle)] - pub extern "C" fn minifi_init_extension( + pub unsafe extern "C" fn minifi_init_extension( extension_context: *mut minifi_native::sys::minifi_extension_context ) { diff --git a/minifi_rust/minifi_native/src/mock/mock_controller_service_context.rs b/minifi_rust/minifi_native/src/mock/mock_controller_service_context.rs index e24c0bce6d..2fd6924ffd 100644 --- a/minifi_rust/minifi_native/src/mock/mock_controller_service_context.rs +++ b/minifi_rust/minifi_native/src/mock/mock_controller_service_context.rs @@ -11,6 +11,12 @@ impl GetProperty for MockControllerServiceContext { } } +impl Default for MockControllerServiceContext { + fn default() -> Self { + Self::new() + } +} + impl MockControllerServiceContext { pub fn new() -> Self { Self { diff --git a/minifi_rust/minifi_native/src/mock/mock_flow_file.rs b/minifi_rust/minifi_native/src/mock/mock_flow_file.rs index 06b2c61966..5c1daa8bef 100644 --- a/minifi_rust/minifi_native/src/mock/mock_flow_file.rs +++ b/minifi_rust/minifi_native/src/mock/mock_flow_file.rs @@ -10,6 +10,12 @@ pub struct MockFlowFile { impl FlowFile for MockFlowFile {} +impl Default for MockFlowFile { + fn default() -> Self { + Self::new() + } +} + impl MockFlowFile { pub fn new() -> MockFlowFile { MockFlowFile { diff --git a/minifi_rust/minifi_native/src/mock/mock_logger.rs b/minifi_rust/minifi_native/src/mock/mock_logger.rs index a6451ac9b4..a611c58378 100644 --- a/minifi_rust/minifi_native/src/mock/mock_logger.rs +++ b/minifi_rust/minifi_native/src/mock/mock_logger.rs @@ -22,6 +22,12 @@ impl Logger for MockLogger { } } +impl Default for MockLogger { + fn default() -> Self { + Self::new() + } +} + impl MockLogger { pub fn new() -> Self { MockLogger { diff --git a/minifi_rust/minifi_native/src/mock/mock_process_context.rs b/minifi_rust/minifi_native/src/mock/mock_process_context.rs index 0739647bbf..56cd50eb76 100644 --- a/minifi_rust/minifi_native/src/mock/mock_process_context.rs +++ b/minifi_rust/minifi_native/src/mock/mock_process_context.rs @@ -112,6 +112,12 @@ impl ProcessContext for MockProcessContext { } } +impl Default for MockProcessContext { + fn default() -> Self { + Self::new() + } +} + impl MockProcessContext { pub fn new() -> Self { Self { diff --git a/minifi_rust/minifi_native/src/mock/mock_process_session.rs b/minifi_rust/minifi_native/src/mock/mock_process_session.rs index d9c5864df1..e80b9a76b4 100644 --- a/minifi_rust/minifi_native/src/mock/mock_process_session.rs +++ b/minifi_rust/minifi_native/src/mock/mock_process_session.rs @@ -125,6 +125,12 @@ impl ProcessSession for MockProcessSession { } } +impl Default for MockProcessSession { + fn default() -> Self { + Self::new() + } +} + impl MockProcessSession { pub fn new() -> Self { Self { diff --git a/minifi_rust/minifi_native_sys/build.rs b/minifi_rust/minifi_native_sys/build.rs index 5ed0728aa7..cfd03062df 100644 --- a/minifi_rust/minifi_native_sys/build.rs +++ b/minifi_rust/minifi_native_sys/build.rs @@ -2,16 +2,16 @@ use std::env; use std::path::{Path, PathBuf}; use std::process::Command; -struct SDK { +struct Sdk { header_path: PathBuf, #[cfg_attr(not(windows), allow(dead_code))] def_path: PathBuf, behave_path: PathBuf, } -impl SDK { +impl Sdk { fn from_local_repo(repo_root: &Path) -> Option { - let canon = std::fs::canonicalize(&repo_root).ok()?; + let canon = std::fs::canonicalize(repo_root).ok()?; println!( "cargo:info=MINIFI_SDK_PATH from from_local_repo: {:?} -> {:?}", repo_root, canon @@ -168,7 +168,7 @@ fn generate_minifi_c_api_lib(def_path: &Path) { fn main() { println!("cargo:rerun-if-env-changed=MINIFI_SDK_PATH"); - let sdk = SDK::new_from_env_variable() + let sdk = Sdk::new_from_env_variable() .expect("Couldn't find valid SDK. Ensure MINIFI_SDK_PATH is set correctly."); #[cfg(windows)] diff --git a/minifi_rust/minifi_rs_behave/src/main.rs b/minifi_rust/minifi_rs_behave/src/main.rs index 0dc99686ae..c30d3c0c1c 100644 --- a/minifi_rust/minifi_rs_behave/src/main.rs +++ b/minifi_rust/minifi_rs_behave/src/main.rs @@ -73,7 +73,7 @@ impl BehaveRunner { } fn install_minifi_behave(&self) -> ExitStatus { - Command::new(&self.get_venv_python()) + Command::new(self.get_venv_python()) .arg("-m") .arg("pip") .arg("install") @@ -101,9 +101,9 @@ impl BehaveRunner { } fn run_tests(&self, root: &Path) -> ExitStatus { - let mut cmd = Command::new(&self.get_venv_behave()); + let mut cmd = Command::new(self.get_venv_behave()); - cmd.args(&Self::find_features(root)); + cmd.args(Self::find_features(root)); cmd.arg("--show-progress-bar"); cmd.arg("--parallel-processes"); @@ -150,6 +150,6 @@ fn main() { let install_status = behave_runner.install_minifi_behave(); assert!(install_status.success(), "Pip install failed"); - let status = behave_runner.run_tests(&root); + let status = behave_runner.run_tests(root); assert!(status.success(), "Behave tests failed"); } From 123aaeeb2abcdf9402b405ff002cf433f868ab63 Mon Sep 17 00:00:00 2001 From: Martin Zink Date: Wed, 1 Jul 2026 13:00:24 +0200 Subject: [PATCH 09/26] claude code review --- .../src/processors/asciify_german.rs | 24 +++--- .../src/processors/asciify_german/tests.rs | 20 +++++ .../src/processors/generate_flow_file.rs | 19 ++++- .../src/processors/get_file.rs | 39 +++++---- .../src/processors/log_attribute.rs | 21 +++-- .../src/processors/log_attribute/tests.rs | 29 +++++++ .../src/processors/put_file.rs | 12 +-- .../minifi_native/src/api/process_session.rs | 8 +- .../c_ffi/c_ffi_controller_service_context.rs | 17 +++- .../c_ffi_controller_service_definition.rs | 5 +- .../minifi_native/src/c_ffi/c_ffi_logger.rs | 27 +++--- .../src/c_ffi/c_ffi_process_context.rs | 2 +- .../src/c_ffi/c_ffi_process_session.rs | 82 +++++++++++++------ .../minifi_native/src/c_ffi/c_ffi_streams.rs | 4 - .../minifi_native/src/mock/mock_flow_file.rs | 11 ++- .../src/mock/mock_process_context.rs | 22 ++++- .../src/mock/mock_process_session.rs | 24 +++++- minifi_rust/minifi_native_sys/build.rs | 45 ++++++++-- 18 files changed, 301 insertions(+), 110 deletions(-) diff --git a/minifi_rust/extensions/minifi_rs_playground/src/processors/asciify_german.rs b/minifi_rust/extensions/minifi_rs_playground/src/processors/asciify_german.rs index 7491fc60dd..55936935c3 100644 --- a/minifi_rust/extensions/minifi_rs_playground/src/processors/asciify_german.rs +++ b/minifi_rust/extensions/minifi_rs_playground/src/processors/asciify_german.rs @@ -37,17 +37,19 @@ impl FlowFileStreamTransform for AsciifyGerman { } 0xC3 => { let mut next = [0u8; 1]; - if input_stream.read(&mut next)? > 0 { - match next[0] { - 0xA4 => output_stream.write_all(b"ae")?, // ä - 0xB6 => output_stream.write_all(b"oe")?, // ö - 0xBC => output_stream.write_all(b"ue")?, // ü - 0x84 => output_stream.write_all(b"Ae")?, // Ä - 0x96 => output_stream.write_all(b"Oe")?, // Ö - 0x9C => output_stream.write_all(b"Ue")?, // Ü - 0x9F => output_stream.write_all(b"ss")?, // ß - _ => return Ok(TransformStreamResult::route_without_changes(&FAILURE)), - } + if input_stream.read(&mut next)? == 0 { + // Truncated multi-byte sequence at EOF — treat as malformed input. + return Ok(TransformStreamResult::route_without_changes(&FAILURE)); + } + match next[0] { + 0xA4 => output_stream.write_all(b"ae")?, // ä + 0xB6 => output_stream.write_all(b"oe")?, // ö + 0xBC => output_stream.write_all(b"ue")?, // ü + 0x84 => output_stream.write_all(b"Ae")?, // Ä + 0x96 => output_stream.write_all(b"Oe")?, // Ö + 0x9C => output_stream.write_all(b"Ue")?, // Ü + 0x9F => output_stream.write_all(b"ss")?, // ß + _ => return Ok(TransformStreamResult::route_without_changes(&FAILURE)), } } _ => return Ok(TransformStreamResult::route_without_changes(&FAILURE)), diff --git a/minifi_rust/extensions/minifi_rs_playground/src/processors/asciify_german/tests.rs b/minifi_rust/extensions/minifi_rs_playground/src/processors/asciify_german/tests.rs index 268a95ff92..ec2ded4204 100644 --- a/minifi_rust/extensions/minifi_rs_playground/src/processors/asciify_german/tests.rs +++ b/minifi_rust/extensions/minifi_rs_playground/src/processors/asciify_german/tests.rs @@ -52,3 +52,23 @@ fn simple_failure_test() { } assert_eq!(output_vec, "Ueldoeg".as_bytes()); } + +#[test] +fn truncated_umlaut_at_eof_routes_to_failure() { + let context = MockProcessContext::new(); + let logger = MockLogger::new(); + + let asciify_german = AsciifyGerman::schedule(&context, &logger).expect("Should succeed"); + // A lone 0xC3 is the leading byte of every German umlaut in UTF-8; on EOF the + // sequence is incomplete and the processor must report failure rather than + // silently truncating. + let input_bytes: &[u8] = &[b'a', 0xC3]; + let mut input_stream = BufReader::new(input_bytes); + let mut output_vec: Vec = Vec::new(); + + let result = asciify_german + .transform(&context, &mut input_stream, &mut output_vec, &logger) + .expect("Should succeed"); + assert_eq!(result.write_status(), IoState::Cancel); + assert_eq!(result.target_relationship_name(), FAILURE.name); +} diff --git a/minifi_rust/extensions/minifi_rs_playground/src/processors/generate_flow_file.rs b/minifi_rust/extensions/minifi_rs_playground/src/processors/generate_flow_file.rs index fa894ef599..971b6a07d3 100644 --- a/minifi_rust/extensions/minifi_rs_playground/src/processors/generate_flow_file.rs +++ b/minifi_rust/extensions/minifi_rs_playground/src/processors/generate_flow_file.rs @@ -136,10 +136,21 @@ impl Trigger for GenerateFlowFileRs { let non_unique_data_buffer: &[u8]; let custom_text_for_batch: Option; - if self.mode == Mode::CustomText - && let Some(custom_text) = context.get_property(&properties::CUSTOM_TEXT, None)? - { - custom_text_for_batch = Some(custom_text); + if self.mode == Mode::CustomText { + // CustomText mode must have the Custom Text property set at + // trigger time — falling back to `data_generated_during_on_schedule` + // (which is empty for this mode) would silently produce empty + // flow files. + custom_text_for_batch = Some( + context + .get_property(&properties::CUSTOM_TEXT, None)? + .ok_or_else(|| { + MinifiError::trigger_err( + "GenerateFlowFile is in CustomText mode but the \"Custom Text\" \ + property is not set at trigger time", + ) + })?, + ); non_unique_data_buffer = custom_text_for_batch.as_ref().unwrap().as_bytes(); } else { non_unique_data_buffer = self.data_generated_during_on_schedule.as_slice(); diff --git a/minifi_rust/extensions/minifi_rs_playground/src/processors/get_file.rs b/minifi_rust/extensions/minifi_rs_playground/src/processors/get_file.rs index 1366b71bb6..96bd419a87 100644 --- a/minifi_rust/extensions/minifi_rs_playground/src/processors/get_file.rs +++ b/minifi_rust/extensions/minifi_rs_playground/src/processors/get_file.rs @@ -13,7 +13,7 @@ use minifi_native::{ use std::collections::VecDeque; use std::error; use std::fs::File; -use std::path::PathBuf; +use std::path::{Path, PathBuf}; use std::sync::Mutex; use std::time::{Duration, Instant, SystemTime}; use walkdir::{DirEntry, WalkDir}; @@ -92,27 +92,32 @@ impl GetFileRs { } fn perform_listing(&self) { - let mut directory_listings = self.directory_listing.lock().unwrap(); let mut walker = WalkDir::new(&self.input_directory); - if !self.recursive { walker = walker.max_depth(1); } + let mut new_paths: VecDeque = VecDeque::new(); let mut files_added = 0u32; let mut bytes_added = 0u64; for entry in walker.into_iter().filter_map(Result::ok) { if self.entry_matches_criteria(&entry).unwrap_or(false) { let file_size = entry.metadata().map(|m| m.len()).unwrap_or(0); - directory_listings.paths.push_back(entry.into_path()); + new_paths.push_back(entry.into_path()); files_added += 1; bytes_added += file_size; } } + + { + let mut directory_listings = self.directory_listing.lock().unwrap(); + directory_listings.paths.extend(new_paths); + directory_listings.last_polling_time = Some(Instant::now()); + } + let mut metrics = self.metrics.lock().unwrap(); metrics.accepted_files += files_added; metrics.input_bytes += bytes_added; - directory_listings.last_polling_time = Some(Instant::now()); } fn entry_matches_criteria(&self, dir_entry: &DirEntry) -> Result> { @@ -153,9 +158,9 @@ impl GetFileRs { &self, session: &mut PS, logger: &L, - path: PathBuf, + path: &Path, ) -> Result<(), MinifiError> { - info!(logger, "GetFile process {:?}", &path); + info!(logger, "GetFile process {:?}", path); let mut ff = session .create() .expect("Successful FlowFile creation is expected"); @@ -172,17 +177,14 @@ impl GetFileRs { )?; session.write_stream(&ff, |output_stream| { - let mut file = File::open(&path)?; + let mut file = File::open(path)?; std::io::copy(&mut file, output_stream)?; Ok(((), IoState::Ok)) })?; - if !self.keep_source_file { - match std::fs::remove_file(&path) { - Ok(_) => {} - Err(err) => { - warn!(logger, "Failed to remove source file {:?}", err); - } - } + if !self.keep_source_file + && let Err(err) = std::fs::remove_file(path) + { + warn!(logger, "Failed to remove source file {:?}", err); } session.transfer(ff, relationships::SUCCESS.name)?; Ok(()) @@ -289,7 +291,12 @@ impl Trigger for GetFileRs { let files = self.poll_listing(self.batch_size); for file in files { - self.get_single_file(session, logger, file)?; + // A single mid-batch failure (deleted, permission flipped, + // transient I/O) should not orphan the flow files already + // transferred earlier in the same trigger. Log and continue. + if let Err(err) = self.get_single_file(session, logger, &file) { + warn!(logger, "Failed to ingest {:?}: {}", file, err); + } } context.report_metrics(self.calculate_metrics())?; Ok(OnTriggerResult::Ok) diff --git a/minifi_rust/extensions/minifi_rs_playground/src/processors/log_attribute.rs b/minifi_rust/extensions/minifi_rs_playground/src/processors/log_attribute.rs index 56f88f9a72..b37c7c5a67 100644 --- a/minifi_rust/extensions/minifi_rs_playground/src/processors/log_attribute.rs +++ b/minifi_rust/extensions/minifi_rs_playground/src/processors/log_attribute.rs @@ -31,13 +31,15 @@ impl LogAttributeRs { log_msg.push_str("\nFlowFile Attributes Map Content"); session.on_attributes(flow_file, |key, value| { if let Some(attributes_to_ignore) = &self.attributes_to_ignore - && attributes_to_ignore.iter().any(|ign| ign == key) { - return; - } + && attributes_to_ignore.iter().any(|ign| ign == key) + { + return; + } if let Some(attributes_to_log) = &self.attributes_to_log - && !attributes_to_log.iter().any(|ign| ign == key) { - return; - } + && !attributes_to_log.iter().any(|ign| ign == key) + { + return; + } log_msg.push_str(format!("\nkey:{} value:{}", &key, &value).as_str()); }); if self.log_payload { @@ -118,8 +120,11 @@ impl Schedule for LogAttributeRs { context: &P, property: &Property, ) -> Result>, MinifiError> { - Ok(context - .get_property(property)?.map(|s| s.split(",").map(|s| s.to_string()).collect::>())) + Ok(context.get_property(property)?.map(|s| { + s.split(',') + .map(|s| s.trim().to_string()) + .collect::>() + })) } let attributes_to_log = get_csv_property(context, &properties::ATTRIBUTES_TO_LOG)?; diff --git a/minifi_rust/extensions/minifi_rs_playground/src/processors/log_attribute/tests.rs b/minifi_rust/extensions/minifi_rs_playground/src/processors/log_attribute/tests.rs index c87070b9f3..1a272b4077 100644 --- a/minifi_rust/extensions/minifi_rs_playground/src/processors/log_attribute/tests.rs +++ b/minifi_rust/extensions/minifi_rs_playground/src/processors/log_attribute/tests.rs @@ -91,6 +91,35 @@ Payload: tester(vec, LogLevel::Critical, Box::new(properties_set), expected); } +#[test] +fn attributes_to_log_csv_trims_whitespace() { + let mut flow_file = MockFlowFile::with_content(b"payload"); + flow_file + .attributes + .insert(String::from("apple"), String::from("apfel")); + flow_file + .attributes + .insert(String::from("pear"), String::from("birne")); + flow_file + .attributes + .insert(String::from("cherry"), String::from("kirsche")); + let vec = vec![flow_file]; + + // Value with spaces around commas: filter must match on trimmed names. + let expected = "Logging for flow file +-------------------------------------------------- +FlowFile Attributes Map Content +key:apple value:apfel +key:pear value:birne +--------------------------------------------------" + .to_string(); + let properties_set = [ + ("Log Payload", "false"), + ("Attributes to Log", "apple, pear"), + ]; + tester(vec, LogLevel::Info, Box::new(properties_set), expected); +} + #[test] fn default_level_multiple_attributes() { let mut flow_file = MockFlowFile::with_content("Lorem ipsum dolor sit amet, consectetur adipiscing elit. Integer facilisis diam sit amet nisl interdum, vitae interdum arcu viverra. Nam placerat mi in erat pellentesque, at ultrices orci faucibus. Cras sollicitudin iaculis posuere. Sed tempus, dolor nec lacinia suscipit, tellus odio venenatis odio, nec sollicitudin dolor augue non urna. Aliquam tincidunt viverra ipsum eget hendrerit. Suspendisse varius, augue vel fermentum varius, velit elit euismod lacus, a placerat purus est a lacus. Aenean nibh neque, consectetur hendrerit egestas vitae, commodo non quam. Nullam luctus tempor ante, sed tempus quam imperdiet in. Maecenas gravida erat orci, in consequat urna pretium nec. In sodales iaculis magna at vehicula.".as_bytes()); diff --git a/minifi_rust/extensions/minifi_rs_playground/src/processors/put_file.rs b/minifi_rust/extensions/minifi_rs_playground/src/processors/put_file.rs index 61dfc057f8..b0d0e76649 100644 --- a/minifi_rust/extensions/minifi_rs_playground/src/processors/put_file.rs +++ b/minifi_rust/extensions/minifi_rs_playground/src/processors/put_file.rs @@ -75,6 +75,7 @@ impl PutFileRs { { parent.exists() && WalkDir::new(parent) + .max_depth(1) .into_iter() .filter_map(Result::ok) .filter(|e| e.file_type().is_file()) @@ -96,15 +97,16 @@ impl PutFileRs { let file_name = context .get_attribute("filename")? .unwrap_or(context.get_id()?); - Ok(PathBuf::from(directory + "/" + file_name.as_str())) + Ok(PathBuf::from(directory).join(file_name)) } fn prepare_destination(&self, destination: &Path) -> std::io::Result<()> { if let Some(parent) = destination.parent() - && self.try_make_dirs { - std::fs::create_dir_all(parent)?; - self.unix_permissions.set_directory_permissions(parent)?; - } + && self.try_make_dirs + { + std::fs::create_dir_all(parent)?; + self.unix_permissions.set_directory_permissions(parent)?; + } Ok(()) } diff --git a/minifi_rust/minifi_native/src/api/process_session.rs b/minifi_rust/minifi_native/src/api/process_session.rs index f58dc9cc60..fe0171ba7a 100644 --- a/minifi_rust/minifi_native/src/api/process_session.rs +++ b/minifi_rust/minifi_native/src/api/process_session.rs @@ -1,9 +1,9 @@ use crate::MinifiError; use crate::api::flow_file::FlowFile; -pub trait InputStream: std::io::BufRead + Send + std::fmt::Debug {} -pub trait OutputStream: std::io::Write + Send + std::fmt::Debug {} -impl OutputStream for T {} -impl InputStream for T {} +pub trait InputStream: std::io::BufRead + std::fmt::Debug {} +pub trait OutputStream: std::io::Write + std::fmt::Debug {} +impl OutputStream for T {} +impl InputStream for T {} #[derive(Debug, Clone, Copy, Eq, PartialEq)] pub enum IoState { diff --git a/minifi_rust/minifi_native/src/c_ffi/c_ffi_controller_service_context.rs b/minifi_rust/minifi_native/src/c_ffi/c_ffi_controller_service_context.rs index b39bbfd3e5..12535a290c 100644 --- a/minifi_rust/minifi_native/src/c_ffi/c_ffi_controller_service_context.rs +++ b/minifi_rust/minifi_native/src/c_ffi/c_ffi_controller_service_context.rs @@ -2,10 +2,12 @@ use crate::c_ffi::c_ffi_primitives::StringView; use crate::{GetProperty, MinifiError, Property}; use minifi_native_sys::{ minifi_controller_service_context, minifi_controller_service_context_get_property, - minifi_status_MINIFI_STATUS_SUCCESS, minifi_string_view, + minifi_status_MINIFI_STATUS_PROPERTY_NOT_SET, minifi_status_MINIFI_STATUS_SUCCESS, + minifi_string_view, }; use std::borrow::Cow; use std::ffi::c_void; +use std::num::NonZeroU32; pub struct CffiControllerServiceContext<'a> { ptr: *mut minifi_controller_service_context, @@ -28,7 +30,7 @@ unsafe extern "C" fn property_callback( unsafe { let result_target = &mut *(output_option as *mut Option); - if property_c_value.data.is_null() || property_c_value.length == 0 { + if property_c_value.data.is_null() { *result_target = None; return; } @@ -58,12 +60,21 @@ impl<'a> GetProperty for CffiControllerServiceContext<'a> { #[allow(non_upper_case_globals)] match status { minifi_status_MINIFI_STATUS_SUCCESS => Ok(result), - _ => match property.is_required { + minifi_status_MINIFI_STATUS_PROPERTY_NOT_SET => match property.is_required { true => Err(MinifiError::MissingRequiredProperty(Cow::from( property.name, ))), false => Ok(None), }, + err_code => Err(MinifiError::StatusError(( + format!( + "minifi_controller_service_context_get_property({:?})", + property.name + ) + .into(), + // SAFETY: err_code is non-zero because SUCCESS is handled above. + unsafe { NonZeroU32::new_unchecked(err_code) }, + ))), } } } diff --git a/minifi_rust/minifi_native/src/c_ffi/c_ffi_controller_service_definition.rs b/minifi_rust/minifi_native/src/c_ffi/c_ffi_controller_service_definition.rs index 2fcd7ede5b..7425297214 100644 --- a/minifi_rust/minifi_native/src/c_ffi/c_ffi_controller_service_definition.rs +++ b/minifi_rust/minifi_native/src/c_ffi/c_ffi_controller_service_definition.rs @@ -121,7 +121,10 @@ where let name = unsafe { let slice = std::slice::from_raw_parts(interface_name.data as *const u8, interface_name.length); - std::str::from_utf8_unchecked(slice) + match std::str::from_utf8(slice) { + Ok(s) => s, + Err(_) => return std::ptr::null_mut(), + } }; let wrapper = unsafe { &*(self_ptr as *const ControllerService) }; diff --git a/minifi_rust/minifi_native/src/c_ffi/c_ffi_logger.rs b/minifi_rust/minifi_native/src/c_ffi/c_ffi_logger.rs index bb1ea0980e..78fa29f0e4 100644 --- a/minifi_rust/minifi_native/src/c_ffi/c_ffi_logger.rs +++ b/minifi_rust/minifi_native/src/c_ffi/c_ffi_logger.rs @@ -34,18 +34,23 @@ impl CffiLogger { impl Logger for CffiLogger { fn log(&self, level: LogLevel, args: fmt::Arguments) { + // A logged message that contains an interior NUL byte would make + // `CString::new` fail + let mut message = fmt::format(args); + if message.as_bytes().contains(&0) { + message = message.replace('\0', "\\0"); + } + // SAFETY: NULs have been stripped, so CString::new cannot fail. + let c_message = CString::new(message).expect("NUL bytes were replaced above"); unsafe { - let message = fmt::format(args); - if let Ok(c_message) = CString::new(message) { - minifi_logger_log_string( - self.ptr, - level.into(), - minifi_string_view { - data: c_message.as_ptr(), - length: c_message.as_bytes().len(), - }, - ); - } + minifi_logger_log_string( + self.ptr, + level.into(), + minifi_string_view { + data: c_message.as_ptr(), + length: c_message.as_bytes().len(), + }, + ); } } diff --git a/minifi_rust/minifi_native/src/c_ffi/c_ffi_process_context.rs b/minifi_rust/minifi_native/src/c_ffi/c_ffi_process_context.rs index d95e809d23..161ef7546a 100644 --- a/minifi_rust/minifi_native/src/c_ffi/c_ffi_process_context.rs +++ b/minifi_rust/minifi_native/src/c_ffi/c_ffi_process_context.rs @@ -33,7 +33,7 @@ unsafe extern "C" fn get_property_callback( unsafe { let result_target = &mut *(output_option as *mut Option); - if property_c_value.data.is_null() || property_c_value.length == 0 { + if property_c_value.data.is_null() { *result_target = None; return; } diff --git a/minifi_rust/minifi_native/src/c_ffi/c_ffi_process_session.rs b/minifi_rust/minifi_native/src/c_ffi/c_ffi_process_session.rs index 7efc353e44..3c137fcccf 100644 --- a/minifi_rust/minifi_native/src/c_ffi/c_ffi_process_session.rs +++ b/minifi_rust/minifi_native/src/c_ffi/c_ffi_process_session.rs @@ -45,30 +45,46 @@ impl<'a> CffiProcessSession<'a> { produce_batch: F, ) -> Result<(), MinifiError> where - F: FnMut(&mut [u8]) -> Option, + F: FnMut(&mut [u8]) -> Result, MinifiError>, { unsafe { - struct State<'b, F: FnMut(&mut [u8]) -> Option> { + struct State<'b, F> + where + F: FnMut(&mut [u8]) -> Result, MinifiError>, + { callback: F, buffer: &'b mut [u8], + error: Option, } let mut buffer = [0u8; 8192]; let mut state = State { callback: produce_batch, buffer: &mut buffer, + error: None, }; - unsafe extern "C" fn cb Option>( + unsafe extern "C" fn cb( user_ctx: *mut c_void, output_stream: *mut minifi_output_stream, - ) -> i64 { + ) -> i64 + where + F: FnMut(&mut [u8]) -> Result, MinifiError>, + { unsafe { let state = &mut *(user_ctx as *mut State); let mut overall_writes = 0; - // The user fills our provided buffer - while let Some(n) = (state.callback)(state.buffer) { + loop { + let n = match (state.callback)(state.buffer) { + Ok(Some(n)) => n, + Ok(None) => break, // EOF + Err(err) => { + state.error = Some(err); + return minifi_io_status_MINIFI_IO_ERROR; + } + }; + let written = minifi_output_stream_write( output_stream, state.buffer.as_ptr() as *const c_char, @@ -84,12 +100,18 @@ impl<'a> CffiProcessSession<'a> { } } - match minifi_process_session_write( + let status = minifi_process_session_write( self.ptr, flow_file.get_ptr(), Some(cb::), &mut state as *mut _ as *mut c_void, - ) { + ); + + if let Some(err) = state.error.take() { + return Err(err); + } + + match status { #[allow(non_upper_case_globals)] minifi_status_MINIFI_STATUS_SUCCESS => Ok(()), error_code => Err(MinifiError::StatusError(( @@ -295,12 +317,10 @@ impl<'a> ProcessSession for CffiProcessSession<'a> { flow_file: &Self::FlowFile, mut stream: Box, ) -> Result<(), MinifiError> { - self.write_in_batches(flow_file, |buffer| { - match stream.read(buffer) { - Ok(0) => None, // EOF - Ok(n) => Some(n), - Err(_e) => None, // Signal failure/EOF - } + self.write_in_batches(flow_file, |buffer| match stream.read(buffer) { + Ok(0) => Ok(None), // EOF + Ok(n) => Ok(Some(n)), + Err(e) => Err(MinifiError::IoError(e)), }) } @@ -349,23 +369,32 @@ impl<'a> ProcessSession for CffiProcessSession<'a> { } } - unsafe { - let session_status = minifi_process_session_write( + let session_status = unsafe { + minifi_process_session_write( self.ptr, flow_file.get_ptr(), Some(write_cb::), &mut ctx as *mut _ as *mut c_void, - ); - if session_status != minifi_status_MINIFI_STATUS_SUCCESS { - return Err(MinifiError::StatusError(( - "minifi_process_session_write".into(), - NonZeroU32::new_unchecked(session_status), - ))); - } + ) + }; + + // If the user closure ran, its outcome (including a deliberate + // Ok((_, IoState::Cancel))) always wins over the C session status: + // MINIFI_IO_CANCEL from the callback makes the session return + // non-success even though from Rust's perspective the write succeeded. + if let Some(result) = ctx.result.take() { + return result; } - ctx.result - .expect("Agent returned with success, so ctx.result should be set") + // Callback never ran — surface the raw C status. + if session_status != minifi_status_MINIFI_STATUS_SUCCESS { + return Err(MinifiError::StatusError(( + "minifi_process_session_write".into(), + unsafe { NonZeroU32::new_unchecked(session_status) }, + ))); + } + + Err(MinifiError::UnknownError) } fn read(&self, flow_file: &Self::FlowFile) -> Option> { @@ -380,7 +409,8 @@ impl<'a> ProcessSession for CffiProcessSession<'a> { let stream_size = minifi_input_stream_size(input_stream); if stream_size == 0 { - *result_target = None; + // Empty content is a legal state, not an error + *result_target = Some(Vec::new()); return 0; } let mut buffer: Vec = Vec::with_capacity(stream_size); diff --git a/minifi_rust/minifi_native/src/c_ffi/c_ffi_streams.rs b/minifi_rust/minifi_native/src/c_ffi/c_ffi_streams.rs index 964653e75d..82fd1ec19c 100644 --- a/minifi_rust/minifi_native/src/c_ffi/c_ffi_streams.rs +++ b/minifi_rust/minifi_native/src/c_ffi/c_ffi_streams.rs @@ -13,8 +13,6 @@ pub struct CffiInputStream<'a> { _marker: std::marker::PhantomData<&'a ()>, } -unsafe impl<'a> Send for CffiInputStream<'a> {} - impl<'a> CffiInputStream<'a> { pub fn new(ptr: *mut minifi_input_stream) -> Self { Self { @@ -93,8 +91,6 @@ impl<'a> CffiOutputStream<'a> { } } -unsafe impl<'a> Send for CffiOutputStream<'a> {} - impl<'a> std::io::Write for CffiOutputStream<'a> { fn write(&mut self, buf: &[u8]) -> std::io::Result { unsafe { diff --git a/minifi_rust/minifi_native/src/mock/mock_flow_file.rs b/minifi_rust/minifi_native/src/mock/mock_flow_file.rs index 5c1daa8bef..96ca074590 100644 --- a/minifi_rust/minifi_native/src/mock/mock_flow_file.rs +++ b/minifi_rust/minifi_native/src/mock/mock_flow_file.rs @@ -1,6 +1,7 @@ use crate::api::FlowFile; use std::cell::RefCell; use std::collections::HashMap; +use std::sync::atomic::{AtomicU64, Ordering}; pub struct MockFlowFile { pub content: RefCell>, @@ -16,12 +17,18 @@ impl Default for MockFlowFile { } } +fn next_mock_flow_file_id() -> String { + static COUNTER: AtomicU64 = AtomicU64::new(0); + let n = COUNTER.fetch_add(1, Ordering::Relaxed); + format!("mock-flow-file-{:016x}", n) +} + impl MockFlowFile { pub fn new() -> MockFlowFile { MockFlowFile { content: RefCell::new(Vec::new()), attributes: HashMap::new(), - id: "67e55044-10b1-426f-9247-bb680e5fe0c8".to_string(), // TODO generate something? + id: next_mock_flow_file_id(), } } @@ -29,7 +36,7 @@ impl MockFlowFile { Self { content: RefCell::new(content.to_vec()), attributes: HashMap::new(), - id: "67e55044-10b1-426f-9247-bb680e5fe0c8".to_string(), // TODO generate something? + id: next_mock_flow_file_id(), } } diff --git a/minifi_rust/minifi_native/src/mock/mock_process_context.rs b/minifi_rust/minifi_native/src/mock/mock_process_context.rs index 56cd50eb76..3ea6e015b2 100644 --- a/minifi_rust/minifi_native/src/mock/mock_process_context.rs +++ b/minifi_rust/minifi_native/src/mock/mock_process_context.rs @@ -78,12 +78,21 @@ impl ProcessContext for MockProcessContext { fn get_raw_controller_service( &self, - _property: &Property, + property: &Property, ) -> Result, MinifiError> where Cs: RawControllerService + ComponentIdentifier + 'static, { - panic!("Not implemented yet"); + // Mirror `get_controller_service`: resolve the property to a + // service name and downcast the registered `Box`. + if let Some(service_name) = self.get_property(property, None)? { + Ok(self + .controller_services + .get(&service_name) + .and_then(|c| c.downcast_ref::())) + } else { + Ok(None) + } } fn get_controller_service(&self, property: &Property) -> Result, MinifiError> @@ -104,7 +113,14 @@ impl ProcessContext for MockProcessContext { &self, _property: &Property, ) -> Result>, MinifiError> { - todo!() + // A fully-typed mock for `dyn Trait` interfaces would need per-property + // registration keyed by both property name and interface name; the + // combination of `?Sized + ControllerServiceApi` (no `'static`, no + // `Any`) makes safe storage awkward. Return `None` by default so + // processors that call this method can still be scheduled and + // triggered under the mock — tests that need a live implementation + // should exercise it in the FFI path via `cargo behave`. + Ok(None) } fn report_metrics(&self, _metrics: Vec<(String, f64)>) -> Result<(), MinifiError> { diff --git a/minifi_rust/minifi_native/src/mock/mock_process_session.rs b/minifi_rust/minifi_native/src/mock/mock_process_session.rs index e80b9a76b4..e8871d99eb 100644 --- a/minifi_rust/minifi_native/src/mock/mock_process_session.rs +++ b/minifi_rust/minifi_native/src/mock/mock_process_session.rs @@ -75,8 +75,10 @@ impl ProcessSession for MockProcessSession { flow_file: &Self::FlowFile, mut stream: Box, ) -> Result<(), MinifiError> { + let mut buf = flow_file.content.borrow_mut(); + buf.clear(); stream - .read_to_end(&mut flow_file.content.borrow_mut()) + .read_to_end(&mut buf) .expect("Mock data should be readable"); Ok(()) } @@ -98,11 +100,13 @@ impl ProcessSession for MockProcessSession { Some(flow_file.content.borrow().clone()) } - fn read_stream(&self, _flow_file: &Self::FlowFile, _callback: F) -> Result + fn read_stream(&self, flow_file: &Self::FlowFile, callback: F) -> Result where F: FnOnce(&mut dyn InputStream) -> Result, { - unimplemented!("Not implemented yet") + let bytes = flow_file.content.borrow().clone(); + let mut cursor = std::io::Cursor::new(bytes); + callback(&mut cursor) } fn read_in_batches( @@ -168,6 +172,20 @@ mod tests { assert_eq!(vec, b"Hello, World!"); } + #[test] + fn test_write_from_stream_replaces_content() { + let session = MockProcessSession::new(); + let flow_file = MockFlowFile::new(); + *flow_file.content.borrow_mut() = b"Hello, World!".to_vec(); + + let source: &[u8] = b"Hi"; + session + .write_from_stream(&flow_file, Box::new(source)) + .expect("write_from_stream should succeed"); + + assert_eq!(*flow_file.content.borrow(), b"Hi"); + } + #[test] fn test_write_stream_replaces_content() { let session = MockProcessSession::new(); diff --git a/minifi_rust/minifi_native_sys/build.rs b/minifi_rust/minifi_native_sys/build.rs index cfd03062df..5abcc9a22f 100644 --- a/minifi_rust/minifi_native_sys/build.rs +++ b/minifi_rust/minifi_native_sys/build.rs @@ -76,10 +76,10 @@ impl Sdk { if sdk_path.starts_with("https://") { let zip_path = out_dir.join("downloaded-sdk.zip"); let extract_dir = out_dir.join("extracted-sdk"); - if !extract_dir.exists() { + Self::ensure_extracted(&extract_dir, || { Self::download_file(&sdk_path, &zip_path); Self::extract_zip(&zip_path, &extract_dir); - } + }); Self::from_local_sdk_path(&extract_dir) } else { let local_path = if sdk_path.starts_with(".") { @@ -89,9 +89,9 @@ impl Sdk { }; if local_path.is_file() && local_path.extension().unwrap_or_default() == "zip" { let extract_dir = out_dir.join("extracted-local-sdk"); - if !extract_dir.exists() { + Self::ensure_extracted(&extract_dir, || { Self::extract_zip(&local_path, &extract_dir); - } + }); Self::from_local_sdk_path(&extract_dir) } else { Self::from_local_sdk_path(&local_path) @@ -100,6 +100,30 @@ impl Sdk { } } + /// Run `extract` unless a previous run completed successfully (marked by a + /// sentinel file inside `dest_dir`). A partial extraction — for example if + /// `curl`/`tar` panicked mid-way — leaves `dest_dir` on disk without the + /// sentinel, so the next build reruns the extraction rather than working + /// with a broken tree. + fn ensure_extracted(dest_dir: &Path, extract: F) { + let sentinel = dest_dir.join(".extraction-complete"); + if sentinel.exists() { + return; + } + if dest_dir.exists() { + std::fs::remove_dir_all(dest_dir).unwrap_or_else(|e| { + panic!( + "Failed to clean up stale extraction directory {:?}: {}", + dest_dir, e + ) + }); + } + extract(); + std::fs::write(&sentinel, b"ok").unwrap_or_else(|e| { + panic!("Failed to write extraction sentinel {:?}: {}", sentinel, e) + }); + } + fn download_file(url: &str, dest: &Path) { println!("cargo:warning=Downloading SDK from {}...", url); let status = Command::new("curl") @@ -175,10 +199,15 @@ fn main() { generate_minifi_c_api_lib(&sdk.def_path); println!("cargo:rerun-if-changed={}", sdk.header_path.display()); - println!( - "cargo:behave_path={}", - sdk.behave_path.canonicalize().unwrap().display() - ); + println!("cargo:rerun-if-changed={}", sdk.def_path.display()); + let behave_path_canonical = sdk.behave_path.canonicalize().unwrap_or_else(|e| { + panic!( + "Failed to canonicalize behave path {:?}: {}. Ensure the behave framework \ + is present alongside the SDK (set MINIFI_SDK_PATH to a full SDK checkout).", + sdk.behave_path, e + ) + }); + println!("cargo:behave_path={}", behave_path_canonical.display()); let bindings = bindgen::Builder::default() .header(sdk.header_path.to_str().unwrap()) From 3f63290ee8a0fa55bd1e565cbfd5a14f1d099957 Mon Sep 17 00:00:00 2001 From: Martin Zink Date: Wed, 1 Jul 2026 14:27:56 +0200 Subject: [PATCH 10/26] arm clippy fixes --- .../src/c_ffi/c_ffi_controller_service_context.rs | 2 +- .../src/c_ffi/c_ffi_controller_service_definition.rs | 2 +- minifi_rust/minifi_native/src/c_ffi/c_ffi_primitives.rs | 4 ++-- minifi_rust/minifi_native/src/c_ffi/c_ffi_process_context.rs | 2 +- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/minifi_rust/minifi_native/src/c_ffi/c_ffi_controller_service_context.rs b/minifi_rust/minifi_native/src/c_ffi/c_ffi_controller_service_context.rs index 12535a290c..87cbbf4d29 100644 --- a/minifi_rust/minifi_native/src/c_ffi/c_ffi_controller_service_context.rs +++ b/minifi_rust/minifi_native/src/c_ffi/c_ffi_controller_service_context.rs @@ -36,7 +36,7 @@ unsafe extern "C" fn property_callback( } let value_slice = - std::slice::from_raw_parts(property_c_value.data as *const u8, property_c_value.length); + std::slice::from_raw_parts(property_c_value.data.cast::(), property_c_value.length); if let Ok(string_value) = String::from_utf8(value_slice.to_vec()) { *result_target = Some(string_value); } diff --git a/minifi_rust/minifi_native/src/c_ffi/c_ffi_controller_service_definition.rs b/minifi_rust/minifi_native/src/c_ffi/c_ffi_controller_service_definition.rs index 7425297214..e8048c0de5 100644 --- a/minifi_rust/minifi_native/src/c_ffi/c_ffi_controller_service_definition.rs +++ b/minifi_rust/minifi_native/src/c_ffi/c_ffi_controller_service_definition.rs @@ -120,7 +120,7 @@ where ) -> *mut std::ffi::c_void { let name = unsafe { let slice = - std::slice::from_raw_parts(interface_name.data as *const u8, interface_name.length); + std::slice::from_raw_parts(interface_name.data.cast::(), interface_name.length); match std::str::from_utf8(slice) { Ok(s) => s, Err(_) => return std::ptr::null_mut(), diff --git a/minifi_rust/minifi_native/src/c_ffi/c_ffi_primitives.rs b/minifi_rust/minifi_native/src/c_ffi/c_ffi_primitives.rs index 78b144cee0..660c860e26 100644 --- a/minifi_rust/minifi_native/src/c_ffi/c_ffi_primitives.rs +++ b/minifi_rust/minifi_native/src/c_ffi/c_ffi_primitives.rs @@ -59,7 +59,7 @@ impl ConvertMinifiStringView for minifi_string_view { return Err(FfiConversionError::NullPointer); } unsafe { - let slice = std::slice::from_raw_parts(self.data as *const u8, self.length); + let slice = std::slice::from_raw_parts(self.data.cast::(), self.length); String::from_utf8(slice.to_vec()).map_err(|_| FfiConversionError::InvalidUtf8) } } @@ -69,7 +69,7 @@ impl ConvertMinifiStringView for minifi_string_view { return Err(FfiConversionError::NullPointer); } unsafe { - let slice = std::slice::from_raw_parts(self.data as *const u8, self.length); + let slice = std::slice::from_raw_parts(self.data.cast::(), self.length); str::from_utf8(slice).map_err(|_| FfiConversionError::InvalidUtf8) } } diff --git a/minifi_rust/minifi_native/src/c_ffi/c_ffi_process_context.rs b/minifi_rust/minifi_native/src/c_ffi/c_ffi_process_context.rs index 161ef7546a..a887ab7130 100644 --- a/minifi_rust/minifi_native/src/c_ffi/c_ffi_process_context.rs +++ b/minifi_rust/minifi_native/src/c_ffi/c_ffi_process_context.rs @@ -39,7 +39,7 @@ unsafe extern "C" fn get_property_callback( } let value_slice = - std::slice::from_raw_parts(property_c_value.data as *const u8, property_c_value.length); + std::slice::from_raw_parts(property_c_value.data.cast::(), property_c_value.length); if let Ok(string_value) = String::from_utf8(value_slice.to_vec()) { *result_target = Some(string_value); } From f58cc911e6f3e2e02cde40eaf068fe13fc387980 Mon Sep 17 00:00:00 2001 From: Martin Zink Date: Wed, 1 Jul 2026 14:34:31 +0200 Subject: [PATCH 11/26] add Send for CffiInputStream/CffiOutputStream --- minifi_rust/minifi_native/src/api/process_session.rs | 8 ++++---- minifi_rust/minifi_native/src/c_ffi/c_ffi_streams.rs | 4 ++++ 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/minifi_rust/minifi_native/src/api/process_session.rs b/minifi_rust/minifi_native/src/api/process_session.rs index fe0171ba7a..f58dc9cc60 100644 --- a/minifi_rust/minifi_native/src/api/process_session.rs +++ b/minifi_rust/minifi_native/src/api/process_session.rs @@ -1,9 +1,9 @@ use crate::MinifiError; use crate::api::flow_file::FlowFile; -pub trait InputStream: std::io::BufRead + std::fmt::Debug {} -pub trait OutputStream: std::io::Write + std::fmt::Debug {} -impl OutputStream for T {} -impl InputStream for T {} +pub trait InputStream: std::io::BufRead + Send + std::fmt::Debug {} +pub trait OutputStream: std::io::Write + Send + std::fmt::Debug {} +impl OutputStream for T {} +impl InputStream for T {} #[derive(Debug, Clone, Copy, Eq, PartialEq)] pub enum IoState { diff --git a/minifi_rust/minifi_native/src/c_ffi/c_ffi_streams.rs b/minifi_rust/minifi_native/src/c_ffi/c_ffi_streams.rs index 82fd1ec19c..964653e75d 100644 --- a/minifi_rust/minifi_native/src/c_ffi/c_ffi_streams.rs +++ b/minifi_rust/minifi_native/src/c_ffi/c_ffi_streams.rs @@ -13,6 +13,8 @@ pub struct CffiInputStream<'a> { _marker: std::marker::PhantomData<&'a ()>, } +unsafe impl<'a> Send for CffiInputStream<'a> {} + impl<'a> CffiInputStream<'a> { pub fn new(ptr: *mut minifi_input_stream) -> Self { Self { @@ -91,6 +93,8 @@ impl<'a> CffiOutputStream<'a> { } } +unsafe impl<'a> Send for CffiOutputStream<'a> {} + impl<'a> std::io::Write for CffiOutputStream<'a> { fn write(&mut self, buf: &[u8]) -> std::io::Result { unsafe { From 840db4575af99bcc8e8050d963d03320024c75f2 Mon Sep 17 00:00:00 2001 From: Martin Zink Date: Wed, 1 Jul 2026 14:53:55 +0200 Subject: [PATCH 12/26] clippy fixes --- .../src/processors/get_file/tests.rs | 2 +- .../minifi_native/src/c_ffi/c_ffi_streams.rs | 10 ++++++++ .../minifi_native/src/mock/mock_logger.rs | 24 +++++++------------ .../src/mock/mock_process_session.rs | 8 ++++--- 4 files changed, 24 insertions(+), 20 deletions(-) diff --git a/minifi_rust/extensions/minifi_rs_playground/src/processors/get_file/tests.rs b/minifi_rust/extensions/minifi_rs_playground/src/processors/get_file/tests.rs index 143fadaf7f..d886ffc5c5 100644 --- a/minifi_rust/extensions/minifi_rs_playground/src/processors/get_file/tests.rs +++ b/minifi_rust/extensions/minifi_rs_playground/src/processors/get_file/tests.rs @@ -120,7 +120,7 @@ fn test_complex_dir_with_filter( .flow_file .attributes .get("filename") - .and_then(|filename| Some(filename.contains(expected_filename_part))) + .map(|filename| filename.contains(expected_filename_part)) .unwrap_or(false) })); let sum_file_len = transferred_flow_files diff --git a/minifi_rust/minifi_native/src/c_ffi/c_ffi_streams.rs b/minifi_rust/minifi_native/src/c_ffi/c_ffi_streams.rs index 964653e75d..f2923a93a7 100644 --- a/minifi_rust/minifi_native/src/c_ffi/c_ffi_streams.rs +++ b/minifi_rust/minifi_native/src/c_ffi/c_ffi_streams.rs @@ -13,6 +13,13 @@ pub struct CffiInputStream<'a> { _marker: std::marker::PhantomData<&'a ()>, } +// SAFETY: The wrapped `minifi_input_stream*` points at a +// `minifi::io::InputStream` on the C++ side. That interface has no +// thread-affine state (no `thread_local`, no owning-thread assertions in the +// concrete implementations — see +// `minifi-api/common/include/minifi-cpp/io/InputStream.h` and the impls under +// `libminifi/src/core/`). The stream also cannot escape the FFI callback it +// was constructed in: the `'a` lifetime is bound to that call frame. unsafe impl<'a> Send for CffiInputStream<'a> {} impl<'a> CffiInputStream<'a> { @@ -93,6 +100,9 @@ impl<'a> CffiOutputStream<'a> { } } +// SAFETY: Same reasoning as `CffiInputStream` above — the underlying +// `minifi::io::OutputStream` has no thread affinity, and the `'a` lifetime keeps +// the stream inside its FFI callback frame. unsafe impl<'a> Send for CffiOutputStream<'a> {} impl<'a> std::io::Write for CffiOutputStream<'a> { diff --git a/minifi_rust/minifi_native/src/mock/mock_logger.rs b/minifi_rust/minifi_native/src/mock/mock_logger.rs index a611c58378..73dbdd40fb 100644 --- a/minifi_rust/minifi_native/src/mock/mock_logger.rs +++ b/minifi_rust/minifi_native/src/mock/mock_logger.rs @@ -66,22 +66,14 @@ mod tests { let mut call_count = 0; - trace!( - mock_logger, - "This is a trace message {}", - || -> u32 { - call_count += 1; - call_count - }() - ); - error!( - mock_logger, - "This is an error message {}", - || -> u32 { - call_count += 1; - call_count - }() - ); + trace!(mock_logger, "This is a trace message {}", { + call_count += 1; + call_count + }); + error!(mock_logger, "This is an error message {}", { + call_count += 1; + call_count + }); assert_eq!(mock_logger.logs.lock().unwrap().len(), 1); assert_eq!(call_count, 1); diff --git a/minifi_rust/minifi_native/src/mock/mock_process_session.rs b/minifi_rust/minifi_native/src/mock/mock_process_session.rs index e8871d99eb..c3ba3fc546 100644 --- a/minifi_rust/minifi_native/src/mock/mock_process_session.rs +++ b/minifi_rust/minifi_native/src/mock/mock_process_session.rs @@ -156,11 +156,13 @@ mod tests { #[test] fn test_read_in_batches() { let session = MockProcessSession::new(); - let mut flow_file = MockFlowFile::new(); - flow_file.content = RefCell::from("Hello, World!".to_string().as_bytes().to_vec()); + let flow_file = MockFlowFile::new(); + flow_file + .content + .replace("Hello, World!".as_bytes().to_vec()); let mut vec: Vec = Vec::new(); - let res = session.read_in_batches(&mut flow_file, 1, |batch| { + let res = session.read_in_batches(&flow_file, 1, |batch| { assert_eq!(batch.len(), 1); vec.push(batch[0]); Ok(()) From ba6ad686c68901fc009ee98a02b6ebc4d12ca90c Mon Sep 17 00:00:00 2001 From: Martin Zink Date: Wed, 1 Jul 2026 14:56:12 +0200 Subject: [PATCH 13/26] ci clippy changes --- .github/workflows/ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e3db0a3eb6..db756244a7 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -631,13 +631,13 @@ jobs: - id: cargo_check name: Cargo check continue-on-error: true - run: cargo fmt --check + run: cargo fmt --all --check working-directory: minifi_rust - id: clippy_check name: Clippy check continue-on-error: true - run: cargo clippy + run: cargo clippy --workspace --all-targets -- -D warnings working-directory: minifi_rust - name: Check Linter Statuses From 6b3e659e00995551862815cf3a728ddffa339da3 Mon Sep 17 00:00:00 2001 From: Martin Zink Date: Mon, 6 Jul 2026 17:08:13 +0200 Subject: [PATCH 14/26] rebase fixes --- .../src/c_ffi/c_ffi_controller_service_definition.rs | 10 +++++----- .../src/c_ffi/c_ffi_controller_service_list.rs | 8 ++++---- .../src/c_ffi/c_ffi_processor_definition.rs | 8 ++++---- .../minifi_native/src/c_ffi/c_ffi_processor_list.rs | 8 ++++---- 4 files changed, 17 insertions(+), 17 deletions(-) diff --git a/minifi_rust/minifi_native/src/c_ffi/c_ffi_controller_service_definition.rs b/minifi_rust/minifi_native/src/c_ffi/c_ffi_controller_service_definition.rs index e8048c0de5..2c78f75b80 100644 --- a/minifi_rust/minifi_native/src/c_ffi/c_ffi_controller_service_definition.rs +++ b/minifi_rust/minifi_native/src/c_ffi/c_ffi_controller_service_definition.rs @@ -7,7 +7,7 @@ use crate::{ LogLevel, Property, ProvidedInterface, }; use minifi_native_sys::{ - minifi_controller_service_callbacks, minifi_controller_service_class_definition, + minifi_controller_service_callbacks, minifi_controller_service_definition, minifi_controller_service_context, minifi_controller_service_metadata, minifi_status, minifi_string_view, }; @@ -15,19 +15,19 @@ use std::ffi::c_void; #[derive(Debug)] pub struct ControllerServiceClassDefinition<'a> { - inner: minifi_controller_service_class_definition, + inner: minifi_controller_service_definition, _lifetime: std::marker::PhantomData<&'a ()>, } impl<'a> ControllerServiceClassDefinition<'a> { - pub(crate) fn new(inner: minifi_controller_service_class_definition) -> Self { + pub(crate) fn new(inner: minifi_controller_service_definition) -> Self { Self { inner, _lifetime: std::marker::PhantomData, } } - pub unsafe fn as_raw(&self) -> minifi_controller_service_class_definition { + pub unsafe fn as_raw(&self) -> minifi_controller_service_definition { self.inner } } @@ -152,7 +152,7 @@ where { fn class_description(&'_ self) -> ControllerServiceClassDefinition<'_> { unsafe { - ControllerServiceClassDefinition::new(minifi_controller_service_class_definition { + ControllerServiceClassDefinition::new(minifi_controller_service_definition { full_name: self.name.as_minifi_c_type(), description: self.description_text.as_minifi_c_type(), properties_count: self.c_properties.len(), diff --git a/minifi_rust/minifi_native/src/c_ffi/c_ffi_controller_service_list.rs b/minifi_rust/minifi_native/src/c_ffi/c_ffi_controller_service_list.rs index ced9461200..9b28676065 100644 --- a/minifi_rust/minifi_native/src/c_ffi/c_ffi_controller_service_list.rs +++ b/minifi_rust/minifi_native/src/c_ffi/c_ffi_controller_service_list.rs @@ -1,11 +1,11 @@ use crate::c_ffi::RegisterableControllerService; use crate::c_ffi::c_ffi_controller_service_definition::DynRawControllerServiceDefinition; -use minifi_native_sys::minifi_controller_service_class_definition; +use minifi_native_sys::minifi_controller_service_definition; pub struct CffiControllerServiceList { controller_service_definitions: Vec>, minifi_controller_service_class_description_list: - Vec, + Vec, } impl Default for CffiControllerServiceList { @@ -53,11 +53,11 @@ impl CffiControllerServiceList { /// # Safety /// - /// The returned *minifi_controller_service_class_definition only valid until self lives + /// The returned *minifi_controller_service_definition only valid until self lives /// TODO(mzink) maybe some lifetimes? pub unsafe fn get_controller_service_ptr( &self, - ) -> *const minifi_controller_service_class_definition { + ) -> *const minifi_controller_service_definition { self.minifi_controller_service_class_description_list .as_ptr() } diff --git a/minifi_rust/minifi_native/src/c_ffi/c_ffi_processor_definition.rs b/minifi_rust/minifi_native/src/c_ffi/c_ffi_processor_definition.rs index 9393944abd..92eba14392 100644 --- a/minifi_rust/minifi_native/src/c_ffi/c_ffi_processor_definition.rs +++ b/minifi_rust/minifi_native/src/c_ffi/c_ffi_processor_definition.rs @@ -184,19 +184,19 @@ where #[derive(Debug)] pub struct ProcessorClassDefinition<'a> { - inner: minifi_processor_class_definition, + inner: minifi_processor_definition, _marker: std::marker::PhantomData<&'a ()>, } impl<'a> ProcessorClassDefinition<'a> { - pub(crate) fn new(inner: minifi_processor_class_definition) -> Self { + pub(crate) fn new(inner: minifi_processor_definition) -> Self { Self { inner, _marker: std::marker::PhantomData, } } - pub unsafe fn as_raw(&self) -> minifi_processor_class_definition { + pub unsafe fn as_raw(&self) -> minifi_processor_definition { self.inner } } @@ -211,7 +211,7 @@ where { fn class_description(&'_ self) -> ProcessorClassDefinition<'_> { unsafe { - ProcessorClassDefinition::new(minifi_processor_class_definition { + ProcessorClassDefinition::new(minifi_processor_definition { full_name: self.name.as_minifi_c_type(), description: self.description_text.as_minifi_c_type(), properties_count: self.c_properties.len(), diff --git a/minifi_rust/minifi_native/src/c_ffi/c_ffi_processor_list.rs b/minifi_rust/minifi_native/src/c_ffi/c_ffi_processor_list.rs index fddd9c8189..7efea7d524 100644 --- a/minifi_rust/minifi_native/src/c_ffi/c_ffi_processor_list.rs +++ b/minifi_rust/minifi_native/src/c_ffi/c_ffi_processor_list.rs @@ -1,10 +1,10 @@ use crate::c_ffi::RawRegisterableProcessor; use crate::c_ffi::c_ffi_processor_definition::DynRawProcessorDefinition; -use minifi_native_sys::minifi_processor_class_definition; +use minifi_native_sys::minifi_processor_definition; pub struct CffiProcessorList { processor_definitions: Vec>, - minifi_processor_class_description_list: Vec, + minifi_processor_class_description_list: Vec, } impl Default for CffiProcessorList { @@ -51,9 +51,9 @@ impl CffiProcessorList { /// # Safety /// - /// The returned *minifi_processor_class_definition only valid until self lives + /// The returned *minifi_processor_definition only valid until self lives /// TODO(mzink) maybe some lifetimes? - pub unsafe fn get_processor_ptr(&self) -> *const minifi_processor_class_definition { + pub unsafe fn get_processor_ptr(&self) -> *const minifi_processor_definition { self.minifi_processor_class_description_list.as_ptr() } } From f33babc0d90c0d2d3961d22ecc899b1e869287ad Mon Sep 17 00:00:00 2001 From: Martin Zink Date: Mon, 6 Jul 2026 17:29:29 +0200 Subject: [PATCH 15/26] update manifest --- .../ubuntu_22_04_clang_arm_manifest.json | 648 +++++++++++++++++- 1 file changed, 637 insertions(+), 11 deletions(-) diff --git a/.github/references/ubuntu_22_04_clang_arm_manifest.json b/.github/references/ubuntu_22_04_clang_arm_manifest.json index bcb495d3e9..7d0e17c3f6 100644 --- a/.github/references/ubuntu_22_04_clang_arm_manifest.json +++ b/.github/references/ubuntu_22_04_clang_arm_manifest.json @@ -86,6 +86,24 @@ "supportsDynamicProperties": "false", "type": "org.apache.nifi.minifi.processors.ApplyTemplate" }, + { + "inputRequirement": "INPUT_REQUIRED", + "isSingleThreaded": "false", + "supportedRelationships": [ + { + "name": "failure", + "description": "Non-german flowfiles are routed here" + }, + { + "name": "success", + "description": "All asciified flowfiles are routed here" + } + ], + "typeDescription": "This processor switches German characters with their ascii counterparts. (to test stream API)", + "supportsDynamicRelationships": "false", + "supportsDynamicProperties": "false", + "type": "minifi_rs_playground.processors.asciify_german.AsciifyGerman" + }, { "propertyDescriptors": { "Attribute name prefix": { @@ -1132,6 +1150,15 @@ "supportsDynamicProperties": "false", "type": "org.apache.nifi.minifi.processors.ConvertRecord" }, + { + "inputRequirement": "INPUT_FORBIDDEN", + "isSingleThreaded": "true", + "supportedRelationships": [], + "typeDescription": "For testing lazy logging", + "supportsDynamicRelationships": "false", + "supportsDynamicProperties": "false", + "type": "minifi_rs_playground.processors.count_actual_logging.CountActualLogging" + }, { "propertyDescriptors": { "Max Buffer Age": { @@ -1870,6 +1897,20 @@ "supportsDynamicProperties": "true", "type": "org.apache.nifi.minifi.aws.processors.DeleteS3Object" }, + { + "inputRequirement": "INPUT_REQUIRED", + "isSingleThreaded": "true", + "supportedRelationships": [ + { + "name": "success", + "description": {} + } + ], + "typeDescription": "Duplicate text", + "supportsDynamicRelationships": "false", + "supportsDynamicProperties": "false", + "type": "minifi_rs_playground.processors.duplicate_text.DuplicateStreamText" + }, { "propertyDescriptors": { "Destination": { @@ -3707,6 +3748,76 @@ "supportsDynamicProperties": "false", "type": "org.apache.nifi.minifi.processors.GenerateFlowFile" }, + { + "propertyDescriptors": { + "Batch Size": { + "name": "Batch Size", + "description": "The number of FlowFiles to be transferred in each invocation", + "validator": "NON_NEGATIVE_INTEGER_VALIDATOR", + "required": "true", + "sensitive": "false", + "expressionLanguageScope": "NONE", + "defaultValue": "1" + }, + "Custom Text": { + "name": "Custom Text", + "description": "If Data Format is text and if Unique FlowFiles is false, then this custom text will be used as content of the generated FlowFiles and the File Size will be ignored. Finally, if Expression Language is used, evaluation will be performed only once per batch of generated FlowFiles", + "validator": "VALID", + "required": "false", + "sensitive": "false", + "expressionLanguageScope": "FLOWFILE_ATTRIBUTES" + }, + "Data Format": { + "name": "Data Format", + "description": "Specifies whether the data should be Text or Binary", + "validator": "VALID", + "required": "true", + "sensitive": "false", + "expressionLanguageScope": "NONE", + "defaultValue": "Binary", + "allowableValues": [ + { + "value": "Text", + "displayName": "Text" + }, + { + "value": "Binary", + "displayName": "Binary" + } + ] + }, + "File Size": { + "name": "File Size", + "description": "The size of the file that will be used", + "validator": "DATA_SIZE_VALIDATOR", + "required": "true", + "sensitive": "false", + "expressionLanguageScope": "FLOWFILE_ATTRIBUTES", + "defaultValue": "1 kB" + }, + "Unique FlowFiles": { + "name": "Unique FlowFiles", + "description": "If true, each FlowFile that is generated will be unique. If false, a random value will be generated and all FlowFiles will get the same content but this offers much higher throughput (but see the description of Custom Text for special non-random use cases)", + "validator": "BOOLEAN_VALIDATOR", + "required": "true", + "sensitive": "false", + "expressionLanguageScope": "NONE", + "defaultValue": "true" + } + }, + "inputRequirement": "INPUT_FORBIDDEN", + "isSingleThreaded": "false", + "supportedRelationships": [ + { + "name": "success", + "description": "success" + } + ], + "typeDescription": "This processor creates FlowFiles with random data or custom content. GenerateFlowFile is useful for load testing, configuration, and simulation.", + "supportsDynamicRelationships": "false", + "supportsDynamicProperties": "false", + "type": "minifi_rs_playground.processors.generate_flow_file.GenerateFlowFileRs" + }, { "propertyDescriptors": { "Bucket Name": { @@ -3926,6 +4037,106 @@ "supportsDynamicProperties": "false", "type": "org.apache.nifi.minifi.processors.GetFile" }, + { + "propertyDescriptors": { + "Batch Size": { + "name": "Batch Size", + "description": "The maximum number of files to pull in each iteration", + "validator": "NON_NEGATIVE_INTEGER_VALIDATOR", + "required": "true", + "sensitive": "false", + "expressionLanguageScope": "NONE", + "defaultValue": "10" + }, + "Ignore Hidden Files": { + "name": "Ignore Hidden Files", + "description": "Indicates whether or not hidden files should be ignored", + "validator": "BOOLEAN_VALIDATOR", + "required": "true", + "sensitive": "false", + "expressionLanguageScope": "NONE", + "defaultValue": "true" + }, + "Input Directory": { + "name": "Input Directory", + "description": "The input directory from which to pull files", + "validator": "NON_BLANK_VALIDATOR", + "required": "true", + "sensitive": "false", + "expressionLanguageScope": "FLOWFILE_ATTRIBUTES" + }, + "Keep Source File": { + "name": "Keep Source File", + "description": "If true, the file is not deleted after it has been copied to the Content Repository", + "validator": "BOOLEAN_VALIDATOR", + "required": "false", + "sensitive": "false", + "expressionLanguageScope": "NONE", + "defaultValue": "false" + }, + "Maximum File Age": { + "name": "Maximum File Age", + "description": "The maximum age that a file must be in order to be pulled; any file older than this amount of time (according to last modification date) will be ignored", + "validator": "TIME_PERIOD_VALIDATOR", + "required": "false", + "sensitive": "false", + "expressionLanguageScope": "NONE" + }, + "Maximum File Size": { + "name": "Maximum File Size", + "description": "The maximum size that a file can be in order to be pulled", + "validator": "DATA_SIZE_VALIDATOR", + "required": "false", + "sensitive": "false", + "expressionLanguageScope": "NONE" + }, + "Minimum File Age": { + "name": "Minimum File Age", + "description": "The minimum age that a file must be in order to be pulled; any file younger than this amount of time (according to last modification date) will be ignored", + "validator": "TIME_PERIOD_VALIDATOR", + "required": "false", + "sensitive": "false", + "expressionLanguageScope": "NONE" + }, + "Minimum File Size": { + "name": "Minimum File Size", + "description": "The minimum size that a file can be in order to be pulled", + "validator": "DATA_SIZE_VALIDATOR", + "required": "false", + "sensitive": "false", + "expressionLanguageScope": "NONE" + }, + "Polling Interval": { + "name": "Polling Interval", + "description": "Indicates how long to wait before performing a directory listing", + "validator": "TIME_PERIOD_VALIDATOR", + "required": "false", + "sensitive": "false", + "expressionLanguageScope": "NONE" + }, + "Recurse Subdirectories": { + "name": "Recurse Subdirectories", + "description": "Indicates whether or not to pull files from subdirectories", + "validator": "BOOLEAN_VALIDATOR", + "required": "false", + "sensitive": "false", + "expressionLanguageScope": "NONE", + "defaultValue": "true" + } + }, + "inputRequirement": "INPUT_FORBIDDEN", + "isSingleThreaded": "false", + "supportedRelationships": [ + { + "name": "success", + "description": "FlowFiles are transferred here after logging" + } + ], + "typeDescription": "Creates FlowFiles from files in a directory. MiNiFi will ignore files for which it doesn't have read permissions.", + "supportsDynamicRelationships": "false", + "supportsDynamicProperties": "false", + "type": "minifi_rs_playground.processors.get_file.GetFileRs" + }, { "propertyDescriptors": { "Endpoint List": { @@ -4426,6 +4637,84 @@ "supportsDynamicProperties": "false", "type": "org.apache.nifi.minifi.processors.KamikazeProcessor" }, + { + "propertyDescriptors": { + "Schedule Behaviour": { + "name": "Schedule Behaviour", + "description": "What to do during the on_schedule method", + "validator": "VALID", + "required": "true", + "sensitive": "false", + "expressionLanguageScope": "NONE", + "defaultValue": "ReturnOk", + "allowableValues": [ + { + "value": "ReturnErr", + "displayName": "ReturnErr" + }, + { + "value": "ReturnOk", + "displayName": "ReturnOk" + }, + { + "value": "GetNotRegisteredProperty", + "displayName": "GetNotRegisteredProperty" + }, + { + "value": "GetInvalidControllerService", + "displayName": "GetInvalidControllerService" + }, + { + "value": "Panic", + "displayName": "Panic" + } + ] + }, + "Trigger Behaviour": { + "name": "Trigger Behaviour", + "description": "What to do during the trigger method", + "validator": "VALID", + "required": "true", + "sensitive": "false", + "expressionLanguageScope": "NONE", + "defaultValue": "ReturnOk", + "allowableValues": [ + { + "value": "ReturnErr", + "displayName": "ReturnErr" + }, + { + "value": "ReturnOk", + "displayName": "ReturnOk" + }, + { + "value": "GetNotRegisteredProperty", + "displayName": "GetNotRegisteredProperty" + }, + { + "value": "GetInvalidControllerService", + "displayName": "GetInvalidControllerService" + }, + { + "value": "Panic", + "displayName": "Panic" + } + ] + } + }, + "inputRequirement": "INPUT_ALLOWED", + "isSingleThreaded": "false", + "supportedRelationships": [ + { + "name": "success", + "description": "success relationship" + } + ], + "typeDescription": "This processor can fail or panic in on_trigger and on_schedule calls based on configuration. Only for testing purposes.", + "supportsDynamicRelationships": "false", + "supportsDynamicProperties": "false", + "type": "minifi_rs_playground.processors.kamikaze_processor.KamikazeProcessorRs" + }, { "propertyDescriptors": { "Azure Storage Credentials Service": { @@ -6035,19 +6324,173 @@ "supportsDynamicProperties": "false", "type": "org.apache.nifi.minifi.processors.LogAttribute" }, - { - "inputRequirement": "INPUT_ALLOWED", - "isSingleThreaded": "false", - "supportedRelationships": [], - "typeDescription": "This processor logs a message on destruction. Only for testing purposes.", - "supportsDynamicRelationships": "false", - "supportsDynamicProperties": "false", - "type": "org.apache.nifi.minifi.processors.LogOnDestructionProcessor" - }, { "propertyDescriptors": { - "After": { - "name": "After", + "Attributes to Ignore": { + "name": "Attributes to Ignore", + "description": "A comma-separated list of Attributes to ignore. If not specified, no attributes will be ignored.", + "validator": "VALID", + "required": "false", + "sensitive": "false", + "expressionLanguageScope": "NONE" + }, + "Attributes to Log": { + "name": "Attributes to Log", + "description": "A comma-separated list of Attributes to Log. If not specified, all attributes will be logged.", + "validator": "VALID", + "required": "false", + "sensitive": "false", + "expressionLanguageScope": "NONE" + }, + "FlowFiles To Log": { + "name": "FlowFiles To Log", + "description": "Number of flow files to log. If set to zero all flow files will be logged. Please note that this may block other threads from running if not used judiciously.", + "validator": "VALID", + "required": "true", + "sensitive": "false", + "expressionLanguageScope": "NONE", + "defaultValue": "1" + }, + "Hexencode Payload": { + "name": "Hexencode Payload", + "description": "If true, the FlowFile's payload will be logged in a hexencoded format", + "validator": "BOOLEAN_VALIDATOR", + "required": "true", + "sensitive": "false", + "expressionLanguageScope": "NONE", + "defaultValue": "false" + }, + "Log Level": { + "name": "Log Level", + "description": "The Log Level to use when logging the Attributes", + "validator": "VALID", + "required": "true", + "sensitive": "false", + "expressionLanguageScope": "NONE", + "defaultValue": "Info", + "allowableValues": [ + { + "value": "Trace", + "displayName": "Trace" + }, + { + "value": "Debug", + "displayName": "Debug" + }, + { + "value": "Info", + "displayName": "Info" + }, + { + "value": "Warn", + "displayName": "Warn" + }, + { + "value": "Error", + "displayName": "Error" + }, + { + "value": "Critical", + "displayName": "Critical" + }, + { + "value": "Off", + "displayName": "Off" + } + ] + }, + "Log Payload": { + "name": "Log Payload", + "description": "If true, the FlowFile's payload will be logged, in addition to its attributes. Otherwise, just the Attributes will be logged.", + "validator": "BOOLEAN_VALIDATOR", + "required": "true", + "sensitive": "false", + "expressionLanguageScope": "NONE", + "defaultValue": "false" + }, + "Log Prefix": { + "name": "Log Prefix", + "description": "Log prefix appended to the log lines. It helps to distinguish the output of multiple LogAttribute processors.", + "validator": "VALID", + "required": "false", + "sensitive": "false", + "expressionLanguageScope": "NONE" + } + }, + "inputRequirement": "INPUT_REQUIRED", + "isSingleThreaded": "false", + "supportedRelationships": [ + { + "name": "success", + "description": "FlowFiles are transferred here after logging" + } + ], + "typeDescription": "Logs attributes of flow files in the MiNiFi application log.", + "supportsDynamicRelationships": "false", + "supportsDynamicProperties": "false", + "type": "minifi_rs_playground.processors.log_attribute.LogAttributeRs" + }, + { + "inputRequirement": "INPUT_ALLOWED", + "isSingleThreaded": "false", + "supportedRelationships": [], + "typeDescription": "This processor logs a message on destruction. Only for testing purposes.", + "supportsDynamicRelationships": "false", + "supportsDynamicProperties": "false", + "type": "org.apache.nifi.minifi.processors.LogOnDestructionProcessor" + }, + { + "propertyDescriptors": { + "Lorem Ipsum Controller Service": { + "typeProvidedByValue": { + "type": "minifi_rs_playground.controller_services.lorem_ipsum_controller_service.LoremIpsumControllerService", + "group": "minifi_rs_playground.controller_services.lorem_ipsum_controller_service", + "artifact": "minifi_rs_playground" + }, + "name": "Lorem Ipsum Controller Service", + "description": "Name of the lorem ipsum controller service", + "validator": "VALID", + "required": "true", + "sensitive": "false", + "expressionLanguageScope": "NONE" + }, + "Write Method": { + "name": "Write Method", + "description": "Which API to test", + "validator": "VALID", + "required": "true", + "sensitive": "false", + "expressionLanguageScope": "NONE", + "defaultValue": "Buffer", + "allowableValues": [ + { + "value": "Buffer", + "displayName": "Buffer" + }, + { + "value": "Stream", + "displayName": "Stream" + } + ] + } + }, + "inputRequirement": "INPUT_FORBIDDEN", + "isSingleThreaded": "false", + "supportedRelationships": [ + { + "name": "success", + "description": "All flowfile are routed here" + } + ], + "typeDescription": "Processor to test Controller Service API", + "supportsDynamicRelationships": "false", + "supportsDynamicProperties": "false", + "type": "minifi_rs_playground.processors.lorem_ipsum_cs_user.LoremIpsumCSUser" + }, + { + "propertyDescriptors": { + "After": { + "name": "After", "description": "For operations which result in new entries, places the new entry after the entry specified by this property.", "validator": "VALID", "required": "false", @@ -7831,6 +8274,91 @@ "supportsDynamicProperties": "false", "type": "org.apache.nifi.minifi.processors.PutFile" }, + { + "propertyDescriptors": { + "Conflict Resolution Strategy": { + "name": "Conflict Resolution Strategy", + "description": "Indicates what should happen when a file with the same name already exists in the output directory", + "validator": "VALID", + "required": "true", + "sensitive": "false", + "expressionLanguageScope": "NONE", + "defaultValue": "fail", + "allowableValues": [ + { + "value": "fail", + "displayName": "fail" + }, + { + "value": "replace", + "displayName": "replace" + }, + { + "value": "ignore", + "displayName": "ignore" + } + ] + }, + "Create Missing Directories": { + "name": "Create Missing Directories", + "description": "If true, then missing destination directories will be created. If false, flowfiles are penalized and sent to failure.", + "validator": "BOOLEAN_VALIDATOR", + "required": "true", + "sensitive": "false", + "expressionLanguageScope": "NONE", + "defaultValue": "true" + }, + "Directory": { + "name": "Directory", + "description": "The output directory to which to put files", + "validator": "NON_BLANK_VALIDATOR", + "required": "true", + "sensitive": "false", + "expressionLanguageScope": "FLOWFILE_ATTRIBUTES", + "defaultValue": "." + }, + "Directory Permissions": { + "name": "Directory Permissions", + "description": "Sets the permissions on the directories being created if 'Create Missing Directories' property is set. Must be an octal number (e.g. 644 or 0755). Not supported on Windows systems.", + "validator": "VALID", + "required": "false", + "sensitive": "false", + "expressionLanguageScope": "NONE" + }, + "Maximum File Count": { + "name": "Maximum File Count", + "description": "Specifies the maximum number of files that can exist in the output directory", + "validator": "NON_NEGATIVE_INTEGER_VALIDATOR", + "required": "false", + "sensitive": "false", + "expressionLanguageScope": "NONE" + }, + "Permissions": { + "name": "Permissions", + "description": "Sets the permissions on the output file to the value of this attribute. Must be an octal number (e.g. 644 or 0755). Not supported on Windows systems.", + "validator": "VALID", + "required": "false", + "sensitive": "false", + "expressionLanguageScope": "NONE" + } + }, + "inputRequirement": "INPUT_REQUIRED", + "isSingleThreaded": "false", + "supportedRelationships": [ + { + "name": "failure", + "description": "Failed files (conflict, write failure, etc.) are transferred to failure" + }, + { + "name": "success", + "description": "Flowfiles that are successfully written to a file are routed to this relationship" + } + ], + "typeDescription": "Writes the contents of a FlowFile to the local file system.", + "supportsDynamicRelationships": "false", + "supportsDynamicProperties": "false", + "type": "minifi_rs_playground.processors.put_file.PutFileRs" + }, { "propertyDescriptors": { "Bucket": { @@ -10912,6 +11440,43 @@ "supportsDynamicRelationships": "false", "supportsDynamicProperties": "false", "type": "org.apache.nifi.minifi.api_testing.ZooProcessor" + }, + { + "propertyDescriptors": { + "Can fly service": { + "typeProvidedByValue": { + "type": "minifi_rs_playground.controller_services.animal_controller_apis.CanFlyControllerApi", + "group": "minifi_rs_playground.controller_services.animal_controller_apis", + "artifact": "minifi_rs_playground" + }, + "name": "Can fly service", + "description": "Test CanFlyService", + "validator": "VALID", + "required": "true", + "sensitive": "false", + "expressionLanguageScope": "NONE" + }, + "Number of Legs service": { + "typeProvidedByValue": { + "type": "minifi_rs_playground.controller_services.animal_controller_apis.NumberOfLegsControllerApi", + "group": "minifi_rs_playground.controller_services.animal_controller_apis", + "artifact": "minifi_rs_playground" + }, + "name": "Number of Legs service", + "description": "Test NumberOfLegsService", + "validator": "VALID", + "required": "true", + "sensitive": "false", + "expressionLanguageScope": "NONE" + } + }, + "inputRequirement": "INPUT_FORBIDDEN", + "isSingleThreaded": "false", + "supportedRelationships": [], + "typeDescription": "Test ZooProcessorRs", + "supportsDynamicRelationships": "false", + "supportsDynamicProperties": "false", + "type": "minifi_rs_playground.processors.zoo_processor.ZooProcessorRs" } ], "controllerServices": [ @@ -11100,6 +11665,43 @@ } ] }, + { + "propertyDescriptors": { + "Extra information": { + "name": "Extra information", + "description": "We need this to verify the casting was done correctly", + "validator": "VALID", + "required": "false", + "sensitive": "false", + "expressionLanguageScope": "NONE" + }, + "Has Jetpack": { + "name": "Has Jetpack", + "description": "Whether or not the dog has a jetpack", + "validator": "BOOLEAN_VALIDATOR", + "required": "true", + "sensitive": "false", + "expressionLanguageScope": "NONE", + "defaultValue": "false" + } + }, + "typeDescription": "Test DogControllerRs", + "supportsDynamicRelationships": "false", + "supportsDynamicProperties": "false", + "type": "minifi_rs_playground.controller_services.dog_controller_service.DogControllerRs", + "providedApiImplementations": [ + { + "artifact": "minifi_rs_playground", + "group": "minifi_rs_playground.controller_services.animal_controller_apis", + "type": "minifi_rs_playground.controller_services.animal_controller_apis.CanFlyControllerApi" + }, + { + "artifact": "minifi_rs_playground", + "group": "minifi_rs_playground.controller_services.animal_controller_apis", + "type": "minifi_rs_playground.controller_services.animal_controller_apis.NumberOfLegsControllerApi" + } + ] + }, { "typeDescription": "Test DuckController", "supportsDynamicRelationships": "false", @@ -11118,6 +11720,30 @@ } ] }, + { + "typeDescription": "Test DuckControllerRs", + "supportsDynamicRelationships": "false", + "supportsDynamicProperties": "false", + "type": "minifi_rs_playground.controller_services.duck_controller_service.DuckControllerRs", + "providedApiImplementations": [ + { + "artifact": "minifi_rs_playground", + "group": "minifi_rs_playground.controller_services.animal_controller_apis", + "type": "minifi_rs_playground.controller_services.animal_controller_apis.CanFlyControllerApi" + }, + { + "artifact": "minifi_rs_playground", + "group": "minifi_rs_playground.controller_services.animal_controller_apis", + "type": "minifi_rs_playground.controller_services.animal_controller_apis.NumberOfLegsControllerApi" + } + ] + }, + { + "typeDescription": "Dummy Controller Service", + "supportsDynamicRelationships": "false", + "supportsDynamicProperties": "false", + "type": "minifi_rs_playground.controller_services.dummy_controller_service.DummyControllerService" + }, { "propertyDescriptors": { "API Key": { From b0f7d6b9e3e3a440a36a315469d02b8248e6d443 Mon Sep 17 00:00:00 2001 From: Martin Zink Date: Tue, 7 Jul 2026 10:24:21 +0200 Subject: [PATCH 16/26] c23 update --- minifi_rust/minifi_native_sys/build.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/minifi_rust/minifi_native_sys/build.rs b/minifi_rust/minifi_native_sys/build.rs index 5abcc9a22f..a031131dda 100644 --- a/minifi_rust/minifi_native_sys/build.rs +++ b/minifi_rust/minifi_native_sys/build.rs @@ -211,6 +211,7 @@ fn main() { let bindings = bindgen::Builder::default() .header(sdk.header_path.to_str().unwrap()) + .clang_arg("-std=c23") .parse_callbacks(Box::new(bindgen::CargoCallbacks::new())) .generate() .expect("Unable to generate bindings"); From ad0bec5125249ac9b019259524aa4c29dd19d14f Mon Sep 17 00:00:00 2001 From: Martin Zink Date: Tue, 7 Jul 2026 10:24:55 +0200 Subject: [PATCH 17/26] cargo fmt --- .../src/c_ffi/c_ffi_controller_service_definition.rs | 4 ++-- .../src/c_ffi/c_ffi_controller_service_list.rs | 7 ++----- 2 files changed, 4 insertions(+), 7 deletions(-) diff --git a/minifi_rust/minifi_native/src/c_ffi/c_ffi_controller_service_definition.rs b/minifi_rust/minifi_native/src/c_ffi/c_ffi_controller_service_definition.rs index 2c78f75b80..5173959cf2 100644 --- a/minifi_rust/minifi_native/src/c_ffi/c_ffi_controller_service_definition.rs +++ b/minifi_rust/minifi_native/src/c_ffi/c_ffi_controller_service_definition.rs @@ -7,8 +7,8 @@ use crate::{ LogLevel, Property, ProvidedInterface, }; use minifi_native_sys::{ - minifi_controller_service_callbacks, minifi_controller_service_definition, - minifi_controller_service_context, minifi_controller_service_metadata, minifi_status, + minifi_controller_service_callbacks, minifi_controller_service_context, + minifi_controller_service_definition, minifi_controller_service_metadata, minifi_status, minifi_string_view, }; use std::ffi::c_void; diff --git a/minifi_rust/minifi_native/src/c_ffi/c_ffi_controller_service_list.rs b/minifi_rust/minifi_native/src/c_ffi/c_ffi_controller_service_list.rs index 9b28676065..1a8a582b63 100644 --- a/minifi_rust/minifi_native/src/c_ffi/c_ffi_controller_service_list.rs +++ b/minifi_rust/minifi_native/src/c_ffi/c_ffi_controller_service_list.rs @@ -4,8 +4,7 @@ use minifi_native_sys::minifi_controller_service_definition; pub struct CffiControllerServiceList { controller_service_definitions: Vec>, - minifi_controller_service_class_description_list: - Vec, + minifi_controller_service_class_description_list: Vec, } impl Default for CffiControllerServiceList { @@ -55,9 +54,7 @@ impl CffiControllerServiceList { /// /// The returned *minifi_controller_service_definition only valid until self lives /// TODO(mzink) maybe some lifetimes? - pub unsafe fn get_controller_service_ptr( - &self, - ) -> *const minifi_controller_service_definition { + pub unsafe fn get_controller_service_ptr(&self) -> *const minifi_controller_service_definition { self.minifi_controller_service_class_description_list .as_ptr() } From ebcbd63e4c7b188625dc5f37de0069641a4bc37f Mon Sep 17 00:00:00 2001 From: Martin Zink Date: Tue, 7 Jul 2026 10:27:08 +0200 Subject: [PATCH 18/26] update manifest --- .../ubuntu_22_04_clang_arm_manifest.json | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/.github/references/ubuntu_22_04_clang_arm_manifest.json b/.github/references/ubuntu_22_04_clang_arm_manifest.json index 7d0e17c3f6..7e2a935edf 100644 --- a/.github/references/ubuntu_22_04_clang_arm_manifest.json +++ b/.github/references/ubuntu_22_04_clang_arm_manifest.json @@ -11927,6 +11927,23 @@ } ] }, + { + "propertyDescriptors": { + "Length": { + "name": "Length", + "description": "How many words to generate", + "validator": "NON_NEGATIVE_INTEGER_VALIDATOR", + "required": "true", + "sensitive": "false", + "expressionLanguageScope": "NONE", + "defaultValue": "25" + } + }, + "typeDescription": "Simple Rusty Controller Service to test API", + "supportsDynamicRelationships": "false", + "supportsDynamicProperties": "false", + "type": "minifi_rs_playground.controller_services.lorem_ipsum_controller_service.LoremIpsumControllerService" + }, { "propertyDescriptors": { "Default Prioritizer": { From 0dddb32114bbf71aa716d6a276eedb1c8dd47ca9 Mon Sep 17 00:00:00 2001 From: Martin Zink Date: Tue, 7 Jul 2026 10:47:45 +0200 Subject: [PATCH 19/26] support c2x --- minifi_rust/minifi_native_sys/build.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/minifi_rust/minifi_native_sys/build.rs b/minifi_rust/minifi_native_sys/build.rs index a031131dda..51435cfa4e 100644 --- a/minifi_rust/minifi_native_sys/build.rs +++ b/minifi_rust/minifi_native_sys/build.rs @@ -211,7 +211,7 @@ fn main() { let bindings = bindgen::Builder::default() .header(sdk.header_path.to_str().unwrap()) - .clang_arg("-std=c23") + .clang_arg("-std=c2x") .parse_callbacks(Box::new(bindgen::CargoCallbacks::new())) .generate() .expect("Unable to generate bindings"); From e8a2505dc7715020a24be14b0d7f2ea930d53c96 Mon Sep 17 00:00:00 2001 From: Martin Zink Date: Tue, 7 Jul 2026 16:24:46 +0200 Subject: [PATCH 20/26] tag docker image without explicit version --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index db756244a7..62c4eef1ed 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -576,7 +576,7 @@ jobs: path: build - name: Load Docker image run: | - docker load --input ./build/minifi_docker.tar && docker tag apacheminificpp:1.0.0 apacheminificpp:behave + docker load --input ./build/minifi_docker.tar | awk '{print $NF}' | xargs -I {} docker tag {} apacheminificpp:behave - id: install_deps name: Install dependencies for Docker Verify run: | From 7b2807bed28c2ada97f9479c9f3abb2f658c83ff Mon Sep 17 00:00:00 2001 From: Martin Zink Date: Tue, 7 Jul 2026 16:28:58 +0200 Subject: [PATCH 21/26] cargo_check -> cargo_fmt_check --- .github/workflows/ci.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 62c4eef1ed..c33fff8919 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -603,7 +603,7 @@ jobs: name: minifi_rs_behave path: minifi_rust/minifi_rs_behave/output linters: - name: "C++ lint + Shellcheck + Flake8 + Cargo check" + name: "C++ lint + Shellcheck + Flake8 + Cargo fmt check + Clippy check" runs-on: ubuntu-22.04-arm timeout-minutes: 15 steps: @@ -628,8 +628,8 @@ jobs: continue-on-error: true run: ./run_flake8.sh . - - id: cargo_check - name: Cargo check + - id: cargo_fmt_check + name: Cargo fmt check continue-on-error: true run: cargo fmt --all --check working-directory: minifi_rust @@ -641,7 +641,7 @@ jobs: working-directory: minifi_rust - name: Check Linter Statuses - if: steps.cpp_lint.outcome == 'failure' || steps.shellcheck.outcome == 'failure' || steps.flake8_check.outcome == 'failure' || steps.cargo_check.outcome == 'failure' || steps.clippy_check.outcome == 'failure' + if: steps.cpp_lint.outcome == 'failure' || steps.shellcheck.outcome == 'failure' || steps.flake8_check.outcome == 'failure' || steps.cargo_fmt_check.outcome == 'failure' || steps.clippy_check.outcome == 'failure' run: | echo "One or more linters failed. Failing the workflow." exit 1 From d99eaa679e6e3356d830fdb3911d7ce4e853d7f7 Mon Sep 17 00:00:00 2001 From: Martin Zink Date: Mon, 20 Jul 2026 10:36:07 +0200 Subject: [PATCH 22/26] Renames: Concurrent -> MultiThreaded, Exclusive -> SingleThreaded --- minifi_rust/README.md | 4 ++-- .../minifi_rs_playground/src/lib.rs | 24 +++++++++---------- .../processor_wrappers/complex_processor.rs | 8 +++---- .../processor_wrappers/flow_file_source.rs | 8 +++---- .../flow_file_stream_transform.rs | 12 ++++------ .../processor_wrappers/flow_file_transform.rs | 8 +++---- .../minifi_native/src/api/raw_processor.rs | 16 ++++++------- .../src/c_ffi/c_ffi_processor_definition.rs | 8 +++---- minifi_rust/minifi_native/src/lib.rs | 2 +- 9 files changed, 44 insertions(+), 46 deletions(-) diff --git a/minifi_rust/README.md b/minifi_rust/README.md index 950f9153b0..6defc2c4d8 100644 --- a/minifi_rust/README.md +++ b/minifi_rust/README.md @@ -48,8 +48,8 @@ Building an extension is straightforward. The framework provides a declare_minif ```rust declare_minifi_extension!( processors: [ - (FlowFileSourceProcessorType, Concurrent, MyFlowFileSource), - (FlowFileTransformProcessorType, Exclusive, MyDataTransformer), + (FlowFileSourceProcessorType, MultiThreaded, MyFlowFileSource), + (FlowFileTransformProcessorType, SingleThreaded, MyDataTransformer), ], controllers: [ MyCustomControllerService, diff --git a/minifi_rust/extensions/minifi_rs_playground/src/lib.rs b/minifi_rust/extensions/minifi_rs_playground/src/lib.rs index b73087b499..53527a37ab 100644 --- a/minifi_rust/extensions/minifi_rs_playground/src/lib.rs +++ b/minifi_rust/extensions/minifi_rs_playground/src/lib.rs @@ -17,22 +17,22 @@ use crate::processors::put_file::PutFileRs; use crate::processors::zoo_processor::ZooProcessorRs; use minifi_native::{ - ComplexProcessorType, Concurrent, Exclusive, FlowFileSourceProcessorType, - FlowFileStreamTransformProcessorType, FlowFileTransformProcessorType, + ComplexProcessorType, FlowFileSourceProcessorType, FlowFileStreamTransformProcessorType, + FlowFileTransformProcessorType, MultiThreaded, SingleThreaded, }; minifi_native::declare_minifi_extension!( processors: [ - (ComplexProcessorType, Concurrent, GenerateFlowFileRs), - (ComplexProcessorType, Concurrent, LogAttributeRs), - (ComplexProcessorType, Concurrent, GetFileRs), - (ComplexProcessorType, Concurrent, KamikazeProcessorRs), - (ComplexProcessorType, Exclusive, CountActualLogging), - (FlowFileSourceProcessorType, Concurrent, LoremIpsumCSUser), - (FlowFileTransformProcessorType, Concurrent, PutFileRs), - (FlowFileStreamTransformProcessorType, Concurrent, AsciifyGerman), - (FlowFileStreamTransformProcessorType, Exclusive, DuplicateStreamText), - (ComplexProcessorType, Concurrent, ZooProcessorRs), + (ComplexProcessorType, MultiThreaded, GenerateFlowFileRs), + (ComplexProcessorType, MultiThreaded, LogAttributeRs), + (ComplexProcessorType, MultiThreaded, GetFileRs), + (ComplexProcessorType, MultiThreaded, KamikazeProcessorRs), + (ComplexProcessorType, SingleThreaded, CountActualLogging), + (FlowFileSourceProcessorType, MultiThreaded, LoremIpsumCSUser), + (FlowFileTransformProcessorType, MultiThreaded, PutFileRs), + (FlowFileStreamTransformProcessorType, MultiThreaded, AsciifyGerman), + (FlowFileStreamTransformProcessorType, SingleThreaded, DuplicateStreamText), + (ComplexProcessorType, MultiThreaded, ZooProcessorRs), ], controllers: [ LoremIpsumControllerService, diff --git a/minifi_rust/minifi_native/src/api/processor_wrappers/complex_processor.rs b/minifi_rust/minifi_native/src/api/processor_wrappers/complex_processor.rs index a907f77e59..5f79e7a0df 100644 --- a/minifi_rust/minifi_native/src/api/processor_wrappers/complex_processor.rs +++ b/minifi_rust/minifi_native/src/api/processor_wrappers/complex_processor.rs @@ -1,7 +1,7 @@ use crate::api::raw_processor::{MultiThreadedTrigger, SingleThreadedTrigger}; use crate::{ - ComponentIdentifier, Concurrent, Exclusive, Logger, MinifiError, OnTriggerResult, - ProcessContext, ProcessSession, Processor, ProcessorDefinition, Schedule, + ComponentIdentifier, Logger, MinifiError, MultiThreaded, OnTriggerResult, ProcessContext, + ProcessSession, Processor, ProcessorDefinition, Schedule, SingleThreaded, }; pub trait MutTrigger { @@ -33,7 +33,7 @@ pub trait Trigger { pub struct ComplexProcessorType {} impl SingleThreadedTrigger - for Processor + for Processor where Implementation: Schedule + MutTrigger + ComponentIdentifier + ProcessorDefinition, L: Logger, @@ -58,7 +58,7 @@ where } impl MultiThreadedTrigger - for Processor + for Processor where Implementation: Schedule + Trigger + ComponentIdentifier + ProcessorDefinition, L: Logger, diff --git a/minifi_rust/minifi_native/src/api/processor_wrappers/flow_file_source.rs b/minifi_rust/minifi_native/src/api/processor_wrappers/flow_file_source.rs index 35ff73bc4e..1a64a874a8 100644 --- a/minifi_rust/minifi_native/src/api/processor_wrappers/flow_file_source.rs +++ b/minifi_rust/minifi_native/src/api/processor_wrappers/flow_file_source.rs @@ -1,8 +1,8 @@ use crate::api::processor_wrappers::utils::flow_file_content::Content; use crate::api::raw_processor::{MultiThreadedTrigger, SingleThreadedTrigger}; use crate::{ - Concurrent, Exclusive, GetControllerService, GetProperty, Logger, MinifiError, OnTriggerResult, - ProcessContext, ProcessSession, Processor, Relationship, Schedule, + GetControllerService, GetProperty, Logger, MinifiError, MultiThreaded, OnTriggerResult, + ProcessContext, ProcessSession, Processor, Relationship, Schedule, SingleThreaded, }; use std::collections::HashMap; @@ -76,7 +76,7 @@ where pub struct FlowFileSourceProcessorType {} impl MultiThreadedTrigger - for Processor + for Processor where Implementation: Schedule + FlowFileSource, L: Logger, @@ -102,7 +102,7 @@ where } impl SingleThreadedTrigger - for Processor + for Processor where Implementation: Schedule + MutFlowFileSource, L: Logger, diff --git a/minifi_rust/minifi_native/src/api/processor_wrappers/flow_file_stream_transform.rs b/minifi_rust/minifi_native/src/api/processor_wrappers/flow_file_stream_transform.rs index 68a05ebdf3..aa62edd920 100644 --- a/minifi_rust/minifi_native/src/api/processor_wrappers/flow_file_stream_transform.rs +++ b/minifi_rust/minifi_native/src/api/processor_wrappers/flow_file_stream_transform.rs @@ -2,9 +2,9 @@ use crate::api::process_session::IoState; use crate::api::processor_wrappers::utils::context_session_flowfile_bundle::ContextSessionFlowFileBundle; use crate::api::raw_processor::{MultiThreadedTrigger, SingleThreadedTrigger}; use crate::{ - Concurrent, Exclusive, GetAttribute, GetControllerService, GetProperty, InputStream, LogLevel, - Logger, MinifiError, OnTriggerResult, OutputStream, ProcessContext, ProcessSession, Processor, - Relationship, Schedule, + GetAttribute, GetControllerService, GetProperty, InputStream, LogLevel, Logger, MinifiError, + MultiThreaded, OnTriggerResult, OutputStream, ProcessContext, ProcessSession, Processor, + Relationship, Schedule, SingleThreaded, }; use std::collections::HashMap; @@ -115,9 +115,8 @@ where } } -// Concurrent Implementation (Multi-Threaded) impl MultiThreadedTrigger - for Processor + for Processor where Implementation: Schedule + FlowFileStreamTransform, L: Logger, @@ -143,9 +142,8 @@ where } } -// Exclusive Implementation (Single-Threaded) impl SingleThreadedTrigger - for Processor + for Processor where Implementation: Schedule + MutFlowFileStreamTransform, L: Logger, diff --git a/minifi_rust/minifi_native/src/api/processor_wrappers/flow_file_transform.rs b/minifi_rust/minifi_native/src/api/processor_wrappers/flow_file_transform.rs index a594ef7368..2d97f23c91 100644 --- a/minifi_rust/minifi_native/src/api/processor_wrappers/flow_file_transform.rs +++ b/minifi_rust/minifi_native/src/api/processor_wrappers/flow_file_transform.rs @@ -6,8 +6,8 @@ use crate::api::processor_wrappers::utils::flow_file_content::Content; use crate::api::property::{GetControllerService, GetProperty}; use crate::api::raw_processor::{MultiThreadedTrigger, SingleThreadedTrigger}; use crate::{ - Concurrent, Exclusive, GetAttribute, LogLevel, Logger, MinifiError, OnTriggerResult, - ProcessContext, ProcessSession, Relationship, Schedule, info, + GetAttribute, LogLevel, Logger, MinifiError, MultiThreaded, OnTriggerResult, ProcessContext, + ProcessSession, Relationship, Schedule, SingleThreaded, info, }; use std::collections::HashMap; @@ -131,7 +131,7 @@ where } impl MultiThreadedTrigger - for Processor + for Processor where Implementation: Schedule + FlowFileTransform, L: Logger, @@ -158,7 +158,7 @@ where } impl SingleThreadedTrigger - for Processor + for Processor where Implementation: Schedule + MutFlowFileTransform, L: Logger, diff --git a/minifi_rust/minifi_native/src/api/raw_processor.rs b/minifi_rust/minifi_native/src/api/raw_processor.rs index ea1d3d7f67..7fd6f3d0ca 100644 --- a/minifi_rust/minifi_native/src/api/raw_processor.rs +++ b/minifi_rust/minifi_native/src/api/raw_processor.rs @@ -30,24 +30,24 @@ pub trait ThreadingModel: sealed::Sealed { } #[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub struct Concurrent; -impl ThreadingModel for Concurrent { +pub struct MultiThreaded; +impl ThreadingModel for MultiThreaded { const IS_EXCLUSIVE: bool = false; } #[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub struct Exclusive; -impl ThreadingModel for Exclusive { +pub struct SingleThreaded; +impl ThreadingModel for SingleThreaded { const IS_EXCLUSIVE: bool = true; } mod sealed { pub trait Sealed {} - impl Sealed for super::Concurrent {} - impl Sealed for super::Exclusive {} + impl Sealed for super::MultiThreaded {} + impl Sealed for super::SingleThreaded {} } -pub trait SingleThreadedTrigger: RawProcessor { +pub trait SingleThreadedTrigger: RawProcessor { fn trigger( &mut self, context: &mut PC, @@ -58,7 +58,7 @@ pub trait SingleThreadedTrigger: RawProcessor { PS: ProcessSession; } -pub trait MultiThreadedTrigger: RawProcessor { +pub trait MultiThreadedTrigger: RawProcessor { fn trigger( &self, context: &mut PC, diff --git a/minifi_rust/minifi_native/src/c_ffi/c_ffi_processor_definition.rs b/minifi_rust/minifi_native/src/c_ffi/c_ffi_processor_definition.rs index 92eba14392..cb279403c6 100644 --- a/minifi_rust/minifi_native/src/c_ffi/c_ffi_processor_definition.rs +++ b/minifi_rust/minifi_native/src/c_ffi/c_ffi_processor_definition.rs @@ -10,8 +10,8 @@ use crate::c_ffi::CffiLogger; use crate::c_ffi::c_ffi_output_attribute::COutputAttributes; use crate::c_ffi::c_ffi_property::CProperties; use crate::{ - ComponentIdentifier, Concurrent, Exclusive, LogLevel, OutputAttribute, Processor, - ProcessorDefinition, Property, Schedule, + ComponentIdentifier, LogLevel, MultiThreaded, OutputAttribute, Processor, ProcessorDefinition, + Property, Schedule, SingleThreaded, }; use crate::{OnTriggerResult, Relationship}; use minifi_native_sys::*; @@ -27,7 +27,7 @@ pub trait DispatchOnTrigger { ) -> minifi_status; } -impl DispatchOnTrigger for T +impl DispatchOnTrigger for T where T: MultiThreadedTrigger, { @@ -55,7 +55,7 @@ where } } -impl DispatchOnTrigger for T +impl DispatchOnTrigger for T where T: SingleThreadedTrigger, { diff --git a/minifi_rust/minifi_native/src/lib.rs b/minifi_rust/minifi_native/src/lib.rs index 027efa55c0..a91544290f 100644 --- a/minifi_rust/minifi_native/src/lib.rs +++ b/minifi_rust/minifi_native/src/lib.rs @@ -24,7 +24,7 @@ pub use api::processor_wrappers::utils::flow_file_content::Content; pub use api::processor::{Processor, Schedule}; -pub use api::raw_processor::{Concurrent, Exclusive}; +pub use api::raw_processor::{MultiThreaded, SingleThreaded}; pub use api::logger::{LogLevel, Logger}; From ad70c8c90b930e9124c0578320e8a564c051ab9e Mon Sep 17 00:00:00 2001 From: Martin Zink Date: Mon, 20 Jul 2026 10:44:48 +0200 Subject: [PATCH 23/26] added tech preview disclaimer --- minifi_rust/README.md | 32 +++++++++++++++++++------------- 1 file changed, 19 insertions(+), 13 deletions(-) diff --git a/minifi_rust/README.md b/minifi_rust/README.md index 6defc2c4d8..abcd621c67 100644 --- a/minifi_rust/README.md +++ b/minifi_rust/README.md @@ -1,6 +1,14 @@ -# MiNiFi Native Rust +# MiNiFi Native Rust (tech preview) -This repository provides a safe, idiomatic, and high-performance Rust framework for building native extensions (processors) for [Apache NiFi MiNiFi C++](https://github.com/apache/nifi-minifi-cpp) +> ⚠️ **Tech Preview** +> +> The Rust bindings are a technology preview. The API surface (both the +> `minifi-native` safe API and the `minifi-native-sys` FFI layer) is still +> evolving, and we are **not yet committed to backward compatibility** for +> the Rust bindings. Expect breaking changes between MiNiFi C++ releases +> until the bindings are declared stable. + +This repository provides a safe, idiomatic, and high-performance Rust framework for building native extensions (processors) for [Apache NiFi MiNiFi C++](https://github.com/apache/nifi-minifi-cpp). It is designed to offer a robust developer experience, allowing you to write powerful and reliable data processing components in safe Rust. @@ -11,12 +19,11 @@ The framework completely encapsulates the unsafe C FFI (Foreign Function Interfa - **Zero-Cost Abstraction**: The safe API wrapper is designed to compile down with zero runtime overhead compared to writing raw C code. - **Ergonomics**: Provide a clean, idiomatic Rust API that is a pleasure to use. Developers should not need to think about unsafe code or C++ interoperability. - **Testability**: Every component of a processor's logic should be unit-testable in a pure Rust environment, without needing a C++ host. - - **Cross Platform**: The library should work on all platforms that are supported by [Apache NiFi MiNiFi C++](https://github.com/apache/nifi-minifi-cpp) + - **Cross Platform**: The library should work on all platforms that are supported by [Apache NiFi MiNiFi C++](https://github.com/apache/nifi-minifi-cpp). - macOS (aarch64) - Linux (x86_64, aarch64) - Windows (x86_64) - The project is structured as a Cargo workspace with a clear, layered architecture: ### [minifi-native-sys](minifi_native_sys) Contains the raw, unsafe FFI bindings to the minifi-api.h C API. @@ -32,18 +39,18 @@ Pure rust traits that simplify the requirements for a working processor Concrete structs (CffiSession, CffiLogger, etc.) that implement the API traits by calling the unsafe functions from minifi-native-sys. #### Thread safety The trait system differentiates between thread-safe (&self) and single-threaded (&mut self) processors at compile time. -#### Comprehensive Mocking: +#### Comprehensive Mocking A full suite of mock objects allows for fast and reliable unit testing of all processor logic. ### [minifi_native_macros](minifi_native_macros) -Helper crate that includes the procedural macros +Helper crate that includes the procedural macros. ### [minifi_rs_behave](minifi_rs_behave) -Run the behave integration tests using Minifi's docker framework. This will test the release artifacts against the latest released [MiNiFi native docker container](https://hub.docker.com/r/apache/nifi-minifi-cpp). +Run the behave integration tests using MiNiFi's Docker framework. This will test the release artifacts against the latest released [MiNiFi native docker container](https://hub.docker.com/r/apache/nifi-minifi-cpp). There is a handy alias to initiate all behave tests. `cargo behave` ## Creating an Extension -Building an extension is straightforward. The framework provides a declare_minifi_extension! macro that automatically generates the C-compatible entry points and registers your components. +Building an extension is straightforward. The framework provides a `declare_minifi_extension!` macro that automatically generates the C-compatible entry points and registers your components. ```rust declare_minifi_extension!( @@ -57,15 +64,14 @@ declare_minifi_extension!( ); ``` - ## Deployment -Build your extension as a dynamic library (cd extensions/your_extension && cargo build --release). +Build your extension as a dynamic library: `cd extensions/your_extension && cargo build --release`. -Locate the output artifact in target/release/ (it will be a .so on Linux, .dll on Windows, or .dylib on macOS). +Locate the output artifact in `target/release/` (it will be a `.so` on Linux, `.dll` on Windows, or `.dylib` on macOS). -Copy the library file into the MiNiFi C++ application's extensions/ directory. +Copy the library file into the MiNiFi C++ application's `extensions/` directory. -Restart the MiNiFi Native agent to automatically discover and load the new processors. +Restart the MiNiFi C++ agent to automatically discover and load the new processors. ## Included Extensions ### [minifi_rs_playground](extensions/minifi_rs_playground) From 671732ecd3d3f0a8dd30d2f9848893ba1e9f77a4 Mon Sep 17 00:00:00 2001 From: Martin Zink Date: Mon, 20 Jul 2026 13:30:43 +0200 Subject: [PATCH 24/26] apache short licenses in every source file --- .../minifi_rs_playground/features/basic.feature | 17 ++++++++++++++++- .../features/controller_apis.feature | 16 +++++++++++++++- .../features/environment.py | 17 +++++++++++++++++ .../features/error-handling.feature | 15 +++++++++++++++ .../features/lazy-logging.feature | 15 +++++++++++++++ .../features/metrics.feature | 15 +++++++++++++++ .../features/steps/steps.py | 17 +++++++++++++++++ .../features/streaming.feature | 15 +++++++++++++++ .../animal_controller_apis.rs | 17 +++++++++++++++++ .../dog_controller_service.rs | 17 +++++++++++++++++ .../duck_controller_service.rs | 17 +++++++++++++++++ .../dummy_controller_service.rs | 17 +++++++++++++++++ .../lorem_ipsum_controller_service.rs | 17 +++++++++++++++++ .../properties.rs | 17 +++++++++++++++++ .../src/controller_services/mod.rs | 17 +++++++++++++++++ .../extensions/minifi_rs_playground/src/lib.rs | 17 +++++++++++++++++ .../src/processors/asciify_german.rs | 17 +++++++++++++++++ .../asciify_german/processor_definition.rs | 17 +++++++++++++++++ .../processors/asciify_german/relationships.rs | 17 +++++++++++++++++ .../src/processors/asciify_german/tests.rs | 17 +++++++++++++++++ .../src/processors/count_actual_logging.rs | 17 +++++++++++++++++ .../src/processors/duplicate_text.rs | 17 +++++++++++++++++ .../src/processors/generate_flow_file.rs | 17 +++++++++++++++++ .../generate_flow_file/processor_definition.rs | 17 +++++++++++++++++ .../processors/generate_flow_file/properties.rs | 17 +++++++++++++++++ .../generate_flow_file/relationships.rs | 17 +++++++++++++++++ .../src/processors/generate_flow_file/tests.rs | 17 +++++++++++++++++ .../src/processors/get_file.rs | 17 +++++++++++++++++ .../processors/get_file/output_attributes.rs | 17 +++++++++++++++++ .../processors/get_file/processor_definition.rs | 17 +++++++++++++++++ .../src/processors/get_file/properties.rs | 17 +++++++++++++++++ .../src/processors/get_file/relationships.rs | 17 +++++++++++++++++ .../src/processors/get_file/tests.rs | 17 +++++++++++++++++ .../src/processors/kamikaze_processor.rs | 17 +++++++++++++++++ .../kamikaze_processor/processor_definition.rs | 17 +++++++++++++++++ .../processors/kamikaze_processor/properties.rs | 17 +++++++++++++++++ .../kamikaze_processor/relationships.rs | 17 +++++++++++++++++ .../src/processors/kamikaze_processor/tests.rs | 17 +++++++++++++++++ .../src/processors/log_attribute.rs | 17 +++++++++++++++++ .../log_attribute/processor_definition.rs | 17 +++++++++++++++++ .../src/processors/log_attribute/properties.rs | 17 +++++++++++++++++ .../processors/log_attribute/relationships.rs | 17 +++++++++++++++++ .../src/processors/log_attribute/tests.rs | 17 +++++++++++++++++ .../src/processors/lorem_ipsum_cs_user.rs | 17 +++++++++++++++++ .../lorem_ipsum_cs_user/processor_definition.rs | 17 +++++++++++++++++ .../lorem_ipsum_cs_user/properties.rs | 17 +++++++++++++++++ .../lorem_ipsum_cs_user/relationships.rs | 17 +++++++++++++++++ .../src/processors/lorem_ipsum_cs_user/tests.rs | 17 +++++++++++++++++ .../minifi_rs_playground/src/processors/mod.rs | 17 +++++++++++++++++ .../src/processors/put_file.rs | 17 +++++++++++++++++ .../processors/put_file/processor_definition.rs | 17 +++++++++++++++++ .../src/processors/put_file/properties.rs | 17 +++++++++++++++++ .../src/processors/put_file/relationships.rs | 17 +++++++++++++++++ .../src/processors/put_file/tests.rs | 17 +++++++++++++++++ .../processors/put_file/unix_only_properties.rs | 17 +++++++++++++++++ .../src/processors/zoo_processor.rs | 17 +++++++++++++++++ minifi_rust/minifi_native/src/api.rs | 17 +++++++++++++++++ minifi_rust/minifi_native/src/api/attribute.rs | 17 +++++++++++++++++ .../src/api/component_definition_traits.rs | 17 +++++++++++++++++ .../minifi_native/src/api/controller_service.rs | 17 +++++++++++++++++ minifi_rust/minifi_native/src/api/errors.rs | 17 +++++++++++++++++ minifi_rust/minifi_native/src/api/flow_file.rs | 17 +++++++++++++++++ minifi_rust/minifi_native/src/api/logger.rs | 17 +++++++++++++++++ .../minifi_native/src/api/process_context.rs | 17 +++++++++++++++++ .../minifi_native/src/api/process_session.rs | 17 +++++++++++++++++ minifi_rust/minifi_native/src/api/processor.rs | 17 +++++++++++++++++ .../minifi_native/src/api/processor_wrappers.rs | 17 +++++++++++++++++ .../api/processor_wrappers/complex_processor.rs | 17 +++++++++++++++++ .../api/processor_wrappers/flow_file_source.rs | 17 +++++++++++++++++ .../flow_file_stream_transform.rs | 17 +++++++++++++++++ .../processor_wrappers/flow_file_transform.rs | 17 +++++++++++++++++ .../src/api/processor_wrappers/utils.rs | 17 +++++++++++++++++ .../utils/context_session_flowfile_bundle.rs | 17 +++++++++++++++++ .../utils/flow_file_content.rs | 17 +++++++++++++++++ minifi_rust/minifi_native/src/api/property.rs | 17 +++++++++++++++++ .../minifi_native/src/api/provided_interface.rs | 17 +++++++++++++++++ .../src/api/raw_controller_service.rs | 17 +++++++++++++++++ .../minifi_native/src/api/raw_processor.rs | 17 +++++++++++++++++ .../minifi_native/src/api/relationship.rs | 17 +++++++++++++++++ minifi_rust/minifi_native/src/c_ffi.rs | 17 +++++++++++++++++ .../c_ffi/c_ffi_controller_service_context.rs | 17 +++++++++++++++++ .../c_ffi_controller_service_definition.rs | 17 +++++++++++++++++ .../src/c_ffi/c_ffi_controller_service_list.rs | 17 +++++++++++++++++ .../minifi_native/src/c_ffi/c_ffi_flow_file.rs | 17 +++++++++++++++++ .../minifi_native/src/c_ffi/c_ffi_logger.rs | 17 +++++++++++++++++ .../src/c_ffi/c_ffi_output_attribute.rs | 17 +++++++++++++++++ .../minifi_native/src/c_ffi/c_ffi_primitives.rs | 17 +++++++++++++++++ .../src/c_ffi/c_ffi_process_context.rs | 17 +++++++++++++++++ .../src/c_ffi/c_ffi_process_session.rs | 17 +++++++++++++++++ .../src/c_ffi/c_ffi_processor_definition.rs | 17 +++++++++++++++++ .../src/c_ffi/c_ffi_processor_list.rs | 17 +++++++++++++++++ .../minifi_native/src/c_ffi/c_ffi_property.rs | 17 +++++++++++++++++ .../src/c_ffi/c_ffi_relationship.rs | 17 +++++++++++++++++ .../minifi_native/src/c_ffi/c_ffi_streams.rs | 17 +++++++++++++++++ minifi_rust/minifi_native/src/lib.rs | 17 +++++++++++++++++ minifi_rust/minifi_native/src/mock.rs | 17 +++++++++++++++++ .../src/mock/mock_controller_service_context.rs | 17 +++++++++++++++++ .../minifi_native/src/mock/mock_flow_file.rs | 17 +++++++++++++++++ .../minifi_native/src/mock/mock_logger.rs | 17 +++++++++++++++++ .../src/mock/mock_process_context.rs | 17 +++++++++++++++++ .../src/mock/mock_process_session.rs | 17 +++++++++++++++++ minifi_rust/minifi_native_macros/src/lib.rs | 17 +++++++++++++++++ minifi_rust/minifi_native_sys/build.rs | 17 +++++++++++++++++ minifi_rust/minifi_native_sys/src/lib.rs | 17 +++++++++++++++++ minifi_rust/minifi_rs_behave/build.rs | 17 +++++++++++++++++ minifi_rust/minifi_rs_behave/src/main.rs | 17 +++++++++++++++++ 106 files changed, 1791 insertions(+), 2 deletions(-) diff --git a/minifi_rust/extensions/minifi_rs_playground/features/basic.feature b/minifi_rust/extensions/minifi_rs_playground/features/basic.feature index 4c642f8530..698266740d 100644 --- a/minifi_rust/extensions/minifi_rs_playground/features/basic.feature +++ b/minifi_rust/extensions/minifi_rs_playground/features/basic.feature @@ -1,6 +1,21 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + @SUPPORTS_WINDOWS Feature: Basic scenarios - + Scenario: The rust library is loaded into minifi Given log property "logger.org::apache::nifi::minifi::core::extension::ExtensionManager" is set to "TRACE,stderr" And log property "logger.org::apache::nifi::minifi::core::ClassLoader" is set to "TRACE,stderr" diff --git a/minifi_rust/extensions/minifi_rs_playground/features/controller_apis.feature b/minifi_rust/extensions/minifi_rs_playground/features/controller_apis.feature index 49411ce8b5..c93f896160 100644 --- a/minifi_rust/extensions/minifi_rs_playground/features/controller_apis.feature +++ b/minifi_rust/extensions/minifi_rs_playground/features/controller_apis.feature @@ -1,3 +1,18 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + @SUPPORTS_WINDOWS Feature: Testing controller service api casting @@ -11,4 +26,3 @@ Feature: Testing controller service api casting Then the Minifi logs contain the following message: "[minifi_rs_playground::processors::zoo_processor::ZooProcessorRs] [critical] Can DogControllerRs { has_jetpack: true, extra_info: "The dog (Canis familiaris or Canis lupus familiaris) is a domesticated descendant of wolves." } fly? true" in less than 10 seconds And the Minifi logs do not contain errors And the Minifi logs do not contain warnings - diff --git a/minifi_rust/extensions/minifi_rs_playground/features/environment.py b/minifi_rust/extensions/minifi_rs_playground/features/environment.py index c5d67ba23d..3ba4288e03 100644 --- a/minifi_rust/extensions/minifi_rs_playground/features/environment.py +++ b/minifi_rust/extensions/minifi_rs_playground/features/environment.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + import os from typing import List diff --git a/minifi_rust/extensions/minifi_rs_playground/features/error-handling.feature b/minifi_rust/extensions/minifi_rs_playground/features/error-handling.feature index 543c4a45a9..5eb3329f8a 100644 --- a/minifi_rust/extensions/minifi_rs_playground/features/error-handling.feature +++ b/minifi_rust/extensions/minifi_rs_playground/features/error-handling.feature @@ -1,3 +1,18 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + @SUPPORTS_WINDOWS Feature: API error handling and logging diff --git a/minifi_rust/extensions/minifi_rs_playground/features/lazy-logging.feature b/minifi_rust/extensions/minifi_rs_playground/features/lazy-logging.feature index 89e11197ba..ece06e3663 100644 --- a/minifi_rust/extensions/minifi_rs_playground/features/lazy-logging.feature +++ b/minifi_rust/extensions/minifi_rs_playground/features/lazy-logging.feature @@ -1,3 +1,18 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + @SUPPORTS_WINDOWS Feature: Logs should be lazily evaluated diff --git a/minifi_rust/extensions/minifi_rs_playground/features/metrics.feature b/minifi_rust/extensions/minifi_rs_playground/features/metrics.feature index 7ebf844e2a..8c335649b2 100644 --- a/minifi_rust/extensions/minifi_rs_playground/features/metrics.feature +++ b/minifi_rust/extensions/minifi_rs_playground/features/metrics.feature @@ -1,3 +1,18 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + @SUPPORTS_WINDOWS Feature: Testing custom and default metrics diff --git a/minifi_rust/extensions/minifi_rs_playground/features/steps/steps.py b/minifi_rust/extensions/minifi_rs_playground/features/steps/steps.py index 181105116e..f9a0ed64a5 100644 --- a/minifi_rust/extensions/minifi_rs_playground/features/steps/steps.py +++ b/minifi_rust/extensions/minifi_rs_playground/features/steps/steps.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + from behave import then, when, given import humanfriendly diff --git a/minifi_rust/extensions/minifi_rs_playground/features/streaming.feature b/minifi_rust/extensions/minifi_rs_playground/features/streaming.feature index d6fea8933d..2189156516 100644 --- a/minifi_rust/extensions/minifi_rs_playground/features/streaming.feature +++ b/minifi_rust/extensions/minifi_rs_playground/features/streaming.feature @@ -1,3 +1,18 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + @SUPPORTS_WINDOWS Feature: Testing streaming reads and writes diff --git a/minifi_rust/extensions/minifi_rs_playground/src/controller_services/animal_controller_apis.rs b/minifi_rust/extensions/minifi_rs_playground/src/controller_services/animal_controller_apis.rs index f2f1e26760..d63c0026c1 100644 --- a/minifi_rust/extensions/minifi_rs_playground/src/controller_services/animal_controller_apis.rs +++ b/minifi_rust/extensions/minifi_rs_playground/src/controller_services/animal_controller_apis.rs @@ -1,3 +1,20 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + use minifi_native::macros::controller_service_api; use std::fmt::Debug; diff --git a/minifi_rust/extensions/minifi_rs_playground/src/controller_services/dog_controller_service.rs b/minifi_rust/extensions/minifi_rs_playground/src/controller_services/dog_controller_service.rs index 939c789fe4..a1eb4ad636 100644 --- a/minifi_rust/extensions/minifi_rs_playground/src/controller_services/dog_controller_service.rs +++ b/minifi_rust/extensions/minifi_rs_playground/src/controller_services/dog_controller_service.rs @@ -1,3 +1,20 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + use crate::controller_services::animal_controller_apis::{ CanFlyControllerApi, NumberOfLegsControllerApi, }; diff --git a/minifi_rust/extensions/minifi_rs_playground/src/controller_services/duck_controller_service.rs b/minifi_rust/extensions/minifi_rs_playground/src/controller_services/duck_controller_service.rs index 74a123ea1e..13c00854d1 100644 --- a/minifi_rust/extensions/minifi_rs_playground/src/controller_services/duck_controller_service.rs +++ b/minifi_rust/extensions/minifi_rs_playground/src/controller_services/duck_controller_service.rs @@ -1,3 +1,20 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + use crate::controller_services::animal_controller_apis::{ CanFlyControllerApi, NumberOfLegsControllerApi, }; diff --git a/minifi_rust/extensions/minifi_rs_playground/src/controller_services/dummy_controller_service.rs b/minifi_rust/extensions/minifi_rs_playground/src/controller_services/dummy_controller_service.rs index d69efb8f98..868f339ca1 100644 --- a/minifi_rust/extensions/minifi_rs_playground/src/controller_services/dummy_controller_service.rs +++ b/minifi_rust/extensions/minifi_rs_playground/src/controller_services/dummy_controller_service.rs @@ -1,3 +1,20 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + use minifi_native::macros::ComponentIdentifier; use minifi_native::{ ControllerServiceDefinition, EnableControllerService, GetProperty, Logger, MinifiError, diff --git a/minifi_rust/extensions/minifi_rs_playground/src/controller_services/lorem_ipsum_controller_service.rs b/minifi_rust/extensions/minifi_rs_playground/src/controller_services/lorem_ipsum_controller_service.rs index 6b3809917d..d642292817 100644 --- a/minifi_rust/extensions/minifi_rs_playground/src/controller_services/lorem_ipsum_controller_service.rs +++ b/minifi_rust/extensions/minifi_rs_playground/src/controller_services/lorem_ipsum_controller_service.rs @@ -1,3 +1,20 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + mod properties; use crate::controller_services::lorem_ipsum_controller_service::properties::LENGTH; diff --git a/minifi_rust/extensions/minifi_rs_playground/src/controller_services/lorem_ipsum_controller_service/properties.rs b/minifi_rust/extensions/minifi_rs_playground/src/controller_services/lorem_ipsum_controller_service/properties.rs index f15fd95f21..b57cacc347 100644 --- a/minifi_rust/extensions/minifi_rs_playground/src/controller_services/lorem_ipsum_controller_service/properties.rs +++ b/minifi_rust/extensions/minifi_rs_playground/src/controller_services/lorem_ipsum_controller_service/properties.rs @@ -1,3 +1,20 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + use minifi_native::{Property, StandardPropertyValidator}; pub(crate) const LENGTH: Property = Property { diff --git a/minifi_rust/extensions/minifi_rs_playground/src/controller_services/mod.rs b/minifi_rust/extensions/minifi_rs_playground/src/controller_services/mod.rs index 774dd42076..c971d5df3c 100644 --- a/minifi_rust/extensions/minifi_rs_playground/src/controller_services/mod.rs +++ b/minifi_rust/extensions/minifi_rs_playground/src/controller_services/mod.rs @@ -1,3 +1,20 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + pub(crate) mod animal_controller_apis; pub(crate) mod dog_controller_service; pub(crate) mod duck_controller_service; diff --git a/minifi_rust/extensions/minifi_rs_playground/src/lib.rs b/minifi_rust/extensions/minifi_rs_playground/src/lib.rs index 53527a37ab..b06d6ad5cb 100644 --- a/minifi_rust/extensions/minifi_rs_playground/src/lib.rs +++ b/minifi_rust/extensions/minifi_rs_playground/src/lib.rs @@ -1,3 +1,20 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + mod controller_services; mod processors; diff --git a/minifi_rust/extensions/minifi_rs_playground/src/processors/asciify_german.rs b/minifi_rust/extensions/minifi_rs_playground/src/processors/asciify_german.rs index 55936935c3..84db2b3d8b 100644 --- a/minifi_rust/extensions/minifi_rs_playground/src/processors/asciify_german.rs +++ b/minifi_rust/extensions/minifi_rs_playground/src/processors/asciify_german.rs @@ -1,3 +1,20 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + use crate::processors::asciify_german::relationships::FAILURE; use minifi_native::macros::ComponentIdentifier; use minifi_native::{ diff --git a/minifi_rust/extensions/minifi_rs_playground/src/processors/asciify_german/processor_definition.rs b/minifi_rust/extensions/minifi_rs_playground/src/processors/asciify_german/processor_definition.rs index 62f135741d..e2429fe974 100644 --- a/minifi_rust/extensions/minifi_rs_playground/src/processors/asciify_german/processor_definition.rs +++ b/minifi_rust/extensions/minifi_rs_playground/src/processors/asciify_german/processor_definition.rs @@ -1,3 +1,20 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + use crate::processors::asciify_german::AsciifyGerman; use minifi_native::{ OutputAttribute, ProcessorDefinition, ProcessorInputRequirement, Property, Relationship, diff --git a/minifi_rust/extensions/minifi_rs_playground/src/processors/asciify_german/relationships.rs b/minifi_rust/extensions/minifi_rs_playground/src/processors/asciify_german/relationships.rs index ccb190d200..e624aa3bc2 100644 --- a/minifi_rust/extensions/minifi_rs_playground/src/processors/asciify_german/relationships.rs +++ b/minifi_rust/extensions/minifi_rs_playground/src/processors/asciify_german/relationships.rs @@ -1,3 +1,20 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + use minifi_native::Relationship; pub(crate) const SUCCESS: Relationship = Relationship { diff --git a/minifi_rust/extensions/minifi_rs_playground/src/processors/asciify_german/tests.rs b/minifi_rust/extensions/minifi_rs_playground/src/processors/asciify_german/tests.rs index ec2ded4204..c9b93c6581 100644 --- a/minifi_rust/extensions/minifi_rs_playground/src/processors/asciify_german/tests.rs +++ b/minifi_rust/extensions/minifi_rs_playground/src/processors/asciify_german/tests.rs @@ -1,3 +1,20 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + use super::*; use crate::processors::asciify_german::relationships::SUCCESS; use minifi_native::{IoState, MockLogger, MockProcessContext}; diff --git a/minifi_rust/extensions/minifi_rs_playground/src/processors/count_actual_logging.rs b/minifi_rust/extensions/minifi_rs_playground/src/processors/count_actual_logging.rs index 2aa3b4c4a2..b1b4ccf2aa 100644 --- a/minifi_rust/extensions/minifi_rs_playground/src/processors/count_actual_logging.rs +++ b/minifi_rust/extensions/minifi_rs_playground/src/processors/count_actual_logging.rs @@ -1,3 +1,20 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + use minifi_native::macros::ComponentIdentifier; use minifi_native::{ GetProperty, Logger, MinifiError, MutTrigger, OnTriggerResult, OutputAttribute, ProcessContext, diff --git a/minifi_rust/extensions/minifi_rs_playground/src/processors/duplicate_text.rs b/minifi_rust/extensions/minifi_rs_playground/src/processors/duplicate_text.rs index 9f81bc2990..ef0d9c9171 100644 --- a/minifi_rust/extensions/minifi_rs_playground/src/processors/duplicate_text.rs +++ b/minifi_rust/extensions/minifi_rs_playground/src/processors/duplicate_text.rs @@ -1,3 +1,20 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + use minifi_native::macros::ComponentIdentifier; use minifi_native::{ GetAttribute, GetControllerService, GetProperty, InputStream, Logger, MinifiError, diff --git a/minifi_rust/extensions/minifi_rs_playground/src/processors/generate_flow_file.rs b/minifi_rust/extensions/minifi_rs_playground/src/processors/generate_flow_file.rs index 971b6a07d3..711993d116 100644 --- a/minifi_rust/extensions/minifi_rs_playground/src/processors/generate_flow_file.rs +++ b/minifi_rust/extensions/minifi_rs_playground/src/processors/generate_flow_file.rs @@ -1,3 +1,20 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + use minifi_native::macros::ComponentIdentifier; use minifi_native::{ GetProperty, Logger, MinifiError, OnTriggerResult, ProcessContext, ProcessSession, Schedule, diff --git a/minifi_rust/extensions/minifi_rs_playground/src/processors/generate_flow_file/processor_definition.rs b/minifi_rust/extensions/minifi_rs_playground/src/processors/generate_flow_file/processor_definition.rs index 7e0c21b520..13955b9896 100644 --- a/minifi_rust/extensions/minifi_rs_playground/src/processors/generate_flow_file/processor_definition.rs +++ b/minifi_rust/extensions/minifi_rs_playground/src/processors/generate_flow_file/processor_definition.rs @@ -1,3 +1,20 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + use super::properties::*; use super::{GenerateFlowFileRs, relationships}; use minifi_native::{ diff --git a/minifi_rust/extensions/minifi_rs_playground/src/processors/generate_flow_file/properties.rs b/minifi_rust/extensions/minifi_rs_playground/src/processors/generate_flow_file/properties.rs index 58840edb99..16d0b8a718 100644 --- a/minifi_rust/extensions/minifi_rs_playground/src/processors/generate_flow_file/properties.rs +++ b/minifi_rust/extensions/minifi_rs_playground/src/processors/generate_flow_file/properties.rs @@ -1,3 +1,20 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + use minifi_native::{Property, StandardPropertyValidator}; pub(crate) const FILE_SIZE: Property = Property { diff --git a/minifi_rust/extensions/minifi_rs_playground/src/processors/generate_flow_file/relationships.rs b/minifi_rust/extensions/minifi_rs_playground/src/processors/generate_flow_file/relationships.rs index e108d52230..81fd6f9040 100644 --- a/minifi_rust/extensions/minifi_rs_playground/src/processors/generate_flow_file/relationships.rs +++ b/minifi_rust/extensions/minifi_rs_playground/src/processors/generate_flow_file/relationships.rs @@ -1,3 +1,20 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + use minifi_native::Relationship; pub(crate) const SUCCESS: Relationship = Relationship { diff --git a/minifi_rust/extensions/minifi_rs_playground/src/processors/generate_flow_file/tests.rs b/minifi_rust/extensions/minifi_rs_playground/src/processors/generate_flow_file/tests.rs index 2a1f4108f8..39b2facd47 100644 --- a/minifi_rust/extensions/minifi_rs_playground/src/processors/generate_flow_file/tests.rs +++ b/minifi_rust/extensions/minifi_rs_playground/src/processors/generate_flow_file/tests.rs @@ -1,3 +1,20 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + use super::*; use crate::processors::generate_flow_file::properties::{ BATCH_SIZE, CUSTOM_TEXT, DATA_FORMAT, UNIQUE_FLOW_FILES, diff --git a/minifi_rust/extensions/minifi_rs_playground/src/processors/get_file.rs b/minifi_rust/extensions/minifi_rs_playground/src/processors/get_file.rs index 96bd419a87..e2644e8035 100644 --- a/minifi_rust/extensions/minifi_rs_playground/src/processors/get_file.rs +++ b/minifi_rust/extensions/minifi_rs_playground/src/processors/get_file.rs @@ -1,3 +1,20 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + use crate::processors::get_file::output_attributes::{ ABSOLUTE_PATH_OUTPUT_ATTRIBUTE, FILENAME_OUTPUT_ATTRIBUTE, }; diff --git a/minifi_rust/extensions/minifi_rs_playground/src/processors/get_file/output_attributes.rs b/minifi_rust/extensions/minifi_rs_playground/src/processors/get_file/output_attributes.rs index 62b1fbc11f..1e6e632993 100644 --- a/minifi_rust/extensions/minifi_rs_playground/src/processors/get_file/output_attributes.rs +++ b/minifi_rust/extensions/minifi_rs_playground/src/processors/get_file/output_attributes.rs @@ -1,3 +1,20 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + use minifi_native::OutputAttribute; pub(crate) const FILENAME_OUTPUT_ATTRIBUTE: OutputAttribute = OutputAttribute { diff --git a/minifi_rust/extensions/minifi_rs_playground/src/processors/get_file/processor_definition.rs b/minifi_rust/extensions/minifi_rs_playground/src/processors/get_file/processor_definition.rs index b3bd89146d..713be5a5a9 100644 --- a/minifi_rust/extensions/minifi_rs_playground/src/processors/get_file/processor_definition.rs +++ b/minifi_rust/extensions/minifi_rs_playground/src/processors/get_file/processor_definition.rs @@ -1,3 +1,20 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + use crate::processors::get_file::output_attributes::{ ABSOLUTE_PATH_OUTPUT_ATTRIBUTE, FILENAME_OUTPUT_ATTRIBUTE, }; diff --git a/minifi_rust/extensions/minifi_rs_playground/src/processors/get_file/properties.rs b/minifi_rust/extensions/minifi_rs_playground/src/processors/get_file/properties.rs index 5945ce29d5..b43009b28e 100644 --- a/minifi_rust/extensions/minifi_rs_playground/src/processors/get_file/properties.rs +++ b/minifi_rust/extensions/minifi_rs_playground/src/processors/get_file/properties.rs @@ -1,3 +1,20 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + use minifi_native::{Property, StandardPropertyValidator}; pub(crate) const DIRECTORY: Property = Property { diff --git a/minifi_rust/extensions/minifi_rs_playground/src/processors/get_file/relationships.rs b/minifi_rust/extensions/minifi_rs_playground/src/processors/get_file/relationships.rs index 5f3c286715..b8ec9ae3ce 100644 --- a/minifi_rust/extensions/minifi_rs_playground/src/processors/get_file/relationships.rs +++ b/minifi_rust/extensions/minifi_rs_playground/src/processors/get_file/relationships.rs @@ -1,3 +1,20 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + use minifi_native::Relationship; pub(crate) const SUCCESS: Relationship = Relationship { diff --git a/minifi_rust/extensions/minifi_rs_playground/src/processors/get_file/tests.rs b/minifi_rust/extensions/minifi_rs_playground/src/processors/get_file/tests.rs index d886ffc5c5..34449bd419 100644 --- a/minifi_rust/extensions/minifi_rs_playground/src/processors/get_file/tests.rs +++ b/minifi_rust/extensions/minifi_rs_playground/src/processors/get_file/tests.rs @@ -1,3 +1,20 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + use super::*; use crate::processors::get_file::relationships::SUCCESS; use filetime::FileTime; diff --git a/minifi_rust/extensions/minifi_rs_playground/src/processors/kamikaze_processor.rs b/minifi_rust/extensions/minifi_rs_playground/src/processors/kamikaze_processor.rs index b6ed74b499..42c0163c69 100644 --- a/minifi_rust/extensions/minifi_rs_playground/src/processors/kamikaze_processor.rs +++ b/minifi_rust/extensions/minifi_rs_playground/src/processors/kamikaze_processor.rs @@ -1,3 +1,20 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + mod properties; mod relationships; diff --git a/minifi_rust/extensions/minifi_rs_playground/src/processors/kamikaze_processor/processor_definition.rs b/minifi_rust/extensions/minifi_rs_playground/src/processors/kamikaze_processor/processor_definition.rs index 5c38f209da..cd21163ec6 100644 --- a/minifi_rust/extensions/minifi_rs_playground/src/processors/kamikaze_processor/processor_definition.rs +++ b/minifi_rust/extensions/minifi_rs_playground/src/processors/kamikaze_processor/processor_definition.rs @@ -1,3 +1,20 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + use super::*; use minifi_native::{ OutputAttribute, ProcessorDefinition, ProcessorInputRequirement, Property, Relationship, diff --git a/minifi_rust/extensions/minifi_rs_playground/src/processors/kamikaze_processor/properties.rs b/minifi_rust/extensions/minifi_rs_playground/src/processors/kamikaze_processor/properties.rs index 02fab5b106..6ee86b1bda 100644 --- a/minifi_rust/extensions/minifi_rs_playground/src/processors/kamikaze_processor/properties.rs +++ b/minifi_rust/extensions/minifi_rs_playground/src/processors/kamikaze_processor/properties.rs @@ -1,3 +1,20 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + use crate::processors::kamikaze_processor::KamikazeBehaviour; use minifi_native::{Property, StandardPropertyValidator}; use strum::VariantNames; diff --git a/minifi_rust/extensions/minifi_rs_playground/src/processors/kamikaze_processor/relationships.rs b/minifi_rust/extensions/minifi_rs_playground/src/processors/kamikaze_processor/relationships.rs index 1bdfca63f5..33e94d41a6 100644 --- a/minifi_rust/extensions/minifi_rs_playground/src/processors/kamikaze_processor/relationships.rs +++ b/minifi_rust/extensions/minifi_rs_playground/src/processors/kamikaze_processor/relationships.rs @@ -1,3 +1,20 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + use minifi_native::Relationship; pub(crate) const SUCCESS: Relationship = Relationship { diff --git a/minifi_rust/extensions/minifi_rs_playground/src/processors/kamikaze_processor/tests.rs b/minifi_rust/extensions/minifi_rs_playground/src/processors/kamikaze_processor/tests.rs index 9bd44336f8..f5560a25a8 100644 --- a/minifi_rust/extensions/minifi_rs_playground/src/processors/kamikaze_processor/tests.rs +++ b/minifi_rust/extensions/minifi_rs_playground/src/processors/kamikaze_processor/tests.rs @@ -1,3 +1,20 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + use super::*; use crate::processors::kamikaze_processor::properties::{SCHEDULE_BEHAVIOUR, TRIGGER_BEHAVIOUR}; use minifi_native::MinifiError::{ScheduleError, TriggerError}; diff --git a/minifi_rust/extensions/minifi_rs_playground/src/processors/log_attribute.rs b/minifi_rust/extensions/minifi_rs_playground/src/processors/log_attribute.rs index b37c7c5a67..a9150c016d 100644 --- a/minifi_rust/extensions/minifi_rs_playground/src/processors/log_attribute.rs +++ b/minifi_rust/extensions/minifi_rs_playground/src/processors/log_attribute.rs @@ -1,3 +1,20 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + use crate::processors::log_attribute::properties::{FLOW_FILES_TO_LOG, LOG_LEVEL, LOG_PAYLOAD}; use minifi_native::macros::ComponentIdentifier; use minifi_native::{ diff --git a/minifi_rust/extensions/minifi_rs_playground/src/processors/log_attribute/processor_definition.rs b/minifi_rust/extensions/minifi_rs_playground/src/processors/log_attribute/processor_definition.rs index 0d4c08fdb1..8c2dc4c030 100644 --- a/minifi_rust/extensions/minifi_rs_playground/src/processors/log_attribute/processor_definition.rs +++ b/minifi_rust/extensions/minifi_rs_playground/src/processors/log_attribute/processor_definition.rs @@ -1,3 +1,20 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + use crate::processors::log_attribute::properties::*; use crate::processors::log_attribute::{LogAttributeRs, relationships}; use minifi_native::{ diff --git a/minifi_rust/extensions/minifi_rs_playground/src/processors/log_attribute/properties.rs b/minifi_rust/extensions/minifi_rs_playground/src/processors/log_attribute/properties.rs index ea0de2b32d..48002c92fc 100644 --- a/minifi_rust/extensions/minifi_rs_playground/src/processors/log_attribute/properties.rs +++ b/minifi_rust/extensions/minifi_rs_playground/src/processors/log_attribute/properties.rs @@ -1,3 +1,20 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + use minifi_native::{LogLevel, Property, StandardPropertyValidator}; use strum::VariantNames; diff --git a/minifi_rust/extensions/minifi_rs_playground/src/processors/log_attribute/relationships.rs b/minifi_rust/extensions/minifi_rs_playground/src/processors/log_attribute/relationships.rs index 5f3c286715..b8ec9ae3ce 100644 --- a/minifi_rust/extensions/minifi_rs_playground/src/processors/log_attribute/relationships.rs +++ b/minifi_rust/extensions/minifi_rs_playground/src/processors/log_attribute/relationships.rs @@ -1,3 +1,20 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + use minifi_native::Relationship; pub(crate) const SUCCESS: Relationship = Relationship { diff --git a/minifi_rust/extensions/minifi_rs_playground/src/processors/log_attribute/tests.rs b/minifi_rust/extensions/minifi_rs_playground/src/processors/log_attribute/tests.rs index 1a272b4077..8c770fa6bf 100644 --- a/minifi_rust/extensions/minifi_rs_playground/src/processors/log_attribute/tests.rs +++ b/minifi_rust/extensions/minifi_rs_playground/src/processors/log_attribute/tests.rs @@ -1,3 +1,20 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + use super::*; use minifi_native::{ ComponentIdentifier, MockFlowFile, MockLogger, MockProcessContext, MockProcessSession, diff --git a/minifi_rust/extensions/minifi_rs_playground/src/processors/lorem_ipsum_cs_user.rs b/minifi_rust/extensions/minifi_rs_playground/src/processors/lorem_ipsum_cs_user.rs index 6361669d36..c90e04970a 100644 --- a/minifi_rust/extensions/minifi_rs_playground/src/processors/lorem_ipsum_cs_user.rs +++ b/minifi_rust/extensions/minifi_rs_playground/src/processors/lorem_ipsum_cs_user.rs @@ -1,3 +1,20 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + mod properties; use crate::controller_services::lorem_ipsum_controller_service::LoremIpsumControllerService; diff --git a/minifi_rust/extensions/minifi_rs_playground/src/processors/lorem_ipsum_cs_user/processor_definition.rs b/minifi_rust/extensions/minifi_rs_playground/src/processors/lorem_ipsum_cs_user/processor_definition.rs index 2ba155a80f..6c94730f52 100644 --- a/minifi_rust/extensions/minifi_rs_playground/src/processors/lorem_ipsum_cs_user/processor_definition.rs +++ b/minifi_rust/extensions/minifi_rs_playground/src/processors/lorem_ipsum_cs_user/processor_definition.rs @@ -1,3 +1,20 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + use super::LoremIpsumCSUser; use super::properties::*; use crate::processors::lorem_ipsum_cs_user::relationships::SUCCESS; diff --git a/minifi_rust/extensions/minifi_rs_playground/src/processors/lorem_ipsum_cs_user/properties.rs b/minifi_rust/extensions/minifi_rs_playground/src/processors/lorem_ipsum_cs_user/properties.rs index 687cbbec01..b78ab6d23b 100644 --- a/minifi_rust/extensions/minifi_rs_playground/src/processors/lorem_ipsum_cs_user/properties.rs +++ b/minifi_rust/extensions/minifi_rs_playground/src/processors/lorem_ipsum_cs_user/properties.rs @@ -1,3 +1,20 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + use crate::controller_services::lorem_ipsum_controller_service::LoremIpsumControllerService; use minifi_native::ComponentIdentifier; use minifi_native::{Property, StandardPropertyValidator}; diff --git a/minifi_rust/extensions/minifi_rs_playground/src/processors/lorem_ipsum_cs_user/relationships.rs b/minifi_rust/extensions/minifi_rs_playground/src/processors/lorem_ipsum_cs_user/relationships.rs index 97d0d98286..4c7c2d0530 100644 --- a/minifi_rust/extensions/minifi_rs_playground/src/processors/lorem_ipsum_cs_user/relationships.rs +++ b/minifi_rust/extensions/minifi_rs_playground/src/processors/lorem_ipsum_cs_user/relationships.rs @@ -1,3 +1,20 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + use minifi_native::Relationship; pub(crate) const SUCCESS: Relationship = Relationship { diff --git a/minifi_rust/extensions/minifi_rs_playground/src/processors/lorem_ipsum_cs_user/tests.rs b/minifi_rust/extensions/minifi_rs_playground/src/processors/lorem_ipsum_cs_user/tests.rs index e1eb6b54dc..8a3ad0340f 100644 --- a/minifi_rust/extensions/minifi_rs_playground/src/processors/lorem_ipsum_cs_user/tests.rs +++ b/minifi_rust/extensions/minifi_rs_playground/src/processors/lorem_ipsum_cs_user/tests.rs @@ -1,3 +1,20 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + use crate::processors::lorem_ipsum_cs_user::LoremIpsumCSUser; use minifi_native::{ComponentIdentifier, MockLogger, MockProcessContext, Schedule}; diff --git a/minifi_rust/extensions/minifi_rs_playground/src/processors/mod.rs b/minifi_rust/extensions/minifi_rs_playground/src/processors/mod.rs index f1a885c728..0ae3a307dd 100644 --- a/minifi_rust/extensions/minifi_rs_playground/src/processors/mod.rs +++ b/minifi_rust/extensions/minifi_rs_playground/src/processors/mod.rs @@ -1,3 +1,20 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + pub(crate) mod asciify_german; pub(crate) mod count_actual_logging; pub(crate) mod duplicate_text; diff --git a/minifi_rust/extensions/minifi_rs_playground/src/processors/put_file.rs b/minifi_rust/extensions/minifi_rs_playground/src/processors/put_file.rs index b0d0e76649..c9faf8b13b 100644 --- a/minifi_rust/extensions/minifi_rs_playground/src/processors/put_file.rs +++ b/minifi_rust/extensions/minifi_rs_playground/src/processors/put_file.rs @@ -1,3 +1,20 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + use crate::processors::put_file::relationships::{FAILURE, SUCCESS}; use minifi_native::macros::ComponentIdentifier; use minifi_native::{ diff --git a/minifi_rust/extensions/minifi_rs_playground/src/processors/put_file/processor_definition.rs b/minifi_rust/extensions/minifi_rs_playground/src/processors/put_file/processor_definition.rs index 245782a870..a492c67e6f 100644 --- a/minifi_rust/extensions/minifi_rs_playground/src/processors/put_file/processor_definition.rs +++ b/minifi_rust/extensions/minifi_rs_playground/src/processors/put_file/processor_definition.rs @@ -1,3 +1,20 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + use super::*; use minifi_native::{ OutputAttribute, ProcessorDefinition, ProcessorInputRequirement, Property, Relationship, diff --git a/minifi_rust/extensions/minifi_rs_playground/src/processors/put_file/properties.rs b/minifi_rust/extensions/minifi_rs_playground/src/processors/put_file/properties.rs index bea299be62..4facd860f2 100644 --- a/minifi_rust/extensions/minifi_rs_playground/src/processors/put_file/properties.rs +++ b/minifi_rust/extensions/minifi_rs_playground/src/processors/put_file/properties.rs @@ -1,3 +1,20 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + use minifi_native::{Property, StandardPropertyValidator}; use strum::VariantNames; diff --git a/minifi_rust/extensions/minifi_rs_playground/src/processors/put_file/relationships.rs b/minifi_rust/extensions/minifi_rs_playground/src/processors/put_file/relationships.rs index a3387b65b1..cd5c506eef 100644 --- a/minifi_rust/extensions/minifi_rs_playground/src/processors/put_file/relationships.rs +++ b/minifi_rust/extensions/minifi_rs_playground/src/processors/put_file/relationships.rs @@ -1,3 +1,20 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + use minifi_native::Relationship; pub(crate) const SUCCESS: Relationship = Relationship { diff --git a/minifi_rust/extensions/minifi_rs_playground/src/processors/put_file/tests.rs b/minifi_rust/extensions/minifi_rs_playground/src/processors/put_file/tests.rs index 076d8d270d..acdd196832 100644 --- a/minifi_rust/extensions/minifi_rs_playground/src/processors/put_file/tests.rs +++ b/minifi_rust/extensions/minifi_rs_playground/src/processors/put_file/tests.rs @@ -1,3 +1,20 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + use super::*; use crate::processors::put_file::relationships::{FAILURE, SUCCESS}; use minifi_native::{MockLogger, MockProcessContext}; diff --git a/minifi_rust/extensions/minifi_rs_playground/src/processors/put_file/unix_only_properties.rs b/minifi_rust/extensions/minifi_rs_playground/src/processors/put_file/unix_only_properties.rs index 8c98a861f8..f7cbd7811b 100644 --- a/minifi_rust/extensions/minifi_rs_playground/src/processors/put_file/unix_only_properties.rs +++ b/minifi_rust/extensions/minifi_rs_playground/src/processors/put_file/unix_only_properties.rs @@ -1,3 +1,20 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + use minifi_native::{Property, StandardPropertyValidator}; pub(crate) const PERMISSIONS: Property = Property { diff --git a/minifi_rust/extensions/minifi_rs_playground/src/processors/zoo_processor.rs b/minifi_rust/extensions/minifi_rs_playground/src/processors/zoo_processor.rs index 52295dfe30..05661a2d31 100644 --- a/minifi_rust/extensions/minifi_rs_playground/src/processors/zoo_processor.rs +++ b/minifi_rust/extensions/minifi_rs_playground/src/processors/zoo_processor.rs @@ -1,3 +1,20 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + use crate::controller_services::animal_controller_apis::{ CanFlyControllerApi, NumberOfLegsControllerApi, }; diff --git a/minifi_rust/minifi_native/src/api.rs b/minifi_rust/minifi_native/src/api.rs index 494a998b63..6152b94ca5 100644 --- a/minifi_rust/minifi_native/src/api.rs +++ b/minifi_rust/minifi_native/src/api.rs @@ -1,3 +1,20 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + pub(crate) mod attribute; pub(crate) mod component_definition_traits; pub(crate) mod controller_service; diff --git a/minifi_rust/minifi_native/src/api/attribute.rs b/minifi_rust/minifi_native/src/api/attribute.rs index f80b9eff42..9e6b316ebd 100644 --- a/minifi_rust/minifi_native/src/api/attribute.rs +++ b/minifi_rust/minifi_native/src/api/attribute.rs @@ -1,3 +1,20 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + use crate::MinifiError; pub struct OutputAttribute { diff --git a/minifi_rust/minifi_native/src/api/component_definition_traits.rs b/minifi_rust/minifi_native/src/api/component_definition_traits.rs index 947c10518e..b0d67b2a94 100644 --- a/minifi_rust/minifi_native/src/api/component_definition_traits.rs +++ b/minifi_rust/minifi_native/src/api/component_definition_traits.rs @@ -1,3 +1,20 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + use crate::api::provided_interface::ProvidedInterface; use crate::{OutputAttribute, ProcessorInputRequirement, Property, Relationship}; diff --git a/minifi_rust/minifi_native/src/api/controller_service.rs b/minifi_rust/minifi_native/src/api/controller_service.rs index 875dc02766..70ad597204 100644 --- a/minifi_rust/minifi_native/src/api/controller_service.rs +++ b/minifi_rust/minifi_native/src/api/controller_service.rs @@ -1,3 +1,20 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + use crate::api::RawControllerService; use crate::{ComponentIdentifier, GetProperty, LogLevel, Logger, MinifiError}; diff --git a/minifi_rust/minifi_native/src/api/errors.rs b/minifi_rust/minifi_native/src/api/errors.rs index b0baf22d2a..19f1f705a7 100644 --- a/minifi_rust/minifi_native/src/api/errors.rs +++ b/minifi_rust/minifi_native/src/api/errors.rs @@ -1,3 +1,20 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + use minifi_native_sys::minifi_status; use std::borrow::Cow; use std::ffi::NulError; diff --git a/minifi_rust/minifi_native/src/api/flow_file.rs b/minifi_rust/minifi_native/src/api/flow_file.rs index 81aa2bb734..d0d0cd81ea 100644 --- a/minifi_rust/minifi_native/src/api/flow_file.rs +++ b/minifi_rust/minifi_native/src/api/flow_file.rs @@ -1,3 +1,20 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + use crate::MinifiError; pub trait FlowFile {} diff --git a/minifi_rust/minifi_native/src/api/logger.rs b/minifi_rust/minifi_native/src/api/logger.rs index a7c3fbfacb..b7df518176 100644 --- a/minifi_rust/minifi_native/src/api/logger.rs +++ b/minifi_rust/minifi_native/src/api/logger.rs @@ -1,3 +1,20 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + use std::fmt; use std::fmt::Debug; diff --git a/minifi_rust/minifi_native/src/api/process_context.rs b/minifi_rust/minifi_native/src/api/process_context.rs index 1a967ffea3..0f114cf479 100644 --- a/minifi_rust/minifi_native/src/api/process_context.rs +++ b/minifi_rust/minifi_native/src/api/process_context.rs @@ -1,3 +1,20 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + use crate::StandardPropertyValidator::*; use crate::api::RawControllerService; use crate::api::component_definition_traits::ComponentIdentifier; diff --git a/minifi_rust/minifi_native/src/api/process_session.rs b/minifi_rust/minifi_native/src/api/process_session.rs index f58dc9cc60..fbcfa4475e 100644 --- a/minifi_rust/minifi_native/src/api/process_session.rs +++ b/minifi_rust/minifi_native/src/api/process_session.rs @@ -1,3 +1,20 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + use crate::MinifiError; use crate::api::flow_file::FlowFile; pub trait InputStream: std::io::BufRead + Send + std::fmt::Debug {} diff --git a/minifi_rust/minifi_native/src/api/processor.rs b/minifi_rust/minifi_native/src/api/processor.rs index c8900036f0..8ac4bfb12f 100644 --- a/minifi_rust/minifi_native/src/api/processor.rs +++ b/minifi_rust/minifi_native/src/api/processor.rs @@ -1,3 +1,20 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + use crate::api::{RawProcessor, ThreadingModel}; use crate::{GetProperty, LogLevel, Logger, MinifiError, ProcessContext}; use std::marker::PhantomData; diff --git a/minifi_rust/minifi_native/src/api/processor_wrappers.rs b/minifi_rust/minifi_native/src/api/processor_wrappers.rs index ee5827f785..e0ec668319 100644 --- a/minifi_rust/minifi_native/src/api/processor_wrappers.rs +++ b/minifi_rust/minifi_native/src/api/processor_wrappers.rs @@ -1,3 +1,20 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + pub(crate) mod complex_processor; pub(crate) mod flow_file_source; pub(crate) mod flow_file_stream_transform; diff --git a/minifi_rust/minifi_native/src/api/processor_wrappers/complex_processor.rs b/minifi_rust/minifi_native/src/api/processor_wrappers/complex_processor.rs index 5f79e7a0df..24f29384ff 100644 --- a/minifi_rust/minifi_native/src/api/processor_wrappers/complex_processor.rs +++ b/minifi_rust/minifi_native/src/api/processor_wrappers/complex_processor.rs @@ -1,3 +1,20 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + use crate::api::raw_processor::{MultiThreadedTrigger, SingleThreadedTrigger}; use crate::{ ComponentIdentifier, Logger, MinifiError, MultiThreaded, OnTriggerResult, ProcessContext, diff --git a/minifi_rust/minifi_native/src/api/processor_wrappers/flow_file_source.rs b/minifi_rust/minifi_native/src/api/processor_wrappers/flow_file_source.rs index 1a64a874a8..1f049ff125 100644 --- a/minifi_rust/minifi_native/src/api/processor_wrappers/flow_file_source.rs +++ b/minifi_rust/minifi_native/src/api/processor_wrappers/flow_file_source.rs @@ -1,3 +1,20 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + use crate::api::processor_wrappers::utils::flow_file_content::Content; use crate::api::raw_processor::{MultiThreadedTrigger, SingleThreadedTrigger}; use crate::{ diff --git a/minifi_rust/minifi_native/src/api/processor_wrappers/flow_file_stream_transform.rs b/minifi_rust/minifi_native/src/api/processor_wrappers/flow_file_stream_transform.rs index aa62edd920..ece4244b90 100644 --- a/minifi_rust/minifi_native/src/api/processor_wrappers/flow_file_stream_transform.rs +++ b/minifi_rust/minifi_native/src/api/processor_wrappers/flow_file_stream_transform.rs @@ -1,3 +1,20 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + use crate::api::process_session::IoState; use crate::api::processor_wrappers::utils::context_session_flowfile_bundle::ContextSessionFlowFileBundle; use crate::api::raw_processor::{MultiThreadedTrigger, SingleThreadedTrigger}; diff --git a/minifi_rust/minifi_native/src/api/processor_wrappers/flow_file_transform.rs b/minifi_rust/minifi_native/src/api/processor_wrappers/flow_file_transform.rs index 2d97f23c91..74f8a0ee1d 100644 --- a/minifi_rust/minifi_native/src/api/processor_wrappers/flow_file_transform.rs +++ b/minifi_rust/minifi_native/src/api/processor_wrappers/flow_file_transform.rs @@ -1,3 +1,20 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + use crate::api::InputStream; use crate::api::flow_file::GetId; use crate::api::processor::Processor; diff --git a/minifi_rust/minifi_native/src/api/processor_wrappers/utils.rs b/minifi_rust/minifi_native/src/api/processor_wrappers/utils.rs index e10592e96c..e22b3bab93 100644 --- a/minifi_rust/minifi_native/src/api/processor_wrappers/utils.rs +++ b/minifi_rust/minifi_native/src/api/processor_wrappers/utils.rs @@ -1,2 +1,19 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + pub(crate) mod context_session_flowfile_bundle; pub(crate) mod flow_file_content; diff --git a/minifi_rust/minifi_native/src/api/processor_wrappers/utils/context_session_flowfile_bundle.rs b/minifi_rust/minifi_native/src/api/processor_wrappers/utils/context_session_flowfile_bundle.rs index 72bc5933b6..86817f7e34 100644 --- a/minifi_rust/minifi_native/src/api/processor_wrappers/utils/context_session_flowfile_bundle.rs +++ b/minifi_rust/minifi_native/src/api/processor_wrappers/utils/context_session_flowfile_bundle.rs @@ -1,3 +1,20 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + use crate::api::attribute::GetAttribute; use crate::api::flow_file::GetId; use crate::api::property::{GetControllerService, GetProperty}; diff --git a/minifi_rust/minifi_native/src/api/processor_wrappers/utils/flow_file_content.rs b/minifi_rust/minifi_native/src/api/processor_wrappers/utils/flow_file_content.rs index 6bd5492605..bf1dcf210c 100644 --- a/minifi_rust/minifi_native/src/api/processor_wrappers/utils/flow_file_content.rs +++ b/minifi_rust/minifi_native/src/api/processor_wrappers/utils/flow_file_content.rs @@ -1,3 +1,20 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + use std::fmt::Formatter; pub enum Content<'a> { diff --git a/minifi_rust/minifi_native/src/api/property.rs b/minifi_rust/minifi_native/src/api/property.rs index 63f8210a58..952b6eb688 100644 --- a/minifi_rust/minifi_native/src/api/property.rs +++ b/minifi_rust/minifi_native/src/api/property.rs @@ -1,3 +1,20 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + use crate::StandardPropertyValidator::{ BoolValidator, DataSizeValidator, TimePeriodValidator, U64Validator, }; diff --git a/minifi_rust/minifi_native/src/api/provided_interface.rs b/minifi_rust/minifi_native/src/api/provided_interface.rs index bee680e128..2f1f148471 100644 --- a/minifi_rust/minifi_native/src/api/provided_interface.rs +++ b/minifi_rust/minifi_native/src/api/provided_interface.rs @@ -1,3 +1,20 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + pub trait ControllerServiceApi { const INTERFACE_NAME: &'static str; } diff --git a/minifi_rust/minifi_native/src/api/raw_controller_service.rs b/minifi_rust/minifi_native/src/api/raw_controller_service.rs index 3348a398e0..2a38b89bd0 100644 --- a/minifi_rust/minifi_native/src/api/raw_controller_service.rs +++ b/minifi_rust/minifi_native/src/api/raw_controller_service.rs @@ -1,3 +1,20 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + use crate::{GetProperty, LogLevel, Logger, MinifiError}; /// This RawControllerService will be instantiated, and called on by the agent diff --git a/minifi_rust/minifi_native/src/api/raw_processor.rs b/minifi_rust/minifi_native/src/api/raw_processor.rs index 7fd6f3d0ca..51a4a05504 100644 --- a/minifi_rust/minifi_native/src/api/raw_processor.rs +++ b/minifi_rust/minifi_native/src/api/raw_processor.rs @@ -1,3 +1,20 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + use crate::api::errors::MinifiError; use crate::{LogLevel, Logger, ProcessContext, ProcessSession}; diff --git a/minifi_rust/minifi_native/src/api/relationship.rs b/minifi_rust/minifi_native/src/api/relationship.rs index dc80477fa8..45041a2ba5 100644 --- a/minifi_rust/minifi_native/src/api/relationship.rs +++ b/minifi_rust/minifi_native/src/api/relationship.rs @@ -1,3 +1,20 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + #[derive(Debug, Eq, PartialEq)] pub struct Relationship { pub name: &'static str, diff --git a/minifi_rust/minifi_native/src/c_ffi.rs b/minifi_rust/minifi_native/src/c_ffi.rs index d426cda447..878af16e23 100644 --- a/minifi_rust/minifi_native/src/c_ffi.rs +++ b/minifi_rust/minifi_native/src/c_ffi.rs @@ -1,3 +1,20 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + mod c_ffi_controller_service_context; mod c_ffi_controller_service_definition; mod c_ffi_controller_service_list; diff --git a/minifi_rust/minifi_native/src/c_ffi/c_ffi_controller_service_context.rs b/minifi_rust/minifi_native/src/c_ffi/c_ffi_controller_service_context.rs index 87cbbf4d29..2284c8dc75 100644 --- a/minifi_rust/minifi_native/src/c_ffi/c_ffi_controller_service_context.rs +++ b/minifi_rust/minifi_native/src/c_ffi/c_ffi_controller_service_context.rs @@ -1,3 +1,20 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + use crate::c_ffi::c_ffi_primitives::StringView; use crate::{GetProperty, MinifiError, Property}; use minifi_native_sys::{ diff --git a/minifi_rust/minifi_native/src/c_ffi/c_ffi_controller_service_definition.rs b/minifi_rust/minifi_native/src/c_ffi/c_ffi_controller_service_definition.rs index 5173959cf2..738071dc7a 100644 --- a/minifi_rust/minifi_native/src/c_ffi/c_ffi_controller_service_definition.rs +++ b/minifi_rust/minifi_native/src/c_ffi/c_ffi_controller_service_definition.rs @@ -1,3 +1,20 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + use crate::api::RawControllerService; use crate::c_ffi::c_ffi_controller_service_context::CffiControllerServiceContext; use crate::c_ffi::c_ffi_property::CProperties; diff --git a/minifi_rust/minifi_native/src/c_ffi/c_ffi_controller_service_list.rs b/minifi_rust/minifi_native/src/c_ffi/c_ffi_controller_service_list.rs index 1a8a582b63..54a7ad35ba 100644 --- a/minifi_rust/minifi_native/src/c_ffi/c_ffi_controller_service_list.rs +++ b/minifi_rust/minifi_native/src/c_ffi/c_ffi_controller_service_list.rs @@ -1,3 +1,20 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + use crate::c_ffi::RegisterableControllerService; use crate::c_ffi::c_ffi_controller_service_definition::DynRawControllerServiceDefinition; use minifi_native_sys::minifi_controller_service_definition; diff --git a/minifi_rust/minifi_native/src/c_ffi/c_ffi_flow_file.rs b/minifi_rust/minifi_native/src/c_ffi/c_ffi_flow_file.rs index 0fdeac4b50..580c04626c 100644 --- a/minifi_rust/minifi_native/src/c_ffi/c_ffi_flow_file.rs +++ b/minifi_rust/minifi_native/src/c_ffi/c_ffi_flow_file.rs @@ -1,3 +1,20 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + use crate::api::FlowFile; use minifi_native_sys::minifi_flow_file; diff --git a/minifi_rust/minifi_native/src/c_ffi/c_ffi_logger.rs b/minifi_rust/minifi_native/src/c_ffi/c_ffi_logger.rs index 78fa29f0e4..2a6cb8dedb 100644 --- a/minifi_rust/minifi_native/src/c_ffi/c_ffi_logger.rs +++ b/minifi_rust/minifi_native/src/c_ffi/c_ffi_logger.rs @@ -1,3 +1,20 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + use crate::api::LogLevel; use crate::api::Logger; use minifi_native_sys::{ diff --git a/minifi_rust/minifi_native/src/c_ffi/c_ffi_output_attribute.rs b/minifi_rust/minifi_native/src/c_ffi/c_ffi_output_attribute.rs index 40858bdd04..3597d671d5 100644 --- a/minifi_rust/minifi_native/src/c_ffi/c_ffi_output_attribute.rs +++ b/minifi_rust/minifi_native/src/c_ffi/c_ffi_output_attribute.rs @@ -1,3 +1,20 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + use crate::OutputAttribute; use crate::c_ffi::StaticStrAsMinifiCStr; use minifi_native_sys::{minifi_output_attribute_definition, minifi_string_view}; diff --git a/minifi_rust/minifi_native/src/c_ffi/c_ffi_primitives.rs b/minifi_rust/minifi_native/src/c_ffi/c_ffi_primitives.rs index 660c860e26..7892b2f895 100644 --- a/minifi_rust/minifi_native/src/c_ffi/c_ffi_primitives.rs +++ b/minifi_rust/minifi_native/src/c_ffi/c_ffi_primitives.rs @@ -1,3 +1,20 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + use crate::ProcessorInputRequirement; use minifi_native_sys::{ minifi_input_requirement, minifi_input_requirement_MINIFI_INPUT_ALLOWED, diff --git a/minifi_rust/minifi_native/src/c_ffi/c_ffi_process_context.rs b/minifi_rust/minifi_native/src/c_ffi/c_ffi_process_context.rs index a887ab7130..60dfc033c0 100644 --- a/minifi_rust/minifi_native/src/c_ffi/c_ffi_process_context.rs +++ b/minifi_rust/minifi_native/src/c_ffi/c_ffi_process_context.rs @@ -1,3 +1,20 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + use super::c_ffi_flow_file::CffiFlowFile; use super::c_ffi_primitives::StringView; use crate::api::ProcessContext; diff --git a/minifi_rust/minifi_native/src/c_ffi/c_ffi_process_session.rs b/minifi_rust/minifi_native/src/c_ffi/c_ffi_process_session.rs index 3c137fcccf..73a2e26368 100644 --- a/minifi_rust/minifi_native/src/c_ffi/c_ffi_process_session.rs +++ b/minifi_rust/minifi_native/src/c_ffi/c_ffi_process_session.rs @@ -1,3 +1,20 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + use super::c_ffi_flow_file::CffiFlowFile; use crate::MinifiError; use crate::api::process_session::{IoState, OutputStream}; diff --git a/minifi_rust/minifi_native/src/c_ffi/c_ffi_processor_definition.rs b/minifi_rust/minifi_native/src/c_ffi/c_ffi_processor_definition.rs index cb279403c6..4f59009511 100644 --- a/minifi_rust/minifi_native/src/c_ffi/c_ffi_processor_definition.rs +++ b/minifi_rust/minifi_native/src/c_ffi/c_ffi_processor_definition.rs @@ -1,3 +1,20 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + use std::ffi::c_void; use std::ptr; diff --git a/minifi_rust/minifi_native/src/c_ffi/c_ffi_processor_list.rs b/minifi_rust/minifi_native/src/c_ffi/c_ffi_processor_list.rs index 7efea7d524..bdb8cc11b4 100644 --- a/minifi_rust/minifi_native/src/c_ffi/c_ffi_processor_list.rs +++ b/minifi_rust/minifi_native/src/c_ffi/c_ffi_processor_list.rs @@ -1,3 +1,20 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + use crate::c_ffi::RawRegisterableProcessor; use crate::c_ffi::c_ffi_processor_definition::DynRawProcessorDefinition; use minifi_native_sys::minifi_processor_definition; diff --git a/minifi_rust/minifi_native/src/c_ffi/c_ffi_property.rs b/minifi_rust/minifi_native/src/c_ffi/c_ffi_property.rs index bfb7d61395..d2fe2c1b7d 100644 --- a/minifi_rust/minifi_native/src/c_ffi/c_ffi_property.rs +++ b/minifi_rust/minifi_native/src/c_ffi/c_ffi_property.rs @@ -1,3 +1,20 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + use super::c_ffi_primitives::StaticStrAsMinifiCStr; use crate::{Property, StandardPropertyValidator}; use minifi_native_sys::{ diff --git a/minifi_rust/minifi_native/src/c_ffi/c_ffi_relationship.rs b/minifi_rust/minifi_native/src/c_ffi/c_ffi_relationship.rs index 86d3ee9083..a1895afff4 100644 --- a/minifi_rust/minifi_native/src/c_ffi/c_ffi_relationship.rs +++ b/minifi_rust/minifi_native/src/c_ffi/c_ffi_relationship.rs @@ -1,3 +1,20 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + use super::c_ffi_primitives::StaticStrAsMinifiCStr; use crate::Relationship; use minifi_native_sys::minifi_relationship_definition; diff --git a/minifi_rust/minifi_native/src/c_ffi/c_ffi_streams.rs b/minifi_rust/minifi_native/src/c_ffi/c_ffi_streams.rs index f2923a93a7..6b191e5916 100644 --- a/minifi_rust/minifi_native/src/c_ffi/c_ffi_streams.rs +++ b/minifi_rust/minifi_native/src/c_ffi/c_ffi_streams.rs @@ -1,3 +1,20 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + use minifi_native_sys::{ minifi_input_stream, minifi_input_stream_read, minifi_output_stream, minifi_output_stream_write, }; diff --git a/minifi_rust/minifi_native/src/lib.rs b/minifi_rust/minifi_native/src/lib.rs index a91544290f..c467b68e87 100644 --- a/minifi_rust/minifi_native/src/lib.rs +++ b/minifi_rust/minifi_native/src/lib.rs @@ -1,3 +1,20 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + mod api; pub mod c_ffi; pub mod mock; diff --git a/minifi_rust/minifi_native/src/mock.rs b/minifi_rust/minifi_native/src/mock.rs index e98b637468..49e7c7c8d6 100644 --- a/minifi_rust/minifi_native/src/mock.rs +++ b/minifi_rust/minifi_native/src/mock.rs @@ -1,3 +1,20 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + mod mock_controller_service_context; mod mock_flow_file; mod mock_logger; diff --git a/minifi_rust/minifi_native/src/mock/mock_controller_service_context.rs b/minifi_rust/minifi_native/src/mock/mock_controller_service_context.rs index 2fd6924ffd..2a594ef8a9 100644 --- a/minifi_rust/minifi_native/src/mock/mock_controller_service_context.rs +++ b/minifi_rust/minifi_native/src/mock/mock_controller_service_context.rs @@ -1,3 +1,20 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + use crate::mock::mock_process_context::MockPropertyMap; use crate::{GetProperty, MinifiError, Property}; diff --git a/minifi_rust/minifi_native/src/mock/mock_flow_file.rs b/minifi_rust/minifi_native/src/mock/mock_flow_file.rs index 96ca074590..c998335a24 100644 --- a/minifi_rust/minifi_native/src/mock/mock_flow_file.rs +++ b/minifi_rust/minifi_native/src/mock/mock_flow_file.rs @@ -1,3 +1,20 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + use crate::api::FlowFile; use std::cell::RefCell; use std::collections::HashMap; diff --git a/minifi_rust/minifi_native/src/mock/mock_logger.rs b/minifi_rust/minifi_native/src/mock/mock_logger.rs index 73dbdd40fb..2ac4854775 100644 --- a/minifi_rust/minifi_native/src/mock/mock_logger.rs +++ b/minifi_rust/minifi_native/src/mock/mock_logger.rs @@ -1,3 +1,20 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + use crate::LogLevel::Trace; use crate::api::LogLevel; use crate::api::Logger; diff --git a/minifi_rust/minifi_native/src/mock/mock_process_context.rs b/minifi_rust/minifi_native/src/mock/mock_process_context.rs index 3ea6e015b2..be05daa16d 100644 --- a/minifi_rust/minifi_native/src/mock/mock_process_context.rs +++ b/minifi_rust/minifi_native/src/mock/mock_process_context.rs @@ -1,3 +1,20 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + use crate::api::{GetId, ProcessContext, RawControllerService}; use crate::{ ComponentIdentifier, ControllerServiceApi, EnableControllerService, GetAttribute, MinifiError, diff --git a/minifi_rust/minifi_native/src/mock/mock_process_session.rs b/minifi_rust/minifi_native/src/mock/mock_process_session.rs index c3ba3fc546..05f4766e03 100644 --- a/minifi_rust/minifi_native/src/mock/mock_process_session.rs +++ b/minifi_rust/minifi_native/src/mock/mock_process_session.rs @@ -1,3 +1,20 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + use crate::api::process_session::IoState; use crate::api::{InputStream, ProcessSession}; use crate::{MinifiError, MockFlowFile}; diff --git a/minifi_rust/minifi_native_macros/src/lib.rs b/minifi_rust/minifi_native_macros/src/lib.rs index 4b486e84f9..fa8e563361 100644 --- a/minifi_rust/minifi_native_macros/src/lib.rs +++ b/minifi_rust/minifi_native_macros/src/lib.rs @@ -1,3 +1,20 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + use proc_macro::TokenStream; use quote::quote; use syn::{DeriveInput, ItemTrait, parse_macro_input}; diff --git a/minifi_rust/minifi_native_sys/build.rs b/minifi_rust/minifi_native_sys/build.rs index 51435cfa4e..3403c4bbe3 100644 --- a/minifi_rust/minifi_native_sys/build.rs +++ b/minifi_rust/minifi_native_sys/build.rs @@ -1,3 +1,20 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + use std::env; use std::path::{Path, PathBuf}; use std::process::Command; diff --git a/minifi_rust/minifi_native_sys/src/lib.rs b/minifi_rust/minifi_native_sys/src/lib.rs index fd4dddab9a..f5cfecc7b6 100644 --- a/minifi_rust/minifi_native_sys/src/lib.rs +++ b/minifi_rust/minifi_native_sys/src/lib.rs @@ -1,3 +1,20 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + // minifi-sys/src/lib.rs #![allow(non_upper_case_globals)] diff --git a/minifi_rust/minifi_rs_behave/build.rs b/minifi_rust/minifi_rs_behave/build.rs index a675fa0e2b..680d75d895 100644 --- a/minifi_rust/minifi_rs_behave/build.rs +++ b/minifi_rust/minifi_rs_behave/build.rs @@ -1,3 +1,20 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + fn main() { println!("cargo:rerun-if-env-changed=DEP_MINIFI_BEHAVE_PATH"); let behave_path = diff --git a/minifi_rust/minifi_rs_behave/src/main.rs b/minifi_rust/minifi_rs_behave/src/main.rs index c30d3c0c1c..f0d8c12bc4 100644 --- a/minifi_rust/minifi_rs_behave/src/main.rs +++ b/minifi_rust/minifi_rs_behave/src/main.rs @@ -1,3 +1,20 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + use glob::glob; use std::env; use std::path::{Path, PathBuf}; From 4074b0d2dddfc9d2aad1fd863618b6d9e5838232 Mon Sep 17 00:00:00 2001 From: Martin Zink Date: Mon, 20 Jul 2026 13:44:27 +0200 Subject: [PATCH 25/26] update README.md --- minifi_rust/README.md | 31 +++++++++++++++++++++++-------- 1 file changed, 23 insertions(+), 8 deletions(-) diff --git a/minifi_rust/README.md b/minifi_rust/README.md index abcd621c67..69d9602b3e 100644 --- a/minifi_rust/README.md +++ b/minifi_rust/README.md @@ -2,11 +2,20 @@ > ⚠️ **Tech Preview** > -> The Rust bindings are a technology preview. The API surface (both the -> `minifi-native` safe API and the `minifi-native-sys` FFI layer) is still -> evolving, and we are **not yet committed to backward compatibility** for -> the Rust bindings. Expect breaking changes between MiNiFi C++ releases -> until the bindings are declared stable. +> Only the **Rust bindings** are a technology preview. The Rust source API +> (both the `minifi-native` safe API and the `minifi-native-sys` FFI layer) +> is still evolving, and we are **not yet committed to source-level backward +> compatibility** for the Rust bindings — expect breaking changes in the +> Rust API between MiNiFi C++ releases until the bindings are declared +> stable. +> +> Under the hood, the bindings target the **stable MiNiFi C API**, which +> *does* provide backward compatibility. This means the compiled artifact +> is unaffected by Rust-side churn: an extension built against an older +> version of these bindings will keep loading into newer MiNiFi C++ +> releases. You only need to rebuild against the new bindings if you want +> to pick up new Rust API features — upgrading the agent alone does not +> force a rebuild. This repository provides a safe, idiomatic, and high-performance Rust framework for building native extensions (processors) for [Apache NiFi MiNiFi C++](https://github.com/apache/nifi-minifi-cpp). @@ -32,9 +41,15 @@ Provides the public, safe, and idiomatic Rust API. This is the crate that develo #### API Traits Pure Rust traits (Processor, ProcessSession, Logger, etc.) that define the abstract behavior of the MiNiFi environment. #### Higher level API -Pure rust traits that simplify the requirements for a working processor - - FlowFileTransform - - FlowFileSource +Pure rust traits that simplify the requirements for a working processor. Pick the one that matches your processor's shape — the wrapper takes care of getting/creating the flowfile, wiring up streams, applying attributes, and transferring to the right relationship, so your code only needs to describe the transformation itself. + - [FlowFileTransform](minifi_native/src/api/processor_wrappers/flow_file_transform.rs) + - Consumes a single incoming flowfile, optionally rewrites its content (returned as an in-memory buffer) and/or adds attributes, then routes it to a relationship. Best for buffered, one-in / one-out transforms. + - [FlowFileStreamingTransform](minifi_native/src/api/processor_wrappers/flow_file_stream_transform.rs) + - Same one-in / one-out shape as `FlowFileTransform`, but you get both an `InputStream` and an `OutputStream` and write content incrementally. Use this when the payload is too large to hold in memory or when you want to stream data through as you process it. + - [FlowFileSource](minifi_native/src/api/processor_wrappers/flow_file_source.rs) + - Produces zero or more new flowfiles per trigger without consuming an input. Use this for processors that generate data (timers, pollers, external fetchers) — return an empty vec to yield when there is nothing to emit. + - [ComplexProcessor](minifi_native/src/api/processor_wrappers/complex_processor.rs) + - Escape hatch that hands you the raw `ProcessContext`, `ProcessSession`, and `Logger` — the same shape as MiNiFi C++'s `onTrigger` or NiFi Java's `Processor#onTrigger`. Use this when your processor doesn't fit the one-in / one-out / source molds: batching multiple flowfiles per trigger, custom routing logic, penalizing/rolling back, or anything that requires manual `session.get()` / `session.transfer()` bookkeeping. #### FFI Wrappers Concrete structs (CffiSession, CffiLogger, etc.) that implement the API traits by calling the unsafe functions from minifi-native-sys. #### Thread safety From ea4ef884d2a189c608b2ceda920e2c20f526b3f5 Mon Sep 17 00:00:00 2001 From: Martin Zink Date: Mon, 20 Jul 2026 14:56:43 +0200 Subject: [PATCH 26/26] fix clippy --- .../minifi_rs_playground/src/processors/log_attribute.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/minifi_rust/extensions/minifi_rs_playground/src/processors/log_attribute.rs b/minifi_rust/extensions/minifi_rs_playground/src/processors/log_attribute.rs index a9150c016d..adeee2ae30 100644 --- a/minifi_rust/extensions/minifi_rs_playground/src/processors/log_attribute.rs +++ b/minifi_rust/extensions/minifi_rs_playground/src/processors/log_attribute.rs @@ -57,7 +57,7 @@ impl LogAttributeRs { { return; } - log_msg.push_str(format!("\nkey:{} value:{}", &key, &value).as_str()); + log_msg.push_str(format!("\nkey:{} value:{}", key, value).as_str()); }); if self.log_payload { log_msg.push_str("\nPayload:\n");