From 9004aa002aa0802f7d02615449e5cd6d03985c20 Mon Sep 17 00:00:00 2001 From: Martin Zink Date: Wed, 8 Jul 2026 10:30:49 +0200 Subject: [PATCH 01/32] MINIFICPP-2892 Tensor extensions based on sonos/tract --- .../ubuntu_22_04_clang_arm_manifest.json | 626 ++++++++++++++++++ .../minifi_rs_playground.md | 1 + .../extensions/minifi_tensor/Cargo.toml | 25 + .../minifi_tensor/features/basic.feature | 33 + .../features/classification.feature | 128 ++++ .../minifi_tensor/features/detection.feature | 131 ++++ .../minifi_tensor/features/environment.py | 79 +++ .../features/image_to_tensor.feature | 105 +++ .../features/resources/.gitignore | 5 + .../minifi_tensor/features/steps/steps.py | 23 + .../extensions/minifi_tensor/minifi_tensor.md | 305 +++++++++ .../extensions/minifi_tensor/src/lib.rs | 53 ++ .../low_level_processors/classify_output.rs | 426 ++++++++++++ .../classify_output/classify_output_def.rs | 162 +++++ .../filter_bounding_boxes.rs | 485 ++++++++++++++ .../filter_bounding_boxes_def.rs | 153 +++++ .../low_level_processors/image_to_tensor.rs | 610 +++++++++++++++++ .../image_to_tensor/image_to_tensor_def.rs | 191 ++++++ .../invoke_tract_model.rs | 254 +++++++ .../invoke_tract_model_def.rs | 88 +++ .../src/low_level_processors/mod.rs | 5 + .../src/processors/classify_image.rs | 79 +++ .../classify_image/classify_object_def.rs | 81 +++ .../src/processors/detect_object.rs | 112 ++++ .../detect_object/detect_object_def.rs | 99 +++ .../src/processors/draw_bounding_box.rs | 165 +++++ .../minifi_tensor/src/processors/mod.rs | 20 + .../minifi_tensor/src/services/mod.rs | 18 + .../src/services/tract_model_service.rs | 179 +++++ .../tract_model_service/service_definition.rs | 50 ++ .../minifi_tensor/src/utils/bounding_box.rs | 181 +++++ .../minifi_tensor/src/utils/dimensions.rs | 50 ++ .../extensions/minifi_tensor/src/utils/mod.rs | 22 + .../src/utils/per_channel_f32.rs | 69 ++ .../src/utils/score_activation.rs | 16 + .../minifi_tensor/src/utils/tensor_helpers.rs | 156 +++++ minifi_rust/minifi_native/src/lib.rs | 13 +- minifi_rust/minifi_rs_behave/Cargo.toml | 1 - .../minifi_rs_behave/Dockerfile.alpine | 2 +- .../minifi_rs_behave/Dockerfile.debian | 4 +- minifi_rust/minifi_rs_behave/linux_build.sh | 9 + 41 files changed, 5205 insertions(+), 9 deletions(-) create mode 100644 minifi_rust/extensions/minifi_tensor/Cargo.toml create mode 100644 minifi_rust/extensions/minifi_tensor/features/basic.feature create mode 100644 minifi_rust/extensions/minifi_tensor/features/classification.feature create mode 100644 minifi_rust/extensions/minifi_tensor/features/detection.feature create mode 100644 minifi_rust/extensions/minifi_tensor/features/environment.py create mode 100644 minifi_rust/extensions/minifi_tensor/features/image_to_tensor.feature create mode 100644 minifi_rust/extensions/minifi_tensor/features/resources/.gitignore create mode 100644 minifi_rust/extensions/minifi_tensor/features/steps/steps.py create mode 100644 minifi_rust/extensions/minifi_tensor/minifi_tensor.md create mode 100644 minifi_rust/extensions/minifi_tensor/src/lib.rs create mode 100644 minifi_rust/extensions/minifi_tensor/src/low_level_processors/classify_output.rs create mode 100644 minifi_rust/extensions/minifi_tensor/src/low_level_processors/classify_output/classify_output_def.rs create mode 100644 minifi_rust/extensions/minifi_tensor/src/low_level_processors/filter_bounding_boxes.rs create mode 100644 minifi_rust/extensions/minifi_tensor/src/low_level_processors/filter_bounding_boxes/filter_bounding_boxes_def.rs create mode 100644 minifi_rust/extensions/minifi_tensor/src/low_level_processors/image_to_tensor.rs create mode 100644 minifi_rust/extensions/minifi_tensor/src/low_level_processors/image_to_tensor/image_to_tensor_def.rs create mode 100644 minifi_rust/extensions/minifi_tensor/src/low_level_processors/invoke_tract_model.rs create mode 100644 minifi_rust/extensions/minifi_tensor/src/low_level_processors/invoke_tract_model/invoke_tract_model_def.rs create mode 100644 minifi_rust/extensions/minifi_tensor/src/low_level_processors/mod.rs create mode 100644 minifi_rust/extensions/minifi_tensor/src/processors/classify_image.rs create mode 100644 minifi_rust/extensions/minifi_tensor/src/processors/classify_image/classify_object_def.rs create mode 100644 minifi_rust/extensions/minifi_tensor/src/processors/detect_object.rs create mode 100644 minifi_rust/extensions/minifi_tensor/src/processors/detect_object/detect_object_def.rs create mode 100644 minifi_rust/extensions/minifi_tensor/src/processors/draw_bounding_box.rs create mode 100644 minifi_rust/extensions/minifi_tensor/src/processors/mod.rs create mode 100644 minifi_rust/extensions/minifi_tensor/src/services/mod.rs create mode 100644 minifi_rust/extensions/minifi_tensor/src/services/tract_model_service.rs create mode 100644 minifi_rust/extensions/minifi_tensor/src/services/tract_model_service/service_definition.rs create mode 100644 minifi_rust/extensions/minifi_tensor/src/utils/bounding_box.rs create mode 100644 minifi_rust/extensions/minifi_tensor/src/utils/dimensions.rs create mode 100644 minifi_rust/extensions/minifi_tensor/src/utils/mod.rs create mode 100644 minifi_rust/extensions/minifi_tensor/src/utils/per_channel_f32.rs create mode 100644 minifi_rust/extensions/minifi_tensor/src/utils/score_activation.rs create mode 100644 minifi_rust/extensions/minifi_tensor/src/utils/tensor_helpers.rs diff --git a/.github/references/ubuntu_22_04_clang_arm_manifest.json b/.github/references/ubuntu_22_04_clang_arm_manifest.json index 139b3854d7..ec0bfbdf49 100644 --- a/.github/references/ubuntu_22_04_clang_arm_manifest.json +++ b/.github/references/ubuntu_22_04_clang_arm_manifest.json @@ -12772,5 +12772,631 @@ "artifact": "minifi_rs_playground", "version": "0.1.0" } +}, +{ + "bundles": { + "componentManifest": { + "processors": [ + { + "propertyDescriptors": { + "Color format": { + "name": "Color format", + "description": "Colour space of the tensor fed to the model. RGB and BGR produce three-channel tensors (channel order determined by the format); Grayscale produces a single-channel luma tensor.", + "validator": "VALID", + "required": "true", + "sensitive": "false", + "expressionLanguageScope": "NONE", + "defaultValue": "RGB", + "allowableValues": [ + { + "value": "RGB", + "displayName": "RGB" + }, + { + "value": "BGR", + "displayName": "BGR" + }, + { + "value": "GRAYSCALE", + "displayName": "GRAYSCALE" + } + ] + }, + "Confidence Threshold": { + "name": "Confidence Threshold", + "description": "Minimum confidence a class must reach to be included in the output JSON. Applied AFTER activation, so the units match the chosen activation (0.0..=1.0 for Softmax/Sigmoid, model-native for None). Set to 0.0 to always emit exactly Top K predictions.", + "validator": "NUMBER_VALIDATOR", + "required": "true", + "sensitive": "false", + "expressionLanguageScope": "FLOWFILE_ATTRIBUTES", + "defaultValue": "0.0" + }, + "Label index offset": { + "name": "Label index offset", + "description": "Offset added to the model's class ID when looking up a name in the labels file. Defaults to 0 (labels file line N = class N). Set to 1 for label files that start with a dummy/background entry — e.g. the ONNX MobileNetV2 model emits 1000 class scores while 'imagenet_slim_labels.txt' has 1001 lines (line 0 = 'dummy'), so class ID 653 maps to line 654 = 'military uniform'.", + "validator": "NON_NEGATIVE_INTEGER_VALIDATOR", + "required": "true", + "sensitive": "false", + "expressionLanguageScope": "NONE", + "defaultValue": "0" + }, + "Labels file path": { + "name": "Labels file path", + "description": "Optional path to a newline-separated labels file (line N = name of class N). Loaded once at service enable time. When set, each prediction in the output JSON gains a 'class_name' field and the 'class.top1.name' flow file attribute is populated. Leave empty to emit numeric class IDs only.", + "validator": "VALID", + "required": "false", + "sensitive": "false", + "expressionLanguageScope": "NONE" + }, + "Letterbox pad value": { + "name": "Letterbox pad value", + "description": "Value written for padding pixels when 'Resize mode' is 'Letterbox'. This is a normalised value (post mean/std), so 0.0 corresponds to a neutral input for most networks. Ignored when 'Resize mode' is 'Stretch'.", + "validator": "NUMBER_VALIDATOR", + "required": "true", + "sensitive": "false", + "expressionLanguageScope": "NONE", + "defaultValue": "0.0" + }, + "Mean": { + "name": "Mean", + "description": "Mean subtracted from each pixel before dividing by 'Standard Deviation'. Accepts either a single value (broadcast to all channels) or three comma-separated values applied per channel in the order dictated by 'Color format'. Example: '0.485, 0.456, 0.406' for ImageNet-style RGB normalisation.", + "validator": "VALID", + "required": "false", + "sensitive": "false", + "expressionLanguageScope": "NONE", + "defaultValue": "0.0" + }, + "Output attribute name": { + "name": "Output attribute name", + "description": "Specify the attribute to use as output, if not provided, the content is overridden instead.", + "validator": "VALID", + "required": "false", + "sensitive": "false", + "expressionLanguageScope": "FLOWFILE_ATTRIBUTES" + }, + "Pixel divisor": { + "name": "Pixel divisor", + "description": "Divisor applied to raw u8 pixel values before subtracting 'Mean' and dividing by 'Standard Deviation'. Defaults to 1.0 (mean/std interpreted in [0, 255] pixel space, e.g. UltraFace's mean=127, std=128). Set to 255 to bring pixels into [0.0, 1.0] first so ImageNet-style mean/std values like '0.485, 0.456, 0.406' / '0.229, 0.224, 0.225' can be used directly, matching the PyTorch / torchvision / ONNX MobileNet convention. Must be non-zero.", + "validator": "NUMBER_VALIDATOR", + "required": "true", + "sensitive": "false", + "expressionLanguageScope": "NONE", + "defaultValue": "1.0" + }, + "Resize filter": { + "name": "Resize filter", + "description": "Interpolation filter applied when resizing the decoded image. Nearest is fastest but blocky; Bilinear is a good default; Bicubic and Lanczos3 are higher-quality but slower.", + "validator": "VALID", + "required": "true", + "sensitive": "false", + "expressionLanguageScope": "NONE", + "defaultValue": "Bilinear", + "allowableValues": [ + { + "value": "Nearest", + "displayName": "Nearest" + }, + { + "value": "Bilinear", + "displayName": "Bilinear" + }, + { + "value": "Bicubic", + "displayName": "Bicubic" + }, + { + "value": "Lanczos3", + "displayName": "Lanczos3" + } + ] + }, + "Resize mode": { + "name": "Resize mode", + "description": "How the source image is fitted into the target dimensions. 'Stretch' scales each axis independently, distorting aspect ratio. 'Letterbox' preserves aspect ratio and pads the remaining border with 'Letterbox pad value' (applied in normalised output space).", + "validator": "VALID", + "required": "true", + "sensitive": "false", + "expressionLanguageScope": "NONE", + "defaultValue": "Stretch", + "allowableValues": [ + { + "value": "Stretch", + "displayName": "Stretch" + }, + { + "value": "Letterbox", + "displayName": "Letterbox" + } + ] + }, + "Score activation": { + "name": "Score activation", + "description": "Activation applied to the raw score vector before ranking. Softmax = mutually-exclusive classes (ImageNet-trained ResNet/MobileNet/EfficientNet raw logits). Sigmoid = independent classes (multi-label classifiers). None = the model already emits probabilities/scores; rank the raw values.", + "validator": "VALID", + "required": "true", + "sensitive": "false", + "expressionLanguageScope": "NONE", + "defaultValue": "Softmax", + "allowableValues": [ + { + "value": "Softmax", + "displayName": "Softmax" + }, + { + "value": "Sigmoid", + "displayName": "Sigmoid" + }, + { + "value": "None", + "displayName": "None" + } + ] + }, + "Score output index": { + "name": "Score output index", + "description": "Zero-based index of the model output tensor that holds classification scores. The processor slices the concatenated payload from InvokeTractModel according to the 'tensor.N.bytes' attributes. Almost always 0 for single-head classifiers.", + "validator": "NON_NEGATIVE_INTEGER_VALIDATOR", + "required": "true", + "sensitive": "false", + "expressionLanguageScope": "NONE", + "defaultValue": "0" + }, + "Standard Deviation": { + "name": "Standard Deviation", + "description": "Divisor applied after subtracting 'Mean'. Accepts a single value (broadcast) or three comma-separated values (per channel). Must be non-zero. Example: '255.0' to scale u8 pixels into [0.0, 1.0]; '0.229, 0.224, 0.225' for ImageNet.", + "validator": "VALID", + "required": "false", + "sensitive": "false", + "expressionLanguageScope": "NONE", + "defaultValue": "255.0" + }, + "Target height": { + "name": "Target height", + "description": "Height in pixels the decoded image is resized to before normalisation and inference.", + "validator": "NON_NEGATIVE_INTEGER_VALIDATOR", + "required": "true", + "sensitive": "false", + "expressionLanguageScope": "NONE" + }, + "Target width": { + "name": "Target width", + "description": "Width in pixels the decoded image is resized to before normalisation and inference.", + "validator": "NON_NEGATIVE_INTEGER_VALIDATOR", + "required": "true", + "sensitive": "false", + "expressionLanguageScope": "NONE" + }, + "Tensor shape format": { + "name": "Tensor shape format", + "description": "Memory layout of the tensor fed to the model. CHW (channels-first) is typical for PyTorch/ONNX detectors. HWC (channels-last) matches TensorFlow/TFLite. Ignored for Grayscale (always effectively 1xHxW).", + "validator": "VALID", + "required": "true", + "sensitive": "false", + "expressionLanguageScope": "NONE", + "defaultValue": "CHW", + "allowableValues": [ + { + "value": "CHW", + "displayName": "CHW" + }, + { + "value": "HWC", + "displayName": "HWC" + } + ] + }, + "Top K": { + "name": "Top K", + "description": "Number of highest-scoring classes to include in the output JSON, in descending order of confidence. Values above the total class count are clamped. Set to 1 for pure top-1 classification.", + "validator": "NON_NEGATIVE_INTEGER_VALIDATOR", + "required": "true", + "sensitive": "false", + "expressionLanguageScope": "NONE", + "defaultValue": "5" + }, + "Tract model service": { + "typeProvidedByValue": { + "type": "minifi_tensor.services.tract_model_service.TractModelService", + "group": "org.apache.nifi.minifi.rust", + "artifact": "minifi_tensor" + }, + "name": "Tract model service", + "description": "Reference to a TractModelService controller service. The referenced service owns the compiled model (ONNX or NNEF) that will be evaluated for each incoming flow file.", + "validator": "VALID", + "required": "true", + "sensitive": "false", + "expressionLanguageScope": "NONE" + } + }, + "inputRequirement": "INPUT_REQUIRED", + "isSingleThreaded": "false", + "supportedRelationships": [ + { + "name": "failure", + "description": "The image could not be decoded, the input tensor could not be built, the model failed to run, or the model outputs could not be interpreted as scores + boxes." + }, + { + "name": "success", + "description": "Inference and post-processing completed. The flow file content is the original, unchanged image; the classifications are written to the configured output attribute as a JSON array (may be empty)." + } + ], + "typeDescription": "Runs a full image-classification pass in a single processor: decodes the image from the flow file content, resizes and normalises it into an input tensor, runs one inference against the compiled model owned by the referenced TractModelService, and post-processes the score vector (score activation, Top-K selection, confidence filtering, optional label lookup) into predictions. Collapses the ImageToTensor -> InvokeTractModel -> ClassifyOutput chain into one node. The flow file content is left unchanged (the original image); the Top-K classifications are written as a JSON array to the configured output attribute.", + "supportsDynamicRelationships": "false", + "supportsDynamicProperties": "false", + "type": "minifi_tensor.processors.classify_image.ClassifyImage" + }, + { + "propertyDescriptors": { + "Background class index": { + "name": "Background class index", + "description": "Index of the 'background / no-object' class. Boxes whose winning class equals this index are dropped. In score-matrix mode this is only honoured when the score tensor has more than one class per box; in 'Class output index' mode it is matched against each box's class id.", + "validator": "NON_NEGATIVE_INTEGER_VALIDATOR", + "required": "false", + "sensitive": "false", + "expressionLanguageScope": "NONE" + }, + "Box format": { + "name": "Box format", + "description": "Layout of the four floats per box in the box output tensor. Xyxy = [x_min, y_min, x_max, y_max] (SSD, MobileNet-SSD, most PyTorch exports). Yxyx = [y_min, x_min, y_max, x_max] (TensorFlow Object Detection API). Cxcywh = [cx, cy, w, h] (YOLOv3/5/8 raw output).", + "validator": "VALID", + "required": "true", + "sensitive": "false", + "expressionLanguageScope": "NONE", + "defaultValue": "Xyxy", + "allowableValues": [ + { + "value": "Xyxy", + "displayName": "Xyxy" + }, + { + "value": "Yxyx", + "displayName": "Yxyx" + }, + { + "value": "Cxcywh", + "displayName": "Cxcywh" + } + ] + }, + "Box output index": { + "name": "Box output index", + "description": "Zero-based index of the model output tensor that holds box coordinates. Must differ from 'Score output index'.", + "validator": "NON_NEGATIVE_INTEGER_VALIDATOR", + "required": "true", + "sensitive": "false", + "expressionLanguageScope": "NONE", + "defaultValue": "1" + }, + "Class output index": { + "name": "Class output index", + "description": "Zero-based index of a model output tensor that holds one class id per box. Set this for detectors that emit boxes, per-box scores, and class ids as three separate parallel tensors, with NMS already folded into the graph (TensorFlow Object Detection API; YOLO / EfficientNMS 'end2end' exports). When set, 'Score output index' is read as one score per box (not a [boxes, classes] matrix) and no argmax is performed; the class id tensor may be integer- or float-typed. Leave empty for models that emit a per-class score matrix.", + "validator": "NON_NEGATIVE_INTEGER_VALIDATOR", + "required": "false", + "sensitive": "false", + "expressionLanguageScope": "NONE" + }, + "Color format": { + "name": "Color format", + "description": "Colour space of the tensor fed to the model. RGB and BGR produce three-channel tensors (channel order determined by the format); Grayscale produces a single-channel luma tensor.", + "validator": "VALID", + "required": "true", + "sensitive": "false", + "expressionLanguageScope": "NONE", + "defaultValue": "RGB", + "allowableValues": [ + { + "value": "RGB", + "displayName": "RGB" + }, + { + "value": "BGR", + "displayName": "BGR" + }, + { + "value": "GRAYSCALE", + "displayName": "GRAYSCALE" + } + ] + }, + "Confidence Threshold": { + "name": "Confidence Threshold", + "description": "Minimum per-box class probability (0.0 to 1.0) required to keep a bounding box after applying the chosen 'Score activation'. Boxes below the threshold are discarded before NMS.", + "validator": "NUMBER_VALIDATOR", + "required": "true", + "sensitive": "false", + "expressionLanguageScope": "FLOWFILE_ATTRIBUTES", + "defaultValue": "0.7" + }, + "IoU Threshold": { + "name": "IoU Threshold", + "description": "Intersection-over-union cutoff used during non-maximum suppression. Boxes of the same class whose IoU with a higher-confidence peer exceeds this value are suppressed. Typical values: 0.45 (SSD/YOLO default), 0.5, 0.3 for stricter deduplication.", + "validator": "NUMBER_VALIDATOR", + "required": "true", + "sensitive": "false", + "expressionLanguageScope": "NONE", + "defaultValue": "0.45" + }, + "Letterbox pad value": { + "name": "Letterbox pad value", + "description": "Value written for padding pixels when 'Resize mode' is 'Letterbox'. This is a normalised value (post mean/std), so 0.0 corresponds to a neutral input for most networks. Ignored when 'Resize mode' is 'Stretch'.", + "validator": "NUMBER_VALIDATOR", + "required": "true", + "sensitive": "false", + "expressionLanguageScope": "NONE", + "defaultValue": "0.0" + }, + "Mean": { + "name": "Mean", + "description": "Mean subtracted from each pixel before dividing by 'Standard Deviation'. Accepts either a single value (broadcast to all channels) or three comma-separated values applied per channel in the order dictated by 'Color format'. Example: '0.485, 0.456, 0.406' for ImageNet-style RGB normalisation.", + "validator": "VALID", + "required": "false", + "sensitive": "false", + "expressionLanguageScope": "NONE", + "defaultValue": "0.0" + }, + "Output attribute name": { + "name": "Output attribute name", + "description": "Specify the attribute to use as output, if not provided, the content is overridden instead.", + "validator": "VALID", + "required": "false", + "sensitive": "false", + "expressionLanguageScope": "FLOWFILE_ATTRIBUTES" + }, + "Pixel divisor": { + "name": "Pixel divisor", + "description": "Divisor applied to raw u8 pixel values before subtracting 'Mean' and dividing by 'Standard Deviation'. Defaults to 1.0 (mean/std interpreted in [0, 255] pixel space, e.g. UltraFace's mean=127, std=128). Set to 255 to bring pixels into [0.0, 1.0] first so ImageNet-style mean/std values like '0.485, 0.456, 0.406' / '0.229, 0.224, 0.225' can be used directly, matching the PyTorch / torchvision / ONNX MobileNet convention. Must be non-zero.", + "validator": "NUMBER_VALIDATOR", + "required": "true", + "sensitive": "false", + "expressionLanguageScope": "NONE", + "defaultValue": "1.0" + }, + "Resize filter": { + "name": "Resize filter", + "description": "Interpolation filter applied when resizing the decoded image. Nearest is fastest but blocky; Bilinear is a good default; Bicubic and Lanczos3 are higher-quality but slower.", + "validator": "VALID", + "required": "true", + "sensitive": "false", + "expressionLanguageScope": "NONE", + "defaultValue": "Bilinear", + "allowableValues": [ + { + "value": "Nearest", + "displayName": "Nearest" + }, + { + "value": "Bilinear", + "displayName": "Bilinear" + }, + { + "value": "Bicubic", + "displayName": "Bicubic" + }, + { + "value": "Lanczos3", + "displayName": "Lanczos3" + } + ] + }, + "Resize mode": { + "name": "Resize mode", + "description": "How the source image is fitted into the target dimensions. 'Stretch' scales each axis independently, distorting aspect ratio. 'Letterbox' preserves aspect ratio and pads the remaining border with 'Letterbox pad value' (applied in normalised output space).", + "validator": "VALID", + "required": "true", + "sensitive": "false", + "expressionLanguageScope": "NONE", + "defaultValue": "Stretch", + "allowableValues": [ + { + "value": "Stretch", + "displayName": "Stretch" + }, + { + "value": "Letterbox", + "displayName": "Letterbox" + } + ] + }, + "Score activation": { + "name": "Score activation", + "description": "Activation applied to raw per-class scores before selecting the winning class. Softmax = mutually-exclusive classes (SSD/MobileNet-SSD raw logits). Sigmoid = independent classes (YOLOv5/v8 style). None = the model already emits probabilities/scores; use raw argmax with the raw score as confidence.", + "validator": "VALID", + "required": "true", + "sensitive": "false", + "expressionLanguageScope": "NONE", + "defaultValue": "Softmax", + "allowableValues": [ + { + "value": "Softmax", + "displayName": "Softmax" + }, + { + "value": "Sigmoid", + "displayName": "Sigmoid" + }, + { + "value": "None", + "displayName": "None" + } + ] + }, + "Score output index": { + "name": "Score output index", + "description": "Zero-based index of the model output tensor that holds classification scores. The processor slices the concatenated payload from InvokeTractModel according to the 'tensor.N.bytes' attributes.", + "validator": "NON_NEGATIVE_INTEGER_VALIDATOR", + "required": "true", + "sensitive": "false", + "expressionLanguageScope": "NONE", + "defaultValue": "0" + }, + "Standard Deviation": { + "name": "Standard Deviation", + "description": "Divisor applied after subtracting 'Mean'. Accepts a single value (broadcast) or three comma-separated values (per channel). Must be non-zero. Example: '255.0' to scale u8 pixels into [0.0, 1.0]; '0.229, 0.224, 0.225' for ImageNet.", + "validator": "VALID", + "required": "false", + "sensitive": "false", + "expressionLanguageScope": "NONE", + "defaultValue": "255.0" + }, + "Target height": { + "name": "Target height", + "description": "Height in pixels the decoded image is resized to before normalisation and inference.", + "validator": "NON_NEGATIVE_INTEGER_VALIDATOR", + "required": "true", + "sensitive": "false", + "expressionLanguageScope": "NONE" + }, + "Target width": { + "name": "Target width", + "description": "Width in pixels the decoded image is resized to before normalisation and inference.", + "validator": "NON_NEGATIVE_INTEGER_VALIDATOR", + "required": "true", + "sensitive": "false", + "expressionLanguageScope": "NONE" + }, + "Tensor shape format": { + "name": "Tensor shape format", + "description": "Memory layout of the tensor fed to the model. CHW (channels-first) is typical for PyTorch/ONNX detectors. HWC (channels-last) matches TensorFlow/TFLite. Ignored for Grayscale (always effectively 1xHxW).", + "validator": "VALID", + "required": "true", + "sensitive": "false", + "expressionLanguageScope": "NONE", + "defaultValue": "CHW", + "allowableValues": [ + { + "value": "CHW", + "displayName": "CHW" + }, + { + "value": "HWC", + "displayName": "HWC" + } + ] + }, + "Tract model service": { + "typeProvidedByValue": { + "type": "minifi_tensor.services.tract_model_service.TractModelService", + "group": "org.apache.nifi.minifi.rust", + "artifact": "minifi_tensor" + }, + "name": "Tract model service", + "description": "Reference to a TractModelService controller service. The referenced service owns the compiled model (ONNX or NNEF) that will be evaluated for each incoming flow file.", + "validator": "VALID", + "required": "true", + "sensitive": "false", + "expressionLanguageScope": "NONE" + } + }, + "inputRequirement": "INPUT_REQUIRED", + "isSingleThreaded": "false", + "supportedRelationships": [ + { + "name": "failure", + "description": "The image could not be decoded, the input tensor could not be built, the model failed to run, or the model outputs could not be interpreted as scores + boxes." + }, + { + "name": "success", + "description": "Inference and post-processing completed. The flow file content is the original, unchanged image; the detected boxes are written to the configured output attribute as a JSON array (may be empty)." + } + ], + "typeDescription": "Runs a full object-detection pass in a single processor: decodes the image from the flow file content, resizes and normalises it into an input tensor, runs one inference against the compiled model owned by the referenced TractModelService, and post-processes the model outputs (score activation, confidence filtering, box decoding, per-class non-maximum suppression) into bounding boxes. Collapses the ImageToTensor -> InvokeTractModel -> FilterBoundingBoxes chain into one node. The flow file content is left unchanged (the original image); the detected boxes are written as a JSON array to the configured output attribute so a downstream DrawBoundingBox can annotate the image.", + "supportsDynamicRelationships": "false", + "supportsDynamicProperties": "false", + "type": "minifi_tensor.processors.detect_object.DetectObject" + }, + { + "propertyDescriptors": { + "Bounding boxes": { + "name": "Bounding boxes", + "description": "JSON array of bounding boxes to draw onto the image (fields class_id, confidence, x_min, y_min, x_max, y_max; coordinates normalised to [0,1] against the image). Typically the attribute produced by an upstream DetectObject or FilterBoundingBoxes processor.", + "validator": "VALID", + "required": "false", + "sensitive": "false", + "expressionLanguageScope": "NONE", + "defaultValue": "${enrichment.value}" + }, + "Line color": { + "name": "Line color", + "description": "Outline colour as '[R, G, B]' u8 channels (0-255).", + "validator": "VALID", + "required": "false", + "sensitive": "false", + "expressionLanguageScope": "NONE", + "defaultValue": "[0, 255, 0]" + }, + "Line thickness": { + "name": "Line thickness", + "description": "Thickness in pixels of the drawn box outline.", + "validator": "NON_NEGATIVE_INTEGER_VALIDATOR", + "required": "true", + "sensitive": "false", + "expressionLanguageScope": "NONE", + "defaultValue": "5" + } + }, + "inputRequirement": "INPUT_REQUIRED", + "isSingleThreaded": "false", + "supportedRelationships": [ + { + "name": "failure", + "description": "Invalid FlowFiles are routed here" + }, + { + "name": "success", + "description": "Flowfiles are routed here after drawing the bounding boxes" + } + ], + "typeDescription": "Decodes the image from the flow file content, draws each bounding box supplied via the 'Bounding boxes' property onto it, and re-encodes the annotated image as PNG. Pair with an upstream DetectObject / FilterBoundingBoxes to visualise detections.", + "supportsDynamicRelationships": "false", + "supportsDynamicProperties": "false", + "type": "minifi_tensor.processors.draw_bounding_box.DrawBoundingBox" + } + ], + "controllerServices": [ + { + "propertyDescriptors": { + "Model File Path": { + "name": "Model File Path", + "description": "Absolute path to the model on the edge device. For ONNX this is a `.onnx` file; for NNEF this is a `.nnef.tgz` archive, a `.nnef` tarball, or the root directory of an unpacked NNEF model. The model is loaded, parsed, and compiled for the host CPU once when the controller service is enabled; subsequent inference calls reuse the compiled runnable.", + "validator": "VALID", + "required": "true", + "sensitive": "false", + "expressionLanguageScope": "NONE" + }, + "Model format": { + "name": "Model format", + "description": "Format of the file/directory referenced by 'Model File Path'. 'Auto' picks Onnx when the path ends in `.onnx` and Nnef when it ends in `.nnef`, `.nnef.tgz`, `.nnef.tar`, `.nnef.tar.gz`, or points at a directory. Set explicitly when the path uses a non-standard extension.", + "validator": "VALID", + "required": "true", + "sensitive": "false", + "expressionLanguageScope": "NONE", + "defaultValue": "Auto", + "allowableValues": [ + { + "value": "Auto", + "displayName": "Auto" + }, + { + "value": "Onnx", + "displayName": "Onnx" + }, + { + "value": "Nnef", + "displayName": "Nnef" + } + ] + } + }, + "typeDescription": "Provides a shared, CPU-optimized neural network for inference. Supports ONNX (`.onnx`) and NNEF (directory or tarball) models; the format can be auto-detected from the file extension or set explicitly.", + "supportsDynamicRelationships": "false", + "supportsDynamicProperties": "false", + "type": "minifi_tensor.services.tract_model_service.TractModelService" + } + ] + }, + "group": "org.apache.nifi.minifi.rust", + "artifact": "minifi_tensor", + "version": "0.1.0" + } } ] diff --git a/minifi_rust/extensions/minifi_rs_playground/minifi_rs_playground.md b/minifi_rust/extensions/minifi_rs_playground/minifi_rs_playground.md index e05adf0e0e..74ecd008fd 100644 --- a/minifi_rust/extensions/minifi_rs_playground/minifi_rs_playground.md +++ b/minifi_rust/extensions/minifi_rs_playground/minifi_rs_playground.md @@ -218,6 +218,7 @@ In the list below, the names of required properties appear in bold. Any other pr | Name | Default Value | Allowable Values | Description | |------------------------------------|---------------|-------------------|--------------------------------------------| +| Dummy Controller Service | | | Optional dummy controller service | | **Lorem Ipsum Controller Service** | | | Name of the lorem ipsum controller service | | **Write Method** | Buffer | Buffer
Stream | Which API to test | diff --git a/minifi_rust/extensions/minifi_tensor/Cargo.toml b/minifi_rust/extensions/minifi_tensor/Cargo.toml new file mode 100644 index 0000000000..6c48a961d3 --- /dev/null +++ b/minifi_rust/extensions/minifi_tensor/Cargo.toml @@ -0,0 +1,25 @@ +[package] +name = "minifi_tensor" +version = "0.1.0" +edition = "2024" + +[lib] +crate-type = ["cdylib"] + +[features] +low-level-processors = [] +all = ["low-level-processors"] + +[dependencies] +minifi_native = { path = "../../minifi_native" } +strum = "0.28.0" +strum_macros = "0.28.0" +image = "0.25.10" +imageproc = "0.27.0" +tract = "0.23.4" +ndarray = "0.17.2" +serde = { version = "1.0.229", features = ["derive"] } +serde_json = "1.0.151" + +[dev-dependencies] +minifi_native = { path = "../../minifi_native", features = ["test-utils"] } diff --git a/minifi_rust/extensions/minifi_tensor/features/basic.feature b/minifi_rust/extensions/minifi_tensor/features/basic.feature new file mode 100644 index 0000000000..0718f98c1b --- /dev/null +++ b/minifi_rust/extensions/minifi_tensor/features/basic.feature @@ -0,0 +1,33 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +@SUPPORTS_WINDOWS +Feature: minifi_tensor extension loads + + Scenario: All processors and the TractModelService register cleanly + Given log property "logger.org::apache::nifi::minifi::core::extension::ExtensionManager" is set to "TRACE,stderr" + And log property "logger.org::apache::nifi::minifi::core::ClassLoader" is set to "TRACE,stderr" + + When the MiNiFi instance starts up + + Then the Minifi logs contain the following message: "Registering class 'ImageToTensor' at '/minifi_tensor'" in less than 10 seconds + And the Minifi logs contain the following message: "Registering class 'InvokeTractModel' at '/minifi_tensor'" in less than 1 seconds + And the Minifi logs contain the following message: "Registering class 'FilterBoundingBoxes' at '/minifi_tensor'" in less than 1 seconds + And the Minifi logs contain the following message: "Registering class 'ClassifyOutput' at '/minifi_tensor'" in less than 1 seconds + And the Minifi logs contain the following message: "Registering class 'TractModelService' at '/minifi_tensor'" in less than 1 seconds + And the Minifi logs contain the following message: "Registering class 'ClassifyImage' at '/minifi_tensor'" in less than 1 seconds + And the Minifi logs contain the following message: "Registering class 'DetectObject' at '/minifi_tensor'" in less than 1 seconds + And the Minifi logs do not contain errors + And the Minifi logs do not contain warnings diff --git a/minifi_rust/extensions/minifi_tensor/features/classification.feature b/minifi_rust/extensions/minifi_tensor/features/classification.feature new file mode 100644 index 0000000000..20be0f6b1e --- /dev/null +++ b/minifi_rust/extensions/minifi_tensor/features/classification.feature @@ -0,0 +1,128 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +@SUPPORTS_WINDOWS +Feature: Image classification with MobileNetV2 + + # Based on https://github.com/sonos/tract/tree/main/examples/onnx-mobilenet-v2 + Scenario: Grace Hopper image is classified using MobileNetV2 ONNX and imagenet labels (ImageToTensor + InvokeTract + ClassifyOutput) + Given a host resource file "grace_hopper.jpg" is copied to the "/tmp/input/grace_hopper.jpg" path in the MiNiFi container + And a host resource file "mobilenetv2-7.onnx" is copied to the "/tmp/models/mobilenetv2-7.onnx" path in the MiNiFi container + And a host resource file "imagenet_slim_labels.txt" is copied to the "/tmp/models/imagenet_slim_labels.txt" path in the MiNiFi container + + And a TractModelService controller service named "MobileNet" is set up and the "Model File Path" property set to "/tmp/models/mobilenetv2-7.onnx" + And the "Model format" property of the MobileNet controller service is set to "Onnx" + + And a GetFile processor with the "Input Directory" property set to "/tmp/input" + And the "Keep Source File" property of the GetFile processor is set to "false" + + And an ImageToTensor processor with the "Target width" property set to "224" + And the "Target height" property of the ImageToTensor processor is set to "224" + And the "Resize filter" property of the ImageToTensor processor is set to "Bilinear" + And the "Resize mode" property of the ImageToTensor processor is set to "Stretch" + And the "Color format" property of the ImageToTensor processor is set to "RGB" + And the "Tensor shape format" property of the ImageToTensor processor is set to "CHW" + And the "Mean" property of the ImageToTensor processor is set to "0.485, 0.456, 0.406" + And the "Standard Deviation" property of the ImageToTensor processor is set to "0.229, 0.224, 0.225" + And the "Pixel divisor" property of the ImageToTensor processor is set to "255" + + And an InvokeTractModel processor with the "Tract model service" property set to "MobileNet" + + And a ClassifyOutput processor with the "Top K" property set to "3" + And the "Score activation" property of the ClassifyOutput processor is set to "Softmax" + And the "Labels file path" property of the ClassifyOutput processor is set to "/tmp/models/imagenet_slim_labels.txt" + # ImageNet slim labels start with a dummy entry on line 0 + And the "Label index offset" property of the ClassifyOutput processor is set to "1" + And the "Confidence Threshold" property of the ClassifyOutput processor is set to "0.0" + + And a PutFile processor with the "Directory" property set to "/tmp/output" + + And a LogAttribute processor with the "FlowFiles To Log" property set to "0" + And LogAttribute is EVENT_DRIVEN + + And the "success" relationship of the GetFile processor is connected to the ImageToTensor + And the "success" relationship of the ImageToTensor processor is connected to the InvokeTractModel + And the "success" relationship of the InvokeTractModel processor is connected to the ClassifyOutput + And the "success" relationship of the ClassifyOutput processor is connected to the LogAttribute + And the "success" relationship of the LogAttribute processor is connected to the PutFile + And ImageToTensor's failure relationship is auto-terminated + And InvokeTractModel's failure relationship is auto-terminated + And ClassifyOutput's failure relationship is auto-terminated + And PutFile's success relationship is auto-terminated + And PutFile's failure relationship is auto-terminated + + When the MiNiFi instance starts up + + # Grace Hopper wears a US Navy uniform; MobileNetV2 on ImageNet consistently + # picks "military uniform" (occasionally "suit", "Windsor tie", or + # "bulletproof vest" as close runner-ups). Match any of those to keep the + # test resilient to small numeric differences across tract versions. + Then the Minifi logs match the following regex: "key:class.top1.name value:(military uniform|bulletproof vest|suit|Windsor tie)" in less than 60 seconds + And the Minifi logs match the following regex: "key:class.count value:[1-3]" in less than 1 seconds + And the Minifi logs contain the following message: "key:mime.type value:application/json" in less than 1 seconds + And at least one file in "/tmp/output" content match the following regex: "\"class_name\":\"(military uniform|bulletproof vest|suit|Windsor tie)\"" in less than 30 seconds + And the Minifi logs do not contain errors + + # Based on https://github.com/sonos/tract/tree/main/examples/onnx-mobilenet-v2 + Scenario: Grace Hopper image is classified using MobileNetV2 ONNX and imagenet labels (ClassifyImage) + Given a host resource file "grace_hopper.jpg" is copied to the "/tmp/input/grace_hopper.jpg" path in the MiNiFi container + And a host resource file "mobilenetv2-7.onnx" is copied to the "/tmp/models/mobilenetv2-7.onnx" path in the MiNiFi container + And a host resource file "imagenet_slim_labels.txt" is copied to the "/tmp/models/imagenet_slim_labels.txt" path in the MiNiFi container + + And a TractModelService controller service named "MobileNet" is set up and the "Model File Path" property set to "/tmp/models/mobilenetv2-7.onnx" + And the "Model format" property of the MobileNet controller service is set to "Onnx" + + And a GetFile processor with the "Input Directory" property set to "/tmp/input" + And the "Keep Source File" property of the GetFile processor is set to "false" + + And a ClassifyImage processor with the "Target width" property set to "224" + And the "Target height" property of the ClassifyImage processor is set to "224" + And the "Resize filter" property of the ClassifyImage processor is set to "Bilinear" + And the "Resize mode" property of the ClassifyImage processor is set to "Stretch" + And the "Color format" property of the ClassifyImage processor is set to "RGB" + And the "Tensor shape format" property of the ClassifyImage processor is set to "CHW" + And the "Mean" property of the ClassifyImage processor is set to "0.485, 0.456, 0.406" + And the "Standard Deviation" property of the ClassifyImage processor is set to "0.229, 0.224, 0.225" + And the "Pixel divisor" property of the ClassifyImage processor is set to "255" + And the "Tract model service" property of the ClassifyImage processor is set to "MobileNet" + And the "Top K" property of the ClassifyImage processor is set to "3" + And the "Score activation" property of the ClassifyImage processor is set to "Softmax" + And the "Labels file path" property of the ClassifyImage processor is set to "/tmp/models/imagenet_slim_labels.txt" + And the "Label index offset" property of the ClassifyImage processor is set to "1" + And the "Confidence Threshold" property of the ClassifyImage processor is set to "0.0" + + And a PutFile processor with the "Directory" property set to "/tmp/output" + + And a LogAttribute processor with the "FlowFiles To Log" property set to "0" + And LogAttribute is EVENT_DRIVEN + + And the "success" relationship of the GetFile processor is connected to the ClassifyImage + And the "success" relationship of the ClassifyImage processor is connected to the LogAttribute + And the "success" relationship of the LogAttribute processor is connected to the PutFile + And ClassifyImage's failure relationship is auto-terminated + And PutFile's success relationship is auto-terminated + And PutFile's failure relationship is auto-terminated + + When the MiNiFi instance starts up + + # Grace Hopper wears a US Navy uniform; MobileNetV2 on ImageNet consistently + # picks "military uniform" (occasionally "suit", "Windsor tie", or + # "bulletproof vest" as close runner-ups). Match any of those to keep the + # test resilient to small numeric differences across tract versions. + Then the Minifi logs match the following regex: "key:class.top1.name value:(military uniform|bulletproof vest|suit|Windsor tie)" in less than 60 seconds + And the Minifi logs match the following regex: "key:class.count value:[1-3]" in less than 1 seconds + And the Minifi logs contain the following message: "key:mime.type value:application/json" in less than 1 seconds + And at least one file in "/tmp/output" content match the following regex: "\"class_name\":\"(military uniform|bulletproof vest|suit|Windsor tie)\"" in less than 30 seconds + And the Minifi logs do not contain errors diff --git a/minifi_rust/extensions/minifi_tensor/features/detection.feature b/minifi_rust/extensions/minifi_tensor/features/detection.feature new file mode 100644 index 0000000000..660bca9f8b --- /dev/null +++ b/minifi_rust/extensions/minifi_tensor/features/detection.feature @@ -0,0 +1,131 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +@SUPPORTS_WINDOWS +Feature: Face detection with UltraFace (SSD) + + # Based on https://github.com/sonos/tract/blob/main/examples/face_detection_yolov8onnx_example/src/main.rs + Scenario: Grace Hopper image yields at least one face detection (ImageToTensor + InvokeTract + FilterBoundingBoxes) + Given a host resource file "grace_hopper.jpg" is copied to the "/tmp/input/grace_hopper.jpg" path in the MiNiFi container + And a host resource file "version-RFB-320.onnx" is copied to the "/tmp/models/ultraface.onnx" path in the MiNiFi container + + And a TractModelService controller service named "UltraFace" is set up and the "Model File Path" property set to "/tmp/models/ultraface.onnx" + And the "Model format" property of the UltraFace controller service is set to "Onnx" + + And a GetFile processor with the "Input Directory" property set to "/tmp/input" + And the "Keep Source File" property of the GetFile processor is set to "false" + + And an ImageToTensor processor with the "Target width" property set to "320" + And the "Target height" property of the ImageToTensor processor is set to "240" + And the "Resize filter" property of the ImageToTensor processor is set to "Bilinear" + And the "Resize mode" property of the ImageToTensor processor is set to "Letterbox" + And the "Color format" property of the ImageToTensor processor is set to "RGB" + And the "Tensor shape format" property of the ImageToTensor processor is set to "CHW" + And the "Mean" property of the ImageToTensor processor is set to "127.0" + And the "Standard Deviation" property of the ImageToTensor processor is set to "128.0" + + And an InvokeTractModel processor with the "Tract model service" property set to "UltraFace" + + # UltraFace: output 0 = scores [1, N, 2] (softmax over background/face), + # output 1 = boxes [1, N, 4] in Xyxy normalised to 0..1. Class 0 is + # background so leave the defaults. + And a FilterBoundingBoxes processor with the "Confidence Threshold" property set to "0.5" + And the "IoU Threshold" property of the FilterBoundingBoxes processor is set to "0.45" + And the "Score output index" property of the FilterBoundingBoxes processor is set to "0" + And the "Box output index" property of the FilterBoundingBoxes processor is set to "1" + And the "Box format" property of the FilterBoundingBoxes processor is set to "Xyxy" + And the "Score activation" property of the FilterBoundingBoxes processor is set to "Softmax" + And the "Background class index" property of the FilterBoundingBoxes processor is set to "0" + + And a PutFile processor with the "Directory" property set to "/tmp/output" + + And a LogAttribute processor with the "FlowFiles To Log" property set to "0" + And LogAttribute is EVENT_DRIVEN + + And the "success" relationship of the GetFile processor is connected to the ImageToTensor + And the "success" relationship of the ImageToTensor processor is connected to the InvokeTractModel + And the "success" relationship of the InvokeTractModel processor is connected to the FilterBoundingBoxes + And the "success" relationship of the FilterBoundingBoxes processor is connected to the LogAttribute + And the "success" relationship of the LogAttribute processor is connected to the PutFile + And ImageToTensor's failure relationship is auto-terminated + And InvokeTractModel's failure relationship is auto-terminated + And FilterBoundingBoxes's failure relationship is auto-terminated + And PutFile's success relationship is auto-terminated + And PutFile's failure relationship is auto-terminated + + When the MiNiFi instance starts up + + Then the Minifi logs match the following regex: "key:object.count value:[1-9][0-9]*" in less than 60 seconds + And the Minifi logs contain the following message: "key:mime.type value:application/json" in less than 1 seconds + And at least one file in "/tmp/output" content match the following regex: "\"class_id\":1" in less than 30 seconds + And at least one file in "/tmp/output" content match the following regex: "\"confidence\":0\.[5-9][0-9]*" in less than 30 seconds + And the Minifi logs do not contain errors + + # Based on https://github.com/sonos/tract/blob/main/examples/face_detection_yolov8onnx_example/src/main.rs + Scenario: Grace Hopper image yields at least one face detection (DetectObject) + Given a host resource file "grace_hopper.jpg" is copied to the "/tmp/input/grace_hopper.jpg" path in the MiNiFi container + And a host resource file "version-RFB-320.onnx" is copied to the "/tmp/models/ultraface.onnx" path in the MiNiFi container + + And a TractModelService controller service named "UltraFace" is set up and the "Model File Path" property set to "/tmp/models/ultraface.onnx" + And the "Model format" property of the UltraFace controller service is set to "Onnx" + + And a GetFile processor with the "Input Directory" property set to "/tmp/input" + And the "Keep Source File" property of the GetFile processor is set to "false" + + And a DetectObject processor with the "Target width" property set to "320" + And the "Target height" property of the DetectObject processor is set to "240" + And the "Resize filter" property of the DetectObject processor is set to "Bilinear" + And the "Resize mode" property of the DetectObject processor is set to "Letterbox" + And the "Color format" property of the DetectObject processor is set to "RGB" + And the "Tensor shape format" property of the DetectObject processor is set to "CHW" + And the "Mean" property of the DetectObject processor is set to "127.0" + And the "Standard Deviation" property of the DetectObject processor is set to "128.0" + And the "Tract model service" property of the DetectObject processor is set to "UltraFace" + # UltraFace: output 0 = scores [1, N, 2] (softmax over background/face), + # output 1 = boxes [1, N, 4] in Xyxy normalised to 0..1. Class 0 is + # background so leave the defaults. + And the "Confidence Threshold" property of the DetectObject processor is set to "0.5" + And the "IoU Threshold" property of the DetectObject processor is set to "0.45" + And the "Score output index" property of the DetectObject processor is set to "0" + And the "Box output index" property of the DetectObject processor is set to "1" + And the "Box format" property of the DetectObject processor is set to "Xyxy" + And the "Score activation" property of the DetectObject processor is set to "Softmax" + And the "Background class index" property of the DetectObject processor is set to "0" + And the "Output attribute name" property of the DetectObject processor is set to "detected_objects" + + And a DrawBoundingBox processor with the "Bounding boxes" property set to "${detected_objects}" + And the "Line color" property of the DrawBoundingBox processor is set to "0, 255, 0" + And the "Line thickness" property of the DrawBoundingBox processor is set to "5" + + And a LogAttribute processor with the "FlowFiles To Log" property set to "0" + And LogAttribute is EVENT_DRIVEN + + And a PutFile processor with the "Directory" property set to "/tmp/output" + + And the "success" relationship of the GetFile processor is connected to the DetectObject + And the "success" relationship of the DetectObject processor is connected to the LogAttribute + And the "success" relationship of the LogAttribute processor is connected to the DrawBoundingBox + And the "success" relationship of the DrawBoundingBox processor is connected to the PutFile + And DetectObject's failure relationship is auto-terminated + And DrawBoundingBox's failure relationship is auto-terminated + And PutFile's success relationship is auto-terminated + And PutFile's failure relationship is auto-terminated + + When the MiNiFi instance starts up + + Then the Minifi logs match the following regex: "key:object.count value:[1-9][0-9]*" in less than 60 seconds + And the Minifi logs match the following regex: "key:detected_objects value:.*\"class_id\":1" in less than 1 seconds + And the Minifi logs match the following regex: "key:detected_objects value:.*\"confidence\":0\.[5-9][0-9]*" in less than 1 seconds + And the Minifi logs do not contain errors diff --git a/minifi_rust/extensions/minifi_tensor/features/environment.py b/minifi_rust/extensions/minifi_tensor/features/environment.py new file mode 100644 index 0000000000..a787ac9d42 --- /dev/null +++ b/minifi_rust/extensions/minifi_tensor/features/environment.py @@ -0,0 +1,79 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +import os +import urllib.request + +from minifi_behave.core.hooks import ( + add_extension_to_minifi_container, + common_after_scenario, + common_before_scenario, +) + +# Model / label / image assets fetched on first use. All hosted on +# public buckets or the sonos/tract repo +REMOTE_ASSETS: dict[str, str] = { + # ImageNet MobileNetV2 classifier (~14 MB) — the reference model used by tract's unit tests + "mobilenetv2-7.onnx": "https://s3.amazonaws.com/tract-ci-builds/tests/mobilenetv2-7.onnx", + # 1000-class ImageNet labels (line N = class N; line 0 is "dummy") + "imagenet_slim_labels.txt": "https://raw.githubusercontent.com/sonos/tract/main/examples/" + "onnx-mobilenet-v2/imagenet_slim_labels.txt", + # Same test image tract's example uses. MobileNetV2 confidently + # classifies this as "military uniform". + "grace_hopper.jpg": "https://raw.githubusercontent.com/sonos/tract/main/examples/" + "onnx-mobilenet-v2/grace_hopper.jpg", + # UltraFace RFB-320 (~1.2 MB): 2-output SSD-style detector matching the + # existing FilterBoundingBoxes defaults (Xyxy boxes, class 0 = background, + # softmax over 2 classes: background/face). 320x240 RGB, mean=127, std=128. + "version-RFB-320.onnx": "https://github.com/onnx/models/raw/refs/heads/main/validated/vision/" + "body_analysis/ultraface/models/version-RFB-320.onnx", +} + + +def _ensure_asset(cache_dir: str, filename: str) -> str: + dest = os.path.join(cache_dir, filename) + if os.path.exists(dest): + return dest + url = REMOTE_ASSETS[filename] + os.makedirs(cache_dir, exist_ok=True) + tmp = dest + ".part" + print(f"[minifi_tensor tests] fetching {filename} from {url}") + urllib.request.urlretrieve(url, tmp) + os.replace(tmp, dest) + return dest + + +def before_all(context): + dir_path = os.path.dirname(os.path.realpath(__file__)) + build_path = os.path.normpath(os.path.join(dir_path, "../../../target/release/")) + deps_build_path = os.path.normpath(os.path.join(dir_path, "../../../target/release/deps/")) + add_extension_to_minifi_container("minifi_tensor", [build_path, deps_build_path], context) + + context.tensor_resource_dir = os.path.join(dir_path, "resources") + os.makedirs(context.tensor_resource_dir, exist_ok=True) + for name in REMOTE_ASSETS: + _ensure_asset(context.tensor_resource_dir, name) + + +def before_scenario(context, scenario): + context.minifi_container_image = "apacheminificpp:minifi_tensor" + common_before_scenario(context, scenario) + context.resource_dir = context.tensor_resource_dir + + +def after_scenario(context, scenario): + common_after_scenario(context, scenario) diff --git a/minifi_rust/extensions/minifi_tensor/features/image_to_tensor.feature b/minifi_rust/extensions/minifi_tensor/features/image_to_tensor.feature new file mode 100644 index 0000000000..bf94b28886 --- /dev/null +++ b/minifi_rust/extensions/minifi_tensor/features/image_to_tensor.feature @@ -0,0 +1,105 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +@SUPPORTS_WINDOWS +Feature: ImageToTensor preprocesses image bytes into normalised tensors + + Scenario: RGB CHW 224x224 tensor is produced with the expected shape and dtype + Given a host resource file "grace_hopper.jpg" is copied to the "/tmp/input/grace_hopper.jpg" path in the MiNiFi container + And a GetFile processor with the "Input Directory" property set to "/tmp/input" + And the "Keep Source File" property of the GetFile processor is set to "false" + And an ImageToTensor processor with the "Target width" property set to "224" + And the "Target height" property of the ImageToTensor processor is set to "224" + And the "Resize filter" property of the ImageToTensor processor is set to "Bilinear" + And the "Resize mode" property of the ImageToTensor processor is set to "Stretch" + And the "Color format" property of the ImageToTensor processor is set to "RGB" + And the "Tensor shape format" property of the ImageToTensor processor is set to "CHW" + And the "Mean" property of the ImageToTensor processor is set to "0.485, 0.456, 0.406" + And the "Standard Deviation" property of the ImageToTensor processor is set to "0.229, 0.224, 0.225" + And a LogAttribute processor with the "FlowFiles To Log" property set to "0" + And LogAttribute is EVENT_DRIVEN + And the "success" relationship of the GetFile processor is connected to the ImageToTensor + And the "success" relationship of the ImageToTensor processor is connected to the LogAttribute + And LogAttribute's success relationship is auto-terminated + And ImageToTensor's failure relationship is auto-terminated + + When the MiNiFi instance starts up + + Then the Minifi logs contain the following message: "key:tensor.0.shape value:1,3,224,224" in less than 30 seconds + And the Minifi logs contain the following message: "key:tensor.0.dtype value:F32" in less than 1 seconds + And the Minifi logs do not contain errors + + Scenario: HWC layout is reflected in the tensor.shape attribute + Given a host resource file "grace_hopper.jpg" is copied to the "/tmp/input/grace_hopper.jpg" path in the MiNiFi container + And a GetFile processor with the "Input Directory" property set to "/tmp/input" + And the "Keep Source File" property of the GetFile processor is set to "false" + And an ImageToTensor processor with the "Target width" property set to "300" + And the "Target height" property of the ImageToTensor processor is set to "300" + And the "Tensor shape format" property of the ImageToTensor processor is set to "HWC" + And the "Color format" property of the ImageToTensor processor is set to "RGB" + And the "Mean" property of the ImageToTensor processor is set to "0.0" + And the "Standard Deviation" property of the ImageToTensor processor is set to "255.0" + And a LogAttribute processor with the "FlowFiles To Log" property set to "0" + And LogAttribute is EVENT_DRIVEN + And the "success" relationship of the GetFile processor is connected to the ImageToTensor + And the "success" relationship of the ImageToTensor processor is connected to the LogAttribute + And LogAttribute's success relationship is auto-terminated + And ImageToTensor's failure relationship is auto-terminated + + When the MiNiFi instance starts up + + Then the Minifi logs contain the following message: "key:tensor.0.shape value:1,300,300,3" in less than 30 seconds + And the Minifi logs do not contain errors + + Scenario: Letterbox mode still produces the target dimensions + Given a host resource file "grace_hopper.jpg" is copied to the "/tmp/input/grace_hopper.jpg" path in the MiNiFi container + And a GetFile processor with the "Input Directory" property set to "/tmp/input" + And the "Keep Source File" property of the GetFile processor is set to "false" + And an ImageToTensor processor with the "Target width" property set to "320" + And the "Target height" property of the ImageToTensor processor is set to "240" + And the "Resize mode" property of the ImageToTensor processor is set to "Letterbox" + And the "Letterbox pad value" property of the ImageToTensor processor is set to "0.0" + And the "Color format" property of the ImageToTensor processor is set to "RGB" + And the "Tensor shape format" property of the ImageToTensor processor is set to "CHW" + And the "Mean" property of the ImageToTensor processor is set to "127.0" + And the "Standard Deviation" property of the ImageToTensor processor is set to "128.0" + And a LogAttribute processor with the "FlowFiles To Log" property set to "0" + And LogAttribute is EVENT_DRIVEN + And the "success" relationship of the GetFile processor is connected to the ImageToTensor + And the "success" relationship of the ImageToTensor processor is connected to the LogAttribute + And LogAttribute's success relationship is auto-terminated + And ImageToTensor's failure relationship is auto-terminated + + When the MiNiFi instance starts up + + Then the Minifi logs contain the following message: "key:tensor.0.shape value:1,3,240,320" in less than 30 seconds + And the Minifi logs do not contain errors + + Scenario: Invalid image bytes route to failure without crashing MiNiFi + Given a directory at "/tmp/input" has a file "not_an_image.bin" with the content "garbage bytes not a real image" + And a GetFile processor with the "Input Directory" property set to "/tmp/input" + And the "Keep Source File" property of the GetFile processor is set to "false" + And an ImageToTensor processor with the "Target width" property set to "224" + And the "Target height" property of the ImageToTensor processor is set to "224" + And a PutFile processor with the "Directory" property set to "/tmp/output" + And the "success" relationship of the GetFile processor is connected to the ImageToTensor + And the "failure" relationship of the ImageToTensor processor is connected to the PutFile + And ImageToTensor's success relationship is auto-terminated + And PutFile's success relationship is auto-terminated + And PutFile's failure relationship is auto-terminated + + When the MiNiFi instance starts up + + Then at least one file with the content "garbage bytes not a real image" is placed in the "/tmp/output" directory in less than 30 seconds diff --git a/minifi_rust/extensions/minifi_tensor/features/resources/.gitignore b/minifi_rust/extensions/minifi_tensor/features/resources/.gitignore new file mode 100644 index 0000000000..2537751f50 --- /dev/null +++ b/minifi_rust/extensions/minifi_tensor/features/resources/.gitignore @@ -0,0 +1,5 @@ +# Test-time downloads (models, labels, images). Fetched on first test run by +# environment.py. Do not commit — several are megabytes and licensed by their +# upstream projects. +* +!.gitignore diff --git a/minifi_rust/extensions/minifi_tensor/features/steps/steps.py b/minifi_rust/extensions/minifi_tensor/features/steps/steps.py new file mode 100644 index 0000000000..231d2d53bb --- /dev/null +++ b/minifi_rust/extensions/minifi_tensor/features/steps/steps.py @@ -0,0 +1,23 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +from minifi_behave.steps import ( + checking_steps, # noqa: F401 + configuration_steps, # noqa: F401 + core_steps, # noqa: F401 + flow_building_steps, # noqa: F401 +) diff --git a/minifi_rust/extensions/minifi_tensor/minifi_tensor.md b/minifi_rust/extensions/minifi_tensor/minifi_tensor.md new file mode 100644 index 0000000000..65a0081494 --- /dev/null +++ b/minifi_rust/extensions/minifi_tensor/minifi_tensor.md @@ -0,0 +1,305 @@ + + +## Table of Contents + +### Processors + +- [ClassifyImage](#ClassifyImage) +- [ClassifyOutput](#ClassifyOutput) +- [DetectObject](#DetectObject) +- [DrawBoundingBox](#DrawBoundingBox) +- [FilterBoundingBoxes](#FilterBoundingBoxes) +- [ImageToTensor](#ImageToTensor) +- [InvokeTractModel](#InvokeTractModel) +### Controller Services + +- [TractModelService](#TractModelService) + + +## ClassifyImage + +### Description + +Runs a full image-classification pass in a single processor: decodes the image from the flow file content, resizes and normalises it into an input tensor, runs one inference against the compiled model owned by the referenced TractModelService, and post-processes the score vector (score activation, Top-K selection, confidence filtering, optional label lookup) into predictions. Collapses the ImageToTensor -> InvokeTractModel -> ClassifyOutput chain into one node. The flow file content is left unchanged (the original image); the Top-K classifications are written as a JSON array to the configured output attribute. + +### Properties + +In the list below, the names of required properties appear in bold. Any other properties (not in bold) are considered optional. The table also indicates any default values, and whether a property supports the NiFi Expression Language. + +| Name | Default Value | Allowable Values | Description | +|--------------------------|---------------|-----------------------------------------------|----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| **Color format** | RGB | RGB
BGR
GRAYSCALE | Colour space of the tensor fed to the model. RGB and BGR produce three-channel tensors (channel order determined by the format); Grayscale produces a single-channel luma tensor. | +| **Confidence Threshold** | 0.0 | | Minimum confidence a class must reach to be included in the output JSON. Applied AFTER activation, so the units match the chosen activation (0.0..=1.0 for Softmax/Sigmoid, model-native for None). Set to 0.0 to always emit exactly Top K predictions.
**Supports Expression Language: true** | +| **Label index offset** | 0 | | Offset added to the model's class ID when looking up a name in the labels file. Defaults to 0 (labels file line N = class N). Set to 1 for label files that start with a dummy/background entry — e.g. the ONNX MobileNetV2 model emits 1000 class scores while 'imagenet_slim_labels.txt' has 1001 lines (line 0 = 'dummy'), so class ID 653 maps to line 654 = 'military uniform'. | +| Labels file path | | | Optional path to a newline-separated labels file (line N = name of class N). Loaded once at service enable time. When set, each prediction in the output JSON gains a 'class_name' field and the 'class.top1.name' flow file attribute is populated. Leave empty to emit numeric class IDs only. | +| **Letterbox pad value** | 0.0 | | Value written for padding pixels when 'Resize mode' is 'Letterbox'. This is a normalised value (post mean/std), so 0.0 corresponds to a neutral input for most networks. Ignored when 'Resize mode' is 'Stretch'. | +| Mean | 0.0 | | Mean subtracted from each pixel before dividing by 'Standard Deviation'. Accepts either a single value (broadcast to all channels) or three comma-separated values applied per channel in the order dictated by 'Color format'. Example: '0.485, 0.456, 0.406' for ImageNet-style RGB normalisation. | +| Output attribute name | | | Specify the attribute to use as output, if not provided, the content is overridden instead.
**Supports Expression Language: true** | +| **Pixel divisor** | 1.0 | | Divisor applied to raw u8 pixel values before subtracting 'Mean' and dividing by 'Standard Deviation'. Defaults to 1.0 (mean/std interpreted in [0, 255] pixel space, e.g. UltraFace's mean=127, std=128). Set to 255 to bring pixels into [0.0, 1.0] first so ImageNet-style mean/std values like '0.485, 0.456, 0.406' / '0.229, 0.224, 0.225' can be used directly, matching the PyTorch / torchvision / ONNX MobileNet convention. Must be non-zero. | +| **Resize filter** | Bilinear | Nearest
Bilinear
Bicubic
Lanczos3 | Interpolation filter applied when resizing the decoded image. Nearest is fastest but blocky; Bilinear is a good default; Bicubic and Lanczos3 are higher-quality but slower. | +| **Resize mode** | Stretch | Stretch
Letterbox | How the source image is fitted into the target dimensions. 'Stretch' scales each axis independently, distorting aspect ratio. 'Letterbox' preserves aspect ratio and pads the remaining border with 'Letterbox pad value' (applied in normalised output space). | +| **Score activation** | Softmax | Softmax
Sigmoid
None | Activation applied to the raw score vector before ranking. Softmax = mutually-exclusive classes (ImageNet-trained ResNet/MobileNet/EfficientNet raw logits). Sigmoid = independent classes (multi-label classifiers). None = the model already emits probabilities/scores; rank the raw values. | +| **Score output index** | 0 | | Zero-based index of the model output tensor that holds classification scores. The processor slices the concatenated payload from InvokeTractModel according to the 'tensor.N.bytes' attributes. Almost always 0 for single-head classifiers. | +| Standard Deviation | 255.0 | | Divisor applied after subtracting 'Mean'. Accepts a single value (broadcast) or three comma-separated values (per channel). Must be non-zero. Example: '255.0' to scale u8 pixels into [0.0, 1.0]; '0.229, 0.224, 0.225' for ImageNet. | +| **Target height** | | | Height in pixels the decoded image is resized to before normalisation and inference. | +| **Target width** | | | Width in pixels the decoded image is resized to before normalisation and inference. | +| **Tensor shape format** | CHW | CHW
HWC | Memory layout of the tensor fed to the model. CHW (channels-first) is typical for PyTorch/ONNX detectors. HWC (channels-last) matches TensorFlow/TFLite. Ignored for Grayscale (always effectively 1xHxW). | +| **Top K** | 5 | | Number of highest-scoring classes to include in the output JSON, in descending order of confidence. Values above the total class count are clamped. Set to 1 for pure top-1 classification. | +| **Tract model service** | | | Reference to a TractModelService controller service. The referenced service owns the compiled model (ONNX or NNEF) that will be evaluated for each incoming flow file. | + +### Relationships + +| Name | Description | +|---------|-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| failure | The image could not be decoded, the input tensor could not be built, the model failed to run, or the model outputs could not be interpreted as scores + boxes. | +| success | Inference and post-processing completed. The flow file content is the original, unchanged image; the classifications are written to the configured output attribute as a JSON array (may be empty). | + + +## ClassifyOutput + +### Description + +Post-processes the output of a classification model invoked via InvokeTractModel. Reads the flattened score vector from the configured output tensor index, applies the chosen activation (softmax / sigmoid / none), and emits the Top K classes as a JSON array [{class_id, confidence, class_name?}, ...]. Optional labels file maps numeric class IDs to human-readable names. Works with ImageNet-style ResNet/MobileNet/EfficientNet checkpoints (Softmax over raw logits) as well as multi-label classifiers (Sigmoid) and models that already emit probabilities (None). Assumes a single flattened score vector — upstream ImageToTensor produces batch=1 tensors, so this is the common case. + +### Properties + +In the list below, the names of required properties appear in bold. Any other properties (not in bold) are considered optional. The table also indicates any default values, and whether a property supports the NiFi Expression Language. + +| Name | Default Value | Allowable Values | Description | +|--------------------------|---------------|------------------------------|----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| **Confidence Threshold** | 0.0 | | Minimum confidence a class must reach to be included in the output JSON. Applied AFTER activation, so the units match the chosen activation (0.0..=1.0 for Softmax/Sigmoid, model-native for None). Set to 0.0 to always emit exactly Top K predictions.
**Supports Expression Language: true** | +| **Label index offset** | 0 | | Offset added to the model's class ID when looking up a name in the labels file. Defaults to 0 (labels file line N = class N). Set to 1 for label files that start with a dummy/background entry — e.g. the ONNX MobileNetV2 model emits 1000 class scores while 'imagenet_slim_labels.txt' has 1001 lines (line 0 = 'dummy'), so class ID 653 maps to line 654 = 'military uniform'. | +| Labels file path | | | Optional path to a newline-separated labels file (line N = name of class N). Loaded once at service enable time. When set, each prediction in the output JSON gains a 'class_name' field and the 'class.top1.name' flow file attribute is populated. Leave empty to emit numeric class IDs only. | +| Output attribute name | | | Specify the attribute to use as output, if not provided, the content is overridden instead.
**Supports Expression Language: true** | +| **Score activation** | Softmax | Softmax
Sigmoid
None | Activation applied to the raw score vector before ranking. Softmax = mutually-exclusive classes (ImageNet-trained ResNet/MobileNet/EfficientNet raw logits). Sigmoid = independent classes (multi-label classifiers). None = the model already emits probabilities/scores; rank the raw values. | +| **Score output index** | 0 | | Zero-based index of the model output tensor that holds classification scores. The processor slices the concatenated payload from InvokeTractModel according to the 'tensor.N.bytes' attributes. Almost always 0 for single-head classifiers. | +| **Top K** | 5 | | Number of highest-scoring classes to include in the output JSON, in descending order of confidence. Values above the total class count are clamped. Set to 1 for pure top-1 classification. | + +### Relationships + +| Name | Description | +|---------|----------------------------------------------------------------------------------------------------------------------------------------------------------| +| failure | The upstream output attributes were missing/invalid or the score tensor could not be interpreted as f32 values. | +| success | Classification completed. The flow file content is a JSON array of the Top K predictions (possibly fewer if the confidence threshold filtered some out). | + +### Output Attributes + +| Attribute | Relationship | Description | +|-----------------------|--------------|----------------------------------------------------------------------------------------------------------------------------------------------------| +| class.count | success | Number of predictions retained after Top K selection and confidence filtering. | +| class.top1.confidence | success | Confidence (post-activation) of the highest-confidence prediction, when at least one prediction cleared the confidence threshold. | +| class.top1.id | success | Numeric class ID of the highest-confidence prediction, when at least one prediction cleared the confidence threshold. | +| class.top1.name | success | Label of the highest-confidence prediction. Only present when 'Labels file path' was configured and at least one prediction cleared the threshold. | +| mime.type | success | Always 'application/json' — the output payload is a JSON array of prediction objects with fields class_id, confidence, and optional class_name. | + + +## DetectObject + +### Description + +Runs a full object-detection pass in a single processor: decodes the image from the flow file content, resizes and normalises it into an input tensor, runs one inference against the compiled model owned by the referenced TractModelService, and post-processes the model outputs (score activation, confidence filtering, box decoding, per-class non-maximum suppression) into bounding boxes. Collapses the ImageToTensor -> InvokeTractModel -> FilterBoundingBoxes chain into one node. The flow file content is left unchanged (the original image); the detected boxes are written as a JSON array to the configured output attribute so a downstream DrawBoundingBox can annotate the image. + +### Properties + +In the list below, the names of required properties appear in bold. Any other properties (not in bold) are considered optional. The table also indicates any default values, and whether a property supports the NiFi Expression Language. + +| Name | Default Value | Allowable Values | Description | +|--------------------------|---------------|-----------------------------------------------|----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| Background class index | | | Index of the 'background / no-object' class. Boxes whose winning class equals this index are dropped. In score-matrix mode this is only honoured when the score tensor has more than one class per box; in 'Class output index' mode it is matched against each box's class id. | +| **Box format** | Xyxy | Xyxy
Yxyx
Cxcywh | Layout of the four floats per box in the box output tensor. Xyxy = [x_min, y_min, x_max, y_max] (SSD, MobileNet-SSD, most PyTorch exports). Yxyx = [y_min, x_min, y_max, x_max] (TensorFlow Object Detection API). Cxcywh = [cx, cy, w, h] (YOLOv3/5/8 raw output). | +| **Box output index** | 1 | | Zero-based index of the model output tensor that holds box coordinates. Must differ from 'Score output index'. | +| Class output index | | | Zero-based index of a model output tensor that holds one class id per box. Set this for detectors that emit boxes, per-box scores, and class ids as three separate parallel tensors, with NMS already folded into the graph (TensorFlow Object Detection API; YOLO / EfficientNMS 'end2end' exports). When set, 'Score output index' is read as one score per box (not a [boxes, classes] matrix) and no argmax is performed; the class id tensor may be integer- or float-typed. Leave empty for models that emit a per-class score matrix. | +| **Color format** | RGB | RGB
BGR
GRAYSCALE | Colour space of the tensor fed to the model. RGB and BGR produce three-channel tensors (channel order determined by the format); Grayscale produces a single-channel luma tensor. | +| **Confidence Threshold** | 0.7 | | Minimum per-box class probability (0.0 to 1.0) required to keep a bounding box after applying the chosen 'Score activation'. Boxes below the threshold are discarded before NMS.
**Supports Expression Language: true** | +| **IoU Threshold** | 0.45 | | Intersection-over-union cutoff used during non-maximum suppression. Boxes of the same class whose IoU with a higher-confidence peer exceeds this value are suppressed. Typical values: 0.45 (SSD/YOLO default), 0.5, 0.3 for stricter deduplication. | +| **Letterbox pad value** | 0.0 | | Value written for padding pixels when 'Resize mode' is 'Letterbox'. This is a normalised value (post mean/std), so 0.0 corresponds to a neutral input for most networks. Ignored when 'Resize mode' is 'Stretch'. | +| Mean | 0.0 | | Mean subtracted from each pixel before dividing by 'Standard Deviation'. Accepts either a single value (broadcast to all channels) or three comma-separated values applied per channel in the order dictated by 'Color format'. Example: '0.485, 0.456, 0.406' for ImageNet-style RGB normalisation. | +| Output attribute name | | | Specify the attribute to use as output, if not provided, the content is overridden instead.
**Supports Expression Language: true** | +| **Pixel divisor** | 1.0 | | Divisor applied to raw u8 pixel values before subtracting 'Mean' and dividing by 'Standard Deviation'. Defaults to 1.0 (mean/std interpreted in [0, 255] pixel space, e.g. UltraFace's mean=127, std=128). Set to 255 to bring pixels into [0.0, 1.0] first so ImageNet-style mean/std values like '0.485, 0.456, 0.406' / '0.229, 0.224, 0.225' can be used directly, matching the PyTorch / torchvision / ONNX MobileNet convention. Must be non-zero. | +| **Resize filter** | Bilinear | Nearest
Bilinear
Bicubic
Lanczos3 | Interpolation filter applied when resizing the decoded image. Nearest is fastest but blocky; Bilinear is a good default; Bicubic and Lanczos3 are higher-quality but slower. | +| **Resize mode** | Stretch | Stretch
Letterbox | How the source image is fitted into the target dimensions. 'Stretch' scales each axis independently, distorting aspect ratio. 'Letterbox' preserves aspect ratio and pads the remaining border with 'Letterbox pad value' (applied in normalised output space). | +| **Score activation** | Softmax | Softmax
Sigmoid
None | Activation applied to raw per-class scores before selecting the winning class. Softmax = mutually-exclusive classes (SSD/MobileNet-SSD raw logits). Sigmoid = independent classes (YOLOv5/v8 style). None = the model already emits probabilities/scores; use raw argmax with the raw score as confidence. | +| **Score output index** | 0 | | Zero-based index of the model output tensor that holds classification scores. The processor slices the concatenated payload from InvokeTractModel according to the 'tensor.N.bytes' attributes. | +| Standard Deviation | 255.0 | | Divisor applied after subtracting 'Mean'. Accepts a single value (broadcast) or three comma-separated values (per channel). Must be non-zero. Example: '255.0' to scale u8 pixels into [0.0, 1.0]; '0.229, 0.224, 0.225' for ImageNet. | +| **Target height** | | | Height in pixels the decoded image is resized to before normalisation and inference. | +| **Target width** | | | Width in pixels the decoded image is resized to before normalisation and inference. | +| **Tensor shape format** | CHW | CHW
HWC | Memory layout of the tensor fed to the model. CHW (channels-first) is typical for PyTorch/ONNX detectors. HWC (channels-last) matches TensorFlow/TFLite. Ignored for Grayscale (always effectively 1xHxW). | +| **Tract model service** | | | Reference to a TractModelService controller service. The referenced service owns the compiled model (ONNX or NNEF) that will be evaluated for each incoming flow file. | + +### Relationships + +| Name | Description | +|---------|----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| failure | The image could not be decoded, the input tensor could not be built, the model failed to run, or the model outputs could not be interpreted as scores + boxes. | +| success | Inference and post-processing completed. The flow file content is the original, unchanged image; the detected boxes are written to the configured output attribute as a JSON array (may be empty). | + +### Output Attributes + +| Attribute | Relationship | Description | +|-------------------------|--------------|------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| | success | JSON array of the surviving bounding boxes (fields class_id, confidence, x_min, y_min, x_max, y_max; coordinates normalised to [0,1] against the original image). The attribute name is configurable via the 'Output attribute name' property. | +| object.count | success | Number of bounding boxes retained after confidence filtering and NMS. | + + +## DrawBoundingBox + +### Description + +Decodes the image from the flow file content, draws each bounding box supplied via the 'Bounding boxes' property onto it, and re-encodes the annotated image as PNG. Pair with an upstream DetectObject / FilterBoundingBoxes to visualise detections. + +### Properties + +In the list below, the names of required properties appear in bold. Any other properties (not in bold) are considered optional. The table also indicates any default values, and whether a property supports the NiFi Expression Language. + +| Name | Default Value | Allowable Values | Description | +|--------------------|---------------------|------------------|------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| Bounding boxes | ${enrichment.value} | | JSON array of bounding boxes to draw onto the image (fields class_id, confidence, x_min, y_min, x_max, y_max; coordinates normalised to [0,1] against the image). Typically the attribute produced by an upstream DetectObject or FilterBoundingBoxes processor. | +| Line color | [0, 255, 0] | | Outline colour as '[R, G, B]' u8 channels (0-255). | +| **Line thickness** | 5 | | Thickness in pixels of the drawn box outline. | + +### Relationships + +| Name | Description | +|---------|------------------------------------------------------------| +| failure | Invalid FlowFiles are routed here | +| success | Flowfiles are routed here after drawing the bounding boxes | + + +## FilterBoundingBoxes + +### Description + +Post-processes the concatenated output of InvokeTractModel for object-detection models. Reads the classification score tensor and the box coordinate tensor from the flow file payload (indices configurable), applies the configured score activation, filters by confidence, decodes box coordinates from the configured layout, and applies per-class non-maximum suppression at the configured IoU threshold. Handles both per-class score matrices (argmax per box) and detectors that emit boxes / per-box scores / class ids as separate tensors with NMS folded into the graph (set 'Class output index'). Works with SSD-, YOLO-, and TensorFlow-style detectors by tuning properties — no code changes needed for common model families. Emits a JSON array of the surviving boxes. + +### Properties + +In the list below, the names of required properties appear in bold. Any other properties (not in bold) are considered optional. The table also indicates any default values, and whether a property supports the NiFi Expression Language. + +| Name | Default Value | Allowable Values | Description | +|--------------------------|---------------|------------------------------|----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| Background class index | | | Index of the 'background / no-object' class. Boxes whose winning class equals this index are dropped. In score-matrix mode this is only honoured when the score tensor has more than one class per box; in 'Class output index' mode it is matched against each box's class id. | +| **Box format** | Xyxy | Xyxy
Yxyx
Cxcywh | Layout of the four floats per box in the box output tensor. Xyxy = [x_min, y_min, x_max, y_max] (SSD, MobileNet-SSD, most PyTorch exports). Yxyx = [y_min, x_min, y_max, x_max] (TensorFlow Object Detection API). Cxcywh = [cx, cy, w, h] (YOLOv3/5/8 raw output). | +| **Box output index** | 1 | | Zero-based index of the model output tensor that holds box coordinates. Must differ from 'Score output index'. | +| Class output index | | | Zero-based index of a model output tensor that holds one class id per box. Set this for detectors that emit boxes, per-box scores, and class ids as three separate parallel tensors, with NMS already folded into the graph (TensorFlow Object Detection API; YOLO / EfficientNMS 'end2end' exports). When set, 'Score output index' is read as one score per box (not a [boxes, classes] matrix) and no argmax is performed; the class id tensor may be integer- or float-typed. Leave empty for models that emit a per-class score matrix. | +| **Confidence Threshold** | 0.7 | | Minimum per-box class probability (0.0 to 1.0) required to keep a bounding box after applying the chosen 'Score activation'. Boxes below the threshold are discarded before NMS.
**Supports Expression Language: true** | +| **IoU Threshold** | 0.45 | | Intersection-over-union cutoff used during non-maximum suppression. Boxes of the same class whose IoU with a higher-confidence peer exceeds this value are suppressed. Typical values: 0.45 (SSD/YOLO default), 0.5, 0.3 for stricter deduplication. | +| Output attribute name | | | Specify the attribute to use as output, if not provided, the content is overridden instead.
**Supports Expression Language: true** | +| **Score activation** | Softmax | Softmax
Sigmoid
None | Activation applied to raw per-class scores before selecting the winning class. Softmax = mutually-exclusive classes (SSD/MobileNet-SSD raw logits). Sigmoid = independent classes (YOLOv5/v8 style). None = the model already emits probabilities/scores; use raw argmax with the raw score as confidence. | +| **Score output index** | 0 | | Zero-based index of the model output tensor that holds classification scores. The processor slices the concatenated payload from InvokeTractModel according to the 'tensor.N.bytes' attributes. | + +### Relationships + +| Name | Description | +|---------|--------------------------------------------------------------------------------------------------------------------------------------------| +| failure | The upstream output attributes were missing/invalid, the payload was truncated, or the tensors could not be interpreted as scores + boxes. | +| success | Filtering completed. The flow file content is a JSON array of the surviving bounding boxes (may be empty). | + +### Output Attributes + +| Attribute | Relationship | Description | +|--------------|--------------|-------------------------------------------------------------------------------------------------------------------------------------------| +| mime.type | success | Always 'application/json' — the output payload is a JSON array of objects with fields class_id, confidence, x_min, y_min, x_max, y_max. | +| object.count | success | Number of bounding boxes retained after confidence filtering and NMS. | + + +## ImageToTensor + +### Description + +Decodes an image from the flow file content and converts it into a normalised numeric tensor suitable for feeding into a downstream inference processor such as InvokeTractModel. Supports RGB / BGR / Grayscale, CHW / HWC layouts, stretch or letterbox resizing, and scalar or per-channel mean/std normalisation. The output payload is the raw little-endian f32 tensor; the 'tensor.shape' and 'tensor.dtype' attributes describe its layout. + +### Properties + +In the list below, the names of required properties appear in bold. Any other properties (not in bold) are considered optional. The table also indicates any default values, and whether a property supports the NiFi Expression Language. + +| Name | Default Value | Allowable Values | Description | +|-------------------------|---------------|-----------------------------------------------|----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| **Color format** | RGB | RGB
BGR
GRAYSCALE | Colour space of the tensor fed to the model. RGB and BGR produce three-channel tensors (channel order determined by the format); Grayscale produces a single-channel luma tensor. | +| **Letterbox pad value** | 0.0 | | Value written for padding pixels when 'Resize mode' is 'Letterbox'. This is a normalised value (post mean/std), so 0.0 corresponds to a neutral input for most networks. Ignored when 'Resize mode' is 'Stretch'. | +| Mean | 0.0 | | Mean subtracted from each pixel before dividing by 'Standard Deviation'. Accepts either a single value (broadcast to all channels) or three comma-separated values applied per channel in the order dictated by 'Color format'. Example: '0.485, 0.456, 0.406' for ImageNet-style RGB normalisation. | +| **Pixel divisor** | 1.0 | | Divisor applied to raw u8 pixel values before subtracting 'Mean' and dividing by 'Standard Deviation'. Defaults to 1.0 (mean/std interpreted in [0, 255] pixel space, e.g. UltraFace's mean=127, std=128). Set to 255 to bring pixels into [0.0, 1.0] first so ImageNet-style mean/std values like '0.485, 0.456, 0.406' / '0.229, 0.224, 0.225' can be used directly, matching the PyTorch / torchvision / ONNX MobileNet convention. Must be non-zero. | +| **Resize filter** | Bilinear | Nearest
Bilinear
Bicubic
Lanczos3 | Interpolation filter applied when resizing the decoded image. Nearest is fastest but blocky; Bilinear is a good default; Bicubic and Lanczos3 are higher-quality but slower. | +| **Resize mode** | Stretch | Stretch
Letterbox | How the source image is fitted into the target dimensions. 'Stretch' scales each axis independently, distorting aspect ratio. 'Letterbox' preserves aspect ratio and pads the remaining border with 'Letterbox pad value' (applied in normalised output space). | +| Standard Deviation | 255.0 | | Divisor applied after subtracting 'Mean'. Accepts a single value (broadcast) or three comma-separated values (per channel). Must be non-zero. Example: '255.0' to scale u8 pixels into [0.0, 1.0]; '0.229, 0.224, 0.225' for ImageNet. | +| **Target height** | | | Height in pixels the decoded image is resized to before normalisation and inference. | +| **Target width** | | | Width in pixels the decoded image is resized to before normalisation and inference. | +| **Tensor shape format** | CHW | CHW
HWC | Memory layout of the tensor fed to the model. CHW (channels-first) is typical for PyTorch/ONNX detectors. HWC (channels-last) matches TensorFlow/TFLite. Ignored for Grayscale (always effectively 1xHxW). | + +### Relationships + +| Name | Description | +|---------|--------------------------------------------------------| +| failure | The input flow file could not be decoded as an image. | +| success | The input image was decoded and converted to a tensor. | + +### Output Attributes + +| Attribute | Relationship | Description | +|-----------------------|--------------|---------------------------------------------------------------------------------------------------------------------------------------------------------| +| image.original.height | success | The height of the original image before the resizing. | +| image.original.width | success | The width of the original image before the resizing. | +| image.target.height | success | The height of the image after the resizing. | +| image.target.width | success | The width of the image after the resizing. | +| tensor.0.dtype | success | Element type of the values in the output tensor. Currently always 'F32'. | +| tensor.0.shape | success | Comma-separated dimensions of the output tensor in the chosen layout, always including a leading batch dimension of 1 (e.g. '1,3,224,224' for RGB CHW). | +| tensors.len | success | Number of tensors in the output FlowFile. Currently always '1' | + + +## InvokeTractModel + +### Description + +Runs a single inference against the compiled model owned by the referenced TractModelService. Reads the input tensor from the flow file content plus the 'tensor.shape' and (optionally) 'tensor.dtype' attributes produced by an upstream processor such as ImageToTensor. The flow file's new content is every output tensor's raw bytes concatenated in model order; per-tensor shape, byte length, and dtype are written to attributes. Only a single input tensor and only f32 input dtype are supported in this pass. + +### Properties + +In the list below, the names of required properties appear in bold. Any other properties (not in bold) are considered optional. The table also indicates any default values, and whether a property supports the NiFi Expression Language. + +| Name | Default Value | Allowable Values | Description | +|-------------------------|---------------|------------------|------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| **Tract model service** | | | Reference to a TractModelService controller service. The referenced service owns the compiled model (ONNX or NNEF) that will be evaluated for each incoming flow file. | + +### Relationships + +| Name | Description | +|---------|---------------------------------------------------------------------------------------------------------------------------------------------| +| failure | The input tensor could not be built (missing/invalid tensor.shape, unsupported tensor.dtype, malformed payload) or the model failed to run. | +| success | Inference completed. The flow file's content is the concatenation of every output tensor's raw bytes in model output order. | + +### Output Attributes + +| Attribute | Relationship | Description | +|------------------|--------------|--------------------------------------------------------------------------------------------------------------------------------------------| +| tensor.{i}.bytes | success | Byte length of output tensor at index i within the concatenated payload. Consumers slice the payload sequentially using these lengths. | +| tensor.{i}.dtype | success | Element type of output tensor at index i | +| tensor.{i}.shape | success | Comma-separated dimensions of output tensor at index i. | +| tensors.len | success | Number of output tensors produced by the model. Downstream processors can loop from 0 up to this count when reading per-output attributes. | + + +## TractModelService + +### Description + +Provides a shared, CPU-optimized neural network for inference. Supports ONNX (`.onnx`) and NNEF (directory or tarball) models; the format can be auto-detected from the file extension or set explicitly. + +### Properties + +In the list below, the names of required properties appear in bold. Any other properties (not in bold) are considered optional. The table also indicates any default values, and whether a property supports the NiFi Expression Language. + +| Name | Default Value | Allowable Values | Description | +|---------------------|---------------|------------------------|-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| **Model File Path** | | | Absolute path to the model on the edge device. For ONNX this is a `.onnx` file; for NNEF this is a `.nnef.tgz` archive, a `.nnef` tarball, or the root directory of an unpacked NNEF model. The model is loaded, parsed, and compiled for the host CPU once when the controller service is enabled; subsequent inference calls reuse the compiled runnable. | +| **Model format** | Auto | Auto
Onnx
Nnef | Format of the file/directory referenced by 'Model File Path'. 'Auto' picks Onnx when the path ends in `.onnx` and Nnef when it ends in `.nnef`, `.nnef.tgz`, `.nnef.tar`, `.nnef.tar.gz`, or points at a directory. Set explicitly when the path uses a non-standard extension. | diff --git a/minifi_rust/extensions/minifi_tensor/src/lib.rs b/minifi_rust/extensions/minifi_tensor/src/lib.rs new file mode 100644 index 0000000000..c150f09f9d --- /dev/null +++ b/minifi_rust/extensions/minifi_tensor/src/lib.rs @@ -0,0 +1,53 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#[cfg(feature = "low-level-processors")] +use crate::low_level_processors::{ + classify_output::ClassifyOutput, filter_bounding_boxes::FilterBoundingBoxes, + image_to_tensor::ImageToTensor, invoke_tract_model::InvokeTractModel, +}; +use crate::processors::classify_image::ClassifyImage; +use crate::processors::detect_object::DetectObject; +use crate::processors::draw_bounding_box::DrawBoundingBox; + +use crate::services::tract_model_service::TractModelService; +use minifi_native::{FlowFileTransformProcessorType, MultiThreaded}; + +mod low_level_processors; +mod processors; +mod services; +mod utils; + +minifi_native::declare_minifi_extension!( + group_name: "org.apache.nifi.minifi.rust", + processors: [ + (FlowFileTransformProcessorType, MultiThreaded, DetectObject), + (FlowFileTransformProcessorType, MultiThreaded, ClassifyImage), + (FlowFileTransformProcessorType, MultiThreaded, DrawBoundingBox), + #[cfg(feature = "low-level-processors")] + (FlowFileTransformProcessorType, MultiThreaded, ImageToTensor), + #[cfg(feature = "low-level-processors")] + (FlowFileTransformProcessorType, MultiThreaded, InvokeTractModel), + #[cfg(feature = "low-level-processors")] + (FlowFileTransformProcessorType, MultiThreaded, FilterBoundingBoxes), + #[cfg(feature = "low-level-processors")] + (FlowFileTransformProcessorType, MultiThreaded, ClassifyOutput), + ], + controllers: [ + TractModelService + ] +); diff --git a/minifi_rust/extensions/minifi_tensor/src/low_level_processors/classify_output.rs b/minifi_rust/extensions/minifi_tensor/src/low_level_processors/classify_output.rs new file mode 100644 index 0000000000..a67701bda5 --- /dev/null +++ b/minifi_rust/extensions/minifi_tensor/src/low_level_processors/classify_output.rs @@ -0,0 +1,426 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use crate::utils::score_activation::ScoreActivation; +use crate::utils::tensor_helpers::{deserialize_tensors, tensor_as_f32, tensor_shape}; +use classify_output_def::SUCCESS; +pub(crate) use classify_output_def::{ + CONFIDENCE_THRESHOLD, LABEL_INDEX_OFFSET, LABELS_FILE_PATH, OUTPUT_ATTRIBUTE_NAME, + SCORE_ACTIVATION, SCORE_OUTPUT_INDEX, TOP_K, +}; +use minifi_native::macros::ComponentIdentifier; +use minifi_native::{ + Content, FlowFileTransform, GetAttribute, GetId, GetProperty, InputStream, Logger, MinifiError, + ProcessError, RouteErrorExt, Schedule, TransformedFlowFile, +}; +use serde::Serialize; +use std::path::Path; +use tract::Tensor; + +mod classify_output_def; + +#[derive(Serialize, Clone, Debug, PartialEq)] +struct Prediction { + class_id: usize, + confidence: f32, + #[serde(skip_serializing_if = "Option::is_none")] + class_name: Option, +} + +fn load_labels(path: &Path) -> Result, MinifiError> { + let content = std::fs::read_to_string(path).map_err(|e| { + MinifiError::custom(format!("Failed to read labels file '{:?}': {}", path, e)) + })?; + Ok(content + .lines() + .map(|line| line.trim_end().to_string()) + .collect()) +} + +fn top_k(mut scored: Vec<(usize, f32)>, k: usize) -> Vec<(usize, f32)> { + scored.sort_by(|&(ai, a), &(bi, b)| b.total_cmp(&a).then(ai.cmp(&bi))); + scored.truncate(k); + scored +} + +#[derive(ComponentIdentifier)] +pub(crate) struct ClassifyOutput { + top_k: usize, + score_output_index: usize, + score_activation: ScoreActivation, + confidence_threshold: f32, + labels: Vec, + label_index_offset: usize, +} + +impl Schedule for ClassifyOutput { + fn schedule( + context: &Ctx, + _logger: &L, + ) -> Result + where + Self: Sized, + { + let top_k = context.get_property(&TOP_K)?; + if top_k == 0 { + return Err(MinifiError::validation("Top K must be >= 1")); + } + let score_output_index = context.get_property(&SCORE_OUTPUT_INDEX)?; + let score_activation = context.get_property(&SCORE_ACTIVATION)?; + let confidence_threshold = context.get_property(&CONFIDENCE_THRESHOLD)?; + + let labels = match context.get_property(&LABELS_FILE_PATH)? { + Some(path) => load_labels(&path)?, + _ => Vec::new(), + }; + let label_index_offset = context.get_property(&LABEL_INDEX_OFFSET)?; + + Ok(Self { + top_k, + score_output_index, + score_activation, + confidence_threshold, + labels, + label_index_offset, + }) + } +} + +impl ClassifyOutput { + fn label_for(&self, class_id: usize) -> Option { + self.labels + .get(class_id.checked_add(self.label_index_offset)?) + .cloned() + } + + pub(crate) fn classify<'a, Context: GetProperty + GetAttribute + GetId>( + &self, + context: &Context, + tensors: Vec, + ) -> Result, ProcessError> { + let score_floats = + tensor_as_f32(&tensors, self.score_output_index).route_err_to_failure()?; + if score_floats.is_empty() { + return Err(MinifiError::custom("Score tensor is empty; nothing to classify").into()); + } + + // A classifier head is a single score vector: shape [num_classes] or + // [1, .., num_classes]. We rank over the flattened class axis, so any + // leading axis > 1 (a real batch) would silently mix rows and yield + // class ids past num_classes. Reject it rather than produce garbage. + // (`ImageToTensor` emits batch=1 today; this just enforces the contract.) + let shape = tensor_shape(&tensors, self.score_output_index).route_err_to_failure()?; + if shape.iter().rev().skip(1).any(|&d| d != 1) { + return Err(MinifiError::custom(format!( + "ClassifyOutput expects a single score vector (shape [num_classes] or \ + [1, .., num_classes]); got {shape:?}. A batch dimension > 1 is not supported." + ))) + .route_err_to_failure(); + } + + let finite: Vec<(usize, f32)> = score_floats + .iter() + .copied() + .enumerate() + .filter(|&(_, s)| s.is_finite()) + .collect(); + + let (max_logit, sum_exp) = match self.score_activation { + ScoreActivation::Softmax => { + let max = finite + .iter() + .map(|&(_, s)| s) + .reduce(f32::max) + .unwrap_or(f32::NEG_INFINITY); + let sum = finite.iter().map(|&(_, s)| (s - max).exp()).sum::(); + (max, sum) + } + _ => (0.0, 1.0), + }; + + let predictions: Vec = top_k(finite, self.top_k) + .into_iter() + .filter_map(|(class_id, raw)| { + let confidence = match self.score_activation { + ScoreActivation::Softmax => (raw - max_logit).exp() / sum_exp, + ScoreActivation::Sigmoid => 1.0 / (1.0 + (-raw).exp()), + ScoreActivation::None => raw, + }; + + if confidence >= self.confidence_threshold { + Some(Prediction { + class_id, + confidence, + class_name: self.label_for(class_id), + }) + } else { + None + } + }) + .collect(); + + let (content, extra_attribute) = match context.get_property(&OUTPUT_ATTRIBUTE_NAME)? { + None => ( + Some(Content::Buffer( + serde_json::to_vec(&predictions).route_err_to_failure()?, + )), + None, + ), + Some(output_attr) => ( + None, + Some((output_attr, serde_json::to_string(&predictions).unwrap())), + ), + }; + + let mut transformed = TransformedFlowFile::new(&SUCCESS, content) + .with_attribute("mime.type", "application/json") + .with_attribute("class.count", predictions.len().to_string()); + + if let Some(top) = predictions.first() { + transformed = transformed + .with_attribute("class.top1.id", top.class_id.to_string()) + .with_attribute("class.top1.confidence", top.confidence.to_string()); + if let Some(name) = &top.class_name { + transformed = transformed.with_attribute("class.top1.name", name.clone()); + } + } + + if let Some((key, value)) = extra_attribute { + transformed = transformed.with_attribute(key, value); + } + Ok(transformed) + } +} + +impl FlowFileTransform for ClassifyOutput { + fn transform<'a, Context: GetProperty + GetAttribute + GetId, LoggerImpl: Logger>( + &self, + context: &Context, + input_stream: &'a mut dyn InputStream, + _logger: &LoggerImpl, + ) -> Result, ProcessError> { + let tensors = deserialize_tensors(context, input_stream).route_err_to_failure()?; + self.classify(context, tensors) + } +} + +#[cfg(test)] +mod tests { + use super::classify_output_def::FAILURE; + use super::*; + use minifi_native::{MockLogger, MockProcessContext}; + use std::io::Cursor; + + fn make_processor(top_k: usize, activation: ScoreActivation) -> ClassifyOutput { + ClassifyOutput { + top_k, + score_output_index: 0, + score_activation: activation, + confidence_threshold: 0.0, + labels: Vec::new(), + label_index_offset: 0, + } + } + + #[test] + fn test_top_k_descending_and_clamped() { + let scored = vec![(0, 0.1), (1, 0.9), (2, 0.5), (3, 0.3)]; + assert_eq!(top_k(scored.clone(), 2), vec![(1, 0.9), (2, 0.5)]); + assert_eq!( + top_k(scored, 10), + vec![(1, 0.9), (2, 0.5), (3, 0.3), (0, 0.1)] + ); + } + + #[test] + fn test_top_k_tiebreak_by_lower_index() { + let scored = vec![(0, 0.5), (1, 0.5), (2, 0.5)]; + assert_eq!(top_k(scored, 3), vec![(0, 0.5), (1, 0.5), (2, 0.5)]); + } + + fn build_payload(scores: &[f32]) -> Vec { + let mut bytes = Vec::with_capacity(scores.len() * 4); + for s in scores { + bytes.extend_from_slice(&s.to_le_bytes()); + } + bytes + } + + fn context_with_scores(scores: &[f32]) -> MockProcessContext { + let mut ctx = MockProcessContext::new(); + ctx.attributes.insert("tensors.len".into(), "1".into()); + ctx.attributes + .insert("tensor.0.bytes".to_string(), (scores.len() * 4).to_string()); + ctx.attributes + .insert("tensor.0.shape".to_string(), format!("1,{}", scores.len())); + ctx.attributes + .insert("tensor.0.dtype".to_string(), "F32".to_string()); + ctx + } + + #[test] + fn test_transform_returns_top_k_json() { + let processor = make_processor(3, ScoreActivation::Softmax); + let logits = vec![1.0f32, 4.0, 2.0, 0.5, 3.0]; + let context = context_with_scores(&logits); + let payload = build_payload(&logits); + let mut stream = Cursor::new(payload); + let result = processor + .transform(&context, &mut stream, &MockLogger::new()) + .expect("transform succeeds"); + + assert_eq!(result.target_relationship(), SUCCESS.name); + assert_eq!(result.attribute("class.top1.id").unwrap(), "1"); + assert_eq!(result.attribute("class.count").unwrap(), "3"); + + let json_bytes = result.into_bytes().unwrap().unwrap(); + let json = String::from_utf8(json_bytes).unwrap(); + // Expect three entries, ranked class_id 1, then 4, then 2. + assert!(json.contains("\"class_id\":1")); + assert!(json.contains("\"class_id\":4")); + assert!(json.contains("\"class_id\":2")); + } + + #[test] + fn test_transform_omits_class_name_when_labels_absent() { + let processor = make_processor(1, ScoreActivation::None); + let scores = vec![0.1f32, 0.9]; + let context = context_with_scores(&scores); + let mut stream = Cursor::new(build_payload(&scores)); + let result = processor + .transform(&context, &mut stream, &MockLogger::new()) + .unwrap(); + // Snapshot the top1.name absence before into_bytes consumes `result`. + let has_top1_name = result.attribute("class.top1.name").is_some(); + let json = String::from_utf8(result.into_bytes().unwrap().unwrap()).unwrap(); + assert!(!json.contains("class_name")); + assert!(!has_top1_name); + } + + #[test] + fn test_transform_looks_up_labels() { + let mut processor = make_processor(1, ScoreActivation::None); + processor.labels = vec![ + "tench".into(), + "goldfish".into(), + "great_white_shark".into(), + ]; + let scores = vec![0.1f32, 0.9, 0.5]; + let context = context_with_scores(&scores); + let mut stream = Cursor::new(build_payload(&scores)); + let result = processor + .transform(&context, &mut stream, &MockLogger::new()) + .unwrap(); + assert_eq!(result.attribute("class.top1.name").unwrap(), "goldfish"); + let json = String::from_utf8(result.into_bytes().unwrap().unwrap()).unwrap(); + assert!(json.contains("\"class_name\":\"goldfish\"")); + } + + #[test] + fn test_label_index_offset_shifts_lookup() { + // Mirrors the ImageNet slim labels layout: line 0 is a dummy entry, so + // model class 0 should resolve to labels[1] = "tench". + let mut processor = make_processor(1, ScoreActivation::None); + processor.labels = vec![ + "dummy".into(), + "tench".into(), + "goldfish".into(), + "great_white_shark".into(), + ]; + processor.label_index_offset = 1; + let scores = vec![0.9f32, 0.1, 0.0]; // model class 0 wins + let context = context_with_scores(&scores); + let mut stream = Cursor::new(build_payload(&scores)); + let result = processor + .transform(&context, &mut stream, &MockLogger::new()) + .unwrap(); + assert_eq!( + result.attribute("class.top1.name").unwrap(), + "tench", + "class id 0 with offset 1 should land on labels[1]" + ); + } + + #[test] + fn test_transform_filters_below_confidence_threshold() { + let mut processor = make_processor(3, ScoreActivation::None); + processor.confidence_threshold = 0.6; + let scores = vec![0.1f32, 0.9, 0.5]; + let context = context_with_scores(&scores); + let mut stream = Cursor::new(build_payload(&scores)); + let result = processor + .transform(&context, &mut stream, &MockLogger::new()) + .unwrap(); + assert_eq!( + result.attribute("class.count").unwrap(), + "1", + "only the 0.9-scored class should survive" + ); + } + + #[test] + fn test_transform_ignores_non_finite_scores() { + // A NaN score must not steal the top slot: with top_k = 1 the sole + // prediction should be the finite class 1, not the NaN class 0. + let processor = make_processor(1, ScoreActivation::None); + let scores = vec![f32::NAN, 0.9]; + let context = context_with_scores(&scores); + let mut stream = Cursor::new(build_payload(&scores)); + let result = processor + .transform(&context, &mut stream, &MockLogger::new()) + .unwrap(); + assert_eq!(result.attribute("class.count").unwrap(), "1"); + assert_eq!(result.attribute("class.top1.id").unwrap(), "1"); + } + + #[test] + fn test_transform_batched_scores_route_to_failure() { + // A real batch dimension ([2, 3]) can't be flattened into one score + // vector without mixing rows. It's input-dependent, so it must route to + // failure (not raise a fatal/rollback error). + let processor = make_processor(1, ScoreActivation::None); + let scores = vec![0.1f32, 0.9, 0.2, 0.8, 0.3, 0.7]; + let mut context = context_with_scores(&scores); + context + .attributes + .insert("tensor.0.shape".into(), "2,3".into()); + let mut stream = Cursor::new(build_payload(&scores)); + let err = processor + .transform(&context, &mut stream, &MockLogger::new()) + .expect_err("batched scores should be rejected"); + match err { + ProcessError::Route(route) => assert_eq!(route.relationship.as_ref(), FAILURE.name), + other => panic!("expected route to failure, got {other:?}"), + } + } + + #[test] + fn test_transform_missing_bytes_attribute_routes_to_failure() { + let processor = make_processor(1, ScoreActivation::Softmax); + let context = MockProcessContext::new(); // no tensor.0.bytes + let mut stream = Cursor::new(vec![0u8; 4]); + let err = processor + .transform(&context, &mut stream, &MockLogger::new()) + .expect_err("missing attribute should route to failure via a Route error"); + match err { + ProcessError::Route(route) => { + assert_eq!(route.relationship.as_ref(), FAILURE.name) + } + other => panic!("expected route to failure, got {other:?}"), + } + } +} diff --git a/minifi_rust/extensions/minifi_tensor/src/low_level_processors/classify_output/classify_output_def.rs b/minifi_rust/extensions/minifi_tensor/src/low_level_processors/classify_output/classify_output_def.rs new file mode 100644 index 0000000000..1a9e4457b4 --- /dev/null +++ b/minifi_rust/extensions/minifi_tensor/src/low_level_processors/classify_output/classify_output_def.rs @@ -0,0 +1,162 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use super::{ClassifyOutput, ScoreActivation}; +use minifi_native::{ + OutputAttribute, ProcessorDefinition, ProcessorInputRequirement, Property, PropertyDefinition, + Relationship, property_definitions, +}; +use std::path::PathBuf; + +pub(crate) const TOP_K: Property = Property::new( + "Top K", + "Number of highest-scoring classes to include in the output JSON, in descending \ + order of confidence. Values above the total class count are clamped. Set to 1 \ + for pure top-1 classification.", +) +.with_default("5"); + +pub(crate) const SCORE_OUTPUT_INDEX: Property = Property::new( + "Score output index", + "Zero-based index of the model output tensor that holds classification scores. \ + The processor slices the concatenated payload from InvokeTractModel according \ + to the 'tensor.N.bytes' attributes. Almost always 0 for single-head \ + classifiers.", +) +.with_default("0"); + +pub(crate) const SCORE_ACTIVATION: Property = Property::new( + "Score activation", + "Activation applied to the raw score vector before ranking. \ + Softmax = mutually-exclusive classes (ImageNet-trained ResNet/MobileNet/\ + EfficientNet raw logits). \ + Sigmoid = independent classes (multi-label classifiers). \ + None = the model already emits probabilities/scores; rank the raw values.", +) +.with_default(ScoreActivation::Softmax.into_str()); + +pub(crate) const CONFIDENCE_THRESHOLD: Property = Property::new( + "Confidence Threshold", + "Minimum confidence a class must reach to be included in the output JSON. \ + Applied AFTER activation, so the units match the chosen activation \ + (0.0..=1.0 for Softmax/Sigmoid, model-native for None). Set to 0.0 to always \ + emit exactly Top K predictions.", +) +.with_default("0.0") +.supports_expression_language(); + +pub(crate) const LABELS_FILE_PATH: Property> = Property::new( + "Labels file path", + "Optional path to a newline-separated labels file (line N = name of class N). \ + Loaded once at service enable time. When set, each prediction in the output \ + JSON gains a 'class_name' field and the 'class.top1.name' flow file attribute \ + is populated. Leave empty to emit numeric class IDs only.", +); + +pub(crate) const LABEL_INDEX_OFFSET: Property = Property::new( + "Label index offset", + "Offset added to the model's class ID when looking up a name in the labels file. \ + Defaults to 0 (labels file line N = class N). Set to 1 for label files that \ + start with a dummy/background entry — e.g. the ONNX MobileNetV2 model emits \ + 1000 class scores while 'imagenet_slim_labels.txt' has 1001 lines (line 0 = \ + 'dummy'), so class ID 653 maps to line 654 = 'military uniform'.", +) +.with_default("0"); + +pub(super) const SUCCESS: Relationship = Relationship { + name: "success", + description: "Classification completed. The flow file content is a JSON array of the Top K \ + predictions (possibly fewer if the confidence threshold filtered some out).", +}; + +pub(super) const FAILURE: Relationship = Relationship { + name: "failure", + description: "The upstream output attributes were missing/invalid or the score tensor could \ + not be interpreted as f32 values.", +}; + +const MIME_TYPE_ATTR: OutputAttribute = OutputAttribute { + name: "mime.type", + relationships: &["success"], + description: "Always 'application/json' — the output payload is a JSON array of prediction \ + objects with fields class_id, confidence, and optional class_name.", +}; + +const CLASS_COUNT_ATTR: OutputAttribute = OutputAttribute { + name: "class.count", + relationships: &["success"], + description: "Number of predictions retained after Top K selection and confidence filtering.", +}; + +const CLASS_TOP1_ID_ATTR: OutputAttribute = OutputAttribute { + name: "class.top1.id", + relationships: &["success"], + description: "Numeric class ID of the highest-confidence prediction, when at least one \ + prediction cleared the confidence threshold.", +}; + +const CLASS_TOP1_CONFIDENCE_ATTR: OutputAttribute = OutputAttribute { + name: "class.top1.confidence", + relationships: &["success"], + description: "Confidence (post-activation) of the highest-confidence prediction, when at \ + least one prediction cleared the confidence threshold.", +}; + +const CLASS_TOP1_NAME_ATTR: OutputAttribute = OutputAttribute { + name: "class.top1.name", + relationships: &["success"], + description: "Label of the highest-confidence prediction. Only present when 'Labels file \ + path' was configured and at least one prediction cleared the threshold.", +}; + +pub(crate) const OUTPUT_ATTRIBUTE_NAME: Property> = Property::new( + "Output attribute name", + "Specify the attribute to use as output, if not provided, the content is overridden instead.", +) +.supports_expression_language(); + +impl ProcessorDefinition for ClassifyOutput { + const DESCRIPTION: &'static str = "Post-processes the output of a classification model invoked via InvokeTractModel. Reads \ + the flattened score vector from the configured output tensor index, applies the chosen \ + activation (softmax / sigmoid / none), and emits the Top K classes as a JSON array \ + [{class_id, confidence, class_name?}, ...]. Optional labels file maps numeric class IDs \ + to human-readable names. Works with ImageNet-style ResNet/MobileNet/EfficientNet \ + checkpoints (Softmax over raw logits) as well as multi-label classifiers (Sigmoid) and \ + models that already emit probabilities (None). Assumes a single flattened score \ + vector — upstream ImageToTensor produces batch=1 tensors, so this is the common case."; + const INPUT_REQUIREMENT: ProcessorInputRequirement = ProcessorInputRequirement::Required; + const SUPPORTS_DYNAMIC_PROPERTIES: bool = false; + const SUPPORTS_DYNAMIC_RELATIONSHIPS: bool = false; + const OUTPUT_ATTRIBUTES: &'static [OutputAttribute] = &[ + MIME_TYPE_ATTR, + CLASS_COUNT_ATTR, + CLASS_TOP1_ID_ATTR, + CLASS_TOP1_CONFIDENCE_ATTR, + CLASS_TOP1_NAME_ATTR, + ]; + const RELATIONSHIPS: &'static [Relationship] = &[SUCCESS, FAILURE]; + + const PROPERTIES: &[PropertyDefinition] = property_definitions![ + TOP_K, + SCORE_OUTPUT_INDEX, + SCORE_ACTIVATION, + CONFIDENCE_THRESHOLD, + LABELS_FILE_PATH, + LABEL_INDEX_OFFSET, + OUTPUT_ATTRIBUTE_NAME + ]; +} diff --git a/minifi_rust/extensions/minifi_tensor/src/low_level_processors/filter_bounding_boxes.rs b/minifi_rust/extensions/minifi_tensor/src/low_level_processors/filter_bounding_boxes.rs new file mode 100644 index 0000000000..8f6723b34a --- /dev/null +++ b/minifi_rust/extensions/minifi_tensor/src/low_level_processors/filter_bounding_boxes.rs @@ -0,0 +1,485 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +mod filter_bounding_boxes_def; + +use crate::utils::bounding_box::BoundingBox; +use crate::utils::dimensions::Dimensions; +use crate::utils::score_activation::ScoreActivation; +use crate::utils::tensor_helpers::{deserialize_tensors, tensor_as_f32}; +use filter_bounding_boxes_def::SUCCESS; +pub(crate) use filter_bounding_boxes_def::{ + BACKGROUND_CLASS_INDEX, BOX_FORMAT, BOX_OUTPUT_INDEX, CLASS_OUTPUT_INDEX, CONFIDENCE_THRESHOLD, + IOU_THRESHOLD, OUTPUT_ATTRIBUTE_NAME, SCORE_ACTIVATION, SCORE_OUTPUT_INDEX, +}; +use minifi_native::macros::{ComponentIdentifier, PropertyType}; +use minifi_native::{ + Content, FlowFileTransform, GetAttribute, GetId, GetProperty, InputStream, Logger, MinifiError, + ProcessError, RouteErrorExt, Schedule, TransformedFlowFile, debug, +}; +use strum_macros::{Display, EnumString, IntoStaticStr, VariantNames}; +use tract::Tensor; + +#[derive( + Debug, Clone, Copy, PartialEq, Display, EnumString, VariantNames, IntoStaticStr, PropertyType, +)] +#[strum(serialize_all = "PascalCase", const_into_str)] +pub(crate) enum BoxFormat { + /// `[x_min, y_min, x_max, y_max]` — SSD, MobileNet-SSD, most PyTorch models. + Xyxy, + /// `[y_min, x_min, y_max, x_max]` — TensorFlow Object Detection API. + Yxyx, + /// `[cx, cy, w, h]` — YOLOv3/5/8 raw output (center + size). + Cxcywh, +} + +/// Convert the four floats at `box_floats[offset..offset+4]` into a canonical +/// `(x_min, y_min, x_max, y_max)` tuple, regardless of the source layout. +fn decode_box(box_floats: &[f32], offset: usize, format: BoxFormat) -> (f32, f32, f32, f32) { + let a = box_floats[offset]; + let b = box_floats[offset + 1]; + let c = box_floats[offset + 2]; + let d = box_floats[offset + 3]; + match format { + BoxFormat::Xyxy => (a, b, c, d), + BoxFormat::Yxyx => (b, a, d, c), + BoxFormat::Cxcywh => { + let (cx, cy, w, h) = (a, b, c, d); + (cx - w / 2.0, cy - h / 2.0, cx + w / 2.0, cy + h / 2.0) + } + } +} + +struct ScoredClass { + class_id: usize, + confidence: f32, +} + +fn score_box( + logits: &[f32], + activation: ScoreActivation, + background_class_index: Option, +) -> ScoredClass { + let num_classes = logits.len(); + + let best_valid = logits + .iter() + .enumerate() + .filter(|&(_, &logit)| logit.is_finite()) + .filter(|&(id, _)| match background_class_index { + Some(bg_idx) => !(num_classes > 1 && id == bg_idx), + None => true, + }) + .max_by(|a, b| a.1.total_cmp(b.1)); + + let (class_id, &best_logit) = match best_valid { + Some(val) => val, + None => { + return ScoredClass { + class_id: 0, + confidence: f32::NEG_INFINITY, + }; + } + }; + + let confidence = match activation { + ScoreActivation::Softmax => { + let max_logit = logits + .iter() + .copied() + .filter(|l| l.is_finite()) + .reduce(f32::max) + .unwrap_or(f32::NEG_INFINITY); + let sum_exp: f32 = logits + .iter() + .filter(|l| l.is_finite()) + .map(|&l| (l - max_logit).exp()) + .sum(); + + (best_logit - max_logit).exp() / sum_exp + } + ScoreActivation::Sigmoid => 1.0 / (1.0 + (-best_logit).exp()), + ScoreActivation::None => best_logit, + }; + + ScoredClass { + class_id, + confidence, + } +} + +/// Turn a single per-box score into a confidence for the "separate class-id +/// tensor" path. +/// Sigmoid maps a raw logit to a probability; +/// Softmax has no meaning over a single scalar, thus pass-through. +/// None passes the score through. +fn activate_scalar(score: f32, activation: ScoreActivation) -> f32 { + match activation { + ScoreActivation::Sigmoid => 1.0 / (1.0 + (-score).exp()), + ScoreActivation::Softmax | ScoreActivation::None => score, + } +} + +#[derive(ComponentIdentifier)] +pub(crate) struct FilterBoundingBoxes { + confidence_threshold: f32, + iou_threshold: f32, + score_output_index: usize, + box_output_index: usize, + box_format: BoxFormat, + score_activation: ScoreActivation, + background_class_index: Option, + class_output_index: Option, +} + +impl Schedule for FilterBoundingBoxes { + fn schedule( + context: &Ctx, + _logger: &L, + ) -> Result { + let confidence_threshold = context.get_property(&CONFIDENCE_THRESHOLD)?; + let iou_threshold = context.get_property(&IOU_THRESHOLD)?; + let score_output_index = context.get_property(&SCORE_OUTPUT_INDEX)?; + let box_output_index = context.get_property(&BOX_OUTPUT_INDEX)?; + let box_format = context.get_property(&BOX_FORMAT)?; + let score_activation = context.get_property(&SCORE_ACTIVATION)?; + let background_class_index = context.get_property(&BACKGROUND_CLASS_INDEX)?; + let class_output_index = context.get_property(&CLASS_OUTPUT_INDEX)?; + + Ok(Self { + confidence_threshold, + iou_threshold, + score_output_index, + box_output_index, + box_format, + score_activation, + background_class_index, + class_output_index, + }) + } +} + +impl FilterBoundingBoxes { + pub(crate) fn filter<'a, Context: GetProperty, LoggerImpl: Logger>( + &self, + context: &Context, + logger: &LoggerImpl, + tensors: Vec, + orig_dim: Dimensions, + target_dim: Dimensions, + ) -> Result, ProcessError> { + let score_floats = + tensor_as_f32(&tensors, self.score_output_index).route_err_to_failure()?; + let box_floats = tensor_as_f32(&tensors, self.box_output_index).route_err_to_failure()?; + + let scale = (target_dim.width / orig_dim.width).min(target_dim.height / orig_dim.height); + let pad_x = (target_dim.width - (orig_dim.width * scale)) / 2.0; + let pad_y = (target_dim.height - (orig_dim.height * scale)) / 2.0; + + if !box_floats.len().is_multiple_of(4) { + return Err(MinifiError::custom( + "Box tensor byte length is not a multiple of 16 (4 f32 per box)", + ) + .into()); + } + let num_boxes = box_floats.len() / 4; + if num_boxes == 0 { + debug!(logger, "No boxes to filter; emitting empty array"); + return Ok(TransformedFlowFile::new(&SUCCESS, None) + .with_content(b"[]".to_vec().into()) + .with_attribute("object.count", "0") + .with_attribute("mime.type", "application/json")); + } + + let make_box = |i: usize, class_id: usize, confidence: f32| -> BoundingBox { + let (raw_x_min, raw_y_min, raw_x_max, raw_y_max) = + decode_box(&box_floats, i * 4, self.box_format); + let true_x_min = (((raw_x_min * target_dim.width) - pad_x) / scale) / orig_dim.width; + let true_y_min = (((raw_y_min * target_dim.height) - pad_y) / scale) / orig_dim.height; + let true_x_max = (((raw_x_max * target_dim.width) - pad_x) / scale) / orig_dim.width; + let true_y_max = (((raw_y_max * target_dim.height) - pad_y) / scale) / orig_dim.height; + BoundingBox { + class_id, + confidence, + x_min: true_x_min.clamp(0.0, 1.0), + y_min: true_y_min.clamp(0.0, 1.0), + x_max: true_x_max.clamp(0.0, 1.0), + y_max: true_y_max.clamp(0.0, 1.0), + } + }; + + let mut valid_boxes = Vec::new(); + + match self.class_output_index { + // Separate class-id tensor: one score and one class id per box + Some(class_index) => { + let class_floats = tensor_as_f32(&tensors, class_index).route_err_to_failure()?; + if score_floats.len() != num_boxes || class_floats.len() != num_boxes { + return Err(MinifiError::custom(format!( + "'Class output index' mode expects one score and one class id per box \ + (num_boxes={}, scores={}, classes={})", + num_boxes, + score_floats.len(), + class_floats.len() + )) + .into()); + } + debug!( + logger, + "Filtering {} boxes with separate class-id tensor (activation={:?}, \ + box_format={:?})...", + num_boxes, + self.score_activation, + self.box_format + ); + for i in 0..num_boxes { + let confidence = activate_scalar(score_floats[i], self.score_activation); + if confidence < self.confidence_threshold { + continue; + } + let raw_class = class_floats[i]; + if raw_class < 0.0 { + continue; + } + let class_id = raw_class.round() as usize; + if self.background_class_index == Some(class_id) { + continue; + } + valid_boxes.push(make_box(i, class_id, confidence)); + } + } + // Per-class score matrix: argmax over classes per box. + None => { + if !score_floats.len().is_multiple_of(num_boxes) { + return Err(MinifiError::custom(format!( + "Scores length ({}) not divisible by number of boxes ({})", + score_floats.len(), + num_boxes + )) + .into()); + } + let num_classes = score_floats.len() / num_boxes; + debug!( + logger, + "Filtering {} boxes across {} potential classes (activation={:?}, \ + box_format={:?})...", + num_boxes, + num_classes, + self.score_activation, + self.box_format + ); + for i in 0..num_boxes { + let logits = &score_floats[i * num_classes..(i + 1) * num_classes]; + let scored = + score_box(logits, self.score_activation, self.background_class_index); + if scored.confidence >= self.confidence_threshold { + valid_boxes.push(make_box(i, scored.class_id, scored.confidence)); + } + } + } + } + + debug!( + logger, + "Found {} boxes exceeding the {} threshold.", + valid_boxes.len(), + self.confidence_threshold + ); + + let filtered_boxes = + BoundingBox::apply_non_maximum_suppression(valid_boxes, self.iou_threshold); + + let json_output = serde_json::to_vec(&filtered_boxes).route_err_to_failure()?; + + let (content, extra_attribute) = match context.get_property(&OUTPUT_ATTRIBUTE_NAME)? { + None => (Some(Content::Buffer(json_output)), None), + Some(output_attr) => ( + None, + Some((output_attr, serde_json::to_string(&filtered_boxes).unwrap())), + ), + }; + + let mut transformed = TransformedFlowFile::new(&SUCCESS, content) + .with_attribute("object.count", filtered_boxes.len().to_string()) + .with_attribute("mime.type", "application/json"); + if let Some((key, value)) = extra_attribute { + transformed = transformed.with_attribute(key, value); + } + Ok(transformed) + } +} + +impl FlowFileTransform for FilterBoundingBoxes { + fn transform<'a, Context: GetProperty + GetAttribute + GetId, LoggerImpl: Logger>( + &self, + context: &Context, + input_stream: &'a mut dyn InputStream, + logger: &LoggerImpl, + ) -> Result, ProcessError> { + let orig_dim = Dimensions::original_from_attributes(context).route_err_to_failure()?; + let target_dim = Dimensions::target_from_attributes(context).route_err_to_failure()?; + + let tensors = deserialize_tensors(context, input_stream).route_err_to_failure()?; + + self.filter(context, logger, tensors, orig_dim, target_dim) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_decode_box_xyxy_is_identity() { + let raw = [10.0, 20.0, 30.0, 40.0]; + let (x0, y0, x1, y1) = decode_box(&raw, 0, BoxFormat::Xyxy); + assert_eq!((x0, y0, x1, y1), (10.0, 20.0, 30.0, 40.0)); + } + + #[test] + fn test_decode_box_yxyx_swaps_axes() { + let raw = [20.0, 10.0, 40.0, 30.0]; // (y_min, x_min, y_max, x_max) + let (x0, y0, x1, y1) = decode_box(&raw, 0, BoxFormat::Yxyx); + assert_eq!((x0, y0, x1, y1), (10.0, 20.0, 30.0, 40.0)); + } + + #[test] + fn test_decode_box_cxcywh_converts_to_corners() { + // Center (100, 200), width 20, height 40 → corners (90,180)-(110,220) + let raw = [100.0, 200.0, 20.0, 40.0]; + let (x0, y0, x1, y1) = decode_box(&raw, 0, BoxFormat::Cxcywh); + assert!((x0 - 90.0).abs() < 1e-6); + assert!((y0 - 180.0).abs() < 1e-6); + assert!((x1 - 110.0).abs() < 1e-6); + assert!((y1 - 220.0).abs() < 1e-6); + } + + #[test] + fn test_score_box_softmax_picks_argmax_excluding_background() { + // 3 classes: bg strongly favored, but bg is class 0 which we skip. + let logits = [5.0, 0.5, 2.0]; + let scored = score_box(&logits, ScoreActivation::Softmax, Some(0)); + assert_eq!(scored.class_id, 2); + assert!(scored.confidence > 0.0 && scored.confidence < 1.0); + } + + #[test] + fn test_score_box_sigmoid_treats_classes_independently() { + // Sigmoid(2.0) ≈ 0.881, so class 1 wins over class 0 regardless of + // relative magnitudes. + let logits = [-3.0, 2.0]; + let scored = score_box(&logits, ScoreActivation::Sigmoid, None); + assert_eq!(scored.class_id, 1); + assert!((scored.confidence - 0.8807).abs() < 1e-3); + } + + #[test] + fn test_score_box_none_takes_raw_argmax() { + let logits = [0.1, 0.7, 0.2]; + let scored = score_box(&logits, ScoreActivation::None, None); + assert_eq!(scored.class_id, 1); + assert!((scored.confidence - 0.7).abs() < 1e-6); + } + + #[test] + fn test_score_box_background_disabled_keeps_class_zero() { + let logits = [5.0, 0.5]; + let scored = score_box(&logits, ScoreActivation::Softmax, None); + assert_eq!(scored.class_id, 0); + } + + #[test] + fn test_activate_scalar_sigmoid_and_passthrough() { + assert!((activate_scalar(0.0, ScoreActivation::Sigmoid) - 0.5).abs() < 1e-6); + assert_eq!(activate_scalar(0.42, ScoreActivation::None), 0.42); + // Softmax over a scalar has no meaning → pass-through. + assert_eq!(activate_scalar(0.42, ScoreActivation::Softmax), 0.42); + } + + fn class_index_processor() -> FilterBoundingBoxes { + FilterBoundingBoxes { + confidence_threshold: 0.5, + iou_threshold: 0.45, + score_output_index: 0, + box_output_index: 1, + box_format: BoxFormat::Xyxy, + score_activation: ScoreActivation::None, + background_class_index: None, + class_output_index: Some(2), + } + } + + /// TF-OD / EfficientNMS style: separate scores, boxes, and (integer) class-id + /// tensors, one entry per box, NMS already applied by the model. + #[test] + fn test_filter_with_separate_class_index_tensor() { + use minifi_native::{MockLogger, MockProcessContext}; + use tract::__ndarray_interop::TensorInterface; + + let scores = Tensor::from_slice::(&[2], &[0.9, 0.2]).unwrap(); + let boxes = + Tensor::from_slice::(&[2, 4], &[0.1, 0.1, 0.2, 0.2, 0.5, 0.5, 0.9, 0.9]).unwrap(); + // class ids arrive as i64, exercising the cast path too. + let classes = Tensor::from_slice::(&[2], &[5, 3]).unwrap(); + + let processor = class_index_processor(); + let dim = Dimensions { + width: 100.0, + height: 100.0, + }; + let result = processor + .filter( + &MockProcessContext::new(), + &MockLogger::new(), + vec![scores, boxes, classes], + dim, + dim, + ) + .expect("filter should succeed"); + + // Only box 0 (score 0.9) clears the 0.5 threshold; box 1 (0.2) is dropped. + assert_eq!(result.attribute("object.count").unwrap(), "1"); + let json = String::from_utf8(result.into_bytes().unwrap().unwrap()).unwrap(); + assert!(json.contains("\"class_id\":5")); + assert!(!json.contains("\"class_id\":3")); + } + + #[test] + fn test_filter_class_index_mode_rejects_mismatched_lengths() { + use minifi_native::{MockLogger, MockProcessContext}; + use tract::__ndarray_interop::TensorInterface; + + // 2 boxes but only 1 score → the parallel-tensor contract is violated. + let scores = Tensor::from_slice::(&[1], &[0.9]).unwrap(); + let boxes = + Tensor::from_slice::(&[2, 4], &[0.1, 0.1, 0.2, 0.2, 0.5, 0.5, 0.9, 0.9]).unwrap(); + let classes = Tensor::from_slice::(&[2], &[5, 3]).unwrap(); + + let dim = Dimensions { + width: 100.0, + height: 100.0, + }; + let result = class_index_processor().filter( + &MockProcessContext::new(), + &MockLogger::new(), + vec![scores, boxes, classes], + dim, + dim, + ); + assert!(result.is_err(), "mismatched score/box counts should error"); + } +} diff --git a/minifi_rust/extensions/minifi_tensor/src/low_level_processors/filter_bounding_boxes/filter_bounding_boxes_def.rs b/minifi_rust/extensions/minifi_tensor/src/low_level_processors/filter_bounding_boxes/filter_bounding_boxes_def.rs new file mode 100644 index 0000000000..ae5d071458 --- /dev/null +++ b/minifi_rust/extensions/minifi_tensor/src/low_level_processors/filter_bounding_boxes/filter_bounding_boxes_def.rs @@ -0,0 +1,153 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use super::{BoxFormat, FilterBoundingBoxes, ScoreActivation}; +use minifi_native::{ + OutputAttribute, ProcessorDefinition, ProcessorInputRequirement, Property, PropertyDefinition, + Relationship, property_definitions, +}; + +pub(crate) const CONFIDENCE_THRESHOLD: Property = Property::new( + "Confidence Threshold", + "Minimum per-box class probability (0.0 to 1.0) required to keep a bounding box \ + after applying the chosen 'Score activation'. Boxes below the threshold are \ + discarded before NMS.", +) +.with_default("0.7") +.supports_expression_language(); + +pub(crate) const IOU_THRESHOLD: Property = Property::new( + "IoU Threshold", + "Intersection-over-union cutoff used during non-maximum suppression. Boxes of \ + the same class whose IoU with a higher-confidence peer exceeds this value are \ + suppressed. Typical values: 0.45 (SSD/YOLO default), 0.5, 0.3 for stricter \ + deduplication.", +) +.with_default("0.45"); + +pub(crate) const SCORE_OUTPUT_INDEX: Property = Property::new( + "Score output index", + "Zero-based index of the model output tensor that holds classification scores. \ + The processor slices the concatenated payload from InvokeTractModel according \ + to the 'tensor.N.bytes' attributes.", +) +.with_default("0"); + +pub(crate) const BOX_OUTPUT_INDEX: Property = Property::new( + "Box output index", + "Zero-based index of the model output tensor that holds box coordinates. Must \ + differ from 'Score output index'.", +) +.with_default("1"); + +pub(crate) const CLASS_OUTPUT_INDEX: Property> = Property::new( + "Class output index", + "Zero-based index of a model output tensor that holds one class id per box. Set this \ + for detectors that emit boxes, per-box scores, and class ids as three separate \ + parallel tensors, with NMS already folded into the graph (TensorFlow Object \ + Detection API; YOLO / EfficientNMS 'end2end' exports). When set, 'Score output \ + index' is read as one score per box (not a [boxes, classes] matrix) and no \ + argmax is performed; the class id tensor may be integer- or float-typed. Leave \ + empty for models that emit a per-class score matrix.", +); + +pub(crate) const BOX_FORMAT: Property = Property::new( + "Box format", + "Layout of the four floats per box in the box output tensor. Xyxy = \ + [x_min, y_min, x_max, y_max] (SSD, MobileNet-SSD, most PyTorch exports). \ + Yxyx = [y_min, x_min, y_max, x_max] (TensorFlow Object Detection API). \ + Cxcywh = [cx, cy, w, h] (YOLOv3/5/8 raw output).", +) +.with_default(BoxFormat::Xyxy.into_str()); + +pub(crate) const SCORE_ACTIVATION: Property = Property::new( + "Score activation", + "Activation applied to raw per-class scores before selecting the winning class. \ + Softmax = mutually-exclusive classes (SSD/MobileNet-SSD raw logits). \ + Sigmoid = independent classes (YOLOv5/v8 style). \ + None = the model already emits probabilities/scores; use raw argmax with the \ + raw score as confidence.", +) +.with_default(ScoreActivation::Softmax.into_str()); + +pub(crate) const BACKGROUND_CLASS_INDEX: Property> = Property::new( + "Background class index", + "Index of the 'background / no-object' class. Boxes whose winning class equals this \ + index are dropped. In score-matrix mode this is only honoured when the score \ + tensor has more than one class per box; in 'Class output index' mode it is \ + matched against each box's class id.", +); + +pub(crate) const OUTPUT_ATTRIBUTE_NAME: Property> = Property::new( + "Output attribute name", + "Specify the attribute to use as output, if not provided, the content is overridden instead.", +) +.supports_expression_language(); + +pub const SUCCESS: Relationship = Relationship { + name: "success", + description: "Filtering completed. The flow file content is a JSON array of the surviving \ + bounding boxes (may be empty).", +}; + +pub const FAILURE: Relationship = Relationship { + name: "failure", + description: "The upstream output attributes were missing/invalid, the payload was truncated, \ + or the tensors could not be interpreted as scores + boxes.", +}; + +const OBJECT_COUNT_ATTR: OutputAttribute = OutputAttribute { + name: "object.count", + relationships: &["success"], + description: "Number of bounding boxes retained after confidence filtering and NMS.", +}; + +const MIME_TYPE_ATTR: OutputAttribute = OutputAttribute { + name: "mime.type", + relationships: &["success"], + description: "Always 'application/json' — the output payload is a JSON array of objects with \ + fields class_id, confidence, x_min, y_min, x_max, y_max.", +}; + +impl ProcessorDefinition for FilterBoundingBoxes { + const DESCRIPTION: &'static str = "Post-processes the concatenated output of InvokeTractModel for object-detection models. \ + Reads the classification score tensor and the box coordinate tensor from the flow file \ + payload (indices configurable), applies the configured score activation, filters by \ + confidence, decodes box coordinates from the configured layout, and applies per-class \ + non-maximum suppression at the configured IoU threshold. Handles both per-class score \ + matrices (argmax per box) and detectors that emit boxes / per-box scores / class ids as \ + separate tensors with NMS folded into the graph (set 'Class output index'). Works with \ + SSD-, YOLO-, and TensorFlow-style detectors by tuning properties — no code changes needed \ + for common model families. Emits a JSON array of the surviving boxes."; + const INPUT_REQUIREMENT: ProcessorInputRequirement = ProcessorInputRequirement::Required; + const SUPPORTS_DYNAMIC_PROPERTIES: bool = false; + const SUPPORTS_DYNAMIC_RELATIONSHIPS: bool = false; + const OUTPUT_ATTRIBUTES: &'static [OutputAttribute] = &[OBJECT_COUNT_ATTR, MIME_TYPE_ATTR]; + const RELATIONSHIPS: &'static [Relationship] = &[SUCCESS, FAILURE]; + + const PROPERTIES: &[PropertyDefinition] = property_definitions![ + CONFIDENCE_THRESHOLD, + IOU_THRESHOLD, + SCORE_OUTPUT_INDEX, + BOX_OUTPUT_INDEX, + CLASS_OUTPUT_INDEX, + BOX_FORMAT, + SCORE_ACTIVATION, + BACKGROUND_CLASS_INDEX, + OUTPUT_ATTRIBUTE_NAME, + ]; +} diff --git a/minifi_rust/extensions/minifi_tensor/src/low_level_processors/image_to_tensor.rs b/minifi_rust/extensions/minifi_tensor/src/low_level_processors/image_to_tensor.rs new file mode 100644 index 0000000000..8889389d4b --- /dev/null +++ b/minifi_rust/extensions/minifi_tensor/src/low_level_processors/image_to_tensor.rs @@ -0,0 +1,610 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +pub(crate) mod image_to_tensor_def; + +use crate::utils::dimensions::Dimensions; +use crate::utils::per_channel_f32::PerChannelF32; +use crate::utils::tensor_helpers::{MinifiDatumType, load_as_image}; +pub(crate) use image_to_tensor_def::{ + COLOR_FORMAT, LETTERBOX_PAD_VALUE, MEAN, PIXEL_DIVISOR, RESIZE_FILTER, RESIZE_MODE, STD_DEV, + TARGET_HEIGHT, TARGET_WIDTH, TENSOR_SHAPE_FORMAT, +}; +use image_to_tensor_def::{ + IMG_ORG_HEIGHT_ATTR, IMG_ORG_WIDTH_ATTR, SUCCESS, TENSOR_DTYPE_ATTR, TENSOR_SHAPE_ATTR, +}; +use image_to_tensor_def::{IMG_TRG_HEIGHT_ATTR, IMG_TRG_WIDTH_ATTR, TENSORS_LEN_ATTR}; +use minifi_native::macros::{ComponentIdentifier, PropertyType}; +use minifi_native::{ + FlowFileTransform, GetAttribute, GetControllerService, GetId, GetProperty, InputStream, Logger, + MinifiError, ProcessError, RouteErrorExt, Schedule, TransformedFlowFile, +}; +use strum_macros::{Display, EnumString, IntoStaticStr, VariantNames}; +use tract::Tensor; + +tract::impl_ndarray_interop!(); + +#[derive( + Debug, Clone, Copy, PartialEq, Display, EnumString, VariantNames, IntoStaticStr, PropertyType, +)] +#[strum(serialize_all = "PascalCase", const_into_str)] +pub(crate) enum ResizeFilter { + Nearest, + Bilinear, + Bicubic, + Lanczos3, +} + +impl From for image::imageops::FilterType { + fn from(filter: ResizeFilter) -> Self { + match filter { + ResizeFilter::Nearest => image::imageops::FilterType::Nearest, + ResizeFilter::Bilinear => image::imageops::FilterType::Triangle, + ResizeFilter::Bicubic => image::imageops::FilterType::CatmullRom, + ResizeFilter::Lanczos3 => image::imageops::FilterType::Lanczos3, + } + } +} + +#[derive( + Debug, Clone, Copy, PartialEq, Display, EnumString, VariantNames, IntoStaticStr, PropertyType, +)] +#[strum(serialize_all = "UPPERCASE", const_into_str)] +pub(crate) enum ColorFormat { + Rgb, + Bgr, + Grayscale, +} + +#[derive( + Debug, Clone, Copy, PartialEq, Display, EnumString, VariantNames, IntoStaticStr, PropertyType, +)] +#[strum(serialize_all = "UPPERCASE", const_into_str)] +pub(crate) enum TensorShapeFormat { + Chw, // center, height, width + Hwc, // height, width, center +} + +#[derive( + Debug, Clone, Copy, PartialEq, Display, EnumString, VariantNames, IntoStaticStr, PropertyType, +)] +#[strum(serialize_all = "PascalCase", const_into_str)] +pub(crate) enum ResizeMode { + Stretch, + Letterbox, +} + +#[derive(ComponentIdentifier)] +pub(crate) struct ImageToTensor { + target_width: u32, + target_height: u32, + resize_filter: ResizeFilter, + resize_mode: ResizeMode, + color_format: ColorFormat, + tensor_shape_format: TensorShapeFormat, + mean: PerChannelF32, + std_dev: PerChannelF32, + pixel_divisor: f32, + letterbox_pad_value: f32, +} + +impl Schedule for ImageToTensor { + fn schedule( + context: &Ctx, + _logger: &L, + ) -> Result + where + Self: Sized, + { + let target_width = context.get_property(&TARGET_WIDTH)?; + let target_height = context.get_property(&TARGET_HEIGHT)?; + let resize_filter = context.get_property(&RESIZE_FILTER)?; + let resize_mode = context.get_property(&RESIZE_MODE)?; + let color_format = context.get_property(&COLOR_FORMAT)?; + let tensor_shape_format = context.get_property(&TENSOR_SHAPE_FORMAT)?; + let mean = context.get_property(&MEAN)?; + let std_dev = context.get_property(&STD_DEV)?; + if std_dev.contains_zero() { + return Err(MinifiError::validation( + "Standard Deviation components must be non-zero", + )); + } + let pixel_divisor = context.get_property(&PIXEL_DIVISOR)?; + if pixel_divisor == 0.0 { + return Err(MinifiError::validation("Pixel divisor must be non-zero")); + } + let letterbox_pad_value = context.get_property(&LETTERBOX_PAD_VALUE)?; + + Ok(Self { + target_width, + target_height, + resize_filter, + resize_mode, + color_format, + tensor_shape_format, + mean, + std_dev, + pixel_divisor, + letterbox_pad_value, + }) + } +} + +struct MaskedRgbImage { + img: image::RgbImage, + mask: Vec, +} + +impl ImageToTensor { + fn stretch_resize(&self, img: image::DynamicImage) -> MaskedRgbImage { + let resized = img + .resize_exact( + self.target_width, + self.target_height, + self.resize_filter.into(), + ) + .to_rgb8(); + let mask = vec![true; (self.target_width * self.target_height) as usize]; + MaskedRgbImage { img: resized, mask } + } + + fn letterbox_resize(&self, img: image::DynamicImage) -> MaskedRgbImage { + let (src_w, src_h) = (img.width() as f32, img.height() as f32); + let scale = (self.target_width as f32 / src_w).min(self.target_height as f32 / src_h); + let new_w = (src_w * scale).round().max(1.0) as u32; + let new_h = (src_h * scale).round().max(1.0) as u32; + let scaled = img + .resize_exact(new_w, new_h, self.resize_filter.into()) + .to_rgb8(); + + let pad_x = (self.target_width - new_w) / 2; + let pad_y = (self.target_height - new_h) / 2; + + let mut canvas = image::RgbImage::from_pixel( + self.target_width, + self.target_height, + image::Rgb([0, 0, 0]), + ); + image::imageops::overlay(&mut canvas, &scaled, pad_x as i64, pad_y as i64); + + // Mask to track which pixel is part of source and which is padding + let mut mask = vec![false; (self.target_width * self.target_height) as usize]; + for y in pad_y..(new_h + pad_y) { + for x in pad_x..(new_w + pad_x) { + mask[(y * self.target_width + x) as usize] = true; + } + } + MaskedRgbImage { img: canvas, mask } + } + + fn resize_rgb(&self, img: image::DynamicImage) -> MaskedRgbImage { + match self.resize_mode { + ResizeMode::Stretch => self.stretch_resize(img), + ResizeMode::Letterbox => self.letterbox_resize(img), + } + } + + pub fn tensor_bytes(&self, img: image::DynamicImage) -> Vec { + let num_channels: usize = match self.color_format { + ColorFormat::Grayscale => 1, + _ => 3, + }; + let total_pixels = (self.target_width * self.target_height) as usize; + let mut tensor_bytes = Vec::with_capacity(total_pixels * num_channels * 4); + + let masked_img = self.resize_rgb(img); + + if self.color_format == ColorFormat::Grayscale { + let mean = self.mean.per_channel(0); + let std_dev = self.std_dev.per_channel(0); + for (idx, pixel) in masked_img.img.pixels().enumerate() { + let val = if masked_img.mask[idx] { + // Rec. 601 luma coefficients — matches image::to_luma8(). + let luma = + 0.299 * pixel[0] as f32 + 0.587 * pixel[1] as f32 + 0.114 * pixel[2] as f32; + (luma / self.pixel_divisor - mean) / std_dev + } else { + self.letterbox_pad_value + }; + tensor_bytes.extend_from_slice(&val.to_le_bytes()); + } + } else { + let channel_order: [usize; 3] = match self.color_format { + ColorFormat::Rgb => [0, 1, 2], + ColorFormat::Bgr => [2, 1, 0], + _ => unreachable!(), + }; + + let normalized = |px_idx: usize, out_c: usize, src_c: usize| -> f32 { + if !masked_img.mask[px_idx] { + return self.letterbox_pad_value; + } + let raw = masked_img.img.get_pixel( + (px_idx as u32) % self.target_width, + (px_idx as u32) / self.target_width, + )[src_c] as f32; + (raw / self.pixel_divisor - self.mean.per_channel(out_c)) + / self.std_dev.per_channel(out_c) + }; + + match self.tensor_shape_format { + TensorShapeFormat::Chw => { + for (out_c, &src_c) in channel_order.iter().enumerate() { + for px_idx in 0..total_pixels { + let v = normalized(px_idx, out_c, src_c); + tensor_bytes.extend_from_slice(&v.to_le_bytes()); + } + } + } + TensorShapeFormat::Hwc => { + for px_idx in 0..total_pixels { + for (out_c, &src_c) in channel_order.iter().enumerate() { + let v = normalized(px_idx, out_c, src_c); + tensor_bytes.extend_from_slice(&v.to_le_bytes()); + } + } + } + } + } + + tensor_bytes + } + pub fn get_shape(&self) -> Vec { + let num_channels: usize = match self.color_format { + ColorFormat::Grayscale => 1, + _ => 3, + }; + match (self.color_format, self.tensor_shape_format) { + (ColorFormat::Grayscale, _) => { + vec![ + 1, + 1, + self.target_height as usize, + self.target_width as usize, + ] + } + (_, TensorShapeFormat::Chw) => vec![ + 1, + num_channels, + self.target_height as usize, + self.target_width as usize, + ], + (_, TensorShapeFormat::Hwc) => vec![ + 1, + self.target_height as usize, + self.target_width as usize, + num_channels, + ], + } + } + + fn get_shape_str(&self) -> String { + self.get_shape() + .iter() + .map(|n| n.to_string()) + .collect::>() + .join(",") + } + pub fn get_target_dim(&self) -> Dimensions { + Dimensions { + width: self.target_width as f32, + height: self.target_height as f32, + } + } + + pub fn get_tensor(&self, img: image::DynamicImage) -> Result { + let f32_data: Vec = self + .tensor_bytes(img) + .as_chunks::<4>() + .0 + .iter() + .map(|c| f32::from_le_bytes(*c)) + .collect(); + let shape = self.get_shape(); + let array = ndarray::Array::from_shape_vec(shape, f32_data).map_err(MinifiError::other)?; + Ok(array.tract()?) + } +} + +impl FlowFileTransform for ImageToTensor { + fn transform< + 'a, + Context: GetProperty + GetControllerService + GetAttribute + GetId, + LoggerImpl: Logger, + >( + &self, + _context: &Context, + input_stream: &'a mut dyn InputStream, + _logger: &LoggerImpl, + ) -> Result, ProcessError> { + let img = load_as_image(input_stream).route_err_to_failure()?; + let orig_dim = Dimensions::from_image(&img); + + let tensor_bytes = self.tensor_bytes(img); + + Ok( + TransformedFlowFile::new(&SUCCESS, Some(tensor_bytes.into())).with_attributes([ + (&TENSORS_LEN_ATTR, "1".to_string()), + (&TENSOR_SHAPE_ATTR, self.get_shape_str()), + (&TENSOR_DTYPE_ATTR, MinifiDatumType::F32.to_string()), + (&IMG_ORG_HEIGHT_ATTR, orig_dim.height.to_string()), + (&IMG_ORG_WIDTH_ATTR, orig_dim.width.to_string()), + (&IMG_TRG_HEIGHT_ATTR, self.target_height.to_string()), + (&IMG_TRG_WIDTH_ATTR, self.target_width.to_string()), + ]), + ) + } +} + +#[cfg(test)] +mod tests { + use super::image_to_tensor_def::{FAILURE, SUCCESS}; + use super::*; + use image::{ImageFormat, RgbImage}; + use minifi_native::{MockLogger, MockProcessContext}; + use std::io::Cursor; + + fn create_test_image_bytes() -> Vec { + let mut img = RgbImage::new(2, 2); + // Set one pixel to pure white (255, 255, 255) -> should scale to 1.0 + img.put_pixel(0, 0, image::Rgb([255, 255, 255])); + // Set one pixel to pure black (0, 0, 0) -> should scale to 0.0 + img.put_pixel(1, 1, image::Rgb([0, 0, 0])); + + let mut bytes: Vec = Vec::new(); + let mut cursor = Cursor::new(&mut bytes); + img.write_to(&mut cursor, ImageFormat::Png).unwrap(); + bytes + } + + fn default_processor() -> ImageToTensor { + ImageToTensor { + target_width: 2, + target_height: 2, + resize_filter: ResizeFilter::Nearest, + resize_mode: ResizeMode::Stretch, + color_format: ColorFormat::Rgb, + tensor_shape_format: TensorShapeFormat::Chw, + mean: PerChannelF32::SingleChannel(0.0), + std_dev: PerChannelF32::SingleChannel(255.0), + pixel_divisor: 1.0, + letterbox_pad_value: 0.0, + } + } + + fn payload_as_f32(bytes: &[u8]) -> Vec { + bytes + .as_chunks::<4>() + .0 + .iter() + .map(|c| f32::from_le_bytes(*c)) + .collect() + } + + #[test] + fn test_successful_rgb_chw_transform() { + let processor = default_processor(); + + let context = MockProcessContext::new(); + let input_bytes = create_test_image_bytes(); + let mut input_stream = Cursor::new(input_bytes); + + let result = processor + .transform(&context, &mut input_stream, &MockLogger::new()) + .expect("Transform should succeed"); + + assert_eq!(result.target_relationship(), SUCCESS.name); + + assert_eq!( + result + .attribute("tensor.0.shape") + .expect("Missing shape attribute"), + "1,3,2,2" + ); + assert_eq!( + result + .attribute("tensor.0.dtype") + .expect("Missing dtype attribute"), + "F32" + ); + + let payload: Vec = result + .into_bytes() + .unwrap() + .expect("there should be a payload"); + + assert_eq!(payload.len(), 48); + + let first_f32 = payload_as_f32(&payload)[0]; + assert_eq!(first_f32, 1.0); + } + + #[test] + fn test_successful_grayscale_transform() { + let processor = ImageToTensor { + color_format: ColorFormat::Grayscale, + ..default_processor() + }; + + let context = MockProcessContext::new(); + let input_bytes = create_test_image_bytes(); + let mut input_stream = Cursor::new(input_bytes); + + let result = processor + .transform(&context, &mut input_stream, &MockLogger::new()) + .expect("Transform should succeed"); + + assert_eq!(result.target_relationship(), SUCCESS.name); + assert_eq!( + result + .attribute("tensor.0.shape") + .expect("Missing shape attribute"), + "1,1,2,2" + ); + + let payload = result + .into_bytes() + .unwrap() + .expect("there should be a payload"); + // 2x2 image * 1 channel * 4 bytes per f32 = 16 bytes + assert_eq!(payload.len(), 16); + } + + #[test] + fn test_invalid_image_routes_to_failure() { + let processor = ImageToTensor { + target_width: 224, + target_height: 224, + resize_filter: ResizeFilter::Bilinear, + ..default_processor() + }; + + let context = MockProcessContext::new(); + + let invalid_bytes = vec![0x00, 0x01, 0x02, 0x03, 0x04]; + let mut input_stream = Cursor::new(invalid_bytes); + + let err = processor + .transform(&context, &mut input_stream, &MockLogger::new()) + .expect_err("Invalid image should route to FAILURE via a Route error"); + + match err { + ProcessError::Route(route) => { + assert_eq!(route.relationship.as_ref(), FAILURE.name) + } + other => panic!("expected route to failure, got {other:?}"), + } + } + + #[test] + fn test_hwc_shape_attribute_matches_layout() { + let processor = ImageToTensor { + tensor_shape_format: TensorShapeFormat::Hwc, + ..default_processor() + }; + let context = MockProcessContext::new(); + let mut input_stream = Cursor::new(create_test_image_bytes()); + let result = processor + .transform(&context, &mut input_stream, &MockLogger::new()) + .unwrap(); + assert_eq!(result.attribute("tensor.0.shape").unwrap(), "1,2,2,3"); + } + + #[test] + fn test_per_channel_mean_and_std() { + // Use per-channel mean/std where R uses (mean=0, std=255) → 1.0, + // G uses (mean=255, std=255) → 0.0, B uses (mean=0, std=127.5) → 2.0. + let processor = ImageToTensor { + mean: PerChannelF32::TriChannel([0.0, 255.0, 0.0]), + std_dev: PerChannelF32::TriChannel([255.0, 255.0, 127.5]), + ..default_processor() + }; + let context = MockProcessContext::new(); + let mut input_stream = Cursor::new(create_test_image_bytes()); + let result = processor + .transform(&context, &mut input_stream, &MockLogger::new()) + .unwrap(); + let payload = payload_as_f32(&result.into_bytes().unwrap().unwrap()); + // CHW layout: 4 R values, 4 G values, 4 B values. Pixel (0,0) is white. + assert!( + (payload[0] - 1.0).abs() < 1e-6, + "R channel pixel(0,0) = {}", + payload[0] + ); + assert!( + (payload[4] - 0.0).abs() < 1e-6, + "G channel pixel(0,0) = {}", + payload[4] + ); + assert!( + (payload[8] - 2.0).abs() < 1e-6, + "B channel pixel(0,0) = {}", + payload[8] + ); + } + + #[test] + fn test_letterbox_pads_non_square_image() { + // Create a 4x2 image (aspect 2:1) — letterboxed into 4x4 should get + // top/bottom padding of one row each, and the middle two rows should + // contain the source content. + let mut img = RgbImage::new(4, 2); + for y in 0..2 { + for x in 0..4 { + img.put_pixel(x, y, image::Rgb([255, 255, 255])); + } + } + let mut bytes: Vec = Vec::new(); + img.write_to(&mut Cursor::new(&mut bytes), ImageFormat::Png) + .unwrap(); + + let processor = ImageToTensor { + target_width: 4, + target_height: 4, + resize_mode: ResizeMode::Letterbox, + letterbox_pad_value: -1.0, + ..default_processor() + }; + let context = MockProcessContext::new(); + let mut input_stream = Cursor::new(bytes); + let result = processor + .transform(&context, &mut input_stream, &MockLogger::new()) + .unwrap(); + let payload = payload_as_f32(&result.into_bytes().unwrap().unwrap()); + + // CHW, R channel first. 16 pixels per channel. Rows: 0 = pad, 1-2 = content, 3 = pad. + // R channel: + // row 0 (pixels 0..4) = pad value (-1.0) + // rows 1..3 (pixels 4..12) = content (1.0 after /255) + // row 3 (pixels 12..16) = pad value (-1.0) + for (i, v) in payload.iter().enumerate().take(4) { + assert!((v + 1.0).abs() < 1e-6, "expected pad at r[{}]", i); + } + for (i, v) in payload.iter().enumerate().take(12).skip(4) { + assert!((v - 1.0).abs() < 1e-6, "expected content at r[{}]", i); + } + for (i, v) in payload.iter().enumerate().take(16).skip(12) { + assert!((v + 1.0).abs() < 1e-6, "expected pad at r[{}]", i); + } + } + + #[test] + fn test_pixel_divisor_matches_tract_imagenet_recipe() { + // Reproduces the tract onnx-mobilenet-v2 example's normalization: + // (raw / 255.0 - mean_[0,1]) / std_[0,1] + // A pure-white pixel R channel should become (1.0 - 0.485) / 0.229 ≈ 2.2489. + let processor = ImageToTensor { + mean: PerChannelF32::TriChannel([0.485, 0.456, 0.406]), + std_dev: PerChannelF32::TriChannel([0.229, 0.224, 0.225]), + pixel_divisor: 255.0, + ..default_processor() + }; + let context = MockProcessContext::new(); + let mut input_stream = Cursor::new(create_test_image_bytes()); + let result = processor + .transform(&context, &mut input_stream, &MockLogger::new()) + .unwrap(); + let payload = payload_as_f32(&result.into_bytes().unwrap().unwrap()); + // Pixel (0,0) is white (255,255,255). CHW → payload[0] is R for pixel(0,0). + let expected_r = (1.0_f32 - 0.485) / 0.229; + assert!( + (payload[0] - expected_r).abs() < 1e-5, + "R channel pixel(0,0) = {}, expected {}", + payload[0], + expected_r + ); + } +} diff --git a/minifi_rust/extensions/minifi_tensor/src/low_level_processors/image_to_tensor/image_to_tensor_def.rs b/minifi_rust/extensions/minifi_tensor/src/low_level_processors/image_to_tensor/image_to_tensor_def.rs new file mode 100644 index 0000000000..9faa88f21e --- /dev/null +++ b/minifi_rust/extensions/minifi_tensor/src/low_level_processors/image_to_tensor/image_to_tensor_def.rs @@ -0,0 +1,191 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use super::{ColorFormat, ImageToTensor, ResizeFilter, ResizeMode, TensorShapeFormat}; +use crate::utils::per_channel_f32::PerChannelF32; +use minifi_native::{ + OutputAttribute, ProcessorDefinition, ProcessorInputRequirement, Property, PropertyDefinition, + Relationship, property_definitions, +}; + +pub(crate) const TARGET_WIDTH: Property = Property::new( + "Target width", + "Width in pixels the decoded image is resized to before normalisation and \ + inference.", +); + +pub(crate) const TARGET_HEIGHT: Property = Property::new( + "Target height", + "Height in pixels the decoded image is resized to before normalisation and \ + inference.", +); + +pub(crate) const RESIZE_FILTER: Property = Property::new( + "Resize filter", + "Interpolation filter applied when resizing the decoded image. Nearest is fastest \ + but blocky; Bilinear is a good default; Bicubic and Lanczos3 are higher-quality \ + but slower.", +) +.with_default(ResizeFilter::Bilinear.into_str()); + +pub(crate) const RESIZE_MODE: Property = Property::new( + "Resize mode", + "How the source image is fitted into the target dimensions. 'Stretch' scales each \ + axis independently, distorting aspect ratio. 'Letterbox' preserves aspect ratio \ + and pads the remaining border with 'Letterbox pad value' (applied in normalised \ + output space).", +) +.with_default(ResizeMode::Stretch.into_str()); + +pub(crate) const LETTERBOX_PAD_VALUE: Property = Property::new( + "Letterbox pad value", + "Value written for padding pixels when 'Resize mode' is 'Letterbox'. This is a \ + normalised value (post mean/std), so 0.0 corresponds to a neutral input for most \ + networks. Ignored when 'Resize mode' is 'Stretch'.", +) +.with_default("0.0"); + +pub(crate) const COLOR_FORMAT: Property = Property::new( + "Color format", + "Colour space of the tensor fed to the model. RGB and BGR produce three-channel \ + tensors (channel order determined by the format); Grayscale produces a \ + single-channel luma tensor.", +) +.with_default(ColorFormat::Rgb.into_str()); + +pub(crate) const TENSOR_SHAPE_FORMAT: Property = Property::new( + "Tensor shape format", + "Memory layout of the tensor fed to the model. CHW (channels-first) is typical \ + for PyTorch/ONNX detectors. HWC (channels-last) matches TensorFlow/TFLite. \ + Ignored for Grayscale (always effectively 1xHxW).", +) +.with_default(TensorShapeFormat::Chw.into_str()); + +pub(crate) const MEAN: Property = Property::new( + "Mean", + "Mean subtracted from each pixel before dividing by 'Standard Deviation'. Accepts \ + either a single value (broadcast to all channels) or three comma-separated \ + values applied per channel in the order dictated by 'Color format'. Example: \ + '0.485, 0.456, 0.406' for ImageNet-style RGB normalisation.", +) +.with_default("0.0"); + +pub(crate) const STD_DEV: Property = Property::new( + "Standard Deviation", + "Divisor applied after subtracting 'Mean'. Accepts a single value (broadcast) or \ + three comma-separated values (per channel). Must be non-zero. Example: '255.0' \ + to scale u8 pixels into [0.0, 1.0]; '0.229, 0.224, 0.225' for ImageNet.", +) +.with_default("255.0"); + +pub(crate) const PIXEL_DIVISOR: Property = Property::new( + "Pixel divisor", + "Divisor applied to raw u8 pixel values before subtracting 'Mean' and dividing \ + by 'Standard Deviation'. Defaults to 1.0 (mean/std interpreted in [0, 255] pixel \ + space, e.g. UltraFace's mean=127, std=128). Set to 255 to bring pixels into \ + [0.0, 1.0] first so ImageNet-style mean/std values like '0.485, 0.456, 0.406' / \ + '0.229, 0.224, 0.225' can be used directly, matching the PyTorch / torchvision / \ + ONNX MobileNet convention. Must be non-zero.", +) +.with_default("1.0"); + +pub(super) const SUCCESS: Relationship = Relationship { + name: "success", + description: "The input image was decoded and converted to a tensor.", +}; + +pub(super) const FAILURE: Relationship = Relationship { + name: "failure", + description: "The input flow file could not be decoded as an image.", +}; + +pub(super) const TENSORS_LEN_ATTR: OutputAttribute = OutputAttribute { + name: "tensors.len", + relationships: &["success"], + description: "Number of tensors in the output FlowFile. Currently always '1'", +}; + +pub(super) const TENSOR_SHAPE_ATTR: OutputAttribute = OutputAttribute { + name: "tensor.0.shape", + relationships: &["success"], + description: "Comma-separated dimensions of the output tensor in the chosen layout, always \ + including a leading batch dimension of 1 (e.g. '1,3,224,224' for RGB CHW).", +}; + +pub(super) const TENSOR_DTYPE_ATTR: OutputAttribute = OutputAttribute { + name: "tensor.0.dtype", + relationships: &["success"], + description: "Element type of the values in the output tensor. Currently always 'F32'.", +}; + +pub(super) const IMG_ORG_HEIGHT_ATTR: OutputAttribute = OutputAttribute { + name: "image.original.height", + relationships: &["success"], + description: "The height of the original image before the resizing.", +}; + +pub(super) const IMG_ORG_WIDTH_ATTR: OutputAttribute = OutputAttribute { + name: "image.original.width", + relationships: &["success"], + description: "The width of the original image before the resizing.", +}; + +pub(super) const IMG_TRG_HEIGHT_ATTR: OutputAttribute = OutputAttribute { + name: "image.target.height", + relationships: &["success"], + description: "The height of the image after the resizing.", +}; + +pub(super) const IMG_TRG_WIDTH_ATTR: OutputAttribute = OutputAttribute { + name: "image.target.width", + relationships: &["success"], + description: "The width of the image after the resizing.", +}; + +impl ProcessorDefinition for ImageToTensor { + const DESCRIPTION: &'static str = "Decodes an image from the flow file content and converts it into a normalised numeric \ + tensor suitable for feeding into a downstream inference processor such as \ + InvokeTractModel. Supports RGB / BGR / Grayscale, CHW / HWC layouts, stretch or \ + letterbox resizing, and scalar or per-channel mean/std normalisation. The output payload \ + is the raw little-endian f32 tensor; the 'tensor.shape' and 'tensor.dtype' attributes \ + describe its layout."; + const INPUT_REQUIREMENT: ProcessorInputRequirement = ProcessorInputRequirement::Required; + const SUPPORTS_DYNAMIC_PROPERTIES: bool = false; + const SUPPORTS_DYNAMIC_RELATIONSHIPS: bool = false; + const OUTPUT_ATTRIBUTES: &'static [OutputAttribute] = &[ + TENSORS_LEN_ATTR, + TENSOR_SHAPE_ATTR, + TENSOR_DTYPE_ATTR, + IMG_ORG_WIDTH_ATTR, + IMG_ORG_HEIGHT_ATTR, + IMG_TRG_WIDTH_ATTR, + IMG_TRG_HEIGHT_ATTR, + ]; + const RELATIONSHIPS: &'static [Relationship] = &[SUCCESS, FAILURE]; + const PROPERTIES: &[PropertyDefinition] = property_definitions![ + TARGET_WIDTH, + TARGET_HEIGHT, + RESIZE_FILTER, + RESIZE_MODE, + LETTERBOX_PAD_VALUE, + COLOR_FORMAT, + TENSOR_SHAPE_FORMAT, + MEAN, + STD_DEV, + PIXEL_DIVISOR, + ]; +} diff --git a/minifi_rust/extensions/minifi_tensor/src/low_level_processors/invoke_tract_model.rs b/minifi_rust/extensions/minifi_tensor/src/low_level_processors/invoke_tract_model.rs new file mode 100644 index 0000000000..21e5742327 --- /dev/null +++ b/minifi_rust/extensions/minifi_tensor/src/low_level_processors/invoke_tract_model.rs @@ -0,0 +1,254 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use crate::utils::tensor_helpers::MinifiDatumType; +pub(crate) use invoke_tract_model_def::TRACT_MODEL_SERVICE; +use invoke_tract_model_def::*; +use minifi_native::macros::ComponentIdentifier; +use minifi_native::{ + FlowFileTransform, GetAttribute, GetControllerService, GetId, GetProperty, InputStream, Logger, + MinifiError, ProcessError, RouteErrorExt, Schedule, TransformedFlowFile, +}; +use std::error::Error; +use tract::__ndarray_interop::TensorInterface; +use tract::Tensor; +tract::impl_ndarray_interop!(); + +mod invoke_tract_model_def; + +#[derive(ComponentIdentifier)] +pub(crate) struct InvokeTractModel {} + +impl Schedule for InvokeTractModel { + fn schedule( + _context: &Ctx, + _logger: &L, + ) -> Result + where + Self: Sized, + { + Ok(Self {}) + } +} + +impl InvokeTractModel { + fn get_payload_as_f32_array( + input_stream: &mut dyn InputStream, + ) -> Result, Box> { + let mut raw_bytes = Vec::new(); + input_stream.read_to_end(&mut raw_bytes)?; + if raw_bytes.len() % 4 != 0 { + return Err(MinifiError::custom("Input bytes length is not a multiple of 4").into()); + } + let mut f32_data = Vec::with_capacity(raw_bytes.len() / 4); + for chunk in raw_bytes.as_chunks::<4>().0.iter() { + let val = f32::from_le_bytes(*chunk); + f32_data.push(val); + } + Ok(f32_data) + } + + fn get_shape_from_attribute( + context: &Context, + ) -> Result, Box> { + let shape_str = context + .get_attribute("tensor.0.shape") + .and_then(|opt| opt.ok_or(MinifiError::custom("Missing tensor.0.shape")))?; + shape_str + .split(',') + .map(|s| s.trim().parse::()) + .collect::, _>>() + .map_err(|e| e.into()) + } + + fn check_input_dtype(context: &Context) -> Result<(), Box> { + match context.get_attribute("tensor.0.dtype")? { + None => Ok(()), + Some(dtype) => dtype.parse::().map(|_| ()).map_err(|_| { + MinifiError::custom(format!( + "Unsupported tensor.0.dtype '{}' (only '{}' is supported)", + dtype, + MinifiDatumType::F32 + )) + .into() + }), + } + } + + fn get_input_tensor( + context: &Context, + input_stream: &mut dyn InputStream, + ) -> Result> { + Self::check_input_dtype(context)?; + let shape = Self::get_shape_from_attribute(context)?; + let f32_data = Self::get_payload_as_f32_array(input_stream)?; + + let array = ndarray::Array::from_shape_vec(shape, f32_data) + .map_err(|e| MinifiError::custom(format!("Failed to create array: {}", e)))?; + + Ok(array.tract()?) + } +} + +impl FlowFileTransform for InvokeTractModel { + fn transform< + 'a, + Context: GetProperty + GetControllerService + GetAttribute + GetId, + LoggerImpl: Logger, + >( + &self, + context: &Context, + input_stream: &'a mut dyn InputStream, + _logger: &LoggerImpl, + ) -> Result, ProcessError> { + let controller_service = context.get_controller_service(&TRACT_MODEL_SERVICE)?; + + let input_tensor: Tensor = Self::get_input_tensor(context, input_stream) + .map_err(|e| MinifiError::custom(e.to_string())) + .route_err_to_failure()?; + + let output_tensors = controller_service + .run_inference(vec![input_tensor]) + .route_err_to_failure()?; + let mut output_bytes = Vec::new(); + let mut transformed = TransformedFlowFile::new(&SUCCESS, None) + .with_attribute("tensors.len", output_tensors.len().to_string()); + + for (i, tensor) in output_tensors.iter().enumerate() { + let (datum_type, out_shape, raw_tensor_bytes) = tensor + .as_bytes() + .map_err(|e| MinifiError::custom(format!("Failed to read tensor bytes: {}", e)))?; + + output_bytes.extend_from_slice(raw_tensor_bytes); + + let out_shape_str = out_shape + .iter() + .map(|d| d.to_string()) + .collect::>() + .join(","); + + let tensor_bytes = raw_tensor_bytes.len().to_string(); + transformed = transformed.with_attributes([ + (format!("tensor.{}.shape", i), out_shape_str), + (format!("tensor.{}.bytes", i), tensor_bytes), + (format!("tensor.{}.dtype", i), format!("{:?}", datum_type)), + ]); + } + + Ok(transformed.with_content(output_bytes.into())) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use minifi_native::{MockLogger, MockProcessContext}; + use std::io::Cursor; + + #[test] + fn test_get_shape_parses_correctly() { + let mut context = MockProcessContext::new(); + // Insert a valid shape with spaces + context + .attributes + .insert("tensor.0.shape".to_string(), "1, 3, 224, 224".to_string()); + + let shape = + InvokeTractModel::get_shape_from_attribute(&context).expect("Should parse valid shape"); + assert_eq!(shape, vec![1, 3, 224, 224]); + } + + #[test] + fn test_get_shape_missing_attribute_fails() { + let context = MockProcessContext::new(); // Empty attributes + let result = InvokeTractModel::get_shape_from_attribute(&context); + assert!(result.is_err(), "Should fail when tensor.shape is missing"); + } + + #[test] + fn test_get_shape_invalid_data_fails() { + let mut context = MockProcessContext::new(); + context + .attributes + .insert("tensor.0.shape".to_string(), "1, apple, 224".to_string()); + + let result = InvokeTractModel::get_shape_from_attribute(&context); + assert!( + result.is_err(), + "Should fail when shape contains non-integers" + ); + } + + #[test] + fn test_get_payload_as_f32_array_success() { + let expected_floats: [f32; 3] = [1.0, 2.5, -3.5]; + let mut bytes = Vec::new(); + for f in expected_floats.iter() { + bytes.extend_from_slice(&f.to_le_bytes()); + } + let mut stream = Cursor::new(bytes); + let result = InvokeTractModel::get_payload_as_f32_array(&mut stream) + .expect("Should parse valid byte stream"); + assert_eq!(result, vec![1.0, 2.5, -3.5]); + } + + #[test] + fn test_get_payload_invalid_length_fails() { + let bytes = vec![0u8, 1, 2, 3, 4]; + let mut stream = Cursor::new(bytes); + + let result = InvokeTractModel::get_payload_as_f32_array(&mut stream); + assert!( + result.is_err(), + "Should error out if byte length is not a multiple of 4" + ); + } + + #[test] + fn test_check_input_dtype_accepts_missing_and_f32() { + let mut context = MockProcessContext::new(); + assert!(InvokeTractModel::check_input_dtype(&context).is_ok()); + context + .attributes + .insert("tensor.0.dtype".to_string(), "F32".to_string()); + assert!(InvokeTractModel::check_input_dtype(&context).is_ok()); + } + + #[test] + fn test_check_input_dtype_rejects_other() { + let mut context = MockProcessContext::new(); + context + .attributes + .insert("tensor.0.dtype".to_string(), "u8".to_string()); + assert!(InvokeTractModel::check_input_dtype(&context).is_err()); + } + + #[test] + fn test_transform_missing_controller_service_throws_error() { + let processor = InvokeTractModel {}; + let context = MockProcessContext::new(); + let mut stream = Cursor::new(vec![]); + let logger = MockLogger::new(); + + let result = processor.transform(&context, &mut stream, &logger); + + assert!( + result.is_err(), + "Should throw an error when TractModelService is missing" + ); + } +} diff --git a/minifi_rust/extensions/minifi_tensor/src/low_level_processors/invoke_tract_model/invoke_tract_model_def.rs b/minifi_rust/extensions/minifi_tensor/src/low_level_processors/invoke_tract_model/invoke_tract_model_def.rs new file mode 100644 index 0000000000..0fc09edb02 --- /dev/null +++ b/minifi_rust/extensions/minifi_tensor/src/low_level_processors/invoke_tract_model/invoke_tract_model_def.rs @@ -0,0 +1,88 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use crate::services::tract_model_service::TractModelService; +use minifi_native::{ + OutputAttribute, ProcessorDefinition, ProcessorInputRequirement, Property, PropertyDefinition, + Relationship, property_definitions, +}; + +pub(crate) const TRACT_MODEL_SERVICE: Property = Property::new( + "Tract model service", + "Reference to a TractModelService controller service. The referenced service \ + owns the compiled model (ONNX or NNEF) that will be evaluated for each \ + incoming flow file.", +); + +pub(super) const SUCCESS: Relationship = Relationship { + name: "success", + description: "Inference completed. The flow file's content is the concatenation of every \ + output tensor's raw bytes in model output order.", +}; + +pub(super) const FAILURE: Relationship = Relationship { + name: "failure", + description: "The input tensor could not be built (missing/invalid tensor.shape, unsupported \ + tensor.dtype, malformed payload) or the model failed to run.", +}; + +const OUTPUT_COUNT_ATTR: OutputAttribute = OutputAttribute { + name: "tensors.len", + relationships: &["success"], + description: "Number of output tensors produced by the model. Downstream processors can loop \ + from 0 up to this count when reading per-output attributes.", +}; + +const OUTPUT_SHAPE_ATTR: OutputAttribute = OutputAttribute { + name: "tensor.{i}.shape", + relationships: &["success"], + description: "Comma-separated dimensions of output tensor at index i.", +}; + +const OUTPUT_BYTES_ATTR: OutputAttribute = OutputAttribute { + name: "tensor.{i}.bytes", + relationships: &["success"], + description: "Byte length of output tensor at index i within the concatenated payload. \ + Consumers slice the payload sequentially using these lengths.", +}; + +const OUTPUT_DTYPE_ATTR: OutputAttribute = OutputAttribute { + name: "tensor.{i}.dtype", + relationships: &["success"], + description: "Element type of output tensor at index i", +}; + +impl ProcessorDefinition for super::InvokeTractModel { + const DESCRIPTION: &'static str = "Runs a single inference against the compiled model owned by the referenced \ + TractModelService. Reads the input tensor from the flow file content plus the \ + 'tensor.shape' and (optionally) 'tensor.dtype' attributes produced by an upstream \ + processor such as ImageToTensor. The flow file's new content is every output tensor's \ + raw bytes concatenated in model order; per-tensor shape, byte length, and dtype are \ + written to attributes. Only a single input tensor and only f32 input dtype are \ + supported in this pass."; + const INPUT_REQUIREMENT: ProcessorInputRequirement = ProcessorInputRequirement::Required; + const SUPPORTS_DYNAMIC_PROPERTIES: bool = false; + const SUPPORTS_DYNAMIC_RELATIONSHIPS: bool = false; + const OUTPUT_ATTRIBUTES: &'static [OutputAttribute] = &[ + OUTPUT_COUNT_ATTR, + OUTPUT_SHAPE_ATTR, + OUTPUT_BYTES_ATTR, + OUTPUT_DTYPE_ATTR, + ]; + const RELATIONSHIPS: &'static [Relationship] = &[SUCCESS, FAILURE]; + const PROPERTIES: &[PropertyDefinition] = property_definitions![TRACT_MODEL_SERVICE]; +} diff --git a/minifi_rust/extensions/minifi_tensor/src/low_level_processors/mod.rs b/minifi_rust/extensions/minifi_tensor/src/low_level_processors/mod.rs new file mode 100644 index 0000000000..071791e20a --- /dev/null +++ b/minifi_rust/extensions/minifi_tensor/src/low_level_processors/mod.rs @@ -0,0 +1,5 @@ +pub(crate) mod classify_output; +pub(crate) mod filter_bounding_boxes; +pub(crate) mod image_to_tensor; +#[cfg(feature = "low-level-processors")] +pub(crate) mod invoke_tract_model; diff --git a/minifi_rust/extensions/minifi_tensor/src/processors/classify_image.rs b/minifi_rust/extensions/minifi_tensor/src/processors/classify_image.rs new file mode 100644 index 0000000000..a9cafa94ba --- /dev/null +++ b/minifi_rust/extensions/minifi_tensor/src/processors/classify_image.rs @@ -0,0 +1,79 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +mod classify_object_def; + +use crate::low_level_processors::classify_output::ClassifyOutput; +use crate::low_level_processors::image_to_tensor::ImageToTensor; +use crate::utils::tensor_helpers::load_as_image; +use classify_object_def::TRACT_MODEL_SERVICE; +use minifi_native::macros::ComponentIdentifier; +use minifi_native::{ + FlowFileTransform, GetAttribute, GetControllerService, GetId, GetProperty, InputStream, Logger, + MinifiError, ProcessError, RouteErrorExt, Schedule, TransformedFlowFile, +}; +use tract::Tensor; + +#[derive(ComponentIdentifier)] +pub(crate) struct ClassifyImage { + image_to_tensor: ImageToTensor, + classify_output: ClassifyOutput, +} + +impl Schedule for ClassifyImage { + fn schedule(context: &Ctx, logger: &L) -> Result + where + Self: Sized, + { + let image_to_tensor = ImageToTensor::schedule(context, logger)?; + let classify_output = ClassifyOutput::schedule(context, logger)?; + Ok(Self { + image_to_tensor, + classify_output, + }) + } +} + +impl FlowFileTransform for ClassifyImage { + fn transform< + 'a, + Context: GetProperty + GetControllerService + GetAttribute + GetId, + LoggerImpl: Logger, + >( + &self, + context: &Context, + input_stream: &'a mut dyn InputStream, + _logger: &LoggerImpl, + ) -> Result, ProcessError> { + let tract_model_service = context.get_controller_service(&TRACT_MODEL_SERVICE)?; + let img = load_as_image(input_stream).route_err_to_failure()?; + + // ImageToTensor + let input_tensor: Tensor = self + .image_to_tensor + .get_tensor(img) + .route_err_to_failure()?; + + // InvokeTract + let output_tensors = tract_model_service + .run_inference(vec![input_tensor]) + .route_err_to_failure()?; + + // ClassifyOutput + self.classify_output.classify(context, output_tensors) + } +} diff --git a/minifi_rust/extensions/minifi_tensor/src/processors/classify_image/classify_object_def.rs b/minifi_rust/extensions/minifi_tensor/src/processors/classify_image/classify_object_def.rs new file mode 100644 index 0000000000..88b35a72cd --- /dev/null +++ b/minifi_rust/extensions/minifi_tensor/src/processors/classify_image/classify_object_def.rs @@ -0,0 +1,81 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use crate::low_level_processors::{classify_output, image_to_tensor}; +use crate::processors::classify_image::ClassifyImage; +use crate::services::tract_model_service::TractModelService; +use minifi_native::{ + OutputAttribute, ProcessorDefinition, ProcessorInputRequirement, Property, PropertyDefinition, + Relationship, property_definitions, +}; + +pub(crate) const TRACT_MODEL_SERVICE: Property = Property::new( + "Tract model service", + "Reference to a TractModelService controller service. The referenced service \ + owns the compiled model (ONNX or NNEF) that will be evaluated for each \ + incoming flow file.", +); + +pub(super) const SUCCESS: Relationship = Relationship { + name: "success", + description: "Inference and post-processing completed. The flow file content is the original, \ + unchanged image; the classifications are written to the configured output \ + attribute as a JSON array (may be empty).", +}; + +pub(super) const FAILURE: Relationship = Relationship { + name: "failure", + description: "The image could not be decoded, the input tensor could not be built, the model \ + failed to run, or the model outputs could not be interpreted as scores + boxes.", +}; + +impl ProcessorDefinition for ClassifyImage { + const DESCRIPTION: &'static str = "Runs a full image-classification pass in a single processor: decodes the image from the \ + flow file content, resizes and normalises it into an input tensor, runs one inference \ + against the compiled model owned by the referenced TractModelService, and post-processes \ + the score vector (score activation, Top-K selection, confidence filtering, optional label \ + lookup) into predictions. Collapses the ImageToTensor -> InvokeTractModel -> \ + ClassifyOutput chain into one node. The flow file content is left unchanged (the original \ + image); the Top-K classifications are written as a JSON array to the configured output \ + attribute."; + const INPUT_REQUIREMENT: ProcessorInputRequirement = ProcessorInputRequirement::Required; + const SUPPORTS_DYNAMIC_PROPERTIES: bool = false; + const SUPPORTS_DYNAMIC_RELATIONSHIPS: bool = false; + const OUTPUT_ATTRIBUTES: &'static [OutputAttribute] = &[]; + const RELATIONSHIPS: &'static [Relationship] = &[SUCCESS, FAILURE]; + + const PROPERTIES: &'static [PropertyDefinition] = property_definitions![ + image_to_tensor::TARGET_WIDTH, + image_to_tensor::TARGET_HEIGHT, + image_to_tensor::RESIZE_FILTER, + image_to_tensor::RESIZE_MODE, + image_to_tensor::LETTERBOX_PAD_VALUE, + image_to_tensor::COLOR_FORMAT, + image_to_tensor::TENSOR_SHAPE_FORMAT, + image_to_tensor::MEAN, + image_to_tensor::STD_DEV, + image_to_tensor::PIXEL_DIVISOR, + TRACT_MODEL_SERVICE, + classify_output::TOP_K, + classify_output::SCORE_OUTPUT_INDEX, + classify_output::SCORE_ACTIVATION, + classify_output::CONFIDENCE_THRESHOLD, + classify_output::LABELS_FILE_PATH, + classify_output::LABEL_INDEX_OFFSET, + classify_output::OUTPUT_ATTRIBUTE_NAME + ]; +} diff --git a/minifi_rust/extensions/minifi_tensor/src/processors/detect_object.rs b/minifi_rust/extensions/minifi_tensor/src/processors/detect_object.rs new file mode 100644 index 0000000000..6a08fe7546 --- /dev/null +++ b/minifi_rust/extensions/minifi_tensor/src/processors/detect_object.rs @@ -0,0 +1,112 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +mod detect_object_def; + +use crate::low_level_processors::filter_bounding_boxes::FilterBoundingBoxes; +use crate::low_level_processors::image_to_tensor::ImageToTensor; +use crate::utils::dimensions::Dimensions; +use crate::utils::tensor_helpers::load_as_image; +use detect_object_def::TRACT_MODEL_SERVICE; +use minifi_native::macros::ComponentIdentifier; +use minifi_native::{ + FlowFileTransform, GetAttribute, GetControllerService, GetId, GetProperty, InputStream, Logger, + MinifiError, ProcessError, RouteErrorExt, Schedule, TransformedFlowFile, +}; +use tract::Tensor; + +tract::impl_ndarray_interop!(); + +#[derive(ComponentIdentifier)] +pub(crate) struct DetectObject { + image_to_tensor: ImageToTensor, + filter_bounding_boxes: FilterBoundingBoxes, +} + +impl Schedule for DetectObject { + fn schedule(context: &Ctx, logger: &L) -> Result + where + Self: Sized, + { + let image_to_tensor = ImageToTensor::schedule(context, logger)?; + let filter_bounding_boxes = FilterBoundingBoxes::schedule(context, logger)?; + Ok(Self { + image_to_tensor, + filter_bounding_boxes, + }) + } +} + +impl FlowFileTransform for DetectObject { + fn transform< + 'a, + Context: GetProperty + GetControllerService + GetAttribute + GetId, + LoggerImpl: Logger, + >( + &self, + context: &Context, + input_stream: &'a mut dyn InputStream, + logger: &LoggerImpl, + ) -> Result, ProcessError> { + let tract_model_service = context.get_controller_service(&TRACT_MODEL_SERVICE)?; + let img = load_as_image(input_stream).route_err_to_failure()?; + let orig_dim = Dimensions::from_image(&img); + let target_dim = self.image_to_tensor.get_target_dim(); + + // ImageToTensor + let input_tensor: Tensor = self + .image_to_tensor + .get_tensor(img) + .route_err_to_failure()?; + + // InvokeTract + let output_tensors = tract_model_service + .run_inference(vec![input_tensor]) + .route_err_to_failure()?; + + // FilterBoundingBox + self.filter_bounding_boxes + .filter(context, logger, output_tensors, orig_dim, target_dim) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::low_level_processors::image_to_tensor::{ + PIXEL_DIVISOR, TARGET_HEIGHT, TARGET_WIDTH, + }; + use minifi_native::{MockLogger, MockProcessContext}; + use std::io::Cursor; + + #[test] + fn test_missing_controller_service_errors() { + let mut context = MockProcessContext::new(); + context.properties.insert(TARGET_HEIGHT.name(), "100"); + context.properties.insert(TARGET_WIDTH.name(), "100"); + context.properties.insert(PIXEL_DIVISOR.name(), "1.0"); + let processor = + DetectObject::schedule(&context, &MockLogger::new()).expect("Expected to schedule"); + let mut input_stream = Cursor::new(vec![]); + + let result = processor.transform(&context, &mut input_stream, &MockLogger::new()); + assert!( + result.is_err(), + "Should error when TractModelService is missing" + ); + } +} diff --git a/minifi_rust/extensions/minifi_tensor/src/processors/detect_object/detect_object_def.rs b/minifi_rust/extensions/minifi_tensor/src/processors/detect_object/detect_object_def.rs new file mode 100644 index 0000000000..c4e56e9ab1 --- /dev/null +++ b/minifi_rust/extensions/minifi_tensor/src/processors/detect_object/detect_object_def.rs @@ -0,0 +1,99 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use crate::low_level_processors::{filter_bounding_boxes, image_to_tensor}; +use crate::processors::detect_object::DetectObject; +use crate::services::tract_model_service::TractModelService; +use minifi_native::{ + OutputAttribute, ProcessorDefinition, ProcessorInputRequirement, Property, PropertyDefinition, + Relationship, property_definitions, +}; + +pub(crate) const TRACT_MODEL_SERVICE: Property = Property::new( + "Tract model service", + "Reference to a TractModelService controller service. The referenced service \ + owns the compiled model (ONNX or NNEF) that will be evaluated for each \ + incoming flow file.", +); + +pub(super) const SUCCESS: Relationship = Relationship { + name: "success", + description: "Inference and post-processing completed. The flow file content is the original, \ + unchanged image; the detected boxes are written to the configured output \ + attribute as a JSON array (may be empty).", +}; + +pub(super) const FAILURE: Relationship = Relationship { + name: "failure", + description: "The image could not be decoded, the input tensor could not be built, the model \ + failed to run, or the model outputs could not be interpreted as scores + boxes.", +}; + +const OBJECT_COUNT_ATTR: OutputAttribute = OutputAttribute { + name: "object.count", + relationships: &["success"], + description: "Number of bounding boxes retained after confidence filtering and NMS.", +}; + +const DETECTED_OBJECTS_ATTR: OutputAttribute = OutputAttribute { + name: "", + relationships: &["success"], + description: "JSON array of the surviving bounding boxes (fields class_id, confidence, x_min, \ + y_min, x_max, y_max; coordinates normalised to [0,1] against the original \ + image). The attribute name is configurable via the 'Output attribute name' \ + property.", +}; + +impl ProcessorDefinition for DetectObject { + const DESCRIPTION: &'static str = "Runs a full object-detection pass in a single processor: decodes the image from the flow \ + file content, resizes and normalises it into an input tensor, runs one inference against \ + the compiled model owned by the referenced TractModelService, and post-processes the \ + model outputs (score activation, confidence filtering, box decoding, per-class \ + non-maximum suppression) into bounding boxes. Collapses the ImageToTensor -> \ + InvokeTractModel -> FilterBoundingBoxes chain into one node. The flow file content is \ + left unchanged (the original image); the detected boxes are written as a JSON array to \ + the configured output attribute so a downstream DrawBoundingBox can annotate the image."; + const INPUT_REQUIREMENT: ProcessorInputRequirement = ProcessorInputRequirement::Required; + const SUPPORTS_DYNAMIC_PROPERTIES: bool = false; + const SUPPORTS_DYNAMIC_RELATIONSHIPS: bool = false; + const OUTPUT_ATTRIBUTES: &'static [OutputAttribute] = + &[OBJECT_COUNT_ATTR, DETECTED_OBJECTS_ATTR]; + const RELATIONSHIPS: &'static [Relationship] = &[SUCCESS, FAILURE]; + + const PROPERTIES: &'static [PropertyDefinition] = property_definitions![ + image_to_tensor::TARGET_WIDTH, + image_to_tensor::TARGET_HEIGHT, + image_to_tensor::RESIZE_FILTER, + image_to_tensor::RESIZE_MODE, + image_to_tensor::LETTERBOX_PAD_VALUE, + image_to_tensor::COLOR_FORMAT, + image_to_tensor::TENSOR_SHAPE_FORMAT, + image_to_tensor::MEAN, + image_to_tensor::STD_DEV, + image_to_tensor::PIXEL_DIVISOR, + TRACT_MODEL_SERVICE, + filter_bounding_boxes::CONFIDENCE_THRESHOLD, + filter_bounding_boxes::IOU_THRESHOLD, + filter_bounding_boxes::SCORE_OUTPUT_INDEX, + filter_bounding_boxes::BOX_OUTPUT_INDEX, + filter_bounding_boxes::CLASS_OUTPUT_INDEX, + filter_bounding_boxes::BOX_FORMAT, + filter_bounding_boxes::SCORE_ACTIVATION, + filter_bounding_boxes::BACKGROUND_CLASS_INDEX, + filter_bounding_boxes::OUTPUT_ATTRIBUTE_NAME, + ]; +} diff --git a/minifi_rust/extensions/minifi_tensor/src/processors/draw_bounding_box.rs b/minifi_rust/extensions/minifi_tensor/src/processors/draw_bounding_box.rs new file mode 100644 index 0000000000..e665144a99 --- /dev/null +++ b/minifi_rust/extensions/minifi_tensor/src/processors/draw_bounding_box.rs @@ -0,0 +1,165 @@ +use crate::utils::bounding_box::{BoundingBox, BoundingBoxes}; +use image::{ImageFormat, Rgb, load_from_memory}; +use minifi_native::macros::ComponentIdentifier; +use minifi_native::{ + FlowFileTransform, GetAttribute, GetControllerService, GetId, GetProperty, InputStream, Logger, + MinifiError, OutputAttribute, ProcessError, ProcessorDefinition, ProcessorInputRequirement, + Property, PropertyConstraints, PropertyType, Relationship, RouteErrorExt, Schedule, + TransformedFlowFile, +}; +use minifi_native::{PropertyDefinition, PropertySchema, property_definitions}; +use std::io::Cursor; + +pub(crate) const SUCCESS: Relationship = Relationship { + name: "success", + description: "Flowfiles are routed here after drawing the bounding boxes", +}; + +pub(crate) const FAILURE: Relationship = Relationship { + name: "failure", + description: "Invalid FlowFiles are routed here", +}; + +pub(crate) const BOUNDING_BOXES: Property = Property::new( + "Bounding boxes", + "JSON array of bounding boxes to draw onto the image (fields class_id, confidence, x_min, \ + y_min, x_max, y_max; coordinates normalised to [0,1] against the image). Typically the \ + attribute produced by an upstream DetectObject or FilterBoundingBoxes processor.", +) +.with_default("${enrichment.value}"); + +const LINE_THICKNESS: Property = Property::new( + "Line thickness", + "Thickness in pixels of the drawn box outline.", +) +.with_default("5"); + +const LINE_COLOR: Property = Property::new( + "Line color", + "Outline colour as '[R, G, B]' u8 channels (0-255).", +) +.with_default("[0, 255, 0]"); + +#[derive(Debug, ComponentIdentifier)] +pub(crate) struct DrawBoundingBox {} + +impl Schedule for DrawBoundingBox { + fn schedule( + _context: &Ctx, + _logger: &L, + ) -> Result + where + Self: Sized, + { + Ok(Self {}) + } +} + +struct LineColor {} + +impl PropertySchema for LineColor { + const CONSTRAINT: Option = None; + const IS_REQUIRED: bool = false; +} + +impl PropertyType for LineColor { + type Output = Rgb; + + fn parse(s: &str) -> Result { + let clean_str = s.trim().trim_matches(|c| c == '[' || c == ']'); + let mut iter = clean_str.split(','); + + let mut next_channel = || { + iter.next() + .ok_or_else(|| MinifiError::validation("expected R,G,B color channels"))? + .trim() + .parse::() + .map_err(MinifiError::from) + }; + + Ok(Rgb::([ + next_channel()?, + next_channel()?, + next_channel()?, + ])) + } +} + +impl FlowFileTransform for DrawBoundingBox { + fn transform< + 'a, + Context: GetProperty + GetControllerService + GetAttribute + GetId, + LoggerImpl: Logger, + >( + &self, + context: &Context, + input_stream: &'a mut dyn InputStream, + _logger: &LoggerImpl, + ) -> Result, ProcessError> { + let line_thickness = context + .get_property(&LINE_THICKNESS) + .route_err_to_failure()?; + let line_color = context.get_property(&LINE_COLOR).route_err_to_failure()?; + let boxes: Vec = context + .get_property(&BOUNDING_BOXES) + .route_err_to_failure()?; + + let mut image_bytes = Vec::new(); + input_stream.read_to_end(&mut image_bytes)?; + + let mut img = load_from_memory(&image_bytes) + .map(|dyn_img| dyn_img.to_rgb8()) + .route_err_to_failure()?; + + boxes + .iter() + .for_each(|bbox| bbox.draw_onto(&mut img, line_thickness, line_color)); + + let mut output_bytes = Vec::new(); + img.write_to(&mut Cursor::new(&mut output_bytes), ImageFormat::Png) + .route_err_to_failure()?; + + Ok(TransformedFlowFile::new( + &SUCCESS, + Some(output_bytes.into()), + )) + } +} + +impl ProcessorDefinition for DrawBoundingBox { + const DESCRIPTION: &'static str = "Decodes the image from the flow file content, draws each bounding box supplied via the \ + 'Bounding boxes' property onto it, and re-encodes the annotated image as PNG. Pair with an \ + upstream DetectObject / FilterBoundingBoxes to visualise detections."; + const INPUT_REQUIREMENT: ProcessorInputRequirement = ProcessorInputRequirement::Required; + const SUPPORTS_DYNAMIC_PROPERTIES: bool = false; + const SUPPORTS_DYNAMIC_RELATIONSHIPS: bool = false; + const OUTPUT_ATTRIBUTES: &'static [OutputAttribute] = &[]; + const RELATIONSHIPS: &'static [Relationship] = &[SUCCESS, FAILURE]; + const PROPERTIES: &[PropertyDefinition] = + property_definitions![BOUNDING_BOXES, LINE_COLOR, LINE_THICKNESS]; +} + +#[cfg(test)] +mod tests { + use crate::processors::draw_bounding_box::{LINE_COLOR, LINE_THICKNESS}; + use minifi_native::{GetProperty, MockControllerServiceContext}; + + #[test] + fn test_parsing_colors() { + let mock_context = MockControllerServiceContext::default(); + let default_color = mock_context + .get_property(&LINE_COLOR) + .expect("we should parse this"); + let green = image::Rgb([0, 255, 0]); + assert_eq!(default_color, green); + } + + #[test] + fn test_parsing_line_thickness() { + let mock_context = MockControllerServiceContext::default(); + let default_thickness = mock_context + .get_property(&LINE_THICKNESS) + .expect("we should parse this"); + assert_eq!(default_thickness, 5); + } +} diff --git a/minifi_rust/extensions/minifi_tensor/src/processors/mod.rs b/minifi_rust/extensions/minifi_tensor/src/processors/mod.rs new file mode 100644 index 0000000000..83c58ebb03 --- /dev/null +++ b/minifi_rust/extensions/minifi_tensor/src/processors/mod.rs @@ -0,0 +1,20 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +pub(crate) mod classify_image; +pub(crate) mod detect_object; +pub(crate) mod draw_bounding_box; diff --git a/minifi_rust/extensions/minifi_tensor/src/services/mod.rs b/minifi_rust/extensions/minifi_tensor/src/services/mod.rs new file mode 100644 index 0000000000..44ed575366 --- /dev/null +++ b/minifi_rust/extensions/minifi_tensor/src/services/mod.rs @@ -0,0 +1,18 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +pub(crate) mod tract_model_service; diff --git a/minifi_rust/extensions/minifi_tensor/src/services/tract_model_service.rs b/minifi_rust/extensions/minifi_tensor/src/services/tract_model_service.rs new file mode 100644 index 0000000000..9e1b467679 --- /dev/null +++ b/minifi_rust/extensions/minifi_tensor/src/services/tract_model_service.rs @@ -0,0 +1,179 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use crate::services::tract_model_service::service_definition::{MODEL_FILE_PATH, MODEL_FORMAT}; +use minifi_native::macros::{ComponentIdentifier, PropertyType}; +use minifi_native::{EnableControllerService, GetProperty, Logger, MinifiError, trace}; +use std::path::Path; +use strum_macros::{Display, EnumString, IntoStaticStr, VariantNames}; +use tract::__ndarray_interop::anyhow; +use tract::prelude::*; + +mod service_definition; + +#[derive( + Debug, Clone, Copy, PartialEq, Display, EnumString, VariantNames, IntoStaticStr, PropertyType, +)] +#[strum(serialize_all = "PascalCase", const_into_str)] +pub(crate) enum ModelFormat { + Auto, + Onnx, + Nnef, +} + +/// Resolved format after any auto-detection. +#[derive(Debug, Clone, Copy, PartialEq)] +enum ResolvedFormat { + Onnx, + Nnef, +} + +impl ModelFormat { + fn resolve(self, path: &Path) -> Result { + match self { + ModelFormat::Onnx => Ok(ResolvedFormat::Onnx), + ModelFormat::Nnef => Ok(ResolvedFormat::Nnef), + ModelFormat::Auto => { + let path_str = path.to_string_lossy().to_ascii_lowercase(); + if path_str.ends_with(".onnx") { + Ok(ResolvedFormat::Onnx) + } else if path_str.ends_with(".nnef") + || path_str.ends_with(".nnef.tgz") + || path_str.ends_with(".nnef.tar") + || path_str.ends_with(".nnef.tar.gz") + || path.is_dir() + { + Ok(ResolvedFormat::Nnef) + } else { + Err(MinifiError::custom(format!( + "Could not auto-detect model format from '{:?}'. Set 'Model format' to \ + 'Onnx' or 'Nnef' explicitly.", + path + ))) + } + } + } + } +} + +#[derive(ComponentIdentifier)] +pub(crate) struct TractModelService { + runnable_model: Runnable, +} + +impl EnableControllerService for TractModelService { + fn enable(context: &Ctx, logger: &L) -> Result + where + Self: Sized, + { + let model_path = context.get_property(&MODEL_FILE_PATH)?; + let format = context.get_property(&MODEL_FORMAT)?; + let resolved = format.resolve(&model_path)?; + + trace!( + logger, + "Loading Tract model ({:?}) from: {:?}", resolved, model_path + ); + + let model = match resolved { + ResolvedFormat::Onnx => onnx()?.load(&model_path)?.into_model()?, + ResolvedFormat::Nnef => nnef()?.load(&model_path)?, + }; + + let runtime = runtime_for_name("default")?; + let runnable_model = runtime.prepare(model)?; + + trace!(logger, "Successfully loaded and compiled Tract model."); + + Ok(Self { runnable_model }) + } +} + +impl TractModelService { + pub fn run_inference( + &self, + inputs: impl IntoIterator, + ) -> anyhow::Result> { + let vec_inputs: Vec = inputs.into_iter().collect(); + + self.runnable_model.run(vec_inputs) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use std::path::PathBuf; + use std::str::FromStr; + + #[test] + fn test_resolve_explicit_formats_bypass_detection() { + assert_eq!( + ModelFormat::Onnx + .resolve(&PathBuf::from_str("/tmp/no-extension").unwrap()) + .unwrap(), + ResolvedFormat::Onnx + ); + assert_eq!( + ModelFormat::Nnef + .resolve(&PathBuf::from_str("/tmp/no-extension").unwrap()) + .unwrap(), + ResolvedFormat::Nnef + ); + } + + #[test] + fn test_resolve_auto_by_onnx_extension() { + assert_eq!( + ModelFormat::Auto + .resolve(&PathBuf::from_str("/models/face.onnx").unwrap()) + .unwrap(), + ResolvedFormat::Onnx + ); + assert_eq!( + ModelFormat::Auto + .resolve(&PathBuf::from_str("/MODELS/FACE.ONNX").unwrap()) + .unwrap(), + ResolvedFormat::Onnx + ); + } + + #[test] + fn test_resolve_auto_by_nnef_extension() { + assert_eq!( + ModelFormat::Auto + .resolve(&PathBuf::from_str("/models/mobilenet.nnef.tgz").unwrap()) + .unwrap(), + ResolvedFormat::Nnef + ); + assert_eq!( + ModelFormat::Auto + .resolve(&PathBuf::from_str("/models/mobilenet.nnef").unwrap()) + .unwrap(), + ResolvedFormat::Nnef + ); + } + + #[test] + fn test_resolve_auto_errors_on_unknown() { + assert!( + ModelFormat::Auto + .resolve(&PathBuf::from_str("/tmp/no-hint").unwrap()) + .is_err() + ); + } +} diff --git a/minifi_rust/extensions/minifi_tensor/src/services/tract_model_service/service_definition.rs b/minifi_rust/extensions/minifi_tensor/src/services/tract_model_service/service_definition.rs new file mode 100644 index 0000000000..312a470d80 --- /dev/null +++ b/minifi_rust/extensions/minifi_tensor/src/services/tract_model_service/service_definition.rs @@ -0,0 +1,50 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use crate::services::tract_model_service::ModelFormat; +use minifi_native::{ + ControllerServiceDefinition, Property, PropertyDefinition, ProvidedInterface, + property_definitions, +}; +use std::path::PathBuf; + +pub(crate) const MODEL_FILE_PATH: Property = Property::new( + "Model File Path", + "Absolute path to the model on the edge device. For ONNX this is a `.onnx` \ + file; for NNEF this is a `.nnef.tgz` archive, a `.nnef` tarball, or the root \ + directory of an unpacked NNEF model. The model is loaded, parsed, and compiled \ + for the host CPU once when the controller service is enabled; subsequent \ + inference calls reuse the compiled runnable.", +); + +pub(crate) const MODEL_FORMAT: Property = Property::new( + "Model format", + "Format of the file/directory referenced by 'Model File Path'. 'Auto' picks \ + Onnx when the path ends in `.onnx` and Nnef when it ends in `.nnef`, \ + `.nnef.tgz`, `.nnef.tar`, `.nnef.tar.gz`, or points at a directory. Set \ + explicitly when the path uses a non-standard extension.", +) +.with_default(ModelFormat::Auto.into_str()); + +impl ControllerServiceDefinition for super::TractModelService { + const DESCRIPTION: &'static str = "Provides a shared, CPU-optimized neural network for inference. Supports ONNX (`.onnx`) \ + and NNEF (directory or tarball) models; the format can be auto-detected from the file \ + extension or set explicitly."; + const PROPERTIES: &'static [PropertyDefinition] = + property_definitions![MODEL_FILE_PATH, MODEL_FORMAT]; + const PROVIDED_APIS: &'static [ProvidedInterface] = &[]; +} diff --git a/minifi_rust/extensions/minifi_tensor/src/utils/bounding_box.rs b/minifi_rust/extensions/minifi_tensor/src/utils/bounding_box.rs new file mode 100644 index 0000000000..8e8c00c246 --- /dev/null +++ b/minifi_rust/extensions/minifi_tensor/src/utils/bounding_box.rs @@ -0,0 +1,181 @@ +use image::{Rgb, RgbImage}; +use minifi_native::{MinifiError, PropertyConstraints, PropertySchema, PropertyType}; +use serde::{Deserialize, Serialize}; + +#[derive(Serialize, Deserialize, Clone, Debug)] +pub struct BoundingBox { + pub(crate) class_id: usize, + pub(crate) confidence: f32, + pub(crate) x_min: f32, + pub(crate) y_min: f32, + pub(crate) x_max: f32, + pub(crate) y_max: f32, +} + +fn draw_thick_rect( + img: &mut RgbImage, + left: u32, + top: u32, + right: u32, + bottom: u32, + thickness: u32, + color: Rgb, +) { + let box_width = right.saturating_sub(left); + let box_height = bottom.saturating_sub(top); + + for t in 0..thickness { + if box_width > 2 * t && box_height > 2 * t { + let rect = imageproc::rect::Rect::at((left + t) as i32, (top + t) as i32) + .of_size(box_width - 2 * t, box_height - 2 * t); + imageproc::drawing::draw_hollow_rect_mut(img, rect, color); + } + } +} + +impl BoundingBox { + pub fn class_id(&self) -> usize { + self.class_id + } + pub fn confidence(&self) -> f32 { + self.confidence + } + + pub(crate) fn calculate_intersection_over_union(box1: &BoundingBox, box2: &BoundingBox) -> f32 { + let x_left = box1.x_min.max(box2.x_min); + let y_top = box1.y_min.max(box2.y_min); + let x_right = box1.x_max.min(box2.x_max); + let y_bottom = box1.y_max.min(box2.y_max); + + if x_right < x_left || y_bottom < y_top { + return 0.0; + } + + let intersection_area = (x_right - x_left) * (y_bottom - y_top); + let box1_area = (box1.x_max - box1.x_min) * (box1.y_max - box1.y_min); + let box2_area = (box2.x_max - box2.x_min) * (box2.y_max - box2.y_min); + + intersection_area / (box1_area + box2_area - intersection_area) + } + + pub(crate) fn apply_non_maximum_suppression( + mut boxes: Vec, + iou_threshold: f32, + ) -> Vec { + boxes.sort_by(|a, b| { + b.confidence() + .partial_cmp(&a.confidence()) + .unwrap_or(std::cmp::Ordering::Equal) + }); + + let mut keep = Vec::new(); + let mut is_suppressed = vec![false; boxes.len()]; + + for i in 0..boxes.len() { + if is_suppressed[i] { + continue; + } + + keep.push(boxes[i].clone()); + + for j in (i + 1)..boxes.len() { + if is_suppressed[j] { + continue; + } + + if boxes[i].class_id() == boxes[j].class_id() { + let iou = BoundingBox::calculate_intersection_over_union(&boxes[i], &boxes[j]); + if iou > iou_threshold { + is_suppressed[j] = true; + } + } + } + } + keep + } + + pub(crate) fn draw_onto(&self, img: &mut RgbImage, line_thickness: u32, line_color: Rgb) { + let width = img.width() as f32; + let height = img.height() as f32; + + let left = (self.x_min * width).round() as u32; + let top = (self.y_min * height).round() as u32; + let right = (self.x_max * width).round() as u32; + let bottom = (self.y_max * height).round() as u32; + + draw_thick_rect(img, left, top, right, bottom, line_thickness, line_color); + } +} + +pub(crate) struct BoundingBoxes {} + +impl PropertySchema for BoundingBoxes { + const CONSTRAINT: Option = None; + const IS_REQUIRED: bool = false; +} + +impl PropertyType for BoundingBoxes { + type Output = Vec; + + fn parse(s: &str) -> Result { + serde_json::from_str::>(s).map_err(MinifiError::other) + } +} + +#[cfg(test)] +mod tests { + use crate::utils::bounding_box::BoundingBox; + + #[test] + fn test_iou_zero_when_disjoint() { + let a = BoundingBox { + class_id: 0, + confidence: 1.0, + x_min: 0.0, + y_min: 0.0, + x_max: 1.0, + y_max: 1.0, + }; + let b = BoundingBox { + class_id: 0, + confidence: 1.0, + x_min: 2.0, + y_min: 2.0, + x_max: 3.0, + y_max: 3.0, + }; + assert_eq!(BoundingBox::calculate_intersection_over_union(&a, &b), 0.0); + } + + #[test] + fn test_nms_suppresses_overlapping_same_class() { + let boxes = vec![ + BoundingBox { + class_id: 1, + confidence: 0.9, + x_min: 0.0, + y_min: 0.0, + x_max: 1.0, + y_max: 1.0, + }, + BoundingBox { + class_id: 1, + confidence: 0.8, + x_min: 0.1, + y_min: 0.1, + x_max: 1.0, + y_max: 1.0, + }, + BoundingBox { + class_id: 2, + confidence: 0.7, + x_min: 0.1, + y_min: 0.1, + x_max: 1.0, + y_max: 1.0, + }, + ]; + let kept = BoundingBox::apply_non_maximum_suppression(boxes, 0.5); + assert_eq!(kept.len(), 2); // same-class dupe suppressed, other-class kept + } +} diff --git a/minifi_rust/extensions/minifi_tensor/src/utils/dimensions.rs b/minifi_rust/extensions/minifi_tensor/src/utils/dimensions.rs new file mode 100644 index 0000000000..9a78016b9b --- /dev/null +++ b/minifi_rust/extensions/minifi_tensor/src/utils/dimensions.rs @@ -0,0 +1,50 @@ +use minifi_native::{GetAttribute, MinifiError}; + +#[derive(Debug, Clone, Copy, PartialEq)] +pub(crate) struct Dimensions { + pub(crate) width: f32, + pub(crate) height: f32, +} + +impl Dimensions { + pub(crate) fn from_image(img: &image::DynamicImage) -> Self { + Self { + width: img.width() as f32, + height: img.height() as f32, + } + } + + pub(crate) fn original_from_attributes( + context: &Context, + ) -> Result { + let orig_w = context + .get_required_attribute("image.original.width")? + .parse::()?; + + let orig_h = context + .get_required_attribute("image.original.height")? + .parse::()?; + + Ok(Dimensions { + width: orig_w, + height: orig_h, + }) + } + + pub(crate) fn target_from_attributes( + context: &Context, + ) -> Result { + let orig_w = context + .get_required_attribute("image.target.width")? + .parse::()?; + + let orig_h = context + .get_required_attribute("image.target.height")? + .parse::()?; + + Ok(Dimensions { + width: orig_w, + height: orig_h, + }) + } +} diff --git a/minifi_rust/extensions/minifi_tensor/src/utils/mod.rs b/minifi_rust/extensions/minifi_tensor/src/utils/mod.rs new file mode 100644 index 0000000000..faf8b16f30 --- /dev/null +++ b/minifi_rust/extensions/minifi_tensor/src/utils/mod.rs @@ -0,0 +1,22 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +pub(crate) mod bounding_box; +pub(crate) mod dimensions; +pub(crate) mod per_channel_f32; +pub(crate) mod score_activation; +pub(crate) mod tensor_helpers; diff --git a/minifi_rust/extensions/minifi_tensor/src/utils/per_channel_f32.rs b/minifi_rust/extensions/minifi_tensor/src/utils/per_channel_f32.rs new file mode 100644 index 0000000000..887b6aa564 --- /dev/null +++ b/minifi_rust/extensions/minifi_tensor/src/utils/per_channel_f32.rs @@ -0,0 +1,69 @@ +use crate::utils::per_channel_f32::PerChannelF32::TriChannel; +use minifi_native::{MinifiError, PropertyConstraints, PropertySchema, PropertyType}; + +#[derive(PartialEq, Debug)] +pub(crate) enum PerChannelF32 { + SingleChannel(f32), + TriChannel([f32; 3]), +} + +impl PerChannelF32 { + pub fn contains_zero(&self) -> bool { + match self { + Self::SingleChannel(f) => *f == 0.0f32, + TriChannel(channels) => channels.contains(&0.0f32), + } + } + + pub fn per_channel(&self, channel_id: usize) -> f32 { + match self { + Self::SingleChannel(f) => *f, + TriChannel(channels) => channels[channel_id], + } + } +} + +impl PropertySchema for PerChannelF32 { + const CONSTRAINT: Option = None; + const IS_REQUIRED: bool = false; +} + +impl PropertyType for PerChannelF32 { + type Output = PerChannelF32; + + fn parse(input: &str) -> Result { + let parts: Vec<&str> = input.split(',').map(|s| s.trim()).collect(); + match parts.len() { + 1 => Ok(Self::SingleChannel(parts[0].parse::()?)), + 3 => { + let f = parts[0].parse::()?; + let s = parts[1].parse::()?; + let t = parts[2].parse::()?; + Ok(TriChannel([f, s, t])) + } + _n => Err(MinifiError::validation( + "expected 1 or 3 comma-separated floats", + )), + } + } +} + +#[cfg(test)] +mod tests { + use crate::utils::per_channel_f32::PerChannelF32; + use minifi_native::PropertyType; + + #[test] + fn test_parse_per_channel_f32_accepts_scalar_and_triple() { + assert_eq!( + PerChannelF32::parse("0.5").unwrap(), + PerChannelF32::SingleChannel(0.5) + ); + assert_eq!( + PerChannelF32::parse("0.485, 0.456, 0.406").unwrap(), + PerChannelF32::TriChannel([0.485, 0.456, 0.406]) + ); + assert!(PerChannelF32::parse("1, 2").is_err()); + assert!(PerChannelF32::parse("1, 2, three").is_err()); + } +} diff --git a/minifi_rust/extensions/minifi_tensor/src/utils/score_activation.rs b/minifi_rust/extensions/minifi_tensor/src/utils/score_activation.rs new file mode 100644 index 0000000000..12125693e0 --- /dev/null +++ b/minifi_rust/extensions/minifi_tensor/src/utils/score_activation.rs @@ -0,0 +1,16 @@ +use minifi_native::macros::PropertyType; +use strum_macros::{Display, EnumString, IntoStaticStr, VariantNames}; + +#[derive( + Debug, Clone, Copy, PartialEq, Display, EnumString, VariantNames, IntoStaticStr, PropertyType, +)] +#[strum(serialize_all = "PascalCase", const_into_str)] +pub(crate) enum ScoreActivation { + /// Cross-class softmax; classes are mutually exclusive (typical for + /// ImageNet-trained ResNet/MobileNet/EfficientNet ONNX exports). + Softmax, + /// Per-class sigmoid; classes are independent (multi-label classifiers). + Sigmoid, + /// Pass-through — the model already emits probabilities. + None, +} diff --git a/minifi_rust/extensions/minifi_tensor/src/utils/tensor_helpers.rs b/minifi_rust/extensions/minifi_tensor/src/utils/tensor_helpers.rs new file mode 100644 index 0000000000..2c883be7d4 --- /dev/null +++ b/minifi_rust/extensions/minifi_tensor/src/utils/tensor_helpers.rs @@ -0,0 +1,156 @@ +use image::{DynamicImage, ImageResult}; +use minifi_native::{GetAttribute, InputStream, MinifiError}; +use strum_macros::{Display, EnumString}; +use tract::__ndarray_interop::TensorInterface; +use tract::Tensor; +use tract::prelude::DatumType; + +tract::impl_ndarray_interop!(); + +fn parse_tensor_shape( + context: &Context, + id: usize, +) -> Result, MinifiError> { + let shape_str = context.get_required_attribute(&format!("tensor.{}.shape", id))?; + + if shape_str.trim().is_empty() { + return Ok(Vec::new()); + } + + let shape = shape_str + .split(',') + .map(|s| s.trim().parse::()) + .collect::, _>>()?; + + Ok(shape) +} + +#[derive(Debug, Clone, Copy, PartialEq, Display, EnumString)] +#[strum(serialize_all = "PascalCase", const_into_str)] +pub(crate) enum MinifiDatumType { + F32, +} + +impl From for DatumType { + fn from(value: MinifiDatumType) -> Self { + match value { + MinifiDatumType::F32 => DatumType::F32, + } + } +} + +fn numeric_datum_type_from_str(s: &str) -> Option { + Some(match s { + "U8" => DatumType::U8, + "U16" => DatumType::U16, + "U32" => DatumType::U32, + "U64" => DatumType::U64, + "I8" => DatumType::I8, + "I16" => DatumType::I16, + "I32" => DatumType::I32, + "I64" => DatumType::I64, + "F16" => DatumType::F16, + "F32" => DatumType::F32, + "F64" => DatumType::F64, + _ => return None, + }) +} + +fn parse_tensor_dtype( + context: &Context, + id: usize, +) -> Result { + let dtype_str = context.get_required_attribute(&format!("tensor.{}.dtype", id))?; + numeric_datum_type_from_str(&dtype_str).ok_or_else(|| { + MinifiError::custom(format!( + "Unsupported tensor.{}.dtype '{}': only numeric tensors can be read", + id, dtype_str + )) + }) +} + +pub(crate) fn deserialize_tensors( + context: &Context, + input_stream: &mut dyn InputStream, +) -> Result, MinifiError> { + let mut result = vec![]; + + let mut flow_file_contents = Vec::new(); + input_stream.read_to_end(&mut flow_file_contents)?; + let number_of_tensors = context + .get_required_attribute("tensors.len")? + .parse::()?; + + let mut cursor = 0usize; + for i in 0..number_of_tensors { + let tensor_len = context + .get_required_attribute(&format!("tensor.{}.bytes", i))? + .parse::()?; + let tensor_shape = parse_tensor_shape(context, i)?; + let tensor_dtype = parse_tensor_dtype(context, i)?; + let tensor_data = &flow_file_contents[cursor..cursor + tensor_len]; + result.push(Tensor::from_bytes( + tensor_dtype, + &tensor_shape, + tensor_data, + )?); + cursor += tensor_len; + } + + Ok(result) +} + +pub(crate) fn tensor_as_f32(tensors: &[Tensor], index: usize) -> Result, MinifiError> { + let tensor = tensors + .get(index) + .ok_or(MinifiError::custom("Invalid shape of tensors"))?; + let casted = tensor.convert_to(DatumType::F32)?; + Ok(casted.as_slice::()?.to_vec()) +} + +pub(crate) fn tensor_shape(tensors: &[Tensor], index: usize) -> Result, MinifiError> { + let tensor = tensors + .get(index) + .ok_or(MinifiError::custom("Invalid shape of tensors"))?; + Ok(tensor.shape()?.to_vec()) +} + +pub(crate) fn load_as_image(input_stream: &mut dyn InputStream) -> ImageResult { + let mut raw_bytes = Vec::new(); + input_stream.read_to_end(&mut raw_bytes)?; + + image::load_from_memory(&raw_bytes) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_tensor_as_f32_reads_f32_tensor() { + let t = Tensor::from_slice::(&[3], &[0.5, -1.5, 2.0]).unwrap(); + assert_eq!(tensor_as_f32(&[t], 0).unwrap(), vec![0.5, -1.5, 2.0]); + } + + #[test] + fn test_tensor_as_f32_casts_integer_tensor() { + // i64 class-index tensor (e.g. TF-OD / EfficientNMS 'detection_classes') + // must be cast to f32, not byte-reinterpreted. + let t = Tensor::from_slice::(&[4], &[0, 1, 17, 80]).unwrap(); + assert_eq!(tensor_as_f32(&[t], 0).unwrap(), vec![0.0, 1.0, 17.0, 80.0]); + } + + #[test] + fn test_tensor_as_f32_index_out_of_range_errors() { + assert!(tensor_as_f32(&[], 0).is_err()); + } + + #[test] + fn test_numeric_datum_type_accepts_numeric_rejects_other() { + assert_eq!(numeric_datum_type_from_str("I64"), Some(DatumType::I64)); + assert_eq!(numeric_datum_type_from_str("F32"), Some(DatumType::F32)); + assert_eq!(numeric_datum_type_from_str("U8"), Some(DatumType::U8)); + assert_eq!(numeric_datum_type_from_str("String"), None); + assert_eq!(numeric_datum_type_from_str("Bool"), None); + } +} diff --git a/minifi_rust/minifi_native/src/lib.rs b/minifi_rust/minifi_native/src/lib.rs index b855291a29..f8509301d6 100644 --- a/minifi_rust/minifi_native/src/lib.rs +++ b/minifi_rust/minifi_native/src/lib.rs @@ -83,10 +83,10 @@ macro_rules! declare_minifi_extension { ( // Group name group_name: $group:expr, - // Match a tuple of three types for each processor - processors: [ $( ($kind:ty, $thread:ty, $impl:ty) ),* $(,)? ], - // Match a single type for each controller service - controllers: [ $( $ctrl:ty ),* $(,)? ] + // Match a tuple of three types for each processor, WITH optional attributes + processors: [ $( $(#[$proc_meta:meta])* ($kind:ty, $thread:ty, $impl:ty) ),* $(,)? ], + // Match a single type for each controller service, WITH optional attributes + controllers: [ $( $(#[$ctrl_meta:meta])* $ctrl:ty ),* $(,)? ] ) => { #[unsafe(no_mangle)] @@ -108,8 +108,9 @@ macro_rules! declare_minifi_extension { let extension = minifi_native::sys::minifi_register_extension(extension_context, &extension_definition); - $( + // Re-apply the captured attributes (e.g., #[cfg(...)]) to this block + $(#[$proc_meta])* { let processor_def = minifi_native::Processor::< $impl, @@ -125,6 +126,8 @@ macro_rules! declare_minifi_extension { )* $( + // Re-apply the captured attributes to this block + $(#[$ctrl_meta])* { let controller_def = minifi_native::ControllerService::< diff --git a/minifi_rust/minifi_rs_behave/Cargo.toml b/minifi_rust/minifi_rs_behave/Cargo.toml index c13e8521e5..7656efd370 100644 --- a/minifi_rust/minifi_rs_behave/Cargo.toml +++ b/minifi_rust/minifi_rs_behave/Cargo.toml @@ -5,7 +5,6 @@ edition = "2024" [dependencies] glob = "0.3.3" -minifi_rs_playground = { path = "../extensions/minifi_rs_playground" } minifi_native_sys = { path = "../minifi_native_sys" } [lints.rust] diff --git a/minifi_rust/minifi_rs_behave/Dockerfile.alpine b/minifi_rust/minifi_rs_behave/Dockerfile.alpine index 89f2c01987..19b69f53d6 100644 --- a/minifi_rust/minifi_rs_behave/Dockerfile.alpine +++ b/minifi_rust/minifi_rs_behave/Dockerfile.alpine @@ -18,7 +18,7 @@ COPY target/.docker_sdk.zi[p] /app/target/ RUN cargo chef cook --release --recipe-path recipe.json COPY . . -RUN cargo build --release +RUN cargo build --release --all-features # Export Stage FROM scratch AS bin-export diff --git a/minifi_rust/minifi_rs_behave/Dockerfile.debian b/minifi_rust/minifi_rs_behave/Dockerfile.debian index e3283496b1..48507ef2e2 100644 --- a/minifi_rust/minifi_rs_behave/Dockerfile.debian +++ b/minifi_rust/minifi_rs_behave/Dockerfile.debian @@ -1,4 +1,4 @@ -FROM rust:slim-bullseye AS chef +FROM rust:slim-bookworm AS chef RUN apt-get update && apt-get install -y clang lld pkg-config curl tar && cargo install cargo-chef WORKDIR /app @@ -18,7 +18,7 @@ COPY target/.docker_sdk.zi[p] /app/target/ RUN cargo chef cook --release --recipe-path recipe.json COPY . . -RUN cargo build --release +RUN cargo build --release --all-features # Export Stage FROM scratch AS bin-export diff --git a/minifi_rust/minifi_rs_behave/linux_build.sh b/minifi_rust/minifi_rs_behave/linux_build.sh index 44c9c924a8..b751802270 100755 --- a/minifi_rust/minifi_rs_behave/linux_build.sh +++ b/minifi_rust/minifi_rs_behave/linux_build.sh @@ -81,10 +81,19 @@ TARGET_DIR="target/release" mkdir -p "$TARGET_DIR" # 3. Build using Docker +# When running under GitHub Actions, persist the chef layer across runs +CACHE_ARGS=() +if [ -n "${ACTIONS_CACHE_URL}${ACTIONS_RESULTS_URL}" ]; then + echo "GitHub Actions cache detected — enabling type=gha buildx cache" + CACHE_ARGS+=(--cache-from "type=gha,scope=behave-$FLAVOR") + CACHE_ARGS+=(--cache-to "type=gha,mode=max,scope=behave-$FLAVOR") +fi + docker buildx build \ -f "$DOCKERFILE" \ --target bin-export \ --build-arg MINIFI_SDK_PATH="$DOCKER_SDK_ARG" \ + "${CACHE_ARGS[@]}" \ --output type=local,dest="$TARGET_DIR" \ . From 423a292e6ac55a8d6970b0900499d588f5f05399 Mon Sep 17 00:00:00 2001 From: Martin Zink Date: Wed, 9 Sep 2026 13:45:56 +0200 Subject: [PATCH 02/32] review changes --- behave_framework/pyproject.toml | 3 +- .../minifi_tensor/features/environment.py | 89 ++++++++++++++----- .../filter_bounding_boxes.rs | 8 +- .../low_level_processors/image_to_tensor.rs | 4 +- .../src/services/tract_model_service.rs | 5 +- 5 files changed, 79 insertions(+), 30 deletions(-) diff --git a/behave_framework/pyproject.toml b/behave_framework/pyproject.toml index d166376709..04c3039522 100644 --- a/behave_framework/pyproject.toml +++ b/behave_framework/pyproject.toml @@ -9,7 +9,8 @@ dependencies = [ "PyYAML==6.0.3", "humanfriendly==10.0", "cryptography==50.0.0", - "pyjks==20.0.0" + "pyjks==20.0.0", + "certifi==2026.7.22" ] [tool.setuptools] diff --git a/minifi_rust/extensions/minifi_tensor/features/environment.py b/minifi_rust/extensions/minifi_tensor/features/environment.py index a787ac9d42..c1b8a2fde7 100644 --- a/minifi_rust/extensions/minifi_tensor/features/environment.py +++ b/minifi_rust/extensions/minifi_tensor/features/environment.py @@ -15,48 +15,91 @@ # specific language governing permissions and limitations # under the License. +import hashlib import os +import shutil +import ssl import urllib.request +import certifi + from minifi_behave.core.hooks import ( add_extension_to_minifi_container, common_after_scenario, common_before_scenario, ) + +_SSL_CONTEXT = ssl.create_default_context(cafile=certifi.where()) + + +class RemoteAsset: + def __init__(self, url: str, sha256: str): + self.url = url + self.sha256 = sha256 + + def acquire(self, cache_dir: str, filename: str) -> str: + dest = os.path.join(cache_dir, filename) + if os.path.exists(dest) and self._verify(dest): + return dest + os.makedirs(cache_dir, exist_ok=True) + tmp = dest + ".part" + print(f"[minifi_tensor tests] fetching {filename} from {self.url}") + with urllib.request.urlopen(self.url, context=_SSL_CONTEXT) as response, open(tmp, "wb") as out: + shutil.copyfileobj(response, out) + if not self._verify(tmp): + actual = self._digest(tmp) + os.remove(tmp) + raise RuntimeError( + f"sha256 mismatch for {filename}: expected {self.sha256}, got {actual}" + ) + os.replace(tmp, dest) + return dest + + def _verify(self, path: str) -> bool: + return self._digest(path) == self.sha256 + + @staticmethod + def _digest(path: str) -> str: + h = hashlib.sha256() + with open(path, "rb") as f: + for chunk in iter(lambda: f.read(1 << 20), b""): + h.update(chunk) + return h.hexdigest() + + # Model / label / image assets fetched on first use. All hosted on # public buckets or the sonos/tract repo -REMOTE_ASSETS: dict[str, str] = { +REMOTE_ASSETS: dict[str, RemoteAsset] = { # ImageNet MobileNetV2 classifier (~14 MB) — the reference model used by tract's unit tests - "mobilenetv2-7.onnx": "https://s3.amazonaws.com/tract-ci-builds/tests/mobilenetv2-7.onnx", + "mobilenetv2-7.onnx": RemoteAsset( + "https://s3.amazonaws.com/tract-ci-builds/tests/mobilenetv2-7.onnx", + "c1c513582d56afceff8516c73804e484c81c6a830712ab6d682253f4a3cd042f", + ), # 1000-class ImageNet labels (line N = class N; line 0 is "dummy") - "imagenet_slim_labels.txt": "https://raw.githubusercontent.com/sonos/tract/main/examples/" - "onnx-mobilenet-v2/imagenet_slim_labels.txt", + "imagenet_slim_labels.txt": RemoteAsset( + "https://raw.githubusercontent.com/sonos/tract/main/examples/" + "onnx-mobilenet-v2/imagenet_slim_labels.txt", + "e8d2cef25bb7b3c8c6923ad3c463b47de8b8535cadf4bd62a2ca2532c587eb9f", + ), # Same test image tract's example uses. MobileNetV2 confidently # classifies this as "military uniform". - "grace_hopper.jpg": "https://raw.githubusercontent.com/sonos/tract/main/examples/" - "onnx-mobilenet-v2/grace_hopper.jpg", + "grace_hopper.jpg": RemoteAsset( + "https://raw.githubusercontent.com/sonos/tract/main/examples/" + "onnx-mobilenet-v2/grace_hopper.jpg", + "e1f57e98cf38076c0f9a058d74ffddf90f20453e436033784606b63c8ed2e49a", + ), # UltraFace RFB-320 (~1.2 MB): 2-output SSD-style detector matching the # existing FilterBoundingBoxes defaults (Xyxy boxes, class 0 = background, # softmax over 2 classes: background/face). 320x240 RGB, mean=127, std=128. - "version-RFB-320.onnx": "https://github.com/onnx/models/raw/refs/heads/main/validated/vision/" - "body_analysis/ultraface/models/version-RFB-320.onnx", + "version-RFB-320.onnx": RemoteAsset( + "https://github.com/onnx/models/raw/refs/heads/main/validated/vision/" + "body_analysis/ultraface/models/version-RFB-320.onnx", + "34cd7e60aeff28744c657de7a3dc64e872d506741de66987f3426f2b79f88017", + ), } -def _ensure_asset(cache_dir: str, filename: str) -> str: - dest = os.path.join(cache_dir, filename) - if os.path.exists(dest): - return dest - url = REMOTE_ASSETS[filename] - os.makedirs(cache_dir, exist_ok=True) - tmp = dest + ".part" - print(f"[minifi_tensor tests] fetching {filename} from {url}") - urllib.request.urlretrieve(url, tmp) - os.replace(tmp, dest) - return dest - - def before_all(context): dir_path = os.path.dirname(os.path.realpath(__file__)) build_path = os.path.normpath(os.path.join(dir_path, "../../../target/release/")) @@ -65,8 +108,8 @@ def before_all(context): context.tensor_resource_dir = os.path.join(dir_path, "resources") os.makedirs(context.tensor_resource_dir, exist_ok=True) - for name in REMOTE_ASSETS: - _ensure_asset(context.tensor_resource_dir, name) + for name, asset in REMOTE_ASSETS.items(): + asset.acquire(context.tensor_resource_dir, name) def before_scenario(context, scenario): diff --git a/minifi_rust/extensions/minifi_tensor/src/low_level_processors/filter_bounding_boxes.rs b/minifi_rust/extensions/minifi_tensor/src/low_level_processors/filter_bounding_boxes.rs index 8f6723b34a..d7b8b2a602 100644 --- a/minifi_rust/extensions/minifi_tensor/src/low_level_processors/filter_bounding_boxes.rs +++ b/minifi_rust/extensions/minifi_tensor/src/low_level_processors/filter_bounding_boxes.rs @@ -255,6 +255,9 @@ impl FilterBoundingBoxes { if raw_class < 0.0 { continue; } + if !confidence.is_finite() || !raw_class.is_finite() { + continue; + } let class_id = raw_class.round() as usize; if self.background_class_index == Some(class_id) { continue; @@ -309,7 +312,10 @@ impl FilterBoundingBoxes { None => (Some(Content::Buffer(json_output)), None), Some(output_attr) => ( None, - Some((output_attr, serde_json::to_string(&filtered_boxes).unwrap())), + Some(( + output_attr, + serde_json::to_string(&filtered_boxes).route_err_to_failure()?, + )), ), }; diff --git a/minifi_rust/extensions/minifi_tensor/src/low_level_processors/image_to_tensor.rs b/minifi_rust/extensions/minifi_tensor/src/low_level_processors/image_to_tensor.rs index 8889389d4b..aff7abb6b9 100644 --- a/minifi_rust/extensions/minifi_tensor/src/low_level_processors/image_to_tensor.rs +++ b/minifi_rust/extensions/minifi_tensor/src/low_level_processors/image_to_tensor.rs @@ -74,8 +74,8 @@ pub(crate) enum ColorFormat { )] #[strum(serialize_all = "UPPERCASE", const_into_str)] pub(crate) enum TensorShapeFormat { - Chw, // center, height, width - Hwc, // height, width, center + Chw, // channel, height, width + Hwc, // height, width, channel } #[derive( diff --git a/minifi_rust/extensions/minifi_tensor/src/services/tract_model_service.rs b/minifi_rust/extensions/minifi_tensor/src/services/tract_model_service.rs index 9e1b467679..99c9d53cbd 100644 --- a/minifi_rust/extensions/minifi_tensor/src/services/tract_model_service.rs +++ b/minifi_rust/extensions/minifi_tensor/src/services/tract_model_service.rs @@ -20,7 +20,6 @@ use minifi_native::macros::{ComponentIdentifier, PropertyType}; use minifi_native::{EnableControllerService, GetProperty, Logger, MinifiError, trace}; use std::path::Path; use strum_macros::{Display, EnumString, IntoStaticStr, VariantNames}; -use tract::__ndarray_interop::anyhow; use tract::prelude::*; mod service_definition; @@ -107,10 +106,10 @@ impl TractModelService { pub fn run_inference( &self, inputs: impl IntoIterator, - ) -> anyhow::Result> { + ) -> Result, MinifiError> { let vec_inputs: Vec = inputs.into_iter().collect(); - self.runnable_model.run(vec_inputs) + Ok(self.runnable_model.run(vec_inputs)?) } } From cf0a2d93e5a3ea8a89873bb07a51b55f962554bc Mon Sep 17 00:00:00 2001 From: Martin Zink Date: Wed, 9 Sep 2026 13:54:36 +0200 Subject: [PATCH 03/32] ruff fix --- .../extensions/minifi_tensor/features/environment.py | 12 +++--------- 1 file changed, 3 insertions(+), 9 deletions(-) diff --git a/minifi_rust/extensions/minifi_tensor/features/environment.py b/minifi_rust/extensions/minifi_tensor/features/environment.py index c1b8a2fde7..9211634c58 100644 --- a/minifi_rust/extensions/minifi_tensor/features/environment.py +++ b/minifi_rust/extensions/minifi_tensor/features/environment.py @@ -22,14 +22,12 @@ import urllib.request import certifi - from minifi_behave.core.hooks import ( add_extension_to_minifi_container, common_after_scenario, common_before_scenario, ) - _SSL_CONTEXT = ssl.create_default_context(cafile=certifi.where()) @@ -50,9 +48,7 @@ def acquire(self, cache_dir: str, filename: str) -> str: if not self._verify(tmp): actual = self._digest(tmp) os.remove(tmp) - raise RuntimeError( - f"sha256 mismatch for {filename}: expected {self.sha256}, got {actual}" - ) + raise RuntimeError(f"sha256 mismatch for {filename}: expected {self.sha256}, got {actual}") os.replace(tmp, dest) return dest @@ -78,15 +74,13 @@ def _digest(path: str) -> str: ), # 1000-class ImageNet labels (line N = class N; line 0 is "dummy") "imagenet_slim_labels.txt": RemoteAsset( - "https://raw.githubusercontent.com/sonos/tract/main/examples/" - "onnx-mobilenet-v2/imagenet_slim_labels.txt", + "https://raw.githubusercontent.com/sonos/tract/main/examples/onnx-mobilenet-v2/imagenet_slim_labels.txt", "e8d2cef25bb7b3c8c6923ad3c463b47de8b8535cadf4bd62a2ca2532c587eb9f", ), # Same test image tract's example uses. MobileNetV2 confidently # classifies this as "military uniform". "grace_hopper.jpg": RemoteAsset( - "https://raw.githubusercontent.com/sonos/tract/main/examples/" - "onnx-mobilenet-v2/grace_hopper.jpg", + "https://raw.githubusercontent.com/sonos/tract/main/examples/onnx-mobilenet-v2/grace_hopper.jpg", "e1f57e98cf38076c0f9a058d74ffddf90f20453e436033784606b63c8ed2e49a", ), # UltraFace RFB-320 (~1.2 MB): 2-output SSD-style detector matching the From e8ebb2b02fdf4af76b5a40330fb8aa7b2ba21093 Mon Sep 17 00:00:00 2001 From: Martin Zink Date: Wed, 9 Sep 2026 14:09:38 +0200 Subject: [PATCH 04/32] fix cargo clippy --- .../minifi_tensor/src/low_level_processors/classify_output.rs | 4 ++-- .../minifi_tensor/src/low_level_processors/image_to_tensor.rs | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/minifi_rust/extensions/minifi_tensor/src/low_level_processors/classify_output.rs b/minifi_rust/extensions/minifi_tensor/src/low_level_processors/classify_output.rs index a67701bda5..ba4b25b99a 100644 --- a/minifi_rust/extensions/minifi_tensor/src/low_level_processors/classify_output.rs +++ b/minifi_rust/extensions/minifi_tensor/src/low_level_processors/classify_output.rs @@ -403,7 +403,7 @@ mod tests { .transform(&context, &mut stream, &MockLogger::new()) .expect_err("batched scores should be rejected"); match err { - ProcessError::Route(route) => assert_eq!(route.relationship.as_ref(), FAILURE.name), + ProcessError::Route(route) => assert_eq!(route.relationship, FAILURE.name), other => panic!("expected route to failure, got {other:?}"), } } @@ -418,7 +418,7 @@ mod tests { .expect_err("missing attribute should route to failure via a Route error"); match err { ProcessError::Route(route) => { - assert_eq!(route.relationship.as_ref(), FAILURE.name) + assert_eq!(route.relationship, FAILURE.name) } other => panic!("expected route to failure, got {other:?}"), } diff --git a/minifi_rust/extensions/minifi_tensor/src/low_level_processors/image_to_tensor.rs b/minifi_rust/extensions/minifi_tensor/src/low_level_processors/image_to_tensor.rs index aff7abb6b9..2a747e51b7 100644 --- a/minifi_rust/extensions/minifi_tensor/src/low_level_processors/image_to_tensor.rs +++ b/minifi_rust/extensions/minifi_tensor/src/low_level_processors/image_to_tensor.rs @@ -483,7 +483,7 @@ mod tests { match err { ProcessError::Route(route) => { - assert_eq!(route.relationship.as_ref(), FAILURE.name) + assert_eq!(route.relationship, FAILURE.name) } other => panic!("expected route to failure, got {other:?}"), } From cb351bbc431bce6a47a80845b5a95ce7a7d9b537 Mon Sep 17 00:00:00 2001 From: Martin Zink Date: Wed, 9 Sep 2026 14:33:39 +0200 Subject: [PATCH 05/32] review changes --- .../minifi_tensor/features/environment.py | 13 ++- .../low_level_processors/classify_output.rs | 5 +- .../filter_bounding_boxes.rs | 91 +++++++++++++++++-- .../low_level_processors/image_to_tensor.rs | 14 ++- .../image_to_tensor/image_to_tensor_def.rs | 9 ++ .../src/processors/detect_object.rs | 10 +- .../src/processors/draw_bounding_box.rs | 3 +- 7 files changed, 130 insertions(+), 15 deletions(-) diff --git a/minifi_rust/extensions/minifi_tensor/features/environment.py b/minifi_rust/extensions/minifi_tensor/features/environment.py index 9211634c58..47fb5a9f61 100644 --- a/minifi_rust/extensions/minifi_tensor/features/environment.py +++ b/minifi_rust/extensions/minifi_tensor/features/environment.py @@ -29,6 +29,7 @@ ) _SSL_CONTEXT = ssl.create_default_context(cafile=certifi.where()) +_DOWNLOAD_TIMEOUT_S = 60 class RemoteAsset: @@ -43,8 +44,16 @@ def acquire(self, cache_dir: str, filename: str) -> str: os.makedirs(cache_dir, exist_ok=True) tmp = dest + ".part" print(f"[minifi_tensor tests] fetching {filename} from {self.url}") - with urllib.request.urlopen(self.url, context=_SSL_CONTEXT) as response, open(tmp, "wb") as out: - shutil.copyfileobj(response, out) + try: + with ( + urllib.request.urlopen(self.url, context=_SSL_CONTEXT, timeout=_DOWNLOAD_TIMEOUT_S) as response, + open(tmp, "wb") as out, + ): + shutil.copyfileobj(response, out) + except OSError as e: + if os.path.exists(tmp): + os.remove(tmp) + raise RuntimeError(f"failed to fetch {filename} from {self.url}: {e}") from e if not self._verify(tmp): actual = self._digest(tmp) os.remove(tmp) diff --git a/minifi_rust/extensions/minifi_tensor/src/low_level_processors/classify_output.rs b/minifi_rust/extensions/minifi_tensor/src/low_level_processors/classify_output.rs index ba4b25b99a..fd482b2993 100644 --- a/minifi_rust/extensions/minifi_tensor/src/low_level_processors/classify_output.rs +++ b/minifi_rust/extensions/minifi_tensor/src/low_level_processors/classify_output.rs @@ -182,7 +182,10 @@ impl ClassifyOutput { ), Some(output_attr) => ( None, - Some((output_attr, serde_json::to_string(&predictions).unwrap())), + Some(( + output_attr, + serde_json::to_string(&predictions).route_err_to_failure()?, + )), ), }; diff --git a/minifi_rust/extensions/minifi_tensor/src/low_level_processors/filter_bounding_boxes.rs b/minifi_rust/extensions/minifi_tensor/src/low_level_processors/filter_bounding_boxes.rs index d7b8b2a602..fa19577716 100644 --- a/minifi_rust/extensions/minifi_tensor/src/low_level_processors/filter_bounding_boxes.rs +++ b/minifi_rust/extensions/minifi_tensor/src/low_level_processors/filter_bounding_boxes.rs @@ -17,6 +17,7 @@ mod filter_bounding_boxes_def; +use crate::low_level_processors::image_to_tensor::ResizeMode; use crate::utils::bounding_box::BoundingBox; use crate::utils::dimensions::Dimensions; use crate::utils::score_activation::ScoreActivation; @@ -181,14 +182,27 @@ impl FilterBoundingBoxes { tensors: Vec, orig_dim: Dimensions, target_dim: Dimensions, + resize_mode: ResizeMode, ) -> Result, ProcessError> { let score_floats = tensor_as_f32(&tensors, self.score_output_index).route_err_to_failure()?; let box_floats = tensor_as_f32(&tensors, self.box_output_index).route_err_to_failure()?; - let scale = (target_dim.width / orig_dim.width).min(target_dim.height / orig_dim.height); - let pad_x = (target_dim.width - (orig_dim.width * scale)) / 2.0; - let pad_y = (target_dim.height - (orig_dim.height * scale)) / 2.0; + let (scale_x, scale_y, pad_x, pad_y) = match resize_mode { + ResizeMode::Letterbox => { + let scale = + (target_dim.width / orig_dim.width).min(target_dim.height / orig_dim.height); + let pad_x = (target_dim.width - (orig_dim.width * scale)) / 2.0; + let pad_y = (target_dim.height - (orig_dim.height * scale)) / 2.0; + (scale, scale, pad_x, pad_y) + } + ResizeMode::Stretch => ( + target_dim.width / orig_dim.width, + target_dim.height / orig_dim.height, + 0.0, + 0.0, + ), + }; if !box_floats.len().is_multiple_of(4) { return Err(MinifiError::custom( @@ -208,10 +222,12 @@ impl FilterBoundingBoxes { let make_box = |i: usize, class_id: usize, confidence: f32| -> BoundingBox { let (raw_x_min, raw_y_min, raw_x_max, raw_y_max) = decode_box(&box_floats, i * 4, self.box_format); - let true_x_min = (((raw_x_min * target_dim.width) - pad_x) / scale) / orig_dim.width; - let true_y_min = (((raw_y_min * target_dim.height) - pad_y) / scale) / orig_dim.height; - let true_x_max = (((raw_x_max * target_dim.width) - pad_x) / scale) / orig_dim.width; - let true_y_max = (((raw_y_max * target_dim.height) - pad_y) / scale) / orig_dim.height; + let true_x_min = (((raw_x_min * target_dim.width) - pad_x) / scale_x) / orig_dim.width; + let true_y_min = + (((raw_y_min * target_dim.height) - pad_y) / scale_y) / orig_dim.height; + let true_x_max = (((raw_x_max * target_dim.width) - pad_x) / scale_x) / orig_dim.width; + let true_y_max = + (((raw_y_max * target_dim.height) - pad_y) / scale_y) / orig_dim.height; BoundingBox { class_id, confidence, @@ -329,6 +345,15 @@ impl FilterBoundingBoxes { } } +fn resize_mode_from_attributes(context: &Context) -> ResizeMode { + context + .get_attribute("image.resize.mode") + .ok() + .flatten() + .and_then(|raw| raw.parse::().ok()) + .unwrap_or(ResizeMode::Letterbox) +} + impl FlowFileTransform for FilterBoundingBoxes { fn transform<'a, Context: GetProperty + GetAttribute + GetId, LoggerImpl: Logger>( &self, @@ -338,10 +363,11 @@ impl FlowFileTransform for FilterBoundingBoxes { ) -> Result, ProcessError> { let orig_dim = Dimensions::original_from_attributes(context).route_err_to_failure()?; let target_dim = Dimensions::target_from_attributes(context).route_err_to_failure()?; + let resize_mode = resize_mode_from_attributes(context); let tensors = deserialize_tensors(context, input_stream).route_err_to_failure()?; - self.filter(context, logger, tensors, orig_dim, target_dim) + self.filter(context, logger, tensors, orig_dim, target_dim, resize_mode) } } @@ -454,6 +480,7 @@ mod tests { vec![scores, boxes, classes], dim, dim, + ResizeMode::Letterbox, ) .expect("filter should succeed"); @@ -464,6 +491,53 @@ mod tests { assert!(!json.contains("\"class_id\":3")); } + #[test] + fn test_resize_mode_changes_coordinate_un_mapping() { + use minifi_native::{MockLogger, MockProcessContext}; + use tract::__ndarray_interop::TensorInterface; + + let run = |mode: ResizeMode| -> BoundingBox { + // One interior box (avoids clamping) in normalised Xyxy. + let scores = Tensor::from_slice::(&[1], &[0.9]).unwrap(); + let boxes = Tensor::from_slice::(&[1, 4], &[0.4, 0.4, 0.6, 0.6]).unwrap(); + let classes = Tensor::from_slice::(&[1], &[5]).unwrap(); + + let result = class_index_processor() + .filter( + &MockProcessContext::new(), + &MockLogger::new(), + vec![scores, boxes, classes], + Dimensions { + width: 200.0, + height: 100.0, + }, + Dimensions { + width: 100.0, + height: 100.0, + }, + mode, + ) + .expect("filter should succeed"); + let json = result.into_bytes().unwrap().unwrap(); + let boxes: Vec = serde_json::from_slice(&json).unwrap(); + boxes.into_iter().next().expect("one box expected") + }; + + // Stretch: scale_x = 100/200, scale_y = 100/100, no padding → identity in + // normalised space. + let stretched = run(ResizeMode::Stretch); + assert!((stretched.y_min - 0.4).abs() < 1e-5); + assert!((stretched.y_max - 0.6).abs() < 1e-5); + + // Letterbox: uniform scale 0.5, symmetric vertical pad of 25px removed → + // y expands to 0.3..0.7. x is unchanged (pad_x = 0, same scale on x). + let letterboxed = run(ResizeMode::Letterbox); + assert!((letterboxed.x_min - 0.4).abs() < 1e-5); + assert!((letterboxed.x_max - 0.6).abs() < 1e-5); + assert!((letterboxed.y_min - 0.3).abs() < 1e-5); + assert!((letterboxed.y_max - 0.7).abs() < 1e-5); + } + #[test] fn test_filter_class_index_mode_rejects_mismatched_lengths() { use minifi_native::{MockLogger, MockProcessContext}; @@ -485,6 +559,7 @@ mod tests { vec![scores, boxes, classes], dim, dim, + ResizeMode::Letterbox, ); assert!(result.is_err(), "mismatched score/box counts should error"); } diff --git a/minifi_rust/extensions/minifi_tensor/src/low_level_processors/image_to_tensor.rs b/minifi_rust/extensions/minifi_tensor/src/low_level_processors/image_to_tensor.rs index 2a747e51b7..cbe5895ceb 100644 --- a/minifi_rust/extensions/minifi_tensor/src/low_level_processors/image_to_tensor.rs +++ b/minifi_rust/extensions/minifi_tensor/src/low_level_processors/image_to_tensor.rs @@ -24,7 +24,8 @@ pub(crate) use image_to_tensor_def::{ TARGET_HEIGHT, TARGET_WIDTH, TENSOR_SHAPE_FORMAT, }; use image_to_tensor_def::{ - IMG_ORG_HEIGHT_ATTR, IMG_ORG_WIDTH_ATTR, SUCCESS, TENSOR_DTYPE_ATTR, TENSOR_SHAPE_ATTR, + IMG_ORG_HEIGHT_ATTR, IMG_ORG_WIDTH_ATTR, IMG_RESIZE_MODE_ATTR, SUCCESS, TENSOR_DTYPE_ATTR, + TENSOR_SHAPE_ATTR, }; use image_to_tensor_def::{IMG_TRG_HEIGHT_ATTR, IMG_TRG_WIDTH_ATTR, TENSORS_LEN_ATTR}; use minifi_native::macros::{ComponentIdentifier, PropertyType}; @@ -305,6 +306,10 @@ impl ImageToTensor { } } + pub fn get_resize_mode(&self) -> ResizeMode { + self.resize_mode + } + pub fn get_tensor(&self, img: image::DynamicImage) -> Result { let f32_data: Vec = self .tensor_bytes(img) @@ -344,6 +349,7 @@ impl FlowFileTransform for ImageToTensor { (&IMG_ORG_WIDTH_ATTR, orig_dim.width.to_string()), (&IMG_TRG_HEIGHT_ATTR, self.target_height.to_string()), (&IMG_TRG_WIDTH_ATTR, self.target_width.to_string()), + (&IMG_RESIZE_MODE_ATTR, self.resize_mode.to_string()), ]), ) } @@ -420,6 +426,12 @@ mod tests { .expect("Missing dtype attribute"), "F32" ); + assert_eq!( + result + .attribute("image.resize.mode") + .expect("Missing resize mode attribute"), + "Stretch" + ); let payload: Vec = result .into_bytes() diff --git a/minifi_rust/extensions/minifi_tensor/src/low_level_processors/image_to_tensor/image_to_tensor_def.rs b/minifi_rust/extensions/minifi_tensor/src/low_level_processors/image_to_tensor/image_to_tensor_def.rs index 9faa88f21e..8fcc196728 100644 --- a/minifi_rust/extensions/minifi_tensor/src/low_level_processors/image_to_tensor/image_to_tensor_def.rs +++ b/minifi_rust/extensions/minifi_tensor/src/low_level_processors/image_to_tensor/image_to_tensor_def.rs @@ -156,6 +156,14 @@ pub(super) const IMG_TRG_WIDTH_ATTR: OutputAttribute = OutputAttribute { description: "The width of the image after the resizing.", }; +pub(super) const IMG_RESIZE_MODE_ATTR: OutputAttribute = OutputAttribute { + name: "image.resize.mode", + relationships: &["success"], + description: "The resize mode ('Stretch' or 'Letterbox') applied to fit the image into the \ + target dimensions. Downstream processors such as FilterBoundingBoxes use this \ + to invert the coordinate mapping correctly.", +}; + impl ProcessorDefinition for ImageToTensor { const DESCRIPTION: &'static str = "Decodes an image from the flow file content and converts it into a normalised numeric \ tensor suitable for feeding into a downstream inference processor such as \ @@ -174,6 +182,7 @@ impl ProcessorDefinition for ImageToTensor { IMG_ORG_HEIGHT_ATTR, IMG_TRG_WIDTH_ATTR, IMG_TRG_HEIGHT_ATTR, + IMG_RESIZE_MODE_ATTR, ]; const RELATIONSHIPS: &'static [Relationship] = &[SUCCESS, FAILURE]; const PROPERTIES: &[PropertyDefinition] = property_definitions![ diff --git a/minifi_rust/extensions/minifi_tensor/src/processors/detect_object.rs b/minifi_rust/extensions/minifi_tensor/src/processors/detect_object.rs index 6a08fe7546..fbf6301cec 100644 --- a/minifi_rust/extensions/minifi_tensor/src/processors/detect_object.rs +++ b/minifi_rust/extensions/minifi_tensor/src/processors/detect_object.rs @@ -79,8 +79,14 @@ impl FlowFileTransform for DetectObject { .route_err_to_failure()?; // FilterBoundingBox - self.filter_bounding_boxes - .filter(context, logger, output_tensors, orig_dim, target_dim) + self.filter_bounding_boxes.filter( + context, + logger, + output_tensors, + orig_dim, + target_dim, + self.image_to_tensor.get_resize_mode(), + ) } } diff --git a/minifi_rust/extensions/minifi_tensor/src/processors/draw_bounding_box.rs b/minifi_rust/extensions/minifi_tensor/src/processors/draw_bounding_box.rs index e665144a99..b8b51b6200 100644 --- a/minifi_rust/extensions/minifi_tensor/src/processors/draw_bounding_box.rs +++ b/minifi_rust/extensions/minifi_tensor/src/processors/draw_bounding_box.rs @@ -26,7 +26,8 @@ pub(crate) const BOUNDING_BOXES: Property = Property::new( y_min, x_max, y_max; coordinates normalised to [0,1] against the image). Typically the \ attribute produced by an upstream DetectObject or FilterBoundingBoxes processor.", ) -.with_default("${enrichment.value}"); +.with_default("${enrichment.value}") +.supports_expression_language(); const LINE_THICKNESS: Property = Property::new( "Line thickness", From 5ed98d210f2683573990b532b8ad03c1e98baa41 Mon Sep 17 00:00:00 2001 From: Martin Zink Date: Wed, 9 Sep 2026 16:05:36 +0200 Subject: [PATCH 06/32] fix --- .github/references/ubuntu_22_04_clang_arm_manifest.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/references/ubuntu_22_04_clang_arm_manifest.json b/.github/references/ubuntu_22_04_clang_arm_manifest.json index ec0bfbdf49..5b7c779c83 100644 --- a/.github/references/ubuntu_22_04_clang_arm_manifest.json +++ b/.github/references/ubuntu_22_04_clang_arm_manifest.json @@ -13312,7 +13312,7 @@ "validator": "VALID", "required": "false", "sensitive": "false", - "expressionLanguageScope": "NONE", + "expressionLanguageScope": "FLOWFILE_ATTRIBUTES", "defaultValue": "${enrichment.value}" }, "Line color": { From 392582112b74dcecf79375f3abd7098cdc8e982b Mon Sep 17 00:00:00 2001 From: Martin Zink Date: Fri, 11 Sep 2026 12:06:06 +0200 Subject: [PATCH 07/32] fix panic on malformed tensor payload in tensor_helpers.rs --- .../minifi_tensor/src/utils/tensor_helpers.rs | 56 ++++++++++++++++++- 1 file changed, 55 insertions(+), 1 deletion(-) diff --git a/minifi_rust/extensions/minifi_tensor/src/utils/tensor_helpers.rs b/minifi_rust/extensions/minifi_tensor/src/utils/tensor_helpers.rs index 2c883be7d4..c7aa79517a 100644 --- a/minifi_rust/extensions/minifi_tensor/src/utils/tensor_helpers.rs +++ b/minifi_rust/extensions/minifi_tensor/src/utils/tensor_helpers.rs @@ -88,6 +88,11 @@ pub(crate) fn deserialize_tensors( .parse::()?; let tensor_shape = parse_tensor_shape(context, i)?; let tensor_dtype = parse_tensor_dtype(context, i)?; + if cursor + tensor_len > flow_file_contents.len() { + return Err(MinifiError::custom( + "FlowFile contents are not in sync with tensor attributes", + )); + } let tensor_data = &flow_file_contents[cursor..cursor + tensor_len]; result.push(Tensor::from_bytes( tensor_dtype, @@ -97,7 +102,13 @@ pub(crate) fn deserialize_tensors( cursor += tensor_len; } - Ok(result) + if cursor != flow_file_contents.len() { + Err(MinifiError::custom( + "FlowFile contents are not in sync with tensor attributes", + )) + } else { + Ok(result) + } } pub(crate) fn tensor_as_f32(tensors: &[Tensor], index: usize) -> Result, MinifiError> { @@ -125,6 +136,9 @@ pub(crate) fn load_as_image(input_stream: &mut dyn InputStream) -> ImageResult = vec![0.0f32; 6]; + + let bytes: Vec = floats.into_iter().flat_map(|f| f.to_le_bytes()).collect(); + let input_stream = Cursor::new(bytes); + assert_matches!( + deserialize_tensors(&context, &mut input_stream.clone()), + Err(MinifiError::MissingRequiredAttribute(msg)) if msg == "tensors.len" + ); + context.attributes.insert("tensors.len".into(), "1".into()); + assert_matches!( + deserialize_tensors(&context, &mut input_stream.clone()), + Err(MinifiError::MissingRequiredAttribute(msg)) if msg == "tensor.0.bytes" + ); + context + .attributes + .insert("tensor.0.bytes".into(), "24".into()); + assert_matches!( + deserialize_tensors(&context, &mut input_stream.clone()), + Err(MinifiError::MissingRequiredAttribute(msg)) if msg == "tensor.0.shape" + ); + context + .attributes + .insert("tensor.0.shape".into(), "2,3".into()); + assert_matches!( + deserialize_tensors(&context, &mut input_stream.clone()), + Err(MinifiError::MissingRequiredAttribute(msg)) if msg == "tensor.0.dtype" + ); + context + .attributes + .insert("tensor.0.dtype".into(), "F32".into()); + + assert_matches!( + deserialize_tensors(&context, &mut input_stream.clone()), + Ok(_) + ); + } } From 56900af92ab31b6ee3d3606e02bf529a398b323a Mon Sep 17 00:00:00 2001 From: Martin Zink Date: Fri, 11 Sep 2026 12:06:23 +0200 Subject: [PATCH 08/32] fix doc drift in descriptions --- .../features/image_to_tensor.feature | 2 +- .../extensions/minifi_tensor/minifi_tensor.md | 29 ++++++++++--------- .../image_to_tensor/image_to_tensor_def.rs | 2 +- .../invoke_tract_model.rs | 5 +++- .../invoke_tract_model_def.rs | 9 +++--- 5 files changed, 25 insertions(+), 22 deletions(-) diff --git a/minifi_rust/extensions/minifi_tensor/features/image_to_tensor.feature b/minifi_rust/extensions/minifi_tensor/features/image_to_tensor.feature index bf94b28886..a7e82d9155 100644 --- a/minifi_rust/extensions/minifi_tensor/features/image_to_tensor.feature +++ b/minifi_rust/extensions/minifi_tensor/features/image_to_tensor.feature @@ -41,7 +41,7 @@ Feature: ImageToTensor preprocesses image bytes into normalised tensors And the Minifi logs contain the following message: "key:tensor.0.dtype value:F32" in less than 1 seconds And the Minifi logs do not contain errors - Scenario: HWC layout is reflected in the tensor.shape attribute + Scenario: HWC layout is reflected in the tensor.0.shape attribute Given a host resource file "grace_hopper.jpg" is copied to the "/tmp/input/grace_hopper.jpg" path in the MiNiFi container And a GetFile processor with the "Input Directory" property set to "/tmp/input" And the "Keep Source File" property of the GetFile processor is set to "false" diff --git a/minifi_rust/extensions/minifi_tensor/minifi_tensor.md b/minifi_rust/extensions/minifi_tensor/minifi_tensor.md index 65a0081494..64f5b3b687 100644 --- a/minifi_rust/extensions/minifi_tensor/minifi_tensor.md +++ b/minifi_rust/extensions/minifi_tensor/minifi_tensor.md @@ -164,11 +164,11 @@ Decodes the image from the flow file content, draws each bounding box supplied v In the list below, the names of required properties appear in bold. Any other properties (not in bold) are considered optional. The table also indicates any default values, and whether a property supports the NiFi Expression Language. -| Name | Default Value | Allowable Values | Description | -|--------------------|---------------------|------------------|------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| -| Bounding boxes | ${enrichment.value} | | JSON array of bounding boxes to draw onto the image (fields class_id, confidence, x_min, y_min, x_max, y_max; coordinates normalised to [0,1] against the image). Typically the attribute produced by an upstream DetectObject or FilterBoundingBoxes processor. | -| Line color | [0, 255, 0] | | Outline colour as '[R, G, B]' u8 channels (0-255). | -| **Line thickness** | 5 | | Thickness in pixels of the drawn box outline. | +| Name | Default Value | Allowable Values | Description | +|--------------------|---------------------|------------------|-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| Bounding boxes | ${enrichment.value} | | JSON array of bounding boxes to draw onto the image (fields class_id, confidence, x_min, y_min, x_max, y_max; coordinates normalised to [0,1] against the image). Typically the attribute produced by an upstream DetectObject or FilterBoundingBoxes processor.
**Supports Expression Language: true** | +| Line color | [0, 255, 0] | | Outline colour as '[R, G, B]' u8 channels (0-255). | +| **Line thickness** | 5 | | Thickness in pixels of the drawn box outline. | ### Relationships @@ -247,15 +247,16 @@ In the list below, the names of required properties appear in bold. Any other pr ### Output Attributes -| Attribute | Relationship | Description | -|-----------------------|--------------|---------------------------------------------------------------------------------------------------------------------------------------------------------| -| image.original.height | success | The height of the original image before the resizing. | -| image.original.width | success | The width of the original image before the resizing. | -| image.target.height | success | The height of the image after the resizing. | -| image.target.width | success | The width of the image after the resizing. | -| tensor.0.dtype | success | Element type of the values in the output tensor. Currently always 'F32'. | -| tensor.0.shape | success | Comma-separated dimensions of the output tensor in the chosen layout, always including a leading batch dimension of 1 (e.g. '1,3,224,224' for RGB CHW). | -| tensors.len | success | Number of tensors in the output FlowFile. Currently always '1' | +| Attribute | Relationship | Description | +|-----------------------|--------------|--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| image.original.height | success | The height of the original image before the resizing. | +| image.original.width | success | The width of the original image before the resizing. | +| image.resize.mode | success | The resize mode ('Stretch' or 'Letterbox') applied to fit the image into the target dimensions. Downstream processors such as FilterBoundingBoxes use this to invert the coordinate mapping correctly. | +| image.target.height | success | The height of the image after the resizing. | +| image.target.width | success | The width of the image after the resizing. | +| tensor.0.dtype | success | Element type of the values in the output tensor. Currently always 'F32'. | +| tensor.0.shape | success | Comma-separated dimensions of the output tensor in the chosen layout, always including a leading batch dimension of 1 (e.g. '1,3,224,224' for RGB CHW). | +| tensors.len | success | Number of tensors in the output FlowFile. Currently always '1' | ## InvokeTractModel diff --git a/minifi_rust/extensions/minifi_tensor/src/low_level_processors/image_to_tensor/image_to_tensor_def.rs b/minifi_rust/extensions/minifi_tensor/src/low_level_processors/image_to_tensor/image_to_tensor_def.rs index 8fcc196728..aa35564905 100644 --- a/minifi_rust/extensions/minifi_tensor/src/low_level_processors/image_to_tensor/image_to_tensor_def.rs +++ b/minifi_rust/extensions/minifi_tensor/src/low_level_processors/image_to_tensor/image_to_tensor_def.rs @@ -169,7 +169,7 @@ impl ProcessorDefinition for ImageToTensor { tensor suitable for feeding into a downstream inference processor such as \ InvokeTractModel. Supports RGB / BGR / Grayscale, CHW / HWC layouts, stretch or \ letterbox resizing, and scalar or per-channel mean/std normalisation. The output payload \ - is the raw little-endian f32 tensor; the 'tensor.shape' and 'tensor.dtype' attributes \ + is the raw little-endian f32 tensor; the 'tensors.len', 'tensor.{i}.shape' and 'tensor.{i}.dtype' attributes \ describe its layout."; const INPUT_REQUIREMENT: ProcessorInputRequirement = ProcessorInputRequirement::Required; const SUPPORTS_DYNAMIC_PROPERTIES: bool = false; diff --git a/minifi_rust/extensions/minifi_tensor/src/low_level_processors/invoke_tract_model.rs b/minifi_rust/extensions/minifi_tensor/src/low_level_processors/invoke_tract_model.rs index 21e5742327..a751213b5e 100644 --- a/minifi_rust/extensions/minifi_tensor/src/low_level_processors/invoke_tract_model.rs +++ b/minifi_rust/extensions/minifi_tensor/src/low_level_processors/invoke_tract_model.rs @@ -176,7 +176,10 @@ mod tests { fn test_get_shape_missing_attribute_fails() { let context = MockProcessContext::new(); // Empty attributes let result = InvokeTractModel::get_shape_from_attribute(&context); - assert!(result.is_err(), "Should fail when tensor.shape is missing"); + assert!( + result.is_err(), + "Should fail when tensor.0.shape is missing" + ); } #[test] diff --git a/minifi_rust/extensions/minifi_tensor/src/low_level_processors/invoke_tract_model/invoke_tract_model_def.rs b/minifi_rust/extensions/minifi_tensor/src/low_level_processors/invoke_tract_model/invoke_tract_model_def.rs index 0fc09edb02..bc1a2bb66e 100644 --- a/minifi_rust/extensions/minifi_tensor/src/low_level_processors/invoke_tract_model/invoke_tract_model_def.rs +++ b/minifi_rust/extensions/minifi_tensor/src/low_level_processors/invoke_tract_model/invoke_tract_model_def.rs @@ -36,8 +36,8 @@ pub(super) const SUCCESS: Relationship = Relationship { pub(super) const FAILURE: Relationship = Relationship { name: "failure", - description: "The input tensor could not be built (missing/invalid tensor.shape, unsupported \ - tensor.dtype, malformed payload) or the model failed to run.", + description: "The input tensor could not be built (missing/invalid tensor.0.shape, unsupported \ + tensor.0.dtype, malformed payload) or the model failed to run.", }; const OUTPUT_COUNT_ATTR: OutputAttribute = OutputAttribute { @@ -69,11 +69,10 @@ const OUTPUT_DTYPE_ATTR: OutputAttribute = OutputAttribute { impl ProcessorDefinition for super::InvokeTractModel { const DESCRIPTION: &'static str = "Runs a single inference against the compiled model owned by the referenced \ TractModelService. Reads the input tensor from the flow file content plus the \ - 'tensor.shape' and (optionally) 'tensor.dtype' attributes produced by an upstream \ + 'tensor.0.shape' and (optionally) 'tensor.0.dtype' attributes produced by an upstream \ processor such as ImageToTensor. The flow file's new content is every output tensor's \ raw bytes concatenated in model order; per-tensor shape, byte length, and dtype are \ - written to attributes. Only a single input tensor and only f32 input dtype are \ - supported in this pass."; + written to attributes."; const INPUT_REQUIREMENT: ProcessorInputRequirement = ProcessorInputRequirement::Required; const SUPPORTS_DYNAMIC_PROPERTIES: bool = false; const SUPPORTS_DYNAMIC_RELATIONSHIPS: bool = false; From 02b0095bf5819daa542901a6e8518e8097606f1b Mon Sep 17 00:00:00 2001 From: Martin Zink Date: Fri, 11 Sep 2026 12:12:37 +0200 Subject: [PATCH 09/32] fix 0/0 in IoU --- .../extensions/minifi_tensor/src/utils/bounding_box.rs | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/minifi_rust/extensions/minifi_tensor/src/utils/bounding_box.rs b/minifi_rust/extensions/minifi_tensor/src/utils/bounding_box.rs index 8e8c00c246..44b0fb7cb3 100644 --- a/minifi_rust/extensions/minifi_tensor/src/utils/bounding_box.rs +++ b/minifi_rust/extensions/minifi_tensor/src/utils/bounding_box.rs @@ -55,7 +55,13 @@ impl BoundingBox { let box1_area = (box1.x_max - box1.x_min) * (box1.y_max - box1.y_min); let box2_area = (box2.x_max - box2.x_min) * (box2.y_max - box2.y_min); - intersection_area / (box1_area + box2_area - intersection_area) + let divisor = box1_area + box2_area - intersection_area; + + if intersection_area == 0f32 || divisor == 0f32 { + return 0f32; + } + + intersection_area / divisor } pub(crate) fn apply_non_maximum_suppression( From a613332378103c4ff7cf23a774ba0dc228d346d6 Mon Sep 17 00:00:00 2001 From: Martin Zink Date: Fri, 11 Sep 2026 12:14:00 +0200 Subject: [PATCH 10/32] classify_image.rs Failure description fix --- .../extensions/minifi_tensor/src/processors/classify_image.rs | 4 ++-- .../{classify_object_def.rs => classify_image_def.rs} | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) rename minifi_rust/extensions/minifi_tensor/src/processors/classify_image/{classify_object_def.rs => classify_image_def.rs} (99%) diff --git a/minifi_rust/extensions/minifi_tensor/src/processors/classify_image.rs b/minifi_rust/extensions/minifi_tensor/src/processors/classify_image.rs index a9cafa94ba..2b61a428e0 100644 --- a/minifi_rust/extensions/minifi_tensor/src/processors/classify_image.rs +++ b/minifi_rust/extensions/minifi_tensor/src/processors/classify_image.rs @@ -15,12 +15,12 @@ // specific language governing permissions and limitations // under the License. -mod classify_object_def; +mod classify_image_def; use crate::low_level_processors::classify_output::ClassifyOutput; use crate::low_level_processors::image_to_tensor::ImageToTensor; use crate::utils::tensor_helpers::load_as_image; -use classify_object_def::TRACT_MODEL_SERVICE; +use classify_image_def::TRACT_MODEL_SERVICE; use minifi_native::macros::ComponentIdentifier; use minifi_native::{ FlowFileTransform, GetAttribute, GetControllerService, GetId, GetProperty, InputStream, Logger, diff --git a/minifi_rust/extensions/minifi_tensor/src/processors/classify_image/classify_object_def.rs b/minifi_rust/extensions/minifi_tensor/src/processors/classify_image/classify_image_def.rs similarity index 99% rename from minifi_rust/extensions/minifi_tensor/src/processors/classify_image/classify_object_def.rs rename to minifi_rust/extensions/minifi_tensor/src/processors/classify_image/classify_image_def.rs index 88b35a72cd..6a4791c950 100644 --- a/minifi_rust/extensions/minifi_tensor/src/processors/classify_image/classify_object_def.rs +++ b/minifi_rust/extensions/minifi_tensor/src/processors/classify_image/classify_image_def.rs @@ -40,7 +40,7 @@ pub(super) const SUCCESS: Relationship = Relationship { pub(super) const FAILURE: Relationship = Relationship { name: "failure", description: "The image could not be decoded, the input tensor could not be built, the model \ - failed to run, or the model outputs could not be interpreted as scores + boxes.", + failed to run, or the model outputs could not be interpreted as classification.", }; impl ProcessorDefinition for ClassifyImage { From 7fa38630e2920ba267a5a5e18ca74bf6f946057b Mon Sep 17 00:00:00 2001 From: Martin Zink Date: Fri, 11 Sep 2026 14:45:12 +0200 Subject: [PATCH 11/32] remove code duplication from InvokeTract model and add result_via_output_attribute to filter_bounding_boxes.rs --- .../src/processors/asciify_german.rs | 7 +- .../src/processors/get_file.rs | 4 +- .../filter_bounding_boxes.rs | 64 +++---- .../invoke_tract_model.rs | 156 +----------------- minifi_rust/minifi_native/src/api/errors.rs | 10 ++ .../processor_wrappers/flow_file_transform.rs | 5 +- 6 files changed, 63 insertions(+), 183 deletions(-) diff --git a/minifi_rust/extensions/minifi_rs_playground/src/processors/asciify_german.rs b/minifi_rust/extensions/minifi_rs_playground/src/processors/asciify_german.rs index 6d67c7d051..37f70e4860 100644 --- a/minifi_rust/extensions/minifi_rs_playground/src/processors/asciify_german.rs +++ b/minifi_rust/extensions/minifi_rs_playground/src/processors/asciify_german.rs @@ -21,7 +21,7 @@ use crate::processors::asciify_german::relationships::FAILURE; use minifi_native::macros::ComponentIdentifier; use minifi_native::{ FlowFileStreamTransform, GetProperty, InputStream, Logger, MinifiError, OutputStream, - ProcessError, RouteErrorExt, Schedule, TransformStreamResult, + ProcessError, Schedule, TransformStreamResult, }; mod relationships; @@ -56,8 +56,9 @@ impl FlowFileStreamTransform for AsciifyGerman { 0xC3 => { let mut next = [0u8; 1]; if input_stream.read(&mut next)? == 0 { - Err(MinifiError::custom("Truncated multi-byte sequence at EOF")) - .route_err_to_failure()? + return Err(ProcessError::route_to_failure( + "Truncated multi-byte sequence at EOF", + )); } match next[0] { 0xA4 => output_stream.write_all(b"ae")?, // ä diff --git a/minifi_rust/extensions/minifi_rs_playground/src/processors/get_file.rs b/minifi_rust/extensions/minifi_rs_playground/src/processors/get_file.rs index 7ec9f41300..b1f77ed267 100644 --- a/minifi_rust/extensions/minifi_rs_playground/src/processors/get_file.rs +++ b/minifi_rust/extensions/minifi_rs_playground/src/processors/get_file.rs @@ -27,7 +27,7 @@ use crate::processors::get_file::properties::{ use minifi_native::macros::ComponentIdentifier; use minifi_native::{ GetProperty, IoState, Logger, MinifiError, OnTriggerResult, ProcessContext, ProcessError, - ProcessSession, Schedule, Trigger, debug, info, trace, warn, + ProcessSession, Schedule, Trigger, debug, trace, warn, }; use std::collections::VecDeque; use std::error; @@ -179,7 +179,7 @@ impl GetFileRs { logger: &L, path: &Path, ) -> Result<(), MinifiError> { - info!(logger, "GetFile process {:?}", path); + trace!(logger, "GetFile process {:?}", path); let mut ff = session .create() .expect("Successful FlowFile creation is expected"); diff --git a/minifi_rust/extensions/minifi_tensor/src/low_level_processors/filter_bounding_boxes.rs b/minifi_rust/extensions/minifi_tensor/src/low_level_processors/filter_bounding_boxes.rs index fa19577716..559efdb621 100644 --- a/minifi_rust/extensions/minifi_tensor/src/low_level_processors/filter_bounding_boxes.rs +++ b/minifi_rust/extensions/minifi_tensor/src/low_level_processors/filter_bounding_boxes.rs @@ -30,7 +30,7 @@ pub(crate) use filter_bounding_boxes_def::{ use minifi_native::macros::{ComponentIdentifier, PropertyType}; use minifi_native::{ Content, FlowFileTransform, GetAttribute, GetId, GetProperty, InputStream, Logger, MinifiError, - ProcessError, RouteErrorExt, Schedule, TransformedFlowFile, debug, + ProcessError, RouteErrorExt, Schedule, TransformedFlowFile, debug, trace, }; use strum_macros::{Display, EnumString, IntoStaticStr, VariantNames}; use tract::Tensor; @@ -175,6 +175,33 @@ impl Schedule for FilterBoundingBoxes { } impl FilterBoundingBoxes { + fn result_via_output_attribute<'a, Context: GetProperty>( + &self, + context: &Context, + filtered_boxes: Vec, + ) -> Result, MinifiError> { + let output_attr = context.get_property(&OUTPUT_ATTRIBUTE_NAME)?; + let content = if output_attr.is_some() { + None + } else { + Some(Content::Buffer( + serde_json::to_vec(&filtered_boxes).map_err(MinifiError::other)?, + )) + }; + + let mut transformed = TransformedFlowFile::new(&SUCCESS, content) + .with_attribute("object.count", filtered_boxes.len().to_string()); + if let Some(attr) = output_attr { + transformed = transformed.with_attribute( + attr, + serde_json::to_string(&filtered_boxes).map_err(MinifiError::other)?, + ) + } else { + transformed = transformed.with_attribute("mime.type", "application/json"); + } + Ok(transformed) + } + pub(crate) fn filter<'a, Context: GetProperty, LoggerImpl: Logger>( &self, context: &Context, @@ -213,10 +240,9 @@ impl FilterBoundingBoxes { let num_boxes = box_floats.len() / 4; if num_boxes == 0 { debug!(logger, "No boxes to filter; emitting empty array"); - return Ok(TransformedFlowFile::new(&SUCCESS, None) - .with_content(b"[]".to_vec().into()) - .with_attribute("object.count", "0") - .with_attribute("mime.type", "application/json")); + return self + .result_via_output_attribute(context, vec![]) + .route_err_to_failure(); } let make_box = |i: usize, class_id: usize, confidence: f32| -> BoundingBox { @@ -254,7 +280,7 @@ impl FilterBoundingBoxes { )) .into()); } - debug!( + trace!( logger, "Filtering {} boxes with separate class-id tensor (activation={:?}, \ box_format={:?})...", @@ -292,7 +318,7 @@ impl FilterBoundingBoxes { .into()); } let num_classes = score_floats.len() / num_boxes; - debug!( + trace!( logger, "Filtering {} boxes across {} potential classes (activation={:?}, \ box_format={:?})...", @@ -312,7 +338,7 @@ impl FilterBoundingBoxes { } } - debug!( + trace!( logger, "Found {} boxes exceeding the {} threshold.", valid_boxes.len(), @@ -322,26 +348,8 @@ impl FilterBoundingBoxes { let filtered_boxes = BoundingBox::apply_non_maximum_suppression(valid_boxes, self.iou_threshold); - let json_output = serde_json::to_vec(&filtered_boxes).route_err_to_failure()?; - - let (content, extra_attribute) = match context.get_property(&OUTPUT_ATTRIBUTE_NAME)? { - None => (Some(Content::Buffer(json_output)), None), - Some(output_attr) => ( - None, - Some(( - output_attr, - serde_json::to_string(&filtered_boxes).route_err_to_failure()?, - )), - ), - }; - - let mut transformed = TransformedFlowFile::new(&SUCCESS, content) - .with_attribute("object.count", filtered_boxes.len().to_string()) - .with_attribute("mime.type", "application/json"); - if let Some((key, value)) = extra_attribute { - transformed = transformed.with_attribute(key, value); - } - Ok(transformed) + self.result_via_output_attribute(context, filtered_boxes) + .route_err_to_failure() } } diff --git a/minifi_rust/extensions/minifi_tensor/src/low_level_processors/invoke_tract_model.rs b/minifi_rust/extensions/minifi_tensor/src/low_level_processors/invoke_tract_model.rs index a751213b5e..5485c1b644 100644 --- a/minifi_rust/extensions/minifi_tensor/src/low_level_processors/invoke_tract_model.rs +++ b/minifi_rust/extensions/minifi_tensor/src/low_level_processors/invoke_tract_model.rs @@ -15,7 +15,7 @@ // specific language governing permissions and limitations // under the License. -use crate::utils::tensor_helpers::MinifiDatumType; +use crate::utils::tensor_helpers::deserialize_tensors; pub(crate) use invoke_tract_model_def::TRACT_MODEL_SERVICE; use invoke_tract_model_def::*; use minifi_native::macros::ComponentIdentifier; @@ -23,7 +23,6 @@ use minifi_native::{ FlowFileTransform, GetAttribute, GetControllerService, GetId, GetProperty, InputStream, Logger, MinifiError, ProcessError, RouteErrorExt, Schedule, TransformedFlowFile, }; -use std::error::Error; use tract::__ndarray_interop::TensorInterface; use tract::Tensor; tract::impl_ndarray_interop!(); @@ -45,65 +44,6 @@ impl Schedule for InvokeTractModel { } } -impl InvokeTractModel { - fn get_payload_as_f32_array( - input_stream: &mut dyn InputStream, - ) -> Result, Box> { - let mut raw_bytes = Vec::new(); - input_stream.read_to_end(&mut raw_bytes)?; - if raw_bytes.len() % 4 != 0 { - return Err(MinifiError::custom("Input bytes length is not a multiple of 4").into()); - } - let mut f32_data = Vec::with_capacity(raw_bytes.len() / 4); - for chunk in raw_bytes.as_chunks::<4>().0.iter() { - let val = f32::from_le_bytes(*chunk); - f32_data.push(val); - } - Ok(f32_data) - } - - fn get_shape_from_attribute( - context: &Context, - ) -> Result, Box> { - let shape_str = context - .get_attribute("tensor.0.shape") - .and_then(|opt| opt.ok_or(MinifiError::custom("Missing tensor.0.shape")))?; - shape_str - .split(',') - .map(|s| s.trim().parse::()) - .collect::, _>>() - .map_err(|e| e.into()) - } - - fn check_input_dtype(context: &Context) -> Result<(), Box> { - match context.get_attribute("tensor.0.dtype")? { - None => Ok(()), - Some(dtype) => dtype.parse::().map(|_| ()).map_err(|_| { - MinifiError::custom(format!( - "Unsupported tensor.0.dtype '{}' (only '{}' is supported)", - dtype, - MinifiDatumType::F32 - )) - .into() - }), - } - } - - fn get_input_tensor( - context: &Context, - input_stream: &mut dyn InputStream, - ) -> Result> { - Self::check_input_dtype(context)?; - let shape = Self::get_shape_from_attribute(context)?; - let f32_data = Self::get_payload_as_f32_array(input_stream)?; - - let array = ndarray::Array::from_shape_vec(shape, f32_data) - .map_err(|e| MinifiError::custom(format!("Failed to create array: {}", e)))?; - - Ok(array.tract()?) - } -} - impl FlowFileTransform for InvokeTractModel { fn transform< 'a, @@ -117,12 +57,14 @@ impl FlowFileTransform for InvokeTractModel { ) -> Result, ProcessError> { let controller_service = context.get_controller_service(&TRACT_MODEL_SERVICE)?; - let input_tensor: Tensor = Self::get_input_tensor(context, input_stream) - .map_err(|e| MinifiError::custom(e.to_string())) - .route_err_to_failure()?; + let input_tensors: Vec = + deserialize_tensors(context, input_stream).route_err_to_failure()?; + if input_tensors.len() != 1 { + return Err(ProcessError::route_to_failure("Invalid input")); + }; let output_tensors = controller_service - .run_inference(vec![input_tensor]) + .run_inference(input_tensors) .route_err_to_failure()?; let mut output_bytes = Vec::new(); let mut transformed = TransformedFlowFile::new(&SUCCESS, None) @@ -155,91 +97,11 @@ impl FlowFileTransform for InvokeTractModel { #[cfg(test)] mod tests { - use super::*; + use crate::InvokeTractModel; + use minifi_native::FlowFileTransform; use minifi_native::{MockLogger, MockProcessContext}; use std::io::Cursor; - #[test] - fn test_get_shape_parses_correctly() { - let mut context = MockProcessContext::new(); - // Insert a valid shape with spaces - context - .attributes - .insert("tensor.0.shape".to_string(), "1, 3, 224, 224".to_string()); - - let shape = - InvokeTractModel::get_shape_from_attribute(&context).expect("Should parse valid shape"); - assert_eq!(shape, vec![1, 3, 224, 224]); - } - - #[test] - fn test_get_shape_missing_attribute_fails() { - let context = MockProcessContext::new(); // Empty attributes - let result = InvokeTractModel::get_shape_from_attribute(&context); - assert!( - result.is_err(), - "Should fail when tensor.0.shape is missing" - ); - } - - #[test] - fn test_get_shape_invalid_data_fails() { - let mut context = MockProcessContext::new(); - context - .attributes - .insert("tensor.0.shape".to_string(), "1, apple, 224".to_string()); - - let result = InvokeTractModel::get_shape_from_attribute(&context); - assert!( - result.is_err(), - "Should fail when shape contains non-integers" - ); - } - - #[test] - fn test_get_payload_as_f32_array_success() { - let expected_floats: [f32; 3] = [1.0, 2.5, -3.5]; - let mut bytes = Vec::new(); - for f in expected_floats.iter() { - bytes.extend_from_slice(&f.to_le_bytes()); - } - let mut stream = Cursor::new(bytes); - let result = InvokeTractModel::get_payload_as_f32_array(&mut stream) - .expect("Should parse valid byte stream"); - assert_eq!(result, vec![1.0, 2.5, -3.5]); - } - - #[test] - fn test_get_payload_invalid_length_fails() { - let bytes = vec![0u8, 1, 2, 3, 4]; - let mut stream = Cursor::new(bytes); - - let result = InvokeTractModel::get_payload_as_f32_array(&mut stream); - assert!( - result.is_err(), - "Should error out if byte length is not a multiple of 4" - ); - } - - #[test] - fn test_check_input_dtype_accepts_missing_and_f32() { - let mut context = MockProcessContext::new(); - assert!(InvokeTractModel::check_input_dtype(&context).is_ok()); - context - .attributes - .insert("tensor.0.dtype".to_string(), "F32".to_string()); - assert!(InvokeTractModel::check_input_dtype(&context).is_ok()); - } - - #[test] - fn test_check_input_dtype_rejects_other() { - let mut context = MockProcessContext::new(); - context - .attributes - .insert("tensor.0.dtype".to_string(), "u8".to_string()); - assert!(InvokeTractModel::check_input_dtype(&context).is_err()); - } - #[test] fn test_transform_missing_controller_service_throws_error() { let processor = InvokeTractModel {}; diff --git a/minifi_rust/minifi_native/src/api/errors.rs b/minifi_rust/minifi_native/src/api/errors.rs index 16c652a7c5..938175bbdc 100644 --- a/minifi_rust/minifi_native/src/api/errors.rs +++ b/minifi_rust/minifi_native/src/api/errors.rs @@ -61,6 +61,16 @@ pub enum ProcessError { Fatal(MinifiError), } +impl ProcessError { + pub fn route_to_failure>>(reason: S) -> Self { + ProcessError::Route(RouteError { + relationship: "failure".into(), + source: Box::new(MinifiError::custom(reason)), + log_level: LogLevel::Warn, + }) + } +} + impl From for ProcessError { fn from(err: RouteError) -> Self { ProcessError::Route(err) diff --git a/minifi_rust/minifi_native/src/api/processor_wrappers/flow_file_transform.rs b/minifi_rust/minifi_native/src/api/processor_wrappers/flow_file_transform.rs index 2f470125ee..ba88a7c3aa 100644 --- a/minifi_rust/minifi_native/src/api/processor_wrappers/flow_file_transform.rs +++ b/minifi_rust/minifi_native/src/api/processor_wrappers/flow_file_transform.rs @@ -24,10 +24,9 @@ use crate::api::raw_processor::{MultiThreadedTrigger, SingleThreadedTrigger}; use crate::{ GetAttribute, LogLevel, Logger, MinifiError, MultiThreaded, OnTriggerResult, ProcessContext, ProcessError, ProcessSession, Relationship, Schedule, SingleThreaded, impl_with_attributes, - info, }; -use minifi_native::InputStream; +use minifi_native::{InputStream, trace}; use std::borrow::Cow; pub type FlowFileAttribute = (Cow<'static, str>, Cow<'static, str>); @@ -159,7 +158,7 @@ where } }; - info!(logger, "{:?}", transformed); + trace!(logger, "{:?}", transformed); match transformed.new_content { None => {} Some(Content::Buffer(buffer)) => { From 2e23dd832ed6ed8614aac1f8d0f176e435a56ff3 Mon Sep 17 00:00:00 2001 From: Martin Zink Date: Fri, 11 Sep 2026 15:06:45 +0200 Subject: [PATCH 12/32] rebase fix --- minifi_rust/minifi_native/src/api/errors.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/minifi_rust/minifi_native/src/api/errors.rs b/minifi_rust/minifi_native/src/api/errors.rs index 938175bbdc..d7f80bc132 100644 --- a/minifi_rust/minifi_native/src/api/errors.rs +++ b/minifi_rust/minifi_native/src/api/errors.rs @@ -64,7 +64,7 @@ pub enum ProcessError { impl ProcessError { pub fn route_to_failure>>(reason: S) -> Self { ProcessError::Route(RouteError { - relationship: "failure".into(), + relationship: "failure", source: Box::new(MinifiError::custom(reason)), log_level: LogLevel::Warn, }) From 22afa6f0879f0ec0b94a7240bfb8411d74405d74 Mon Sep 17 00:00:00 2001 From: Martin Zink Date: Fri, 11 Sep 2026 15:24:16 +0200 Subject: [PATCH 13/32] Cargo.toml doesnt need explicit all features --- minifi_rust/extensions/minifi_tensor/Cargo.toml | 1 - 1 file changed, 1 deletion(-) diff --git a/minifi_rust/extensions/minifi_tensor/Cargo.toml b/minifi_rust/extensions/minifi_tensor/Cargo.toml index 6c48a961d3..0169229df9 100644 --- a/minifi_rust/extensions/minifi_tensor/Cargo.toml +++ b/minifi_rust/extensions/minifi_tensor/Cargo.toml @@ -8,7 +8,6 @@ crate-type = ["cdylib"] [features] low-level-processors = [] -all = ["low-level-processors"] [dependencies] minifi_native = { path = "../../minifi_native" } From fedbf16a93a32a950998ca4f1cbd7dc79467bf4d Mon Sep 17 00:00:00 2001 From: Martin Zink Date: Mon, 14 Sep 2026 11:40:33 +0200 Subject: [PATCH 14/32] fix image_to_tensor.rs --- .github/references/ubuntu_22_04_clang_arm_manifest.json | 2 +- .../src/low_level_processors/image_to_tensor.rs | 3 +++ .../image_to_tensor/image_to_tensor_def.rs | 6 ++++++ 3 files changed, 10 insertions(+), 1 deletion(-) diff --git a/.github/references/ubuntu_22_04_clang_arm_manifest.json b/.github/references/ubuntu_22_04_clang_arm_manifest.json index 5b7c779c83..6ba6899d26 100644 --- a/.github/references/ubuntu_22_04_clang_arm_manifest.json +++ b/.github/references/ubuntu_22_04_clang_arm_manifest.json @@ -13013,7 +13013,7 @@ "supportedRelationships": [ { "name": "failure", - "description": "The image could not be decoded, the input tensor could not be built, the model failed to run, or the model outputs could not be interpreted as scores + boxes." + "description": "The image could not be decoded, the input tensor could not be built, the model failed to run, or the model outputs could not be interpreted as classification." }, { "name": "success", diff --git a/minifi_rust/extensions/minifi_tensor/src/low_level_processors/image_to_tensor.rs b/minifi_rust/extensions/minifi_tensor/src/low_level_processors/image_to_tensor.rs index cbe5895ceb..2c92459328 100644 --- a/minifi_rust/extensions/minifi_tensor/src/low_level_processors/image_to_tensor.rs +++ b/minifi_rust/extensions/minifi_tensor/src/low_level_processors/image_to_tensor.rs @@ -16,6 +16,7 @@ // under the License. pub(crate) mod image_to_tensor_def; +use crate::low_level_processors::image_to_tensor::image_to_tensor_def::TENSOR_BYTES_ATTR; use crate::utils::dimensions::Dimensions; use crate::utils::per_channel_f32::PerChannelF32; use crate::utils::tensor_helpers::{MinifiDatumType, load_as_image}; @@ -339,11 +340,13 @@ impl FlowFileTransform for ImageToTensor { let orig_dim = Dimensions::from_image(&img); let tensor_bytes = self.tensor_bytes(img); + let tensor_bytes_len = tensor_bytes.len(); Ok( TransformedFlowFile::new(&SUCCESS, Some(tensor_bytes.into())).with_attributes([ (&TENSORS_LEN_ATTR, "1".to_string()), (&TENSOR_SHAPE_ATTR, self.get_shape_str()), + (&TENSOR_BYTES_ATTR, tensor_bytes_len.to_string()), (&TENSOR_DTYPE_ATTR, MinifiDatumType::F32.to_string()), (&IMG_ORG_HEIGHT_ATTR, orig_dim.height.to_string()), (&IMG_ORG_WIDTH_ATTR, orig_dim.width.to_string()), diff --git a/minifi_rust/extensions/minifi_tensor/src/low_level_processors/image_to_tensor/image_to_tensor_def.rs b/minifi_rust/extensions/minifi_tensor/src/low_level_processors/image_to_tensor/image_to_tensor_def.rs index aa35564905..186f001f15 100644 --- a/minifi_rust/extensions/minifi_tensor/src/low_level_processors/image_to_tensor/image_to_tensor_def.rs +++ b/minifi_rust/extensions/minifi_tensor/src/low_level_processors/image_to_tensor/image_to_tensor_def.rs @@ -119,6 +119,12 @@ pub(super) const TENSORS_LEN_ATTR: OutputAttribute = OutputAttribute { description: "Number of tensors in the output FlowFile. Currently always '1'", }; +pub(super) const TENSOR_BYTES_ATTR: OutputAttribute = OutputAttribute { + name: "tensor.0.bytes", + relationships: &["success"], + description: "Byte length of output tensor.", +}; + pub(super) const TENSOR_SHAPE_ATTR: OutputAttribute = OutputAttribute { name: "tensor.0.shape", relationships: &["success"], From 76e7278695f7a5c15fc31b4705b349ded6e97957 Mon Sep 17 00:00:00 2001 From: Martin Zink Date: Mon, 14 Sep 2026 13:17:11 +0200 Subject: [PATCH 15/32] respect original image format in draw_bounding_box.rs --- .../minifi_tensor/src/processors/draw_bounding_box.rs | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/minifi_rust/extensions/minifi_tensor/src/processors/draw_bounding_box.rs b/minifi_rust/extensions/minifi_tensor/src/processors/draw_bounding_box.rs index b8b51b6200..baef343529 100644 --- a/minifi_rust/extensions/minifi_tensor/src/processors/draw_bounding_box.rs +++ b/minifi_rust/extensions/minifi_tensor/src/processors/draw_bounding_box.rs @@ -108,7 +108,10 @@ impl FlowFileTransform for DrawBoundingBox { let mut image_bytes = Vec::new(); input_stream.read_to_end(&mut image_bytes)?; - let mut img = load_from_memory(&image_bytes) + let format = image::guess_format(&image_bytes) + .route_err_to_failure()?; + + let mut img = image::load_from_memory_with_format(&image_bytes, format) .map(|dyn_img| dyn_img.to_rgb8()) .route_err_to_failure()?; @@ -117,7 +120,7 @@ impl FlowFileTransform for DrawBoundingBox { .for_each(|bbox| bbox.draw_onto(&mut img, line_thickness, line_color)); let mut output_bytes = Vec::new(); - img.write_to(&mut Cursor::new(&mut output_bytes), ImageFormat::Png) + img.write_to(&mut Cursor::new(&mut output_bytes), format) .route_err_to_failure()?; Ok(TransformedFlowFile::new( From a35d641d24498f34fd6ea34a2ce38371c1d44935 Mon Sep 17 00:00:00 2001 From: Martin Zink Date: Mon, 14 Sep 2026 15:53:47 +0200 Subject: [PATCH 16/32] rebase fix --- .../minifi_tensor/src/processors/draw_bounding_box.rs | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/minifi_rust/extensions/minifi_tensor/src/processors/draw_bounding_box.rs b/minifi_rust/extensions/minifi_tensor/src/processors/draw_bounding_box.rs index baef343529..3512f1713e 100644 --- a/minifi_rust/extensions/minifi_tensor/src/processors/draw_bounding_box.rs +++ b/minifi_rust/extensions/minifi_tensor/src/processors/draw_bounding_box.rs @@ -1,5 +1,5 @@ use crate::utils::bounding_box::{BoundingBox, BoundingBoxes}; -use image::{ImageFormat, Rgb, load_from_memory}; +use image::Rgb; use minifi_native::macros::ComponentIdentifier; use minifi_native::{ FlowFileTransform, GetAttribute, GetControllerService, GetId, GetProperty, InputStream, Logger, @@ -108,8 +108,7 @@ impl FlowFileTransform for DrawBoundingBox { let mut image_bytes = Vec::new(); input_stream.read_to_end(&mut image_bytes)?; - let format = image::guess_format(&image_bytes) - .route_err_to_failure()?; + let format = image::guess_format(&image_bytes).route_err_to_failure()?; let mut img = image::load_from_memory_with_format(&image_bytes, format) .map(|dyn_img| dyn_img.to_rgb8()) From d6e39f236a013ecf9712dfb992a9227a5a3afad2 Mon Sep 17 00:00:00 2001 From: Martin Zink Date: Wed, 23 Sep 2026 17:20:25 +0200 Subject: [PATCH 17/32] fixing opus 5's findings -fixed math mismatch between letterbox into/from -added passwords \n delimited instead of password -fixed possible (but inlikely) overflow -added output_attrs to classify_image --- .../extensions/minifi_tensor/minifi_tensor.md | 14 +-- .../low_level_processors/classify_output.rs | 99 ++++++++++++----- .../classify_output/classify_output_def.rs | 16 +-- .../filter_bounding_boxes.rs | 101 +++++++++++------- .../low_level_processors/image_to_tensor.rs | 65 ++++++++--- .../src/processors/classify_image.rs | 5 +- .../classify_image/classify_image_def.rs | 3 +- .../minifi_tensor/src/utils/dimensions.rs | 79 ++++++++++++++ .../src/utils/score_activation.rs | 58 ++++++++++ .../minifi_tensor/src/utils/tensor_helpers.rs | 46 ++++++-- 10 files changed, 382 insertions(+), 104 deletions(-) diff --git a/minifi_rust/extensions/minifi_tensor/minifi_tensor.md b/minifi_rust/extensions/minifi_tensor/minifi_tensor.md index 64f5b3b687..26e0ce54e8 100644 --- a/minifi_rust/extensions/minifi_tensor/minifi_tensor.md +++ b/minifi_rust/extensions/minifi_tensor/minifi_tensor.md @@ -64,7 +64,7 @@ In the list below, the names of required properties appear in bold. Any other pr | Name | Description | |---------|-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| -| failure | The image could not be decoded, the input tensor could not be built, the model failed to run, or the model outputs could not be interpreted as scores + boxes. | +| failure | The image could not be decoded, the input tensor could not be built, the model failed to run, or the model outputs could not be interpreted as classification. | | success | Inference and post-processing completed. The flow file content is the original, unchanged image; the classifications are written to the configured output attribute as a JSON array (may be empty). | @@ -219,7 +219,7 @@ In the list below, the names of required properties appear in bold. Any other pr ### Description -Decodes an image from the flow file content and converts it into a normalised numeric tensor suitable for feeding into a downstream inference processor such as InvokeTractModel. Supports RGB / BGR / Grayscale, CHW / HWC layouts, stretch or letterbox resizing, and scalar or per-channel mean/std normalisation. The output payload is the raw little-endian f32 tensor; the 'tensor.shape' and 'tensor.dtype' attributes describe its layout. +Decodes an image from the flow file content and converts it into a normalised numeric tensor suitable for feeding into a downstream inference processor such as InvokeTractModel. Supports RGB / BGR / Grayscale, CHW / HWC layouts, stretch or letterbox resizing, and scalar or per-channel mean/std normalisation. The output payload is the raw little-endian f32 tensor; the 'tensors.len', 'tensor.{i}.shape' and 'tensor.{i}.dtype' attributes describe its layout. ### Properties @@ -263,7 +263,7 @@ In the list below, the names of required properties appear in bold. Any other pr ### Description -Runs a single inference against the compiled model owned by the referenced TractModelService. Reads the input tensor from the flow file content plus the 'tensor.shape' and (optionally) 'tensor.dtype' attributes produced by an upstream processor such as ImageToTensor. The flow file's new content is every output tensor's raw bytes concatenated in model order; per-tensor shape, byte length, and dtype are written to attributes. Only a single input tensor and only f32 input dtype are supported in this pass. +Runs a single inference against the compiled model owned by the referenced TractModelService. Reads the input tensor from the flow file content plus the 'tensor.0.shape' and (optionally) 'tensor.0.dtype' attributes produced by an upstream processor such as ImageToTensor. The flow file's new content is every output tensor's raw bytes concatenated in model order; per-tensor shape, byte length, and dtype are written to attributes. ### Properties @@ -275,10 +275,10 @@ In the list below, the names of required properties appear in bold. Any other pr ### Relationships -| Name | Description | -|---------|---------------------------------------------------------------------------------------------------------------------------------------------| -| failure | The input tensor could not be built (missing/invalid tensor.shape, unsupported tensor.dtype, malformed payload) or the model failed to run. | -| success | Inference completed. The flow file's content is the concatenation of every output tensor's raw bytes in model output order. | +| Name | Description | +|---------|-------------------------------------------------------------------------------------------------------------------------------------------------| +| failure | The input tensor could not be built (missing/invalid tensor.0.shape, unsupported tensor.0.dtype, malformed payload) or the model failed to run. | +| success | Inference completed. The flow file's content is the concatenation of every output tensor's raw bytes in model output order. | ### Output Attributes diff --git a/minifi_rust/extensions/minifi_tensor/src/low_level_processors/classify_output.rs b/minifi_rust/extensions/minifi_tensor/src/low_level_processors/classify_output.rs index fd482b2993..c5542eb1fb 100644 --- a/minifi_rust/extensions/minifi_tensor/src/low_level_processors/classify_output.rs +++ b/minifi_rust/extensions/minifi_tensor/src/low_level_processors/classify_output.rs @@ -15,17 +15,17 @@ // specific language governing permissions and limitations // under the License. -use crate::utils::score_activation::ScoreActivation; +use crate::utils::score_activation::{ScoreActivation, SoftmaxTerms}; use crate::utils::tensor_helpers::{deserialize_tensors, tensor_as_f32, tensor_shape}; use classify_output_def::SUCCESS; pub(crate) use classify_output_def::{ - CONFIDENCE_THRESHOLD, LABEL_INDEX_OFFSET, LABELS_FILE_PATH, OUTPUT_ATTRIBUTE_NAME, - SCORE_ACTIVATION, SCORE_OUTPUT_INDEX, TOP_K, + CLASSIFY_OUTPUT_ATTRIBUTES, CONFIDENCE_THRESHOLD, LABEL_INDEX_OFFSET, LABELS_FILE_PATH, + OUTPUT_ATTRIBUTE_NAME, SCORE_ACTIVATION, SCORE_OUTPUT_INDEX, TOP_K, }; use minifi_native::macros::ComponentIdentifier; use minifi_native::{ Content, FlowFileTransform, GetAttribute, GetId, GetProperty, InputStream, Logger, MinifiError, - ProcessError, RouteErrorExt, Schedule, TransformedFlowFile, + ProcessError, RouteErrorExt, Schedule, TransformedFlowFile, warn, }; use serde::Serialize; use std::path::Path; @@ -88,6 +88,13 @@ impl Schedule for ClassifyOutput { _ => Vec::new(), }; let label_index_offset = context.get_property(&LABEL_INDEX_OFFSET)?; + if !labels.is_empty() && label_index_offset >= labels.len() { + return Err(MinifiError::validation(format!( + "Label index offset ({}) must be smaller than the number of labels ({})", + label_index_offset, + labels.len() + ))); + } Ok(Self { top_k, @@ -107,9 +114,10 @@ impl ClassifyOutput { .cloned() } - pub(crate) fn classify<'a, Context: GetProperty + GetAttribute + GetId>( + pub(crate) fn classify<'a, Context: GetProperty + GetAttribute + GetId, LoggerImpl: Logger>( &self, context: &Context, + logger: &LoggerImpl, tensors: Vec, ) -> Result, ProcessError> { let score_floats = @@ -139,33 +147,29 @@ impl ClassifyOutput { .filter(|&(_, s)| s.is_finite()) .collect(); - let (max_logit, sum_exp) = match self.score_activation { - ScoreActivation::Softmax => { - let max = finite - .iter() - .map(|&(_, s)| s) - .reduce(f32::max) - .unwrap_or(f32::NEG_INFINITY); - let sum = finite.iter().map(|&(_, s)| (s - max).exp()).sum::(); - (max, sum) - } - _ => (0.0, 1.0), - }; + let softmax_terms = SoftmaxTerms::over(finite.iter().map(|&(_, s)| s)); let predictions: Vec = top_k(finite, self.top_k) .into_iter() .filter_map(|(class_id, raw)| { - let confidence = match self.score_activation { - ScoreActivation::Softmax => (raw - max_logit).exp() / sum_exp, - ScoreActivation::Sigmoid => 1.0 / (1.0 + (-raw).exp()), - ScoreActivation::None => raw, - }; + let confidence = self.score_activation.confidence(raw, softmax_terms); if confidence >= self.confidence_threshold { + let class_name = self.label_for(class_id); + if class_name.is_none() && !self.labels.is_empty() { + warn!( + logger, + "No label for class id {} (offset {}, {} labels loaded); \ + the labels file does not match the model's classes", + class_id, + self.label_index_offset, + self.labels.len() + ); + } Some(Prediction { class_id, confidence, - class_name: self.label_for(class_id), + class_name, }) } else { None @@ -214,10 +218,10 @@ impl FlowFileTransform for ClassifyOutput { &self, context: &Context, input_stream: &'a mut dyn InputStream, - _logger: &LoggerImpl, + logger: &LoggerImpl, ) -> Result, ProcessError> { let tensors = deserialize_tensors(context, input_stream).route_err_to_failure()?; - self.classify(context, tensors) + self.classify(context, logger, tensors) } } @@ -225,7 +229,7 @@ impl FlowFileTransform for ClassifyOutput { mod tests { use super::classify_output_def::FAILURE; use super::*; - use minifi_native::{MockLogger, MockProcessContext}; + use minifi_native::{LogLevel, MockLogger, MockProcessContext}; use std::io::Cursor; fn make_processor(top_k: usize, activation: ScoreActivation) -> ClassifyOutput { @@ -358,6 +362,49 @@ mod tests { ); } + #[test] + fn test_label_lookup_miss_warns_when_labels_are_configured() { + let mut processor = make_processor(1, ScoreActivation::None); + processor.labels = vec!["dummy".into(), "tench".into()]; + processor.label_index_offset = 1; + let scores = vec![0.1f32, 0.2, 0.9]; // model class 2 wins + let context = context_with_scores(&scores); + let mut stream = Cursor::new(build_payload(&scores)); + let logger = MockLogger::new(); + + let result = processor.transform(&context, &mut stream, &logger).unwrap(); + + assert!( + result.attribute("class.top1.name").is_none(), + "an out-of-range label index still yields no name" + ); + let logs = logger.logs.lock().unwrap(); + assert!( + logs.iter() + .any(|(level, msg)| *level == LogLevel::Warn && msg.contains("No label for class")), + "expected a warning about the labels/model mismatch, got: {logs:?}" + ); + } + + #[test] + fn test_label_lookup_stays_silent_without_a_labels_file() { + let processor = make_processor(1, ScoreActivation::None); + let scores = vec![0.1f32, 0.9, 0.5]; + let context = context_with_scores(&scores); + let mut stream = Cursor::new(build_payload(&scores)); + let logger = MockLogger::new(); + + processor.transform(&context, &mut stream, &logger).unwrap(); + + let logs = logger.logs.lock().unwrap(); + assert!( + !logs + .iter() + .any(|(_, msg)| msg.contains("No label for class")), + "should not warn when no labels file is configured, got: {logs:?}" + ); + } + #[test] fn test_transform_filters_below_confidence_threshold() { let mut processor = make_processor(3, ScoreActivation::None); diff --git a/minifi_rust/extensions/minifi_tensor/src/low_level_processors/classify_output/classify_output_def.rs b/minifi_rust/extensions/minifi_tensor/src/low_level_processors/classify_output/classify_output_def.rs index 1a9e4457b4..263c586295 100644 --- a/minifi_rust/extensions/minifi_tensor/src/low_level_processors/classify_output/classify_output_def.rs +++ b/minifi_rust/extensions/minifi_tensor/src/low_level_processors/classify_output/classify_output_def.rs @@ -129,6 +129,14 @@ pub(crate) const OUTPUT_ATTRIBUTE_NAME: Property> = Property::new ) .supports_expression_language(); +pub(crate) const CLASSIFY_OUTPUT_ATTRIBUTES: &[OutputAttribute] = &[ + MIME_TYPE_ATTR, + CLASS_COUNT_ATTR, + CLASS_TOP1_ID_ATTR, + CLASS_TOP1_CONFIDENCE_ATTR, + CLASS_TOP1_NAME_ATTR, +]; + impl ProcessorDefinition for ClassifyOutput { const DESCRIPTION: &'static str = "Post-processes the output of a classification model invoked via InvokeTractModel. Reads \ the flattened score vector from the configured output tensor index, applies the chosen \ @@ -141,13 +149,7 @@ impl ProcessorDefinition for ClassifyOutput { const INPUT_REQUIREMENT: ProcessorInputRequirement = ProcessorInputRequirement::Required; const SUPPORTS_DYNAMIC_PROPERTIES: bool = false; const SUPPORTS_DYNAMIC_RELATIONSHIPS: bool = false; - const OUTPUT_ATTRIBUTES: &'static [OutputAttribute] = &[ - MIME_TYPE_ATTR, - CLASS_COUNT_ATTR, - CLASS_TOP1_ID_ATTR, - CLASS_TOP1_CONFIDENCE_ATTR, - CLASS_TOP1_NAME_ATTR, - ]; + const OUTPUT_ATTRIBUTES: &'static [OutputAttribute] = CLASSIFY_OUTPUT_ATTRIBUTES; const RELATIONSHIPS: &'static [Relationship] = &[SUCCESS, FAILURE]; const PROPERTIES: &[PropertyDefinition] = property_definitions![ diff --git a/minifi_rust/extensions/minifi_tensor/src/low_level_processors/filter_bounding_boxes.rs b/minifi_rust/extensions/minifi_tensor/src/low_level_processors/filter_bounding_boxes.rs index 559efdb621..b4aa74aa01 100644 --- a/minifi_rust/extensions/minifi_tensor/src/low_level_processors/filter_bounding_boxes.rs +++ b/minifi_rust/extensions/minifi_tensor/src/low_level_processors/filter_bounding_boxes.rs @@ -20,7 +20,7 @@ mod filter_bounding_boxes_def; use crate::low_level_processors::image_to_tensor::ResizeMode; use crate::utils::bounding_box::BoundingBox; use crate::utils::dimensions::Dimensions; -use crate::utils::score_activation::ScoreActivation; +use crate::utils::score_activation::{ScoreActivation, SoftmaxTerms}; use crate::utils::tensor_helpers::{deserialize_tensors, tensor_as_f32}; use filter_bounding_boxes_def::SUCCESS; pub(crate) use filter_bounding_boxes_def::{ @@ -97,25 +97,7 @@ fn score_box( } }; - let confidence = match activation { - ScoreActivation::Softmax => { - let max_logit = logits - .iter() - .copied() - .filter(|l| l.is_finite()) - .reduce(f32::max) - .unwrap_or(f32::NEG_INFINITY); - let sum_exp: f32 = logits - .iter() - .filter(|l| l.is_finite()) - .map(|&l| (l - max_logit).exp()) - .sum(); - - (best_logit - max_logit).exp() / sum_exp - } - ScoreActivation::Sigmoid => 1.0 / (1.0 + (-best_logit).exp()), - ScoreActivation::None => best_logit, - }; + let confidence = activation.confidence(best_logit, SoftmaxTerms::over(logits.iter().copied())); ScoredClass { class_id, @@ -123,18 +105,6 @@ fn score_box( } } -/// Turn a single per-box score into a confidence for the "separate class-id -/// tensor" path. -/// Sigmoid maps a raw logit to a probability; -/// Softmax has no meaning over a single scalar, thus pass-through. -/// None passes the score through. -fn activate_scalar(score: f32, activation: ScoreActivation) -> f32 { - match activation { - ScoreActivation::Sigmoid => 1.0 / (1.0 + (-score).exp()), - ScoreActivation::Softmax | ScoreActivation::None => score, - } -} - #[derive(ComponentIdentifier)] pub(crate) struct FilterBoundingBoxes { confidence_threshold: f32, @@ -217,11 +187,13 @@ impl FilterBoundingBoxes { let (scale_x, scale_y, pad_x, pad_y) = match resize_mode { ResizeMode::Letterbox => { - let scale = - (target_dim.width / orig_dim.width).min(target_dim.height / orig_dim.height); - let pad_x = (target_dim.width - (orig_dim.width * scale)) / 2.0; - let pad_y = (target_dim.height - (orig_dim.height * scale)) / 2.0; - (scale, scale, pad_x, pad_y) + let geometry = orig_dim.letterbox_into(target_dim); + ( + geometry.scale, + geometry.scale, + geometry.pad_x as f32, + geometry.pad_y as f32, + ) } ResizeMode::Stretch => ( target_dim.width / orig_dim.width, @@ -289,7 +261,7 @@ impl FilterBoundingBoxes { self.box_format ); for i in 0..num_boxes { - let confidence = activate_scalar(score_floats[i], self.score_activation); + let confidence = self.score_activation.confidence_of_scalar(score_floats[i]); if confidence < self.confidence_threshold { continue; } @@ -444,10 +416,10 @@ mod tests { #[test] fn test_activate_scalar_sigmoid_and_passthrough() { - assert!((activate_scalar(0.0, ScoreActivation::Sigmoid) - 0.5).abs() < 1e-6); - assert_eq!(activate_scalar(0.42, ScoreActivation::None), 0.42); + assert!((ScoreActivation::Sigmoid.confidence_of_scalar(0.0) - 0.5).abs() < 1e-6); + assert_eq!(ScoreActivation::None.confidence_of_scalar(0.42), 0.42); // Softmax over a scalar has no meaning → pass-through. - assert_eq!(activate_scalar(0.42, ScoreActivation::Softmax), 0.42); + assert_eq!(ScoreActivation::Softmax.confidence_of_scalar(0.42), 0.42); } fn class_index_processor() -> FilterBoundingBoxes { @@ -546,6 +518,53 @@ mod tests { assert!((letterboxed.y_max - 0.7).abs() < 1e-5); } + #[test] + fn test_letterbox_un_mapping_uses_the_integer_padding_that_was_applied() { + use minifi_native::{MockLogger, MockProcessContext}; + use tract::__ndarray_interop::TensorInterface; + + // SSD300 fed a 1080p frame: scale = 300/1920 = 0.15625, so the scaled + // height is 1080 * 0.15625 = 168.75 — *not* an integer. ImageToTensor + // rounds to 169 and pads (300 - 169) / 2 = 65. Deriving the padding from + // the unrounded 168.75 instead gives 65.625, and that 0.625 target-pixel + // error becomes 0.625 / 0.15625 = 4 original pixels once divided back + // through the scale. + let scores = Tensor::from_slice::(&[1], &[0.9]).unwrap(); + let boxes = Tensor::from_slice::(&[1, 4], &[0.4, 0.4, 0.6, 0.6]).unwrap(); + let classes = Tensor::from_slice::(&[1], &[5]).unwrap(); + + let result = class_index_processor() + .filter( + &MockProcessContext::new(), + &MockLogger::new(), + vec![scores, boxes, classes], + Dimensions { + width: 1920.0, + height: 1080.0, + }, + Dimensions { + width: 300.0, + height: 300.0, + }, + ResizeMode::Letterbox, + ) + .expect("filter should succeed"); + let json = result.into_bytes().unwrap().unwrap(); + let boxes: Vec = serde_json::from_slice(&json).unwrap(); + let bbox = boxes.into_iter().next().expect("one box expected"); + + // x is unpadded (the width axis is the one that fills the canvas), so it + // round-trips exactly. + assert!((bbox.x_min - 0.4).abs() < 1e-5); + assert!((bbox.x_max - 0.6).abs() < 1e-5); + + // y with the integer pad of 65: ((0.4 * 300) - 65) / 0.15625 / 1080. + // The float-pad variant would yield 0.32222 / 0.67778 instead — a 4px + // error, well outside this tolerance. + assert!((bbox.y_min - 352.0 / 1080.0).abs() < 1e-5); + assert!((bbox.y_max - 736.0 / 1080.0).abs() < 1e-5); + } + #[test] fn test_filter_class_index_mode_rejects_mismatched_lengths() { use minifi_native::{MockLogger, MockProcessContext}; diff --git a/minifi_rust/extensions/minifi_tensor/src/low_level_processors/image_to_tensor.rs b/minifi_rust/extensions/minifi_tensor/src/low_level_processors/image_to_tensor.rs index 2c92459328..376f5a2c2c 100644 --- a/minifi_rust/extensions/minifi_tensor/src/low_level_processors/image_to_tensor.rs +++ b/minifi_rust/extensions/minifi_tensor/src/low_level_processors/image_to_tensor.rs @@ -17,7 +17,7 @@ pub(crate) mod image_to_tensor_def; use crate::low_level_processors::image_to_tensor::image_to_tensor_def::TENSOR_BYTES_ATTR; -use crate::utils::dimensions::Dimensions; +use crate::utils::dimensions::{Dimensions, LetterboxGeometry}; use crate::utils::per_channel_f32::PerChannelF32; use crate::utils::tensor_helpers::{MinifiDatumType, load_as_image}; pub(crate) use image_to_tensor_def::{ @@ -113,6 +113,11 @@ impl Schedule for ImageToTensor { { let target_width = context.get_property(&TARGET_WIDTH)?; let target_height = context.get_property(&TARGET_HEIGHT)?; + if target_width == 0 || target_height == 0 { + return Err(MinifiError::validation( + "Target width and Target height must be greater than zero", + )); + } let resize_filter = context.get_property(&RESIZE_FILTER)?; let resize_mode = context.get_property(&RESIZE_MODE)?; let color_format = context.get_property(&COLOR_FORMAT)?; @@ -151,6 +156,10 @@ struct MaskedRgbImage { } impl ImageToTensor { + fn total_pixels(&self) -> usize { + self.target_width as usize * self.target_height as usize + } + fn stretch_resize(&self, img: image::DynamicImage) -> MaskedRgbImage { let resized = img .resize_exact( @@ -159,22 +168,22 @@ impl ImageToTensor { self.resize_filter.into(), ) .to_rgb8(); - let mask = vec![true; (self.target_width * self.target_height) as usize]; + let mask = vec![true; self.total_pixels()]; MaskedRgbImage { img: resized, mask } } fn letterbox_resize(&self, img: image::DynamicImage) -> MaskedRgbImage { - let (src_w, src_h) = (img.width() as f32, img.height() as f32); - let scale = (self.target_width as f32 / src_w).min(self.target_height as f32 / src_h); - let new_w = (src_w * scale).round().max(1.0) as u32; - let new_h = (src_h * scale).round().max(1.0) as u32; + let LetterboxGeometry { + new_width: new_w, + new_height: new_h, + pad_x, + pad_y, + .. + } = Dimensions::from_image(&img).letterbox_into(self.get_target_dim()); let scaled = img .resize_exact(new_w, new_h, self.resize_filter.into()) .to_rgb8(); - let pad_x = (self.target_width - new_w) / 2; - let pad_y = (self.target_height - new_h) / 2; - let mut canvas = image::RgbImage::from_pixel( self.target_width, self.target_height, @@ -183,10 +192,10 @@ impl ImageToTensor { image::imageops::overlay(&mut canvas, &scaled, pad_x as i64, pad_y as i64); // Mask to track which pixel is part of source and which is padding - let mut mask = vec![false; (self.target_width * self.target_height) as usize]; + let mut mask = vec![false; self.total_pixels()]; for y in pad_y..(new_h + pad_y) { for x in pad_x..(new_w + pad_x) { - mask[(y * self.target_width + x) as usize] = true; + mask[y as usize * self.target_width as usize + x as usize] = true; } } MaskedRgbImage { img: canvas, mask } @@ -204,7 +213,7 @@ impl ImageToTensor { ColorFormat::Grayscale => 1, _ => 3, }; - let total_pixels = (self.target_width * self.target_height) as usize; + let total_pixels = self.total_pixels(); let mut tensor_bytes = Vec::with_capacity(total_pixels * num_channels * 4); let masked_img = self.resize_rgb(img); @@ -214,7 +223,9 @@ impl ImageToTensor { let std_dev = self.std_dev.per_channel(0); for (idx, pixel) in masked_img.img.pixels().enumerate() { let val = if masked_img.mask[idx] { - // Rec. 601 luma coefficients — matches image::to_luma8(). + // Rec. 601 luma coefficients, as used by OpenCV and most ML + // preprocessing pipelines. Note this is deliberately *not* + // image::to_luma8(), which applies Rec. 709. let luma = 0.299 * pixel[0] as f32 + 0.587 * pixel[1] as f32 + 0.114 * pixel[2] as f32; (luma / self.pixel_divisor - mean) / std_dev @@ -403,6 +414,34 @@ mod tests { .collect() } + #[test] + fn test_schedule_rejects_zero_target_dimensions() { + for (width, height) in [("0", "100"), ("100", "0"), ("0", "0")] { + let mut context = MockProcessContext::new(); + context.properties.insert(TARGET_WIDTH.name(), width); + context.properties.insert(TARGET_HEIGHT.name(), height); + context.properties.insert(PIXEL_DIVISOR.name(), "1.0"); + + assert!( + matches!( + ImageToTensor::schedule(&context, &MockLogger::new()), + Err(MinifiError::ValidationError(_)) + ), + "expected {width}x{height} to be rejected" + ); + } + } + + #[test] + fn test_schedule_accepts_non_zero_target_dimensions() { + let mut context = MockProcessContext::new(); + context.properties.insert(TARGET_WIDTH.name(), "224"); + context.properties.insert(TARGET_HEIGHT.name(), "224"); + context.properties.insert(PIXEL_DIVISOR.name(), "1.0"); + + assert!(ImageToTensor::schedule(&context, &MockLogger::new()).is_ok()); + } + #[test] fn test_successful_rgb_chw_transform() { let processor = default_processor(); diff --git a/minifi_rust/extensions/minifi_tensor/src/processors/classify_image.rs b/minifi_rust/extensions/minifi_tensor/src/processors/classify_image.rs index 2b61a428e0..548dfe466e 100644 --- a/minifi_rust/extensions/minifi_tensor/src/processors/classify_image.rs +++ b/minifi_rust/extensions/minifi_tensor/src/processors/classify_image.rs @@ -57,7 +57,7 @@ impl FlowFileTransform for ClassifyImage { &self, context: &Context, input_stream: &'a mut dyn InputStream, - _logger: &LoggerImpl, + logger: &LoggerImpl, ) -> Result, ProcessError> { let tract_model_service = context.get_controller_service(&TRACT_MODEL_SERVICE)?; let img = load_as_image(input_stream).route_err_to_failure()?; @@ -74,6 +74,7 @@ impl FlowFileTransform for ClassifyImage { .route_err_to_failure()?; // ClassifyOutput - self.classify_output.classify(context, output_tensors) + self.classify_output + .classify(context, logger, output_tensors) } } diff --git a/minifi_rust/extensions/minifi_tensor/src/processors/classify_image/classify_image_def.rs b/minifi_rust/extensions/minifi_tensor/src/processors/classify_image/classify_image_def.rs index 6a4791c950..a7e86c57a0 100644 --- a/minifi_rust/extensions/minifi_tensor/src/processors/classify_image/classify_image_def.rs +++ b/minifi_rust/extensions/minifi_tensor/src/processors/classify_image/classify_image_def.rs @@ -55,7 +55,8 @@ impl ProcessorDefinition for ClassifyImage { const INPUT_REQUIREMENT: ProcessorInputRequirement = ProcessorInputRequirement::Required; const SUPPORTS_DYNAMIC_PROPERTIES: bool = false; const SUPPORTS_DYNAMIC_RELATIONSHIPS: bool = false; - const OUTPUT_ATTRIBUTES: &'static [OutputAttribute] = &[]; + const OUTPUT_ATTRIBUTES: &'static [OutputAttribute] = + classify_output::CLASSIFY_OUTPUT_ATTRIBUTES; const RELATIONSHIPS: &'static [Relationship] = &[SUCCESS, FAILURE]; const PROPERTIES: &'static [PropertyDefinition] = property_definitions![ diff --git a/minifi_rust/extensions/minifi_tensor/src/utils/dimensions.rs b/minifi_rust/extensions/minifi_tensor/src/utils/dimensions.rs index 9a78016b9b..677b7dd57b 100644 --- a/minifi_rust/extensions/minifi_tensor/src/utils/dimensions.rs +++ b/minifi_rust/extensions/minifi_tensor/src/utils/dimensions.rs @@ -1,5 +1,22 @@ use minifi_native::{GetAttribute, MinifiError}; +/// The exact placement of an aspect-preserving resize inside a target canvas. +/// +/// `ImageToTensor` applies this when resizing, and `FilterBoundingBoxes` inverts +/// it when un-mapping model coordinates back to the original image. Both must +/// agree down to the pixel, so the arithmetic lives here and nowhere else: +/// deriving the padding from the *unrounded* scaled size instead of `new_w`/ +/// `new_h` drifts by up to half a target pixel, which is several pixels once +/// divided back through `scale`. +#[derive(Debug, Clone, Copy, PartialEq)] +pub(crate) struct LetterboxGeometry { + pub(crate) scale: f32, + pub(crate) new_width: u32, + pub(crate) new_height: u32, + pub(crate) pad_x: u32, + pub(crate) pad_y: u32, +} + #[derive(Debug, Clone, Copy, PartialEq)] pub(crate) struct Dimensions { pub(crate) width: f32, @@ -7,6 +24,27 @@ pub(crate) struct Dimensions { } impl Dimensions { + /// Fit `self` into `target` preserving aspect ratio, centring the result. + /// + /// Assumes both dimensions are non-zero; `ImageToTensor::schedule` rejects a + /// zero 'Target width'/'Target height', and a decoded image always has at + /// least one pixel per axis. + pub(crate) fn letterbox_into(&self, target: Dimensions) -> LetterboxGeometry { + let scale = (target.width / self.width).min(target.height / self.height); + let new_width = (self.width * scale).round().max(1.0) as u32; + let new_height = (self.height * scale).round().max(1.0) as u32; + LetterboxGeometry { + scale, + new_width, + new_height, + // Saturating: `new_*` is clamped up to 1, so it can exceed a target + // axis of 0. Callers reject that config, but wrapping here would + // turn a misconfiguration into a panic or a garbage offset. + pad_x: (target.width as u32).saturating_sub(new_width) / 2, + pad_y: (target.height as u32).saturating_sub(new_height) / 2, + } + } + pub(crate) fn from_image(img: &image::DynamicImage) -> Self { Self { width: img.width() as f32, @@ -48,3 +86,44 @@ impl Dimensions { }) } } + +#[cfg(test)] +mod tests { + use super::*; + + fn dim(width: f32, height: f32) -> Dimensions { + Dimensions { width, height } + } + + #[test] + fn letterbox_pads_from_the_rounded_size() { + // 1080p into SSD300: 1080 * 0.15625 = 168.75 rounds to 169, so the pad is + // (300 - 169) / 2 = 65 — not the 65.625 the unrounded size would give. + let geometry = dim(1920.0, 1080.0).letterbox_into(dim(300.0, 300.0)); + assert_eq!(geometry.scale, 0.15625); + assert_eq!(geometry.new_width, 300); + assert_eq!(geometry.new_height, 169); + assert_eq!(geometry.pad_x, 0); + assert_eq!(geometry.pad_y, 65); + } + + #[test] + fn letterbox_is_exact_when_the_scaled_size_is_integral() { + let geometry = dim(200.0, 100.0).letterbox_into(dim(100.0, 100.0)); + assert_eq!(geometry.scale, 0.5); + assert_eq!(geometry.new_width, 100); + assert_eq!(geometry.new_height, 50); + assert_eq!(geometry.pad_x, 0); + assert_eq!(geometry.pad_y, 25); + } + + #[test] + fn letterbox_keeps_a_degenerate_axis_at_one_pixel() { + // A very wide source against a small target rounds the short axis to 0; + // it is clamped to 1 so the resize stays valid. + let geometry = dim(1000.0, 3.0).letterbox_into(dim(10.0, 10.0)); + assert_eq!(geometry.new_width, 10); + assert_eq!(geometry.new_height, 1); + assert_eq!(geometry.pad_y, 4); + } +} diff --git a/minifi_rust/extensions/minifi_tensor/src/utils/score_activation.rs b/minifi_rust/extensions/minifi_tensor/src/utils/score_activation.rs index 12125693e0..cb10d4978e 100644 --- a/minifi_rust/extensions/minifi_tensor/src/utils/score_activation.rs +++ b/minifi_rust/extensions/minifi_tensor/src/utils/score_activation.rs @@ -14,3 +14,61 @@ pub(crate) enum ScoreActivation { /// Pass-through — the model already emits probabilities. None, } + +/// The `(max_logit, sum_exp)` denominator of a numerically-stable softmax. +/// +/// Subtracting the max before exponentiating keeps `exp` in range for large +/// logits. Non-finite logits are skipped so one NaN cannot poison the whole +/// distribution. +#[derive(Debug, Clone, Copy, PartialEq)] +pub(crate) struct SoftmaxTerms { + max_logit: f32, + sum_exp: f32, +} + +impl SoftmaxTerms { + pub(crate) fn over(logits: impl IntoIterator + Clone) -> Self { + let max_logit = logits + .clone() + .into_iter() + .filter(|l| l.is_finite()) + .reduce(f32::max) + .unwrap_or(f32::NEG_INFINITY); + let sum_exp = logits + .into_iter() + .filter(|l| l.is_finite()) + .map(|l| (l - max_logit).exp()) + .sum(); + Self { max_logit, sum_exp } + } +} + +impl ScoreActivation { + /// Confidence for one logit drawn from a full score vector. + /// + /// `terms` must be computed over that same vector, so `Softmax` normalises + /// against the distribution the logit came from. + pub(crate) fn confidence(self, logit: f32, terms: SoftmaxTerms) -> f32 { + match self { + ScoreActivation::Softmax => (logit - terms.max_logit).exp() / terms.sum_exp, + ScoreActivation::Sigmoid => sigmoid(logit), + ScoreActivation::None => logit, + } + } + + /// Confidence for a standalone score, with no surrounding vector to + /// normalise against — the "separate class-id tensor" detector layout. + /// + /// Sigmoid maps a raw logit to a probability; softmax has no meaning over a + /// single scalar, so it passes through, as does None. + pub(crate) fn confidence_of_scalar(self, score: f32) -> f32 { + match self { + ScoreActivation::Sigmoid => sigmoid(score), + ScoreActivation::Softmax | ScoreActivation::None => score, + } + } +} + +fn sigmoid(x: f32) -> f32 { + 1.0 / (1.0 + (-x).exp()) +} diff --git a/minifi_rust/extensions/minifi_tensor/src/utils/tensor_helpers.rs b/minifi_rust/extensions/minifi_tensor/src/utils/tensor_helpers.rs index c7aa79517a..ec4c6b7fbc 100644 --- a/minifi_rust/extensions/minifi_tensor/src/utils/tensor_helpers.rs +++ b/minifi_rust/extensions/minifi_tensor/src/utils/tensor_helpers.rs @@ -88,18 +88,19 @@ pub(crate) fn deserialize_tensors( .parse::()?; let tensor_shape = parse_tensor_shape(context, i)?; let tensor_dtype = parse_tensor_dtype(context, i)?; - if cursor + tensor_len > flow_file_contents.len() { - return Err(MinifiError::custom( - "FlowFile contents are not in sync with tensor attributes", - )); - } - let tensor_data = &flow_file_contents[cursor..cursor + tensor_len]; + let tensor_end = cursor + .checked_add(tensor_len) + .filter(|end| *end <= flow_file_contents.len()) + .ok_or_else(|| { + MinifiError::custom("FlowFile contents are not in sync with tensor attributes") + })?; + let tensor_data = &flow_file_contents[cursor..tensor_end]; result.push(Tensor::from_bytes( tensor_dtype, &tensor_shape, tensor_data, )?); - cursor += tensor_len; + cursor = tensor_end; } if cursor != flow_file_contents.len() { @@ -207,4 +208,35 @@ mod tests { Ok(_) ); } + + #[test] + fn oversized_tensor_len_attribute_errors_instead_of_panicking() { + // A `tensor.N.bytes` large enough to overflow `cursor + tensor_len` must + // still be rejected: the wrapped sum used to slip past the bounds check + // and panic on the slice (aborting the process, since panic = "abort"). + let mut context = MockProcessContext::new(); + let bytes: Vec = vec![0u8; 8]; + let input_stream = Cursor::new(bytes); + + // Two tensors so the second one is checked against a non-zero cursor — + // `0 + tensor_len` cannot overflow. + context.attributes.insert("tensors.len".into(), "2".into()); + for (i, len) in [("0", "8"), ("1", &usize::MAX.to_string()[..])] { + context + .attributes + .insert(format!("tensor.{i}.bytes"), len.into()); + context + .attributes + .insert(format!("tensor.{i}.shape"), "2".into()); + context + .attributes + .insert(format!("tensor.{i}.dtype"), "F32".into()); + } + + assert_matches!( + deserialize_tensors(&context, &mut input_stream.clone()), + Err(MinifiError::CustomError(msg)) + if msg == "FlowFile contents are not in sync with tensor attributes" + ); + } } From 64eebb711cdbd4bee739a567c1c32de648c99710 Mon Sep 17 00:00:00 2001 From: Martin Zink Date: Wed, 23 Sep 2026 17:30:37 +0200 Subject: [PATCH 18/32] trim labels both direction --- .../minifi_tensor/src/low_level_processors/classify_output.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/minifi_rust/extensions/minifi_tensor/src/low_level_processors/classify_output.rs b/minifi_rust/extensions/minifi_tensor/src/low_level_processors/classify_output.rs index c5542eb1fb..cda39a52b4 100644 --- a/minifi_rust/extensions/minifi_tensor/src/low_level_processors/classify_output.rs +++ b/minifi_rust/extensions/minifi_tensor/src/low_level_processors/classify_output.rs @@ -47,7 +47,7 @@ fn load_labels(path: &Path) -> Result, MinifiError> { })?; Ok(content .lines() - .map(|line| line.trim_end().to_string()) + .map(|line| line.trim().to_string()) .collect()) } From 0d344a1381ecb58594db0d051f5c969db2605f48 Mon Sep 17 00:00:00 2001 From: Martin Zink Date: Thu, 24 Sep 2026 15:26:23 +0200 Subject: [PATCH 19/32] adding missing TENSOR_BYTES_ATTR to image_to_tensor_def.rs --- .../low_level_processors/image_to_tensor/image_to_tensor_def.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/minifi_rust/extensions/minifi_tensor/src/low_level_processors/image_to_tensor/image_to_tensor_def.rs b/minifi_rust/extensions/minifi_tensor/src/low_level_processors/image_to_tensor/image_to_tensor_def.rs index 186f001f15..704edc4ade 100644 --- a/minifi_rust/extensions/minifi_tensor/src/low_level_processors/image_to_tensor/image_to_tensor_def.rs +++ b/minifi_rust/extensions/minifi_tensor/src/low_level_processors/image_to_tensor/image_to_tensor_def.rs @@ -182,6 +182,7 @@ impl ProcessorDefinition for ImageToTensor { const SUPPORTS_DYNAMIC_RELATIONSHIPS: bool = false; const OUTPUT_ATTRIBUTES: &'static [OutputAttribute] = &[ TENSORS_LEN_ATTR, + TENSOR_BYTES_ATTR, TENSOR_SHAPE_ATTR, TENSOR_DTYPE_ATTR, IMG_ORG_WIDTH_ATTR, From dea51ecf22f283e1ec7c333dae758aae512a7308 Mon Sep 17 00:00:00 2001 From: Martin Zink Date: Fri, 25 Sep 2026 10:13:23 +0200 Subject: [PATCH 20/32] "certifi>=2026.7.22" --- behave_framework/pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/behave_framework/pyproject.toml b/behave_framework/pyproject.toml index 04c3039522..84fa62f287 100644 --- a/behave_framework/pyproject.toml +++ b/behave_framework/pyproject.toml @@ -10,7 +10,7 @@ dependencies = [ "humanfriendly==10.0", "cryptography==50.0.0", "pyjks==20.0.0", - "certifi==2026.7.22" + "certifi>=2026.7.22" ] [tool.setuptools] From 92c3020134ee11a7116cd67b3bd05e69d50c96ba Mon Sep 17 00:00:00 2001 From: Martin Zink Date: Fri, 25 Sep 2026 10:13:42 +0200 Subject: [PATCH 21/32] Feature: Face detection with UltraFace Single Shot MultiBox Detector (SSD) --- minifi_rust/extensions/minifi_tensor/features/detection.feature | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/minifi_rust/extensions/minifi_tensor/features/detection.feature b/minifi_rust/extensions/minifi_tensor/features/detection.feature index 660bca9f8b..9e807eb094 100644 --- a/minifi_rust/extensions/minifi_tensor/features/detection.feature +++ b/minifi_rust/extensions/minifi_tensor/features/detection.feature @@ -14,7 +14,7 @@ # limitations under the License. @SUPPORTS_WINDOWS -Feature: Face detection with UltraFace (SSD) +Feature: Face detection with UltraFace Single Shot MultiBox Detector (SSD) # Based on https://github.com/sonos/tract/blob/main/examples/face_detection_yolov8onnx_example/src/main.rs Scenario: Grace Hopper image yields at least one face detection (ImageToTensor + InvokeTract + FilterBoundingBoxes) From db3d02cdc0d9d8dcb286010e589d87eb671c70c9 Mon Sep 17 00:00:00 2001 From: Martin Zink Date: Fri, 25 Sep 2026 10:18:22 +0200 Subject: [PATCH 22/32] added missing apache license --- .../minifi_tensor/src/utils/bounding_box.rs | 17 +++++++++++++++++ .../minifi_tensor/src/utils/dimensions.rs | 17 +++++++++++++++++ .../minifi_tensor/src/utils/per_channel_f32.rs | 17 +++++++++++++++++ .../minifi_tensor/src/utils/score_activation.rs | 17 +++++++++++++++++ .../minifi_tensor/src/utils/tensor_helpers.rs | 17 +++++++++++++++++ 5 files changed, 85 insertions(+) diff --git a/minifi_rust/extensions/minifi_tensor/src/utils/bounding_box.rs b/minifi_rust/extensions/minifi_tensor/src/utils/bounding_box.rs index 44b0fb7cb3..92dbdedf87 100644 --- a/minifi_rust/extensions/minifi_tensor/src/utils/bounding_box.rs +++ b/minifi_rust/extensions/minifi_tensor/src/utils/bounding_box.rs @@ -1,3 +1,20 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + use image::{Rgb, RgbImage}; use minifi_native::{MinifiError, PropertyConstraints, PropertySchema, PropertyType}; use serde::{Deserialize, Serialize}; diff --git a/minifi_rust/extensions/minifi_tensor/src/utils/dimensions.rs b/minifi_rust/extensions/minifi_tensor/src/utils/dimensions.rs index 677b7dd57b..98574b9137 100644 --- a/minifi_rust/extensions/minifi_tensor/src/utils/dimensions.rs +++ b/minifi_rust/extensions/minifi_tensor/src/utils/dimensions.rs @@ -1,3 +1,20 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + use minifi_native::{GetAttribute, MinifiError}; /// The exact placement of an aspect-preserving resize inside a target canvas. diff --git a/minifi_rust/extensions/minifi_tensor/src/utils/per_channel_f32.rs b/minifi_rust/extensions/minifi_tensor/src/utils/per_channel_f32.rs index 887b6aa564..39f198ad4f 100644 --- a/minifi_rust/extensions/minifi_tensor/src/utils/per_channel_f32.rs +++ b/minifi_rust/extensions/minifi_tensor/src/utils/per_channel_f32.rs @@ -1,3 +1,20 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + use crate::utils::per_channel_f32::PerChannelF32::TriChannel; use minifi_native::{MinifiError, PropertyConstraints, PropertySchema, PropertyType}; diff --git a/minifi_rust/extensions/minifi_tensor/src/utils/score_activation.rs b/minifi_rust/extensions/minifi_tensor/src/utils/score_activation.rs index cb10d4978e..a710d86646 100644 --- a/minifi_rust/extensions/minifi_tensor/src/utils/score_activation.rs +++ b/minifi_rust/extensions/minifi_tensor/src/utils/score_activation.rs @@ -1,3 +1,20 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + use minifi_native::macros::PropertyType; use strum_macros::{Display, EnumString, IntoStaticStr, VariantNames}; diff --git a/minifi_rust/extensions/minifi_tensor/src/utils/tensor_helpers.rs b/minifi_rust/extensions/minifi_tensor/src/utils/tensor_helpers.rs index ec4c6b7fbc..e7c60eca1e 100644 --- a/minifi_rust/extensions/minifi_tensor/src/utils/tensor_helpers.rs +++ b/minifi_rust/extensions/minifi_tensor/src/utils/tensor_helpers.rs @@ -1,3 +1,20 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + use image::{DynamicImage, ImageResult}; use minifi_native::{GetAttribute, InputStream, MinifiError}; use strum_macros::{Display, EnumString}; From ab06768b1defe86f1d36fee5558ac9b3578b281f Mon Sep 17 00:00:00 2001 From: Martin Zink Date: Fri, 25 Sep 2026 10:19:22 +0200 Subject: [PATCH 23/32] output attrs changes, route_to_failure changes, --- .../extensions/minifi_tensor/minifi_tensor.md | 35 ++++++++++++------- .../low_level_processors/classify_output.rs | 24 +++++++------ .../classify_output/classify_output_def.rs | 25 +++++++------ .../filter_bounding_boxes.rs | 21 +++++------ .../filter_bounding_boxes_def.rs | 5 ++- .../src/low_level_processors/mod.rs | 17 +++++++++ 6 files changed, 77 insertions(+), 50 deletions(-) diff --git a/minifi_rust/extensions/minifi_tensor/minifi_tensor.md b/minifi_rust/extensions/minifi_tensor/minifi_tensor.md index 26e0ce54e8..7c10f2bc5c 100644 --- a/minifi_rust/extensions/minifi_tensor/minifi_tensor.md +++ b/minifi_rust/extensions/minifi_tensor/minifi_tensor.md @@ -67,6 +67,16 @@ In the list below, the names of required properties appear in bold. Any other pr | failure | The image could not be decoded, the input tensor could not be built, the model failed to run, or the model outputs could not be interpreted as classification. | | success | Inference and post-processing completed. The flow file content is the original, unchanged image; the classifications are written to the configured output attribute as a JSON array (may be empty). | +### Output Attributes + +| Attribute | Relationship | Description | +|-----------------------|--------------|----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| class.count | success | Number of predictions retained after Top K selection and confidence filtering. | +| class.top1.confidence | success | Confidence (post-activation) of the highest-confidence prediction, when at least one prediction cleared the confidence threshold. | +| class.top1.id | success | Numeric class ID of the highest-confidence prediction, when at least one prediction cleared the confidence threshold. | +| class.top1.name | success | Label of the highest-confidence prediction. Only present when 'Labels file path' was configured and at least one prediction cleared the threshold. | +| mime.type | success | If the "Output attribute name" is None, then the content will be overridden with the JSON array of objects with fields class_id, confidence, and optional class_name, and the mime type will be set to 'application/json'. | + ## ClassifyOutput @@ -97,13 +107,13 @@ In the list below, the names of required properties appear in bold. Any other pr ### Output Attributes -| Attribute | Relationship | Description | -|-----------------------|--------------|----------------------------------------------------------------------------------------------------------------------------------------------------| -| class.count | success | Number of predictions retained after Top K selection and confidence filtering. | -| class.top1.confidence | success | Confidence (post-activation) of the highest-confidence prediction, when at least one prediction cleared the confidence threshold. | -| class.top1.id | success | Numeric class ID of the highest-confidence prediction, when at least one prediction cleared the confidence threshold. | -| class.top1.name | success | Label of the highest-confidence prediction. Only present when 'Labels file path' was configured and at least one prediction cleared the threshold. | -| mime.type | success | Always 'application/json' — the output payload is a JSON array of prediction objects with fields class_id, confidence, and optional class_name. | +| Attribute | Relationship | Description | +|-----------------------|--------------|----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| class.count | success | Number of predictions retained after Top K selection and confidence filtering. | +| class.top1.confidence | success | Confidence (post-activation) of the highest-confidence prediction, when at least one prediction cleared the confidence threshold. | +| class.top1.id | success | Numeric class ID of the highest-confidence prediction, when at least one prediction cleared the confidence threshold. | +| class.top1.name | success | Label of the highest-confidence prediction. Only present when 'Labels file path' was configured and at least one prediction cleared the threshold. | +| mime.type | success | If the "Output attribute name" is None, then the content will be overridden with the JSON array of objects with fields class_id, confidence, and optional class_name, and the mime type will be set to 'application/json'. | ## DetectObject @@ -167,7 +177,7 @@ In the list below, the names of required properties appear in bold. Any other pr | Name | Default Value | Allowable Values | Description | |--------------------|---------------------|------------------|-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| | Bounding boxes | ${enrichment.value} | | JSON array of bounding boxes to draw onto the image (fields class_id, confidence, x_min, y_min, x_max, y_max; coordinates normalised to [0,1] against the image). Typically the attribute produced by an upstream DetectObject or FilterBoundingBoxes processor.
**Supports Expression Language: true** | -| Line color | [0, 255, 0] | | Outline colour as '[R, G, B]' u8 channels (0-255). | +| Line color | #00FF00 | | Outline color as a hex string (e.g., '#ff00ff' or '#f0f') | | **Line thickness** | 5 | | Thickness in pixels of the drawn box outline. | ### Relationships @@ -209,10 +219,10 @@ In the list below, the names of required properties appear in bold. Any other pr ### Output Attributes -| Attribute | Relationship | Description | -|--------------|--------------|-------------------------------------------------------------------------------------------------------------------------------------------| -| mime.type | success | Always 'application/json' — the output payload is a JSON array of objects with fields class_id, confidence, x_min, y_min, x_max, y_max. | -| object.count | success | Number of bounding boxes retained after confidence filtering and NMS. | +| Attribute | Relationship | Description | +|--------------|--------------|-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| mime.type | success | If the "Output attribute name" is None, then the content will be overridden with the JSON array of objects with fields class_id, confidence, x_min, y_min, x_max, y_max, and the mime type will be set to 'application/json'. | +| object.count | success | Number of bounding boxes retained after confidence filtering and NMS. | ## ImageToTensor @@ -254,6 +264,7 @@ In the list below, the names of required properties appear in bold. Any other pr | image.resize.mode | success | The resize mode ('Stretch' or 'Letterbox') applied to fit the image into the target dimensions. Downstream processors such as FilterBoundingBoxes use this to invert the coordinate mapping correctly. | | image.target.height | success | The height of the image after the resizing. | | image.target.width | success | The width of the image after the resizing. | +| tensor.0.bytes | success | Byte length of output tensor. | | tensor.0.dtype | success | Element type of the values in the output tensor. Currently always 'F32'. | | tensor.0.shape | success | Comma-separated dimensions of the output tensor in the chosen layout, always including a leading batch dimension of 1 (e.g. '1,3,224,224' for RGB CHW). | | tensors.len | success | Number of tensors in the output FlowFile. Currently always '1' | diff --git a/minifi_rust/extensions/minifi_tensor/src/low_level_processors/classify_output.rs b/minifi_rust/extensions/minifi_tensor/src/low_level_processors/classify_output.rs index cda39a52b4..be2c98e829 100644 --- a/minifi_rust/extensions/minifi_tensor/src/low_level_processors/classify_output.rs +++ b/minifi_rust/extensions/minifi_tensor/src/low_level_processors/classify_output.rs @@ -15,12 +15,15 @@ // specific language governing permissions and limitations // under the License. +use crate::low_level_processors::classify_output::classify_output_def::{ + CLASS_COUNT_ATTR, CLASS_TOP1_CONFIDENCE_ATTR, CLASS_TOP1_ID_ATTR, CLASS_TOP1_NAME_ATTR, +}; use crate::utils::score_activation::{ScoreActivation, SoftmaxTerms}; use crate::utils::tensor_helpers::{deserialize_tensors, tensor_as_f32, tensor_shape}; use classify_output_def::SUCCESS; pub(crate) use classify_output_def::{ CLASSIFY_OUTPUT_ATTRIBUTES, CONFIDENCE_THRESHOLD, LABEL_INDEX_OFFSET, LABELS_FILE_PATH, - OUTPUT_ATTRIBUTE_NAME, SCORE_ACTIVATION, SCORE_OUTPUT_INDEX, TOP_K, + MIME_TYPE_ATTR, OUTPUT_ATTRIBUTE_NAME, SCORE_ACTIVATION, SCORE_OUTPUT_INDEX, TOP_K, }; use minifi_native::macros::ComponentIdentifier; use minifi_native::{ @@ -123,7 +126,9 @@ impl ClassifyOutput { let score_floats = tensor_as_f32(&tensors, self.score_output_index).route_err_to_failure()?; if score_floats.is_empty() { - return Err(MinifiError::custom("Score tensor is empty; nothing to classify").into()); + return Err(ProcessError::route_to_failure( + "Score tensor is empty; nothing to classify", + )); } // A classifier head is a single score vector: shape [num_classes] or @@ -133,11 +138,10 @@ impl ClassifyOutput { // (`ImageToTensor` emits batch=1 today; this just enforces the contract.) let shape = tensor_shape(&tensors, self.score_output_index).route_err_to_failure()?; if shape.iter().rev().skip(1).any(|&d| d != 1) { - return Err(MinifiError::custom(format!( + return Err(ProcessError::route_to_failure(format!( "ClassifyOutput expects a single score vector (shape [num_classes] or \ [1, .., num_classes]); got {shape:?}. A batch dimension > 1 is not supported." - ))) - .route_err_to_failure(); + ))); } let finite: Vec<(usize, f32)> = score_floats @@ -194,15 +198,15 @@ impl ClassifyOutput { }; let mut transformed = TransformedFlowFile::new(&SUCCESS, content) - .with_attribute("mime.type", "application/json") - .with_attribute("class.count", predictions.len().to_string()); + .with_attribute(MIME_TYPE_ATTR.name, "application/json") + .with_attribute(CLASS_COUNT_ATTR.name, predictions.len().to_string()); if let Some(top) = predictions.first() { transformed = transformed - .with_attribute("class.top1.id", top.class_id.to_string()) - .with_attribute("class.top1.confidence", top.confidence.to_string()); + .with_attribute(CLASS_TOP1_ID_ATTR.name, top.class_id.to_string()) + .with_attribute(CLASS_TOP1_CONFIDENCE_ATTR.name, top.confidence.to_string()); if let Some(name) = &top.class_name { - transformed = transformed.with_attribute("class.top1.name", name.clone()); + transformed = transformed.with_attribute(CLASS_TOP1_NAME_ATTR.name, name.clone()); } } diff --git a/minifi_rust/extensions/minifi_tensor/src/low_level_processors/classify_output/classify_output_def.rs b/minifi_rust/extensions/minifi_tensor/src/low_level_processors/classify_output/classify_output_def.rs index 263c586295..7ea3c7fb9f 100644 --- a/minifi_rust/extensions/minifi_tensor/src/low_level_processors/classify_output/classify_output_def.rs +++ b/minifi_rust/extensions/minifi_tensor/src/low_level_processors/classify_output/classify_output_def.rs @@ -77,6 +77,12 @@ pub(crate) const LABEL_INDEX_OFFSET: Property = Property::new( ) .with_default("0"); +pub(crate) const OUTPUT_ATTRIBUTE_NAME: Property> = Property::new( + "Output attribute name", + "Specify the attribute to use as output, if not provided, the content is overridden instead.", +) +.supports_expression_language(); + pub(super) const SUCCESS: Relationship = Relationship { name: "success", description: "Classification completed. The flow file content is a JSON array of the Top K \ @@ -89,46 +95,39 @@ pub(super) const FAILURE: Relationship = Relationship { not be interpreted as f32 values.", }; -const MIME_TYPE_ATTR: OutputAttribute = OutputAttribute { +pub(crate) const MIME_TYPE_ATTR: OutputAttribute = OutputAttribute { name: "mime.type", relationships: &["success"], - description: "Always 'application/json' — the output payload is a JSON array of prediction \ - objects with fields class_id, confidence, and optional class_name.", + description: "If the \"Output attribute name\" is None, then the content will be overridden with the JSON array of objects with fields class_id, confidence, and optional class_name, and the mime type will be set to 'application/json'.", }; -const CLASS_COUNT_ATTR: OutputAttribute = OutputAttribute { +pub(crate) const CLASS_COUNT_ATTR: OutputAttribute = OutputAttribute { name: "class.count", relationships: &["success"], description: "Number of predictions retained after Top K selection and confidence filtering.", }; -const CLASS_TOP1_ID_ATTR: OutputAttribute = OutputAttribute { +pub(crate) const CLASS_TOP1_ID_ATTR: OutputAttribute = OutputAttribute { name: "class.top1.id", relationships: &["success"], description: "Numeric class ID of the highest-confidence prediction, when at least one \ prediction cleared the confidence threshold.", }; -const CLASS_TOP1_CONFIDENCE_ATTR: OutputAttribute = OutputAttribute { +pub(crate) const CLASS_TOP1_CONFIDENCE_ATTR: OutputAttribute = OutputAttribute { name: "class.top1.confidence", relationships: &["success"], description: "Confidence (post-activation) of the highest-confidence prediction, when at \ least one prediction cleared the confidence threshold.", }; -const CLASS_TOP1_NAME_ATTR: OutputAttribute = OutputAttribute { +pub(crate) const CLASS_TOP1_NAME_ATTR: OutputAttribute = OutputAttribute { name: "class.top1.name", relationships: &["success"], description: "Label of the highest-confidence prediction. Only present when 'Labels file \ path' was configured and at least one prediction cleared the threshold.", }; -pub(crate) const OUTPUT_ATTRIBUTE_NAME: Property> = Property::new( - "Output attribute name", - "Specify the attribute to use as output, if not provided, the content is overridden instead.", -) -.supports_expression_language(); - pub(crate) const CLASSIFY_OUTPUT_ATTRIBUTES: &[OutputAttribute] = &[ MIME_TYPE_ATTR, CLASS_COUNT_ATTR, diff --git a/minifi_rust/extensions/minifi_tensor/src/low_level_processors/filter_bounding_boxes.rs b/minifi_rust/extensions/minifi_tensor/src/low_level_processors/filter_bounding_boxes.rs index b4aa74aa01..3f96441f8e 100644 --- a/minifi_rust/extensions/minifi_tensor/src/low_level_processors/filter_bounding_boxes.rs +++ b/minifi_rust/extensions/minifi_tensor/src/low_level_processors/filter_bounding_boxes.rs @@ -25,7 +25,7 @@ use crate::utils::tensor_helpers::{deserialize_tensors, tensor_as_f32}; use filter_bounding_boxes_def::SUCCESS; pub(crate) use filter_bounding_boxes_def::{ BACKGROUND_CLASS_INDEX, BOX_FORMAT, BOX_OUTPUT_INDEX, CLASS_OUTPUT_INDEX, CONFIDENCE_THRESHOLD, - IOU_THRESHOLD, OUTPUT_ATTRIBUTE_NAME, SCORE_ACTIVATION, SCORE_OUTPUT_INDEX, + IOU_THRESHOLD, MIME_TYPE_ATTR, OUTPUT_ATTRIBUTE_NAME, SCORE_ACTIVATION, SCORE_OUTPUT_INDEX, }; use minifi_native::macros::{ComponentIdentifier, PropertyType}; use minifi_native::{ @@ -167,7 +167,7 @@ impl FilterBoundingBoxes { serde_json::to_string(&filtered_boxes).map_err(MinifiError::other)?, ) } else { - transformed = transformed.with_attribute("mime.type", "application/json"); + transformed = transformed.with_attribute(MIME_TYPE_ATTR.name, "application/json"); } Ok(transformed) } @@ -204,10 +204,9 @@ impl FilterBoundingBoxes { }; if !box_floats.len().is_multiple_of(4) { - return Err(MinifiError::custom( + return Err(ProcessError::route_to_failure( "Box tensor byte length is not a multiple of 16 (4 f32 per box)", - ) - .into()); + )); } let num_boxes = box_floats.len() / 4; if num_boxes == 0 { @@ -243,14 +242,13 @@ impl FilterBoundingBoxes { Some(class_index) => { let class_floats = tensor_as_f32(&tensors, class_index).route_err_to_failure()?; if score_floats.len() != num_boxes || class_floats.len() != num_boxes { - return Err(MinifiError::custom(format!( + return Err(ProcessError::route_to_failure(format!( "'Class output index' mode expects one score and one class id per box \ (num_boxes={}, scores={}, classes={})", num_boxes, score_floats.len(), class_floats.len() - )) - .into()); + ))); } trace!( logger, @@ -282,12 +280,11 @@ impl FilterBoundingBoxes { // Per-class score matrix: argmax over classes per box. None => { if !score_floats.len().is_multiple_of(num_boxes) { - return Err(MinifiError::custom(format!( + return Err(ProcessError::route_to_failure(format!( "Scores length ({}) not divisible by number of boxes ({})", score_floats.len(), num_boxes - )) - .into()); + ))); } let num_classes = score_floats.len() / num_boxes; trace!( @@ -331,7 +328,7 @@ fn resize_mode_from_attributes(context: &Context) -> Resi .ok() .flatten() .and_then(|raw| raw.parse::().ok()) - .unwrap_or(ResizeMode::Letterbox) + .unwrap_or(ResizeMode::Stretch) } impl FlowFileTransform for FilterBoundingBoxes { diff --git a/minifi_rust/extensions/minifi_tensor/src/low_level_processors/filter_bounding_boxes/filter_bounding_boxes_def.rs b/minifi_rust/extensions/minifi_tensor/src/low_level_processors/filter_bounding_boxes/filter_bounding_boxes_def.rs index ae5d071458..7ea3d35fb1 100644 --- a/minifi_rust/extensions/minifi_tensor/src/low_level_processors/filter_bounding_boxes/filter_bounding_boxes_def.rs +++ b/minifi_rust/extensions/minifi_tensor/src/low_level_processors/filter_bounding_boxes/filter_bounding_boxes_def.rs @@ -116,11 +116,10 @@ const OBJECT_COUNT_ATTR: OutputAttribute = OutputAttribute { description: "Number of bounding boxes retained after confidence filtering and NMS.", }; -const MIME_TYPE_ATTR: OutputAttribute = OutputAttribute { +pub(crate) const MIME_TYPE_ATTR: OutputAttribute = OutputAttribute { name: "mime.type", relationships: &["success"], - description: "Always 'application/json' — the output payload is a JSON array of objects with \ - fields class_id, confidence, x_min, y_min, x_max, y_max.", + description: "If the \"Output attribute name\" is None, then the content will be overridden with the JSON array of objects with fields class_id, confidence, x_min, y_min, x_max, y_max, and the mime type will be set to 'application/json'.", }; impl ProcessorDefinition for FilterBoundingBoxes { diff --git a/minifi_rust/extensions/minifi_tensor/src/low_level_processors/mod.rs b/minifi_rust/extensions/minifi_tensor/src/low_level_processors/mod.rs index 071791e20a..fe18ceddcb 100644 --- a/minifi_rust/extensions/minifi_tensor/src/low_level_processors/mod.rs +++ b/minifi_rust/extensions/minifi_tensor/src/low_level_processors/mod.rs @@ -1,3 +1,20 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + pub(crate) mod classify_output; pub(crate) mod filter_bounding_boxes; pub(crate) mod image_to_tensor; From d06f0c35740cc4e66f8210b2461f8bc7546d1876 Mon Sep 17 00:00:00 2001 From: Martin Zink Date: Fri, 25 Sep 2026 10:19:34 +0200 Subject: [PATCH 24/32] LineColor changes --- .../src/processors/draw_bounding_box.rs | 64 +++++++++++++------ 1 file changed, 46 insertions(+), 18 deletions(-) diff --git a/minifi_rust/extensions/minifi_tensor/src/processors/draw_bounding_box.rs b/minifi_rust/extensions/minifi_tensor/src/processors/draw_bounding_box.rs index 3512f1713e..073890c256 100644 --- a/minifi_rust/extensions/minifi_tensor/src/processors/draw_bounding_box.rs +++ b/minifi_rust/extensions/minifi_tensor/src/processors/draw_bounding_box.rs @@ -1,3 +1,20 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + use crate::utils::bounding_box::{BoundingBox, BoundingBoxes}; use image::Rgb; use minifi_native::macros::ComponentIdentifier; @@ -37,9 +54,9 @@ const LINE_THICKNESS: Property = Property::new( const LINE_COLOR: Property = Property::new( "Line color", - "Outline colour as '[R, G, B]' u8 channels (0-255).", + "Outline color as a hex string (e.g., '#ff00ff' or '#f0f')", ) -.with_default("[0, 255, 0]"); +.with_default("#00FF00"); #[derive(Debug, ComponentIdentifier)] pub(crate) struct DrawBoundingBox {} @@ -67,22 +84,24 @@ impl PropertyType for LineColor { type Output = Rgb; fn parse(s: &str) -> Result { - let clean_str = s.trim().trim_matches(|c| c == '[' || c == ']'); - let mut iter = clean_str.split(','); - - let mut next_channel = || { - iter.next() - .ok_or_else(|| MinifiError::validation("expected R,G,B color channels"))? - .trim() - .parse::() - .map_err(MinifiError::from) + let Some(hex) = s.trim().strip_prefix('#') else { + return Err(MinifiError::validation("Line color must start with #")); }; - Ok(Rgb::([ - next_channel()?, - next_channel()?, - next_channel()?, - ])) + let (r, g, b) = match hex.len() { + 6 => ( + u8::from_str_radix(&hex[0..2], 16).map_err(MinifiError::from)?, + u8::from_str_radix(&hex[2..4], 16).map_err(MinifiError::from)?, + u8::from_str_radix(&hex[4..6], 16).map_err(MinifiError::from)?, + ), + 3 => ( + u8::from_str_radix(&hex[0..1], 16).map_err(MinifiError::from)? * 17, + u8::from_str_radix(&hex[1..2], 16).map_err(MinifiError::from)? * 17, + u8::from_str_radix(&hex[2..3], 16).map_err(MinifiError::from)? * 17, + ), + _ => return Err(MinifiError::validation("expected 3 or 6 digit hex color")), + }; + Ok(Rgb::([r, g, b])) } } @@ -144,8 +163,9 @@ impl ProcessorDefinition for DrawBoundingBox { #[cfg(test)] mod tests { - use crate::processors::draw_bounding_box::{LINE_COLOR, LINE_THICKNESS}; - use minifi_native::{GetProperty, MockControllerServiceContext}; + use crate::processors::draw_bounding_box::{LINE_COLOR, LINE_THICKNESS, LineColor}; + use minifi_native::{GetProperty, MockControllerServiceContext, PropertyType}; + use std::assert_matches; #[test] fn test_parsing_colors() { @@ -155,6 +175,14 @@ mod tests { .expect("we should parse this"); let green = image::Rgb([0, 255, 0]); assert_eq!(default_color, green); + assert_matches!(LineColor::parse("[0,255,0]"), Err(_)); + assert_matches!(LineColor::parse("#00FG00"), Err(_)); + assert_matches!(LineColor::parse("#FFFFFFF"), Err(_)); + assert_matches!(LineColor::parse("FFFFFF"), Err(_)); + assert_matches!(LineColor::parse("#FFFF"), Err(_)); + assert_matches!(LineColor::parse("#0f0"), Ok(image::Rgb([0, 255, 0]))); + assert_matches!(LineColor::parse("#101010"), Ok(image::Rgb([16, 16, 16]))); + assert_matches!(LineColor::parse("#89A"), Ok(image::Rgb([0x88, 0x99, 0xAA]))); } #[test] From 01eb67e32490ae09943ad8fb9b9f374346944458 Mon Sep 17 00:00:00 2001 From: Martin Zink Date: Fri, 25 Sep 2026 10:37:10 +0200 Subject: [PATCH 25/32] labelsproperty --- .../low_level_processors/classify_output.rs | 69 +++++++++++-------- .../classify_output/classify_output_def.rs | 5 +- 2 files changed, 43 insertions(+), 31 deletions(-) diff --git a/minifi_rust/extensions/minifi_tensor/src/low_level_processors/classify_output.rs b/minifi_rust/extensions/minifi_tensor/src/low_level_processors/classify_output.rs index be2c98e829..842d552860 100644 --- a/minifi_rust/extensions/minifi_tensor/src/low_level_processors/classify_output.rs +++ b/minifi_rust/extensions/minifi_tensor/src/low_level_processors/classify_output.rs @@ -28,10 +28,10 @@ pub(crate) use classify_output_def::{ use minifi_native::macros::ComponentIdentifier; use minifi_native::{ Content, FlowFileTransform, GetAttribute, GetId, GetProperty, InputStream, Logger, MinifiError, - ProcessError, RouteErrorExt, Schedule, TransformedFlowFile, warn, + ProcessError, PropertyConstraints, PropertySchema, PropertyType, RouteErrorExt, Schedule, + TransformedFlowFile, warn, }; use serde::Serialize; -use std::path::Path; use tract::Tensor; mod classify_output_def; @@ -44,14 +44,25 @@ struct Prediction { class_name: Option, } -fn load_labels(path: &Path) -> Result, MinifiError> { - let content = std::fs::read_to_string(path).map_err(|e| { - MinifiError::custom(format!("Failed to read labels file '{:?}': {}", path, e)) - })?; - Ok(content - .lines() - .map(|line| line.trim().to_string()) - .collect()) +pub(crate) struct LabelsProperty {} + +impl PropertySchema for LabelsProperty { + const CONSTRAINT: Option = None; + const IS_REQUIRED: bool = false; +} + +impl PropertyType for LabelsProperty { + type Output = Vec; + + fn parse(s: &str) -> Result { + let content = std::fs::read_to_string(s).map_err(|e| { + MinifiError::custom(format!("Failed to read labels file '{:?}': {}", s, e)) + })?; + Ok(content + .lines() + .map(|line| line.trim().to_string()) + .collect()) + } } fn top_k(mut scored: Vec<(usize, f32)>, k: usize) -> Vec<(usize, f32)> { @@ -66,7 +77,7 @@ pub(crate) struct ClassifyOutput { score_output_index: usize, score_activation: ScoreActivation, confidence_threshold: f32, - labels: Vec, + labels: Option>, label_index_offset: usize, } @@ -86,16 +97,15 @@ impl Schedule for ClassifyOutput { let score_activation = context.get_property(&SCORE_ACTIVATION)?; let confidence_threshold = context.get_property(&CONFIDENCE_THRESHOLD)?; - let labels = match context.get_property(&LABELS_FILE_PATH)? { - Some(path) => load_labels(&path)?, - _ => Vec::new(), - }; + let labels = context.get_property(&LABELS_FILE_PATH)?; let label_index_offset = context.get_property(&LABEL_INDEX_OFFSET)?; - if !labels.is_empty() && label_index_offset >= labels.len() { + if let Some(l) = &labels + && label_index_offset >= l.len() + { return Err(MinifiError::validation(format!( "Label index offset ({}) must be smaller than the number of labels ({})", label_index_offset, - labels.len() + l.len() ))); } @@ -112,9 +122,10 @@ impl Schedule for ClassifyOutput { impl ClassifyOutput { fn label_for(&self, class_id: usize) -> Option { - self.labels - .get(class_id.checked_add(self.label_index_offset)?) - .cloned() + self.labels.as_ref().and_then(|l| { + l.get(class_id.checked_add(self.label_index_offset)?) + .cloned() + }) } pub(crate) fn classify<'a, Context: GetProperty + GetAttribute + GetId, LoggerImpl: Logger>( @@ -160,14 +171,16 @@ impl ClassifyOutput { if confidence >= self.confidence_threshold { let class_name = self.label_for(class_id); - if class_name.is_none() && !self.labels.is_empty() { + if class_name.is_none() + && let Some(l) = &self.labels + { warn!( logger, "No label for class id {} (offset {}, {} labels loaded); \ the labels file does not match the model's classes", class_id, self.label_index_offset, - self.labels.len() + l.len() ); } Some(Prediction { @@ -242,7 +255,7 @@ mod tests { score_output_index: 0, score_activation: activation, confidence_threshold: 0.0, - labels: Vec::new(), + labels: None, label_index_offset: 0, } } @@ -325,11 +338,11 @@ mod tests { #[test] fn test_transform_looks_up_labels() { let mut processor = make_processor(1, ScoreActivation::None); - processor.labels = vec![ + processor.labels = Some(vec![ "tench".into(), "goldfish".into(), "great_white_shark".into(), - ]; + ]); let scores = vec![0.1f32, 0.9, 0.5]; let context = context_with_scores(&scores); let mut stream = Cursor::new(build_payload(&scores)); @@ -346,12 +359,12 @@ mod tests { // Mirrors the ImageNet slim labels layout: line 0 is a dummy entry, so // model class 0 should resolve to labels[1] = "tench". let mut processor = make_processor(1, ScoreActivation::None); - processor.labels = vec![ + processor.labels = Some(vec![ "dummy".into(), "tench".into(), "goldfish".into(), "great_white_shark".into(), - ]; + ]); processor.label_index_offset = 1; let scores = vec![0.9f32, 0.1, 0.0]; // model class 0 wins let context = context_with_scores(&scores); @@ -369,7 +382,7 @@ mod tests { #[test] fn test_label_lookup_miss_warns_when_labels_are_configured() { let mut processor = make_processor(1, ScoreActivation::None); - processor.labels = vec!["dummy".into(), "tench".into()]; + processor.labels = Some(vec!["dummy".into(), "tench".into()]); processor.label_index_offset = 1; let scores = vec![0.1f32, 0.2, 0.9]; // model class 2 wins let context = context_with_scores(&scores); diff --git a/minifi_rust/extensions/minifi_tensor/src/low_level_processors/classify_output/classify_output_def.rs b/minifi_rust/extensions/minifi_tensor/src/low_level_processors/classify_output/classify_output_def.rs index 7ea3c7fb9f..29e8c084aa 100644 --- a/minifi_rust/extensions/minifi_tensor/src/low_level_processors/classify_output/classify_output_def.rs +++ b/minifi_rust/extensions/minifi_tensor/src/low_level_processors/classify_output/classify_output_def.rs @@ -15,12 +15,11 @@ // specific language governing permissions and limitations // under the License. -use super::{ClassifyOutput, ScoreActivation}; +use super::{ClassifyOutput, LabelsProperty, ScoreActivation}; use minifi_native::{ OutputAttribute, ProcessorDefinition, ProcessorInputRequirement, Property, PropertyDefinition, Relationship, property_definitions, }; -use std::path::PathBuf; pub(crate) const TOP_K: Property = Property::new( "Top K", @@ -59,7 +58,7 @@ pub(crate) const CONFIDENCE_THRESHOLD: Property = Property::new( .with_default("0.0") .supports_expression_language(); -pub(crate) const LABELS_FILE_PATH: Property> = Property::new( +pub(crate) const LABELS_FILE_PATH: Property> = Property::new( "Labels file path", "Optional path to a newline-separated labels file (line N = name of class N). \ Loaded once at service enable time. When set, each prediction in the output \ From 9bafc8d6aceb343d18eaba19cd1dae5ecb4f55fa Mon Sep 17 00:00:00 2001 From: Martin Zink Date: Fri, 25 Sep 2026 10:53:23 +0200 Subject: [PATCH 26/32] Apply batched suggestions from code review Co-authored-by: Ferenc Gerlits --- .../image_to_tensor/image_to_tensor_def.rs | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/minifi_rust/extensions/minifi_tensor/src/low_level_processors/image_to_tensor/image_to_tensor_def.rs b/minifi_rust/extensions/minifi_tensor/src/low_level_processors/image_to_tensor/image_to_tensor_def.rs index 704edc4ade..03c486fa39 100644 --- a/minifi_rust/extensions/minifi_tensor/src/low_level_processors/image_to_tensor/image_to_tensor_def.rs +++ b/minifi_rust/extensions/minifi_tensor/src/low_level_processors/image_to_tensor/image_to_tensor_def.rs @@ -61,7 +61,7 @@ pub(crate) const LETTERBOX_PAD_VALUE: Property = Property::new( pub(crate) const COLOR_FORMAT: Property = Property::new( "Color format", - "Colour space of the tensor fed to the model. RGB and BGR produce three-channel \ + "Color space of the tensor fed to the model. RGB and BGR produce three-channel \ tensors (channel order determined by the format); Grayscale produces a \ single-channel luma tensor.", ) @@ -77,7 +77,7 @@ pub(crate) const TENSOR_SHAPE_FORMAT: Property = Property::ne pub(crate) const MEAN: Property = Property::new( "Mean", - "Mean subtracted from each pixel before dividing by 'Standard Deviation'. Accepts \ + "Value subtracted from each pixel before dividing by 'Standard Deviation'. Accepts \ either a single value (broadcast to all channels) or three comma-separated \ values applied per channel in the order dictated by 'Color format'. Example: \ '0.485, 0.456, 0.406' for ImageNet-style RGB normalisation.", @@ -175,8 +175,7 @@ impl ProcessorDefinition for ImageToTensor { tensor suitable for feeding into a downstream inference processor such as \ InvokeTractModel. Supports RGB / BGR / Grayscale, CHW / HWC layouts, stretch or \ letterbox resizing, and scalar or per-channel mean/std normalisation. The output payload \ - is the raw little-endian f32 tensor; the 'tensors.len', 'tensor.{i}.shape' and 'tensor.{i}.dtype' attributes \ - describe its layout."; + is a single raw little-endian f32 tensor; the 'tensor.0.shape' attribute describes its layout."; const INPUT_REQUIREMENT: ProcessorInputRequirement = ProcessorInputRequirement::Required; const SUPPORTS_DYNAMIC_PROPERTIES: bool = false; const SUPPORTS_DYNAMIC_RELATIONSHIPS: bool = false; From 890708adceb0ff9033e544d0623f6a22bb7b3348 Mon Sep 17 00:00:00 2001 From: Martin Zink Date: Fri, 25 Sep 2026 13:08:52 +0200 Subject: [PATCH 27/32] #[strum(serialize = "Grayscale")] --- minifi_rust/extensions/minifi_tensor/minifi_tensor.md | 6 +++--- .../src/low_level_processors/image_to_tensor.rs | 1 + 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/minifi_rust/extensions/minifi_tensor/minifi_tensor.md b/minifi_rust/extensions/minifi_tensor/minifi_tensor.md index 7c10f2bc5c..50b4eed619 100644 --- a/minifi_rust/extensions/minifi_tensor/minifi_tensor.md +++ b/minifi_rust/extensions/minifi_tensor/minifi_tensor.md @@ -41,7 +41,7 @@ In the list below, the names of required properties appear in bold. Any other pr | Name | Default Value | Allowable Values | Description | |--------------------------|---------------|-----------------------------------------------|----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| -| **Color format** | RGB | RGB
BGR
GRAYSCALE | Colour space of the tensor fed to the model. RGB and BGR produce three-channel tensors (channel order determined by the format); Grayscale produces a single-channel luma tensor. | +| **Color format** | RGB | RGB
BGR
Grayscale | Colour space of the tensor fed to the model. RGB and BGR produce three-channel tensors (channel order determined by the format); Grayscale produces a single-channel luma tensor. | | **Confidence Threshold** | 0.0 | | Minimum confidence a class must reach to be included in the output JSON. Applied AFTER activation, so the units match the chosen activation (0.0..=1.0 for Softmax/Sigmoid, model-native for None). Set to 0.0 to always emit exactly Top K predictions.
**Supports Expression Language: true** | | **Label index offset** | 0 | | Offset added to the model's class ID when looking up a name in the labels file. Defaults to 0 (labels file line N = class N). Set to 1 for label files that start with a dummy/background entry — e.g. the ONNX MobileNetV2 model emits 1000 class scores while 'imagenet_slim_labels.txt' has 1001 lines (line 0 = 'dummy'), so class ID 653 maps to line 654 = 'military uniform'. | | Labels file path | | | Optional path to a newline-separated labels file (line N = name of class N). Loaded once at service enable time. When set, each prediction in the output JSON gains a 'class_name' field and the 'class.top1.name' flow file attribute is populated. Leave empty to emit numeric class IDs only. | @@ -132,7 +132,7 @@ In the list below, the names of required properties appear in bold. Any other pr | **Box format** | Xyxy | Xyxy
Yxyx
Cxcywh | Layout of the four floats per box in the box output tensor. Xyxy = [x_min, y_min, x_max, y_max] (SSD, MobileNet-SSD, most PyTorch exports). Yxyx = [y_min, x_min, y_max, x_max] (TensorFlow Object Detection API). Cxcywh = [cx, cy, w, h] (YOLOv3/5/8 raw output). | | **Box output index** | 1 | | Zero-based index of the model output tensor that holds box coordinates. Must differ from 'Score output index'. | | Class output index | | | Zero-based index of a model output tensor that holds one class id per box. Set this for detectors that emit boxes, per-box scores, and class ids as three separate parallel tensors, with NMS already folded into the graph (TensorFlow Object Detection API; YOLO / EfficientNMS 'end2end' exports). When set, 'Score output index' is read as one score per box (not a [boxes, classes] matrix) and no argmax is performed; the class id tensor may be integer- or float-typed. Leave empty for models that emit a per-class score matrix. | -| **Color format** | RGB | RGB
BGR
GRAYSCALE | Colour space of the tensor fed to the model. RGB and BGR produce three-channel tensors (channel order determined by the format); Grayscale produces a single-channel luma tensor. | +| **Color format** | RGB | RGB
BGR
Grayscale | Colour space of the tensor fed to the model. RGB and BGR produce three-channel tensors (channel order determined by the format); Grayscale produces a single-channel luma tensor. | | **Confidence Threshold** | 0.7 | | Minimum per-box class probability (0.0 to 1.0) required to keep a bounding box after applying the chosen 'Score activation'. Boxes below the threshold are discarded before NMS.
**Supports Expression Language: true** | | **IoU Threshold** | 0.45 | | Intersection-over-union cutoff used during non-maximum suppression. Boxes of the same class whose IoU with a higher-confidence peer exceeds this value are suppressed. Typical values: 0.45 (SSD/YOLO default), 0.5, 0.3 for stricter deduplication. | | **Letterbox pad value** | 0.0 | | Value written for padding pixels when 'Resize mode' is 'Letterbox'. This is a normalised value (post mean/std), so 0.0 corresponds to a neutral input for most networks. Ignored when 'Resize mode' is 'Stretch'. | @@ -237,7 +237,7 @@ In the list below, the names of required properties appear in bold. Any other pr | Name | Default Value | Allowable Values | Description | |-------------------------|---------------|-----------------------------------------------|----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| -| **Color format** | RGB | RGB
BGR
GRAYSCALE | Colour space of the tensor fed to the model. RGB and BGR produce three-channel tensors (channel order determined by the format); Grayscale produces a single-channel luma tensor. | +| **Color format** | RGB | RGB
BGR
Grayscale | Colour space of the tensor fed to the model. RGB and BGR produce three-channel tensors (channel order determined by the format); Grayscale produces a single-channel luma tensor. | | **Letterbox pad value** | 0.0 | | Value written for padding pixels when 'Resize mode' is 'Letterbox'. This is a normalised value (post mean/std), so 0.0 corresponds to a neutral input for most networks. Ignored when 'Resize mode' is 'Stretch'. | | Mean | 0.0 | | Mean subtracted from each pixel before dividing by 'Standard Deviation'. Accepts either a single value (broadcast to all channels) or three comma-separated values applied per channel in the order dictated by 'Color format'. Example: '0.485, 0.456, 0.406' for ImageNet-style RGB normalisation. | | **Pixel divisor** | 1.0 | | Divisor applied to raw u8 pixel values before subtracting 'Mean' and dividing by 'Standard Deviation'. Defaults to 1.0 (mean/std interpreted in [0, 255] pixel space, e.g. UltraFace's mean=127, std=128). Set to 255 to bring pixels into [0.0, 1.0] first so ImageNet-style mean/std values like '0.485, 0.456, 0.406' / '0.229, 0.224, 0.225' can be used directly, matching the PyTorch / torchvision / ONNX MobileNet convention. Must be non-zero. | diff --git a/minifi_rust/extensions/minifi_tensor/src/low_level_processors/image_to_tensor.rs b/minifi_rust/extensions/minifi_tensor/src/low_level_processors/image_to_tensor.rs index 376f5a2c2c..65d6e46608 100644 --- a/minifi_rust/extensions/minifi_tensor/src/low_level_processors/image_to_tensor.rs +++ b/minifi_rust/extensions/minifi_tensor/src/low_level_processors/image_to_tensor.rs @@ -68,6 +68,7 @@ impl From for image::imageops::FilterType { pub(crate) enum ColorFormat { Rgb, Bgr, + #[strum(serialize = "Grayscale")] Grayscale, } From cd390d7d686f00cad37abedeace955de436d756d Mon Sep 17 00:00:00 2001 From: Martin Zink Date: Fri, 25 Sep 2026 14:50:08 +0200 Subject: [PATCH 28/32] sigmoid tests in classify_output.rs --- .../low_level_processors/classify_output.rs | 65 +++++++++++++++++++ 1 file changed, 65 insertions(+) diff --git a/minifi_rust/extensions/minifi_tensor/src/low_level_processors/classify_output.rs b/minifi_rust/extensions/minifi_tensor/src/low_level_processors/classify_output.rs index 842d552860..f8731f85a4 100644 --- a/minifi_rust/extensions/minifi_tensor/src/low_level_processors/classify_output.rs +++ b/minifi_rust/extensions/minifi_tensor/src/low_level_processors/classify_output.rs @@ -319,6 +319,71 @@ mod tests { assert!(json.contains("\"class_id\":2")); } + #[test] + fn test_sigmoid_activation_scores_classes_independently() { + // Sigmoid squashes every logit on its own, so the confidences are not a + // distribution: here they sum to ~1.5. A softmax regression would force + // that sum to exactly 1.0. + let processor = make_processor(3, ScoreActivation::Sigmoid); + let logits = vec![0.0f32, 2.0, -2.0]; + let context = context_with_scores(&logits); + let mut stream = Cursor::new(build_payload(&logits)); + let result = processor + .transform(&context, &mut stream, &MockLogger::new()) + .expect("transform succeeds"); + + let json = result.into_bytes().unwrap().unwrap(); + let predictions: Vec = serde_json::from_slice(&json).unwrap(); + assert_eq!(predictions.len(), 3); + + let ids: Vec = predictions + .iter() + .map(|p| p["class_id"].as_u64().unwrap()) + .collect(); + assert_eq!( + ids, + vec![1, 0, 2], + "sigmoid is monotonic, so the ranking follows the raw logits" + ); + + let confidences: Vec = predictions + .iter() + .map(|p| p["confidence"].as_f64().unwrap() as f32) + .collect(); + let sigmoid_of = |logit: f32| 1.0f32 / (1.0 + (-logit).exp()); + assert_eq!(confidences[0], sigmoid_of(2.0)); + assert_eq!(confidences[1], 0.5, "sigmoid(0.0) is exactly one half"); + assert_eq!(confidences[2], sigmoid_of(-2.0)); + + let sum: f32 = confidences.iter().sum(); + assert!( + sum > 1.4, + "per-class sigmoid must not normalise across classes, got {sum}" + ); + } + + #[test] + fn test_sigmoid_activation_threshold_is_inclusive_at_one_half() { + // The threshold is applied after top_k, so top_k = 3 admits all three + // candidates and only the filter decides. sigmoid(0.0) == 0.5 passes the + // `>=` comparison; the negative logit falls below it. + let mut processor = make_processor(3, ScoreActivation::Sigmoid); + processor.confidence_threshold = 0.5; + let logits = vec![2.0f32, -0.5, 0.0]; + let context = context_with_scores(&logits); + let mut stream = Cursor::new(build_payload(&logits)); + let result = processor + .transform(&context, &mut stream, &MockLogger::new()) + .unwrap(); + + assert_eq!( + result.attribute("class.count").unwrap(), + "2", + "only the -0.5 logit should be filtered out" + ); + assert_eq!(result.attribute("class.top1.id").unwrap(), "0"); + } + #[test] fn test_transform_omits_class_name_when_labels_absent() { let processor = make_processor(1, ScoreActivation::None); From afb709d854097d470e43384c5378ad17125c7d28 Mon Sep 17 00:00:00 2001 From: Martin Zink Date: Fri, 25 Sep 2026 15:57:59 +0200 Subject: [PATCH 29/32] move test_resources into target folder --- minifi_rust/extensions/minifi_tensor/features/environment.py | 2 +- .../extensions/minifi_tensor/features/resources/.gitignore | 5 ----- 2 files changed, 1 insertion(+), 6 deletions(-) delete mode 100644 minifi_rust/extensions/minifi_tensor/features/resources/.gitignore diff --git a/minifi_rust/extensions/minifi_tensor/features/environment.py b/minifi_rust/extensions/minifi_tensor/features/environment.py index 47fb5a9f61..ccb2e5870c 100644 --- a/minifi_rust/extensions/minifi_tensor/features/environment.py +++ b/minifi_rust/extensions/minifi_tensor/features/environment.py @@ -109,7 +109,7 @@ def before_all(context): deps_build_path = os.path.normpath(os.path.join(dir_path, "../../../target/release/deps/")) add_extension_to_minifi_container("minifi_tensor", [build_path, deps_build_path], context) - context.tensor_resource_dir = os.path.join(dir_path, "resources") + context.tensor_resource_dir = os.path.join(dir_path, "../../../target/test_resources") os.makedirs(context.tensor_resource_dir, exist_ok=True) for name, asset in REMOTE_ASSETS.items(): asset.acquire(context.tensor_resource_dir, name) diff --git a/minifi_rust/extensions/minifi_tensor/features/resources/.gitignore b/minifi_rust/extensions/minifi_tensor/features/resources/.gitignore deleted file mode 100644 index 2537751f50..0000000000 --- a/minifi_rust/extensions/minifi_tensor/features/resources/.gitignore +++ /dev/null @@ -1,5 +0,0 @@ -# Test-time downloads (models, labels, images). Fetched on first test run by -# environment.py. Do not commit — several are megabytes and licensed by their -# upstream projects. -* -!.gitignore From 0a380d866d9f2ff19e4a96e40df4d23136bf5689 Mon Sep 17 00:00:00 2001 From: Martin Zink Date: Fri, 25 Sep 2026 15:58:15 +0200 Subject: [PATCH 30/32] folded label_index_offset into labels --- .../extensions/minifi_tensor/Cargo.toml | 1 + .../low_level_processors/classify_output.rs | 86 +++++++++---------- .../classify_output/classify_output_def.rs | 5 +- 3 files changed, 43 insertions(+), 49 deletions(-) diff --git a/minifi_rust/extensions/minifi_tensor/Cargo.toml b/minifi_rust/extensions/minifi_tensor/Cargo.toml index 0169229df9..5dfc8f9ea5 100644 --- a/minifi_rust/extensions/minifi_tensor/Cargo.toml +++ b/minifi_rust/extensions/minifi_tensor/Cargo.toml @@ -22,3 +22,4 @@ serde_json = "1.0.151" [dev-dependencies] minifi_native = { path = "../../minifi_native", features = ["test-utils"] } +tempfile = "3.27.0" diff --git a/minifi_rust/extensions/minifi_tensor/src/low_level_processors/classify_output.rs b/minifi_rust/extensions/minifi_tensor/src/low_level_processors/classify_output.rs index f8731f85a4..90a2500f1e 100644 --- a/minifi_rust/extensions/minifi_tensor/src/low_level_processors/classify_output.rs +++ b/minifi_rust/extensions/minifi_tensor/src/low_level_processors/classify_output.rs @@ -78,7 +78,6 @@ pub(crate) struct ClassifyOutput { score_activation: ScoreActivation, confidence_threshold: f32, labels: Option>, - label_index_offset: usize, } impl Schedule for ClassifyOutput { @@ -97,17 +96,27 @@ impl Schedule for ClassifyOutput { let score_activation = context.get_property(&SCORE_ACTIVATION)?; let confidence_threshold = context.get_property(&CONFIDENCE_THRESHOLD)?; - let labels = context.get_property(&LABELS_FILE_PATH)?; let label_index_offset = context.get_property(&LABEL_INDEX_OFFSET)?; - if let Some(l) = &labels - && label_index_offset >= l.len() - { - return Err(MinifiError::validation(format!( - "Label index offset ({}) must be smaller than the number of labels ({})", - label_index_offset, - l.len() - ))); - } + let labels = if let Some(mut labels) = context.get_property(&LABELS_FILE_PATH)? { + if let Some(dummy_indices) = label_index_offset { + if dummy_indices >= labels.len() { + return Err(MinifiError::validation(format!( + "Label index offset ({}) must be smaller than the number of labels ({})", + dummy_indices, + labels.len() + ))); + } + labels.drain(0..dummy_indices); + } + Some(labels) + } else { + if label_index_offset.is_some() { + return Err(MinifiError::validation( + "Label index offset is set without valid labels, either unset label index offset or provide valid labels file", + )); + } + None + }; Ok(Self { top_k, @@ -115,19 +124,11 @@ impl Schedule for ClassifyOutput { score_activation, confidence_threshold, labels, - label_index_offset, }) } } impl ClassifyOutput { - fn label_for(&self, class_id: usize) -> Option { - self.labels.as_ref().and_then(|l| { - l.get(class_id.checked_add(self.label_index_offset)?) - .cloned() - }) - } - pub(crate) fn classify<'a, Context: GetProperty + GetAttribute + GetId, LoggerImpl: Logger>( &self, context: &Context, @@ -170,16 +171,15 @@ impl ClassifyOutput { let confidence = self.score_activation.confidence(raw, softmax_terms); if confidence >= self.confidence_threshold { - let class_name = self.label_for(class_id); + let class_name = self.labels.as_ref().and_then(|l| l.get(class_id).cloned()); if class_name.is_none() && let Some(l) = &self.labels { warn!( logger, - "No label for class id {} (offset {}, {} labels loaded); \ + "No label for class id {} ({} labels loaded); \ the labels file does not match the model's classes", class_id, - self.label_index_offset, l.len() ); } @@ -248,6 +248,8 @@ mod tests { use super::*; use minifi_native::{LogLevel, MockLogger, MockProcessContext}; use std::io::Cursor; + use std::io::Write; + use tempfile::NamedTempFile; fn make_processor(top_k: usize, activation: ScoreActivation) -> ClassifyOutput { ClassifyOutput { @@ -256,7 +258,6 @@ mod tests { score_activation: activation, confidence_threshold: 0.0, labels: None, - label_index_offset: 0, } } @@ -420,35 +421,28 @@ mod tests { } #[test] - fn test_label_index_offset_shifts_lookup() { - // Mirrors the ImageNet slim labels layout: line 0 is a dummy entry, so - // model class 0 should resolve to labels[1] = "tench". - let mut processor = make_processor(1, ScoreActivation::None); - processor.labels = Some(vec![ - "dummy".into(), - "tench".into(), - "goldfish".into(), - "great_white_shark".into(), - ]); - processor.label_index_offset = 1; - let scores = vec![0.9f32, 0.1, 0.0]; // model class 0 wins - let context = context_with_scores(&scores); - let mut stream = Cursor::new(build_payload(&scores)); - let result = processor - .transform(&context, &mut stream, &MockLogger::new()) - .unwrap(); - assert_eq!( - result.attribute("class.top1.name").unwrap(), - "tench", - "class id 0 with offset 1 should land on labels[1]" + fn test_label_offset() { + let mut file = NamedTempFile::new().expect("Failed to create temp file"); + writeln!(file, "dummy").unwrap(); + writeln!(file, "tench").unwrap(); + writeln!(file, "goldfish").unwrap(); + writeln!(file, "great_white_shark").unwrap(); + let mut mock_context = MockProcessContext::default(); + mock_context.properties.insert( + LABELS_FILE_PATH.name().to_string(), + file.path().to_string_lossy(), ); + mock_context + .properties + .insert(LABEL_INDEX_OFFSET.name().to_string(), "1"); + let scheduled = ClassifyOutput::schedule(&mock_context, &MockLogger::new()).unwrap(); + assert_eq!(3, scheduled.labels.unwrap().len()); } #[test] fn test_label_lookup_miss_warns_when_labels_are_configured() { let mut processor = make_processor(1, ScoreActivation::None); - processor.labels = Some(vec!["dummy".into(), "tench".into()]); - processor.label_index_offset = 1; + processor.labels = Some(vec!["abc".into(), "tench".into()]); let scores = vec![0.1f32, 0.2, 0.9]; // model class 2 wins let context = context_with_scores(&scores); let mut stream = Cursor::new(build_payload(&scores)); diff --git a/minifi_rust/extensions/minifi_tensor/src/low_level_processors/classify_output/classify_output_def.rs b/minifi_rust/extensions/minifi_tensor/src/low_level_processors/classify_output/classify_output_def.rs index 29e8c084aa..b28cc8241a 100644 --- a/minifi_rust/extensions/minifi_tensor/src/low_level_processors/classify_output/classify_output_def.rs +++ b/minifi_rust/extensions/minifi_tensor/src/low_level_processors/classify_output/classify_output_def.rs @@ -66,15 +66,14 @@ pub(crate) const LABELS_FILE_PATH: Property> = Property:: is populated. Leave empty to emit numeric class IDs only.", ); -pub(crate) const LABEL_INDEX_OFFSET: Property = Property::new( +pub(crate) const LABEL_INDEX_OFFSET: Property> = Property::new( "Label index offset", "Offset added to the model's class ID when looking up a name in the labels file. \ Defaults to 0 (labels file line N = class N). Set to 1 for label files that \ start with a dummy/background entry — e.g. the ONNX MobileNetV2 model emits \ 1000 class scores while 'imagenet_slim_labels.txt' has 1001 lines (line 0 = \ 'dummy'), so class ID 653 maps to line 654 = 'military uniform'.", -) -.with_default("0"); +); pub(crate) const OUTPUT_ATTRIBUTE_NAME: Property> = Property::new( "Output attribute name", From 281338c4e816ee8675a71a406e66db4486b453ef Mon Sep 17 00:00:00 2001 From: Martin Zink Date: Fri, 25 Sep 2026 16:01:24 +0200 Subject: [PATCH 31/32] docs update (also adding missing RunLlamacpp docs) --- CONTROLLERS.md | 18 +- PROCESSORS.md | 190 ++++++++++++++++++ .../extensions/minifi_tensor/minifi_tensor.md | 18 +- 3 files changed, 216 insertions(+), 10 deletions(-) diff --git a/CONTROLLERS.md b/CONTROLLERS.md index 28fce43e1e..2b09fee53b 100644 --- a/CONTROLLERS.md +++ b/CONTROLLERS.md @@ -31,6 +31,7 @@ limitations under the License. - [RocksDbStateStorage](#RocksDbStateStorage) - [SmbConnectionControllerService](#SmbConnectionControllerService) - [SSLContextService](#SSLContextService) +- [TractModelService](#TractModelService) - [UpdatePolicyControllerService](#UpdatePolicyControllerService) - [VolatileMapStateStorage](#VolatileMapStateStorage) - [XMLReader](#XMLReader) @@ -324,6 +325,22 @@ In the list below, the names of required properties appear in bold. Any other pr | Use System Cert Store | false | true
false | Whether to use the certificates in the OS's certificate store | +## TractModelService + +### Description + +Provides a shared, CPU-optimized neural network for inference. Supports ONNX (`.onnx`) and NNEF (directory or tarball) models; the format can be auto-detected from the file extension or set explicitly. + +### Properties + +In the list below, the names of required properties appear in bold. Any other properties (not in bold) are considered optional. The table also indicates any default values, and whether a property supports the NiFi Expression Language. + +| Name | Default Value | Allowable Values | Description | +|---------------------|---------------|------------------------|-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| **Model File Path** | | | Absolute path to the model on the edge device. For ONNX this is a `.onnx` file; for NNEF this is a `.nnef.tgz` archive, a `.nnef` tarball, or the root directory of an unpacked NNEF model. The model is loaded, parsed, and compiled for the host CPU once when the controller service is enabled; subsequent inference calls reuse the compiled runnable. | +| **Model format** | Auto | Auto
Onnx
Nnef | Format of the file/directory referenced by 'Model File Path'. 'Auto' picks Onnx when the path ends in `.onnx` and Nnef when it ends in `.nnef`, `.nnef.tgz`, `.nnef.tar`, `.nnef.tar.gz`, or points at a directory. Set explicitly when the path uses a non-standard extension. | + + ## UpdatePolicyControllerService ### Description @@ -393,4 +410,3 @@ In the list below, the names of required properties appear in bold. Any other pr | **Pretty Print XML** | false | true
false | Specifies whether or not the XML should be pretty printed | | **Name of Record Tag** | | | Specifies the name of the XML record tag wrapping the record fields. | | **Name of Root Tag** | | | Specifies the name of the XML root tag wrapping the record set. | - diff --git a/PROCESSORS.md b/PROCESSORS.md index 4c7705b2a6..941a56c8b9 100644 --- a/PROCESSORS.md +++ b/PROCESSORS.md @@ -19,6 +19,7 @@ limitations under the License. - [ApplyTemplate](#ApplyTemplate) - [AttributeRollingWindow](#AttributeRollingWindow) - [AttributesToJSON](#AttributesToJSON) +- [ClassifyImage](#ClassifyImage) - [CollectKubernetesPodMetrics](#CollectKubernetesPodMetrics) - [CompressContent](#CompressContent) - [ConsumeJournald](#ConsumeJournald) @@ -31,6 +32,8 @@ limitations under the License. - [DeleteAzureDataLakeStorage](#DeleteAzureDataLakeStorage) - [DeleteGCSObject](#DeleteGCSObject) - [DeleteS3Object](#DeleteS3Object) +- [DetectObject](#DetectObject) +- [DrawBoundingBox](#DrawBoundingBox) - [EvaluateJsonPath](#EvaluateJsonPath) - [ExecuteProcess](#ExecuteProcess) - [ExecuteScript](#ExecuteScript) @@ -95,6 +98,7 @@ limitations under the License. - [RetryFlowFile](#RetryFlowFile) - [RouteOnAttribute](#RouteOnAttribute) - [RouteText](#RouteText) +- [RunLlamaCppInference](#RunLlamaCppInference) - [SegmentContent](#SegmentContent) - [SplitContent](#SplitContent) - [SplitJson](#SplitJson) @@ -214,6 +218,76 @@ In the list below, the names of required properties appear in bold. Any other pr | success | All FlowFiles received are routed to success | +## ClassifyImage + +### Description + +Runs a full image-classification pass in a single processor: decodes the image from the flow file content, resizes and normalises it into an input tensor, runs one inference against the compiled model owned by the referenced TractModelService, and post-processes the score vector (score activation, Top-K selection, confidence filtering, optional label lookup) into predictions. Collapses the ImageToTensor -> InvokeTractModel -> ClassifyOutput chain into one node. The flow file content is left unchanged (the original image); the Top-K classifications are written as a JSON array to the configured output attribute. + +### Properties + +In the list below, the names of required properties appear in bold. Any other properties (not in bold) are considered optional. The table also indicates any default values, and whether a property supports the NiFi Expression Language. + +| Name | Default Value | Allowable Values | Description | +|--------------------------|---------------|-----------------------------------------------|----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| **Target width** | | | Width in pixels the decoded image is resized to before normalisation and inference. | +| **Target height** | | | Height in pixels the decoded image is resized to before normalisation and inference. | +| **Resize filter** | Bilinear | Nearest
Bilinear
Bicubic
Lanczos3 | Interpolation filter applied when resizing the decoded image. Nearest is fastest but blocky; Bilinear is a good default; Bicubic and Lanczos3 are higher-quality but slower. | +| **Resize mode** | Stretch | Stretch
Letterbox | How the source image is fitted into the target dimensions. 'Stretch' scales each axis independently, distorting aspect ratio. 'Letterbox' preserves aspect ratio and pads the remaining border with 'Letterbox pad value' (applied in normalised output space). | +| **Letterbox pad value** | 0.0 | | Value written for padding pixels when 'Resize mode' is 'Letterbox'. This is a normalised value (post mean/std), so 0.0 corresponds to a neutral input for most networks. Ignored when 'Resize mode' is 'Stretch'. | +| **Color format** | RGB | RGB
BGR
Grayscale | Color space of the tensor fed to the model. RGB and BGR produce three-channel tensors (channel order determined by the format); Grayscale produces a single-channel luma tensor. | +| **Tensor shape format** | CHW | CHW
HWC | Memory layout of the tensor fed to the model. CHW (channels-first) is typical for PyTorch/ONNX detectors. HWC (channels-last) matches TensorFlow/TFLite. Ignored for Grayscale (always effectively 1xHxW). | +| Mean | 0.0 | | Value subtracted from each pixel before dividing by 'Standard Deviation'. Accepts either a single value (broadcast to all channels) or three comma-separated values applied per channel in the order dictated by 'Color format'. Example: '0.485, 0.456, 0.406' for ImageNet-style RGB normalisation. | +| Standard Deviation | 255.0 | | Divisor applied after subtracting 'Mean'. Accepts a single value (broadcast) or three comma-separated values (per channel). Must be non-zero. Example: '255.0' to scale u8 pixels into [0.0, 1.0]; '0.229, 0.224, 0.225' for ImageNet. | +| **Pixel divisor** | 1.0 | | Divisor applied to raw u8 pixel values before subtracting 'Mean' and dividing by 'Standard Deviation'. Defaults to 1.0 (mean/std interpreted in [0, 255] pixel space, e.g. UltraFace's mean=127, std=128). Set to 255 to bring pixels into [0.0, 1.0] first so ImageNet-style mean/std values like '0.485, 0.456, 0.406' / '0.229, 0.224, 0.225' can be used directly, matching the PyTorch / torchvision / ONNX MobileNet convention. Must be non-zero. | +| **Tract model service** | | | Reference to a TractModelService controller service. The referenced service owns the compiled model (ONNX or NNEF) that will be evaluated for each incoming flow file. | +| **Top K** | 5 | | Number of highest-scoring classes to include in the output JSON, in descending order of confidence. Values above the total class count are clamped. Set to 1 for pure top-1 classification. | +| **Score output index** | 0 | | Zero-based index of the model output tensor that holds classification scores. The processor slices the concatenated payload from InvokeTractModel according to the 'tensor.N.bytes' attributes. Almost always 0 for single-head classifiers. | +| **Score activation** | Softmax | Softmax
Sigmoid
None | Activation applied to the raw score vector before ranking. Softmax = mutually-exclusive classes (ImageNet-trained ResNet/MobileNet/EfficientNet raw logits). Sigmoid = independent classes (multi-label classifiers). None = the model already emits probabilities/scores; rank the raw values. | +| **Confidence Threshold** | 0.0 | | Minimum confidence a class must reach to be included in the output JSON. Applied AFTER activation, so the units match the chosen activation (0.0..=1.0 for Softmax/Sigmoid, model-native for None). Set to 0.0 to always emit exactly Top K predictions.
**Supports Expression Language: true** | +| Labels file path | | | Optional path to a newline-separated labels file (line N = name of class N). Loaded once at service enable time. When set, each prediction in the output JSON gains a 'class_name' field and the 'class.top1.name' flow file attribute is populated. Leave empty to emit numeric class IDs only. | +| Label index offset | | | Offset added to the model's class ID when looking up a name in the labels file. Defaults to 0 (labels file line N = class N). Set to 1 for label files that start with a dummy/background entry — e.g. the ONNX MobileNetV2 model emits 1000 class scores while 'imagenet_slim_labels.txt' has 1001 lines (line 0 = 'dummy'), so class ID 653 maps to line 654 = 'military uniform'. | +| Output attribute name | | | Specify the attribute to use as output, if not provided, the content is overridden instead.
**Supports Expression Language: true** | + +### Relationships + +| Name | Description | +|---------|-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| success | Inference and post-processing completed. The flow file content is the original, unchanged image; the classifications are written to the configured output attribute as a JSON array (may be empty). | +| failure | The image could not be decoded, the input tensor could not be built, the model failed to run, or the model outputs could not be interpreted as classification. | + +### Output Attributes + +| Attribute | Relationship | Description | +|-----------------------|--------------|----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| mime.type | success | If the "Output attribute name" is None, then the content will be overridden with the JSON array of objects with fields class_id, confidence, and optional class_name, and the mime type will be set to 'application/json'. | +| class.count | success | Number of predictions retained after Top K selection and confidence filtering. | +| class.top1.id | success | Numeric class ID of the highest-confidence prediction, when at least one prediction cleared the confidence threshold. | +| class.top1.confidence | success | Confidence (post-activation) of the highest-confidence prediction, when at least one prediction cleared the confidence threshold. | +| class.top1.name | success | Label of the highest-confidence prediction. Only present when 'Labels file path' was configured and at least one prediction cleared the threshold. | + + +## CollectKubernetesPodMetrics + +### Description + +A processor which collects pod metrics when MiNiFi is run inside Kubernetes. + +### Properties + +In the list below, the names of required properties appear in bold. Any other properties (not in bold) are considered optional. The table also indicates any default values, and whether a property supports the NiFi Expression Language. + +| Name | Default Value | Allowable Values | Description | +|-----------------------------------|---------------|------------------|------------------------------------------------------------| +| **Kubernetes Controller Service** | | | Controller service which provides Kubernetes functionality | + +### Relationships + +| Name | Description | +|---------|------------------------------------------------| +| success | All flow files produced are routed to Success. | + + ## CompressContent ### Description @@ -594,6 +668,78 @@ In the list below, the names of required properties appear in bold. Any other pr | failure | FlowFiles are routed to failure relationship | +## DetectObject + +### Description + +Runs a full object-detection pass in a single processor: decodes the image from the flow file content, resizes and normalises it into an input tensor, runs one inference against the compiled model owned by the referenced TractModelService, and post-processes the model outputs (score activation, confidence filtering, box decoding, per-class non-maximum suppression) into bounding boxes. Collapses the ImageToTensor -> InvokeTractModel -> FilterBoundingBoxes chain into one node. The flow file content is left unchanged (the original image); the detected boxes are written as a JSON array to the configured output attribute so a downstream DrawBoundingBox can annotate the image. + +### Properties + +In the list below, the names of required properties appear in bold. Any other properties (not in bold) are considered optional. The table also indicates any default values, and whether a property supports the NiFi Expression Language. + +| Name | Default Value | Allowable Values | Description | +|--------------------------|---------------|-----------------------------------------------|----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| **Target width** | | | Width in pixels the decoded image is resized to before normalisation and inference. | +| **Target height** | | | Height in pixels the decoded image is resized to before normalisation and inference. | +| **Resize filter** | Bilinear | Nearest
Bilinear
Bicubic
Lanczos3 | Interpolation filter applied when resizing the decoded image. Nearest is fastest but blocky; Bilinear is a good default; Bicubic and Lanczos3 are higher-quality but slower. | +| **Resize mode** | Stretch | Stretch
Letterbox | How the source image is fitted into the target dimensions. 'Stretch' scales each axis independently, distorting aspect ratio. 'Letterbox' preserves aspect ratio and pads the remaining border with 'Letterbox pad value' (applied in normalised output space). | +| **Letterbox pad value** | 0.0 | | Value written for padding pixels when 'Resize mode' is 'Letterbox'. This is a normalised value (post mean/std), so 0.0 corresponds to a neutral input for most networks. Ignored when 'Resize mode' is 'Stretch'. | +| **Color format** | RGB | RGB
BGR
Grayscale | Color space of the tensor fed to the model. RGB and BGR produce three-channel tensors (channel order determined by the format); Grayscale produces a single-channel luma tensor. | +| **Tensor shape format** | CHW | CHW
HWC | Memory layout of the tensor fed to the model. CHW (channels-first) is typical for PyTorch/ONNX detectors. HWC (channels-last) matches TensorFlow/TFLite. Ignored for Grayscale (always effectively 1xHxW). | +| Mean | 0.0 | | Value subtracted from each pixel before dividing by 'Standard Deviation'. Accepts either a single value (broadcast to all channels) or three comma-separated values applied per channel in the order dictated by 'Color format'. Example: '0.485, 0.456, 0.406' for ImageNet-style RGB normalisation. | +| Standard Deviation | 255.0 | | Divisor applied after subtracting 'Mean'. Accepts a single value (broadcast) or three comma-separated values (per channel). Must be non-zero. Example: '255.0' to scale u8 pixels into [0.0, 1.0]; '0.229, 0.224, 0.225' for ImageNet. | +| **Pixel divisor** | 1.0 | | Divisor applied to raw u8 pixel values before subtracting 'Mean' and dividing by 'Standard Deviation'. Defaults to 1.0 (mean/std interpreted in [0, 255] pixel space, e.g. UltraFace's mean=127, std=128). Set to 255 to bring pixels into [0.0, 1.0] first so ImageNet-style mean/std values like '0.485, 0.456, 0.406' / '0.229, 0.224, 0.225' can be used directly, matching the PyTorch / torchvision / ONNX MobileNet convention. Must be non-zero. | +| **Tract model service** | | | Reference to a TractModelService controller service. The referenced service owns the compiled model (ONNX or NNEF) that will be evaluated for each incoming flow file. | +| **Confidence Threshold** | 0.7 | | Minimum per-box class probability (0.0 to 1.0) required to keep a bounding box after applying the chosen 'Score activation'. Boxes below the threshold are discarded before NMS.
**Supports Expression Language: true** | +| **IoU Threshold** | 0.45 | | Intersection-over-union cutoff used during non-maximum suppression. Boxes of the same class whose IoU with a higher-confidence peer exceeds this value are suppressed. Typical values: 0.45 (SSD/YOLO default), 0.5, 0.3 for stricter deduplication. | +| **Score output index** | 0 | | Zero-based index of the model output tensor that holds classification scores. The processor slices the concatenated payload from InvokeTractModel according to the 'tensor.N.bytes' attributes. | +| **Box output index** | 1 | | Zero-based index of the model output tensor that holds box coordinates. Must differ from 'Score output index'. | +| Class output index | | | Zero-based index of a model output tensor that holds one class id per box. Set this for detectors that emit boxes, per-box scores, and class ids as three separate parallel tensors, with NMS already folded into the graph (TensorFlow Object Detection API; YOLO / EfficientNMS 'end2end' exports). When set, 'Score output index' is read as one score per box (not a [boxes, classes] matrix) and no argmax is performed; the class id tensor may be integer- or float-typed. Leave empty for models that emit a per-class score matrix. | +| **Box format** | Xyxy | Xyxy
Yxyx
Cxcywh | Layout of the four floats per box in the box output tensor. Xyxy = [x_min, y_min, x_max, y_max] (SSD, MobileNet-SSD, most PyTorch exports). Yxyx = [y_min, x_min, y_max, x_max] (TensorFlow Object Detection API). Cxcywh = [cx, cy, w, h] (YOLOv3/5/8 raw output). | +| **Score activation** | Softmax | Softmax
Sigmoid
None | Activation applied to raw per-class scores before selecting the winning class. Softmax = mutually-exclusive classes (SSD/MobileNet-SSD raw logits). Sigmoid = independent classes (YOLOv5/v8 style). None = the model already emits probabilities/scores; use raw argmax with the raw score as confidence. | +| Background class index | | | Index of the 'background / no-object' class. Boxes whose winning class equals this index are dropped. In score-matrix mode this is only honoured when the score tensor has more than one class per box; in 'Class output index' mode it is matched against each box's class id. | +| Output attribute name | | | Specify the attribute to use as output, if not provided, the content is overridden instead.
**Supports Expression Language: true** | + +### Relationships + +| Name | Description | +|---------|----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| success | Inference and post-processing completed. The flow file content is the original, unchanged image; the detected boxes are written to the configured output attribute as a JSON array (may be empty). | +| failure | The image could not be decoded, the input tensor could not be built, the model failed to run, or the model outputs could not be interpreted as scores + boxes. | + +### Output Attributes + +| Attribute | Relationship | Description | +|-------------------------|--------------|------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| object.count | success | Number of bounding boxes retained after confidence filtering and NMS. | +| | success | JSON array of the surviving bounding boxes (fields class_id, confidence, x_min, y_min, x_max, y_max; coordinates normalised to [0,1] against the original image). The attribute name is configurable via the 'Output attribute name' property. | + + +## DrawBoundingBox + +### Description + +Decodes the image from the flow file content, draws each bounding box supplied via the 'Bounding boxes' property onto it, and re-encodes the annotated image as PNG. Pair with an upstream DetectObject / FilterBoundingBoxes to visualise detections. + +### Properties + +In the list below, the names of required properties appear in bold. Any other properties (not in bold) are considered optional. The table also indicates any default values, and whether a property supports the NiFi Expression Language. + +| Name | Default Value | Allowable Values | Description | +|--------------------|---------------------|------------------|-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| Bounding boxes | ${enrichment.value} | | JSON array of bounding boxes to draw onto the image (fields class_id, confidence, x_min, y_min, x_max, y_max; coordinates normalised to [0,1] against the image). Typically the attribute produced by an upstream DetectObject or FilterBoundingBoxes processor.
**Supports Expression Language: true** | +| Line color | #00FF00 | | Outline color as a hex string (e.g., '#ff00ff' or '#f0f') | +| **Line thickness** | 5 | | Thickness in pixels of the drawn box outline. | + +### Relationships + +| Name | Description | +|---------|------------------------------------------------------------| +| success | Flowfiles are routed here after drawing the bounding boxes | +| failure | Invalid FlowFiles are routed here | + + ## EvaluateJsonPath ### Description @@ -2948,6 +3094,50 @@ In the list below, the names of required properties appear in bold. Any other pr | RouteText.Group | | The value captured by all capturing groups in the 'Grouping Regular Expression' property. If this property is not set, this attribute will not be added. | +## RunLlamaCppInference + +### Description + +LlamaCpp processor to use llama.cpp library for running language model inference. The inference will be based on the System Prompt and the Prompt property values, together with the content of the incoming flow file. In the Prompt, the content of the incoming flow file can be referred to as 'the input data' or 'the flow file content'. + +### Properties + +In the list below, the names of required properties appear in bold. Any other properties (not in bold) are considered optional. The table also indicates any default values, and whether a property supports the NiFi Expression Language. + +| Name | Default Value | Allowable Values | Description | +|----------------------------------|-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|------------------|-------------------------------------------------------------------------------------------------------------------------| +| **Model Path** | | | The filesystem path of the model file in gguf format. | +| Output Attribute Name | | | Specify the attribute to use as output, if not provided, the content is overridden instead. | +| MultiModal Model Path | | | The filesystem path of the multimodal model (visual, audio) file in gguf format. | +| Temperature | 0.8 | | The temperature to use for sampling. | +| Top K | 40 | | Limit the next token selection to the K most probable tokens. Set <= 0 value to use vocab size. | +| Top P | 0.9 | | Limit the next token selection to a subset of tokens with a cumulative probability above a threshold P. 1.0 = disabled. | +| Min P | | | Sets a minimum base probability threshold for token selection. 0.0 = disabled. | +| **Min Keep** | 0 | | If greater than 0, force samplers to return N possible tokens at minimum. | +| **Text Context Size** | 4096 | | Size of the text context, use 0 to use size set in model. | +| **Logical Maximum Batch Size** | 2048 | | Logical maximum batch size that can be submitted to the llama.cpp decode function. | +| **Physical Maximum Batch Size** | 512 | | Physical maximum batch size. | +| **Max Number Of Sequences** | 1 | | Maximum number of sequences (i.e. distinct states for recurrent models). | +| **Threads For Generation** | 4 | | Number of threads to use for generation. | +| **Threads For Batch Processing** | 4 | | Number of threads to use for batch processing. | +| Prompt | | | The user prompt for the inference.
**Supports Expression Language: true** | +| System Prompt | You are a helpful assistant. You are given a question with some possible input data otherwise called flow file content. You are expected to generate a response based on the question and the input data. | | The system prompt for the inference. | + +### Relationships + +| Name | Description | +|---------|----------------------------------| +| success | Generated results from the model | +| failure | Generation failed | + +### Output Attributes + +| Attribute | Relationship | Description | +|------------------------------|--------------|------------------------------------------------| +| llamacpp.time.to.first.token | success | Time to first token generated in milliseconds. | +| llamacpp.tokens.per.second | success | Tokens generated per second. | + + ## TailEventLog ### Description diff --git a/minifi_rust/extensions/minifi_tensor/minifi_tensor.md b/minifi_rust/extensions/minifi_tensor/minifi_tensor.md index 50b4eed619..1feeaca86f 100644 --- a/minifi_rust/extensions/minifi_tensor/minifi_tensor.md +++ b/minifi_rust/extensions/minifi_tensor/minifi_tensor.md @@ -41,12 +41,12 @@ In the list below, the names of required properties appear in bold. Any other pr | Name | Default Value | Allowable Values | Description | |--------------------------|---------------|-----------------------------------------------|----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| -| **Color format** | RGB | RGB
BGR
Grayscale | Colour space of the tensor fed to the model. RGB and BGR produce three-channel tensors (channel order determined by the format); Grayscale produces a single-channel luma tensor. | +| **Color format** | RGB | RGB
BGR
Grayscale | Color space of the tensor fed to the model. RGB and BGR produce three-channel tensors (channel order determined by the format); Grayscale produces a single-channel luma tensor. | | **Confidence Threshold** | 0.0 | | Minimum confidence a class must reach to be included in the output JSON. Applied AFTER activation, so the units match the chosen activation (0.0..=1.0 for Softmax/Sigmoid, model-native for None). Set to 0.0 to always emit exactly Top K predictions.
**Supports Expression Language: true** | -| **Label index offset** | 0 | | Offset added to the model's class ID when looking up a name in the labels file. Defaults to 0 (labels file line N = class N). Set to 1 for label files that start with a dummy/background entry — e.g. the ONNX MobileNetV2 model emits 1000 class scores while 'imagenet_slim_labels.txt' has 1001 lines (line 0 = 'dummy'), so class ID 653 maps to line 654 = 'military uniform'. | +| Label index offset | | | Offset added to the model's class ID when looking up a name in the labels file. Defaults to 0 (labels file line N = class N). Set to 1 for label files that start with a dummy/background entry — e.g. the ONNX MobileNetV2 model emits 1000 class scores while 'imagenet_slim_labels.txt' has 1001 lines (line 0 = 'dummy'), so class ID 653 maps to line 654 = 'military uniform'. | | Labels file path | | | Optional path to a newline-separated labels file (line N = name of class N). Loaded once at service enable time. When set, each prediction in the output JSON gains a 'class_name' field and the 'class.top1.name' flow file attribute is populated. Leave empty to emit numeric class IDs only. | | **Letterbox pad value** | 0.0 | | Value written for padding pixels when 'Resize mode' is 'Letterbox'. This is a normalised value (post mean/std), so 0.0 corresponds to a neutral input for most networks. Ignored when 'Resize mode' is 'Stretch'. | -| Mean | 0.0 | | Mean subtracted from each pixel before dividing by 'Standard Deviation'. Accepts either a single value (broadcast to all channels) or three comma-separated values applied per channel in the order dictated by 'Color format'. Example: '0.485, 0.456, 0.406' for ImageNet-style RGB normalisation. | +| Mean | 0.0 | | Value subtracted from each pixel before dividing by 'Standard Deviation'. Accepts either a single value (broadcast to all channels) or three comma-separated values applied per channel in the order dictated by 'Color format'. Example: '0.485, 0.456, 0.406' for ImageNet-style RGB normalisation. | | Output attribute name | | | Specify the attribute to use as output, if not provided, the content is overridden instead.
**Supports Expression Language: true** | | **Pixel divisor** | 1.0 | | Divisor applied to raw u8 pixel values before subtracting 'Mean' and dividing by 'Standard Deviation'. Defaults to 1.0 (mean/std interpreted in [0, 255] pixel space, e.g. UltraFace's mean=127, std=128). Set to 255 to bring pixels into [0.0, 1.0] first so ImageNet-style mean/std values like '0.485, 0.456, 0.406' / '0.229, 0.224, 0.225' can be used directly, matching the PyTorch / torchvision / ONNX MobileNet convention. Must be non-zero. | | **Resize filter** | Bilinear | Nearest
Bilinear
Bicubic
Lanczos3 | Interpolation filter applied when resizing the decoded image. Nearest is fastest but blocky; Bilinear is a good default; Bicubic and Lanczos3 are higher-quality but slower. | @@ -91,7 +91,7 @@ In the list below, the names of required properties appear in bold. Any other pr | Name | Default Value | Allowable Values | Description | |--------------------------|---------------|------------------------------|----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| | **Confidence Threshold** | 0.0 | | Minimum confidence a class must reach to be included in the output JSON. Applied AFTER activation, so the units match the chosen activation (0.0..=1.0 for Softmax/Sigmoid, model-native for None). Set to 0.0 to always emit exactly Top K predictions.
**Supports Expression Language: true** | -| **Label index offset** | 0 | | Offset added to the model's class ID when looking up a name in the labels file. Defaults to 0 (labels file line N = class N). Set to 1 for label files that start with a dummy/background entry — e.g. the ONNX MobileNetV2 model emits 1000 class scores while 'imagenet_slim_labels.txt' has 1001 lines (line 0 = 'dummy'), so class ID 653 maps to line 654 = 'military uniform'. | +| Label index offset | | | Offset added to the model's class ID when looking up a name in the labels file. Defaults to 0 (labels file line N = class N). Set to 1 for label files that start with a dummy/background entry — e.g. the ONNX MobileNetV2 model emits 1000 class scores while 'imagenet_slim_labels.txt' has 1001 lines (line 0 = 'dummy'), so class ID 653 maps to line 654 = 'military uniform'. | | Labels file path | | | Optional path to a newline-separated labels file (line N = name of class N). Loaded once at service enable time. When set, each prediction in the output JSON gains a 'class_name' field and the 'class.top1.name' flow file attribute is populated. Leave empty to emit numeric class IDs only. | | Output attribute name | | | Specify the attribute to use as output, if not provided, the content is overridden instead.
**Supports Expression Language: true** | | **Score activation** | Softmax | Softmax
Sigmoid
None | Activation applied to the raw score vector before ranking. Softmax = mutually-exclusive classes (ImageNet-trained ResNet/MobileNet/EfficientNet raw logits). Sigmoid = independent classes (multi-label classifiers). None = the model already emits probabilities/scores; rank the raw values. | @@ -132,11 +132,11 @@ In the list below, the names of required properties appear in bold. Any other pr | **Box format** | Xyxy | Xyxy
Yxyx
Cxcywh | Layout of the four floats per box in the box output tensor. Xyxy = [x_min, y_min, x_max, y_max] (SSD, MobileNet-SSD, most PyTorch exports). Yxyx = [y_min, x_min, y_max, x_max] (TensorFlow Object Detection API). Cxcywh = [cx, cy, w, h] (YOLOv3/5/8 raw output). | | **Box output index** | 1 | | Zero-based index of the model output tensor that holds box coordinates. Must differ from 'Score output index'. | | Class output index | | | Zero-based index of a model output tensor that holds one class id per box. Set this for detectors that emit boxes, per-box scores, and class ids as three separate parallel tensors, with NMS already folded into the graph (TensorFlow Object Detection API; YOLO / EfficientNMS 'end2end' exports). When set, 'Score output index' is read as one score per box (not a [boxes, classes] matrix) and no argmax is performed; the class id tensor may be integer- or float-typed. Leave empty for models that emit a per-class score matrix. | -| **Color format** | RGB | RGB
BGR
Grayscale | Colour space of the tensor fed to the model. RGB and BGR produce three-channel tensors (channel order determined by the format); Grayscale produces a single-channel luma tensor. | +| **Color format** | RGB | RGB
BGR
Grayscale | Color space of the tensor fed to the model. RGB and BGR produce three-channel tensors (channel order determined by the format); Grayscale produces a single-channel luma tensor. | | **Confidence Threshold** | 0.7 | | Minimum per-box class probability (0.0 to 1.0) required to keep a bounding box after applying the chosen 'Score activation'. Boxes below the threshold are discarded before NMS.
**Supports Expression Language: true** | | **IoU Threshold** | 0.45 | | Intersection-over-union cutoff used during non-maximum suppression. Boxes of the same class whose IoU with a higher-confidence peer exceeds this value are suppressed. Typical values: 0.45 (SSD/YOLO default), 0.5, 0.3 for stricter deduplication. | | **Letterbox pad value** | 0.0 | | Value written for padding pixels when 'Resize mode' is 'Letterbox'. This is a normalised value (post mean/std), so 0.0 corresponds to a neutral input for most networks. Ignored when 'Resize mode' is 'Stretch'. | -| Mean | 0.0 | | Mean subtracted from each pixel before dividing by 'Standard Deviation'. Accepts either a single value (broadcast to all channels) or three comma-separated values applied per channel in the order dictated by 'Color format'. Example: '0.485, 0.456, 0.406' for ImageNet-style RGB normalisation. | +| Mean | 0.0 | | Value subtracted from each pixel before dividing by 'Standard Deviation'. Accepts either a single value (broadcast to all channels) or three comma-separated values applied per channel in the order dictated by 'Color format'. Example: '0.485, 0.456, 0.406' for ImageNet-style RGB normalisation. | | Output attribute name | | | Specify the attribute to use as output, if not provided, the content is overridden instead.
**Supports Expression Language: true** | | **Pixel divisor** | 1.0 | | Divisor applied to raw u8 pixel values before subtracting 'Mean' and dividing by 'Standard Deviation'. Defaults to 1.0 (mean/std interpreted in [0, 255] pixel space, e.g. UltraFace's mean=127, std=128). Set to 255 to bring pixels into [0.0, 1.0] first so ImageNet-style mean/std values like '0.485, 0.456, 0.406' / '0.229, 0.224, 0.225' can be used directly, matching the PyTorch / torchvision / ONNX MobileNet convention. Must be non-zero. | | **Resize filter** | Bilinear | Nearest
Bilinear
Bicubic
Lanczos3 | Interpolation filter applied when resizing the decoded image. Nearest is fastest but blocky; Bilinear is a good default; Bicubic and Lanczos3 are higher-quality but slower. | @@ -229,7 +229,7 @@ In the list below, the names of required properties appear in bold. Any other pr ### Description -Decodes an image from the flow file content and converts it into a normalised numeric tensor suitable for feeding into a downstream inference processor such as InvokeTractModel. Supports RGB / BGR / Grayscale, CHW / HWC layouts, stretch or letterbox resizing, and scalar or per-channel mean/std normalisation. The output payload is the raw little-endian f32 tensor; the 'tensors.len', 'tensor.{i}.shape' and 'tensor.{i}.dtype' attributes describe its layout. +Decodes an image from the flow file content and converts it into a normalised numeric tensor suitable for feeding into a downstream inference processor such as InvokeTractModel. Supports RGB / BGR / Grayscale, CHW / HWC layouts, stretch or letterbox resizing, and scalar or per-channel mean/std normalisation. The output payload is a single raw little-endian f32 tensor; the 'tensor.0.shape' attribute describes its layout. ### Properties @@ -237,9 +237,9 @@ In the list below, the names of required properties appear in bold. Any other pr | Name | Default Value | Allowable Values | Description | |-------------------------|---------------|-----------------------------------------------|----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| -| **Color format** | RGB | RGB
BGR
Grayscale | Colour space of the tensor fed to the model. RGB and BGR produce three-channel tensors (channel order determined by the format); Grayscale produces a single-channel luma tensor. | +| **Color format** | RGB | RGB
BGR
Grayscale | Color space of the tensor fed to the model. RGB and BGR produce three-channel tensors (channel order determined by the format); Grayscale produces a single-channel luma tensor. | | **Letterbox pad value** | 0.0 | | Value written for padding pixels when 'Resize mode' is 'Letterbox'. This is a normalised value (post mean/std), so 0.0 corresponds to a neutral input for most networks. Ignored when 'Resize mode' is 'Stretch'. | -| Mean | 0.0 | | Mean subtracted from each pixel before dividing by 'Standard Deviation'. Accepts either a single value (broadcast to all channels) or three comma-separated values applied per channel in the order dictated by 'Color format'. Example: '0.485, 0.456, 0.406' for ImageNet-style RGB normalisation. | +| Mean | 0.0 | | Value subtracted from each pixel before dividing by 'Standard Deviation'. Accepts either a single value (broadcast to all channels) or three comma-separated values applied per channel in the order dictated by 'Color format'. Example: '0.485, 0.456, 0.406' for ImageNet-style RGB normalisation. | | **Pixel divisor** | 1.0 | | Divisor applied to raw u8 pixel values before subtracting 'Mean' and dividing by 'Standard Deviation'. Defaults to 1.0 (mean/std interpreted in [0, 255] pixel space, e.g. UltraFace's mean=127, std=128). Set to 255 to bring pixels into [0.0, 1.0] first so ImageNet-style mean/std values like '0.485, 0.456, 0.406' / '0.229, 0.224, 0.225' can be used directly, matching the PyTorch / torchvision / ONNX MobileNet convention. Must be non-zero. | | **Resize filter** | Bilinear | Nearest
Bilinear
Bicubic
Lanczos3 | Interpolation filter applied when resizing the decoded image. Nearest is fastest but blocky; Bilinear is a good default; Bicubic and Lanczos3 are higher-quality but slower. | | **Resize mode** | Stretch | Stretch
Letterbox | How the source image is fitted into the target dimensions. 'Stretch' scales each axis independently, distorting aspect ratio. 'Letterbox' preserves aspect ratio and pads the remaining border with 'Letterbox pad value' (applied in normalised output space). | From c1275a66266436d949d527471181315d2d14a1e9 Mon Sep 17 00:00:00 2001 From: Martin Zink Date: Fri, 25 Sep 2026 18:13:59 +0200 Subject: [PATCH 32/32] refresh manifest --- .../ubuntu_22_04_clang_arm_manifest.json | 25 +++++++++---------- 1 file changed, 12 insertions(+), 13 deletions(-) diff --git a/.github/references/ubuntu_22_04_clang_arm_manifest.json b/.github/references/ubuntu_22_04_clang_arm_manifest.json index 6ba6899d26..ae6ebd3b39 100644 --- a/.github/references/ubuntu_22_04_clang_arm_manifest.json +++ b/.github/references/ubuntu_22_04_clang_arm_manifest.json @@ -12781,7 +12781,7 @@ "propertyDescriptors": { "Color format": { "name": "Color format", - "description": "Colour space of the tensor fed to the model. RGB and BGR produce three-channel tensors (channel order determined by the format); Grayscale produces a single-channel luma tensor.", + "description": "Color space of the tensor fed to the model. RGB and BGR produce three-channel tensors (channel order determined by the format); Grayscale produces a single-channel luma tensor.", "validator": "VALID", "required": "true", "sensitive": "false", @@ -12797,8 +12797,8 @@ "displayName": "BGR" }, { - "value": "GRAYSCALE", - "displayName": "GRAYSCALE" + "value": "Grayscale", + "displayName": "Grayscale" } ] }, @@ -12815,10 +12815,9 @@ "name": "Label index offset", "description": "Offset added to the model's class ID when looking up a name in the labels file. Defaults to 0 (labels file line N = class N). Set to 1 for label files that start with a dummy/background entry — e.g. the ONNX MobileNetV2 model emits 1000 class scores while 'imagenet_slim_labels.txt' has 1001 lines (line 0 = 'dummy'), so class ID 653 maps to line 654 = 'military uniform'.", "validator": "NON_NEGATIVE_INTEGER_VALIDATOR", - "required": "true", + "required": "false", "sensitive": "false", - "expressionLanguageScope": "NONE", - "defaultValue": "0" + "expressionLanguageScope": "NONE" }, "Labels file path": { "name": "Labels file path", @@ -12839,7 +12838,7 @@ }, "Mean": { "name": "Mean", - "description": "Mean subtracted from each pixel before dividing by 'Standard Deviation'. Accepts either a single value (broadcast to all channels) or three comma-separated values applied per channel in the order dictated by 'Color format'. Example: '0.485, 0.456, 0.406' for ImageNet-style RGB normalisation.", + "description": "Value subtracted from each pixel before dividing by 'Standard Deviation'. Accepts either a single value (broadcast to all channels) or three comma-separated values applied per channel in the order dictated by 'Color format'. Example: '0.485, 0.456, 0.406' for ImageNet-style RGB normalisation.", "validator": "VALID", "required": "false", "sensitive": "false", @@ -13077,7 +13076,7 @@ }, "Color format": { "name": "Color format", - "description": "Colour space of the tensor fed to the model. RGB and BGR produce three-channel tensors (channel order determined by the format); Grayscale produces a single-channel luma tensor.", + "description": "Color space of the tensor fed to the model. RGB and BGR produce three-channel tensors (channel order determined by the format); Grayscale produces a single-channel luma tensor.", "validator": "VALID", "required": "true", "sensitive": "false", @@ -13093,8 +13092,8 @@ "displayName": "BGR" }, { - "value": "GRAYSCALE", - "displayName": "GRAYSCALE" + "value": "Grayscale", + "displayName": "Grayscale" } ] }, @@ -13127,7 +13126,7 @@ }, "Mean": { "name": "Mean", - "description": "Mean subtracted from each pixel before dividing by 'Standard Deviation'. Accepts either a single value (broadcast to all channels) or three comma-separated values applied per channel in the order dictated by 'Color format'. Example: '0.485, 0.456, 0.406' for ImageNet-style RGB normalisation.", + "description": "Value subtracted from each pixel before dividing by 'Standard Deviation'. Accepts either a single value (broadcast to all channels) or three comma-separated values applied per channel in the order dictated by 'Color format'. Example: '0.485, 0.456, 0.406' for ImageNet-style RGB normalisation.", "validator": "VALID", "required": "false", "sensitive": "false", @@ -13317,12 +13316,12 @@ }, "Line color": { "name": "Line color", - "description": "Outline colour as '[R, G, B]' u8 channels (0-255).", + "description": "Outline color as a hex string (e.g., '#ff00ff' or '#f0f')", "validator": "VALID", "required": "false", "sensitive": "false", "expressionLanguageScope": "NONE", - "defaultValue": "[0, 255, 0]" + "defaultValue": "#00FF00" }, "Line thickness": { "name": "Line thickness",