diff --git a/.github/references/ubuntu_22_04_clang_arm_manifest.json b/.github/references/ubuntu_22_04_clang_arm_manifest.json index 139b3854d7..ae6ebd3b39 100644 --- a/.github/references/ubuntu_22_04_clang_arm_manifest.json +++ b/.github/references/ubuntu_22_04_clang_arm_manifest.json @@ -12772,5 +12772,630 @@ "artifact": "minifi_rs_playground", "version": "0.1.0" } +}, +{ + "bundles": { + "componentManifest": { + "processors": [ + { + "propertyDescriptors": { + "Color format": { + "name": "Color format", + "description": "Color space of the tensor fed to the model. RGB and BGR produce three-channel tensors (channel order determined by the format); Grayscale produces a single-channel luma tensor.", + "validator": "VALID", + "required": "true", + "sensitive": "false", + "expressionLanguageScope": "NONE", + "defaultValue": "RGB", + "allowableValues": [ + { + "value": "RGB", + "displayName": "RGB" + }, + { + "value": "BGR", + "displayName": "BGR" + }, + { + "value": "Grayscale", + "displayName": "Grayscale" + } + ] + }, + "Confidence Threshold": { + "name": "Confidence Threshold", + "description": "Minimum confidence a class must reach to be included in the output JSON. Applied AFTER activation, so the units match the chosen activation (0.0..=1.0 for Softmax/Sigmoid, model-native for None). Set to 0.0 to always emit exactly Top K predictions.", + "validator": "NUMBER_VALIDATOR", + "required": "true", + "sensitive": "false", + "expressionLanguageScope": "FLOWFILE_ATTRIBUTES", + "defaultValue": "0.0" + }, + "Label index offset": { + "name": "Label index offset", + "description": "Offset added to the model's class ID when looking up a name in the labels file. Defaults to 0 (labels file line N = class N). Set to 1 for label files that start with a dummy/background entry — e.g. the ONNX MobileNetV2 model emits 1000 class scores while 'imagenet_slim_labels.txt' has 1001 lines (line 0 = 'dummy'), so class ID 653 maps to line 654 = 'military uniform'.", + "validator": "NON_NEGATIVE_INTEGER_VALIDATOR", + "required": "false", + "sensitive": "false", + "expressionLanguageScope": "NONE" + }, + "Labels file path": { + "name": "Labels file path", + "description": "Optional path to a newline-separated labels file (line N = name of class N). Loaded once at service enable time. When set, each prediction in the output JSON gains a 'class_name' field and the 'class.top1.name' flow file attribute is populated. Leave empty to emit numeric class IDs only.", + "validator": "VALID", + "required": "false", + "sensitive": "false", + "expressionLanguageScope": "NONE" + }, + "Letterbox pad value": { + "name": "Letterbox pad value", + "description": "Value written for padding pixels when 'Resize mode' is 'Letterbox'. This is a normalised value (post mean/std), so 0.0 corresponds to a neutral input for most networks. Ignored when 'Resize mode' is 'Stretch'.", + "validator": "NUMBER_VALIDATOR", + "required": "true", + "sensitive": "false", + "expressionLanguageScope": "NONE", + "defaultValue": "0.0" + }, + "Mean": { + "name": "Mean", + "description": "Value subtracted from each pixel before dividing by 'Standard Deviation'. Accepts either a single value (broadcast to all channels) or three comma-separated values applied per channel in the order dictated by 'Color format'. Example: '0.485, 0.456, 0.406' for ImageNet-style RGB normalisation.", + "validator": "VALID", + "required": "false", + "sensitive": "false", + "expressionLanguageScope": "NONE", + "defaultValue": "0.0" + }, + "Output attribute name": { + "name": "Output attribute name", + "description": "Specify the attribute to use as output, if not provided, the content is overridden instead.", + "validator": "VALID", + "required": "false", + "sensitive": "false", + "expressionLanguageScope": "FLOWFILE_ATTRIBUTES" + }, + "Pixel divisor": { + "name": "Pixel divisor", + "description": "Divisor applied to raw u8 pixel values before subtracting 'Mean' and dividing by 'Standard Deviation'. Defaults to 1.0 (mean/std interpreted in [0, 255] pixel space, e.g. UltraFace's mean=127, std=128). Set to 255 to bring pixels into [0.0, 1.0] first so ImageNet-style mean/std values like '0.485, 0.456, 0.406' / '0.229, 0.224, 0.225' can be used directly, matching the PyTorch / torchvision / ONNX MobileNet convention. Must be non-zero.", + "validator": "NUMBER_VALIDATOR", + "required": "true", + "sensitive": "false", + "expressionLanguageScope": "NONE", + "defaultValue": "1.0" + }, + "Resize filter": { + "name": "Resize filter", + "description": "Interpolation filter applied when resizing the decoded image. Nearest is fastest but blocky; Bilinear is a good default; Bicubic and Lanczos3 are higher-quality but slower.", + "validator": "VALID", + "required": "true", + "sensitive": "false", + "expressionLanguageScope": "NONE", + "defaultValue": "Bilinear", + "allowableValues": [ + { + "value": "Nearest", + "displayName": "Nearest" + }, + { + "value": "Bilinear", + "displayName": "Bilinear" + }, + { + "value": "Bicubic", + "displayName": "Bicubic" + }, + { + "value": "Lanczos3", + "displayName": "Lanczos3" + } + ] + }, + "Resize mode": { + "name": "Resize mode", + "description": "How the source image is fitted into the target dimensions. 'Stretch' scales each axis independently, distorting aspect ratio. 'Letterbox' preserves aspect ratio and pads the remaining border with 'Letterbox pad value' (applied in normalised output space).", + "validator": "VALID", + "required": "true", + "sensitive": "false", + "expressionLanguageScope": "NONE", + "defaultValue": "Stretch", + "allowableValues": [ + { + "value": "Stretch", + "displayName": "Stretch" + }, + { + "value": "Letterbox", + "displayName": "Letterbox" + } + ] + }, + "Score activation": { + "name": "Score activation", + "description": "Activation applied to the raw score vector before ranking. Softmax = mutually-exclusive classes (ImageNet-trained ResNet/MobileNet/EfficientNet raw logits). Sigmoid = independent classes (multi-label classifiers). None = the model already emits probabilities/scores; rank the raw values.", + "validator": "VALID", + "required": "true", + "sensitive": "false", + "expressionLanguageScope": "NONE", + "defaultValue": "Softmax", + "allowableValues": [ + { + "value": "Softmax", + "displayName": "Softmax" + }, + { + "value": "Sigmoid", + "displayName": "Sigmoid" + }, + { + "value": "None", + "displayName": "None" + } + ] + }, + "Score output index": { + "name": "Score output index", + "description": "Zero-based index of the model output tensor that holds classification scores. The processor slices the concatenated payload from InvokeTractModel according to the 'tensor.N.bytes' attributes. Almost always 0 for single-head classifiers.", + "validator": "NON_NEGATIVE_INTEGER_VALIDATOR", + "required": "true", + "sensitive": "false", + "expressionLanguageScope": "NONE", + "defaultValue": "0" + }, + "Standard Deviation": { + "name": "Standard Deviation", + "description": "Divisor applied after subtracting 'Mean'. Accepts a single value (broadcast) or three comma-separated values (per channel). Must be non-zero. Example: '255.0' to scale u8 pixels into [0.0, 1.0]; '0.229, 0.224, 0.225' for ImageNet.", + "validator": "VALID", + "required": "false", + "sensitive": "false", + "expressionLanguageScope": "NONE", + "defaultValue": "255.0" + }, + "Target height": { + "name": "Target height", + "description": "Height in pixels the decoded image is resized to before normalisation and inference.", + "validator": "NON_NEGATIVE_INTEGER_VALIDATOR", + "required": "true", + "sensitive": "false", + "expressionLanguageScope": "NONE" + }, + "Target width": { + "name": "Target width", + "description": "Width in pixels the decoded image is resized to before normalisation and inference.", + "validator": "NON_NEGATIVE_INTEGER_VALIDATOR", + "required": "true", + "sensitive": "false", + "expressionLanguageScope": "NONE" + }, + "Tensor shape format": { + "name": "Tensor shape format", + "description": "Memory layout of the tensor fed to the model. CHW (channels-first) is typical for PyTorch/ONNX detectors. HWC (channels-last) matches TensorFlow/TFLite. Ignored for Grayscale (always effectively 1xHxW).", + "validator": "VALID", + "required": "true", + "sensitive": "false", + "expressionLanguageScope": "NONE", + "defaultValue": "CHW", + "allowableValues": [ + { + "value": "CHW", + "displayName": "CHW" + }, + { + "value": "HWC", + "displayName": "HWC" + } + ] + }, + "Top K": { + "name": "Top K", + "description": "Number of highest-scoring classes to include in the output JSON, in descending order of confidence. Values above the total class count are clamped. Set to 1 for pure top-1 classification.", + "validator": "NON_NEGATIVE_INTEGER_VALIDATOR", + "required": "true", + "sensitive": "false", + "expressionLanguageScope": "NONE", + "defaultValue": "5" + }, + "Tract model service": { + "typeProvidedByValue": { + "type": "minifi_tensor.services.tract_model_service.TractModelService", + "group": "org.apache.nifi.minifi.rust", + "artifact": "minifi_tensor" + }, + "name": "Tract model service", + "description": "Reference to a TractModelService controller service. The referenced service owns the compiled model (ONNX or NNEF) that will be evaluated for each incoming flow file.", + "validator": "VALID", + "required": "true", + "sensitive": "false", + "expressionLanguageScope": "NONE" + } + }, + "inputRequirement": "INPUT_REQUIRED", + "isSingleThreaded": "false", + "supportedRelationships": [ + { + "name": "failure", + "description": "The image could not be decoded, the input tensor could not be built, the model failed to run, or the model outputs could not be interpreted as classification." + }, + { + "name": "success", + "description": "Inference and post-processing completed. The flow file content is the original, unchanged image; the classifications are written to the configured output attribute as a JSON array (may be empty)." + } + ], + "typeDescription": "Runs a full image-classification pass in a single processor: decodes the image from the flow file content, resizes and normalises it into an input tensor, runs one inference against the compiled model owned by the referenced TractModelService, and post-processes the score vector (score activation, Top-K selection, confidence filtering, optional label lookup) into predictions. Collapses the ImageToTensor -> InvokeTractModel -> ClassifyOutput chain into one node. The flow file content is left unchanged (the original image); the Top-K classifications are written as a JSON array to the configured output attribute.", + "supportsDynamicRelationships": "false", + "supportsDynamicProperties": "false", + "type": "minifi_tensor.processors.classify_image.ClassifyImage" + }, + { + "propertyDescriptors": { + "Background class index": { + "name": "Background class index", + "description": "Index of the 'background / no-object' class. Boxes whose winning class equals this index are dropped. In score-matrix mode this is only honoured when the score tensor has more than one class per box; in 'Class output index' mode it is matched against each box's class id.", + "validator": "NON_NEGATIVE_INTEGER_VALIDATOR", + "required": "false", + "sensitive": "false", + "expressionLanguageScope": "NONE" + }, + "Box format": { + "name": "Box format", + "description": "Layout of the four floats per box in the box output tensor. Xyxy = [x_min, y_min, x_max, y_max] (SSD, MobileNet-SSD, most PyTorch exports). Yxyx = [y_min, x_min, y_max, x_max] (TensorFlow Object Detection API). Cxcywh = [cx, cy, w, h] (YOLOv3/5/8 raw output).", + "validator": "VALID", + "required": "true", + "sensitive": "false", + "expressionLanguageScope": "NONE", + "defaultValue": "Xyxy", + "allowableValues": [ + { + "value": "Xyxy", + "displayName": "Xyxy" + }, + { + "value": "Yxyx", + "displayName": "Yxyx" + }, + { + "value": "Cxcywh", + "displayName": "Cxcywh" + } + ] + }, + "Box output index": { + "name": "Box output index", + "description": "Zero-based index of the model output tensor that holds box coordinates. Must differ from 'Score output index'.", + "validator": "NON_NEGATIVE_INTEGER_VALIDATOR", + "required": "true", + "sensitive": "false", + "expressionLanguageScope": "NONE", + "defaultValue": "1" + }, + "Class output index": { + "name": "Class output index", + "description": "Zero-based index of a model output tensor that holds one class id per box. Set this for detectors that emit boxes, per-box scores, and class ids as three separate parallel tensors, with NMS already folded into the graph (TensorFlow Object Detection API; YOLO / EfficientNMS 'end2end' exports). When set, 'Score output index' is read as one score per box (not a [boxes, classes] matrix) and no argmax is performed; the class id tensor may be integer- or float-typed. Leave empty for models that emit a per-class score matrix.", + "validator": "NON_NEGATIVE_INTEGER_VALIDATOR", + "required": "false", + "sensitive": "false", + "expressionLanguageScope": "NONE" + }, + "Color format": { + "name": "Color format", + "description": "Color space of the tensor fed to the model. RGB and BGR produce three-channel tensors (channel order determined by the format); Grayscale produces a single-channel luma tensor.", + "validator": "VALID", + "required": "true", + "sensitive": "false", + "expressionLanguageScope": "NONE", + "defaultValue": "RGB", + "allowableValues": [ + { + "value": "RGB", + "displayName": "RGB" + }, + { + "value": "BGR", + "displayName": "BGR" + }, + { + "value": "Grayscale", + "displayName": "Grayscale" + } + ] + }, + "Confidence Threshold": { + "name": "Confidence Threshold", + "description": "Minimum per-box class probability (0.0 to 1.0) required to keep a bounding box after applying the chosen 'Score activation'. Boxes below the threshold are discarded before NMS.", + "validator": "NUMBER_VALIDATOR", + "required": "true", + "sensitive": "false", + "expressionLanguageScope": "FLOWFILE_ATTRIBUTES", + "defaultValue": "0.7" + }, + "IoU Threshold": { + "name": "IoU Threshold", + "description": "Intersection-over-union cutoff used during non-maximum suppression. Boxes of the same class whose IoU with a higher-confidence peer exceeds this value are suppressed. Typical values: 0.45 (SSD/YOLO default), 0.5, 0.3 for stricter deduplication.", + "validator": "NUMBER_VALIDATOR", + "required": "true", + "sensitive": "false", + "expressionLanguageScope": "NONE", + "defaultValue": "0.45" + }, + "Letterbox pad value": { + "name": "Letterbox pad value", + "description": "Value written for padding pixels when 'Resize mode' is 'Letterbox'. This is a normalised value (post mean/std), so 0.0 corresponds to a neutral input for most networks. Ignored when 'Resize mode' is 'Stretch'.", + "validator": "NUMBER_VALIDATOR", + "required": "true", + "sensitive": "false", + "expressionLanguageScope": "NONE", + "defaultValue": "0.0" + }, + "Mean": { + "name": "Mean", + "description": "Value subtracted from each pixel before dividing by 'Standard Deviation'. Accepts either a single value (broadcast to all channels) or three comma-separated values applied per channel in the order dictated by 'Color format'. Example: '0.485, 0.456, 0.406' for ImageNet-style RGB normalisation.", + "validator": "VALID", + "required": "false", + "sensitive": "false", + "expressionLanguageScope": "NONE", + "defaultValue": "0.0" + }, + "Output attribute name": { + "name": "Output attribute name", + "description": "Specify the attribute to use as output, if not provided, the content is overridden instead.", + "validator": "VALID", + "required": "false", + "sensitive": "false", + "expressionLanguageScope": "FLOWFILE_ATTRIBUTES" + }, + "Pixel divisor": { + "name": "Pixel divisor", + "description": "Divisor applied to raw u8 pixel values before subtracting 'Mean' and dividing by 'Standard Deviation'. Defaults to 1.0 (mean/std interpreted in [0, 255] pixel space, e.g. UltraFace's mean=127, std=128). Set to 255 to bring pixels into [0.0, 1.0] first so ImageNet-style mean/std values like '0.485, 0.456, 0.406' / '0.229, 0.224, 0.225' can be used directly, matching the PyTorch / torchvision / ONNX MobileNet convention. Must be non-zero.", + "validator": "NUMBER_VALIDATOR", + "required": "true", + "sensitive": "false", + "expressionLanguageScope": "NONE", + "defaultValue": "1.0" + }, + "Resize filter": { + "name": "Resize filter", + "description": "Interpolation filter applied when resizing the decoded image. Nearest is fastest but blocky; Bilinear is a good default; Bicubic and Lanczos3 are higher-quality but slower.", + "validator": "VALID", + "required": "true", + "sensitive": "false", + "expressionLanguageScope": "NONE", + "defaultValue": "Bilinear", + "allowableValues": [ + { + "value": "Nearest", + "displayName": "Nearest" + }, + { + "value": "Bilinear", + "displayName": "Bilinear" + }, + { + "value": "Bicubic", + "displayName": "Bicubic" + }, + { + "value": "Lanczos3", + "displayName": "Lanczos3" + } + ] + }, + "Resize mode": { + "name": "Resize mode", + "description": "How the source image is fitted into the target dimensions. 'Stretch' scales each axis independently, distorting aspect ratio. 'Letterbox' preserves aspect ratio and pads the remaining border with 'Letterbox pad value' (applied in normalised output space).", + "validator": "VALID", + "required": "true", + "sensitive": "false", + "expressionLanguageScope": "NONE", + "defaultValue": "Stretch", + "allowableValues": [ + { + "value": "Stretch", + "displayName": "Stretch" + }, + { + "value": "Letterbox", + "displayName": "Letterbox" + } + ] + }, + "Score activation": { + "name": "Score activation", + "description": "Activation applied to raw per-class scores before selecting the winning class. Softmax = mutually-exclusive classes (SSD/MobileNet-SSD raw logits). Sigmoid = independent classes (YOLOv5/v8 style). None = the model already emits probabilities/scores; use raw argmax with the raw score as confidence.", + "validator": "VALID", + "required": "true", + "sensitive": "false", + "expressionLanguageScope": "NONE", + "defaultValue": "Softmax", + "allowableValues": [ + { + "value": "Softmax", + "displayName": "Softmax" + }, + { + "value": "Sigmoid", + "displayName": "Sigmoid" + }, + { + "value": "None", + "displayName": "None" + } + ] + }, + "Score output index": { + "name": "Score output index", + "description": "Zero-based index of the model output tensor that holds classification scores. The processor slices the concatenated payload from InvokeTractModel according to the 'tensor.N.bytes' attributes.", + "validator": "NON_NEGATIVE_INTEGER_VALIDATOR", + "required": "true", + "sensitive": "false", + "expressionLanguageScope": "NONE", + "defaultValue": "0" + }, + "Standard Deviation": { + "name": "Standard Deviation", + "description": "Divisor applied after subtracting 'Mean'. Accepts a single value (broadcast) or three comma-separated values (per channel). Must be non-zero. Example: '255.0' to scale u8 pixels into [0.0, 1.0]; '0.229, 0.224, 0.225' for ImageNet.", + "validator": "VALID", + "required": "false", + "sensitive": "false", + "expressionLanguageScope": "NONE", + "defaultValue": "255.0" + }, + "Target height": { + "name": "Target height", + "description": "Height in pixels the decoded image is resized to before normalisation and inference.", + "validator": "NON_NEGATIVE_INTEGER_VALIDATOR", + "required": "true", + "sensitive": "false", + "expressionLanguageScope": "NONE" + }, + "Target width": { + "name": "Target width", + "description": "Width in pixels the decoded image is resized to before normalisation and inference.", + "validator": "NON_NEGATIVE_INTEGER_VALIDATOR", + "required": "true", + "sensitive": "false", + "expressionLanguageScope": "NONE" + }, + "Tensor shape format": { + "name": "Tensor shape format", + "description": "Memory layout of the tensor fed to the model. CHW (channels-first) is typical for PyTorch/ONNX detectors. HWC (channels-last) matches TensorFlow/TFLite. Ignored for Grayscale (always effectively 1xHxW).", + "validator": "VALID", + "required": "true", + "sensitive": "false", + "expressionLanguageScope": "NONE", + "defaultValue": "CHW", + "allowableValues": [ + { + "value": "CHW", + "displayName": "CHW" + }, + { + "value": "HWC", + "displayName": "HWC" + } + ] + }, + "Tract model service": { + "typeProvidedByValue": { + "type": "minifi_tensor.services.tract_model_service.TractModelService", + "group": "org.apache.nifi.minifi.rust", + "artifact": "minifi_tensor" + }, + "name": "Tract model service", + "description": "Reference to a TractModelService controller service. The referenced service owns the compiled model (ONNX or NNEF) that will be evaluated for each incoming flow file.", + "validator": "VALID", + "required": "true", + "sensitive": "false", + "expressionLanguageScope": "NONE" + } + }, + "inputRequirement": "INPUT_REQUIRED", + "isSingleThreaded": "false", + "supportedRelationships": [ + { + "name": "failure", + "description": "The image could not be decoded, the input tensor could not be built, the model failed to run, or the model outputs could not be interpreted as scores + boxes." + }, + { + "name": "success", + "description": "Inference and post-processing completed. The flow file content is the original, unchanged image; the detected boxes are written to the configured output attribute as a JSON array (may be empty)." + } + ], + "typeDescription": "Runs a full object-detection pass in a single processor: decodes the image from the flow file content, resizes and normalises it into an input tensor, runs one inference against the compiled model owned by the referenced TractModelService, and post-processes the model outputs (score activation, confidence filtering, box decoding, per-class non-maximum suppression) into bounding boxes. Collapses the ImageToTensor -> InvokeTractModel -> FilterBoundingBoxes chain into one node. The flow file content is left unchanged (the original image); the detected boxes are written as a JSON array to the configured output attribute so a downstream DrawBoundingBox can annotate the image.", + "supportsDynamicRelationships": "false", + "supportsDynamicProperties": "false", + "type": "minifi_tensor.processors.detect_object.DetectObject" + }, + { + "propertyDescriptors": { + "Bounding boxes": { + "name": "Bounding boxes", + "description": "JSON array of bounding boxes to draw onto the image (fields class_id, confidence, x_min, y_min, x_max, y_max; coordinates normalised to [0,1] against the image). Typically the attribute produced by an upstream DetectObject or FilterBoundingBoxes processor.", + "validator": "VALID", + "required": "false", + "sensitive": "false", + "expressionLanguageScope": "FLOWFILE_ATTRIBUTES", + "defaultValue": "${enrichment.value}" + }, + "Line color": { + "name": "Line color", + "description": "Outline color as a hex string (e.g., '#ff00ff' or '#f0f')", + "validator": "VALID", + "required": "false", + "sensitive": "false", + "expressionLanguageScope": "NONE", + "defaultValue": "#00FF00" + }, + "Line thickness": { + "name": "Line thickness", + "description": "Thickness in pixels of the drawn box outline.", + "validator": "NON_NEGATIVE_INTEGER_VALIDATOR", + "required": "true", + "sensitive": "false", + "expressionLanguageScope": "NONE", + "defaultValue": "5" + } + }, + "inputRequirement": "INPUT_REQUIRED", + "isSingleThreaded": "false", + "supportedRelationships": [ + { + "name": "failure", + "description": "Invalid FlowFiles are routed here" + }, + { + "name": "success", + "description": "Flowfiles are routed here after drawing the bounding boxes" + } + ], + "typeDescription": "Decodes the image from the flow file content, draws each bounding box supplied via the 'Bounding boxes' property onto it, and re-encodes the annotated image as PNG. Pair with an upstream DetectObject / FilterBoundingBoxes to visualise detections.", + "supportsDynamicRelationships": "false", + "supportsDynamicProperties": "false", + "type": "minifi_tensor.processors.draw_bounding_box.DrawBoundingBox" + } + ], + "controllerServices": [ + { + "propertyDescriptors": { + "Model File Path": { + "name": "Model File Path", + "description": "Absolute path to the model on the edge device. For ONNX this is a `.onnx` file; for NNEF this is a `.nnef.tgz` archive, a `.nnef` tarball, or the root directory of an unpacked NNEF model. The model is loaded, parsed, and compiled for the host CPU once when the controller service is enabled; subsequent inference calls reuse the compiled runnable.", + "validator": "VALID", + "required": "true", + "sensitive": "false", + "expressionLanguageScope": "NONE" + }, + "Model format": { + "name": "Model format", + "description": "Format of the file/directory referenced by 'Model File Path'. 'Auto' picks Onnx when the path ends in `.onnx` and Nnef when it ends in `.nnef`, `.nnef.tgz`, `.nnef.tar`, `.nnef.tar.gz`, or points at a directory. Set explicitly when the path uses a non-standard extension.", + "validator": "VALID", + "required": "true", + "sensitive": "false", + "expressionLanguageScope": "NONE", + "defaultValue": "Auto", + "allowableValues": [ + { + "value": "Auto", + "displayName": "Auto" + }, + { + "value": "Onnx", + "displayName": "Onnx" + }, + { + "value": "Nnef", + "displayName": "Nnef" + } + ] + } + }, + "typeDescription": "Provides a shared, CPU-optimized neural network for inference. Supports ONNX (`.onnx`) and NNEF (directory or tarball) models; the format can be auto-detected from the file extension or set explicitly.", + "supportsDynamicRelationships": "false", + "supportsDynamicProperties": "false", + "type": "minifi_tensor.services.tract_model_service.TractModelService" + } + ] + }, + "group": "org.apache.nifi.minifi.rust", + "artifact": "minifi_tensor", + "version": "0.1.0" + } } ] diff --git a/CONTROLLERS.md b/CONTROLLERS.md index 28fce43e1e..2b09fee53b 100644 --- a/CONTROLLERS.md +++ b/CONTROLLERS.md @@ -31,6 +31,7 @@ limitations under the License. - [RocksDbStateStorage](#RocksDbStateStorage) - [SmbConnectionControllerService](#SmbConnectionControllerService) - [SSLContextService](#SSLContextService) +- [TractModelService](#TractModelService) - [UpdatePolicyControllerService](#UpdatePolicyControllerService) - [VolatileMapStateStorage](#VolatileMapStateStorage) - [XMLReader](#XMLReader) @@ -324,6 +325,22 @@ In the list below, the names of required properties appear in bold. Any other pr | Use System Cert Store | false | true
false | Whether to use the certificates in the OS's certificate store | +## TractModelService + +### Description + +Provides a shared, CPU-optimized neural network for inference. Supports ONNX (`.onnx`) and NNEF (directory or tarball) models; the format can be auto-detected from the file extension or set explicitly. + +### Properties + +In the list below, the names of required properties appear in bold. Any other properties (not in bold) are considered optional. The table also indicates any default values, and whether a property supports the NiFi Expression Language. + +| Name | Default Value | Allowable Values | Description | +|---------------------|---------------|------------------------|-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| **Model File Path** | | | Absolute path to the model on the edge device. For ONNX this is a `.onnx` file; for NNEF this is a `.nnef.tgz` archive, a `.nnef` tarball, or the root directory of an unpacked NNEF model. The model is loaded, parsed, and compiled for the host CPU once when the controller service is enabled; subsequent inference calls reuse the compiled runnable. | +| **Model format** | Auto | Auto
Onnx
Nnef | Format of the file/directory referenced by 'Model File Path'. 'Auto' picks Onnx when the path ends in `.onnx` and Nnef when it ends in `.nnef`, `.nnef.tgz`, `.nnef.tar`, `.nnef.tar.gz`, or points at a directory. Set explicitly when the path uses a non-standard extension. | + + ## UpdatePolicyControllerService ### Description @@ -393,4 +410,3 @@ In the list below, the names of required properties appear in bold. Any other pr | **Pretty Print XML** | false | true
false | Specifies whether or not the XML should be pretty printed | | **Name of Record Tag** | | | Specifies the name of the XML record tag wrapping the record fields. | | **Name of Root Tag** | | | Specifies the name of the XML root tag wrapping the record set. | - diff --git a/PROCESSORS.md b/PROCESSORS.md index 4c7705b2a6..941a56c8b9 100644 --- a/PROCESSORS.md +++ b/PROCESSORS.md @@ -19,6 +19,7 @@ limitations under the License. - [ApplyTemplate](#ApplyTemplate) - [AttributeRollingWindow](#AttributeRollingWindow) - [AttributesToJSON](#AttributesToJSON) +- [ClassifyImage](#ClassifyImage) - [CollectKubernetesPodMetrics](#CollectKubernetesPodMetrics) - [CompressContent](#CompressContent) - [ConsumeJournald](#ConsumeJournald) @@ -31,6 +32,8 @@ limitations under the License. - [DeleteAzureDataLakeStorage](#DeleteAzureDataLakeStorage) - [DeleteGCSObject](#DeleteGCSObject) - [DeleteS3Object](#DeleteS3Object) +- [DetectObject](#DetectObject) +- [DrawBoundingBox](#DrawBoundingBox) - [EvaluateJsonPath](#EvaluateJsonPath) - [ExecuteProcess](#ExecuteProcess) - [ExecuteScript](#ExecuteScript) @@ -95,6 +98,7 @@ limitations under the License. - [RetryFlowFile](#RetryFlowFile) - [RouteOnAttribute](#RouteOnAttribute) - [RouteText](#RouteText) +- [RunLlamaCppInference](#RunLlamaCppInference) - [SegmentContent](#SegmentContent) - [SplitContent](#SplitContent) - [SplitJson](#SplitJson) @@ -214,6 +218,76 @@ In the list below, the names of required properties appear in bold. Any other pr | success | All FlowFiles received are routed to success | +## ClassifyImage + +### Description + +Runs a full image-classification pass in a single processor: decodes the image from the flow file content, resizes and normalises it into an input tensor, runs one inference against the compiled model owned by the referenced TractModelService, and post-processes the score vector (score activation, Top-K selection, confidence filtering, optional label lookup) into predictions. Collapses the ImageToTensor -> InvokeTractModel -> ClassifyOutput chain into one node. The flow file content is left unchanged (the original image); the Top-K classifications are written as a JSON array to the configured output attribute. + +### Properties + +In the list below, the names of required properties appear in bold. Any other properties (not in bold) are considered optional. The table also indicates any default values, and whether a property supports the NiFi Expression Language. + +| Name | Default Value | Allowable Values | Description | +|--------------------------|---------------|-----------------------------------------------|----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| **Target width** | | | Width in pixels the decoded image is resized to before normalisation and inference. | +| **Target height** | | | Height in pixels the decoded image is resized to before normalisation and inference. | +| **Resize filter** | Bilinear | Nearest
Bilinear
Bicubic
Lanczos3 | Interpolation filter applied when resizing the decoded image. Nearest is fastest but blocky; Bilinear is a good default; Bicubic and Lanczos3 are higher-quality but slower. | +| **Resize mode** | Stretch | Stretch
Letterbox | How the source image is fitted into the target dimensions. 'Stretch' scales each axis independently, distorting aspect ratio. 'Letterbox' preserves aspect ratio and pads the remaining border with 'Letterbox pad value' (applied in normalised output space). | +| **Letterbox pad value** | 0.0 | | Value written for padding pixels when 'Resize mode' is 'Letterbox'. This is a normalised value (post mean/std), so 0.0 corresponds to a neutral input for most networks. Ignored when 'Resize mode' is 'Stretch'. | +| **Color format** | RGB | RGB
BGR
Grayscale | Color space of the tensor fed to the model. RGB and BGR produce three-channel tensors (channel order determined by the format); Grayscale produces a single-channel luma tensor. | +| **Tensor shape format** | CHW | CHW
HWC | Memory layout of the tensor fed to the model. CHW (channels-first) is typical for PyTorch/ONNX detectors. HWC (channels-last) matches TensorFlow/TFLite. Ignored for Grayscale (always effectively 1xHxW). | +| Mean | 0.0 | | Value subtracted from each pixel before dividing by 'Standard Deviation'. Accepts either a single value (broadcast to all channels) or three comma-separated values applied per channel in the order dictated by 'Color format'. Example: '0.485, 0.456, 0.406' for ImageNet-style RGB normalisation. | +| Standard Deviation | 255.0 | | Divisor applied after subtracting 'Mean'. Accepts a single value (broadcast) or three comma-separated values (per channel). Must be non-zero. Example: '255.0' to scale u8 pixels into [0.0, 1.0]; '0.229, 0.224, 0.225' for ImageNet. | +| **Pixel divisor** | 1.0 | | Divisor applied to raw u8 pixel values before subtracting 'Mean' and dividing by 'Standard Deviation'. Defaults to 1.0 (mean/std interpreted in [0, 255] pixel space, e.g. UltraFace's mean=127, std=128). Set to 255 to bring pixels into [0.0, 1.0] first so ImageNet-style mean/std values like '0.485, 0.456, 0.406' / '0.229, 0.224, 0.225' can be used directly, matching the PyTorch / torchvision / ONNX MobileNet convention. Must be non-zero. | +| **Tract model service** | | | Reference to a TractModelService controller service. The referenced service owns the compiled model (ONNX or NNEF) that will be evaluated for each incoming flow file. | +| **Top K** | 5 | | Number of highest-scoring classes to include in the output JSON, in descending order of confidence. Values above the total class count are clamped. Set to 1 for pure top-1 classification. | +| **Score output index** | 0 | | Zero-based index of the model output tensor that holds classification scores. The processor slices the concatenated payload from InvokeTractModel according to the 'tensor.N.bytes' attributes. Almost always 0 for single-head classifiers. | +| **Score activation** | Softmax | Softmax
Sigmoid
None | Activation applied to the raw score vector before ranking. Softmax = mutually-exclusive classes (ImageNet-trained ResNet/MobileNet/EfficientNet raw logits). Sigmoid = independent classes (multi-label classifiers). None = the model already emits probabilities/scores; rank the raw values. | +| **Confidence Threshold** | 0.0 | | Minimum confidence a class must reach to be included in the output JSON. Applied AFTER activation, so the units match the chosen activation (0.0..=1.0 for Softmax/Sigmoid, model-native for None). Set to 0.0 to always emit exactly Top K predictions.
**Supports Expression Language: true** | +| Labels file path | | | Optional path to a newline-separated labels file (line N = name of class N). Loaded once at service enable time. When set, each prediction in the output JSON gains a 'class_name' field and the 'class.top1.name' flow file attribute is populated. Leave empty to emit numeric class IDs only. | +| Label index offset | | | Offset added to the model's class ID when looking up a name in the labels file. Defaults to 0 (labels file line N = class N). Set to 1 for label files that start with a dummy/background entry — e.g. the ONNX MobileNetV2 model emits 1000 class scores while 'imagenet_slim_labels.txt' has 1001 lines (line 0 = 'dummy'), so class ID 653 maps to line 654 = 'military uniform'. | +| Output attribute name | | | Specify the attribute to use as output, if not provided, the content is overridden instead.
**Supports Expression Language: true** | + +### Relationships + +| Name | Description | +|---------|-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| success | Inference and post-processing completed. The flow file content is the original, unchanged image; the classifications are written to the configured output attribute as a JSON array (may be empty). | +| failure | The image could not be decoded, the input tensor could not be built, the model failed to run, or the model outputs could not be interpreted as classification. | + +### Output Attributes + +| Attribute | Relationship | Description | +|-----------------------|--------------|----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| mime.type | success | If the "Output attribute name" is None, then the content will be overridden with the JSON array of objects with fields class_id, confidence, and optional class_name, and the mime type will be set to 'application/json'. | +| class.count | success | Number of predictions retained after Top K selection and confidence filtering. | +| class.top1.id | success | Numeric class ID of the highest-confidence prediction, when at least one prediction cleared the confidence threshold. | +| class.top1.confidence | success | Confidence (post-activation) of the highest-confidence prediction, when at least one prediction cleared the confidence threshold. | +| class.top1.name | success | Label of the highest-confidence prediction. Only present when 'Labels file path' was configured and at least one prediction cleared the threshold. | + + +## CollectKubernetesPodMetrics + +### Description + +A processor which collects pod metrics when MiNiFi is run inside Kubernetes. + +### Properties + +In the list below, the names of required properties appear in bold. Any other properties (not in bold) are considered optional. The table also indicates any default values, and whether a property supports the NiFi Expression Language. + +| Name | Default Value | Allowable Values | Description | +|-----------------------------------|---------------|------------------|------------------------------------------------------------| +| **Kubernetes Controller Service** | | | Controller service which provides Kubernetes functionality | + +### Relationships + +| Name | Description | +|---------|------------------------------------------------| +| success | All flow files produced are routed to Success. | + + ## CompressContent ### Description @@ -594,6 +668,78 @@ In the list below, the names of required properties appear in bold. Any other pr | failure | FlowFiles are routed to failure relationship | +## DetectObject + +### Description + +Runs a full object-detection pass in a single processor: decodes the image from the flow file content, resizes and normalises it into an input tensor, runs one inference against the compiled model owned by the referenced TractModelService, and post-processes the model outputs (score activation, confidence filtering, box decoding, per-class non-maximum suppression) into bounding boxes. Collapses the ImageToTensor -> InvokeTractModel -> FilterBoundingBoxes chain into one node. The flow file content is left unchanged (the original image); the detected boxes are written as a JSON array to the configured output attribute so a downstream DrawBoundingBox can annotate the image. + +### Properties + +In the list below, the names of required properties appear in bold. Any other properties (not in bold) are considered optional. The table also indicates any default values, and whether a property supports the NiFi Expression Language. + +| Name | Default Value | Allowable Values | Description | +|--------------------------|---------------|-----------------------------------------------|----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| **Target width** | | | Width in pixels the decoded image is resized to before normalisation and inference. | +| **Target height** | | | Height in pixels the decoded image is resized to before normalisation and inference. | +| **Resize filter** | Bilinear | Nearest
Bilinear
Bicubic
Lanczos3 | Interpolation filter applied when resizing the decoded image. Nearest is fastest but blocky; Bilinear is a good default; Bicubic and Lanczos3 are higher-quality but slower. | +| **Resize mode** | Stretch | Stretch
Letterbox | How the source image is fitted into the target dimensions. 'Stretch' scales each axis independently, distorting aspect ratio. 'Letterbox' preserves aspect ratio and pads the remaining border with 'Letterbox pad value' (applied in normalised output space). | +| **Letterbox pad value** | 0.0 | | Value written for padding pixels when 'Resize mode' is 'Letterbox'. This is a normalised value (post mean/std), so 0.0 corresponds to a neutral input for most networks. Ignored when 'Resize mode' is 'Stretch'. | +| **Color format** | RGB | RGB
BGR
Grayscale | Color space of the tensor fed to the model. RGB and BGR produce three-channel tensors (channel order determined by the format); Grayscale produces a single-channel luma tensor. | +| **Tensor shape format** | CHW | CHW
HWC | Memory layout of the tensor fed to the model. CHW (channels-first) is typical for PyTorch/ONNX detectors. HWC (channels-last) matches TensorFlow/TFLite. Ignored for Grayscale (always effectively 1xHxW). | +| Mean | 0.0 | | Value subtracted from each pixel before dividing by 'Standard Deviation'. Accepts either a single value (broadcast to all channels) or three comma-separated values applied per channel in the order dictated by 'Color format'. Example: '0.485, 0.456, 0.406' for ImageNet-style RGB normalisation. | +| Standard Deviation | 255.0 | | Divisor applied after subtracting 'Mean'. Accepts a single value (broadcast) or three comma-separated values (per channel). Must be non-zero. Example: '255.0' to scale u8 pixels into [0.0, 1.0]; '0.229, 0.224, 0.225' for ImageNet. | +| **Pixel divisor** | 1.0 | | Divisor applied to raw u8 pixel values before subtracting 'Mean' and dividing by 'Standard Deviation'. Defaults to 1.0 (mean/std interpreted in [0, 255] pixel space, e.g. UltraFace's mean=127, std=128). Set to 255 to bring pixels into [0.0, 1.0] first so ImageNet-style mean/std values like '0.485, 0.456, 0.406' / '0.229, 0.224, 0.225' can be used directly, matching the PyTorch / torchvision / ONNX MobileNet convention. Must be non-zero. | +| **Tract model service** | | | Reference to a TractModelService controller service. The referenced service owns the compiled model (ONNX or NNEF) that will be evaluated for each incoming flow file. | +| **Confidence Threshold** | 0.7 | | Minimum per-box class probability (0.0 to 1.0) required to keep a bounding box after applying the chosen 'Score activation'. Boxes below the threshold are discarded before NMS.
**Supports Expression Language: true** | +| **IoU Threshold** | 0.45 | | Intersection-over-union cutoff used during non-maximum suppression. Boxes of the same class whose IoU with a higher-confidence peer exceeds this value are suppressed. Typical values: 0.45 (SSD/YOLO default), 0.5, 0.3 for stricter deduplication. | +| **Score output index** | 0 | | Zero-based index of the model output tensor that holds classification scores. The processor slices the concatenated payload from InvokeTractModel according to the 'tensor.N.bytes' attributes. | +| **Box output index** | 1 | | Zero-based index of the model output tensor that holds box coordinates. Must differ from 'Score output index'. | +| Class output index | | | Zero-based index of a model output tensor that holds one class id per box. Set this for detectors that emit boxes, per-box scores, and class ids as three separate parallel tensors, with NMS already folded into the graph (TensorFlow Object Detection API; YOLO / EfficientNMS 'end2end' exports). When set, 'Score output index' is read as one score per box (not a [boxes, classes] matrix) and no argmax is performed; the class id tensor may be integer- or float-typed. Leave empty for models that emit a per-class score matrix. | +| **Box format** | Xyxy | Xyxy
Yxyx
Cxcywh | Layout of the four floats per box in the box output tensor. Xyxy = [x_min, y_min, x_max, y_max] (SSD, MobileNet-SSD, most PyTorch exports). Yxyx = [y_min, x_min, y_max, x_max] (TensorFlow Object Detection API). Cxcywh = [cx, cy, w, h] (YOLOv3/5/8 raw output). | +| **Score activation** | Softmax | Softmax
Sigmoid
None | Activation applied to raw per-class scores before selecting the winning class. Softmax = mutually-exclusive classes (SSD/MobileNet-SSD raw logits). Sigmoid = independent classes (YOLOv5/v8 style). None = the model already emits probabilities/scores; use raw argmax with the raw score as confidence. | +| Background class index | | | Index of the 'background / no-object' class. Boxes whose winning class equals this index are dropped. In score-matrix mode this is only honoured when the score tensor has more than one class per box; in 'Class output index' mode it is matched against each box's class id. | +| Output attribute name | | | Specify the attribute to use as output, if not provided, the content is overridden instead.
**Supports Expression Language: true** | + +### Relationships + +| Name | Description | +|---------|----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| success | Inference and post-processing completed. The flow file content is the original, unchanged image; the detected boxes are written to the configured output attribute as a JSON array (may be empty). | +| failure | The image could not be decoded, the input tensor could not be built, the model failed to run, or the model outputs could not be interpreted as scores + boxes. | + +### Output Attributes + +| Attribute | Relationship | Description | +|-------------------------|--------------|------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| object.count | success | Number of bounding boxes retained after confidence filtering and NMS. | +| | success | JSON array of the surviving bounding boxes (fields class_id, confidence, x_min, y_min, x_max, y_max; coordinates normalised to [0,1] against the original image). The attribute name is configurable via the 'Output attribute name' property. | + + +## DrawBoundingBox + +### Description + +Decodes the image from the flow file content, draws each bounding box supplied via the 'Bounding boxes' property onto it, and re-encodes the annotated image as PNG. Pair with an upstream DetectObject / FilterBoundingBoxes to visualise detections. + +### Properties + +In the list below, the names of required properties appear in bold. Any other properties (not in bold) are considered optional. The table also indicates any default values, and whether a property supports the NiFi Expression Language. + +| Name | Default Value | Allowable Values | Description | +|--------------------|---------------------|------------------|-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| Bounding boxes | ${enrichment.value} | | JSON array of bounding boxes to draw onto the image (fields class_id, confidence, x_min, y_min, x_max, y_max; coordinates normalised to [0,1] against the image). Typically the attribute produced by an upstream DetectObject or FilterBoundingBoxes processor.
**Supports Expression Language: true** | +| Line color | #00FF00 | | Outline color as a hex string (e.g., '#ff00ff' or '#f0f') | +| **Line thickness** | 5 | | Thickness in pixels of the drawn box outline. | + +### Relationships + +| Name | Description | +|---------|------------------------------------------------------------| +| success | Flowfiles are routed here after drawing the bounding boxes | +| failure | Invalid FlowFiles are routed here | + + ## EvaluateJsonPath ### Description @@ -2948,6 +3094,50 @@ In the list below, the names of required properties appear in bold. Any other pr | RouteText.Group | | The value captured by all capturing groups in the 'Grouping Regular Expression' property. If this property is not set, this attribute will not be added. | +## RunLlamaCppInference + +### Description + +LlamaCpp processor to use llama.cpp library for running language model inference. The inference will be based on the System Prompt and the Prompt property values, together with the content of the incoming flow file. In the Prompt, the content of the incoming flow file can be referred to as 'the input data' or 'the flow file content'. + +### Properties + +In the list below, the names of required properties appear in bold. Any other properties (not in bold) are considered optional. The table also indicates any default values, and whether a property supports the NiFi Expression Language. + +| Name | Default Value | Allowable Values | Description | +|----------------------------------|-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|------------------|-------------------------------------------------------------------------------------------------------------------------| +| **Model Path** | | | The filesystem path of the model file in gguf format. | +| Output Attribute Name | | | Specify the attribute to use as output, if not provided, the content is overridden instead. | +| MultiModal Model Path | | | The filesystem path of the multimodal model (visual, audio) file in gguf format. | +| Temperature | 0.8 | | The temperature to use for sampling. | +| Top K | 40 | | Limit the next token selection to the K most probable tokens. Set <= 0 value to use vocab size. | +| Top P | 0.9 | | Limit the next token selection to a subset of tokens with a cumulative probability above a threshold P. 1.0 = disabled. | +| Min P | | | Sets a minimum base probability threshold for token selection. 0.0 = disabled. | +| **Min Keep** | 0 | | If greater than 0, force samplers to return N possible tokens at minimum. | +| **Text Context Size** | 4096 | | Size of the text context, use 0 to use size set in model. | +| **Logical Maximum Batch Size** | 2048 | | Logical maximum batch size that can be submitted to the llama.cpp decode function. | +| **Physical Maximum Batch Size** | 512 | | Physical maximum batch size. | +| **Max Number Of Sequences** | 1 | | Maximum number of sequences (i.e. distinct states for recurrent models). | +| **Threads For Generation** | 4 | | Number of threads to use for generation. | +| **Threads For Batch Processing** | 4 | | Number of threads to use for batch processing. | +| Prompt | | | The user prompt for the inference.
**Supports Expression Language: true** | +| System Prompt | You are a helpful assistant. You are given a question with some possible input data otherwise called flow file content. You are expected to generate a response based on the question and the input data. | | The system prompt for the inference. | + +### Relationships + +| Name | Description | +|---------|----------------------------------| +| success | Generated results from the model | +| failure | Generation failed | + +### Output Attributes + +| Attribute | Relationship | Description | +|------------------------------|--------------|------------------------------------------------| +| llamacpp.time.to.first.token | success | Time to first token generated in milliseconds. | +| llamacpp.tokens.per.second | success | Tokens generated per second. | + + ## TailEventLog ### Description diff --git a/behave_framework/pyproject.toml b/behave_framework/pyproject.toml index d166376709..84fa62f287 100644 --- a/behave_framework/pyproject.toml +++ b/behave_framework/pyproject.toml @@ -9,7 +9,8 @@ dependencies = [ "PyYAML==6.0.3", "humanfriendly==10.0", "cryptography==50.0.0", - "pyjks==20.0.0" + "pyjks==20.0.0", + "certifi>=2026.7.22" ] [tool.setuptools] diff --git a/minifi_rust/extensions/minifi_rs_playground/minifi_rs_playground.md b/minifi_rust/extensions/minifi_rs_playground/minifi_rs_playground.md index e05adf0e0e..74ecd008fd 100644 --- a/minifi_rust/extensions/minifi_rs_playground/minifi_rs_playground.md +++ b/minifi_rust/extensions/minifi_rs_playground/minifi_rs_playground.md @@ -218,6 +218,7 @@ In the list below, the names of required properties appear in bold. Any other pr | Name | Default Value | Allowable Values | Description | |------------------------------------|---------------|-------------------|--------------------------------------------| +| Dummy Controller Service | | | Optional dummy controller service | | **Lorem Ipsum Controller Service** | | | Name of the lorem ipsum controller service | | **Write Method** | Buffer | Buffer
Stream | Which API to test | diff --git a/minifi_rust/extensions/minifi_rs_playground/src/processors/asciify_german.rs b/minifi_rust/extensions/minifi_rs_playground/src/processors/asciify_german.rs index 6d67c7d051..37f70e4860 100644 --- a/minifi_rust/extensions/minifi_rs_playground/src/processors/asciify_german.rs +++ b/minifi_rust/extensions/minifi_rs_playground/src/processors/asciify_german.rs @@ -21,7 +21,7 @@ use crate::processors::asciify_german::relationships::FAILURE; use minifi_native::macros::ComponentIdentifier; use minifi_native::{ FlowFileStreamTransform, GetProperty, InputStream, Logger, MinifiError, OutputStream, - ProcessError, RouteErrorExt, Schedule, TransformStreamResult, + ProcessError, Schedule, TransformStreamResult, }; mod relationships; @@ -56,8 +56,9 @@ impl FlowFileStreamTransform for AsciifyGerman { 0xC3 => { let mut next = [0u8; 1]; if input_stream.read(&mut next)? == 0 { - Err(MinifiError::custom("Truncated multi-byte sequence at EOF")) - .route_err_to_failure()? + return Err(ProcessError::route_to_failure( + "Truncated multi-byte sequence at EOF", + )); } match next[0] { 0xA4 => output_stream.write_all(b"ae")?, // ä diff --git a/minifi_rust/extensions/minifi_rs_playground/src/processors/get_file.rs b/minifi_rust/extensions/minifi_rs_playground/src/processors/get_file.rs index 7ec9f41300..b1f77ed267 100644 --- a/minifi_rust/extensions/minifi_rs_playground/src/processors/get_file.rs +++ b/minifi_rust/extensions/minifi_rs_playground/src/processors/get_file.rs @@ -27,7 +27,7 @@ use crate::processors::get_file::properties::{ use minifi_native::macros::ComponentIdentifier; use minifi_native::{ GetProperty, IoState, Logger, MinifiError, OnTriggerResult, ProcessContext, ProcessError, - ProcessSession, Schedule, Trigger, debug, info, trace, warn, + ProcessSession, Schedule, Trigger, debug, trace, warn, }; use std::collections::VecDeque; use std::error; @@ -179,7 +179,7 @@ impl GetFileRs { logger: &L, path: &Path, ) -> Result<(), MinifiError> { - info!(logger, "GetFile process {:?}", path); + trace!(logger, "GetFile process {:?}", path); let mut ff = session .create() .expect("Successful FlowFile creation is expected"); diff --git a/minifi_rust/extensions/minifi_tensor/Cargo.toml b/minifi_rust/extensions/minifi_tensor/Cargo.toml new file mode 100644 index 0000000000..5dfc8f9ea5 --- /dev/null +++ b/minifi_rust/extensions/minifi_tensor/Cargo.toml @@ -0,0 +1,25 @@ +[package] +name = "minifi_tensor" +version = "0.1.0" +edition = "2024" + +[lib] +crate-type = ["cdylib"] + +[features] +low-level-processors = [] + +[dependencies] +minifi_native = { path = "../../minifi_native" } +strum = "0.28.0" +strum_macros = "0.28.0" +image = "0.25.10" +imageproc = "0.27.0" +tract = "0.23.4" +ndarray = "0.17.2" +serde = { version = "1.0.229", features = ["derive"] } +serde_json = "1.0.151" + +[dev-dependencies] +minifi_native = { path = "../../minifi_native", features = ["test-utils"] } +tempfile = "3.27.0" diff --git a/minifi_rust/extensions/minifi_tensor/features/basic.feature b/minifi_rust/extensions/minifi_tensor/features/basic.feature new file mode 100644 index 0000000000..0718f98c1b --- /dev/null +++ b/minifi_rust/extensions/minifi_tensor/features/basic.feature @@ -0,0 +1,33 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +@SUPPORTS_WINDOWS +Feature: minifi_tensor extension loads + + Scenario: All processors and the TractModelService register cleanly + Given log property "logger.org::apache::nifi::minifi::core::extension::ExtensionManager" is set to "TRACE,stderr" + And log property "logger.org::apache::nifi::minifi::core::ClassLoader" is set to "TRACE,stderr" + + When the MiNiFi instance starts up + + Then the Minifi logs contain the following message: "Registering class 'ImageToTensor' at '/minifi_tensor'" in less than 10 seconds + And the Minifi logs contain the following message: "Registering class 'InvokeTractModel' at '/minifi_tensor'" in less than 1 seconds + And the Minifi logs contain the following message: "Registering class 'FilterBoundingBoxes' at '/minifi_tensor'" in less than 1 seconds + And the Minifi logs contain the following message: "Registering class 'ClassifyOutput' at '/minifi_tensor'" in less than 1 seconds + And the Minifi logs contain the following message: "Registering class 'TractModelService' at '/minifi_tensor'" in less than 1 seconds + And the Minifi logs contain the following message: "Registering class 'ClassifyImage' at '/minifi_tensor'" in less than 1 seconds + And the Minifi logs contain the following message: "Registering class 'DetectObject' at '/minifi_tensor'" in less than 1 seconds + And the Minifi logs do not contain errors + And the Minifi logs do not contain warnings diff --git a/minifi_rust/extensions/minifi_tensor/features/classification.feature b/minifi_rust/extensions/minifi_tensor/features/classification.feature new file mode 100644 index 0000000000..20be0f6b1e --- /dev/null +++ b/minifi_rust/extensions/minifi_tensor/features/classification.feature @@ -0,0 +1,128 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +@SUPPORTS_WINDOWS +Feature: Image classification with MobileNetV2 + + # Based on https://github.com/sonos/tract/tree/main/examples/onnx-mobilenet-v2 + Scenario: Grace Hopper image is classified using MobileNetV2 ONNX and imagenet labels (ImageToTensor + InvokeTract + ClassifyOutput) + Given a host resource file "grace_hopper.jpg" is copied to the "/tmp/input/grace_hopper.jpg" path in the MiNiFi container + And a host resource file "mobilenetv2-7.onnx" is copied to the "/tmp/models/mobilenetv2-7.onnx" path in the MiNiFi container + And a host resource file "imagenet_slim_labels.txt" is copied to the "/tmp/models/imagenet_slim_labels.txt" path in the MiNiFi container + + And a TractModelService controller service named "MobileNet" is set up and the "Model File Path" property set to "/tmp/models/mobilenetv2-7.onnx" + And the "Model format" property of the MobileNet controller service is set to "Onnx" + + And a GetFile processor with the "Input Directory" property set to "/tmp/input" + And the "Keep Source File" property of the GetFile processor is set to "false" + + And an ImageToTensor processor with the "Target width" property set to "224" + And the "Target height" property of the ImageToTensor processor is set to "224" + And the "Resize filter" property of the ImageToTensor processor is set to "Bilinear" + And the "Resize mode" property of the ImageToTensor processor is set to "Stretch" + And the "Color format" property of the ImageToTensor processor is set to "RGB" + And the "Tensor shape format" property of the ImageToTensor processor is set to "CHW" + And the "Mean" property of the ImageToTensor processor is set to "0.485, 0.456, 0.406" + And the "Standard Deviation" property of the ImageToTensor processor is set to "0.229, 0.224, 0.225" + And the "Pixel divisor" property of the ImageToTensor processor is set to "255" + + And an InvokeTractModel processor with the "Tract model service" property set to "MobileNet" + + And a ClassifyOutput processor with the "Top K" property set to "3" + And the "Score activation" property of the ClassifyOutput processor is set to "Softmax" + And the "Labels file path" property of the ClassifyOutput processor is set to "/tmp/models/imagenet_slim_labels.txt" + # ImageNet slim labels start with a dummy entry on line 0 + And the "Label index offset" property of the ClassifyOutput processor is set to "1" + And the "Confidence Threshold" property of the ClassifyOutput processor is set to "0.0" + + And a PutFile processor with the "Directory" property set to "/tmp/output" + + And a LogAttribute processor with the "FlowFiles To Log" property set to "0" + And LogAttribute is EVENT_DRIVEN + + And the "success" relationship of the GetFile processor is connected to the ImageToTensor + And the "success" relationship of the ImageToTensor processor is connected to the InvokeTractModel + And the "success" relationship of the InvokeTractModel processor is connected to the ClassifyOutput + And the "success" relationship of the ClassifyOutput processor is connected to the LogAttribute + And the "success" relationship of the LogAttribute processor is connected to the PutFile + And ImageToTensor's failure relationship is auto-terminated + And InvokeTractModel's failure relationship is auto-terminated + And ClassifyOutput's failure relationship is auto-terminated + And PutFile's success relationship is auto-terminated + And PutFile's failure relationship is auto-terminated + + When the MiNiFi instance starts up + + # Grace Hopper wears a US Navy uniform; MobileNetV2 on ImageNet consistently + # picks "military uniform" (occasionally "suit", "Windsor tie", or + # "bulletproof vest" as close runner-ups). Match any of those to keep the + # test resilient to small numeric differences across tract versions. + Then the Minifi logs match the following regex: "key:class.top1.name value:(military uniform|bulletproof vest|suit|Windsor tie)" in less than 60 seconds + And the Minifi logs match the following regex: "key:class.count value:[1-3]" in less than 1 seconds + And the Minifi logs contain the following message: "key:mime.type value:application/json" in less than 1 seconds + And at least one file in "/tmp/output" content match the following regex: "\"class_name\":\"(military uniform|bulletproof vest|suit|Windsor tie)\"" in less than 30 seconds + And the Minifi logs do not contain errors + + # Based on https://github.com/sonos/tract/tree/main/examples/onnx-mobilenet-v2 + Scenario: Grace Hopper image is classified using MobileNetV2 ONNX and imagenet labels (ClassifyImage) + Given a host resource file "grace_hopper.jpg" is copied to the "/tmp/input/grace_hopper.jpg" path in the MiNiFi container + And a host resource file "mobilenetv2-7.onnx" is copied to the "/tmp/models/mobilenetv2-7.onnx" path in the MiNiFi container + And a host resource file "imagenet_slim_labels.txt" is copied to the "/tmp/models/imagenet_slim_labels.txt" path in the MiNiFi container + + And a TractModelService controller service named "MobileNet" is set up and the "Model File Path" property set to "/tmp/models/mobilenetv2-7.onnx" + And the "Model format" property of the MobileNet controller service is set to "Onnx" + + And a GetFile processor with the "Input Directory" property set to "/tmp/input" + And the "Keep Source File" property of the GetFile processor is set to "false" + + And a ClassifyImage processor with the "Target width" property set to "224" + And the "Target height" property of the ClassifyImage processor is set to "224" + And the "Resize filter" property of the ClassifyImage processor is set to "Bilinear" + And the "Resize mode" property of the ClassifyImage processor is set to "Stretch" + And the "Color format" property of the ClassifyImage processor is set to "RGB" + And the "Tensor shape format" property of the ClassifyImage processor is set to "CHW" + And the "Mean" property of the ClassifyImage processor is set to "0.485, 0.456, 0.406" + And the "Standard Deviation" property of the ClassifyImage processor is set to "0.229, 0.224, 0.225" + And the "Pixel divisor" property of the ClassifyImage processor is set to "255" + And the "Tract model service" property of the ClassifyImage processor is set to "MobileNet" + And the "Top K" property of the ClassifyImage processor is set to "3" + And the "Score activation" property of the ClassifyImage processor is set to "Softmax" + And the "Labels file path" property of the ClassifyImage processor is set to "/tmp/models/imagenet_slim_labels.txt" + And the "Label index offset" property of the ClassifyImage processor is set to "1" + And the "Confidence Threshold" property of the ClassifyImage processor is set to "0.0" + + And a PutFile processor with the "Directory" property set to "/tmp/output" + + And a LogAttribute processor with the "FlowFiles To Log" property set to "0" + And LogAttribute is EVENT_DRIVEN + + And the "success" relationship of the GetFile processor is connected to the ClassifyImage + And the "success" relationship of the ClassifyImage processor is connected to the LogAttribute + And the "success" relationship of the LogAttribute processor is connected to the PutFile + And ClassifyImage's failure relationship is auto-terminated + And PutFile's success relationship is auto-terminated + And PutFile's failure relationship is auto-terminated + + When the MiNiFi instance starts up + + # Grace Hopper wears a US Navy uniform; MobileNetV2 on ImageNet consistently + # picks "military uniform" (occasionally "suit", "Windsor tie", or + # "bulletproof vest" as close runner-ups). Match any of those to keep the + # test resilient to small numeric differences across tract versions. + Then the Minifi logs match the following regex: "key:class.top1.name value:(military uniform|bulletproof vest|suit|Windsor tie)" in less than 60 seconds + And the Minifi logs match the following regex: "key:class.count value:[1-3]" in less than 1 seconds + And the Minifi logs contain the following message: "key:mime.type value:application/json" in less than 1 seconds + And at least one file in "/tmp/output" content match the following regex: "\"class_name\":\"(military uniform|bulletproof vest|suit|Windsor tie)\"" in less than 30 seconds + And the Minifi logs do not contain errors diff --git a/minifi_rust/extensions/minifi_tensor/features/detection.feature b/minifi_rust/extensions/minifi_tensor/features/detection.feature new file mode 100644 index 0000000000..9e807eb094 --- /dev/null +++ b/minifi_rust/extensions/minifi_tensor/features/detection.feature @@ -0,0 +1,131 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +@SUPPORTS_WINDOWS +Feature: Face detection with UltraFace Single Shot MultiBox Detector (SSD) + + # Based on https://github.com/sonos/tract/blob/main/examples/face_detection_yolov8onnx_example/src/main.rs + Scenario: Grace Hopper image yields at least one face detection (ImageToTensor + InvokeTract + FilterBoundingBoxes) + Given a host resource file "grace_hopper.jpg" is copied to the "/tmp/input/grace_hopper.jpg" path in the MiNiFi container + And a host resource file "version-RFB-320.onnx" is copied to the "/tmp/models/ultraface.onnx" path in the MiNiFi container + + And a TractModelService controller service named "UltraFace" is set up and the "Model File Path" property set to "/tmp/models/ultraface.onnx" + And the "Model format" property of the UltraFace controller service is set to "Onnx" + + And a GetFile processor with the "Input Directory" property set to "/tmp/input" + And the "Keep Source File" property of the GetFile processor is set to "false" + + And an ImageToTensor processor with the "Target width" property set to "320" + And the "Target height" property of the ImageToTensor processor is set to "240" + And the "Resize filter" property of the ImageToTensor processor is set to "Bilinear" + And the "Resize mode" property of the ImageToTensor processor is set to "Letterbox" + And the "Color format" property of the ImageToTensor processor is set to "RGB" + And the "Tensor shape format" property of the ImageToTensor processor is set to "CHW" + And the "Mean" property of the ImageToTensor processor is set to "127.0" + And the "Standard Deviation" property of the ImageToTensor processor is set to "128.0" + + And an InvokeTractModel processor with the "Tract model service" property set to "UltraFace" + + # UltraFace: output 0 = scores [1, N, 2] (softmax over background/face), + # output 1 = boxes [1, N, 4] in Xyxy normalised to 0..1. Class 0 is + # background so leave the defaults. + And a FilterBoundingBoxes processor with the "Confidence Threshold" property set to "0.5" + And the "IoU Threshold" property of the FilterBoundingBoxes processor is set to "0.45" + And the "Score output index" property of the FilterBoundingBoxes processor is set to "0" + And the "Box output index" property of the FilterBoundingBoxes processor is set to "1" + And the "Box format" property of the FilterBoundingBoxes processor is set to "Xyxy" + And the "Score activation" property of the FilterBoundingBoxes processor is set to "Softmax" + And the "Background class index" property of the FilterBoundingBoxes processor is set to "0" + + And a PutFile processor with the "Directory" property set to "/tmp/output" + + And a LogAttribute processor with the "FlowFiles To Log" property set to "0" + And LogAttribute is EVENT_DRIVEN + + And the "success" relationship of the GetFile processor is connected to the ImageToTensor + And the "success" relationship of the ImageToTensor processor is connected to the InvokeTractModel + And the "success" relationship of the InvokeTractModel processor is connected to the FilterBoundingBoxes + And the "success" relationship of the FilterBoundingBoxes processor is connected to the LogAttribute + And the "success" relationship of the LogAttribute processor is connected to the PutFile + And ImageToTensor's failure relationship is auto-terminated + And InvokeTractModel's failure relationship is auto-terminated + And FilterBoundingBoxes's failure relationship is auto-terminated + And PutFile's success relationship is auto-terminated + And PutFile's failure relationship is auto-terminated + + When the MiNiFi instance starts up + + Then the Minifi logs match the following regex: "key:object.count value:[1-9][0-9]*" in less than 60 seconds + And the Minifi logs contain the following message: "key:mime.type value:application/json" in less than 1 seconds + And at least one file in "/tmp/output" content match the following regex: "\"class_id\":1" in less than 30 seconds + And at least one file in "/tmp/output" content match the following regex: "\"confidence\":0\.[5-9][0-9]*" in less than 30 seconds + And the Minifi logs do not contain errors + + # Based on https://github.com/sonos/tract/blob/main/examples/face_detection_yolov8onnx_example/src/main.rs + Scenario: Grace Hopper image yields at least one face detection (DetectObject) + Given a host resource file "grace_hopper.jpg" is copied to the "/tmp/input/grace_hopper.jpg" path in the MiNiFi container + And a host resource file "version-RFB-320.onnx" is copied to the "/tmp/models/ultraface.onnx" path in the MiNiFi container + + And a TractModelService controller service named "UltraFace" is set up and the "Model File Path" property set to "/tmp/models/ultraface.onnx" + And the "Model format" property of the UltraFace controller service is set to "Onnx" + + And a GetFile processor with the "Input Directory" property set to "/tmp/input" + And the "Keep Source File" property of the GetFile processor is set to "false" + + And a DetectObject processor with the "Target width" property set to "320" + And the "Target height" property of the DetectObject processor is set to "240" + And the "Resize filter" property of the DetectObject processor is set to "Bilinear" + And the "Resize mode" property of the DetectObject processor is set to "Letterbox" + And the "Color format" property of the DetectObject processor is set to "RGB" + And the "Tensor shape format" property of the DetectObject processor is set to "CHW" + And the "Mean" property of the DetectObject processor is set to "127.0" + And the "Standard Deviation" property of the DetectObject processor is set to "128.0" + And the "Tract model service" property of the DetectObject processor is set to "UltraFace" + # UltraFace: output 0 = scores [1, N, 2] (softmax over background/face), + # output 1 = boxes [1, N, 4] in Xyxy normalised to 0..1. Class 0 is + # background so leave the defaults. + And the "Confidence Threshold" property of the DetectObject processor is set to "0.5" + And the "IoU Threshold" property of the DetectObject processor is set to "0.45" + And the "Score output index" property of the DetectObject processor is set to "0" + And the "Box output index" property of the DetectObject processor is set to "1" + And the "Box format" property of the DetectObject processor is set to "Xyxy" + And the "Score activation" property of the DetectObject processor is set to "Softmax" + And the "Background class index" property of the DetectObject processor is set to "0" + And the "Output attribute name" property of the DetectObject processor is set to "detected_objects" + + And a DrawBoundingBox processor with the "Bounding boxes" property set to "${detected_objects}" + And the "Line color" property of the DrawBoundingBox processor is set to "0, 255, 0" + And the "Line thickness" property of the DrawBoundingBox processor is set to "5" + + And a LogAttribute processor with the "FlowFiles To Log" property set to "0" + And LogAttribute is EVENT_DRIVEN + + And a PutFile processor with the "Directory" property set to "/tmp/output" + + And the "success" relationship of the GetFile processor is connected to the DetectObject + And the "success" relationship of the DetectObject processor is connected to the LogAttribute + And the "success" relationship of the LogAttribute processor is connected to the DrawBoundingBox + And the "success" relationship of the DrawBoundingBox processor is connected to the PutFile + And DetectObject's failure relationship is auto-terminated + And DrawBoundingBox's failure relationship is auto-terminated + And PutFile's success relationship is auto-terminated + And PutFile's failure relationship is auto-terminated + + When the MiNiFi instance starts up + + Then the Minifi logs match the following regex: "key:object.count value:[1-9][0-9]*" in less than 60 seconds + And the Minifi logs match the following regex: "key:detected_objects value:.*\"class_id\":1" in less than 1 seconds + And the Minifi logs match the following regex: "key:detected_objects value:.*\"confidence\":0\.[5-9][0-9]*" in less than 1 seconds + And the Minifi logs do not contain errors diff --git a/minifi_rust/extensions/minifi_tensor/features/environment.py b/minifi_rust/extensions/minifi_tensor/features/environment.py new file mode 100644 index 0000000000..ccb2e5870c --- /dev/null +++ b/minifi_rust/extensions/minifi_tensor/features/environment.py @@ -0,0 +1,125 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +import hashlib +import os +import shutil +import ssl +import urllib.request + +import certifi +from minifi_behave.core.hooks import ( + add_extension_to_minifi_container, + common_after_scenario, + common_before_scenario, +) + +_SSL_CONTEXT = ssl.create_default_context(cafile=certifi.where()) +_DOWNLOAD_TIMEOUT_S = 60 + + +class RemoteAsset: + def __init__(self, url: str, sha256: str): + self.url = url + self.sha256 = sha256 + + def acquire(self, cache_dir: str, filename: str) -> str: + dest = os.path.join(cache_dir, filename) + if os.path.exists(dest) and self._verify(dest): + return dest + os.makedirs(cache_dir, exist_ok=True) + tmp = dest + ".part" + print(f"[minifi_tensor tests] fetching {filename} from {self.url}") + try: + with ( + urllib.request.urlopen(self.url, context=_SSL_CONTEXT, timeout=_DOWNLOAD_TIMEOUT_S) as response, + open(tmp, "wb") as out, + ): + shutil.copyfileobj(response, out) + except OSError as e: + if os.path.exists(tmp): + os.remove(tmp) + raise RuntimeError(f"failed to fetch {filename} from {self.url}: {e}") from e + if not self._verify(tmp): + actual = self._digest(tmp) + os.remove(tmp) + raise RuntimeError(f"sha256 mismatch for {filename}: expected {self.sha256}, got {actual}") + os.replace(tmp, dest) + return dest + + def _verify(self, path: str) -> bool: + return self._digest(path) == self.sha256 + + @staticmethod + def _digest(path: str) -> str: + h = hashlib.sha256() + with open(path, "rb") as f: + for chunk in iter(lambda: f.read(1 << 20), b""): + h.update(chunk) + return h.hexdigest() + + +# Model / label / image assets fetched on first use. All hosted on +# public buckets or the sonos/tract repo +REMOTE_ASSETS: dict[str, RemoteAsset] = { + # ImageNet MobileNetV2 classifier (~14 MB) — the reference model used by tract's unit tests + "mobilenetv2-7.onnx": RemoteAsset( + "https://s3.amazonaws.com/tract-ci-builds/tests/mobilenetv2-7.onnx", + "c1c513582d56afceff8516c73804e484c81c6a830712ab6d682253f4a3cd042f", + ), + # 1000-class ImageNet labels (line N = class N; line 0 is "dummy") + "imagenet_slim_labels.txt": RemoteAsset( + "https://raw.githubusercontent.com/sonos/tract/main/examples/onnx-mobilenet-v2/imagenet_slim_labels.txt", + "e8d2cef25bb7b3c8c6923ad3c463b47de8b8535cadf4bd62a2ca2532c587eb9f", + ), + # Same test image tract's example uses. MobileNetV2 confidently + # classifies this as "military uniform". + "grace_hopper.jpg": RemoteAsset( + "https://raw.githubusercontent.com/sonos/tract/main/examples/onnx-mobilenet-v2/grace_hopper.jpg", + "e1f57e98cf38076c0f9a058d74ffddf90f20453e436033784606b63c8ed2e49a", + ), + # UltraFace RFB-320 (~1.2 MB): 2-output SSD-style detector matching the + # existing FilterBoundingBoxes defaults (Xyxy boxes, class 0 = background, + # softmax over 2 classes: background/face). 320x240 RGB, mean=127, std=128. + "version-RFB-320.onnx": RemoteAsset( + "https://github.com/onnx/models/raw/refs/heads/main/validated/vision/" + "body_analysis/ultraface/models/version-RFB-320.onnx", + "34cd7e60aeff28744c657de7a3dc64e872d506741de66987f3426f2b79f88017", + ), +} + + +def before_all(context): + dir_path = os.path.dirname(os.path.realpath(__file__)) + build_path = os.path.normpath(os.path.join(dir_path, "../../../target/release/")) + deps_build_path = os.path.normpath(os.path.join(dir_path, "../../../target/release/deps/")) + add_extension_to_minifi_container("minifi_tensor", [build_path, deps_build_path], context) + + context.tensor_resource_dir = os.path.join(dir_path, "../../../target/test_resources") + os.makedirs(context.tensor_resource_dir, exist_ok=True) + for name, asset in REMOTE_ASSETS.items(): + asset.acquire(context.tensor_resource_dir, name) + + +def before_scenario(context, scenario): + context.minifi_container_image = "apacheminificpp:minifi_tensor" + common_before_scenario(context, scenario) + context.resource_dir = context.tensor_resource_dir + + +def after_scenario(context, scenario): + common_after_scenario(context, scenario) diff --git a/minifi_rust/extensions/minifi_tensor/features/image_to_tensor.feature b/minifi_rust/extensions/minifi_tensor/features/image_to_tensor.feature new file mode 100644 index 0000000000..a7e82d9155 --- /dev/null +++ b/minifi_rust/extensions/minifi_tensor/features/image_to_tensor.feature @@ -0,0 +1,105 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +@SUPPORTS_WINDOWS +Feature: ImageToTensor preprocesses image bytes into normalised tensors + + Scenario: RGB CHW 224x224 tensor is produced with the expected shape and dtype + Given a host resource file "grace_hopper.jpg" is copied to the "/tmp/input/grace_hopper.jpg" path in the MiNiFi container + And a GetFile processor with the "Input Directory" property set to "/tmp/input" + And the "Keep Source File" property of the GetFile processor is set to "false" + And an ImageToTensor processor with the "Target width" property set to "224" + And the "Target height" property of the ImageToTensor processor is set to "224" + And the "Resize filter" property of the ImageToTensor processor is set to "Bilinear" + And the "Resize mode" property of the ImageToTensor processor is set to "Stretch" + And the "Color format" property of the ImageToTensor processor is set to "RGB" + And the "Tensor shape format" property of the ImageToTensor processor is set to "CHW" + And the "Mean" property of the ImageToTensor processor is set to "0.485, 0.456, 0.406" + And the "Standard Deviation" property of the ImageToTensor processor is set to "0.229, 0.224, 0.225" + And a LogAttribute processor with the "FlowFiles To Log" property set to "0" + And LogAttribute is EVENT_DRIVEN + And the "success" relationship of the GetFile processor is connected to the ImageToTensor + And the "success" relationship of the ImageToTensor processor is connected to the LogAttribute + And LogAttribute's success relationship is auto-terminated + And ImageToTensor's failure relationship is auto-terminated + + When the MiNiFi instance starts up + + Then the Minifi logs contain the following message: "key:tensor.0.shape value:1,3,224,224" in less than 30 seconds + And the Minifi logs contain the following message: "key:tensor.0.dtype value:F32" in less than 1 seconds + And the Minifi logs do not contain errors + + Scenario: HWC layout is reflected in the tensor.0.shape attribute + Given a host resource file "grace_hopper.jpg" is copied to the "/tmp/input/grace_hopper.jpg" path in the MiNiFi container + And a GetFile processor with the "Input Directory" property set to "/tmp/input" + And the "Keep Source File" property of the GetFile processor is set to "false" + And an ImageToTensor processor with the "Target width" property set to "300" + And the "Target height" property of the ImageToTensor processor is set to "300" + And the "Tensor shape format" property of the ImageToTensor processor is set to "HWC" + And the "Color format" property of the ImageToTensor processor is set to "RGB" + And the "Mean" property of the ImageToTensor processor is set to "0.0" + And the "Standard Deviation" property of the ImageToTensor processor is set to "255.0" + And a LogAttribute processor with the "FlowFiles To Log" property set to "0" + And LogAttribute is EVENT_DRIVEN + And the "success" relationship of the GetFile processor is connected to the ImageToTensor + And the "success" relationship of the ImageToTensor processor is connected to the LogAttribute + And LogAttribute's success relationship is auto-terminated + And ImageToTensor's failure relationship is auto-terminated + + When the MiNiFi instance starts up + + Then the Minifi logs contain the following message: "key:tensor.0.shape value:1,300,300,3" in less than 30 seconds + And the Minifi logs do not contain errors + + Scenario: Letterbox mode still produces the target dimensions + Given a host resource file "grace_hopper.jpg" is copied to the "/tmp/input/grace_hopper.jpg" path in the MiNiFi container + And a GetFile processor with the "Input Directory" property set to "/tmp/input" + And the "Keep Source File" property of the GetFile processor is set to "false" + And an ImageToTensor processor with the "Target width" property set to "320" + And the "Target height" property of the ImageToTensor processor is set to "240" + And the "Resize mode" property of the ImageToTensor processor is set to "Letterbox" + And the "Letterbox pad value" property of the ImageToTensor processor is set to "0.0" + And the "Color format" property of the ImageToTensor processor is set to "RGB" + And the "Tensor shape format" property of the ImageToTensor processor is set to "CHW" + And the "Mean" property of the ImageToTensor processor is set to "127.0" + And the "Standard Deviation" property of the ImageToTensor processor is set to "128.0" + And a LogAttribute processor with the "FlowFiles To Log" property set to "0" + And LogAttribute is EVENT_DRIVEN + And the "success" relationship of the GetFile processor is connected to the ImageToTensor + And the "success" relationship of the ImageToTensor processor is connected to the LogAttribute + And LogAttribute's success relationship is auto-terminated + And ImageToTensor's failure relationship is auto-terminated + + When the MiNiFi instance starts up + + Then the Minifi logs contain the following message: "key:tensor.0.shape value:1,3,240,320" in less than 30 seconds + And the Minifi logs do not contain errors + + Scenario: Invalid image bytes route to failure without crashing MiNiFi + Given a directory at "/tmp/input" has a file "not_an_image.bin" with the content "garbage bytes not a real image" + And a GetFile processor with the "Input Directory" property set to "/tmp/input" + And the "Keep Source File" property of the GetFile processor is set to "false" + And an ImageToTensor processor with the "Target width" property set to "224" + And the "Target height" property of the ImageToTensor processor is set to "224" + And a PutFile processor with the "Directory" property set to "/tmp/output" + And the "success" relationship of the GetFile processor is connected to the ImageToTensor + And the "failure" relationship of the ImageToTensor processor is connected to the PutFile + And ImageToTensor's success relationship is auto-terminated + And PutFile's success relationship is auto-terminated + And PutFile's failure relationship is auto-terminated + + When the MiNiFi instance starts up + + Then at least one file with the content "garbage bytes not a real image" is placed in the "/tmp/output" directory in less than 30 seconds diff --git a/minifi_rust/extensions/minifi_tensor/features/steps/steps.py b/minifi_rust/extensions/minifi_tensor/features/steps/steps.py new file mode 100644 index 0000000000..231d2d53bb --- /dev/null +++ b/minifi_rust/extensions/minifi_tensor/features/steps/steps.py @@ -0,0 +1,23 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +from minifi_behave.steps import ( + checking_steps, # noqa: F401 + configuration_steps, # noqa: F401 + core_steps, # noqa: F401 + flow_building_steps, # noqa: F401 +) diff --git a/minifi_rust/extensions/minifi_tensor/minifi_tensor.md b/minifi_rust/extensions/minifi_tensor/minifi_tensor.md new file mode 100644 index 0000000000..1feeaca86f --- /dev/null +++ b/minifi_rust/extensions/minifi_tensor/minifi_tensor.md @@ -0,0 +1,317 @@ + + +## Table of Contents + +### Processors + +- [ClassifyImage](#ClassifyImage) +- [ClassifyOutput](#ClassifyOutput) +- [DetectObject](#DetectObject) +- [DrawBoundingBox](#DrawBoundingBox) +- [FilterBoundingBoxes](#FilterBoundingBoxes) +- [ImageToTensor](#ImageToTensor) +- [InvokeTractModel](#InvokeTractModel) +### Controller Services + +- [TractModelService](#TractModelService) + + +## ClassifyImage + +### Description + +Runs a full image-classification pass in a single processor: decodes the image from the flow file content, resizes and normalises it into an input tensor, runs one inference against the compiled model owned by the referenced TractModelService, and post-processes the score vector (score activation, Top-K selection, confidence filtering, optional label lookup) into predictions. Collapses the ImageToTensor -> InvokeTractModel -> ClassifyOutput chain into one node. The flow file content is left unchanged (the original image); the Top-K classifications are written as a JSON array to the configured output attribute. + +### Properties + +In the list below, the names of required properties appear in bold. Any other properties (not in bold) are considered optional. The table also indicates any default values, and whether a property supports the NiFi Expression Language. + +| Name | Default Value | Allowable Values | Description | +|--------------------------|---------------|-----------------------------------------------|----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| **Color format** | RGB | RGB
BGR
Grayscale | Color space of the tensor fed to the model. RGB and BGR produce three-channel tensors (channel order determined by the format); Grayscale produces a single-channel luma tensor. | +| **Confidence Threshold** | 0.0 | | Minimum confidence a class must reach to be included in the output JSON. Applied AFTER activation, so the units match the chosen activation (0.0..=1.0 for Softmax/Sigmoid, model-native for None). Set to 0.0 to always emit exactly Top K predictions.
**Supports Expression Language: true** | +| Label index offset | | | Offset added to the model's class ID when looking up a name in the labels file. Defaults to 0 (labels file line N = class N). Set to 1 for label files that start with a dummy/background entry — e.g. the ONNX MobileNetV2 model emits 1000 class scores while 'imagenet_slim_labels.txt' has 1001 lines (line 0 = 'dummy'), so class ID 653 maps to line 654 = 'military uniform'. | +| Labels file path | | | Optional path to a newline-separated labels file (line N = name of class N). Loaded once at service enable time. When set, each prediction in the output JSON gains a 'class_name' field and the 'class.top1.name' flow file attribute is populated. Leave empty to emit numeric class IDs only. | +| **Letterbox pad value** | 0.0 | | Value written for padding pixels when 'Resize mode' is 'Letterbox'. This is a normalised value (post mean/std), so 0.0 corresponds to a neutral input for most networks. Ignored when 'Resize mode' is 'Stretch'. | +| Mean | 0.0 | | Value subtracted from each pixel before dividing by 'Standard Deviation'. Accepts either a single value (broadcast to all channels) or three comma-separated values applied per channel in the order dictated by 'Color format'. Example: '0.485, 0.456, 0.406' for ImageNet-style RGB normalisation. | +| Output attribute name | | | Specify the attribute to use as output, if not provided, the content is overridden instead.
**Supports Expression Language: true** | +| **Pixel divisor** | 1.0 | | Divisor applied to raw u8 pixel values before subtracting 'Mean' and dividing by 'Standard Deviation'. Defaults to 1.0 (mean/std interpreted in [0, 255] pixel space, e.g. UltraFace's mean=127, std=128). Set to 255 to bring pixels into [0.0, 1.0] first so ImageNet-style mean/std values like '0.485, 0.456, 0.406' / '0.229, 0.224, 0.225' can be used directly, matching the PyTorch / torchvision / ONNX MobileNet convention. Must be non-zero. | +| **Resize filter** | Bilinear | Nearest
Bilinear
Bicubic
Lanczos3 | Interpolation filter applied when resizing the decoded image. Nearest is fastest but blocky; Bilinear is a good default; Bicubic and Lanczos3 are higher-quality but slower. | +| **Resize mode** | Stretch | Stretch
Letterbox | How the source image is fitted into the target dimensions. 'Stretch' scales each axis independently, distorting aspect ratio. 'Letterbox' preserves aspect ratio and pads the remaining border with 'Letterbox pad value' (applied in normalised output space). | +| **Score activation** | Softmax | Softmax
Sigmoid
None | Activation applied to the raw score vector before ranking. Softmax = mutually-exclusive classes (ImageNet-trained ResNet/MobileNet/EfficientNet raw logits). Sigmoid = independent classes (multi-label classifiers). None = the model already emits probabilities/scores; rank the raw values. | +| **Score output index** | 0 | | Zero-based index of the model output tensor that holds classification scores. The processor slices the concatenated payload from InvokeTractModel according to the 'tensor.N.bytes' attributes. Almost always 0 for single-head classifiers. | +| Standard Deviation | 255.0 | | Divisor applied after subtracting 'Mean'. Accepts a single value (broadcast) or three comma-separated values (per channel). Must be non-zero. Example: '255.0' to scale u8 pixels into [0.0, 1.0]; '0.229, 0.224, 0.225' for ImageNet. | +| **Target height** | | | Height in pixels the decoded image is resized to before normalisation and inference. | +| **Target width** | | | Width in pixels the decoded image is resized to before normalisation and inference. | +| **Tensor shape format** | CHW | CHW
HWC | Memory layout of the tensor fed to the model. CHW (channels-first) is typical for PyTorch/ONNX detectors. HWC (channels-last) matches TensorFlow/TFLite. Ignored for Grayscale (always effectively 1xHxW). | +| **Top K** | 5 | | Number of highest-scoring classes to include in the output JSON, in descending order of confidence. Values above the total class count are clamped. Set to 1 for pure top-1 classification. | +| **Tract model service** | | | Reference to a TractModelService controller service. The referenced service owns the compiled model (ONNX or NNEF) that will be evaluated for each incoming flow file. | + +### Relationships + +| Name | Description | +|---------|-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| failure | The image could not be decoded, the input tensor could not be built, the model failed to run, or the model outputs could not be interpreted as classification. | +| success | Inference and post-processing completed. The flow file content is the original, unchanged image; the classifications are written to the configured output attribute as a JSON array (may be empty). | + +### Output Attributes + +| Attribute | Relationship | Description | +|-----------------------|--------------|----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| class.count | success | Number of predictions retained after Top K selection and confidence filtering. | +| class.top1.confidence | success | Confidence (post-activation) of the highest-confidence prediction, when at least one prediction cleared the confidence threshold. | +| class.top1.id | success | Numeric class ID of the highest-confidence prediction, when at least one prediction cleared the confidence threshold. | +| class.top1.name | success | Label of the highest-confidence prediction. Only present when 'Labels file path' was configured and at least one prediction cleared the threshold. | +| mime.type | success | If the "Output attribute name" is None, then the content will be overridden with the JSON array of objects with fields class_id, confidence, and optional class_name, and the mime type will be set to 'application/json'. | + + +## ClassifyOutput + +### Description + +Post-processes the output of a classification model invoked via InvokeTractModel. Reads the flattened score vector from the configured output tensor index, applies the chosen activation (softmax / sigmoid / none), and emits the Top K classes as a JSON array [{class_id, confidence, class_name?}, ...]. Optional labels file maps numeric class IDs to human-readable names. Works with ImageNet-style ResNet/MobileNet/EfficientNet checkpoints (Softmax over raw logits) as well as multi-label classifiers (Sigmoid) and models that already emit probabilities (None). Assumes a single flattened score vector — upstream ImageToTensor produces batch=1 tensors, so this is the common case. + +### Properties + +In the list below, the names of required properties appear in bold. Any other properties (not in bold) are considered optional. The table also indicates any default values, and whether a property supports the NiFi Expression Language. + +| Name | Default Value | Allowable Values | Description | +|--------------------------|---------------|------------------------------|----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| **Confidence Threshold** | 0.0 | | Minimum confidence a class must reach to be included in the output JSON. Applied AFTER activation, so the units match the chosen activation (0.0..=1.0 for Softmax/Sigmoid, model-native for None). Set to 0.0 to always emit exactly Top K predictions.
**Supports Expression Language: true** | +| Label index offset | | | Offset added to the model's class ID when looking up a name in the labels file. Defaults to 0 (labels file line N = class N). Set to 1 for label files that start with a dummy/background entry — e.g. the ONNX MobileNetV2 model emits 1000 class scores while 'imagenet_slim_labels.txt' has 1001 lines (line 0 = 'dummy'), so class ID 653 maps to line 654 = 'military uniform'. | +| Labels file path | | | Optional path to a newline-separated labels file (line N = name of class N). Loaded once at service enable time. When set, each prediction in the output JSON gains a 'class_name' field and the 'class.top1.name' flow file attribute is populated. Leave empty to emit numeric class IDs only. | +| Output attribute name | | | Specify the attribute to use as output, if not provided, the content is overridden instead.
**Supports Expression Language: true** | +| **Score activation** | Softmax | Softmax
Sigmoid
None | Activation applied to the raw score vector before ranking. Softmax = mutually-exclusive classes (ImageNet-trained ResNet/MobileNet/EfficientNet raw logits). Sigmoid = independent classes (multi-label classifiers). None = the model already emits probabilities/scores; rank the raw values. | +| **Score output index** | 0 | | Zero-based index of the model output tensor that holds classification scores. The processor slices the concatenated payload from InvokeTractModel according to the 'tensor.N.bytes' attributes. Almost always 0 for single-head classifiers. | +| **Top K** | 5 | | Number of highest-scoring classes to include in the output JSON, in descending order of confidence. Values above the total class count are clamped. Set to 1 for pure top-1 classification. | + +### Relationships + +| Name | Description | +|---------|----------------------------------------------------------------------------------------------------------------------------------------------------------| +| failure | The upstream output attributes were missing/invalid or the score tensor could not be interpreted as f32 values. | +| success | Classification completed. The flow file content is a JSON array of the Top K predictions (possibly fewer if the confidence threshold filtered some out). | + +### Output Attributes + +| Attribute | Relationship | Description | +|-----------------------|--------------|----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| class.count | success | Number of predictions retained after Top K selection and confidence filtering. | +| class.top1.confidence | success | Confidence (post-activation) of the highest-confidence prediction, when at least one prediction cleared the confidence threshold. | +| class.top1.id | success | Numeric class ID of the highest-confidence prediction, when at least one prediction cleared the confidence threshold. | +| class.top1.name | success | Label of the highest-confidence prediction. Only present when 'Labels file path' was configured and at least one prediction cleared the threshold. | +| mime.type | success | If the "Output attribute name" is None, then the content will be overridden with the JSON array of objects with fields class_id, confidence, and optional class_name, and the mime type will be set to 'application/json'. | + + +## DetectObject + +### Description + +Runs a full object-detection pass in a single processor: decodes the image from the flow file content, resizes and normalises it into an input tensor, runs one inference against the compiled model owned by the referenced TractModelService, and post-processes the model outputs (score activation, confidence filtering, box decoding, per-class non-maximum suppression) into bounding boxes. Collapses the ImageToTensor -> InvokeTractModel -> FilterBoundingBoxes chain into one node. The flow file content is left unchanged (the original image); the detected boxes are written as a JSON array to the configured output attribute so a downstream DrawBoundingBox can annotate the image. + +### Properties + +In the list below, the names of required properties appear in bold. Any other properties (not in bold) are considered optional. The table also indicates any default values, and whether a property supports the NiFi Expression Language. + +| Name | Default Value | Allowable Values | Description | +|--------------------------|---------------|-----------------------------------------------|----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| Background class index | | | Index of the 'background / no-object' class. Boxes whose winning class equals this index are dropped. In score-matrix mode this is only honoured when the score tensor has more than one class per box; in 'Class output index' mode it is matched against each box's class id. | +| **Box format** | Xyxy | Xyxy
Yxyx
Cxcywh | Layout of the four floats per box in the box output tensor. Xyxy = [x_min, y_min, x_max, y_max] (SSD, MobileNet-SSD, most PyTorch exports). Yxyx = [y_min, x_min, y_max, x_max] (TensorFlow Object Detection API). Cxcywh = [cx, cy, w, h] (YOLOv3/5/8 raw output). | +| **Box output index** | 1 | | Zero-based index of the model output tensor that holds box coordinates. Must differ from 'Score output index'. | +| Class output index | | | Zero-based index of a model output tensor that holds one class id per box. Set this for detectors that emit boxes, per-box scores, and class ids as three separate parallel tensors, with NMS already folded into the graph (TensorFlow Object Detection API; YOLO / EfficientNMS 'end2end' exports). When set, 'Score output index' is read as one score per box (not a [boxes, classes] matrix) and no argmax is performed; the class id tensor may be integer- or float-typed. Leave empty for models that emit a per-class score matrix. | +| **Color format** | RGB | RGB
BGR
Grayscale | Color space of the tensor fed to the model. RGB and BGR produce three-channel tensors (channel order determined by the format); Grayscale produces a single-channel luma tensor. | +| **Confidence Threshold** | 0.7 | | Minimum per-box class probability (0.0 to 1.0) required to keep a bounding box after applying the chosen 'Score activation'. Boxes below the threshold are discarded before NMS.
**Supports Expression Language: true** | +| **IoU Threshold** | 0.45 | | Intersection-over-union cutoff used during non-maximum suppression. Boxes of the same class whose IoU with a higher-confidence peer exceeds this value are suppressed. Typical values: 0.45 (SSD/YOLO default), 0.5, 0.3 for stricter deduplication. | +| **Letterbox pad value** | 0.0 | | Value written for padding pixels when 'Resize mode' is 'Letterbox'. This is a normalised value (post mean/std), so 0.0 corresponds to a neutral input for most networks. Ignored when 'Resize mode' is 'Stretch'. | +| Mean | 0.0 | | Value subtracted from each pixel before dividing by 'Standard Deviation'. Accepts either a single value (broadcast to all channels) or three comma-separated values applied per channel in the order dictated by 'Color format'. Example: '0.485, 0.456, 0.406' for ImageNet-style RGB normalisation. | +| Output attribute name | | | Specify the attribute to use as output, if not provided, the content is overridden instead.
**Supports Expression Language: true** | +| **Pixel divisor** | 1.0 | | Divisor applied to raw u8 pixel values before subtracting 'Mean' and dividing by 'Standard Deviation'. Defaults to 1.0 (mean/std interpreted in [0, 255] pixel space, e.g. UltraFace's mean=127, std=128). Set to 255 to bring pixels into [0.0, 1.0] first so ImageNet-style mean/std values like '0.485, 0.456, 0.406' / '0.229, 0.224, 0.225' can be used directly, matching the PyTorch / torchvision / ONNX MobileNet convention. Must be non-zero. | +| **Resize filter** | Bilinear | Nearest
Bilinear
Bicubic
Lanczos3 | Interpolation filter applied when resizing the decoded image. Nearest is fastest but blocky; Bilinear is a good default; Bicubic and Lanczos3 are higher-quality but slower. | +| **Resize mode** | Stretch | Stretch
Letterbox | How the source image is fitted into the target dimensions. 'Stretch' scales each axis independently, distorting aspect ratio. 'Letterbox' preserves aspect ratio and pads the remaining border with 'Letterbox pad value' (applied in normalised output space). | +| **Score activation** | Softmax | Softmax
Sigmoid
None | Activation applied to raw per-class scores before selecting the winning class. Softmax = mutually-exclusive classes (SSD/MobileNet-SSD raw logits). Sigmoid = independent classes (YOLOv5/v8 style). None = the model already emits probabilities/scores; use raw argmax with the raw score as confidence. | +| **Score output index** | 0 | | Zero-based index of the model output tensor that holds classification scores. The processor slices the concatenated payload from InvokeTractModel according to the 'tensor.N.bytes' attributes. | +| Standard Deviation | 255.0 | | Divisor applied after subtracting 'Mean'. Accepts a single value (broadcast) or three comma-separated values (per channel). Must be non-zero. Example: '255.0' to scale u8 pixels into [0.0, 1.0]; '0.229, 0.224, 0.225' for ImageNet. | +| **Target height** | | | Height in pixels the decoded image is resized to before normalisation and inference. | +| **Target width** | | | Width in pixels the decoded image is resized to before normalisation and inference. | +| **Tensor shape format** | CHW | CHW
HWC | Memory layout of the tensor fed to the model. CHW (channels-first) is typical for PyTorch/ONNX detectors. HWC (channels-last) matches TensorFlow/TFLite. Ignored for Grayscale (always effectively 1xHxW). | +| **Tract model service** | | | Reference to a TractModelService controller service. The referenced service owns the compiled model (ONNX or NNEF) that will be evaluated for each incoming flow file. | + +### Relationships + +| Name | Description | +|---------|----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| failure | The image could not be decoded, the input tensor could not be built, the model failed to run, or the model outputs could not be interpreted as scores + boxes. | +| success | Inference and post-processing completed. The flow file content is the original, unchanged image; the detected boxes are written to the configured output attribute as a JSON array (may be empty). | + +### Output Attributes + +| Attribute | Relationship | Description | +|-------------------------|--------------|------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| | success | JSON array of the surviving bounding boxes (fields class_id, confidence, x_min, y_min, x_max, y_max; coordinates normalised to [0,1] against the original image). The attribute name is configurable via the 'Output attribute name' property. | +| object.count | success | Number of bounding boxes retained after confidence filtering and NMS. | + + +## DrawBoundingBox + +### Description + +Decodes the image from the flow file content, draws each bounding box supplied via the 'Bounding boxes' property onto it, and re-encodes the annotated image as PNG. Pair with an upstream DetectObject / FilterBoundingBoxes to visualise detections. + +### Properties + +In the list below, the names of required properties appear in bold. Any other properties (not in bold) are considered optional. The table also indicates any default values, and whether a property supports the NiFi Expression Language. + +| Name | Default Value | Allowable Values | Description | +|--------------------|---------------------|------------------|-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| Bounding boxes | ${enrichment.value} | | JSON array of bounding boxes to draw onto the image (fields class_id, confidence, x_min, y_min, x_max, y_max; coordinates normalised to [0,1] against the image). Typically the attribute produced by an upstream DetectObject or FilterBoundingBoxes processor.
**Supports Expression Language: true** | +| Line color | #00FF00 | | Outline color as a hex string (e.g., '#ff00ff' or '#f0f') | +| **Line thickness** | 5 | | Thickness in pixels of the drawn box outline. | + +### Relationships + +| Name | Description | +|---------|------------------------------------------------------------| +| failure | Invalid FlowFiles are routed here | +| success | Flowfiles are routed here after drawing the bounding boxes | + + +## FilterBoundingBoxes + +### Description + +Post-processes the concatenated output of InvokeTractModel for object-detection models. Reads the classification score tensor and the box coordinate tensor from the flow file payload (indices configurable), applies the configured score activation, filters by confidence, decodes box coordinates from the configured layout, and applies per-class non-maximum suppression at the configured IoU threshold. Handles both per-class score matrices (argmax per box) and detectors that emit boxes / per-box scores / class ids as separate tensors with NMS folded into the graph (set 'Class output index'). Works with SSD-, YOLO-, and TensorFlow-style detectors by tuning properties — no code changes needed for common model families. Emits a JSON array of the surviving boxes. + +### Properties + +In the list below, the names of required properties appear in bold. Any other properties (not in bold) are considered optional. The table also indicates any default values, and whether a property supports the NiFi Expression Language. + +| Name | Default Value | Allowable Values | Description | +|--------------------------|---------------|------------------------------|----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| Background class index | | | Index of the 'background / no-object' class. Boxes whose winning class equals this index are dropped. In score-matrix mode this is only honoured when the score tensor has more than one class per box; in 'Class output index' mode it is matched against each box's class id. | +| **Box format** | Xyxy | Xyxy
Yxyx
Cxcywh | Layout of the four floats per box in the box output tensor. Xyxy = [x_min, y_min, x_max, y_max] (SSD, MobileNet-SSD, most PyTorch exports). Yxyx = [y_min, x_min, y_max, x_max] (TensorFlow Object Detection API). Cxcywh = [cx, cy, w, h] (YOLOv3/5/8 raw output). | +| **Box output index** | 1 | | Zero-based index of the model output tensor that holds box coordinates. Must differ from 'Score output index'. | +| Class output index | | | Zero-based index of a model output tensor that holds one class id per box. Set this for detectors that emit boxes, per-box scores, and class ids as three separate parallel tensors, with NMS already folded into the graph (TensorFlow Object Detection API; YOLO / EfficientNMS 'end2end' exports). When set, 'Score output index' is read as one score per box (not a [boxes, classes] matrix) and no argmax is performed; the class id tensor may be integer- or float-typed. Leave empty for models that emit a per-class score matrix. | +| **Confidence Threshold** | 0.7 | | Minimum per-box class probability (0.0 to 1.0) required to keep a bounding box after applying the chosen 'Score activation'. Boxes below the threshold are discarded before NMS.
**Supports Expression Language: true** | +| **IoU Threshold** | 0.45 | | Intersection-over-union cutoff used during non-maximum suppression. Boxes of the same class whose IoU with a higher-confidence peer exceeds this value are suppressed. Typical values: 0.45 (SSD/YOLO default), 0.5, 0.3 for stricter deduplication. | +| Output attribute name | | | Specify the attribute to use as output, if not provided, the content is overridden instead.
**Supports Expression Language: true** | +| **Score activation** | Softmax | Softmax
Sigmoid
None | Activation applied to raw per-class scores before selecting the winning class. Softmax = mutually-exclusive classes (SSD/MobileNet-SSD raw logits). Sigmoid = independent classes (YOLOv5/v8 style). None = the model already emits probabilities/scores; use raw argmax with the raw score as confidence. | +| **Score output index** | 0 | | Zero-based index of the model output tensor that holds classification scores. The processor slices the concatenated payload from InvokeTractModel according to the 'tensor.N.bytes' attributes. | + +### Relationships + +| Name | Description | +|---------|--------------------------------------------------------------------------------------------------------------------------------------------| +| failure | The upstream output attributes were missing/invalid, the payload was truncated, or the tensors could not be interpreted as scores + boxes. | +| success | Filtering completed. The flow file content is a JSON array of the surviving bounding boxes (may be empty). | + +### Output Attributes + +| Attribute | Relationship | Description | +|--------------|--------------|-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| mime.type | success | If the "Output attribute name" is None, then the content will be overridden with the JSON array of objects with fields class_id, confidence, x_min, y_min, x_max, y_max, and the mime type will be set to 'application/json'. | +| object.count | success | Number of bounding boxes retained after confidence filtering and NMS. | + + +## ImageToTensor + +### Description + +Decodes an image from the flow file content and converts it into a normalised numeric tensor suitable for feeding into a downstream inference processor such as InvokeTractModel. Supports RGB / BGR / Grayscale, CHW / HWC layouts, stretch or letterbox resizing, and scalar or per-channel mean/std normalisation. The output payload is a single raw little-endian f32 tensor; the 'tensor.0.shape' attribute describes its layout. + +### Properties + +In the list below, the names of required properties appear in bold. Any other properties (not in bold) are considered optional. The table also indicates any default values, and whether a property supports the NiFi Expression Language. + +| Name | Default Value | Allowable Values | Description | +|-------------------------|---------------|-----------------------------------------------|----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| **Color format** | RGB | RGB
BGR
Grayscale | Color space of the tensor fed to the model. RGB and BGR produce three-channel tensors (channel order determined by the format); Grayscale produces a single-channel luma tensor. | +| **Letterbox pad value** | 0.0 | | Value written for padding pixels when 'Resize mode' is 'Letterbox'. This is a normalised value (post mean/std), so 0.0 corresponds to a neutral input for most networks. Ignored when 'Resize mode' is 'Stretch'. | +| Mean | 0.0 | | Value subtracted from each pixel before dividing by 'Standard Deviation'. Accepts either a single value (broadcast to all channels) or three comma-separated values applied per channel in the order dictated by 'Color format'. Example: '0.485, 0.456, 0.406' for ImageNet-style RGB normalisation. | +| **Pixel divisor** | 1.0 | | Divisor applied to raw u8 pixel values before subtracting 'Mean' and dividing by 'Standard Deviation'. Defaults to 1.0 (mean/std interpreted in [0, 255] pixel space, e.g. UltraFace's mean=127, std=128). Set to 255 to bring pixels into [0.0, 1.0] first so ImageNet-style mean/std values like '0.485, 0.456, 0.406' / '0.229, 0.224, 0.225' can be used directly, matching the PyTorch / torchvision / ONNX MobileNet convention. Must be non-zero. | +| **Resize filter** | Bilinear | Nearest
Bilinear
Bicubic
Lanczos3 | Interpolation filter applied when resizing the decoded image. Nearest is fastest but blocky; Bilinear is a good default; Bicubic and Lanczos3 are higher-quality but slower. | +| **Resize mode** | Stretch | Stretch
Letterbox | How the source image is fitted into the target dimensions. 'Stretch' scales each axis independently, distorting aspect ratio. 'Letterbox' preserves aspect ratio and pads the remaining border with 'Letterbox pad value' (applied in normalised output space). | +| Standard Deviation | 255.0 | | Divisor applied after subtracting 'Mean'. Accepts a single value (broadcast) or three comma-separated values (per channel). Must be non-zero. Example: '255.0' to scale u8 pixels into [0.0, 1.0]; '0.229, 0.224, 0.225' for ImageNet. | +| **Target height** | | | Height in pixels the decoded image is resized to before normalisation and inference. | +| **Target width** | | | Width in pixels the decoded image is resized to before normalisation and inference. | +| **Tensor shape format** | CHW | CHW
HWC | Memory layout of the tensor fed to the model. CHW (channels-first) is typical for PyTorch/ONNX detectors. HWC (channels-last) matches TensorFlow/TFLite. Ignored for Grayscale (always effectively 1xHxW). | + +### Relationships + +| Name | Description | +|---------|--------------------------------------------------------| +| failure | The input flow file could not be decoded as an image. | +| success | The input image was decoded and converted to a tensor. | + +### Output Attributes + +| Attribute | Relationship | Description | +|-----------------------|--------------|--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| image.original.height | success | The height of the original image before the resizing. | +| image.original.width | success | The width of the original image before the resizing. | +| image.resize.mode | success | The resize mode ('Stretch' or 'Letterbox') applied to fit the image into the target dimensions. Downstream processors such as FilterBoundingBoxes use this to invert the coordinate mapping correctly. | +| image.target.height | success | The height of the image after the resizing. | +| image.target.width | success | The width of the image after the resizing. | +| tensor.0.bytes | success | Byte length of output tensor. | +| tensor.0.dtype | success | Element type of the values in the output tensor. Currently always 'F32'. | +| tensor.0.shape | success | Comma-separated dimensions of the output tensor in the chosen layout, always including a leading batch dimension of 1 (e.g. '1,3,224,224' for RGB CHW). | +| tensors.len | success | Number of tensors in the output FlowFile. Currently always '1' | + + +## InvokeTractModel + +### Description + +Runs a single inference against the compiled model owned by the referenced TractModelService. Reads the input tensor from the flow file content plus the 'tensor.0.shape' and (optionally) 'tensor.0.dtype' attributes produced by an upstream processor such as ImageToTensor. The flow file's new content is every output tensor's raw bytes concatenated in model order; per-tensor shape, byte length, and dtype are written to attributes. + +### Properties + +In the list below, the names of required properties appear in bold. Any other properties (not in bold) are considered optional. The table also indicates any default values, and whether a property supports the NiFi Expression Language. + +| Name | Default Value | Allowable Values | Description | +|-------------------------|---------------|------------------|------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| **Tract model service** | | | Reference to a TractModelService controller service. The referenced service owns the compiled model (ONNX or NNEF) that will be evaluated for each incoming flow file. | + +### Relationships + +| Name | Description | +|---------|-------------------------------------------------------------------------------------------------------------------------------------------------| +| failure | The input tensor could not be built (missing/invalid tensor.0.shape, unsupported tensor.0.dtype, malformed payload) or the model failed to run. | +| success | Inference completed. The flow file's content is the concatenation of every output tensor's raw bytes in model output order. | + +### Output Attributes + +| Attribute | Relationship | Description | +|------------------|--------------|--------------------------------------------------------------------------------------------------------------------------------------------| +| tensor.{i}.bytes | success | Byte length of output tensor at index i within the concatenated payload. Consumers slice the payload sequentially using these lengths. | +| tensor.{i}.dtype | success | Element type of output tensor at index i | +| tensor.{i}.shape | success | Comma-separated dimensions of output tensor at index i. | +| tensors.len | success | Number of output tensors produced by the model. Downstream processors can loop from 0 up to this count when reading per-output attributes. | + + +## TractModelService + +### Description + +Provides a shared, CPU-optimized neural network for inference. Supports ONNX (`.onnx`) and NNEF (directory or tarball) models; the format can be auto-detected from the file extension or set explicitly. + +### Properties + +In the list below, the names of required properties appear in bold. Any other properties (not in bold) are considered optional. The table also indicates any default values, and whether a property supports the NiFi Expression Language. + +| Name | Default Value | Allowable Values | Description | +|---------------------|---------------|------------------------|-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| **Model File Path** | | | Absolute path to the model on the edge device. For ONNX this is a `.onnx` file; for NNEF this is a `.nnef.tgz` archive, a `.nnef` tarball, or the root directory of an unpacked NNEF model. The model is loaded, parsed, and compiled for the host CPU once when the controller service is enabled; subsequent inference calls reuse the compiled runnable. | +| **Model format** | Auto | Auto
Onnx
Nnef | Format of the file/directory referenced by 'Model File Path'. 'Auto' picks Onnx when the path ends in `.onnx` and Nnef when it ends in `.nnef`, `.nnef.tgz`, `.nnef.tar`, `.nnef.tar.gz`, or points at a directory. Set explicitly when the path uses a non-standard extension. | diff --git a/minifi_rust/extensions/minifi_tensor/src/lib.rs b/minifi_rust/extensions/minifi_tensor/src/lib.rs new file mode 100644 index 0000000000..c150f09f9d --- /dev/null +++ b/minifi_rust/extensions/minifi_tensor/src/lib.rs @@ -0,0 +1,53 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#[cfg(feature = "low-level-processors")] +use crate::low_level_processors::{ + classify_output::ClassifyOutput, filter_bounding_boxes::FilterBoundingBoxes, + image_to_tensor::ImageToTensor, invoke_tract_model::InvokeTractModel, +}; +use crate::processors::classify_image::ClassifyImage; +use crate::processors::detect_object::DetectObject; +use crate::processors::draw_bounding_box::DrawBoundingBox; + +use crate::services::tract_model_service::TractModelService; +use minifi_native::{FlowFileTransformProcessorType, MultiThreaded}; + +mod low_level_processors; +mod processors; +mod services; +mod utils; + +minifi_native::declare_minifi_extension!( + group_name: "org.apache.nifi.minifi.rust", + processors: [ + (FlowFileTransformProcessorType, MultiThreaded, DetectObject), + (FlowFileTransformProcessorType, MultiThreaded, ClassifyImage), + (FlowFileTransformProcessorType, MultiThreaded, DrawBoundingBox), + #[cfg(feature = "low-level-processors")] + (FlowFileTransformProcessorType, MultiThreaded, ImageToTensor), + #[cfg(feature = "low-level-processors")] + (FlowFileTransformProcessorType, MultiThreaded, InvokeTractModel), + #[cfg(feature = "low-level-processors")] + (FlowFileTransformProcessorType, MultiThreaded, FilterBoundingBoxes), + #[cfg(feature = "low-level-processors")] + (FlowFileTransformProcessorType, MultiThreaded, ClassifyOutput), + ], + controllers: [ + TractModelService + ] +); diff --git a/minifi_rust/extensions/minifi_tensor/src/low_level_processors/classify_output.rs b/minifi_rust/extensions/minifi_tensor/src/low_level_processors/classify_output.rs new file mode 100644 index 0000000000..90a2500f1e --- /dev/null +++ b/minifi_rust/extensions/minifi_tensor/src/low_level_processors/classify_output.rs @@ -0,0 +1,552 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use crate::low_level_processors::classify_output::classify_output_def::{ + CLASS_COUNT_ATTR, CLASS_TOP1_CONFIDENCE_ATTR, CLASS_TOP1_ID_ATTR, CLASS_TOP1_NAME_ATTR, +}; +use crate::utils::score_activation::{ScoreActivation, SoftmaxTerms}; +use crate::utils::tensor_helpers::{deserialize_tensors, tensor_as_f32, tensor_shape}; +use classify_output_def::SUCCESS; +pub(crate) use classify_output_def::{ + CLASSIFY_OUTPUT_ATTRIBUTES, CONFIDENCE_THRESHOLD, LABEL_INDEX_OFFSET, LABELS_FILE_PATH, + MIME_TYPE_ATTR, OUTPUT_ATTRIBUTE_NAME, SCORE_ACTIVATION, SCORE_OUTPUT_INDEX, TOP_K, +}; +use minifi_native::macros::ComponentIdentifier; +use minifi_native::{ + Content, FlowFileTransform, GetAttribute, GetId, GetProperty, InputStream, Logger, MinifiError, + ProcessError, PropertyConstraints, PropertySchema, PropertyType, RouteErrorExt, Schedule, + TransformedFlowFile, warn, +}; +use serde::Serialize; +use tract::Tensor; + +mod classify_output_def; + +#[derive(Serialize, Clone, Debug, PartialEq)] +struct Prediction { + class_id: usize, + confidence: f32, + #[serde(skip_serializing_if = "Option::is_none")] + class_name: Option, +} + +pub(crate) struct LabelsProperty {} + +impl PropertySchema for LabelsProperty { + const CONSTRAINT: Option = None; + const IS_REQUIRED: bool = false; +} + +impl PropertyType for LabelsProperty { + type Output = Vec; + + fn parse(s: &str) -> Result { + let content = std::fs::read_to_string(s).map_err(|e| { + MinifiError::custom(format!("Failed to read labels file '{:?}': {}", s, e)) + })?; + Ok(content + .lines() + .map(|line| line.trim().to_string()) + .collect()) + } +} + +fn top_k(mut scored: Vec<(usize, f32)>, k: usize) -> Vec<(usize, f32)> { + scored.sort_by(|&(ai, a), &(bi, b)| b.total_cmp(&a).then(ai.cmp(&bi))); + scored.truncate(k); + scored +} + +#[derive(ComponentIdentifier)] +pub(crate) struct ClassifyOutput { + top_k: usize, + score_output_index: usize, + score_activation: ScoreActivation, + confidence_threshold: f32, + labels: Option>, +} + +impl Schedule for ClassifyOutput { + fn schedule( + context: &Ctx, + _logger: &L, + ) -> Result + where + Self: Sized, + { + let top_k = context.get_property(&TOP_K)?; + if top_k == 0 { + return Err(MinifiError::validation("Top K must be >= 1")); + } + let score_output_index = context.get_property(&SCORE_OUTPUT_INDEX)?; + let score_activation = context.get_property(&SCORE_ACTIVATION)?; + let confidence_threshold = context.get_property(&CONFIDENCE_THRESHOLD)?; + + let label_index_offset = context.get_property(&LABEL_INDEX_OFFSET)?; + let labels = if let Some(mut labels) = context.get_property(&LABELS_FILE_PATH)? { + if let Some(dummy_indices) = label_index_offset { + if dummy_indices >= labels.len() { + return Err(MinifiError::validation(format!( + "Label index offset ({}) must be smaller than the number of labels ({})", + dummy_indices, + labels.len() + ))); + } + labels.drain(0..dummy_indices); + } + Some(labels) + } else { + if label_index_offset.is_some() { + return Err(MinifiError::validation( + "Label index offset is set without valid labels, either unset label index offset or provide valid labels file", + )); + } + None + }; + + Ok(Self { + top_k, + score_output_index, + score_activation, + confidence_threshold, + labels, + }) + } +} + +impl ClassifyOutput { + pub(crate) fn classify<'a, Context: GetProperty + GetAttribute + GetId, LoggerImpl: Logger>( + &self, + context: &Context, + logger: &LoggerImpl, + tensors: Vec, + ) -> Result, ProcessError> { + let score_floats = + tensor_as_f32(&tensors, self.score_output_index).route_err_to_failure()?; + if score_floats.is_empty() { + return Err(ProcessError::route_to_failure( + "Score tensor is empty; nothing to classify", + )); + } + + // A classifier head is a single score vector: shape [num_classes] or + // [1, .., num_classes]. We rank over the flattened class axis, so any + // leading axis > 1 (a real batch) would silently mix rows and yield + // class ids past num_classes. Reject it rather than produce garbage. + // (`ImageToTensor` emits batch=1 today; this just enforces the contract.) + let shape = tensor_shape(&tensors, self.score_output_index).route_err_to_failure()?; + if shape.iter().rev().skip(1).any(|&d| d != 1) { + return Err(ProcessError::route_to_failure(format!( + "ClassifyOutput expects a single score vector (shape [num_classes] or \ + [1, .., num_classes]); got {shape:?}. A batch dimension > 1 is not supported." + ))); + } + + let finite: Vec<(usize, f32)> = score_floats + .iter() + .copied() + .enumerate() + .filter(|&(_, s)| s.is_finite()) + .collect(); + + let softmax_terms = SoftmaxTerms::over(finite.iter().map(|&(_, s)| s)); + + let predictions: Vec = top_k(finite, self.top_k) + .into_iter() + .filter_map(|(class_id, raw)| { + let confidence = self.score_activation.confidence(raw, softmax_terms); + + if confidence >= self.confidence_threshold { + let class_name = self.labels.as_ref().and_then(|l| l.get(class_id).cloned()); + if class_name.is_none() + && let Some(l) = &self.labels + { + warn!( + logger, + "No label for class id {} ({} labels loaded); \ + the labels file does not match the model's classes", + class_id, + l.len() + ); + } + Some(Prediction { + class_id, + confidence, + class_name, + }) + } else { + None + } + }) + .collect(); + + let (content, extra_attribute) = match context.get_property(&OUTPUT_ATTRIBUTE_NAME)? { + None => ( + Some(Content::Buffer( + serde_json::to_vec(&predictions).route_err_to_failure()?, + )), + None, + ), + Some(output_attr) => ( + None, + Some(( + output_attr, + serde_json::to_string(&predictions).route_err_to_failure()?, + )), + ), + }; + + let mut transformed = TransformedFlowFile::new(&SUCCESS, content) + .with_attribute(MIME_TYPE_ATTR.name, "application/json") + .with_attribute(CLASS_COUNT_ATTR.name, predictions.len().to_string()); + + if let Some(top) = predictions.first() { + transformed = transformed + .with_attribute(CLASS_TOP1_ID_ATTR.name, top.class_id.to_string()) + .with_attribute(CLASS_TOP1_CONFIDENCE_ATTR.name, top.confidence.to_string()); + if let Some(name) = &top.class_name { + transformed = transformed.with_attribute(CLASS_TOP1_NAME_ATTR.name, name.clone()); + } + } + + if let Some((key, value)) = extra_attribute { + transformed = transformed.with_attribute(key, value); + } + Ok(transformed) + } +} + +impl FlowFileTransform for ClassifyOutput { + fn transform<'a, Context: GetProperty + GetAttribute + GetId, LoggerImpl: Logger>( + &self, + context: &Context, + input_stream: &'a mut dyn InputStream, + logger: &LoggerImpl, + ) -> Result, ProcessError> { + let tensors = deserialize_tensors(context, input_stream).route_err_to_failure()?; + self.classify(context, logger, tensors) + } +} + +#[cfg(test)] +mod tests { + use super::classify_output_def::FAILURE; + use super::*; + use minifi_native::{LogLevel, MockLogger, MockProcessContext}; + use std::io::Cursor; + use std::io::Write; + use tempfile::NamedTempFile; + + fn make_processor(top_k: usize, activation: ScoreActivation) -> ClassifyOutput { + ClassifyOutput { + top_k, + score_output_index: 0, + score_activation: activation, + confidence_threshold: 0.0, + labels: None, + } + } + + #[test] + fn test_top_k_descending_and_clamped() { + let scored = vec![(0, 0.1), (1, 0.9), (2, 0.5), (3, 0.3)]; + assert_eq!(top_k(scored.clone(), 2), vec![(1, 0.9), (2, 0.5)]); + assert_eq!( + top_k(scored, 10), + vec![(1, 0.9), (2, 0.5), (3, 0.3), (0, 0.1)] + ); + } + + #[test] + fn test_top_k_tiebreak_by_lower_index() { + let scored = vec![(0, 0.5), (1, 0.5), (2, 0.5)]; + assert_eq!(top_k(scored, 3), vec![(0, 0.5), (1, 0.5), (2, 0.5)]); + } + + fn build_payload(scores: &[f32]) -> Vec { + let mut bytes = Vec::with_capacity(scores.len() * 4); + for s in scores { + bytes.extend_from_slice(&s.to_le_bytes()); + } + bytes + } + + fn context_with_scores(scores: &[f32]) -> MockProcessContext { + let mut ctx = MockProcessContext::new(); + ctx.attributes.insert("tensors.len".into(), "1".into()); + ctx.attributes + .insert("tensor.0.bytes".to_string(), (scores.len() * 4).to_string()); + ctx.attributes + .insert("tensor.0.shape".to_string(), format!("1,{}", scores.len())); + ctx.attributes + .insert("tensor.0.dtype".to_string(), "F32".to_string()); + ctx + } + + #[test] + fn test_transform_returns_top_k_json() { + let processor = make_processor(3, ScoreActivation::Softmax); + let logits = vec![1.0f32, 4.0, 2.0, 0.5, 3.0]; + let context = context_with_scores(&logits); + let payload = build_payload(&logits); + let mut stream = Cursor::new(payload); + let result = processor + .transform(&context, &mut stream, &MockLogger::new()) + .expect("transform succeeds"); + + assert_eq!(result.target_relationship(), SUCCESS.name); + assert_eq!(result.attribute("class.top1.id").unwrap(), "1"); + assert_eq!(result.attribute("class.count").unwrap(), "3"); + + let json_bytes = result.into_bytes().unwrap().unwrap(); + let json = String::from_utf8(json_bytes).unwrap(); + // Expect three entries, ranked class_id 1, then 4, then 2. + assert!(json.contains("\"class_id\":1")); + assert!(json.contains("\"class_id\":4")); + assert!(json.contains("\"class_id\":2")); + } + + #[test] + fn test_sigmoid_activation_scores_classes_independently() { + // Sigmoid squashes every logit on its own, so the confidences are not a + // distribution: here they sum to ~1.5. A softmax regression would force + // that sum to exactly 1.0. + let processor = make_processor(3, ScoreActivation::Sigmoid); + let logits = vec![0.0f32, 2.0, -2.0]; + let context = context_with_scores(&logits); + let mut stream = Cursor::new(build_payload(&logits)); + let result = processor + .transform(&context, &mut stream, &MockLogger::new()) + .expect("transform succeeds"); + + let json = result.into_bytes().unwrap().unwrap(); + let predictions: Vec = serde_json::from_slice(&json).unwrap(); + assert_eq!(predictions.len(), 3); + + let ids: Vec = predictions + .iter() + .map(|p| p["class_id"].as_u64().unwrap()) + .collect(); + assert_eq!( + ids, + vec![1, 0, 2], + "sigmoid is monotonic, so the ranking follows the raw logits" + ); + + let confidences: Vec = predictions + .iter() + .map(|p| p["confidence"].as_f64().unwrap() as f32) + .collect(); + let sigmoid_of = |logit: f32| 1.0f32 / (1.0 + (-logit).exp()); + assert_eq!(confidences[0], sigmoid_of(2.0)); + assert_eq!(confidences[1], 0.5, "sigmoid(0.0) is exactly one half"); + assert_eq!(confidences[2], sigmoid_of(-2.0)); + + let sum: f32 = confidences.iter().sum(); + assert!( + sum > 1.4, + "per-class sigmoid must not normalise across classes, got {sum}" + ); + } + + #[test] + fn test_sigmoid_activation_threshold_is_inclusive_at_one_half() { + // The threshold is applied after top_k, so top_k = 3 admits all three + // candidates and only the filter decides. sigmoid(0.0) == 0.5 passes the + // `>=` comparison; the negative logit falls below it. + let mut processor = make_processor(3, ScoreActivation::Sigmoid); + processor.confidence_threshold = 0.5; + let logits = vec![2.0f32, -0.5, 0.0]; + let context = context_with_scores(&logits); + let mut stream = Cursor::new(build_payload(&logits)); + let result = processor + .transform(&context, &mut stream, &MockLogger::new()) + .unwrap(); + + assert_eq!( + result.attribute("class.count").unwrap(), + "2", + "only the -0.5 logit should be filtered out" + ); + assert_eq!(result.attribute("class.top1.id").unwrap(), "0"); + } + + #[test] + fn test_transform_omits_class_name_when_labels_absent() { + let processor = make_processor(1, ScoreActivation::None); + let scores = vec![0.1f32, 0.9]; + let context = context_with_scores(&scores); + let mut stream = Cursor::new(build_payload(&scores)); + let result = processor + .transform(&context, &mut stream, &MockLogger::new()) + .unwrap(); + // Snapshot the top1.name absence before into_bytes consumes `result`. + let has_top1_name = result.attribute("class.top1.name").is_some(); + let json = String::from_utf8(result.into_bytes().unwrap().unwrap()).unwrap(); + assert!(!json.contains("class_name")); + assert!(!has_top1_name); + } + + #[test] + fn test_transform_looks_up_labels() { + let mut processor = make_processor(1, ScoreActivation::None); + processor.labels = Some(vec![ + "tench".into(), + "goldfish".into(), + "great_white_shark".into(), + ]); + let scores = vec![0.1f32, 0.9, 0.5]; + let context = context_with_scores(&scores); + let mut stream = Cursor::new(build_payload(&scores)); + let result = processor + .transform(&context, &mut stream, &MockLogger::new()) + .unwrap(); + assert_eq!(result.attribute("class.top1.name").unwrap(), "goldfish"); + let json = String::from_utf8(result.into_bytes().unwrap().unwrap()).unwrap(); + assert!(json.contains("\"class_name\":\"goldfish\"")); + } + + #[test] + fn test_label_offset() { + let mut file = NamedTempFile::new().expect("Failed to create temp file"); + writeln!(file, "dummy").unwrap(); + writeln!(file, "tench").unwrap(); + writeln!(file, "goldfish").unwrap(); + writeln!(file, "great_white_shark").unwrap(); + let mut mock_context = MockProcessContext::default(); + mock_context.properties.insert( + LABELS_FILE_PATH.name().to_string(), + file.path().to_string_lossy(), + ); + mock_context + .properties + .insert(LABEL_INDEX_OFFSET.name().to_string(), "1"); + let scheduled = ClassifyOutput::schedule(&mock_context, &MockLogger::new()).unwrap(); + assert_eq!(3, scheduled.labels.unwrap().len()); + } + + #[test] + fn test_label_lookup_miss_warns_when_labels_are_configured() { + let mut processor = make_processor(1, ScoreActivation::None); + processor.labels = Some(vec!["abc".into(), "tench".into()]); + let scores = vec![0.1f32, 0.2, 0.9]; // model class 2 wins + let context = context_with_scores(&scores); + let mut stream = Cursor::new(build_payload(&scores)); + let logger = MockLogger::new(); + + let result = processor.transform(&context, &mut stream, &logger).unwrap(); + + assert!( + result.attribute("class.top1.name").is_none(), + "an out-of-range label index still yields no name" + ); + let logs = logger.logs.lock().unwrap(); + assert!( + logs.iter() + .any(|(level, msg)| *level == LogLevel::Warn && msg.contains("No label for class")), + "expected a warning about the labels/model mismatch, got: {logs:?}" + ); + } + + #[test] + fn test_label_lookup_stays_silent_without_a_labels_file() { + let processor = make_processor(1, ScoreActivation::None); + let scores = vec![0.1f32, 0.9, 0.5]; + let context = context_with_scores(&scores); + let mut stream = Cursor::new(build_payload(&scores)); + let logger = MockLogger::new(); + + processor.transform(&context, &mut stream, &logger).unwrap(); + + let logs = logger.logs.lock().unwrap(); + assert!( + !logs + .iter() + .any(|(_, msg)| msg.contains("No label for class")), + "should not warn when no labels file is configured, got: {logs:?}" + ); + } + + #[test] + fn test_transform_filters_below_confidence_threshold() { + let mut processor = make_processor(3, ScoreActivation::None); + processor.confidence_threshold = 0.6; + let scores = vec![0.1f32, 0.9, 0.5]; + let context = context_with_scores(&scores); + let mut stream = Cursor::new(build_payload(&scores)); + let result = processor + .transform(&context, &mut stream, &MockLogger::new()) + .unwrap(); + assert_eq!( + result.attribute("class.count").unwrap(), + "1", + "only the 0.9-scored class should survive" + ); + } + + #[test] + fn test_transform_ignores_non_finite_scores() { + // A NaN score must not steal the top slot: with top_k = 1 the sole + // prediction should be the finite class 1, not the NaN class 0. + let processor = make_processor(1, ScoreActivation::None); + let scores = vec![f32::NAN, 0.9]; + let context = context_with_scores(&scores); + let mut stream = Cursor::new(build_payload(&scores)); + let result = processor + .transform(&context, &mut stream, &MockLogger::new()) + .unwrap(); + assert_eq!(result.attribute("class.count").unwrap(), "1"); + assert_eq!(result.attribute("class.top1.id").unwrap(), "1"); + } + + #[test] + fn test_transform_batched_scores_route_to_failure() { + // A real batch dimension ([2, 3]) can't be flattened into one score + // vector without mixing rows. It's input-dependent, so it must route to + // failure (not raise a fatal/rollback error). + let processor = make_processor(1, ScoreActivation::None); + let scores = vec![0.1f32, 0.9, 0.2, 0.8, 0.3, 0.7]; + let mut context = context_with_scores(&scores); + context + .attributes + .insert("tensor.0.shape".into(), "2,3".into()); + let mut stream = Cursor::new(build_payload(&scores)); + let err = processor + .transform(&context, &mut stream, &MockLogger::new()) + .expect_err("batched scores should be rejected"); + match err { + ProcessError::Route(route) => assert_eq!(route.relationship, FAILURE.name), + other => panic!("expected route to failure, got {other:?}"), + } + } + + #[test] + fn test_transform_missing_bytes_attribute_routes_to_failure() { + let processor = make_processor(1, ScoreActivation::Softmax); + let context = MockProcessContext::new(); // no tensor.0.bytes + let mut stream = Cursor::new(vec![0u8; 4]); + let err = processor + .transform(&context, &mut stream, &MockLogger::new()) + .expect_err("missing attribute should route to failure via a Route error"); + match err { + ProcessError::Route(route) => { + assert_eq!(route.relationship, FAILURE.name) + } + other => panic!("expected route to failure, got {other:?}"), + } + } +} diff --git a/minifi_rust/extensions/minifi_tensor/src/low_level_processors/classify_output/classify_output_def.rs b/minifi_rust/extensions/minifi_tensor/src/low_level_processors/classify_output/classify_output_def.rs new file mode 100644 index 0000000000..b28cc8241a --- /dev/null +++ b/minifi_rust/extensions/minifi_tensor/src/low_level_processors/classify_output/classify_output_def.rs @@ -0,0 +1,161 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use super::{ClassifyOutput, LabelsProperty, ScoreActivation}; +use minifi_native::{ + OutputAttribute, ProcessorDefinition, ProcessorInputRequirement, Property, PropertyDefinition, + Relationship, property_definitions, +}; + +pub(crate) const TOP_K: Property = Property::new( + "Top K", + "Number of highest-scoring classes to include in the output JSON, in descending \ + order of confidence. Values above the total class count are clamped. Set to 1 \ + for pure top-1 classification.", +) +.with_default("5"); + +pub(crate) const SCORE_OUTPUT_INDEX: Property = Property::new( + "Score output index", + "Zero-based index of the model output tensor that holds classification scores. \ + The processor slices the concatenated payload from InvokeTractModel according \ + to the 'tensor.N.bytes' attributes. Almost always 0 for single-head \ + classifiers.", +) +.with_default("0"); + +pub(crate) const SCORE_ACTIVATION: Property = Property::new( + "Score activation", + "Activation applied to the raw score vector before ranking. \ + Softmax = mutually-exclusive classes (ImageNet-trained ResNet/MobileNet/\ + EfficientNet raw logits). \ + Sigmoid = independent classes (multi-label classifiers). \ + None = the model already emits probabilities/scores; rank the raw values.", +) +.with_default(ScoreActivation::Softmax.into_str()); + +pub(crate) const CONFIDENCE_THRESHOLD: Property = Property::new( + "Confidence Threshold", + "Minimum confidence a class must reach to be included in the output JSON. \ + Applied AFTER activation, so the units match the chosen activation \ + (0.0..=1.0 for Softmax/Sigmoid, model-native for None). Set to 0.0 to always \ + emit exactly Top K predictions.", +) +.with_default("0.0") +.supports_expression_language(); + +pub(crate) const LABELS_FILE_PATH: Property> = Property::new( + "Labels file path", + "Optional path to a newline-separated labels file (line N = name of class N). \ + Loaded once at service enable time. When set, each prediction in the output \ + JSON gains a 'class_name' field and the 'class.top1.name' flow file attribute \ + is populated. Leave empty to emit numeric class IDs only.", +); + +pub(crate) const LABEL_INDEX_OFFSET: Property> = Property::new( + "Label index offset", + "Offset added to the model's class ID when looking up a name in the labels file. \ + Defaults to 0 (labels file line N = class N). Set to 1 for label files that \ + start with a dummy/background entry — e.g. the ONNX MobileNetV2 model emits \ + 1000 class scores while 'imagenet_slim_labels.txt' has 1001 lines (line 0 = \ + 'dummy'), so class ID 653 maps to line 654 = 'military uniform'.", +); + +pub(crate) const OUTPUT_ATTRIBUTE_NAME: Property> = Property::new( + "Output attribute name", + "Specify the attribute to use as output, if not provided, the content is overridden instead.", +) +.supports_expression_language(); + +pub(super) const SUCCESS: Relationship = Relationship { + name: "success", + description: "Classification completed. The flow file content is a JSON array of the Top K \ + predictions (possibly fewer if the confidence threshold filtered some out).", +}; + +pub(super) const FAILURE: Relationship = Relationship { + name: "failure", + description: "The upstream output attributes were missing/invalid or the score tensor could \ + not be interpreted as f32 values.", +}; + +pub(crate) const MIME_TYPE_ATTR: OutputAttribute = OutputAttribute { + name: "mime.type", + relationships: &["success"], + description: "If the \"Output attribute name\" is None, then the content will be overridden with the JSON array of objects with fields class_id, confidence, and optional class_name, and the mime type will be set to 'application/json'.", +}; + +pub(crate) const CLASS_COUNT_ATTR: OutputAttribute = OutputAttribute { + name: "class.count", + relationships: &["success"], + description: "Number of predictions retained after Top K selection and confidence filtering.", +}; + +pub(crate) const CLASS_TOP1_ID_ATTR: OutputAttribute = OutputAttribute { + name: "class.top1.id", + relationships: &["success"], + description: "Numeric class ID of the highest-confidence prediction, when at least one \ + prediction cleared the confidence threshold.", +}; + +pub(crate) const CLASS_TOP1_CONFIDENCE_ATTR: OutputAttribute = OutputAttribute { + name: "class.top1.confidence", + relationships: &["success"], + description: "Confidence (post-activation) of the highest-confidence prediction, when at \ + least one prediction cleared the confidence threshold.", +}; + +pub(crate) const CLASS_TOP1_NAME_ATTR: OutputAttribute = OutputAttribute { + name: "class.top1.name", + relationships: &["success"], + description: "Label of the highest-confidence prediction. Only present when 'Labels file \ + path' was configured and at least one prediction cleared the threshold.", +}; + +pub(crate) const CLASSIFY_OUTPUT_ATTRIBUTES: &[OutputAttribute] = &[ + MIME_TYPE_ATTR, + CLASS_COUNT_ATTR, + CLASS_TOP1_ID_ATTR, + CLASS_TOP1_CONFIDENCE_ATTR, + CLASS_TOP1_NAME_ATTR, +]; + +impl ProcessorDefinition for ClassifyOutput { + const DESCRIPTION: &'static str = "Post-processes the output of a classification model invoked via InvokeTractModel. Reads \ + the flattened score vector from the configured output tensor index, applies the chosen \ + activation (softmax / sigmoid / none), and emits the Top K classes as a JSON array \ + [{class_id, confidence, class_name?}, ...]. Optional labels file maps numeric class IDs \ + to human-readable names. Works with ImageNet-style ResNet/MobileNet/EfficientNet \ + checkpoints (Softmax over raw logits) as well as multi-label classifiers (Sigmoid) and \ + models that already emit probabilities (None). Assumes a single flattened score \ + vector — upstream ImageToTensor produces batch=1 tensors, so this is the common case."; + const INPUT_REQUIREMENT: ProcessorInputRequirement = ProcessorInputRequirement::Required; + const SUPPORTS_DYNAMIC_PROPERTIES: bool = false; + const SUPPORTS_DYNAMIC_RELATIONSHIPS: bool = false; + const OUTPUT_ATTRIBUTES: &'static [OutputAttribute] = CLASSIFY_OUTPUT_ATTRIBUTES; + const RELATIONSHIPS: &'static [Relationship] = &[SUCCESS, FAILURE]; + + const PROPERTIES: &[PropertyDefinition] = property_definitions![ + TOP_K, + SCORE_OUTPUT_INDEX, + SCORE_ACTIVATION, + CONFIDENCE_THRESHOLD, + LABELS_FILE_PATH, + LABEL_INDEX_OFFSET, + OUTPUT_ATTRIBUTE_NAME + ]; +} diff --git a/minifi_rust/extensions/minifi_tensor/src/low_level_processors/filter_bounding_boxes.rs b/minifi_rust/extensions/minifi_tensor/src/low_level_processors/filter_bounding_boxes.rs new file mode 100644 index 0000000000..3f96441f8e --- /dev/null +++ b/minifi_rust/extensions/minifi_tensor/src/low_level_processors/filter_bounding_boxes.rs @@ -0,0 +1,590 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +mod filter_bounding_boxes_def; + +use crate::low_level_processors::image_to_tensor::ResizeMode; +use crate::utils::bounding_box::BoundingBox; +use crate::utils::dimensions::Dimensions; +use crate::utils::score_activation::{ScoreActivation, SoftmaxTerms}; +use crate::utils::tensor_helpers::{deserialize_tensors, tensor_as_f32}; +use filter_bounding_boxes_def::SUCCESS; +pub(crate) use filter_bounding_boxes_def::{ + BACKGROUND_CLASS_INDEX, BOX_FORMAT, BOX_OUTPUT_INDEX, CLASS_OUTPUT_INDEX, CONFIDENCE_THRESHOLD, + IOU_THRESHOLD, MIME_TYPE_ATTR, OUTPUT_ATTRIBUTE_NAME, SCORE_ACTIVATION, SCORE_OUTPUT_INDEX, +}; +use minifi_native::macros::{ComponentIdentifier, PropertyType}; +use minifi_native::{ + Content, FlowFileTransform, GetAttribute, GetId, GetProperty, InputStream, Logger, MinifiError, + ProcessError, RouteErrorExt, Schedule, TransformedFlowFile, debug, trace, +}; +use strum_macros::{Display, EnumString, IntoStaticStr, VariantNames}; +use tract::Tensor; + +#[derive( + Debug, Clone, Copy, PartialEq, Display, EnumString, VariantNames, IntoStaticStr, PropertyType, +)] +#[strum(serialize_all = "PascalCase", const_into_str)] +pub(crate) enum BoxFormat { + /// `[x_min, y_min, x_max, y_max]` — SSD, MobileNet-SSD, most PyTorch models. + Xyxy, + /// `[y_min, x_min, y_max, x_max]` — TensorFlow Object Detection API. + Yxyx, + /// `[cx, cy, w, h]` — YOLOv3/5/8 raw output (center + size). + Cxcywh, +} + +/// Convert the four floats at `box_floats[offset..offset+4]` into a canonical +/// `(x_min, y_min, x_max, y_max)` tuple, regardless of the source layout. +fn decode_box(box_floats: &[f32], offset: usize, format: BoxFormat) -> (f32, f32, f32, f32) { + let a = box_floats[offset]; + let b = box_floats[offset + 1]; + let c = box_floats[offset + 2]; + let d = box_floats[offset + 3]; + match format { + BoxFormat::Xyxy => (a, b, c, d), + BoxFormat::Yxyx => (b, a, d, c), + BoxFormat::Cxcywh => { + let (cx, cy, w, h) = (a, b, c, d); + (cx - w / 2.0, cy - h / 2.0, cx + w / 2.0, cy + h / 2.0) + } + } +} + +struct ScoredClass { + class_id: usize, + confidence: f32, +} + +fn score_box( + logits: &[f32], + activation: ScoreActivation, + background_class_index: Option, +) -> ScoredClass { + let num_classes = logits.len(); + + let best_valid = logits + .iter() + .enumerate() + .filter(|&(_, &logit)| logit.is_finite()) + .filter(|&(id, _)| match background_class_index { + Some(bg_idx) => !(num_classes > 1 && id == bg_idx), + None => true, + }) + .max_by(|a, b| a.1.total_cmp(b.1)); + + let (class_id, &best_logit) = match best_valid { + Some(val) => val, + None => { + return ScoredClass { + class_id: 0, + confidence: f32::NEG_INFINITY, + }; + } + }; + + let confidence = activation.confidence(best_logit, SoftmaxTerms::over(logits.iter().copied())); + + ScoredClass { + class_id, + confidence, + } +} + +#[derive(ComponentIdentifier)] +pub(crate) struct FilterBoundingBoxes { + confidence_threshold: f32, + iou_threshold: f32, + score_output_index: usize, + box_output_index: usize, + box_format: BoxFormat, + score_activation: ScoreActivation, + background_class_index: Option, + class_output_index: Option, +} + +impl Schedule for FilterBoundingBoxes { + fn schedule( + context: &Ctx, + _logger: &L, + ) -> Result { + let confidence_threshold = context.get_property(&CONFIDENCE_THRESHOLD)?; + let iou_threshold = context.get_property(&IOU_THRESHOLD)?; + let score_output_index = context.get_property(&SCORE_OUTPUT_INDEX)?; + let box_output_index = context.get_property(&BOX_OUTPUT_INDEX)?; + let box_format = context.get_property(&BOX_FORMAT)?; + let score_activation = context.get_property(&SCORE_ACTIVATION)?; + let background_class_index = context.get_property(&BACKGROUND_CLASS_INDEX)?; + let class_output_index = context.get_property(&CLASS_OUTPUT_INDEX)?; + + Ok(Self { + confidence_threshold, + iou_threshold, + score_output_index, + box_output_index, + box_format, + score_activation, + background_class_index, + class_output_index, + }) + } +} + +impl FilterBoundingBoxes { + fn result_via_output_attribute<'a, Context: GetProperty>( + &self, + context: &Context, + filtered_boxes: Vec, + ) -> Result, MinifiError> { + let output_attr = context.get_property(&OUTPUT_ATTRIBUTE_NAME)?; + let content = if output_attr.is_some() { + None + } else { + Some(Content::Buffer( + serde_json::to_vec(&filtered_boxes).map_err(MinifiError::other)?, + )) + }; + + let mut transformed = TransformedFlowFile::new(&SUCCESS, content) + .with_attribute("object.count", filtered_boxes.len().to_string()); + if let Some(attr) = output_attr { + transformed = transformed.with_attribute( + attr, + serde_json::to_string(&filtered_boxes).map_err(MinifiError::other)?, + ) + } else { + transformed = transformed.with_attribute(MIME_TYPE_ATTR.name, "application/json"); + } + Ok(transformed) + } + + pub(crate) fn filter<'a, Context: GetProperty, LoggerImpl: Logger>( + &self, + context: &Context, + logger: &LoggerImpl, + tensors: Vec, + orig_dim: Dimensions, + target_dim: Dimensions, + resize_mode: ResizeMode, + ) -> Result, ProcessError> { + let score_floats = + tensor_as_f32(&tensors, self.score_output_index).route_err_to_failure()?; + let box_floats = tensor_as_f32(&tensors, self.box_output_index).route_err_to_failure()?; + + let (scale_x, scale_y, pad_x, pad_y) = match resize_mode { + ResizeMode::Letterbox => { + let geometry = orig_dim.letterbox_into(target_dim); + ( + geometry.scale, + geometry.scale, + geometry.pad_x as f32, + geometry.pad_y as f32, + ) + } + ResizeMode::Stretch => ( + target_dim.width / orig_dim.width, + target_dim.height / orig_dim.height, + 0.0, + 0.0, + ), + }; + + if !box_floats.len().is_multiple_of(4) { + return Err(ProcessError::route_to_failure( + "Box tensor byte length is not a multiple of 16 (4 f32 per box)", + )); + } + let num_boxes = box_floats.len() / 4; + if num_boxes == 0 { + debug!(logger, "No boxes to filter; emitting empty array"); + return self + .result_via_output_attribute(context, vec![]) + .route_err_to_failure(); + } + + let make_box = |i: usize, class_id: usize, confidence: f32| -> BoundingBox { + let (raw_x_min, raw_y_min, raw_x_max, raw_y_max) = + decode_box(&box_floats, i * 4, self.box_format); + let true_x_min = (((raw_x_min * target_dim.width) - pad_x) / scale_x) / orig_dim.width; + let true_y_min = + (((raw_y_min * target_dim.height) - pad_y) / scale_y) / orig_dim.height; + let true_x_max = (((raw_x_max * target_dim.width) - pad_x) / scale_x) / orig_dim.width; + let true_y_max = + (((raw_y_max * target_dim.height) - pad_y) / scale_y) / orig_dim.height; + BoundingBox { + class_id, + confidence, + x_min: true_x_min.clamp(0.0, 1.0), + y_min: true_y_min.clamp(0.0, 1.0), + x_max: true_x_max.clamp(0.0, 1.0), + y_max: true_y_max.clamp(0.0, 1.0), + } + }; + + let mut valid_boxes = Vec::new(); + + match self.class_output_index { + // Separate class-id tensor: one score and one class id per box + Some(class_index) => { + let class_floats = tensor_as_f32(&tensors, class_index).route_err_to_failure()?; + if score_floats.len() != num_boxes || class_floats.len() != num_boxes { + return Err(ProcessError::route_to_failure(format!( + "'Class output index' mode expects one score and one class id per box \ + (num_boxes={}, scores={}, classes={})", + num_boxes, + score_floats.len(), + class_floats.len() + ))); + } + trace!( + logger, + "Filtering {} boxes with separate class-id tensor (activation={:?}, \ + box_format={:?})...", + num_boxes, + self.score_activation, + self.box_format + ); + for i in 0..num_boxes { + let confidence = self.score_activation.confidence_of_scalar(score_floats[i]); + if confidence < self.confidence_threshold { + continue; + } + let raw_class = class_floats[i]; + if raw_class < 0.0 { + continue; + } + if !confidence.is_finite() || !raw_class.is_finite() { + continue; + } + let class_id = raw_class.round() as usize; + if self.background_class_index == Some(class_id) { + continue; + } + valid_boxes.push(make_box(i, class_id, confidence)); + } + } + // Per-class score matrix: argmax over classes per box. + None => { + if !score_floats.len().is_multiple_of(num_boxes) { + return Err(ProcessError::route_to_failure(format!( + "Scores length ({}) not divisible by number of boxes ({})", + score_floats.len(), + num_boxes + ))); + } + let num_classes = score_floats.len() / num_boxes; + trace!( + logger, + "Filtering {} boxes across {} potential classes (activation={:?}, \ + box_format={:?})...", + num_boxes, + num_classes, + self.score_activation, + self.box_format + ); + for i in 0..num_boxes { + let logits = &score_floats[i * num_classes..(i + 1) * num_classes]; + let scored = + score_box(logits, self.score_activation, self.background_class_index); + if scored.confidence >= self.confidence_threshold { + valid_boxes.push(make_box(i, scored.class_id, scored.confidence)); + } + } + } + } + + trace!( + logger, + "Found {} boxes exceeding the {} threshold.", + valid_boxes.len(), + self.confidence_threshold + ); + + let filtered_boxes = + BoundingBox::apply_non_maximum_suppression(valid_boxes, self.iou_threshold); + + self.result_via_output_attribute(context, filtered_boxes) + .route_err_to_failure() + } +} + +fn resize_mode_from_attributes(context: &Context) -> ResizeMode { + context + .get_attribute("image.resize.mode") + .ok() + .flatten() + .and_then(|raw| raw.parse::().ok()) + .unwrap_or(ResizeMode::Stretch) +} + +impl FlowFileTransform for FilterBoundingBoxes { + fn transform<'a, Context: GetProperty + GetAttribute + GetId, LoggerImpl: Logger>( + &self, + context: &Context, + input_stream: &'a mut dyn InputStream, + logger: &LoggerImpl, + ) -> Result, ProcessError> { + let orig_dim = Dimensions::original_from_attributes(context).route_err_to_failure()?; + let target_dim = Dimensions::target_from_attributes(context).route_err_to_failure()?; + let resize_mode = resize_mode_from_attributes(context); + + let tensors = deserialize_tensors(context, input_stream).route_err_to_failure()?; + + self.filter(context, logger, tensors, orig_dim, target_dim, resize_mode) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_decode_box_xyxy_is_identity() { + let raw = [10.0, 20.0, 30.0, 40.0]; + let (x0, y0, x1, y1) = decode_box(&raw, 0, BoxFormat::Xyxy); + assert_eq!((x0, y0, x1, y1), (10.0, 20.0, 30.0, 40.0)); + } + + #[test] + fn test_decode_box_yxyx_swaps_axes() { + let raw = [20.0, 10.0, 40.0, 30.0]; // (y_min, x_min, y_max, x_max) + let (x0, y0, x1, y1) = decode_box(&raw, 0, BoxFormat::Yxyx); + assert_eq!((x0, y0, x1, y1), (10.0, 20.0, 30.0, 40.0)); + } + + #[test] + fn test_decode_box_cxcywh_converts_to_corners() { + // Center (100, 200), width 20, height 40 → corners (90,180)-(110,220) + let raw = [100.0, 200.0, 20.0, 40.0]; + let (x0, y0, x1, y1) = decode_box(&raw, 0, BoxFormat::Cxcywh); + assert!((x0 - 90.0).abs() < 1e-6); + assert!((y0 - 180.0).abs() < 1e-6); + assert!((x1 - 110.0).abs() < 1e-6); + assert!((y1 - 220.0).abs() < 1e-6); + } + + #[test] + fn test_score_box_softmax_picks_argmax_excluding_background() { + // 3 classes: bg strongly favored, but bg is class 0 which we skip. + let logits = [5.0, 0.5, 2.0]; + let scored = score_box(&logits, ScoreActivation::Softmax, Some(0)); + assert_eq!(scored.class_id, 2); + assert!(scored.confidence > 0.0 && scored.confidence < 1.0); + } + + #[test] + fn test_score_box_sigmoid_treats_classes_independently() { + // Sigmoid(2.0) ≈ 0.881, so class 1 wins over class 0 regardless of + // relative magnitudes. + let logits = [-3.0, 2.0]; + let scored = score_box(&logits, ScoreActivation::Sigmoid, None); + assert_eq!(scored.class_id, 1); + assert!((scored.confidence - 0.8807).abs() < 1e-3); + } + + #[test] + fn test_score_box_none_takes_raw_argmax() { + let logits = [0.1, 0.7, 0.2]; + let scored = score_box(&logits, ScoreActivation::None, None); + assert_eq!(scored.class_id, 1); + assert!((scored.confidence - 0.7).abs() < 1e-6); + } + + #[test] + fn test_score_box_background_disabled_keeps_class_zero() { + let logits = [5.0, 0.5]; + let scored = score_box(&logits, ScoreActivation::Softmax, None); + assert_eq!(scored.class_id, 0); + } + + #[test] + fn test_activate_scalar_sigmoid_and_passthrough() { + assert!((ScoreActivation::Sigmoid.confidence_of_scalar(0.0) - 0.5).abs() < 1e-6); + assert_eq!(ScoreActivation::None.confidence_of_scalar(0.42), 0.42); + // Softmax over a scalar has no meaning → pass-through. + assert_eq!(ScoreActivation::Softmax.confidence_of_scalar(0.42), 0.42); + } + + fn class_index_processor() -> FilterBoundingBoxes { + FilterBoundingBoxes { + confidence_threshold: 0.5, + iou_threshold: 0.45, + score_output_index: 0, + box_output_index: 1, + box_format: BoxFormat::Xyxy, + score_activation: ScoreActivation::None, + background_class_index: None, + class_output_index: Some(2), + } + } + + /// TF-OD / EfficientNMS style: separate scores, boxes, and (integer) class-id + /// tensors, one entry per box, NMS already applied by the model. + #[test] + fn test_filter_with_separate_class_index_tensor() { + use minifi_native::{MockLogger, MockProcessContext}; + use tract::__ndarray_interop::TensorInterface; + + let scores = Tensor::from_slice::(&[2], &[0.9, 0.2]).unwrap(); + let boxes = + Tensor::from_slice::(&[2, 4], &[0.1, 0.1, 0.2, 0.2, 0.5, 0.5, 0.9, 0.9]).unwrap(); + // class ids arrive as i64, exercising the cast path too. + let classes = Tensor::from_slice::(&[2], &[5, 3]).unwrap(); + + let processor = class_index_processor(); + let dim = Dimensions { + width: 100.0, + height: 100.0, + }; + let result = processor + .filter( + &MockProcessContext::new(), + &MockLogger::new(), + vec![scores, boxes, classes], + dim, + dim, + ResizeMode::Letterbox, + ) + .expect("filter should succeed"); + + // Only box 0 (score 0.9) clears the 0.5 threshold; box 1 (0.2) is dropped. + assert_eq!(result.attribute("object.count").unwrap(), "1"); + let json = String::from_utf8(result.into_bytes().unwrap().unwrap()).unwrap(); + assert!(json.contains("\"class_id\":5")); + assert!(!json.contains("\"class_id\":3")); + } + + #[test] + fn test_resize_mode_changes_coordinate_un_mapping() { + use minifi_native::{MockLogger, MockProcessContext}; + use tract::__ndarray_interop::TensorInterface; + + let run = |mode: ResizeMode| -> BoundingBox { + // One interior box (avoids clamping) in normalised Xyxy. + let scores = Tensor::from_slice::(&[1], &[0.9]).unwrap(); + let boxes = Tensor::from_slice::(&[1, 4], &[0.4, 0.4, 0.6, 0.6]).unwrap(); + let classes = Tensor::from_slice::(&[1], &[5]).unwrap(); + + let result = class_index_processor() + .filter( + &MockProcessContext::new(), + &MockLogger::new(), + vec![scores, boxes, classes], + Dimensions { + width: 200.0, + height: 100.0, + }, + Dimensions { + width: 100.0, + height: 100.0, + }, + mode, + ) + .expect("filter should succeed"); + let json = result.into_bytes().unwrap().unwrap(); + let boxes: Vec = serde_json::from_slice(&json).unwrap(); + boxes.into_iter().next().expect("one box expected") + }; + + // Stretch: scale_x = 100/200, scale_y = 100/100, no padding → identity in + // normalised space. + let stretched = run(ResizeMode::Stretch); + assert!((stretched.y_min - 0.4).abs() < 1e-5); + assert!((stretched.y_max - 0.6).abs() < 1e-5); + + // Letterbox: uniform scale 0.5, symmetric vertical pad of 25px removed → + // y expands to 0.3..0.7. x is unchanged (pad_x = 0, same scale on x). + let letterboxed = run(ResizeMode::Letterbox); + assert!((letterboxed.x_min - 0.4).abs() < 1e-5); + assert!((letterboxed.x_max - 0.6).abs() < 1e-5); + assert!((letterboxed.y_min - 0.3).abs() < 1e-5); + assert!((letterboxed.y_max - 0.7).abs() < 1e-5); + } + + #[test] + fn test_letterbox_un_mapping_uses_the_integer_padding_that_was_applied() { + use minifi_native::{MockLogger, MockProcessContext}; + use tract::__ndarray_interop::TensorInterface; + + // SSD300 fed a 1080p frame: scale = 300/1920 = 0.15625, so the scaled + // height is 1080 * 0.15625 = 168.75 — *not* an integer. ImageToTensor + // rounds to 169 and pads (300 - 169) / 2 = 65. Deriving the padding from + // the unrounded 168.75 instead gives 65.625, and that 0.625 target-pixel + // error becomes 0.625 / 0.15625 = 4 original pixels once divided back + // through the scale. + let scores = Tensor::from_slice::(&[1], &[0.9]).unwrap(); + let boxes = Tensor::from_slice::(&[1, 4], &[0.4, 0.4, 0.6, 0.6]).unwrap(); + let classes = Tensor::from_slice::(&[1], &[5]).unwrap(); + + let result = class_index_processor() + .filter( + &MockProcessContext::new(), + &MockLogger::new(), + vec![scores, boxes, classes], + Dimensions { + width: 1920.0, + height: 1080.0, + }, + Dimensions { + width: 300.0, + height: 300.0, + }, + ResizeMode::Letterbox, + ) + .expect("filter should succeed"); + let json = result.into_bytes().unwrap().unwrap(); + let boxes: Vec = serde_json::from_slice(&json).unwrap(); + let bbox = boxes.into_iter().next().expect("one box expected"); + + // x is unpadded (the width axis is the one that fills the canvas), so it + // round-trips exactly. + assert!((bbox.x_min - 0.4).abs() < 1e-5); + assert!((bbox.x_max - 0.6).abs() < 1e-5); + + // y with the integer pad of 65: ((0.4 * 300) - 65) / 0.15625 / 1080. + // The float-pad variant would yield 0.32222 / 0.67778 instead — a 4px + // error, well outside this tolerance. + assert!((bbox.y_min - 352.0 / 1080.0).abs() < 1e-5); + assert!((bbox.y_max - 736.0 / 1080.0).abs() < 1e-5); + } + + #[test] + fn test_filter_class_index_mode_rejects_mismatched_lengths() { + use minifi_native::{MockLogger, MockProcessContext}; + use tract::__ndarray_interop::TensorInterface; + + // 2 boxes but only 1 score → the parallel-tensor contract is violated. + let scores = Tensor::from_slice::(&[1], &[0.9]).unwrap(); + let boxes = + Tensor::from_slice::(&[2, 4], &[0.1, 0.1, 0.2, 0.2, 0.5, 0.5, 0.9, 0.9]).unwrap(); + let classes = Tensor::from_slice::(&[2], &[5, 3]).unwrap(); + + let dim = Dimensions { + width: 100.0, + height: 100.0, + }; + let result = class_index_processor().filter( + &MockProcessContext::new(), + &MockLogger::new(), + vec![scores, boxes, classes], + dim, + dim, + ResizeMode::Letterbox, + ); + assert!(result.is_err(), "mismatched score/box counts should error"); + } +} diff --git a/minifi_rust/extensions/minifi_tensor/src/low_level_processors/filter_bounding_boxes/filter_bounding_boxes_def.rs b/minifi_rust/extensions/minifi_tensor/src/low_level_processors/filter_bounding_boxes/filter_bounding_boxes_def.rs new file mode 100644 index 0000000000..7ea3d35fb1 --- /dev/null +++ b/minifi_rust/extensions/minifi_tensor/src/low_level_processors/filter_bounding_boxes/filter_bounding_boxes_def.rs @@ -0,0 +1,152 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use super::{BoxFormat, FilterBoundingBoxes, ScoreActivation}; +use minifi_native::{ + OutputAttribute, ProcessorDefinition, ProcessorInputRequirement, Property, PropertyDefinition, + Relationship, property_definitions, +}; + +pub(crate) const CONFIDENCE_THRESHOLD: Property = Property::new( + "Confidence Threshold", + "Minimum per-box class probability (0.0 to 1.0) required to keep a bounding box \ + after applying the chosen 'Score activation'. Boxes below the threshold are \ + discarded before NMS.", +) +.with_default("0.7") +.supports_expression_language(); + +pub(crate) const IOU_THRESHOLD: Property = Property::new( + "IoU Threshold", + "Intersection-over-union cutoff used during non-maximum suppression. Boxes of \ + the same class whose IoU with a higher-confidence peer exceeds this value are \ + suppressed. Typical values: 0.45 (SSD/YOLO default), 0.5, 0.3 for stricter \ + deduplication.", +) +.with_default("0.45"); + +pub(crate) const SCORE_OUTPUT_INDEX: Property = Property::new( + "Score output index", + "Zero-based index of the model output tensor that holds classification scores. \ + The processor slices the concatenated payload from InvokeTractModel according \ + to the 'tensor.N.bytes' attributes.", +) +.with_default("0"); + +pub(crate) const BOX_OUTPUT_INDEX: Property = Property::new( + "Box output index", + "Zero-based index of the model output tensor that holds box coordinates. Must \ + differ from 'Score output index'.", +) +.with_default("1"); + +pub(crate) const CLASS_OUTPUT_INDEX: Property> = Property::new( + "Class output index", + "Zero-based index of a model output tensor that holds one class id per box. Set this \ + for detectors that emit boxes, per-box scores, and class ids as three separate \ + parallel tensors, with NMS already folded into the graph (TensorFlow Object \ + Detection API; YOLO / EfficientNMS 'end2end' exports). When set, 'Score output \ + index' is read as one score per box (not a [boxes, classes] matrix) and no \ + argmax is performed; the class id tensor may be integer- or float-typed. Leave \ + empty for models that emit a per-class score matrix.", +); + +pub(crate) const BOX_FORMAT: Property = Property::new( + "Box format", + "Layout of the four floats per box in the box output tensor. Xyxy = \ + [x_min, y_min, x_max, y_max] (SSD, MobileNet-SSD, most PyTorch exports). \ + Yxyx = [y_min, x_min, y_max, x_max] (TensorFlow Object Detection API). \ + Cxcywh = [cx, cy, w, h] (YOLOv3/5/8 raw output).", +) +.with_default(BoxFormat::Xyxy.into_str()); + +pub(crate) const SCORE_ACTIVATION: Property = Property::new( + "Score activation", + "Activation applied to raw per-class scores before selecting the winning class. \ + Softmax = mutually-exclusive classes (SSD/MobileNet-SSD raw logits). \ + Sigmoid = independent classes (YOLOv5/v8 style). \ + None = the model already emits probabilities/scores; use raw argmax with the \ + raw score as confidence.", +) +.with_default(ScoreActivation::Softmax.into_str()); + +pub(crate) const BACKGROUND_CLASS_INDEX: Property> = Property::new( + "Background class index", + "Index of the 'background / no-object' class. Boxes whose winning class equals this \ + index are dropped. In score-matrix mode this is only honoured when the score \ + tensor has more than one class per box; in 'Class output index' mode it is \ + matched against each box's class id.", +); + +pub(crate) const OUTPUT_ATTRIBUTE_NAME: Property> = Property::new( + "Output attribute name", + "Specify the attribute to use as output, if not provided, the content is overridden instead.", +) +.supports_expression_language(); + +pub const SUCCESS: Relationship = Relationship { + name: "success", + description: "Filtering completed. The flow file content is a JSON array of the surviving \ + bounding boxes (may be empty).", +}; + +pub const FAILURE: Relationship = Relationship { + name: "failure", + description: "The upstream output attributes were missing/invalid, the payload was truncated, \ + or the tensors could not be interpreted as scores + boxes.", +}; + +const OBJECT_COUNT_ATTR: OutputAttribute = OutputAttribute { + name: "object.count", + relationships: &["success"], + description: "Number of bounding boxes retained after confidence filtering and NMS.", +}; + +pub(crate) const MIME_TYPE_ATTR: OutputAttribute = OutputAttribute { + name: "mime.type", + relationships: &["success"], + description: "If the \"Output attribute name\" is None, then the content will be overridden with the JSON array of objects with fields class_id, confidence, x_min, y_min, x_max, y_max, and the mime type will be set to 'application/json'.", +}; + +impl ProcessorDefinition for FilterBoundingBoxes { + const DESCRIPTION: &'static str = "Post-processes the concatenated output of InvokeTractModel for object-detection models. \ + Reads the classification score tensor and the box coordinate tensor from the flow file \ + payload (indices configurable), applies the configured score activation, filters by \ + confidence, decodes box coordinates from the configured layout, and applies per-class \ + non-maximum suppression at the configured IoU threshold. Handles both per-class score \ + matrices (argmax per box) and detectors that emit boxes / per-box scores / class ids as \ + separate tensors with NMS folded into the graph (set 'Class output index'). Works with \ + SSD-, YOLO-, and TensorFlow-style detectors by tuning properties — no code changes needed \ + for common model families. Emits a JSON array of the surviving boxes."; + const INPUT_REQUIREMENT: ProcessorInputRequirement = ProcessorInputRequirement::Required; + const SUPPORTS_DYNAMIC_PROPERTIES: bool = false; + const SUPPORTS_DYNAMIC_RELATIONSHIPS: bool = false; + const OUTPUT_ATTRIBUTES: &'static [OutputAttribute] = &[OBJECT_COUNT_ATTR, MIME_TYPE_ATTR]; + const RELATIONSHIPS: &'static [Relationship] = &[SUCCESS, FAILURE]; + + const PROPERTIES: &[PropertyDefinition] = property_definitions![ + CONFIDENCE_THRESHOLD, + IOU_THRESHOLD, + SCORE_OUTPUT_INDEX, + BOX_OUTPUT_INDEX, + CLASS_OUTPUT_INDEX, + BOX_FORMAT, + SCORE_ACTIVATION, + BACKGROUND_CLASS_INDEX, + OUTPUT_ATTRIBUTE_NAME, + ]; +} diff --git a/minifi_rust/extensions/minifi_tensor/src/low_level_processors/image_to_tensor.rs b/minifi_rust/extensions/minifi_tensor/src/low_level_processors/image_to_tensor.rs new file mode 100644 index 0000000000..65d6e46608 --- /dev/null +++ b/minifi_rust/extensions/minifi_tensor/src/low_level_processors/image_to_tensor.rs @@ -0,0 +1,665 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +pub(crate) mod image_to_tensor_def; + +use crate::low_level_processors::image_to_tensor::image_to_tensor_def::TENSOR_BYTES_ATTR; +use crate::utils::dimensions::{Dimensions, LetterboxGeometry}; +use crate::utils::per_channel_f32::PerChannelF32; +use crate::utils::tensor_helpers::{MinifiDatumType, load_as_image}; +pub(crate) use image_to_tensor_def::{ + COLOR_FORMAT, LETTERBOX_PAD_VALUE, MEAN, PIXEL_DIVISOR, RESIZE_FILTER, RESIZE_MODE, STD_DEV, + TARGET_HEIGHT, TARGET_WIDTH, TENSOR_SHAPE_FORMAT, +}; +use image_to_tensor_def::{ + IMG_ORG_HEIGHT_ATTR, IMG_ORG_WIDTH_ATTR, IMG_RESIZE_MODE_ATTR, SUCCESS, TENSOR_DTYPE_ATTR, + TENSOR_SHAPE_ATTR, +}; +use image_to_tensor_def::{IMG_TRG_HEIGHT_ATTR, IMG_TRG_WIDTH_ATTR, TENSORS_LEN_ATTR}; +use minifi_native::macros::{ComponentIdentifier, PropertyType}; +use minifi_native::{ + FlowFileTransform, GetAttribute, GetControllerService, GetId, GetProperty, InputStream, Logger, + MinifiError, ProcessError, RouteErrorExt, Schedule, TransformedFlowFile, +}; +use strum_macros::{Display, EnumString, IntoStaticStr, VariantNames}; +use tract::Tensor; + +tract::impl_ndarray_interop!(); + +#[derive( + Debug, Clone, Copy, PartialEq, Display, EnumString, VariantNames, IntoStaticStr, PropertyType, +)] +#[strum(serialize_all = "PascalCase", const_into_str)] +pub(crate) enum ResizeFilter { + Nearest, + Bilinear, + Bicubic, + Lanczos3, +} + +impl From for image::imageops::FilterType { + fn from(filter: ResizeFilter) -> Self { + match filter { + ResizeFilter::Nearest => image::imageops::FilterType::Nearest, + ResizeFilter::Bilinear => image::imageops::FilterType::Triangle, + ResizeFilter::Bicubic => image::imageops::FilterType::CatmullRom, + ResizeFilter::Lanczos3 => image::imageops::FilterType::Lanczos3, + } + } +} + +#[derive( + Debug, Clone, Copy, PartialEq, Display, EnumString, VariantNames, IntoStaticStr, PropertyType, +)] +#[strum(serialize_all = "UPPERCASE", const_into_str)] +pub(crate) enum ColorFormat { + Rgb, + Bgr, + #[strum(serialize = "Grayscale")] + Grayscale, +} + +#[derive( + Debug, Clone, Copy, PartialEq, Display, EnumString, VariantNames, IntoStaticStr, PropertyType, +)] +#[strum(serialize_all = "UPPERCASE", const_into_str)] +pub(crate) enum TensorShapeFormat { + Chw, // channel, height, width + Hwc, // height, width, channel +} + +#[derive( + Debug, Clone, Copy, PartialEq, Display, EnumString, VariantNames, IntoStaticStr, PropertyType, +)] +#[strum(serialize_all = "PascalCase", const_into_str)] +pub(crate) enum ResizeMode { + Stretch, + Letterbox, +} + +#[derive(ComponentIdentifier)] +pub(crate) struct ImageToTensor { + target_width: u32, + target_height: u32, + resize_filter: ResizeFilter, + resize_mode: ResizeMode, + color_format: ColorFormat, + tensor_shape_format: TensorShapeFormat, + mean: PerChannelF32, + std_dev: PerChannelF32, + pixel_divisor: f32, + letterbox_pad_value: f32, +} + +impl Schedule for ImageToTensor { + fn schedule( + context: &Ctx, + _logger: &L, + ) -> Result + where + Self: Sized, + { + let target_width = context.get_property(&TARGET_WIDTH)?; + let target_height = context.get_property(&TARGET_HEIGHT)?; + if target_width == 0 || target_height == 0 { + return Err(MinifiError::validation( + "Target width and Target height must be greater than zero", + )); + } + let resize_filter = context.get_property(&RESIZE_FILTER)?; + let resize_mode = context.get_property(&RESIZE_MODE)?; + let color_format = context.get_property(&COLOR_FORMAT)?; + let tensor_shape_format = context.get_property(&TENSOR_SHAPE_FORMAT)?; + let mean = context.get_property(&MEAN)?; + let std_dev = context.get_property(&STD_DEV)?; + if std_dev.contains_zero() { + return Err(MinifiError::validation( + "Standard Deviation components must be non-zero", + )); + } + let pixel_divisor = context.get_property(&PIXEL_DIVISOR)?; + if pixel_divisor == 0.0 { + return Err(MinifiError::validation("Pixel divisor must be non-zero")); + } + let letterbox_pad_value = context.get_property(&LETTERBOX_PAD_VALUE)?; + + Ok(Self { + target_width, + target_height, + resize_filter, + resize_mode, + color_format, + tensor_shape_format, + mean, + std_dev, + pixel_divisor, + letterbox_pad_value, + }) + } +} + +struct MaskedRgbImage { + img: image::RgbImage, + mask: Vec, +} + +impl ImageToTensor { + fn total_pixels(&self) -> usize { + self.target_width as usize * self.target_height as usize + } + + fn stretch_resize(&self, img: image::DynamicImage) -> MaskedRgbImage { + let resized = img + .resize_exact( + self.target_width, + self.target_height, + self.resize_filter.into(), + ) + .to_rgb8(); + let mask = vec![true; self.total_pixels()]; + MaskedRgbImage { img: resized, mask } + } + + fn letterbox_resize(&self, img: image::DynamicImage) -> MaskedRgbImage { + let LetterboxGeometry { + new_width: new_w, + new_height: new_h, + pad_x, + pad_y, + .. + } = Dimensions::from_image(&img).letterbox_into(self.get_target_dim()); + let scaled = img + .resize_exact(new_w, new_h, self.resize_filter.into()) + .to_rgb8(); + + let mut canvas = image::RgbImage::from_pixel( + self.target_width, + self.target_height, + image::Rgb([0, 0, 0]), + ); + image::imageops::overlay(&mut canvas, &scaled, pad_x as i64, pad_y as i64); + + // Mask to track which pixel is part of source and which is padding + let mut mask = vec![false; self.total_pixels()]; + for y in pad_y..(new_h + pad_y) { + for x in pad_x..(new_w + pad_x) { + mask[y as usize * self.target_width as usize + x as usize] = true; + } + } + MaskedRgbImage { img: canvas, mask } + } + + fn resize_rgb(&self, img: image::DynamicImage) -> MaskedRgbImage { + match self.resize_mode { + ResizeMode::Stretch => self.stretch_resize(img), + ResizeMode::Letterbox => self.letterbox_resize(img), + } + } + + pub fn tensor_bytes(&self, img: image::DynamicImage) -> Vec { + let num_channels: usize = match self.color_format { + ColorFormat::Grayscale => 1, + _ => 3, + }; + let total_pixels = self.total_pixels(); + let mut tensor_bytes = Vec::with_capacity(total_pixels * num_channels * 4); + + let masked_img = self.resize_rgb(img); + + if self.color_format == ColorFormat::Grayscale { + let mean = self.mean.per_channel(0); + let std_dev = self.std_dev.per_channel(0); + for (idx, pixel) in masked_img.img.pixels().enumerate() { + let val = if masked_img.mask[idx] { + // Rec. 601 luma coefficients, as used by OpenCV and most ML + // preprocessing pipelines. Note this is deliberately *not* + // image::to_luma8(), which applies Rec. 709. + let luma = + 0.299 * pixel[0] as f32 + 0.587 * pixel[1] as f32 + 0.114 * pixel[2] as f32; + (luma / self.pixel_divisor - mean) / std_dev + } else { + self.letterbox_pad_value + }; + tensor_bytes.extend_from_slice(&val.to_le_bytes()); + } + } else { + let channel_order: [usize; 3] = match self.color_format { + ColorFormat::Rgb => [0, 1, 2], + ColorFormat::Bgr => [2, 1, 0], + _ => unreachable!(), + }; + + let normalized = |px_idx: usize, out_c: usize, src_c: usize| -> f32 { + if !masked_img.mask[px_idx] { + return self.letterbox_pad_value; + } + let raw = masked_img.img.get_pixel( + (px_idx as u32) % self.target_width, + (px_idx as u32) / self.target_width, + )[src_c] as f32; + (raw / self.pixel_divisor - self.mean.per_channel(out_c)) + / self.std_dev.per_channel(out_c) + }; + + match self.tensor_shape_format { + TensorShapeFormat::Chw => { + for (out_c, &src_c) in channel_order.iter().enumerate() { + for px_idx in 0..total_pixels { + let v = normalized(px_idx, out_c, src_c); + tensor_bytes.extend_from_slice(&v.to_le_bytes()); + } + } + } + TensorShapeFormat::Hwc => { + for px_idx in 0..total_pixels { + for (out_c, &src_c) in channel_order.iter().enumerate() { + let v = normalized(px_idx, out_c, src_c); + tensor_bytes.extend_from_slice(&v.to_le_bytes()); + } + } + } + } + } + + tensor_bytes + } + pub fn get_shape(&self) -> Vec { + let num_channels: usize = match self.color_format { + ColorFormat::Grayscale => 1, + _ => 3, + }; + match (self.color_format, self.tensor_shape_format) { + (ColorFormat::Grayscale, _) => { + vec![ + 1, + 1, + self.target_height as usize, + self.target_width as usize, + ] + } + (_, TensorShapeFormat::Chw) => vec![ + 1, + num_channels, + self.target_height as usize, + self.target_width as usize, + ], + (_, TensorShapeFormat::Hwc) => vec![ + 1, + self.target_height as usize, + self.target_width as usize, + num_channels, + ], + } + } + + fn get_shape_str(&self) -> String { + self.get_shape() + .iter() + .map(|n| n.to_string()) + .collect::>() + .join(",") + } + pub fn get_target_dim(&self) -> Dimensions { + Dimensions { + width: self.target_width as f32, + height: self.target_height as f32, + } + } + + pub fn get_resize_mode(&self) -> ResizeMode { + self.resize_mode + } + + pub fn get_tensor(&self, img: image::DynamicImage) -> Result { + let f32_data: Vec = self + .tensor_bytes(img) + .as_chunks::<4>() + .0 + .iter() + .map(|c| f32::from_le_bytes(*c)) + .collect(); + let shape = self.get_shape(); + let array = ndarray::Array::from_shape_vec(shape, f32_data).map_err(MinifiError::other)?; + Ok(array.tract()?) + } +} + +impl FlowFileTransform for ImageToTensor { + fn transform< + 'a, + Context: GetProperty + GetControllerService + GetAttribute + GetId, + LoggerImpl: Logger, + >( + &self, + _context: &Context, + input_stream: &'a mut dyn InputStream, + _logger: &LoggerImpl, + ) -> Result, ProcessError> { + let img = load_as_image(input_stream).route_err_to_failure()?; + let orig_dim = Dimensions::from_image(&img); + + let tensor_bytes = self.tensor_bytes(img); + let tensor_bytes_len = tensor_bytes.len(); + + Ok( + TransformedFlowFile::new(&SUCCESS, Some(tensor_bytes.into())).with_attributes([ + (&TENSORS_LEN_ATTR, "1".to_string()), + (&TENSOR_SHAPE_ATTR, self.get_shape_str()), + (&TENSOR_BYTES_ATTR, tensor_bytes_len.to_string()), + (&TENSOR_DTYPE_ATTR, MinifiDatumType::F32.to_string()), + (&IMG_ORG_HEIGHT_ATTR, orig_dim.height.to_string()), + (&IMG_ORG_WIDTH_ATTR, orig_dim.width.to_string()), + (&IMG_TRG_HEIGHT_ATTR, self.target_height.to_string()), + (&IMG_TRG_WIDTH_ATTR, self.target_width.to_string()), + (&IMG_RESIZE_MODE_ATTR, self.resize_mode.to_string()), + ]), + ) + } +} + +#[cfg(test)] +mod tests { + use super::image_to_tensor_def::{FAILURE, SUCCESS}; + use super::*; + use image::{ImageFormat, RgbImage}; + use minifi_native::{MockLogger, MockProcessContext}; + use std::io::Cursor; + + fn create_test_image_bytes() -> Vec { + let mut img = RgbImage::new(2, 2); + // Set one pixel to pure white (255, 255, 255) -> should scale to 1.0 + img.put_pixel(0, 0, image::Rgb([255, 255, 255])); + // Set one pixel to pure black (0, 0, 0) -> should scale to 0.0 + img.put_pixel(1, 1, image::Rgb([0, 0, 0])); + + let mut bytes: Vec = Vec::new(); + let mut cursor = Cursor::new(&mut bytes); + img.write_to(&mut cursor, ImageFormat::Png).unwrap(); + bytes + } + + fn default_processor() -> ImageToTensor { + ImageToTensor { + target_width: 2, + target_height: 2, + resize_filter: ResizeFilter::Nearest, + resize_mode: ResizeMode::Stretch, + color_format: ColorFormat::Rgb, + tensor_shape_format: TensorShapeFormat::Chw, + mean: PerChannelF32::SingleChannel(0.0), + std_dev: PerChannelF32::SingleChannel(255.0), + pixel_divisor: 1.0, + letterbox_pad_value: 0.0, + } + } + + fn payload_as_f32(bytes: &[u8]) -> Vec { + bytes + .as_chunks::<4>() + .0 + .iter() + .map(|c| f32::from_le_bytes(*c)) + .collect() + } + + #[test] + fn test_schedule_rejects_zero_target_dimensions() { + for (width, height) in [("0", "100"), ("100", "0"), ("0", "0")] { + let mut context = MockProcessContext::new(); + context.properties.insert(TARGET_WIDTH.name(), width); + context.properties.insert(TARGET_HEIGHT.name(), height); + context.properties.insert(PIXEL_DIVISOR.name(), "1.0"); + + assert!( + matches!( + ImageToTensor::schedule(&context, &MockLogger::new()), + Err(MinifiError::ValidationError(_)) + ), + "expected {width}x{height} to be rejected" + ); + } + } + + #[test] + fn test_schedule_accepts_non_zero_target_dimensions() { + let mut context = MockProcessContext::new(); + context.properties.insert(TARGET_WIDTH.name(), "224"); + context.properties.insert(TARGET_HEIGHT.name(), "224"); + context.properties.insert(PIXEL_DIVISOR.name(), "1.0"); + + assert!(ImageToTensor::schedule(&context, &MockLogger::new()).is_ok()); + } + + #[test] + fn test_successful_rgb_chw_transform() { + let processor = default_processor(); + + let context = MockProcessContext::new(); + let input_bytes = create_test_image_bytes(); + let mut input_stream = Cursor::new(input_bytes); + + let result = processor + .transform(&context, &mut input_stream, &MockLogger::new()) + .expect("Transform should succeed"); + + assert_eq!(result.target_relationship(), SUCCESS.name); + + assert_eq!( + result + .attribute("tensor.0.shape") + .expect("Missing shape attribute"), + "1,3,2,2" + ); + assert_eq!( + result + .attribute("tensor.0.dtype") + .expect("Missing dtype attribute"), + "F32" + ); + assert_eq!( + result + .attribute("image.resize.mode") + .expect("Missing resize mode attribute"), + "Stretch" + ); + + let payload: Vec = result + .into_bytes() + .unwrap() + .expect("there should be a payload"); + + assert_eq!(payload.len(), 48); + + let first_f32 = payload_as_f32(&payload)[0]; + assert_eq!(first_f32, 1.0); + } + + #[test] + fn test_successful_grayscale_transform() { + let processor = ImageToTensor { + color_format: ColorFormat::Grayscale, + ..default_processor() + }; + + let context = MockProcessContext::new(); + let input_bytes = create_test_image_bytes(); + let mut input_stream = Cursor::new(input_bytes); + + let result = processor + .transform(&context, &mut input_stream, &MockLogger::new()) + .expect("Transform should succeed"); + + assert_eq!(result.target_relationship(), SUCCESS.name); + assert_eq!( + result + .attribute("tensor.0.shape") + .expect("Missing shape attribute"), + "1,1,2,2" + ); + + let payload = result + .into_bytes() + .unwrap() + .expect("there should be a payload"); + // 2x2 image * 1 channel * 4 bytes per f32 = 16 bytes + assert_eq!(payload.len(), 16); + } + + #[test] + fn test_invalid_image_routes_to_failure() { + let processor = ImageToTensor { + target_width: 224, + target_height: 224, + resize_filter: ResizeFilter::Bilinear, + ..default_processor() + }; + + let context = MockProcessContext::new(); + + let invalid_bytes = vec![0x00, 0x01, 0x02, 0x03, 0x04]; + let mut input_stream = Cursor::new(invalid_bytes); + + let err = processor + .transform(&context, &mut input_stream, &MockLogger::new()) + .expect_err("Invalid image should route to FAILURE via a Route error"); + + match err { + ProcessError::Route(route) => { + assert_eq!(route.relationship, FAILURE.name) + } + other => panic!("expected route to failure, got {other:?}"), + } + } + + #[test] + fn test_hwc_shape_attribute_matches_layout() { + let processor = ImageToTensor { + tensor_shape_format: TensorShapeFormat::Hwc, + ..default_processor() + }; + let context = MockProcessContext::new(); + let mut input_stream = Cursor::new(create_test_image_bytes()); + let result = processor + .transform(&context, &mut input_stream, &MockLogger::new()) + .unwrap(); + assert_eq!(result.attribute("tensor.0.shape").unwrap(), "1,2,2,3"); + } + + #[test] + fn test_per_channel_mean_and_std() { + // Use per-channel mean/std where R uses (mean=0, std=255) → 1.0, + // G uses (mean=255, std=255) → 0.0, B uses (mean=0, std=127.5) → 2.0. + let processor = ImageToTensor { + mean: PerChannelF32::TriChannel([0.0, 255.0, 0.0]), + std_dev: PerChannelF32::TriChannel([255.0, 255.0, 127.5]), + ..default_processor() + }; + let context = MockProcessContext::new(); + let mut input_stream = Cursor::new(create_test_image_bytes()); + let result = processor + .transform(&context, &mut input_stream, &MockLogger::new()) + .unwrap(); + let payload = payload_as_f32(&result.into_bytes().unwrap().unwrap()); + // CHW layout: 4 R values, 4 G values, 4 B values. Pixel (0,0) is white. + assert!( + (payload[0] - 1.0).abs() < 1e-6, + "R channel pixel(0,0) = {}", + payload[0] + ); + assert!( + (payload[4] - 0.0).abs() < 1e-6, + "G channel pixel(0,0) = {}", + payload[4] + ); + assert!( + (payload[8] - 2.0).abs() < 1e-6, + "B channel pixel(0,0) = {}", + payload[8] + ); + } + + #[test] + fn test_letterbox_pads_non_square_image() { + // Create a 4x2 image (aspect 2:1) — letterboxed into 4x4 should get + // top/bottom padding of one row each, and the middle two rows should + // contain the source content. + let mut img = RgbImage::new(4, 2); + for y in 0..2 { + for x in 0..4 { + img.put_pixel(x, y, image::Rgb([255, 255, 255])); + } + } + let mut bytes: Vec = Vec::new(); + img.write_to(&mut Cursor::new(&mut bytes), ImageFormat::Png) + .unwrap(); + + let processor = ImageToTensor { + target_width: 4, + target_height: 4, + resize_mode: ResizeMode::Letterbox, + letterbox_pad_value: -1.0, + ..default_processor() + }; + let context = MockProcessContext::new(); + let mut input_stream = Cursor::new(bytes); + let result = processor + .transform(&context, &mut input_stream, &MockLogger::new()) + .unwrap(); + let payload = payload_as_f32(&result.into_bytes().unwrap().unwrap()); + + // CHW, R channel first. 16 pixels per channel. Rows: 0 = pad, 1-2 = content, 3 = pad. + // R channel: + // row 0 (pixels 0..4) = pad value (-1.0) + // rows 1..3 (pixels 4..12) = content (1.0 after /255) + // row 3 (pixels 12..16) = pad value (-1.0) + for (i, v) in payload.iter().enumerate().take(4) { + assert!((v + 1.0).abs() < 1e-6, "expected pad at r[{}]", i); + } + for (i, v) in payload.iter().enumerate().take(12).skip(4) { + assert!((v - 1.0).abs() < 1e-6, "expected content at r[{}]", i); + } + for (i, v) in payload.iter().enumerate().take(16).skip(12) { + assert!((v + 1.0).abs() < 1e-6, "expected pad at r[{}]", i); + } + } + + #[test] + fn test_pixel_divisor_matches_tract_imagenet_recipe() { + // Reproduces the tract onnx-mobilenet-v2 example's normalization: + // (raw / 255.0 - mean_[0,1]) / std_[0,1] + // A pure-white pixel R channel should become (1.0 - 0.485) / 0.229 ≈ 2.2489. + let processor = ImageToTensor { + mean: PerChannelF32::TriChannel([0.485, 0.456, 0.406]), + std_dev: PerChannelF32::TriChannel([0.229, 0.224, 0.225]), + pixel_divisor: 255.0, + ..default_processor() + }; + let context = MockProcessContext::new(); + let mut input_stream = Cursor::new(create_test_image_bytes()); + let result = processor + .transform(&context, &mut input_stream, &MockLogger::new()) + .unwrap(); + let payload = payload_as_f32(&result.into_bytes().unwrap().unwrap()); + // Pixel (0,0) is white (255,255,255). CHW → payload[0] is R for pixel(0,0). + let expected_r = (1.0_f32 - 0.485) / 0.229; + assert!( + (payload[0] - expected_r).abs() < 1e-5, + "R channel pixel(0,0) = {}, expected {}", + payload[0], + expected_r + ); + } +} diff --git a/minifi_rust/extensions/minifi_tensor/src/low_level_processors/image_to_tensor/image_to_tensor_def.rs b/minifi_rust/extensions/minifi_tensor/src/low_level_processors/image_to_tensor/image_to_tensor_def.rs new file mode 100644 index 0000000000..03c486fa39 --- /dev/null +++ b/minifi_rust/extensions/minifi_tensor/src/low_level_processors/image_to_tensor/image_to_tensor_def.rs @@ -0,0 +1,206 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use super::{ColorFormat, ImageToTensor, ResizeFilter, ResizeMode, TensorShapeFormat}; +use crate::utils::per_channel_f32::PerChannelF32; +use minifi_native::{ + OutputAttribute, ProcessorDefinition, ProcessorInputRequirement, Property, PropertyDefinition, + Relationship, property_definitions, +}; + +pub(crate) const TARGET_WIDTH: Property = Property::new( + "Target width", + "Width in pixels the decoded image is resized to before normalisation and \ + inference.", +); + +pub(crate) const TARGET_HEIGHT: Property = Property::new( + "Target height", + "Height in pixels the decoded image is resized to before normalisation and \ + inference.", +); + +pub(crate) const RESIZE_FILTER: Property = Property::new( + "Resize filter", + "Interpolation filter applied when resizing the decoded image. Nearest is fastest \ + but blocky; Bilinear is a good default; Bicubic and Lanczos3 are higher-quality \ + but slower.", +) +.with_default(ResizeFilter::Bilinear.into_str()); + +pub(crate) const RESIZE_MODE: Property = Property::new( + "Resize mode", + "How the source image is fitted into the target dimensions. 'Stretch' scales each \ + axis independently, distorting aspect ratio. 'Letterbox' preserves aspect ratio \ + and pads the remaining border with 'Letterbox pad value' (applied in normalised \ + output space).", +) +.with_default(ResizeMode::Stretch.into_str()); + +pub(crate) const LETTERBOX_PAD_VALUE: Property = Property::new( + "Letterbox pad value", + "Value written for padding pixels when 'Resize mode' is 'Letterbox'. This is a \ + normalised value (post mean/std), so 0.0 corresponds to a neutral input for most \ + networks. Ignored when 'Resize mode' is 'Stretch'.", +) +.with_default("0.0"); + +pub(crate) const COLOR_FORMAT: Property = Property::new( + "Color format", + "Color space of the tensor fed to the model. RGB and BGR produce three-channel \ + tensors (channel order determined by the format); Grayscale produces a \ + single-channel luma tensor.", +) +.with_default(ColorFormat::Rgb.into_str()); + +pub(crate) const TENSOR_SHAPE_FORMAT: Property = Property::new( + "Tensor shape format", + "Memory layout of the tensor fed to the model. CHW (channels-first) is typical \ + for PyTorch/ONNX detectors. HWC (channels-last) matches TensorFlow/TFLite. \ + Ignored for Grayscale (always effectively 1xHxW).", +) +.with_default(TensorShapeFormat::Chw.into_str()); + +pub(crate) const MEAN: Property = Property::new( + "Mean", + "Value subtracted from each pixel before dividing by 'Standard Deviation'. Accepts \ + either a single value (broadcast to all channels) or three comma-separated \ + values applied per channel in the order dictated by 'Color format'. Example: \ + '0.485, 0.456, 0.406' for ImageNet-style RGB normalisation.", +) +.with_default("0.0"); + +pub(crate) const STD_DEV: Property = Property::new( + "Standard Deviation", + "Divisor applied after subtracting 'Mean'. Accepts a single value (broadcast) or \ + three comma-separated values (per channel). Must be non-zero. Example: '255.0' \ + to scale u8 pixels into [0.0, 1.0]; '0.229, 0.224, 0.225' for ImageNet.", +) +.with_default("255.0"); + +pub(crate) const PIXEL_DIVISOR: Property = Property::new( + "Pixel divisor", + "Divisor applied to raw u8 pixel values before subtracting 'Mean' and dividing \ + by 'Standard Deviation'. Defaults to 1.0 (mean/std interpreted in [0, 255] pixel \ + space, e.g. UltraFace's mean=127, std=128). Set to 255 to bring pixels into \ + [0.0, 1.0] first so ImageNet-style mean/std values like '0.485, 0.456, 0.406' / \ + '0.229, 0.224, 0.225' can be used directly, matching the PyTorch / torchvision / \ + ONNX MobileNet convention. Must be non-zero.", +) +.with_default("1.0"); + +pub(super) const SUCCESS: Relationship = Relationship { + name: "success", + description: "The input image was decoded and converted to a tensor.", +}; + +pub(super) const FAILURE: Relationship = Relationship { + name: "failure", + description: "The input flow file could not be decoded as an image.", +}; + +pub(super) const TENSORS_LEN_ATTR: OutputAttribute = OutputAttribute { + name: "tensors.len", + relationships: &["success"], + description: "Number of tensors in the output FlowFile. Currently always '1'", +}; + +pub(super) const TENSOR_BYTES_ATTR: OutputAttribute = OutputAttribute { + name: "tensor.0.bytes", + relationships: &["success"], + description: "Byte length of output tensor.", +}; + +pub(super) const TENSOR_SHAPE_ATTR: OutputAttribute = OutputAttribute { + name: "tensor.0.shape", + relationships: &["success"], + description: "Comma-separated dimensions of the output tensor in the chosen layout, always \ + including a leading batch dimension of 1 (e.g. '1,3,224,224' for RGB CHW).", +}; + +pub(super) const TENSOR_DTYPE_ATTR: OutputAttribute = OutputAttribute { + name: "tensor.0.dtype", + relationships: &["success"], + description: "Element type of the values in the output tensor. Currently always 'F32'.", +}; + +pub(super) const IMG_ORG_HEIGHT_ATTR: OutputAttribute = OutputAttribute { + name: "image.original.height", + relationships: &["success"], + description: "The height of the original image before the resizing.", +}; + +pub(super) const IMG_ORG_WIDTH_ATTR: OutputAttribute = OutputAttribute { + name: "image.original.width", + relationships: &["success"], + description: "The width of the original image before the resizing.", +}; + +pub(super) const IMG_TRG_HEIGHT_ATTR: OutputAttribute = OutputAttribute { + name: "image.target.height", + relationships: &["success"], + description: "The height of the image after the resizing.", +}; + +pub(super) const IMG_TRG_WIDTH_ATTR: OutputAttribute = OutputAttribute { + name: "image.target.width", + relationships: &["success"], + description: "The width of the image after the resizing.", +}; + +pub(super) const IMG_RESIZE_MODE_ATTR: OutputAttribute = OutputAttribute { + name: "image.resize.mode", + relationships: &["success"], + description: "The resize mode ('Stretch' or 'Letterbox') applied to fit the image into the \ + target dimensions. Downstream processors such as FilterBoundingBoxes use this \ + to invert the coordinate mapping correctly.", +}; + +impl ProcessorDefinition for ImageToTensor { + const DESCRIPTION: &'static str = "Decodes an image from the flow file content and converts it into a normalised numeric \ + tensor suitable for feeding into a downstream inference processor such as \ + InvokeTractModel. Supports RGB / BGR / Grayscale, CHW / HWC layouts, stretch or \ + letterbox resizing, and scalar or per-channel mean/std normalisation. The output payload \ + is a single raw little-endian f32 tensor; the 'tensor.0.shape' attribute describes its layout."; + const INPUT_REQUIREMENT: ProcessorInputRequirement = ProcessorInputRequirement::Required; + const SUPPORTS_DYNAMIC_PROPERTIES: bool = false; + const SUPPORTS_DYNAMIC_RELATIONSHIPS: bool = false; + const OUTPUT_ATTRIBUTES: &'static [OutputAttribute] = &[ + TENSORS_LEN_ATTR, + TENSOR_BYTES_ATTR, + TENSOR_SHAPE_ATTR, + TENSOR_DTYPE_ATTR, + IMG_ORG_WIDTH_ATTR, + IMG_ORG_HEIGHT_ATTR, + IMG_TRG_WIDTH_ATTR, + IMG_TRG_HEIGHT_ATTR, + IMG_RESIZE_MODE_ATTR, + ]; + const RELATIONSHIPS: &'static [Relationship] = &[SUCCESS, FAILURE]; + const PROPERTIES: &[PropertyDefinition] = property_definitions![ + TARGET_WIDTH, + TARGET_HEIGHT, + RESIZE_FILTER, + RESIZE_MODE, + LETTERBOX_PAD_VALUE, + COLOR_FORMAT, + TENSOR_SHAPE_FORMAT, + MEAN, + STD_DEV, + PIXEL_DIVISOR, + ]; +} diff --git a/minifi_rust/extensions/minifi_tensor/src/low_level_processors/invoke_tract_model.rs b/minifi_rust/extensions/minifi_tensor/src/low_level_processors/invoke_tract_model.rs new file mode 100644 index 0000000000..5485c1b644 --- /dev/null +++ b/minifi_rust/extensions/minifi_tensor/src/low_level_processors/invoke_tract_model.rs @@ -0,0 +1,119 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use crate::utils::tensor_helpers::deserialize_tensors; +pub(crate) use invoke_tract_model_def::TRACT_MODEL_SERVICE; +use invoke_tract_model_def::*; +use minifi_native::macros::ComponentIdentifier; +use minifi_native::{ + FlowFileTransform, GetAttribute, GetControllerService, GetId, GetProperty, InputStream, Logger, + MinifiError, ProcessError, RouteErrorExt, Schedule, TransformedFlowFile, +}; +use tract::__ndarray_interop::TensorInterface; +use tract::Tensor; +tract::impl_ndarray_interop!(); + +mod invoke_tract_model_def; + +#[derive(ComponentIdentifier)] +pub(crate) struct InvokeTractModel {} + +impl Schedule for InvokeTractModel { + fn schedule( + _context: &Ctx, + _logger: &L, + ) -> Result + where + Self: Sized, + { + Ok(Self {}) + } +} + +impl FlowFileTransform for InvokeTractModel { + fn transform< + 'a, + Context: GetProperty + GetControllerService + GetAttribute + GetId, + LoggerImpl: Logger, + >( + &self, + context: &Context, + input_stream: &'a mut dyn InputStream, + _logger: &LoggerImpl, + ) -> Result, ProcessError> { + let controller_service = context.get_controller_service(&TRACT_MODEL_SERVICE)?; + + let input_tensors: Vec = + deserialize_tensors(context, input_stream).route_err_to_failure()?; + if input_tensors.len() != 1 { + return Err(ProcessError::route_to_failure("Invalid input")); + }; + + let output_tensors = controller_service + .run_inference(input_tensors) + .route_err_to_failure()?; + let mut output_bytes = Vec::new(); + let mut transformed = TransformedFlowFile::new(&SUCCESS, None) + .with_attribute("tensors.len", output_tensors.len().to_string()); + + for (i, tensor) in output_tensors.iter().enumerate() { + let (datum_type, out_shape, raw_tensor_bytes) = tensor + .as_bytes() + .map_err(|e| MinifiError::custom(format!("Failed to read tensor bytes: {}", e)))?; + + output_bytes.extend_from_slice(raw_tensor_bytes); + + let out_shape_str = out_shape + .iter() + .map(|d| d.to_string()) + .collect::>() + .join(","); + + let tensor_bytes = raw_tensor_bytes.len().to_string(); + transformed = transformed.with_attributes([ + (format!("tensor.{}.shape", i), out_shape_str), + (format!("tensor.{}.bytes", i), tensor_bytes), + (format!("tensor.{}.dtype", i), format!("{:?}", datum_type)), + ]); + } + + Ok(transformed.with_content(output_bytes.into())) + } +} + +#[cfg(test)] +mod tests { + use crate::InvokeTractModel; + use minifi_native::FlowFileTransform; + use minifi_native::{MockLogger, MockProcessContext}; + use std::io::Cursor; + + #[test] + fn test_transform_missing_controller_service_throws_error() { + let processor = InvokeTractModel {}; + let context = MockProcessContext::new(); + let mut stream = Cursor::new(vec![]); + let logger = MockLogger::new(); + + let result = processor.transform(&context, &mut stream, &logger); + + assert!( + result.is_err(), + "Should throw an error when TractModelService is missing" + ); + } +} diff --git a/minifi_rust/extensions/minifi_tensor/src/low_level_processors/invoke_tract_model/invoke_tract_model_def.rs b/minifi_rust/extensions/minifi_tensor/src/low_level_processors/invoke_tract_model/invoke_tract_model_def.rs new file mode 100644 index 0000000000..bc1a2bb66e --- /dev/null +++ b/minifi_rust/extensions/minifi_tensor/src/low_level_processors/invoke_tract_model/invoke_tract_model_def.rs @@ -0,0 +1,87 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use crate::services::tract_model_service::TractModelService; +use minifi_native::{ + OutputAttribute, ProcessorDefinition, ProcessorInputRequirement, Property, PropertyDefinition, + Relationship, property_definitions, +}; + +pub(crate) const TRACT_MODEL_SERVICE: Property = Property::new( + "Tract model service", + "Reference to a TractModelService controller service. The referenced service \ + owns the compiled model (ONNX or NNEF) that will be evaluated for each \ + incoming flow file.", +); + +pub(super) const SUCCESS: Relationship = Relationship { + name: "success", + description: "Inference completed. The flow file's content is the concatenation of every \ + output tensor's raw bytes in model output order.", +}; + +pub(super) const FAILURE: Relationship = Relationship { + name: "failure", + description: "The input tensor could not be built (missing/invalid tensor.0.shape, unsupported \ + tensor.0.dtype, malformed payload) or the model failed to run.", +}; + +const OUTPUT_COUNT_ATTR: OutputAttribute = OutputAttribute { + name: "tensors.len", + relationships: &["success"], + description: "Number of output tensors produced by the model. Downstream processors can loop \ + from 0 up to this count when reading per-output attributes.", +}; + +const OUTPUT_SHAPE_ATTR: OutputAttribute = OutputAttribute { + name: "tensor.{i}.shape", + relationships: &["success"], + description: "Comma-separated dimensions of output tensor at index i.", +}; + +const OUTPUT_BYTES_ATTR: OutputAttribute = OutputAttribute { + name: "tensor.{i}.bytes", + relationships: &["success"], + description: "Byte length of output tensor at index i within the concatenated payload. \ + Consumers slice the payload sequentially using these lengths.", +}; + +const OUTPUT_DTYPE_ATTR: OutputAttribute = OutputAttribute { + name: "tensor.{i}.dtype", + relationships: &["success"], + description: "Element type of output tensor at index i", +}; + +impl ProcessorDefinition for super::InvokeTractModel { + const DESCRIPTION: &'static str = "Runs a single inference against the compiled model owned by the referenced \ + TractModelService. Reads the input tensor from the flow file content plus the \ + 'tensor.0.shape' and (optionally) 'tensor.0.dtype' attributes produced by an upstream \ + processor such as ImageToTensor. The flow file's new content is every output tensor's \ + raw bytes concatenated in model order; per-tensor shape, byte length, and dtype are \ + written to attributes."; + const INPUT_REQUIREMENT: ProcessorInputRequirement = ProcessorInputRequirement::Required; + const SUPPORTS_DYNAMIC_PROPERTIES: bool = false; + const SUPPORTS_DYNAMIC_RELATIONSHIPS: bool = false; + const OUTPUT_ATTRIBUTES: &'static [OutputAttribute] = &[ + OUTPUT_COUNT_ATTR, + OUTPUT_SHAPE_ATTR, + OUTPUT_BYTES_ATTR, + OUTPUT_DTYPE_ATTR, + ]; + const RELATIONSHIPS: &'static [Relationship] = &[SUCCESS, FAILURE]; + const PROPERTIES: &[PropertyDefinition] = property_definitions![TRACT_MODEL_SERVICE]; +} diff --git a/minifi_rust/extensions/minifi_tensor/src/low_level_processors/mod.rs b/minifi_rust/extensions/minifi_tensor/src/low_level_processors/mod.rs new file mode 100644 index 0000000000..fe18ceddcb --- /dev/null +++ b/minifi_rust/extensions/minifi_tensor/src/low_level_processors/mod.rs @@ -0,0 +1,22 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +pub(crate) mod classify_output; +pub(crate) mod filter_bounding_boxes; +pub(crate) mod image_to_tensor; +#[cfg(feature = "low-level-processors")] +pub(crate) mod invoke_tract_model; diff --git a/minifi_rust/extensions/minifi_tensor/src/processors/classify_image.rs b/minifi_rust/extensions/minifi_tensor/src/processors/classify_image.rs new file mode 100644 index 0000000000..548dfe466e --- /dev/null +++ b/minifi_rust/extensions/minifi_tensor/src/processors/classify_image.rs @@ -0,0 +1,80 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +mod classify_image_def; + +use crate::low_level_processors::classify_output::ClassifyOutput; +use crate::low_level_processors::image_to_tensor::ImageToTensor; +use crate::utils::tensor_helpers::load_as_image; +use classify_image_def::TRACT_MODEL_SERVICE; +use minifi_native::macros::ComponentIdentifier; +use minifi_native::{ + FlowFileTransform, GetAttribute, GetControllerService, GetId, GetProperty, InputStream, Logger, + MinifiError, ProcessError, RouteErrorExt, Schedule, TransformedFlowFile, +}; +use tract::Tensor; + +#[derive(ComponentIdentifier)] +pub(crate) struct ClassifyImage { + image_to_tensor: ImageToTensor, + classify_output: ClassifyOutput, +} + +impl Schedule for ClassifyImage { + fn schedule(context: &Ctx, logger: &L) -> Result + where + Self: Sized, + { + let image_to_tensor = ImageToTensor::schedule(context, logger)?; + let classify_output = ClassifyOutput::schedule(context, logger)?; + Ok(Self { + image_to_tensor, + classify_output, + }) + } +} + +impl FlowFileTransform for ClassifyImage { + fn transform< + 'a, + Context: GetProperty + GetControllerService + GetAttribute + GetId, + LoggerImpl: Logger, + >( + &self, + context: &Context, + input_stream: &'a mut dyn InputStream, + logger: &LoggerImpl, + ) -> Result, ProcessError> { + let tract_model_service = context.get_controller_service(&TRACT_MODEL_SERVICE)?; + let img = load_as_image(input_stream).route_err_to_failure()?; + + // ImageToTensor + let input_tensor: Tensor = self + .image_to_tensor + .get_tensor(img) + .route_err_to_failure()?; + + // InvokeTract + let output_tensors = tract_model_service + .run_inference(vec![input_tensor]) + .route_err_to_failure()?; + + // ClassifyOutput + self.classify_output + .classify(context, logger, output_tensors) + } +} diff --git a/minifi_rust/extensions/minifi_tensor/src/processors/classify_image/classify_image_def.rs b/minifi_rust/extensions/minifi_tensor/src/processors/classify_image/classify_image_def.rs new file mode 100644 index 0000000000..a7e86c57a0 --- /dev/null +++ b/minifi_rust/extensions/minifi_tensor/src/processors/classify_image/classify_image_def.rs @@ -0,0 +1,82 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use crate::low_level_processors::{classify_output, image_to_tensor}; +use crate::processors::classify_image::ClassifyImage; +use crate::services::tract_model_service::TractModelService; +use minifi_native::{ + OutputAttribute, ProcessorDefinition, ProcessorInputRequirement, Property, PropertyDefinition, + Relationship, property_definitions, +}; + +pub(crate) const TRACT_MODEL_SERVICE: Property = Property::new( + "Tract model service", + "Reference to a TractModelService controller service. The referenced service \ + owns the compiled model (ONNX or NNEF) that will be evaluated for each \ + incoming flow file.", +); + +pub(super) const SUCCESS: Relationship = Relationship { + name: "success", + description: "Inference and post-processing completed. The flow file content is the original, \ + unchanged image; the classifications are written to the configured output \ + attribute as a JSON array (may be empty).", +}; + +pub(super) const FAILURE: Relationship = Relationship { + name: "failure", + description: "The image could not be decoded, the input tensor could not be built, the model \ + failed to run, or the model outputs could not be interpreted as classification.", +}; + +impl ProcessorDefinition for ClassifyImage { + const DESCRIPTION: &'static str = "Runs a full image-classification pass in a single processor: decodes the image from the \ + flow file content, resizes and normalises it into an input tensor, runs one inference \ + against the compiled model owned by the referenced TractModelService, and post-processes \ + the score vector (score activation, Top-K selection, confidence filtering, optional label \ + lookup) into predictions. Collapses the ImageToTensor -> InvokeTractModel -> \ + ClassifyOutput chain into one node. The flow file content is left unchanged (the original \ + image); the Top-K classifications are written as a JSON array to the configured output \ + attribute."; + const INPUT_REQUIREMENT: ProcessorInputRequirement = ProcessorInputRequirement::Required; + const SUPPORTS_DYNAMIC_PROPERTIES: bool = false; + const SUPPORTS_DYNAMIC_RELATIONSHIPS: bool = false; + const OUTPUT_ATTRIBUTES: &'static [OutputAttribute] = + classify_output::CLASSIFY_OUTPUT_ATTRIBUTES; + const RELATIONSHIPS: &'static [Relationship] = &[SUCCESS, FAILURE]; + + const PROPERTIES: &'static [PropertyDefinition] = property_definitions![ + image_to_tensor::TARGET_WIDTH, + image_to_tensor::TARGET_HEIGHT, + image_to_tensor::RESIZE_FILTER, + image_to_tensor::RESIZE_MODE, + image_to_tensor::LETTERBOX_PAD_VALUE, + image_to_tensor::COLOR_FORMAT, + image_to_tensor::TENSOR_SHAPE_FORMAT, + image_to_tensor::MEAN, + image_to_tensor::STD_DEV, + image_to_tensor::PIXEL_DIVISOR, + TRACT_MODEL_SERVICE, + classify_output::TOP_K, + classify_output::SCORE_OUTPUT_INDEX, + classify_output::SCORE_ACTIVATION, + classify_output::CONFIDENCE_THRESHOLD, + classify_output::LABELS_FILE_PATH, + classify_output::LABEL_INDEX_OFFSET, + classify_output::OUTPUT_ATTRIBUTE_NAME + ]; +} diff --git a/minifi_rust/extensions/minifi_tensor/src/processors/detect_object.rs b/minifi_rust/extensions/minifi_tensor/src/processors/detect_object.rs new file mode 100644 index 0000000000..fbf6301cec --- /dev/null +++ b/minifi_rust/extensions/minifi_tensor/src/processors/detect_object.rs @@ -0,0 +1,118 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +mod detect_object_def; + +use crate::low_level_processors::filter_bounding_boxes::FilterBoundingBoxes; +use crate::low_level_processors::image_to_tensor::ImageToTensor; +use crate::utils::dimensions::Dimensions; +use crate::utils::tensor_helpers::load_as_image; +use detect_object_def::TRACT_MODEL_SERVICE; +use minifi_native::macros::ComponentIdentifier; +use minifi_native::{ + FlowFileTransform, GetAttribute, GetControllerService, GetId, GetProperty, InputStream, Logger, + MinifiError, ProcessError, RouteErrorExt, Schedule, TransformedFlowFile, +}; +use tract::Tensor; + +tract::impl_ndarray_interop!(); + +#[derive(ComponentIdentifier)] +pub(crate) struct DetectObject { + image_to_tensor: ImageToTensor, + filter_bounding_boxes: FilterBoundingBoxes, +} + +impl Schedule for DetectObject { + fn schedule(context: &Ctx, logger: &L) -> Result + where + Self: Sized, + { + let image_to_tensor = ImageToTensor::schedule(context, logger)?; + let filter_bounding_boxes = FilterBoundingBoxes::schedule(context, logger)?; + Ok(Self { + image_to_tensor, + filter_bounding_boxes, + }) + } +} + +impl FlowFileTransform for DetectObject { + fn transform< + 'a, + Context: GetProperty + GetControllerService + GetAttribute + GetId, + LoggerImpl: Logger, + >( + &self, + context: &Context, + input_stream: &'a mut dyn InputStream, + logger: &LoggerImpl, + ) -> Result, ProcessError> { + let tract_model_service = context.get_controller_service(&TRACT_MODEL_SERVICE)?; + let img = load_as_image(input_stream).route_err_to_failure()?; + let orig_dim = Dimensions::from_image(&img); + let target_dim = self.image_to_tensor.get_target_dim(); + + // ImageToTensor + let input_tensor: Tensor = self + .image_to_tensor + .get_tensor(img) + .route_err_to_failure()?; + + // InvokeTract + let output_tensors = tract_model_service + .run_inference(vec![input_tensor]) + .route_err_to_failure()?; + + // FilterBoundingBox + self.filter_bounding_boxes.filter( + context, + logger, + output_tensors, + orig_dim, + target_dim, + self.image_to_tensor.get_resize_mode(), + ) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::low_level_processors::image_to_tensor::{ + PIXEL_DIVISOR, TARGET_HEIGHT, TARGET_WIDTH, + }; + use minifi_native::{MockLogger, MockProcessContext}; + use std::io::Cursor; + + #[test] + fn test_missing_controller_service_errors() { + let mut context = MockProcessContext::new(); + context.properties.insert(TARGET_HEIGHT.name(), "100"); + context.properties.insert(TARGET_WIDTH.name(), "100"); + context.properties.insert(PIXEL_DIVISOR.name(), "1.0"); + let processor = + DetectObject::schedule(&context, &MockLogger::new()).expect("Expected to schedule"); + let mut input_stream = Cursor::new(vec![]); + + let result = processor.transform(&context, &mut input_stream, &MockLogger::new()); + assert!( + result.is_err(), + "Should error when TractModelService is missing" + ); + } +} diff --git a/minifi_rust/extensions/minifi_tensor/src/processors/detect_object/detect_object_def.rs b/minifi_rust/extensions/minifi_tensor/src/processors/detect_object/detect_object_def.rs new file mode 100644 index 0000000000..c4e56e9ab1 --- /dev/null +++ b/minifi_rust/extensions/minifi_tensor/src/processors/detect_object/detect_object_def.rs @@ -0,0 +1,99 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use crate::low_level_processors::{filter_bounding_boxes, image_to_tensor}; +use crate::processors::detect_object::DetectObject; +use crate::services::tract_model_service::TractModelService; +use minifi_native::{ + OutputAttribute, ProcessorDefinition, ProcessorInputRequirement, Property, PropertyDefinition, + Relationship, property_definitions, +}; + +pub(crate) const TRACT_MODEL_SERVICE: Property = Property::new( + "Tract model service", + "Reference to a TractModelService controller service. The referenced service \ + owns the compiled model (ONNX or NNEF) that will be evaluated for each \ + incoming flow file.", +); + +pub(super) const SUCCESS: Relationship = Relationship { + name: "success", + description: "Inference and post-processing completed. The flow file content is the original, \ + unchanged image; the detected boxes are written to the configured output \ + attribute as a JSON array (may be empty).", +}; + +pub(super) const FAILURE: Relationship = Relationship { + name: "failure", + description: "The image could not be decoded, the input tensor could not be built, the model \ + failed to run, or the model outputs could not be interpreted as scores + boxes.", +}; + +const OBJECT_COUNT_ATTR: OutputAttribute = OutputAttribute { + name: "object.count", + relationships: &["success"], + description: "Number of bounding boxes retained after confidence filtering and NMS.", +}; + +const DETECTED_OBJECTS_ATTR: OutputAttribute = OutputAttribute { + name: "", + relationships: &["success"], + description: "JSON array of the surviving bounding boxes (fields class_id, confidence, x_min, \ + y_min, x_max, y_max; coordinates normalised to [0,1] against the original \ + image). The attribute name is configurable via the 'Output attribute name' \ + property.", +}; + +impl ProcessorDefinition for DetectObject { + const DESCRIPTION: &'static str = "Runs a full object-detection pass in a single processor: decodes the image from the flow \ + file content, resizes and normalises it into an input tensor, runs one inference against \ + the compiled model owned by the referenced TractModelService, and post-processes the \ + model outputs (score activation, confidence filtering, box decoding, per-class \ + non-maximum suppression) into bounding boxes. Collapses the ImageToTensor -> \ + InvokeTractModel -> FilterBoundingBoxes chain into one node. The flow file content is \ + left unchanged (the original image); the detected boxes are written as a JSON array to \ + the configured output attribute so a downstream DrawBoundingBox can annotate the image."; + const INPUT_REQUIREMENT: ProcessorInputRequirement = ProcessorInputRequirement::Required; + const SUPPORTS_DYNAMIC_PROPERTIES: bool = false; + const SUPPORTS_DYNAMIC_RELATIONSHIPS: bool = false; + const OUTPUT_ATTRIBUTES: &'static [OutputAttribute] = + &[OBJECT_COUNT_ATTR, DETECTED_OBJECTS_ATTR]; + const RELATIONSHIPS: &'static [Relationship] = &[SUCCESS, FAILURE]; + + const PROPERTIES: &'static [PropertyDefinition] = property_definitions![ + image_to_tensor::TARGET_WIDTH, + image_to_tensor::TARGET_HEIGHT, + image_to_tensor::RESIZE_FILTER, + image_to_tensor::RESIZE_MODE, + image_to_tensor::LETTERBOX_PAD_VALUE, + image_to_tensor::COLOR_FORMAT, + image_to_tensor::TENSOR_SHAPE_FORMAT, + image_to_tensor::MEAN, + image_to_tensor::STD_DEV, + image_to_tensor::PIXEL_DIVISOR, + TRACT_MODEL_SERVICE, + filter_bounding_boxes::CONFIDENCE_THRESHOLD, + filter_bounding_boxes::IOU_THRESHOLD, + filter_bounding_boxes::SCORE_OUTPUT_INDEX, + filter_bounding_boxes::BOX_OUTPUT_INDEX, + filter_bounding_boxes::CLASS_OUTPUT_INDEX, + filter_bounding_boxes::BOX_FORMAT, + filter_bounding_boxes::SCORE_ACTIVATION, + filter_bounding_boxes::BACKGROUND_CLASS_INDEX, + filter_bounding_boxes::OUTPUT_ATTRIBUTE_NAME, + ]; +} diff --git a/minifi_rust/extensions/minifi_tensor/src/processors/draw_bounding_box.rs b/minifi_rust/extensions/minifi_tensor/src/processors/draw_bounding_box.rs new file mode 100644 index 0000000000..073890c256 --- /dev/null +++ b/minifi_rust/extensions/minifi_tensor/src/processors/draw_bounding_box.rs @@ -0,0 +1,196 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use crate::utils::bounding_box::{BoundingBox, BoundingBoxes}; +use image::Rgb; +use minifi_native::macros::ComponentIdentifier; +use minifi_native::{ + FlowFileTransform, GetAttribute, GetControllerService, GetId, GetProperty, InputStream, Logger, + MinifiError, OutputAttribute, ProcessError, ProcessorDefinition, ProcessorInputRequirement, + Property, PropertyConstraints, PropertyType, Relationship, RouteErrorExt, Schedule, + TransformedFlowFile, +}; +use minifi_native::{PropertyDefinition, PropertySchema, property_definitions}; +use std::io::Cursor; + +pub(crate) const SUCCESS: Relationship = Relationship { + name: "success", + description: "Flowfiles are routed here after drawing the bounding boxes", +}; + +pub(crate) const FAILURE: Relationship = Relationship { + name: "failure", + description: "Invalid FlowFiles are routed here", +}; + +pub(crate) const BOUNDING_BOXES: Property = Property::new( + "Bounding boxes", + "JSON array of bounding boxes to draw onto the image (fields class_id, confidence, x_min, \ + y_min, x_max, y_max; coordinates normalised to [0,1] against the image). Typically the \ + attribute produced by an upstream DetectObject or FilterBoundingBoxes processor.", +) +.with_default("${enrichment.value}") +.supports_expression_language(); + +const LINE_THICKNESS: Property = Property::new( + "Line thickness", + "Thickness in pixels of the drawn box outline.", +) +.with_default("5"); + +const LINE_COLOR: Property = Property::new( + "Line color", + "Outline color as a hex string (e.g., '#ff00ff' or '#f0f')", +) +.with_default("#00FF00"); + +#[derive(Debug, ComponentIdentifier)] +pub(crate) struct DrawBoundingBox {} + +impl Schedule for DrawBoundingBox { + fn schedule( + _context: &Ctx, + _logger: &L, + ) -> Result + where + Self: Sized, + { + Ok(Self {}) + } +} + +struct LineColor {} + +impl PropertySchema for LineColor { + const CONSTRAINT: Option = None; + const IS_REQUIRED: bool = false; +} + +impl PropertyType for LineColor { + type Output = Rgb; + + fn parse(s: &str) -> Result { + let Some(hex) = s.trim().strip_prefix('#') else { + return Err(MinifiError::validation("Line color must start with #")); + }; + + let (r, g, b) = match hex.len() { + 6 => ( + u8::from_str_radix(&hex[0..2], 16).map_err(MinifiError::from)?, + u8::from_str_radix(&hex[2..4], 16).map_err(MinifiError::from)?, + u8::from_str_radix(&hex[4..6], 16).map_err(MinifiError::from)?, + ), + 3 => ( + u8::from_str_radix(&hex[0..1], 16).map_err(MinifiError::from)? * 17, + u8::from_str_radix(&hex[1..2], 16).map_err(MinifiError::from)? * 17, + u8::from_str_radix(&hex[2..3], 16).map_err(MinifiError::from)? * 17, + ), + _ => return Err(MinifiError::validation("expected 3 or 6 digit hex color")), + }; + Ok(Rgb::([r, g, b])) + } +} + +impl FlowFileTransform for DrawBoundingBox { + fn transform< + 'a, + Context: GetProperty + GetControllerService + GetAttribute + GetId, + LoggerImpl: Logger, + >( + &self, + context: &Context, + input_stream: &'a mut dyn InputStream, + _logger: &LoggerImpl, + ) -> Result, ProcessError> { + let line_thickness = context + .get_property(&LINE_THICKNESS) + .route_err_to_failure()?; + let line_color = context.get_property(&LINE_COLOR).route_err_to_failure()?; + let boxes: Vec = context + .get_property(&BOUNDING_BOXES) + .route_err_to_failure()?; + + let mut image_bytes = Vec::new(); + input_stream.read_to_end(&mut image_bytes)?; + + let format = image::guess_format(&image_bytes).route_err_to_failure()?; + + let mut img = image::load_from_memory_with_format(&image_bytes, format) + .map(|dyn_img| dyn_img.to_rgb8()) + .route_err_to_failure()?; + + boxes + .iter() + .for_each(|bbox| bbox.draw_onto(&mut img, line_thickness, line_color)); + + let mut output_bytes = Vec::new(); + img.write_to(&mut Cursor::new(&mut output_bytes), format) + .route_err_to_failure()?; + + Ok(TransformedFlowFile::new( + &SUCCESS, + Some(output_bytes.into()), + )) + } +} + +impl ProcessorDefinition for DrawBoundingBox { + const DESCRIPTION: &'static str = "Decodes the image from the flow file content, draws each bounding box supplied via the \ + 'Bounding boxes' property onto it, and re-encodes the annotated image as PNG. Pair with an \ + upstream DetectObject / FilterBoundingBoxes to visualise detections."; + const INPUT_REQUIREMENT: ProcessorInputRequirement = ProcessorInputRequirement::Required; + const SUPPORTS_DYNAMIC_PROPERTIES: bool = false; + const SUPPORTS_DYNAMIC_RELATIONSHIPS: bool = false; + const OUTPUT_ATTRIBUTES: &'static [OutputAttribute] = &[]; + const RELATIONSHIPS: &'static [Relationship] = &[SUCCESS, FAILURE]; + const PROPERTIES: &[PropertyDefinition] = + property_definitions![BOUNDING_BOXES, LINE_COLOR, LINE_THICKNESS]; +} + +#[cfg(test)] +mod tests { + use crate::processors::draw_bounding_box::{LINE_COLOR, LINE_THICKNESS, LineColor}; + use minifi_native::{GetProperty, MockControllerServiceContext, PropertyType}; + use std::assert_matches; + + #[test] + fn test_parsing_colors() { + let mock_context = MockControllerServiceContext::default(); + let default_color = mock_context + .get_property(&LINE_COLOR) + .expect("we should parse this"); + let green = image::Rgb([0, 255, 0]); + assert_eq!(default_color, green); + assert_matches!(LineColor::parse("[0,255,0]"), Err(_)); + assert_matches!(LineColor::parse("#00FG00"), Err(_)); + assert_matches!(LineColor::parse("#FFFFFFF"), Err(_)); + assert_matches!(LineColor::parse("FFFFFF"), Err(_)); + assert_matches!(LineColor::parse("#FFFF"), Err(_)); + assert_matches!(LineColor::parse("#0f0"), Ok(image::Rgb([0, 255, 0]))); + assert_matches!(LineColor::parse("#101010"), Ok(image::Rgb([16, 16, 16]))); + assert_matches!(LineColor::parse("#89A"), Ok(image::Rgb([0x88, 0x99, 0xAA]))); + } + + #[test] + fn test_parsing_line_thickness() { + let mock_context = MockControllerServiceContext::default(); + let default_thickness = mock_context + .get_property(&LINE_THICKNESS) + .expect("we should parse this"); + assert_eq!(default_thickness, 5); + } +} diff --git a/minifi_rust/extensions/minifi_tensor/src/processors/mod.rs b/minifi_rust/extensions/minifi_tensor/src/processors/mod.rs new file mode 100644 index 0000000000..83c58ebb03 --- /dev/null +++ b/minifi_rust/extensions/minifi_tensor/src/processors/mod.rs @@ -0,0 +1,20 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +pub(crate) mod classify_image; +pub(crate) mod detect_object; +pub(crate) mod draw_bounding_box; diff --git a/minifi_rust/extensions/minifi_tensor/src/services/mod.rs b/minifi_rust/extensions/minifi_tensor/src/services/mod.rs new file mode 100644 index 0000000000..44ed575366 --- /dev/null +++ b/minifi_rust/extensions/minifi_tensor/src/services/mod.rs @@ -0,0 +1,18 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +pub(crate) mod tract_model_service; diff --git a/minifi_rust/extensions/minifi_tensor/src/services/tract_model_service.rs b/minifi_rust/extensions/minifi_tensor/src/services/tract_model_service.rs new file mode 100644 index 0000000000..99c9d53cbd --- /dev/null +++ b/minifi_rust/extensions/minifi_tensor/src/services/tract_model_service.rs @@ -0,0 +1,178 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use crate::services::tract_model_service::service_definition::{MODEL_FILE_PATH, MODEL_FORMAT}; +use minifi_native::macros::{ComponentIdentifier, PropertyType}; +use minifi_native::{EnableControllerService, GetProperty, Logger, MinifiError, trace}; +use std::path::Path; +use strum_macros::{Display, EnumString, IntoStaticStr, VariantNames}; +use tract::prelude::*; + +mod service_definition; + +#[derive( + Debug, Clone, Copy, PartialEq, Display, EnumString, VariantNames, IntoStaticStr, PropertyType, +)] +#[strum(serialize_all = "PascalCase", const_into_str)] +pub(crate) enum ModelFormat { + Auto, + Onnx, + Nnef, +} + +/// Resolved format after any auto-detection. +#[derive(Debug, Clone, Copy, PartialEq)] +enum ResolvedFormat { + Onnx, + Nnef, +} + +impl ModelFormat { + fn resolve(self, path: &Path) -> Result { + match self { + ModelFormat::Onnx => Ok(ResolvedFormat::Onnx), + ModelFormat::Nnef => Ok(ResolvedFormat::Nnef), + ModelFormat::Auto => { + let path_str = path.to_string_lossy().to_ascii_lowercase(); + if path_str.ends_with(".onnx") { + Ok(ResolvedFormat::Onnx) + } else if path_str.ends_with(".nnef") + || path_str.ends_with(".nnef.tgz") + || path_str.ends_with(".nnef.tar") + || path_str.ends_with(".nnef.tar.gz") + || path.is_dir() + { + Ok(ResolvedFormat::Nnef) + } else { + Err(MinifiError::custom(format!( + "Could not auto-detect model format from '{:?}'. Set 'Model format' to \ + 'Onnx' or 'Nnef' explicitly.", + path + ))) + } + } + } + } +} + +#[derive(ComponentIdentifier)] +pub(crate) struct TractModelService { + runnable_model: Runnable, +} + +impl EnableControllerService for TractModelService { + fn enable(context: &Ctx, logger: &L) -> Result + where + Self: Sized, + { + let model_path = context.get_property(&MODEL_FILE_PATH)?; + let format = context.get_property(&MODEL_FORMAT)?; + let resolved = format.resolve(&model_path)?; + + trace!( + logger, + "Loading Tract model ({:?}) from: {:?}", resolved, model_path + ); + + let model = match resolved { + ResolvedFormat::Onnx => onnx()?.load(&model_path)?.into_model()?, + ResolvedFormat::Nnef => nnef()?.load(&model_path)?, + }; + + let runtime = runtime_for_name("default")?; + let runnable_model = runtime.prepare(model)?; + + trace!(logger, "Successfully loaded and compiled Tract model."); + + Ok(Self { runnable_model }) + } +} + +impl TractModelService { + pub fn run_inference( + &self, + inputs: impl IntoIterator, + ) -> Result, MinifiError> { + let vec_inputs: Vec = inputs.into_iter().collect(); + + Ok(self.runnable_model.run(vec_inputs)?) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use std::path::PathBuf; + use std::str::FromStr; + + #[test] + fn test_resolve_explicit_formats_bypass_detection() { + assert_eq!( + ModelFormat::Onnx + .resolve(&PathBuf::from_str("/tmp/no-extension").unwrap()) + .unwrap(), + ResolvedFormat::Onnx + ); + assert_eq!( + ModelFormat::Nnef + .resolve(&PathBuf::from_str("/tmp/no-extension").unwrap()) + .unwrap(), + ResolvedFormat::Nnef + ); + } + + #[test] + fn test_resolve_auto_by_onnx_extension() { + assert_eq!( + ModelFormat::Auto + .resolve(&PathBuf::from_str("/models/face.onnx").unwrap()) + .unwrap(), + ResolvedFormat::Onnx + ); + assert_eq!( + ModelFormat::Auto + .resolve(&PathBuf::from_str("/MODELS/FACE.ONNX").unwrap()) + .unwrap(), + ResolvedFormat::Onnx + ); + } + + #[test] + fn test_resolve_auto_by_nnef_extension() { + assert_eq!( + ModelFormat::Auto + .resolve(&PathBuf::from_str("/models/mobilenet.nnef.tgz").unwrap()) + .unwrap(), + ResolvedFormat::Nnef + ); + assert_eq!( + ModelFormat::Auto + .resolve(&PathBuf::from_str("/models/mobilenet.nnef").unwrap()) + .unwrap(), + ResolvedFormat::Nnef + ); + } + + #[test] + fn test_resolve_auto_errors_on_unknown() { + assert!( + ModelFormat::Auto + .resolve(&PathBuf::from_str("/tmp/no-hint").unwrap()) + .is_err() + ); + } +} diff --git a/minifi_rust/extensions/minifi_tensor/src/services/tract_model_service/service_definition.rs b/minifi_rust/extensions/minifi_tensor/src/services/tract_model_service/service_definition.rs new file mode 100644 index 0000000000..312a470d80 --- /dev/null +++ b/minifi_rust/extensions/minifi_tensor/src/services/tract_model_service/service_definition.rs @@ -0,0 +1,50 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use crate::services::tract_model_service::ModelFormat; +use minifi_native::{ + ControllerServiceDefinition, Property, PropertyDefinition, ProvidedInterface, + property_definitions, +}; +use std::path::PathBuf; + +pub(crate) const MODEL_FILE_PATH: Property = Property::new( + "Model File Path", + "Absolute path to the model on the edge device. For ONNX this is a `.onnx` \ + file; for NNEF this is a `.nnef.tgz` archive, a `.nnef` tarball, or the root \ + directory of an unpacked NNEF model. The model is loaded, parsed, and compiled \ + for the host CPU once when the controller service is enabled; subsequent \ + inference calls reuse the compiled runnable.", +); + +pub(crate) const MODEL_FORMAT: Property = Property::new( + "Model format", + "Format of the file/directory referenced by 'Model File Path'. 'Auto' picks \ + Onnx when the path ends in `.onnx` and Nnef when it ends in `.nnef`, \ + `.nnef.tgz`, `.nnef.tar`, `.nnef.tar.gz`, or points at a directory. Set \ + explicitly when the path uses a non-standard extension.", +) +.with_default(ModelFormat::Auto.into_str()); + +impl ControllerServiceDefinition for super::TractModelService { + const DESCRIPTION: &'static str = "Provides a shared, CPU-optimized neural network for inference. Supports ONNX (`.onnx`) \ + and NNEF (directory or tarball) models; the format can be auto-detected from the file \ + extension or set explicitly."; + const PROPERTIES: &'static [PropertyDefinition] = + property_definitions![MODEL_FILE_PATH, MODEL_FORMAT]; + const PROVIDED_APIS: &'static [ProvidedInterface] = &[]; +} diff --git a/minifi_rust/extensions/minifi_tensor/src/utils/bounding_box.rs b/minifi_rust/extensions/minifi_tensor/src/utils/bounding_box.rs new file mode 100644 index 0000000000..92dbdedf87 --- /dev/null +++ b/minifi_rust/extensions/minifi_tensor/src/utils/bounding_box.rs @@ -0,0 +1,204 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use image::{Rgb, RgbImage}; +use minifi_native::{MinifiError, PropertyConstraints, PropertySchema, PropertyType}; +use serde::{Deserialize, Serialize}; + +#[derive(Serialize, Deserialize, Clone, Debug)] +pub struct BoundingBox { + pub(crate) class_id: usize, + pub(crate) confidence: f32, + pub(crate) x_min: f32, + pub(crate) y_min: f32, + pub(crate) x_max: f32, + pub(crate) y_max: f32, +} + +fn draw_thick_rect( + img: &mut RgbImage, + left: u32, + top: u32, + right: u32, + bottom: u32, + thickness: u32, + color: Rgb, +) { + let box_width = right.saturating_sub(left); + let box_height = bottom.saturating_sub(top); + + for t in 0..thickness { + if box_width > 2 * t && box_height > 2 * t { + let rect = imageproc::rect::Rect::at((left + t) as i32, (top + t) as i32) + .of_size(box_width - 2 * t, box_height - 2 * t); + imageproc::drawing::draw_hollow_rect_mut(img, rect, color); + } + } +} + +impl BoundingBox { + pub fn class_id(&self) -> usize { + self.class_id + } + pub fn confidence(&self) -> f32 { + self.confidence + } + + pub(crate) fn calculate_intersection_over_union(box1: &BoundingBox, box2: &BoundingBox) -> f32 { + let x_left = box1.x_min.max(box2.x_min); + let y_top = box1.y_min.max(box2.y_min); + let x_right = box1.x_max.min(box2.x_max); + let y_bottom = box1.y_max.min(box2.y_max); + + if x_right < x_left || y_bottom < y_top { + return 0.0; + } + + let intersection_area = (x_right - x_left) * (y_bottom - y_top); + let box1_area = (box1.x_max - box1.x_min) * (box1.y_max - box1.y_min); + let box2_area = (box2.x_max - box2.x_min) * (box2.y_max - box2.y_min); + + let divisor = box1_area + box2_area - intersection_area; + + if intersection_area == 0f32 || divisor == 0f32 { + return 0f32; + } + + intersection_area / divisor + } + + pub(crate) fn apply_non_maximum_suppression( + mut boxes: Vec, + iou_threshold: f32, + ) -> Vec { + boxes.sort_by(|a, b| { + b.confidence() + .partial_cmp(&a.confidence()) + .unwrap_or(std::cmp::Ordering::Equal) + }); + + let mut keep = Vec::new(); + let mut is_suppressed = vec![false; boxes.len()]; + + for i in 0..boxes.len() { + if is_suppressed[i] { + continue; + } + + keep.push(boxes[i].clone()); + + for j in (i + 1)..boxes.len() { + if is_suppressed[j] { + continue; + } + + if boxes[i].class_id() == boxes[j].class_id() { + let iou = BoundingBox::calculate_intersection_over_union(&boxes[i], &boxes[j]); + if iou > iou_threshold { + is_suppressed[j] = true; + } + } + } + } + keep + } + + pub(crate) fn draw_onto(&self, img: &mut RgbImage, line_thickness: u32, line_color: Rgb) { + let width = img.width() as f32; + let height = img.height() as f32; + + let left = (self.x_min * width).round() as u32; + let top = (self.y_min * height).round() as u32; + let right = (self.x_max * width).round() as u32; + let bottom = (self.y_max * height).round() as u32; + + draw_thick_rect(img, left, top, right, bottom, line_thickness, line_color); + } +} + +pub(crate) struct BoundingBoxes {} + +impl PropertySchema for BoundingBoxes { + const CONSTRAINT: Option = None; + const IS_REQUIRED: bool = false; +} + +impl PropertyType for BoundingBoxes { + type Output = Vec; + + fn parse(s: &str) -> Result { + serde_json::from_str::>(s).map_err(MinifiError::other) + } +} + +#[cfg(test)] +mod tests { + use crate::utils::bounding_box::BoundingBox; + + #[test] + fn test_iou_zero_when_disjoint() { + let a = BoundingBox { + class_id: 0, + confidence: 1.0, + x_min: 0.0, + y_min: 0.0, + x_max: 1.0, + y_max: 1.0, + }; + let b = BoundingBox { + class_id: 0, + confidence: 1.0, + x_min: 2.0, + y_min: 2.0, + x_max: 3.0, + y_max: 3.0, + }; + assert_eq!(BoundingBox::calculate_intersection_over_union(&a, &b), 0.0); + } + + #[test] + fn test_nms_suppresses_overlapping_same_class() { + let boxes = vec![ + BoundingBox { + class_id: 1, + confidence: 0.9, + x_min: 0.0, + y_min: 0.0, + x_max: 1.0, + y_max: 1.0, + }, + BoundingBox { + class_id: 1, + confidence: 0.8, + x_min: 0.1, + y_min: 0.1, + x_max: 1.0, + y_max: 1.0, + }, + BoundingBox { + class_id: 2, + confidence: 0.7, + x_min: 0.1, + y_min: 0.1, + x_max: 1.0, + y_max: 1.0, + }, + ]; + let kept = BoundingBox::apply_non_maximum_suppression(boxes, 0.5); + assert_eq!(kept.len(), 2); // same-class dupe suppressed, other-class kept + } +} diff --git a/minifi_rust/extensions/minifi_tensor/src/utils/dimensions.rs b/minifi_rust/extensions/minifi_tensor/src/utils/dimensions.rs new file mode 100644 index 0000000000..98574b9137 --- /dev/null +++ b/minifi_rust/extensions/minifi_tensor/src/utils/dimensions.rs @@ -0,0 +1,146 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use minifi_native::{GetAttribute, MinifiError}; + +/// The exact placement of an aspect-preserving resize inside a target canvas. +/// +/// `ImageToTensor` applies this when resizing, and `FilterBoundingBoxes` inverts +/// it when un-mapping model coordinates back to the original image. Both must +/// agree down to the pixel, so the arithmetic lives here and nowhere else: +/// deriving the padding from the *unrounded* scaled size instead of `new_w`/ +/// `new_h` drifts by up to half a target pixel, which is several pixels once +/// divided back through `scale`. +#[derive(Debug, Clone, Copy, PartialEq)] +pub(crate) struct LetterboxGeometry { + pub(crate) scale: f32, + pub(crate) new_width: u32, + pub(crate) new_height: u32, + pub(crate) pad_x: u32, + pub(crate) pad_y: u32, +} + +#[derive(Debug, Clone, Copy, PartialEq)] +pub(crate) struct Dimensions { + pub(crate) width: f32, + pub(crate) height: f32, +} + +impl Dimensions { + /// Fit `self` into `target` preserving aspect ratio, centring the result. + /// + /// Assumes both dimensions are non-zero; `ImageToTensor::schedule` rejects a + /// zero 'Target width'/'Target height', and a decoded image always has at + /// least one pixel per axis. + pub(crate) fn letterbox_into(&self, target: Dimensions) -> LetterboxGeometry { + let scale = (target.width / self.width).min(target.height / self.height); + let new_width = (self.width * scale).round().max(1.0) as u32; + let new_height = (self.height * scale).round().max(1.0) as u32; + LetterboxGeometry { + scale, + new_width, + new_height, + // Saturating: `new_*` is clamped up to 1, so it can exceed a target + // axis of 0. Callers reject that config, but wrapping here would + // turn a misconfiguration into a panic or a garbage offset. + pad_x: (target.width as u32).saturating_sub(new_width) / 2, + pad_y: (target.height as u32).saturating_sub(new_height) / 2, + } + } + + pub(crate) fn from_image(img: &image::DynamicImage) -> Self { + Self { + width: img.width() as f32, + height: img.height() as f32, + } + } + + pub(crate) fn original_from_attributes( + context: &Context, + ) -> Result { + let orig_w = context + .get_required_attribute("image.original.width")? + .parse::()?; + + let orig_h = context + .get_required_attribute("image.original.height")? + .parse::()?; + + Ok(Dimensions { + width: orig_w, + height: orig_h, + }) + } + + pub(crate) fn target_from_attributes( + context: &Context, + ) -> Result { + let orig_w = context + .get_required_attribute("image.target.width")? + .parse::()?; + + let orig_h = context + .get_required_attribute("image.target.height")? + .parse::()?; + + Ok(Dimensions { + width: orig_w, + height: orig_h, + }) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn dim(width: f32, height: f32) -> Dimensions { + Dimensions { width, height } + } + + #[test] + fn letterbox_pads_from_the_rounded_size() { + // 1080p into SSD300: 1080 * 0.15625 = 168.75 rounds to 169, so the pad is + // (300 - 169) / 2 = 65 — not the 65.625 the unrounded size would give. + let geometry = dim(1920.0, 1080.0).letterbox_into(dim(300.0, 300.0)); + assert_eq!(geometry.scale, 0.15625); + assert_eq!(geometry.new_width, 300); + assert_eq!(geometry.new_height, 169); + assert_eq!(geometry.pad_x, 0); + assert_eq!(geometry.pad_y, 65); + } + + #[test] + fn letterbox_is_exact_when_the_scaled_size_is_integral() { + let geometry = dim(200.0, 100.0).letterbox_into(dim(100.0, 100.0)); + assert_eq!(geometry.scale, 0.5); + assert_eq!(geometry.new_width, 100); + assert_eq!(geometry.new_height, 50); + assert_eq!(geometry.pad_x, 0); + assert_eq!(geometry.pad_y, 25); + } + + #[test] + fn letterbox_keeps_a_degenerate_axis_at_one_pixel() { + // A very wide source against a small target rounds the short axis to 0; + // it is clamped to 1 so the resize stays valid. + let geometry = dim(1000.0, 3.0).letterbox_into(dim(10.0, 10.0)); + assert_eq!(geometry.new_width, 10); + assert_eq!(geometry.new_height, 1); + assert_eq!(geometry.pad_y, 4); + } +} diff --git a/minifi_rust/extensions/minifi_tensor/src/utils/mod.rs b/minifi_rust/extensions/minifi_tensor/src/utils/mod.rs new file mode 100644 index 0000000000..faf8b16f30 --- /dev/null +++ b/minifi_rust/extensions/minifi_tensor/src/utils/mod.rs @@ -0,0 +1,22 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +pub(crate) mod bounding_box; +pub(crate) mod dimensions; +pub(crate) mod per_channel_f32; +pub(crate) mod score_activation; +pub(crate) mod tensor_helpers; diff --git a/minifi_rust/extensions/minifi_tensor/src/utils/per_channel_f32.rs b/minifi_rust/extensions/minifi_tensor/src/utils/per_channel_f32.rs new file mode 100644 index 0000000000..39f198ad4f --- /dev/null +++ b/minifi_rust/extensions/minifi_tensor/src/utils/per_channel_f32.rs @@ -0,0 +1,86 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use crate::utils::per_channel_f32::PerChannelF32::TriChannel; +use minifi_native::{MinifiError, PropertyConstraints, PropertySchema, PropertyType}; + +#[derive(PartialEq, Debug)] +pub(crate) enum PerChannelF32 { + SingleChannel(f32), + TriChannel([f32; 3]), +} + +impl PerChannelF32 { + pub fn contains_zero(&self) -> bool { + match self { + Self::SingleChannel(f) => *f == 0.0f32, + TriChannel(channels) => channels.contains(&0.0f32), + } + } + + pub fn per_channel(&self, channel_id: usize) -> f32 { + match self { + Self::SingleChannel(f) => *f, + TriChannel(channels) => channels[channel_id], + } + } +} + +impl PropertySchema for PerChannelF32 { + const CONSTRAINT: Option = None; + const IS_REQUIRED: bool = false; +} + +impl PropertyType for PerChannelF32 { + type Output = PerChannelF32; + + fn parse(input: &str) -> Result { + let parts: Vec<&str> = input.split(',').map(|s| s.trim()).collect(); + match parts.len() { + 1 => Ok(Self::SingleChannel(parts[0].parse::()?)), + 3 => { + let f = parts[0].parse::()?; + let s = parts[1].parse::()?; + let t = parts[2].parse::()?; + Ok(TriChannel([f, s, t])) + } + _n => Err(MinifiError::validation( + "expected 1 or 3 comma-separated floats", + )), + } + } +} + +#[cfg(test)] +mod tests { + use crate::utils::per_channel_f32::PerChannelF32; + use minifi_native::PropertyType; + + #[test] + fn test_parse_per_channel_f32_accepts_scalar_and_triple() { + assert_eq!( + PerChannelF32::parse("0.5").unwrap(), + PerChannelF32::SingleChannel(0.5) + ); + assert_eq!( + PerChannelF32::parse("0.485, 0.456, 0.406").unwrap(), + PerChannelF32::TriChannel([0.485, 0.456, 0.406]) + ); + assert!(PerChannelF32::parse("1, 2").is_err()); + assert!(PerChannelF32::parse("1, 2, three").is_err()); + } +} diff --git a/minifi_rust/extensions/minifi_tensor/src/utils/score_activation.rs b/minifi_rust/extensions/minifi_tensor/src/utils/score_activation.rs new file mode 100644 index 0000000000..a710d86646 --- /dev/null +++ b/minifi_rust/extensions/minifi_tensor/src/utils/score_activation.rs @@ -0,0 +1,91 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use minifi_native::macros::PropertyType; +use strum_macros::{Display, EnumString, IntoStaticStr, VariantNames}; + +#[derive( + Debug, Clone, Copy, PartialEq, Display, EnumString, VariantNames, IntoStaticStr, PropertyType, +)] +#[strum(serialize_all = "PascalCase", const_into_str)] +pub(crate) enum ScoreActivation { + /// Cross-class softmax; classes are mutually exclusive (typical for + /// ImageNet-trained ResNet/MobileNet/EfficientNet ONNX exports). + Softmax, + /// Per-class sigmoid; classes are independent (multi-label classifiers). + Sigmoid, + /// Pass-through — the model already emits probabilities. + None, +} + +/// The `(max_logit, sum_exp)` denominator of a numerically-stable softmax. +/// +/// Subtracting the max before exponentiating keeps `exp` in range for large +/// logits. Non-finite logits are skipped so one NaN cannot poison the whole +/// distribution. +#[derive(Debug, Clone, Copy, PartialEq)] +pub(crate) struct SoftmaxTerms { + max_logit: f32, + sum_exp: f32, +} + +impl SoftmaxTerms { + pub(crate) fn over(logits: impl IntoIterator + Clone) -> Self { + let max_logit = logits + .clone() + .into_iter() + .filter(|l| l.is_finite()) + .reduce(f32::max) + .unwrap_or(f32::NEG_INFINITY); + let sum_exp = logits + .into_iter() + .filter(|l| l.is_finite()) + .map(|l| (l - max_logit).exp()) + .sum(); + Self { max_logit, sum_exp } + } +} + +impl ScoreActivation { + /// Confidence for one logit drawn from a full score vector. + /// + /// `terms` must be computed over that same vector, so `Softmax` normalises + /// against the distribution the logit came from. + pub(crate) fn confidence(self, logit: f32, terms: SoftmaxTerms) -> f32 { + match self { + ScoreActivation::Softmax => (logit - terms.max_logit).exp() / terms.sum_exp, + ScoreActivation::Sigmoid => sigmoid(logit), + ScoreActivation::None => logit, + } + } + + /// Confidence for a standalone score, with no surrounding vector to + /// normalise against — the "separate class-id tensor" detector layout. + /// + /// Sigmoid maps a raw logit to a probability; softmax has no meaning over a + /// single scalar, so it passes through, as does None. + pub(crate) fn confidence_of_scalar(self, score: f32) -> f32 { + match self { + ScoreActivation::Sigmoid => sigmoid(score), + ScoreActivation::Softmax | ScoreActivation::None => score, + } + } +} + +fn sigmoid(x: f32) -> f32 { + 1.0 / (1.0 + (-x).exp()) +} diff --git a/minifi_rust/extensions/minifi_tensor/src/utils/tensor_helpers.rs b/minifi_rust/extensions/minifi_tensor/src/utils/tensor_helpers.rs new file mode 100644 index 0000000000..e7c60eca1e --- /dev/null +++ b/minifi_rust/extensions/minifi_tensor/src/utils/tensor_helpers.rs @@ -0,0 +1,259 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use image::{DynamicImage, ImageResult}; +use minifi_native::{GetAttribute, InputStream, MinifiError}; +use strum_macros::{Display, EnumString}; +use tract::__ndarray_interop::TensorInterface; +use tract::Tensor; +use tract::prelude::DatumType; + +tract::impl_ndarray_interop!(); + +fn parse_tensor_shape( + context: &Context, + id: usize, +) -> Result, MinifiError> { + let shape_str = context.get_required_attribute(&format!("tensor.{}.shape", id))?; + + if shape_str.trim().is_empty() { + return Ok(Vec::new()); + } + + let shape = shape_str + .split(',') + .map(|s| s.trim().parse::()) + .collect::, _>>()?; + + Ok(shape) +} + +#[derive(Debug, Clone, Copy, PartialEq, Display, EnumString)] +#[strum(serialize_all = "PascalCase", const_into_str)] +pub(crate) enum MinifiDatumType { + F32, +} + +impl From for DatumType { + fn from(value: MinifiDatumType) -> Self { + match value { + MinifiDatumType::F32 => DatumType::F32, + } + } +} + +fn numeric_datum_type_from_str(s: &str) -> Option { + Some(match s { + "U8" => DatumType::U8, + "U16" => DatumType::U16, + "U32" => DatumType::U32, + "U64" => DatumType::U64, + "I8" => DatumType::I8, + "I16" => DatumType::I16, + "I32" => DatumType::I32, + "I64" => DatumType::I64, + "F16" => DatumType::F16, + "F32" => DatumType::F32, + "F64" => DatumType::F64, + _ => return None, + }) +} + +fn parse_tensor_dtype( + context: &Context, + id: usize, +) -> Result { + let dtype_str = context.get_required_attribute(&format!("tensor.{}.dtype", id))?; + numeric_datum_type_from_str(&dtype_str).ok_or_else(|| { + MinifiError::custom(format!( + "Unsupported tensor.{}.dtype '{}': only numeric tensors can be read", + id, dtype_str + )) + }) +} + +pub(crate) fn deserialize_tensors( + context: &Context, + input_stream: &mut dyn InputStream, +) -> Result, MinifiError> { + let mut result = vec![]; + + let mut flow_file_contents = Vec::new(); + input_stream.read_to_end(&mut flow_file_contents)?; + let number_of_tensors = context + .get_required_attribute("tensors.len")? + .parse::()?; + + let mut cursor = 0usize; + for i in 0..number_of_tensors { + let tensor_len = context + .get_required_attribute(&format!("tensor.{}.bytes", i))? + .parse::()?; + let tensor_shape = parse_tensor_shape(context, i)?; + let tensor_dtype = parse_tensor_dtype(context, i)?; + let tensor_end = cursor + .checked_add(tensor_len) + .filter(|end| *end <= flow_file_contents.len()) + .ok_or_else(|| { + MinifiError::custom("FlowFile contents are not in sync with tensor attributes") + })?; + let tensor_data = &flow_file_contents[cursor..tensor_end]; + result.push(Tensor::from_bytes( + tensor_dtype, + &tensor_shape, + tensor_data, + )?); + cursor = tensor_end; + } + + if cursor != flow_file_contents.len() { + Err(MinifiError::custom( + "FlowFile contents are not in sync with tensor attributes", + )) + } else { + Ok(result) + } +} + +pub(crate) fn tensor_as_f32(tensors: &[Tensor], index: usize) -> Result, MinifiError> { + let tensor = tensors + .get(index) + .ok_or(MinifiError::custom("Invalid shape of tensors"))?; + let casted = tensor.convert_to(DatumType::F32)?; + Ok(casted.as_slice::()?.to_vec()) +} + +pub(crate) fn tensor_shape(tensors: &[Tensor], index: usize) -> Result, MinifiError> { + let tensor = tensors + .get(index) + .ok_or(MinifiError::custom("Invalid shape of tensors"))?; + Ok(tensor.shape()?.to_vec()) +} + +pub(crate) fn load_as_image(input_stream: &mut dyn InputStream) -> ImageResult { + let mut raw_bytes = Vec::new(); + input_stream.read_to_end(&mut raw_bytes)?; + + image::load_from_memory(&raw_bytes) +} + +#[cfg(test)] +mod tests { + use super::*; + use minifi_native::MockProcessContext; + use std::assert_matches; + use std::io::Cursor; + + #[test] + fn test_tensor_as_f32_reads_f32_tensor() { + let t = Tensor::from_slice::(&[3], &[0.5, -1.5, 2.0]).unwrap(); + assert_eq!(tensor_as_f32(&[t], 0).unwrap(), vec![0.5, -1.5, 2.0]); + } + + #[test] + fn test_tensor_as_f32_casts_integer_tensor() { + // i64 class-index tensor (e.g. TF-OD / EfficientNMS 'detection_classes') + // must be cast to f32, not byte-reinterpreted. + let t = Tensor::from_slice::(&[4], &[0, 1, 17, 80]).unwrap(); + assert_eq!(tensor_as_f32(&[t], 0).unwrap(), vec![0.0, 1.0, 17.0, 80.0]); + } + + #[test] + fn test_tensor_as_f32_index_out_of_range_errors() { + assert!(tensor_as_f32(&[], 0).is_err()); + } + + #[test] + fn test_numeric_datum_type_accepts_numeric_rejects_other() { + assert_eq!(numeric_datum_type_from_str("I64"), Some(DatumType::I64)); + assert_eq!(numeric_datum_type_from_str("F32"), Some(DatumType::F32)); + assert_eq!(numeric_datum_type_from_str("U8"), Some(DatumType::U8)); + assert_eq!(numeric_datum_type_from_str("String"), None); + assert_eq!(numeric_datum_type_from_str("Bool"), None); + } + + #[test] + fn misaligned_attrs_and_content_deserialize_tensors() { + let mut context = MockProcessContext::new(); + let floats: Vec = vec![0.0f32; 6]; + + let bytes: Vec = floats.into_iter().flat_map(|f| f.to_le_bytes()).collect(); + let input_stream = Cursor::new(bytes); + assert_matches!( + deserialize_tensors(&context, &mut input_stream.clone()), + Err(MinifiError::MissingRequiredAttribute(msg)) if msg == "tensors.len" + ); + context.attributes.insert("tensors.len".into(), "1".into()); + assert_matches!( + deserialize_tensors(&context, &mut input_stream.clone()), + Err(MinifiError::MissingRequiredAttribute(msg)) if msg == "tensor.0.bytes" + ); + context + .attributes + .insert("tensor.0.bytes".into(), "24".into()); + assert_matches!( + deserialize_tensors(&context, &mut input_stream.clone()), + Err(MinifiError::MissingRequiredAttribute(msg)) if msg == "tensor.0.shape" + ); + context + .attributes + .insert("tensor.0.shape".into(), "2,3".into()); + assert_matches!( + deserialize_tensors(&context, &mut input_stream.clone()), + Err(MinifiError::MissingRequiredAttribute(msg)) if msg == "tensor.0.dtype" + ); + context + .attributes + .insert("tensor.0.dtype".into(), "F32".into()); + + assert_matches!( + deserialize_tensors(&context, &mut input_stream.clone()), + Ok(_) + ); + } + + #[test] + fn oversized_tensor_len_attribute_errors_instead_of_panicking() { + // A `tensor.N.bytes` large enough to overflow `cursor + tensor_len` must + // still be rejected: the wrapped sum used to slip past the bounds check + // and panic on the slice (aborting the process, since panic = "abort"). + let mut context = MockProcessContext::new(); + let bytes: Vec = vec![0u8; 8]; + let input_stream = Cursor::new(bytes); + + // Two tensors so the second one is checked against a non-zero cursor — + // `0 + tensor_len` cannot overflow. + context.attributes.insert("tensors.len".into(), "2".into()); + for (i, len) in [("0", "8"), ("1", &usize::MAX.to_string()[..])] { + context + .attributes + .insert(format!("tensor.{i}.bytes"), len.into()); + context + .attributes + .insert(format!("tensor.{i}.shape"), "2".into()); + context + .attributes + .insert(format!("tensor.{i}.dtype"), "F32".into()); + } + + assert_matches!( + deserialize_tensors(&context, &mut input_stream.clone()), + Err(MinifiError::CustomError(msg)) + if msg == "FlowFile contents are not in sync with tensor attributes" + ); + } +} diff --git a/minifi_rust/minifi_native/src/api/errors.rs b/minifi_rust/minifi_native/src/api/errors.rs index 16c652a7c5..d7f80bc132 100644 --- a/minifi_rust/minifi_native/src/api/errors.rs +++ b/minifi_rust/minifi_native/src/api/errors.rs @@ -61,6 +61,16 @@ pub enum ProcessError { Fatal(MinifiError), } +impl ProcessError { + pub fn route_to_failure>>(reason: S) -> Self { + ProcessError::Route(RouteError { + relationship: "failure", + source: Box::new(MinifiError::custom(reason)), + log_level: LogLevel::Warn, + }) + } +} + impl From for ProcessError { fn from(err: RouteError) -> Self { ProcessError::Route(err) diff --git a/minifi_rust/minifi_native/src/api/processor_wrappers/flow_file_transform.rs b/minifi_rust/minifi_native/src/api/processor_wrappers/flow_file_transform.rs index 2f470125ee..ba88a7c3aa 100644 --- a/minifi_rust/minifi_native/src/api/processor_wrappers/flow_file_transform.rs +++ b/minifi_rust/minifi_native/src/api/processor_wrappers/flow_file_transform.rs @@ -24,10 +24,9 @@ use crate::api::raw_processor::{MultiThreadedTrigger, SingleThreadedTrigger}; use crate::{ GetAttribute, LogLevel, Logger, MinifiError, MultiThreaded, OnTriggerResult, ProcessContext, ProcessError, ProcessSession, Relationship, Schedule, SingleThreaded, impl_with_attributes, - info, }; -use minifi_native::InputStream; +use minifi_native::{InputStream, trace}; use std::borrow::Cow; pub type FlowFileAttribute = (Cow<'static, str>, Cow<'static, str>); @@ -159,7 +158,7 @@ where } }; - info!(logger, "{:?}", transformed); + trace!(logger, "{:?}", transformed); match transformed.new_content { None => {} Some(Content::Buffer(buffer)) => { diff --git a/minifi_rust/minifi_native/src/lib.rs b/minifi_rust/minifi_native/src/lib.rs index b855291a29..f8509301d6 100644 --- a/minifi_rust/minifi_native/src/lib.rs +++ b/minifi_rust/minifi_native/src/lib.rs @@ -83,10 +83,10 @@ macro_rules! declare_minifi_extension { ( // Group name group_name: $group:expr, - // Match a tuple of three types for each processor - processors: [ $( ($kind:ty, $thread:ty, $impl:ty) ),* $(,)? ], - // Match a single type for each controller service - controllers: [ $( $ctrl:ty ),* $(,)? ] + // Match a tuple of three types for each processor, WITH optional attributes + processors: [ $( $(#[$proc_meta:meta])* ($kind:ty, $thread:ty, $impl:ty) ),* $(,)? ], + // Match a single type for each controller service, WITH optional attributes + controllers: [ $( $(#[$ctrl_meta:meta])* $ctrl:ty ),* $(,)? ] ) => { #[unsafe(no_mangle)] @@ -108,8 +108,9 @@ macro_rules! declare_minifi_extension { let extension = minifi_native::sys::minifi_register_extension(extension_context, &extension_definition); - $( + // Re-apply the captured attributes (e.g., #[cfg(...)]) to this block + $(#[$proc_meta])* { let processor_def = minifi_native::Processor::< $impl, @@ -125,6 +126,8 @@ macro_rules! declare_minifi_extension { )* $( + // Re-apply the captured attributes to this block + $(#[$ctrl_meta])* { let controller_def = minifi_native::ControllerService::< diff --git a/minifi_rust/minifi_rs_behave/Cargo.toml b/minifi_rust/minifi_rs_behave/Cargo.toml index c13e8521e5..7656efd370 100644 --- a/minifi_rust/minifi_rs_behave/Cargo.toml +++ b/minifi_rust/minifi_rs_behave/Cargo.toml @@ -5,7 +5,6 @@ edition = "2024" [dependencies] glob = "0.3.3" -minifi_rs_playground = { path = "../extensions/minifi_rs_playground" } minifi_native_sys = { path = "../minifi_native_sys" } [lints.rust] diff --git a/minifi_rust/minifi_rs_behave/Dockerfile.alpine b/minifi_rust/minifi_rs_behave/Dockerfile.alpine index 89f2c01987..19b69f53d6 100644 --- a/minifi_rust/minifi_rs_behave/Dockerfile.alpine +++ b/minifi_rust/minifi_rs_behave/Dockerfile.alpine @@ -18,7 +18,7 @@ COPY target/.docker_sdk.zi[p] /app/target/ RUN cargo chef cook --release --recipe-path recipe.json COPY . . -RUN cargo build --release +RUN cargo build --release --all-features # Export Stage FROM scratch AS bin-export diff --git a/minifi_rust/minifi_rs_behave/Dockerfile.debian b/minifi_rust/minifi_rs_behave/Dockerfile.debian index e3283496b1..48507ef2e2 100644 --- a/minifi_rust/minifi_rs_behave/Dockerfile.debian +++ b/minifi_rust/minifi_rs_behave/Dockerfile.debian @@ -1,4 +1,4 @@ -FROM rust:slim-bullseye AS chef +FROM rust:slim-bookworm AS chef RUN apt-get update && apt-get install -y clang lld pkg-config curl tar && cargo install cargo-chef WORKDIR /app @@ -18,7 +18,7 @@ COPY target/.docker_sdk.zi[p] /app/target/ RUN cargo chef cook --release --recipe-path recipe.json COPY . . -RUN cargo build --release +RUN cargo build --release --all-features # Export Stage FROM scratch AS bin-export diff --git a/minifi_rust/minifi_rs_behave/linux_build.sh b/minifi_rust/minifi_rs_behave/linux_build.sh index 44c9c924a8..b751802270 100755 --- a/minifi_rust/minifi_rs_behave/linux_build.sh +++ b/minifi_rust/minifi_rs_behave/linux_build.sh @@ -81,10 +81,19 @@ TARGET_DIR="target/release" mkdir -p "$TARGET_DIR" # 3. Build using Docker +# When running under GitHub Actions, persist the chef layer across runs +CACHE_ARGS=() +if [ -n "${ACTIONS_CACHE_URL}${ACTIONS_RESULTS_URL}" ]; then + echo "GitHub Actions cache detected — enabling type=gha buildx cache" + CACHE_ARGS+=(--cache-from "type=gha,scope=behave-$FLAVOR") + CACHE_ARGS+=(--cache-to "type=gha,mode=max,scope=behave-$FLAVOR") +fi + docker buildx build \ -f "$DOCKERFILE" \ --target bin-export \ --build-arg MINIFI_SDK_PATH="$DOCKER_SDK_ARG" \ + "${CACHE_ARGS[@]}" \ --output type=local,dest="$TARGET_DIR" \ .