From e106b914de17049994ae589a10f92061002dca0e Mon Sep 17 00:00:00 2001 From: Neal Butler Date: Wed, 14 Oct 2026 14:18:48 -0700 Subject: [PATCH 1/3] Added multipart upload support, updated schema --- FLIR/conservator/file_transfers.py | 37 ++++++++++++++- FLIR/conservator/generated/schema.py | 71 +++++++++++++++++++++++++++- FLIR/conservator/wrappers/media.py | 23 +++++++-- api_version.txt | 2 +- 4 files changed, 124 insertions(+), 9 deletions(-) diff --git a/FLIR/conservator/file_transfers.py b/FLIR/conservator/file_transfers.py index cfc8704f..e4fc7639 100644 --- a/FLIR/conservator/file_transfers.py +++ b/FLIR/conservator/file_transfers.py @@ -38,6 +38,33 @@ def __init__(self, url, local_path): self.local_path = local_path +class _FileSlice: + """ + A read-only, file-like view of `length` bytes of an open file, starting + at `offset`. Lets ``requests`` stream part of a file without loading it + into memory. Defines ``__len__`` so ``requests`` sends the slice's length + as the Content-Length (rather than the length of the underlying file). + """ + + def __init__(self, f, offset, length): + self._f = f + self._remaining = length + self._length = length + self._f.seek(offset) + + def __len__(self): + return self._length + + def read(self, size=-1): + if self._remaining <= 0: + return b"" + if size is None or size < 0 or size > self._remaining: + size = self._remaining + data = self._f.read(size) + self._remaining -= len(data) + return data + + class FileTransferException(Exception): """ Something went wrong when uploading or downloading a file. @@ -176,18 +203,24 @@ def _do_upload_request(self, upload_request): logger.warning("Encountered FileUploadException with %s", upload_request) return False - def upload(self, url, local_path, max_retries=5): + def upload(self, url, local_path, max_retries=5, offset=0, length=None): """ Upload the file at `local_path` to Conservator `url`. + + :param offset: Byte offset in the file to start uploading from. + :param length: Number of bytes to upload. If `None`, uploads from + `offset` to the end of the file. """ url = self.full_url(url) path = os.path.abspath(local_path) logger.info("Uploading '%s'", path) + if length is None: + length = os.path.getsize(path) - offset response = None retries = 0 while retries < max_retries: with open(path, "rb") as f: - response = requests.put(url, f) + response = requests.put(url, _FileSlice(f, offset, length)) if response.ok: break logger.warning("Got status code %s", response.status_code) diff --git a/FLIR/conservator/generated/schema.py b/FLIR/conservator/generated/schema.py index 14d45f67..52ca25cf 100644 --- a/FLIR/conservator/generated/schema.py +++ b/FLIR/conservator/generated/schema.py @@ -1,7 +1,6 @@ import sgqlc.types import FLIR.conservator.generated.date - schema = sgqlc.types.Schema() @@ -125,6 +124,7 @@ class AcceptAnnotation(sgqlc.types.Input): "point", "source", "target_id", + "identity_id", "attributes", "label_id", ) @@ -145,6 +145,7 @@ class AcceptAnnotation(sgqlc.types.Input): sgqlc.types.non_null("InputSource"), graphql_name="source" ) target_id = sgqlc.types.Field(Int, graphql_name="targetId") + identity_id = sgqlc.types.Field(Int, graphql_name="identityId") attributes = sgqlc.types.Field( sgqlc.types.list_of(sgqlc.types.non_null("AddAttributeInput")), graphql_name="attributes", @@ -163,6 +164,7 @@ class AcceptDatasetAnnotationInput(sgqlc.types.Input): "bounding_polygon", "point", "target_id", + "identity_id", "attributes", ) original_id = sgqlc.types.Field( @@ -185,6 +187,7 @@ class AcceptDatasetAnnotationInput(sgqlc.types.Input): ) point = sgqlc.types.Field("InputPoint", graphql_name="point") target_id = sgqlc.types.Field(Int, graphql_name="targetId") + identity_id = sgqlc.types.Field(Int, graphql_name="identityId") attributes = sgqlc.types.Field( sgqlc.types.list_of(sgqlc.types.non_null("AddAttributeInput")), graphql_name="attributes", @@ -333,6 +336,7 @@ class AnnotationCreate(sgqlc.types.Input): "bounding_box", "bounding_polygon", "target_id", + "identity_id", "custom", "source", "attributes", @@ -353,6 +357,7 @@ class AnnotationCreate(sgqlc.types.Input): graphql_name="boundingPolygon", ) target_id = sgqlc.types.Field(Int, graphql_name="targetId") + identity_id = sgqlc.types.Field(Int, graphql_name="identityId") custom = sgqlc.types.Field(String, graphql_name="custom") source = sgqlc.types.Field("AnnotationSourceInput", graphql_name="source") attributes = sgqlc.types.Field( @@ -420,6 +425,7 @@ class CreateDatasetAnnotationInput(sgqlc.types.Input): "bounding_polygon", "point", "target_id", + "identity_id", "source", "attributes", "custom", @@ -443,6 +449,7 @@ class CreateDatasetAnnotationInput(sgqlc.types.Input): ) point = sgqlc.types.Field("InputPoint", graphql_name="point") target_id = sgqlc.types.Field(Int, graphql_name="targetId") + identity_id = sgqlc.types.Field(Int, graphql_name="identityId") source = sgqlc.types.Field(AnnotationSourceInput, graphql_name="source") attributes = sgqlc.types.Field( sgqlc.types.list_of(AddAttributeInput), graphql_name="attributes" @@ -980,7 +987,14 @@ class MoveAssetInput(sgqlc.types.Input): class PredictionCreate(sgqlc.types.Input): __schema__ = schema - __field_names__ = ("labels", "bounding_box", "classifier_id", "target_id", "custom") + __field_names__ = ( + "labels", + "bounding_box", + "classifier_id", + "target_id", + "identity_id", + "custom", + ) labels = sgqlc.types.Field( sgqlc.types.non_null( sgqlc.types.list_of(sgqlc.types.non_null(AllowedLabelCharacters)) @@ -994,6 +1008,7 @@ class PredictionCreate(sgqlc.types.Input): sgqlc.types.non_null(String), graphql_name="classifierId" ) target_id = sgqlc.types.Field(Int, graphql_name="targetId") + identity_id = sgqlc.types.Field(Int, graphql_name="identityId") custom = sgqlc.types.Field(String, graphql_name="custom") @@ -1105,6 +1120,7 @@ class UpdateAnnotationInput(sgqlc.types.Input): "labels", "label_id", "target_id", + "identity_id", "qa_status", "qa_status_note", "custom_metadata", @@ -1115,6 +1131,7 @@ class UpdateAnnotationInput(sgqlc.types.Input): ) label_id = sgqlc.types.Field(GraphqlID, graphql_name="labelId") target_id = sgqlc.types.Field(Int, graphql_name="targetId") + identity_id = sgqlc.types.Field(Int, graphql_name="identityId") qa_status = sgqlc.types.Field(String, graphql_name="qaStatus") qa_status_note = sgqlc.types.Field(String, graphql_name="qaStatusNote") custom_metadata = sgqlc.types.Field(String, graphql_name="customMetadata") @@ -1283,6 +1300,7 @@ class Annotation(sgqlc.types.Type): __field_names__ = ( "id", "target_id", + "identity_id", "label", "labels", "label_id", @@ -1298,6 +1316,7 @@ class Annotation(sgqlc.types.Type): ) id = sgqlc.types.Field(sgqlc.types.non_null(GraphqlID), graphql_name="id") target_id = sgqlc.types.Field(Int, graphql_name="targetId") + identity_id = sgqlc.types.Field(Int, graphql_name="identityId") label = sgqlc.types.Field( sgqlc.types.non_null(AllowedLabelCharacters), graphql_name="label" ) @@ -1949,6 +1968,7 @@ class DatasetAnnotation(sgqlc.types.Type): __field_names__ = ( "id", "target_id", + "identity_id", "label", "labels", "label_id", @@ -1964,6 +1984,7 @@ class DatasetAnnotation(sgqlc.types.Type): ) id = sgqlc.types.Field(sgqlc.types.non_null(GraphqlID), graphql_name="id") target_id = sgqlc.types.Field(Int, graphql_name="targetId") + identity_id = sgqlc.types.Field(Int, graphql_name="identityId") label = sgqlc.types.Field( sgqlc.types.non_null(AllowedLabelCharacters), graphql_name="label" ) @@ -3044,6 +3065,8 @@ class Mutation(sgqlc.types.Type): "request_changes_annotation", "unset_qa_status_annotation", "update_qa_status_note_annotation", + "set_identity_id", + "initialize_identity_ids", "sign_in", "sign_out", "forgot_password", @@ -3527,6 +3550,50 @@ class Mutation(sgqlc.types.Type): ) ), ) + set_identity_id = sgqlc.types.Field( + sgqlc.types.non_null(Annotation), + graphql_name="setIdentityId", + args=sgqlc.types.ArgDict( + ( + ( + "frame_id", + sgqlc.types.Arg( + sgqlc.types.non_null(GraphqlID), + graphql_name="frameId", + default=None, + ), + ), + ( + "annotation_id", + sgqlc.types.Arg( + sgqlc.types.non_null(GraphqlID), + graphql_name="annotationId", + default=None, + ), + ), + ( + "identity_id", + sgqlc.types.Arg(Int, graphql_name="identityId", default=None), + ), + ) + ), + ) + initialize_identity_ids = sgqlc.types.Field( + sgqlc.types.non_null(Boolean), + graphql_name="initializeIdentityIds", + args=sgqlc.types.ArgDict( + ( + ( + "video_id", + sgqlc.types.Arg( + sgqlc.types.non_null(GraphqlID), + graphql_name="videoId", + default=None, + ), + ), + ) + ), + ) sign_in = sgqlc.types.Field( sgqlc.types.non_null(AuthPayload), graphql_name="signIn", diff --git a/FLIR/conservator/wrappers/media.py b/FLIR/conservator/wrappers/media.py index deaafd96..344d46a6 100644 --- a/FLIR/conservator/wrappers/media.py +++ b/FLIR/conservator/wrappers/media.py @@ -17,6 +17,9 @@ from FLIR.conservator.wrappers.type_proxy import requires_fields +# Files larger than this are split into multiple parts when uploading. +MAX_UPLOAD_PART_SIZE = 1_000_000_000 + class MediaUploadException(Exception): """Raised if an exception occurs during a media upload""" @@ -226,12 +229,24 @@ def upload(conservator, upload_request): ) upload_id = media._initiate_upload(original_name) - url = media._generate_signed_upload_url(upload_id) - response = conservator.files.upload(url=url, local_path=file_path) - completion_tag = response.headers["ETag"] + # Split large files into a multipart upload, using the fewest + # parts that keep each part under MAX_UPLOAD_PART_SIZE. + file_size = os.path.getsize(file_path) + num_parts = max(1, math.ceil(file_size / MAX_UPLOAD_PART_SIZE)) + part_size = math.ceil(file_size / num_parts) + + completion_tags = [] + for part_number in range(1, num_parts + 1): + offset = (part_number - 1) * part_size + length = min(part_size, file_size - offset) + url = media._generate_signed_upload_url(upload_id, part_number) + response = conservator.files.upload( + url=url, local_path=file_path, offset=offset, length=length + ) + completion_tags.append(response.headers["ETag"]) media._complete_upload( - original_name, upload_id, completion_tags=[completion_tag] + original_name, upload_id, completion_tags=completion_tags ) media._trigger_processing() diff --git a/api_version.txt b/api_version.txt index 30b911c7..431c4949 100644 --- a/api_version.txt +++ b/api_version.txt @@ -1 +1 @@ -4ccfd9e0b91e795f71318fe618ec7832 +1837770cd6180477fe416e7bb34435ac From 0e9f072cd4e396ff33933354824b8dc293b617b4 Mon Sep 17 00:00:00 2001 From: Neal Butler Date: Wed, 14 Oct 2026 16:56:05 -0700 Subject: [PATCH 2/3] Added missing import --- FLIR/conservator/wrappers/media.py | 1 + 1 file changed, 1 insertion(+) diff --git a/FLIR/conservator/wrappers/media.py b/FLIR/conservator/wrappers/media.py index 344d46a6..5fd7201e 100644 --- a/FLIR/conservator/wrappers/media.py +++ b/FLIR/conservator/wrappers/media.py @@ -1,4 +1,5 @@ import enum +import math import os import traceback from dataclasses import dataclass From dd56b30eb342747a507144cc77331e863c45e0ab Mon Sep 17 00:00:00 2001 From: Neal Butler Date: Wed, 4 Nov 2026 12:11:26 -0700 Subject: [PATCH 3/3] Updated schema file --- FLIR/conservator/generated/schema.py | 91 ++++++++++++++++++++++++++++ api_version.txt | 2 +- 2 files changed, 92 insertions(+), 1 deletion(-) diff --git a/FLIR/conservator/generated/schema.py b/FLIR/conservator/generated/schema.py index 52ca25cf..fa98bb54 100644 --- a/FLIR/conservator/generated/schema.py +++ b/FLIR/conservator/generated/schema.py @@ -589,6 +589,18 @@ class DatasetFrameMetadataInput(sgqlc.types.Input): spectrum = sgqlc.types.Field(Spectrum, graphql_name="spectrum") +class DatasetIdentityIdMappingInput(sgqlc.types.Input): + __schema__ = schema + __field_names__ = ("video_id", "target_id", "identity_id") + video_id = sgqlc.types.Field( + sgqlc.types.non_null(GraphqlID), graphql_name="videoId" + ) + target_id = sgqlc.types.Field(sgqlc.types.non_null(Int), graphql_name="targetId") + identity_id = sgqlc.types.Field( + sgqlc.types.non_null(Int), graphql_name="identityId" + ) + + class DeleteCollectionInput(sgqlc.types.Input): __schema__ = schema __field_names__ = ("id",) @@ -1815,6 +1827,7 @@ class Dataset(sgqlc.types.Type): "allow_annotations_outside_frame", "allow_duplicate_target_id", "has_changes", + "identity_id_mapping", ) id = sgqlc.types.Field(sgqlc.types.non_null(GraphqlID), graphql_name="id") user_id = sgqlc.types.Field(GraphqlID, graphql_name="userId") @@ -1961,6 +1974,10 @@ class Dataset(sgqlc.types.Type): has_changes = sgqlc.types.Field( sgqlc.types.non_null(Boolean), graphql_name="hasChanges" ) + identity_id_mapping = sgqlc.types.Field( + sgqlc.types.non_null(sgqlc.types.list_of("DatasetIdentityIdMapping")), + graphql_name="identityIdMapping", + ) class DatasetAnnotation(sgqlc.types.Type): @@ -2310,6 +2327,21 @@ class DatasetFramesOnly(sgqlc.types.Type): ) +class DatasetIdentityIdMapping(sgqlc.types.Type): + __schema__ = schema + __field_names__ = ("video_id", "video_name", "target_id", "identity_id") + video_id = sgqlc.types.Field( + sgqlc.types.non_null(GraphqlID), graphql_name="videoId" + ) + video_name = sgqlc.types.Field( + sgqlc.types.non_null(String), graphql_name="videoName" + ) + target_id = sgqlc.types.Field(sgqlc.types.non_null(Int), graphql_name="targetId") + identity_id = sgqlc.types.Field( + sgqlc.types.non_null(Int), graphql_name="identityId" + ) + + class DatasetVideo(sgqlc.types.Type): __schema__ = schema __field_names__ = ("_id", "filename") @@ -2352,6 +2384,21 @@ class DatasetVideoStatsResult(sgqlc.types.Type): count = sgqlc.types.Field(sgqlc.types.non_null(Int), graphql_name="count") +class DatasetVideoTargetIdDetails(sgqlc.types.Type): + __schema__ = schema + __field_names__ = ("video_id", "video_file_name", "video_name", "target_ids") + video_id = sgqlc.types.Field( + sgqlc.types.non_null(GraphqlID), graphql_name="videoId" + ) + video_file_name = sgqlc.types.Field( + sgqlc.types.non_null(String), graphql_name="videoFileName" + ) + video_name = sgqlc.types.Field(String, graphql_name="videoName") + target_ids = sgqlc.types.Field( + sgqlc.types.non_null(sgqlc.types.list_of(Int)), graphql_name="targetIds" + ) + + class ErrorArg(sgqlc.types.Type): __schema__ = schema __field_names__ = ("key", "value") @@ -3115,6 +3162,7 @@ class Mutation(sgqlc.types.Type): "move_dataset", "add_datasets_to_collection", "recalculate_dataset_stats", + "set_identity_id_mapping", "create_dataset_annotations", "accept_dataset_annotations", "update_dataset_annotation", @@ -4562,6 +4610,32 @@ class Mutation(sgqlc.types.Type): ) ), ) + set_identity_id_mapping = sgqlc.types.Field( + sgqlc.types.non_null(Dataset), + graphql_name="setIdentityIdMapping", + args=sgqlc.types.ArgDict( + ( + ( + "dataset_id", + sgqlc.types.Arg( + sgqlc.types.non_null(GraphqlID), + graphql_name="datasetId", + default=None, + ), + ), + ( + "identity_id_mapping", + sgqlc.types.Arg( + sgqlc.types.non_null( + sgqlc.types.list_of(DatasetIdentityIdMappingInput) + ), + graphql_name="identityIdMapping", + default=None, + ), + ), + ) + ), + ) create_dataset_annotations = sgqlc.types.Field( sgqlc.types.non_null( sgqlc.types.list_of(sgqlc.types.non_null(DatasetAnnotation)) @@ -7938,6 +8012,7 @@ class Query(sgqlc.types.Type): "get_dataset_jsonl", "get_dataset_videos_jsonl", "get_dataset_frames_jsonl", + "get_dataset_video_target_id_data", "dataset_frame", "dataset_frames", "dataset_frames_by_ids", @@ -8418,6 +8493,22 @@ class Query(sgqlc.types.Type): ) ), ) + get_dataset_video_target_id_data = sgqlc.types.Field( + sgqlc.types.non_null(sgqlc.types.list_of(DatasetVideoTargetIdDetails)), + graphql_name="getDatasetVideoTargetIdData", + args=sgqlc.types.ArgDict( + ( + ( + "dataset_id", + sgqlc.types.Arg( + sgqlc.types.non_null(GraphqlID), + graphql_name="datasetId", + default=None, + ), + ), + ) + ), + ) dataset_frame = sgqlc.types.Field( DatasetFrame, graphql_name="datasetFrame", diff --git a/api_version.txt b/api_version.txt index 431c4949..cf55d239 100644 --- a/api_version.txt +++ b/api_version.txt @@ -1 +1 @@ -1837770cd6180477fe416e7bb34435ac +b672fa85172fac00ec9415d136c4034b