fix: escape dotted and dollar-prefixed keys in task upserts

resources/sample_settings.yaml

@@ -1,4 +1,4 @@
-flowcept_version: 0.10.8 # Version of the Flowcept package. Do not update this manually. The CI updates it. This setting file is compatible with this version.
+flowcept_version: 1.0.3 # Version of the Flowcept package. Do not update this manually. The CI updates it. This setting file is compatible with this version.
 
 project:
   debug: true # Toggle debug mode. This will add a property `debug: true` to all saved data, making it easier to retrieve/delete them later.

src/flowcept/commons/daos/docdb_dao/mongodb_dao.py

@@ -34,6 +34,28 @@
 from time import time, sleep
 
 
+def _escape_key_dots(value):
+    """Escape '.' and leading '$' in nested dict keys.
+
+    The task upsert uses an aggregation-pipeline ``$set``, which rejects such
+    keys anywhere in the document (server error 16412) and fails the whole
+    ``bulk_write`` batch. Escaped with the fullwidth lookalikes so the stored
+    document stays readable.
+    """
+    if isinstance(value, dict):
+        out = {}
+        for k, v in value.items():
+            if isinstance(k, str):
+                k = k.replace(".", "．")
+                if k.startswith("$"):
+                    k = "＄" + k[1:]
+            out[k] = _escape_key_dots(v)
+        return out
+    if isinstance(value, list):
+        return [_escape_key_dots(v) for v in value]
+    return value
+
+
 class MongoDBDAO(DocumentDBDAO):
     """
     A data access object for MongoDB.
@@ -303,7 +325,7 @@ def insert_and_update_many_tasks(self, doc_list: List[Dict], indexing_key=None)
                 requests.append(
                     UpdateOne(
                         filter={indexing_key: indexing_key_value},
-                        update=[{"$set": indexed_buffer[indexing_key_value]}],
+                        update=[{"$set": _escape_key_dots(indexed_buffer[indexing_key_value])}],
                         upsert=True,
                     )
                 )

src/flowcept/version.py

@@ -10,4 +10,4 @@
 
 # ❗❗❗ Once again: DO NOT CHANGE THIS FILE ❗❗❗
 # ✋⚠️⛔❗❗❗ STOP! DANGER!!ONEONEELEVEN! Did you carefully read the warning above?! :)
-__version__ = "1.0.0"
+__version__ = "1.0.3"

tests/doc_db_inserter/doc_db_inserter_test.py

@@ -98,6 +98,25 @@ def test_db_insert_and_update_many(self):
         c1 = self.doc_dao.count_tasks()
         assert c0 == c1
 
+    def test_db_insert_with_dotted_keys(self):
+        """Tasks whose payload keys contain '.' or a leading '$' must still persist."""
+        # Keys like a dataframe column named "yr.doy" inside a captured payload
+        # used to fail the whole bulk_write with server error 16412.
+        c0 = self.doc_dao.count_tasks()
+        uid = str(uuid4())
+        docs = [
+            {
+                "task_id": str(uuid4()),
+                "myid": uid,
+                "debug": True,
+                "status": "FINISHED",
+                "used": {"columns": {"yr.doy": 1, "$weird": "x"}},
+            }
+        ]
+        assert self.doc_dao.insert_and_update_many_tasks(docs, "myid") is True
+        self.doc_dao.delete_task_keys("myid", [uid])
+        assert self.doc_dao.count_tasks() == c0
+
     def test_status_updates(self):
         c0 = self.doc_dao.count_tasks()
         assert c0 >= 0