scalable distributed syncing

scripts/schema-sync-demo.mjs

@@ -0,0 +1,86 @@
+import {
+  compileSchema
+} from "../src/schema-sync/compiler-contract.mjs";
+import {
+  makeManifestAnnounce,
+  makeArtifactPush,
+  makeArtifactPull,
+  makeAck
+} from "../src/schema-sync/sync-protocol.mjs";
+import { InMemoryReconciliationStore } from "../src/schema-sync/reconciliation-store.mjs";
+import { FITNESS_CHALLENGES_SCHEMA } from "../src/categories/fitness-challenges.mjs";
+
+function nodeScopesForFitness() {
+  return ["fitness-challenges:goals", "fitness-challenges:preferences", "fitness-challenges:streaks"]; 
+}
+
+function nodeScopesNoFitnessSensitive() {
+  // Intentionally omit streaks scope; enforcer will redact sensitive sections.
+  return ["fitness-challenges:goals", "fitness-challenges:preferences"]; 
+}
+
+const compilerVersion = "0.1.0";
+
+const compile = compileSchema(FITNESS_CHALLENGES_SCHEMA, {
+  compiler_version: compilerVersion,
+  output_schema_version: "fitness-challenges.output.v1",
+  source_ref: "src/categories/fitness-challenges.mjs",
+  dependencies: []
+});
+
+const { manifest, artifact } = compile;
+
+// Simulate two nodes: A can read everything, B lacks streaks scope.
+const nodeA = new InMemoryReconciliationStore({ node_id: "node_A", scopes: nodeScopesForFitness() });
+const nodeB = new InMemoryReconciliationStore({ node_id: "node_B", scopes: nodeScopesNoFitnessSensitive() });
+
+const msgAnnounce = makeManifestAnnounce({
+  message_id: "m1",
+  node_id: "node_A",
+  manifest,
+  artifact_id: artifact.artifact_id
+});
+
+const msgPush = makeArtifactPush({
+  message_id: "m2",
+  node_id: "node_A",
+  artifact
+});
+
+const msgPull = makeArtifactPull({
+  message_id: "m3",
+  node_id: "node_B",
+  artifact_id: artifact.artifact_id
+});
+
+const msgAck = makeAck({
+  message_id: "m4",
+  node_id: "node_A",
+  response_to_message_id: "m3",
+  artifact_id: artifact.artifact_id
+});
+
+console.log("=== Compiled artifact ===");
+console.log({ artifact_id: artifact.artifact_id });
+
+console.log("\n=== Node A apply manifest + push ===");
+console.log(nodeA.applyMessage(msgAnnounce));
+console.log(nodeA.applyMessage(msgPush));
+
+console.log("\n=== Node B apply manifest + pull record (simulated) ===");
+console.log(nodeB.applyMessage(msgAnnounce));
+console.log(nodeB.applyMessage(msgPull));
+console.log("\n=== Node A ack pull ===");
+console.log(nodeA.applyMessage(msgAck));
+
+console.log("\n=== Node B receives push (will redact sensitive parts) ===");
+console.log(nodeB.applyMessage(msgPush));
+
+const bArtifact = nodeB.getArtifact(artifact.artifact_id);
+console.log("\n=== Node B stored artifact ===");
+console.log({ artifactId: artifact.artifact_id, hasDefinition: !!bArtifact, hasRedaction: !!bArtifact?.compiled_output?.definition?.sections?.sensitive_signals });
+
+// Print whether sensitive field meta is redacted.
+const redacted = bArtifact?.compiled_output?.definition?.sections?.sensitive_signals?.fields?.medical_constraints;
+console.log("Redacted sensitive field meta:", redacted ? { sensitive: redacted.sensitive, redacted: redacted.redacted } : null);
+

src/schema-sync/artifact-model.mjs

@@ -0,0 +1,69 @@
+import crypto from "crypto";
+
+/**
+ * Deterministic artifact identity:
+ * artifact_id = sha256( stable_stringify({manifest, compiled_output}) )
+ */
+
+export function sha256Hex(input) {
+  return crypto.createHash("sha256").update(String(input)).digest("hex");
+}
+
+/**
+ * Stable stringify to guarantee deterministic hashing.
+ * - sorts object keys
+ * - preserves arrays order
+ */
+export function stableStringify(value) {
+  if (value === null || value === undefined) return String(value);
+  const t = typeof value;
+  if (t !== "object") return JSON.stringify(value);
+  if (Array.isArray(value)) return `[${value.map(stableStringify).join(",")}]`;
+
+  const keys = Object.keys(value).sort();
+  return `{${keys.map((k) => `${JSON.stringify(k)}:${stableStringify(value[k])}`).join(",")}}`;
+}
+
+export function computeArtifactId(manifest, compiledOutput) {
+  const canonical = stableStringify({ manifest, compiledOutput });
+  return `schema_artifact_${sha256Hex(canonical)}`;
+}
+
+export function createSchemaManifest({
+  source_ref,
+  compiler_name,
+  compiler_version,
+  input_hash,
+  output_schema_version,
+  dependencies = []
+}) {
+  if (!source_ref) throw new Error("manifest.source_ref is required");
+  if (!compiler_name) throw new Error("manifest.compiler_name is required");
+  if (!compiler_version) throw new Error("manifest.compiler_version is required");
+  if (!output_schema_version) throw new Error("manifest.output_schema_version is required");
+
+  return {
+    schema_version: "schema-manifest.v1",
+    source_ref,
+    compiler_name,
+    compiler_version,
+    input_hash: String(input_hash || ""),
+    output_schema_version,
+    dependencies: Array.isArray(dependencies) ? dependencies : []
+  };
+}
+
+export function createSchemaArtifact({ manifest, compiled_output }) {
+  if (!manifest) throw new Error("artifact.manifest is required");
+  if (!compiled_output) throw new Error("artifact.compiled_output is required");
+
+  const artifact_id = computeArtifactId(manifest, compiled_output);
+  return {
+    schema_version: "schema-artifact.v1",
+    artifact_id,
+    manifest,
+    compiled_output,
+    created_at: new Date().toISOString()
+  };
+}
+

src/schema-sync/cap-envelope.mjs

@@ -0,0 +1,35 @@
+/**
+ * CAP-like envelope helpers for schema-sync messages.
+ * This is a lightweight local contract; transport/auth is expected to be handled
+ * by the caller.
+ */
+
+export function buildCapRequest({ intent, query, request_id }) {
+  return {
+    schema_version: "cap-request.v1",
+    request_id: request_id || `req_${cryptoRandomId()}`,
+    intent,
+    query: query || {}
+  };
+}
+
+export function buildCapPacket({ response_to, status = "success", granted_at, expires_at, permissions, payload }) {
+  return {
+    schema_version: "cap-packet.v1",
+    response_to,
+    status,
+    granted_at: granted_at || new Date().toISOString(),
+    expires_at: expires_at || new Date(Date.now() + 24 * 60 * 60 * 1000).toISOString(),
+    permissions: permissions || {
+      retention: "ephemeral",
+      redistribution: false
+    },
+    payload: payload || []
+  };
+}
+
+function cryptoRandomId() {
+  // Avoid importing crypto here to keep browser bundling simple; used only for IDs.
+  return Math.random().toString(16).slice(2) + Math.random().toString(16).slice(2);
+}
+

src/schema-sync/compiler-contract.mjs

@@ -0,0 +1,58 @@
+import { createSchemaManifest, createSchemaArtifact, sha256Hex } from "./artifact-model.mjs";
+
+/**
+ * Compiler contract:
+ * compileSchema(definition, options) => { manifest, artifact }
+ *
+ * The definition can be either:
+ * - a category module export object (like FITNESS_CHALLENGES_SCHEMA)
+ * - or a plain JSON schema definition
+ */
+
+export const DEFAULT_COMPILER_NAME = "memact.schema.compiler";
+
+export function normalizeDefinition(definition) {
+  if (!definition || typeof definition !== "object") {
+    throw new Error("definition must be an object");
+  }
+  return definition;
+}
+
+export function compileSchema(definition, {
+  compiler_version,
+  output_schema_version = "schema-output.v1",
+  source_ref = "unknown",
+  compiler_name = DEFAULT_COMPILER_NAME,
+  dependencies = []
+} = {}) {
+  const def = normalizeDefinition(definition);
+
+  if (!compiler_version) throw new Error("options.compiler_version is required");
+
+  // Input hash based on stable ordering.
+  const input_hash = sha256Hex(JSON.stringify(def, Object.keys(def).sort()));
+
+  const manifest = createSchemaManifest({
+    source_ref,
+    compiler_name,
+    compiler_version,
+    input_hash,
+    output_schema_version,
+    dependencies
+  });
+
+  // For this repo snapshot, "compiled output" is a validated JSON payload.
+  // In a full implementation, you’d also attach JSON schema, validators, and any
+  // codegen targets.
+  const compiled_output = {
+    schema_type: def.category || def.schema_type || "unknown",
+    definition: def,
+    validation_hints: {
+      requires_user_confirmation: true
+    }
+  };
+
+  const artifact = createSchemaArtifact({ manifest, compiled_output });
+  return { manifest, artifact };
+}
+

src/schema-sync/index.mjs

@@ -0,0 +1,7 @@
+export * from "./artifact-model.mjs";
+export * from "./compiler-contract.mjs";
+export * from "./cap-envelope.mjs";
+export * from "./sync-protocol.mjs";
+export * from "./reconciliation-store.mjs";
+export * from "./permission-enforcer.mjs";
+

src/schema-sync/permission-enforcer.mjs

@@ -0,0 +1,57 @@
+import { registry } from "../registry.mjs";
+
+/**
+ * Enforce sensitive-section rules on schema artifacts.
+ *
+ * Current repo capability: CategoryRegistry can tell sensitive fields for a category.
+ * We leverage that for artifact definitions that include category and sensitive field markers.
+ */
+
+export function getCategoryFromArtifact(artifact) {
+  const def = artifact?.compiled_output?.definition || artifact?.definition;
+  return def?.category;
+}
+
+export function filterArtifactForPermissions({ artifact, scopes = [] }) {
+  const category = getCategoryFromArtifact(artifact);
+  if (!category) return { ok: false, reason: "missing_category" };
+
+  // If no scopes supplied, return sanitized failure.
+  if (!Array.isArray(scopes) || scopes.length === 0) {
+    return { ok: false, reason: "no_scopes" };
+  }
+
+  // Simple model: if category is known, sensitive fields are blocked unless caller has
+  // any scope that starts with `${category}:` or a wildcard.
+  const sensitive = registry.getSensitiveFields(category);
+  const hasWildcard = scopes.includes("*");
+  if (hasWildcard) return { ok: true, artifact };
+
+  // Caller must have at least one scope that matches the category namespace.
+  const hasCategoryScope = scopes.some((s) => String(s).toLowerCase().startsWith(`${String(category).toLowerCase()}:`));
+  if (!hasCategoryScope) {
+    return { ok: false, reason: "scope_not_allowed" };
+  }
+
+  // If artifact includes a definition with sensitive_signals, we keep it but blank
+  // sensitive fields unless caller scope indicates deeper approval.
+  const def = artifact.compiled_output?.definition;
+  const cloned = structuredClone(artifact);
+  const defCloned = cloned.compiled_output.definition;
+
+  if (defCloned?.sections?.sensitive_signals?.fields) {
+    for (const [field, meta] of Object.entries(defCloned.sections.sensitive_signals.fields)) {
+      if (meta?.sensitive) {
+        defCloned.sections.sensitive_signals.fields[field] = {
+          ...meta,
+          description: meta.description,
+          value: undefined,
+          redacted: true
+        };
+      }
+    }
+  }
+
+  return { ok: true, artifact: cloned, sensitive_fields: [...sensitive] };
+}
+