From 48d7b37fd086d5db18ef2c3ddc9486b5ea06a0be Mon Sep 17 00:00:00 2001 From: Simon Pinfold Date: Tue, 29 Sep 2026 10:38:31 -0700 Subject: [PATCH 1/4] feat(telemetry): send app_version with ops-flag evaluation Ops flags were evaluated with comfy_staff as the only person property, so a release condition could not gate a payload on the Desktop version. Send the build's app_version on every evaluation, independent of consent: it describes the binary, not the person, and a grant only newer builds can apply must be gateable before consent is given. comfy_staff keeps its consent+staff gate. Builds from before this change send no version, so a semver_gte condition excludes them. --- POSTHOG_TELEMETRY_EVENTS.md | 24 +++++++ src/main/lib/opsFlag.ts | 6 +- src/main/lib/telemetry.test.ts | 121 ++++++++++++++++++++++++++++----- src/main/lib/telemetry.ts | 42 +++++++++--- 4 files changed, 166 insertions(+), 27 deletions(-) diff --git a/POSTHOG_TELEMETRY_EVENTS.md b/POSTHOG_TELEMETRY_EVENTS.md index ff0305fe2..5bf5002b3 100644 --- a/POSTHOG_TELEMETRY_EVENTS.md +++ b/POSTHOG_TELEMETRY_EVENTS.md @@ -252,3 +252,27 @@ The hosted ComfyUI bridge accepts an arbitrary `event: string` through therefore submit additional names that are not present in this repository. This inventory is exhaustive for event names defined by this repository, but cannot enumerate events originating from external hosted code. + +## Ops-flag targeting properties + +Ops flags (`makeOpsFlag` in `src/main/lib/opsFlag.ts`, e.g. +`desktop_core_beta_features`) are evaluated with the installation-stable id as +the distinct id and these request-scoped `person_properties`. PostHog matches +release conditions against them but stores nothing, and no +`$feature_flag_called` event is sent. The source is `opsFlagPersonProperties` +in `src/main/lib/telemetry.ts`. + +| Property | When sent | +| ------------- | ------------------------------------------------------ | +| `app_version` | Every evaluation, whatever the consent state | +| `comfy_staff` | `'true'` only with consent granted and a staff account | + +- `app_version` is the same value the events carry. Release builds send plain + semver (`1.1.4`, `1.1.4-rc.1`), so a condition can use the semver operators, + e.g. `app_version semver_gte 1.1.4`. +- Builds released before `app_version` was added send no version. A semver + condition does not match a missing property, so a `semver_gte` gate excludes + them. That is the safe direction for a payload that older builds reject. +- Unpackaged development builds send `git describe` output (e.g. + `v1.1.1-5-gabc1234`), not a release version. Test version gates with a + packaged build or a hand-built `/flags` request. diff --git a/src/main/lib/opsFlag.ts b/src/main/lib/opsFlag.ts index 295e064e9..632db2524 100644 --- a/src/main/lib/opsFlag.ts +++ b/src/main/lib/opsFlag.ts @@ -4,8 +4,10 @@ * Ops flags are server config pushed TO the client (availability guards, rollout gates), not * analytics collected FROM the user, so they read through `getOpsFlagResult`, which deliberately * BYPASSES the consent gate — a user who declined telemetry still gets the override, and - * pre-consent surfaces can still resolve a value. The evaluation request supplies only the - * installation-stable key and the flag key; implicit flag events are disabled. + * pre-consent surfaces can still resolve a value. The evaluation request supplies the + * installation-stable key, the flag key and this build's `app_version` (plus `comfy_staff` for a + * consented staff install — see `opsFlagPersonProperties` in telemetry.ts); implicit flag events + * are disabled. * * Kept separate from `experiments.ts` (locked variant assignment, next-boot cache) so an * operational override isn't accidentally consent-gated. Fetched once at boot; running apps diff --git a/src/main/lib/telemetry.test.ts b/src/main/lib/telemetry.test.ts index f92ae8ac5..9d76acd3f 100644 --- a/src/main/lib/telemetry.test.ts +++ b/src/main/lib/telemetry.test.ts @@ -549,9 +549,9 @@ describe('telemetry anonymous flag reads', () => { { key: 'desktop_core_beta_features', distinctId: 'installation-id', - // Empty rather than absent: the request is made pre-consent by design, and this is + // Only the build's version: the request is made pre-consent by design, and this is // where an email would sit if one were ever attached without one. - options: { sendFeatureFlagEvents: false, personProperties: {} } + options: { sendFeatureFlagEvents: false, personProperties: { app_version: '0.0.0' } } } ]) }) @@ -622,7 +622,7 @@ describe('ops-flag person targeting', () => { await evaluate() - expect(lastPersonProperties()).toEqual({ comfy_staff: 'true' }) + expect(lastPersonProperties()).toEqual({ app_version: '0.0.0', comfy_staff: 'true' }) }) it('leaves the distinct id the installation hash, so bucketing is unchanged', async () => { @@ -651,18 +651,18 @@ describe('ops-flag person targeting', () => { expect(captured.map((event) => event.event)).not.toContain('$feature_flag_called') }) - it('sends nothing for a non-staff install', async () => { + it('sends no comfy_staff for a non-staff install', async () => { // Absent rather than `comfy_staff: 'false'`: a `comfy_staff = true` condition does not match - // a missing property, so nothing goes on the wire for the majority of users. + // a missing property, so it stays off the wire for the majority of users. setupTelemetry({ consent: 'granted' }) telemetry.setFlagEvaluationStaff(false) await evaluate() - expect(lastPersonProperties()).toEqual({}) + expect(lastPersonProperties()).toEqual({ app_version: '0.0.0' }) }) - it('sends nothing when consent is undecided', async () => { + it('sends no comfy_staff when consent is undecided', async () => { // The flag FETCH bypasses consent on purpose (ops flags are config pushed to the client); // whether someone is an employee is a fact about them and does not inherit that exemption. setupTelemetry({ consent: null }) @@ -670,21 +670,21 @@ describe('ops-flag person targeting', () => { await evaluate() - expect(lastPersonProperties()).toEqual({}) + expect(lastPersonProperties()).toEqual({ app_version: '0.0.0' }) }) - it('sends nothing when consent is denied', async () => { + it('sends no comfy_staff when consent is denied', async () => { setupTelemetry({ consent: 'denied' }) telemetry.setFlagEvaluationStaff(true) await evaluate() - expect(lastPersonProperties()).toEqual({}) + expect(lastPersonProperties()).toEqual({ app_version: '0.0.0' }) }) - it('still evaluates the flag without consent, carrying no property', async () => { + it('still evaluates the flag without consent, carrying no staff property', async () => { // The request itself must survive the consent gate — a declined user still gets ops - // overrides. Only the property is withheld. + // overrides. Only the staff property is withheld. setupTelemetry({ consent: 'denied' }) telemetry.setFlagEvaluationStaff(true) posthogClientMock.featureFlagResult = { enabled: true, variant: 'beta', payload: null } @@ -692,19 +692,19 @@ describe('ops-flag person targeting', () => { await expect( telemetry.getOpsFlagResult('desktop_core_beta_features', 'installation-id', 100) ).resolves.toMatchObject({ kind: 'value', value: 'beta' }) - expect(lastPersonProperties()).toEqual({}) + expect(lastPersonProperties()).toEqual({ app_version: '0.0.0' }) }) it('stops sending once the install is reclassified, as on sign-out', async () => { setupTelemetry({ consent: 'granted' }) telemetry.setFlagEvaluationStaff(true) await evaluate() - expect(lastPersonProperties()).toEqual({ comfy_staff: 'true' }) + expect(lastPersonProperties()).toEqual({ app_version: '0.0.0', comfy_staff: 'true' }) telemetry.setFlagEvaluationStaff(false) await evaluate() - expect(lastPersonProperties()).toEqual({}) + expect(lastPersonProperties()).toEqual({ app_version: '0.0.0' }) }) it('does not survive a reset, so no classification leaks between launches in-process', async () => { @@ -714,7 +714,7 @@ describe('ops-flag person targeting', () => { setupTelemetry({ consent: 'granted' }) await evaluate() - expect(lastPersonProperties()).toEqual({}) + expect(lastPersonProperties()).toEqual({ app_version: '0.0.0' }) }) // The bug in one test: with only a machine-derived distinct id, a person condition cannot @@ -777,6 +777,95 @@ describe('ops-flag person targeting', () => { ).resolves.toEqual({ kind: 'value', value: false, payload: undefined }) }) }) + + // `app_version` describes the build, not the person, so unlike `comfy_staff` it rides on every + // evaluation. A grant only newer builds can apply must be gateable from the first launch, which + // is pre-consent. + describe('app_version', () => { + it.each([ + ['undecided', null], + ['denied', 'denied'], + ['granted', 'granted'] + ] as const)('is sent when consent is %s', async (_label, consent) => { + setupTelemetry({ consent, appVersion: '1.1.4' }) + + await evaluate() + + expect(lastPersonProperties()).toEqual({ app_version: '1.1.4' }) + }) + + it('rides alongside comfy_staff for a consented staff install', async () => { + setupTelemetry({ consent: 'granted', appVersion: '1.1.4' }) + telemetry.setFlagEvaluationStaff(true) + + await evaluate() + + expect(lastPersonProperties()).toEqual({ app_version: '1.1.4', comfy_staff: 'true' }) + }) + + it('is the version initTelemetry was given, verbatim, prerelease included', async () => { + // No normalisation: the flag condition compares the same string the events carry. + setupTelemetry({ consent: null, appVersion: '1.1.4-rc.1' }) + + await evaluate() + + expect(lastPersonProperties()).toEqual({ app_version: '1.1.4-rc.1' }) + }) + + it('is omitted rather than sent empty when the version is unknown', async () => { + // An empty string is not a version; absent is what a semver condition treats as no match. + setupTelemetry({ consent: null, appVersion: '' }) + + await evaluate() + + expect(lastPersonProperties()).toEqual({}) + }) + + // What the property is for: a payload carrying args older builds reject, gated server-side. + describe('against a stand-in `app_version semver_gte 1.1.4` condition', () => { + /** Release-only `major.minor.patch` comparison — enough for the versions below. A missing + * value never matches, as in PostHog's own evaluation. */ + function isAtLeast(version: string | undefined, min: [number, number, number]): boolean { + if (!version) return false + const [release = ''] = version.split('-') + const [major = 0, minor = 0, patch = 0] = release.split('.').map(Number) + if (major !== min[0]) return major > min[0] + if (minor !== min[1]) return minor > min[1] + return patch >= min[2] + } + + function serveVersionGatedFlag(): void { + posthogClientMock.evaluateCondition = (options) => { + if (!isAtLeast(options?.personProperties?.['app_version'], [1, 1, 4])) { + return { enabled: false } + } + return { + enabled: true, + variant: 'beta', + payload: { flags: [{ arg: '--enable-assets-output-scanning' }] } + } + } + } + + it.each(['1.1.4', '1.2.0'])('matches %s, even before consent', async (appVersion) => { + setupTelemetry({ consent: null, appVersion }) + serveVersionGatedFlag() + + await expect( + telemetry.getOpsFlagResult('desktop_core_beta_features', 'installation-id', 100) + ).resolves.toMatchObject({ kind: 'value', value: 'beta' }) + }) + + it('does not match 1.1.3', async () => { + setupTelemetry({ consent: 'granted', appVersion: '1.1.3' }) + serveVersionGatedFlag() + + await expect( + telemetry.getOpsFlagResult('desktop_core_beta_features', 'installation-id', 100) + ).resolves.toEqual({ kind: 'value', value: false, payload: undefined }) + }) + }) + }) }) // A cold `/flags` POST measured ~2572 ms on Windows and is always cold at boot, so a 2000 ms diff --git a/src/main/lib/telemetry.ts b/src/main/lib/telemetry.ts index add99cef0..df12ea173 100644 --- a/src/main/lib/telemetry.ts +++ b/src/main/lib/telemetry.ts @@ -196,6 +196,7 @@ export function _resetForTest(): void { installationIdProperty = null consentState = 'undecided' flagEvaluationStaff = false + flagEvaluationAppVersion = null pendingSessionStart = null pendingFirstLaunch = null pendingPersonSet = null @@ -364,6 +365,20 @@ let consentState: ConsentState = 'undecided' */ let flagEvaluationStaff = false +/** + * This build's version, sent with every ops-flag evaluation as `app_version` so + * a release condition can gate on it with PostHog's semver operators. Set once + * by `initTelemetry`; `null` until then, and no request goes out before it, + * because `getOpsFlagResult` needs the client that `initTelemetry` creates. + * + * Unlike `flagEvaluationStaff` it is NOT consent-gated. It describes the binary, + * not the person running it: every install of a release sends the same string. + * A grant that only newer builds can apply has to be gateable on the very first + * launch, and pre-consent is exactly when a version gate would otherwise + * evaluate differently from the same build after consent. + */ +let flagEvaluationAppVersion: string | null = null + /** * Bind whether ops-flag evaluation should present this install as staff. * @@ -377,7 +392,12 @@ export function setFlagEvaluationStaff(isStaff: boolean): void { /** * Person properties for an ops-flag evaluation request. * - * Empty unless consent is `'granted'` AND the account is staff. The flag FETCH + * `app_version` always (see `flagEvaluationAppVersion`). Builds from before it + * was added send none, and a semver condition does not match a missing + * property, so a `>=` gate fails closed for them — the direction a grant that + * older builds would reject needs. + * + * `comfy_staff` only when consent is `'granted'` AND the account is staff. The flag FETCH * itself bypasses the consent gate on purpose — ops flags are config pushed TO * the client, and a user who declined telemetry still gets the override — but * that argument covers the installation-stable key and nothing else. Whether a @@ -391,8 +411,10 @@ export function setFlagEvaluationStaff(isStaff: boolean): void { * overwhelming majority of users. */ function opsFlagPersonProperties(): Record { - if (consentState !== 'granted' || !flagEvaluationStaff) return {} - return { comfy_staff: 'true' } + const properties: Record = {} + if (flagEvaluationAppVersion) properties['app_version'] = flagEvaluationAppVersion + if (consentState === 'granted' && flagEvaluationStaff) properties['comfy_staff'] = 'true' + return properties } const PRE_CONSENT_ALLOWED_EVENTS: ReadonlySet = new Set([ @@ -594,6 +616,7 @@ export function initTelemetry(opts: InitOptions): void { if (initialized) return initialized = true bootstrapTimeMs = Date.now() + flagEvaluationAppVersion = opts.appVersion // Set the per-event defaults BEFORE the early-return so disabled // telemetry still gets a coherent snapshot (useful when tests stub @@ -1663,18 +1686,19 @@ const OPS_FLAG_DEADLINE: unique symbol = Symbol('ops-flag-deadline') * creates a PostHog person behind the capture policy. * * The evaluation call supplies the installation-stable evaluation key, the flag - * key, and — only once consent is `'granted'` and the account is staff — - * `comfy_staff`, via `opsFlagPersonProperties`. That property is - * request-scoped: the SDK puts it in the `/flags` POST body as - * `person_properties`, where the server evaluates release conditions against it - * and nothing is stored. The `distinct_id` remains the installation hash, so + * key, `app_version`, and — only once consent is `'granted'` and the account is + * staff — `comfy_staff`, via `opsFlagPersonProperties`. Those properties are + * request-scoped: the SDK puts them in the `/flags` POST body as + * `person_properties`, where the server evaluates release conditions against + * them and nothing is stored. The `distinct_id` remains the installation hash, so * bucketing is unchanged and the machine is still not linked to any account in * the person store. * * It exists because the installation hash CANNOT be resolved to a person: a * condition on any person attribute can never match a machine-derived id, so * staff targeting silently returned nothing for every install. Supplying a - * property on the request is what makes such a condition evaluable at all. + * property on the request is what makes such a condition evaluable at all. The + * same holds for `app_version`: it is what lets a grant be version-gated. * * This is the ONLY evaluation. An earlier design added a second, authenticated * one once a cloud view resolved auth; it could not work, because the From acfb402bcff40f8227fb25419f56b6bad7eaa763 Mon Sep 17 00:00:00 2001 From: Simon Pinfold Date: Tue, 29 Sep 2026 10:42:01 -0700 Subject: [PATCH 2/4] test,docs: model SemVer prerelease ordering for version-gated ops flags PostHog's server evaluates semver conditions with the semver crate, so 1.1.4-rc.1 sorts below 1.1.4. Make the test stand-in match, cover an RC, document how to include RCs, and correct the dev-build version example (getAppVersion strips the leading v). --- POSTHOG_TELEMETRY_EVENTS.md | 10 +++++++--- src/main/lib/telemetry.test.ts | 15 +++++++++------ 2 files changed, 16 insertions(+), 9 deletions(-) diff --git a/POSTHOG_TELEMETRY_EVENTS.md b/POSTHOG_TELEMETRY_EVENTS.md index 5bf5002b3..b2ec30ed5 100644 --- a/POSTHOG_TELEMETRY_EVENTS.md +++ b/POSTHOG_TELEMETRY_EVENTS.md @@ -273,6 +273,10 @@ in `src/main/lib/telemetry.ts`. - Builds released before `app_version` was added send no version. A semver condition does not match a missing property, so a `semver_gte` gate excludes them. That is the safe direction for a payload that older builds reject. -- Unpackaged development builds send `git describe` output (e.g. - `v1.1.1-5-gabc1234`), not a release version. Test version gates with a - packaged build or a hand-built `/flags` request. +- PostHog's server compares versions by SemVer, so a prerelease sorts below its + release: `1.1.4-rc.1` does not satisfy `semver_gte 1.1.4`. To include the + release candidates of 1.1.4, gate on `semver_gte 1.1.4-rc.0`. +- Unpackaged development builds send `git describe` output with the leading `v` + removed (e.g. `1.1.1-5-gabc1234`). That compares as a prerelease of the last + tag, below the tag itself. Test version gates with a packaged build or a + hand-built `/flags` request. diff --git a/src/main/lib/telemetry.test.ts b/src/main/lib/telemetry.test.ts index 9d76acd3f..803a52a42 100644 --- a/src/main/lib/telemetry.test.ts +++ b/src/main/lib/telemetry.test.ts @@ -823,15 +823,17 @@ describe('ops-flag person targeting', () => { // What the property is for: a payload carrying args older builds reject, gated server-side. describe('against a stand-in `app_version semver_gte 1.1.4` condition', () => { - /** Release-only `major.minor.patch` comparison — enough for the versions below. A missing - * value never matches, as in PostHog's own evaluation. */ + /** `version >= min` for a release `min`, enough for the versions below. Like PostHog's + * server (the `semver` crate), a prerelease sorts BELOW its release, and a missing value + * never matches. */ function isAtLeast(version: string | undefined, min: [number, number, number]): boolean { if (!version) return false - const [release = ''] = version.split('-') + const [release = '', ...prerelease] = version.split('-') const [major = 0, minor = 0, patch = 0] = release.split('.').map(Number) if (major !== min[0]) return major > min[0] if (minor !== min[1]) return minor > min[1] - return patch >= min[2] + if (patch !== min[2]) return patch > min[2] + return prerelease.length === 0 } function serveVersionGatedFlag(): void { @@ -856,8 +858,9 @@ describe('ops-flag person targeting', () => { ).resolves.toMatchObject({ kind: 'value', value: 'beta' }) }) - it('does not match 1.1.3', async () => { - setupTelemetry({ consent: 'granted', appVersion: '1.1.3' }) + it.each(['1.1.3', '1.1.4-rc.1'])('does not match %s', async (appVersion) => { + // An RC of the gated release is below it, so it needs its own `semver_gte 1.1.4-rc.0`. + setupTelemetry({ consent: 'granted', appVersion }) serveVersionGatedFlag() await expect( From be7ac72831ba860fc12e53d33efcf3a03296ba80 Mon Sep 17 00:00:00 2001 From: Simon Pinfold Date: Tue, 29 Sep 2026 11:12:47 -0700 Subject: [PATCH 3/4] docs(telemetry): scope the version gate to live evaluation; note it is not access control A semver gate evaluates closed for builds without app_version, but the persisted ops-flags.json answer is reapplied on an unreachable launch, so a downgraded build can still apply a cached grant. The version is client-reported, so a gate on it is a compatibility gate only. Also note the bare-commit fallback of git describe --always for dev builds. --- POSTHOG_TELEMETRY_EVENTS.md | 10 ++++++++-- src/main/lib/telemetry.ts | 10 ++++++++-- 2 files changed, 16 insertions(+), 4 deletions(-) diff --git a/POSTHOG_TELEMETRY_EVENTS.md b/POSTHOG_TELEMETRY_EVENTS.md index b2ec30ed5..285341f9c 100644 --- a/POSTHOG_TELEMETRY_EVENTS.md +++ b/POSTHOG_TELEMETRY_EVENTS.md @@ -272,11 +272,17 @@ in `src/main/lib/telemetry.ts`. e.g. `app_version semver_gte 1.1.4`. - Builds released before `app_version` was added send no version. A semver condition does not match a missing property, so a `semver_gte` gate excludes - them. That is the safe direction for a payload that older builds reject. + them. That is the safe direction for a payload that older builds reject. It + holds for the live evaluation only: `ops-flags.json` keeps the last answer, so + a build downgraded from one that received the payload still applies it on a + launch whose flag fetch does not complete. +- `app_version` is reported by the client, so a version gate is a compatibility + gate, not access control. - PostHog's server compares versions by SemVer, so a prerelease sorts below its release: `1.1.4-rc.1` does not satisfy `semver_gte 1.1.4`. To include the release candidates of 1.1.4, gate on `semver_gte 1.1.4-rc.0`. - Unpackaged development builds send `git describe` output with the leading `v` removed (e.g. `1.1.1-5-gabc1234`). That compares as a prerelease of the last - tag, below the tag itself. Test version gates with a packaged build or a + tag, below the tag itself. With no reachable tag it is a bare commit id, not a + version. Test version gates with a packaged build or a hand-built `/flags` request. diff --git a/src/main/lib/telemetry.ts b/src/main/lib/telemetry.ts index df12ea173..9d32ab635 100644 --- a/src/main/lib/telemetry.ts +++ b/src/main/lib/telemetry.ts @@ -394,8 +394,14 @@ export function setFlagEvaluationStaff(isStaff: boolean): void { * * `app_version` always (see `flagEvaluationAppVersion`). Builds from before it * was added send none, and a semver condition does not match a missing - * property, so a `>=` gate fails closed for them — the direction a grant that - * older builds would reject needs. + * property, so a `>=` gate evaluates closed for them — the direction a grant + * that older builds would reject needs. That covers the live evaluation only: + * `ops-flags.json` keeps the last answer without the version that earned it, so + * after a downgrade an `unreachable` launch still applies a grant a newer build + * persisted. + * + * The version is self-asserted, so a version gate is a compatibility gate, not + * a trust boundary: never gate anything a user could want withheld on it. * * `comfy_staff` only when consent is `'granted'` AND the account is staff. The flag FETCH * itself bypasses the consent gate on purpose — ops flags are config pushed TO From 4d059361011a61fd56ab13c2a8435d665b9053bb Mon Sep 17 00:00:00 2001 From: Simon Pinfold Date: Wed, 30 Sep 2026 21:00:56 -0700 Subject: [PATCH 4/4] docs(telemetry): note prerelease ordering and the dev-build version format The section in POSTHOG_TELEMETRY_EVENTS.md went with that file; keep the two notes the docblock did not already carry. --- src/main/lib/telemetry.ts | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/main/lib/telemetry.ts b/src/main/lib/telemetry.ts index b3e1678ab..b0f86b6b4 100644 --- a/src/main/lib/telemetry.ts +++ b/src/main/lib/telemetry.ts @@ -401,6 +401,11 @@ export function setFlagEvaluationStaff(isStaff: boolean): void { * after a downgrade an `unreachable` launch still applies a grant a newer build * persisted. * + * PostHog orders prereleases by SemVer, so `1.1.5-rc.1` is below `1.1.5`; gate + * on `X-0` (e.g. `semver_gte 1.1.5-0`) to include X's prereleases. Unpackaged + * builds send `git describe` output (`1.1.4-5-gabc1234`, a prerelease of the + * last tag), or a bare commit id when no tag is reachable. + * * The version is self-asserted, so a version gate is a compatibility gate, not * a trust boundary: never gate anything a user could want withheld on it. *