Branches remove attention thieves instead of adding headlines

.github/workflows/ci.yml

@@ -25,6 +25,9 @@ jobs:
       - name: suppression and invisible-edit rejection
         working-directory: backend
         run: python test_eval_guard.py
+      - name: branch edits stay local and never add text
+        working-directory: backend
+        run: python test_branches.py
 
   frontend-build:
     name: frontend typecheck + build

.gitignore

@@ -48,3 +48,4 @@ backend/demo_out/
 
 # Static web demo build (npm run build:demo)
 web-dist/
+.firebase/

README.md

@@ -43,7 +43,7 @@ It is a small win, and it is the kind Pixel now accepts. An earlier version of t
 
 <p align="center"><img src="assets/media/every-edit.png" width="820" alt="The Red Bull ad and four Nano Banana edits of it, each scored lower than the original"></p>
 
-This is a recorded optimizer run on the same Red Bull ad, with every edit saved. Each one looks more like a finished campaign than the original, and each one scored lower. Every one of them added a headline or a slogan, and text pulls the eye off the can. So Pixel kept the original and reported a change of zero. The edit that won above added no text at all.
+This is a recorded optimizer run on the same Red Bull ad, with every edit saved. Each one looks more like a finished campaign than the original, and each one scored lower. Every one of them added a headline or a slogan, and text pulls the eye off the can. So Pixel kept the original and reported a change of zero. The edit that won above added no text at all. Those were the old preset branches. The current ones remove the attention thieves one at a time and paste back only what changed. On the Nike billboard, 4 of 5 branches now win (59 to 77). [`docs/PIXEL_FINDINGS.md`](docs/PIXEL_FINDINGS.md) has the measurements behind that change.
 
 The first version of Pixel could not do this. It measured raw attention inside the brand box, so "make the logo bigger" always won, and it floored every result at the baseline, so the number could only go up. The version here fixes both:
 

backend/agents.py

@@ -16,6 +16,8 @@
 
 import branch
 import competitive
+import compose
+from branch_pool import branches
 import deepgaze_runner as dg
 import eval_guard
 import gemini
@@ -44,42 +46,17 @@ def scout(brand: str, brief: dict) -> dict:
     return _SCOUT_CACHE[key]
 
 
+def edit_branch(image: Image.Image, branch_: dict) -> tuple[Image.Image, str]:
+    """Run one branch edit and keep only its real changes near the branch's region."""
+    variant, desc = gemini.edit_image(image, branch_["directive"])
+    if str(desc).startswith("["):
+        return variant, desc
+    return compose.keep_changes(image, variant, branch_["region"], branch_.get("protect")), desc
+
+
 def _directive_pool(before: dict, insights: dict, brand: str) -> list[str]:
-    """Diverse edit hypotheses, spanning safe→moderately strong. ENHANCEMENT-first so the
-    winner looks like a *better ad*, not a scorched background with a floating logo; the
-    Judge-gated fitness (_make_scorer) penalizes any variant that crosses into garish, so
-    the search can push moderately without an ugly edit winning on raw attention."""
-    target = f"the {brand} logo and product"
-    # Content-level levers that actually move DeepGaze (gentle re-lighting barely does). The first
-    # three run on the live (depth=1) path, so they're the highest-value, most distinct edits:
-    # de-clutter, reframe head-on, add brand text. Judge-gated fitness still vetoes un-ad-like edits.
-    pool = [
-        f"make SEVERAL coordinated changes at once to turn this into a polished campaign: remove the "
-        f"clutter and any objects, hands, props or stray text crowding or blocking {target}; reframe "
-        f"it head-on and enlarge it as the clear hero; clean and simplify the background; and add a "
-        f"bold, legible on-brand headline, call-to-action and wordmark on {target}",
-        f"remove or clean away any objects, hands, props or background obstructions that crowd or "
-        f"block {target} so the product is fully visible, unobstructed, and the clear hero of the shot",
-        f"reframe {target} to a clean, head-on, front-facing hero angle — square it to the camera so "
-        f"it faces the viewer directly and reads instantly — keep it in roughly the same spot",
-        f"add a bold, legible, on-brand headline and call-to-action and strengthen the logo/wordmark "
-        f"right at {target} as a strong, high-contrast focal point",
-        f"clearly enlarge, brighten and sharpen {target} so it becomes the single biggest, boldest "
-        f"focal element while gently dimming and de-cluttering the surroundings — a polished, real ad",
-        f"recolor the background to a clean, on-brand solid or subtle gradient so {target} stands out "
-        f"as the hero, keeping the shot photographic and realistic",
-    ]
-    # Targeted suppression of each named competing element (clearly tone down, not blackout).
-    pool += [f"clearly tone down or remove the {d['desc']} — it is stealing attention from "
-             f"{target}; reduce its brightness, color and prominence so the brand wins the eye"
-             for d in before["distractors"]]
-    pool += [t["apply"] for t in insights.get("tactics", [])]
-    seen, uniq = set(), []
-    for d in pool:
-        if d and d not in seen:
-            seen.add(d)
-            uniq.append(d)
-    return uniq
+    """Directive strings for the branching search (see `branches`)."""
+    return [b["directive"] for b in branches(before, brand)]
 
 
 def _serialize_tree(tree: list[dict]) -> list[dict]:
@@ -131,7 +108,8 @@ def _step_prep(state: dict) -> dict:
 def _step_context(state: dict) -> dict:
     """Distractor naming + competitor RAG (Scout) — independent, run concurrently."""
     with cf.ThreadPoolExecutor(max_workers=2) as ex:
-        f_names = ex.submit(gemini.label_distractors, state["image"], state["before"]["distractors"])
+        f_names = ex.submit(gemini.label_distractors, state["image"], state["before"]["distractors"],
+                            state["brand"])
         f_insights = ex.submit(scout, state["brand"], state["brief"])
         f_names.result()
         state["insights"] = f_insights.result()
@@ -140,7 +118,9 @@ def _step_context(state: dict) -> dict:
 
 def _step_optimize(state: dict) -> dict:
     """Retoucher: branching tree-search over edits, scored by the Judge-gated fitness."""
-    pool = _directive_pool(state["before"], state["insights"], state["brand"])
+    pool_ = branches(state["before"], state["brand"])
+    pool = [b["directive"] for b in pool_]
+    by_directive = {b["directive"]: b for b in pool_}
 
     def propose(_img, node, n):
         start = (node["depth"] * n) % len(pool)  # rotate so each round tries fresh avenues
@@ -149,7 +129,7 @@ def propose(_img, node, n):
     judged: dict[int, dict] = {}
     state["result"] = branch.search(
         state["image"], state["baseline"], propose,
-        edit=lambda img, directive: gemini.edit_image(img, directive),
+        edit=lambda img, directive: edit_branch(img, by_directive[directive]),
         score=_make_scorer(state["target"], state["brand"], judged),
         breadth=settings.breadth, max_depth=state["depth"], beam_width=1,
         target_score=settings.target_score, epsilon=settings.epsilon,

backend/branch_pool.py

@@ -0,0 +1,38 @@
+"""The branch edits Pixel tries on an ad. Plain data, no model calls, so it is easy to test."""
+from __future__ import annotations
+
+
+def branches(before: dict, brand: str) -> list[dict]:
+    """Branch edits for one image, strongest first. Each is {"directive", "region",
+    "protect"}: one concrete change, the normalized boxes the change must stay near, and
+    the boxes it must never touch (see compose.keep_changes).
+
+    Every branch removes one attention thief, the strongest first, because that is the
+    edit the measurements back: on five ads, removing the named thief beat the original
+    in 9 of 10 tries (mean +2.1 points), while the old presets (headline, CTA, several
+    changes at once, dimming the surroundings) won 2 of 20 (mean -9.6). See
+    docs/PIXEL_FINDINGS.md. Last comes a small polish of the product itself, which rarely
+    wins but is the only honest move left on an ad with no thieves. No branch adds text
+    or dims the scene."""
+    out = []
+    for d in before.get("distractors", []):
+        if d.get("own"):  # the brand's own slogan or logo pulling the eye is not a thief
+            continue
+        what = d["desc"]
+        if what.endswith(" region"):  # unnamed: Gemini couldn't label it
+            what = f"distracting object in the {what} of the photo"
+        out.append({
+            "directive": f"remove the {what} completely and fill its area with what would "
+                         f"naturally be behind it. Keep the {brand} ad, product, logo and slogan "
+                         f"exactly as they are",
+            "region": [d["region"]],
+            "protect": [before["target_box"]],
+        })
+    # Asked for a crisper "label", the model wrote a new one, so this names no text at all.
+    out.append({
+        "directive": f"make the surface of the {brand} product a little crisper and cleaner with "
+                     f"clean highlights, without adding, changing or removing any text or logo",
+        "region": [before["target_box"]],
+        "protect": [],
+    })
+    return out

backend/compose.py

@@ -0,0 +1,106 @@
+"""Keep a generated edit only where it was meant to happen.
+
+Nano Banana returns a whole new frame even for a one-object edit: it shifts the picture
+by a few pixels, re-renders textures and nudges colors everywhere. On the recorded runs
+that drift alone moved the score by a few points either way. So each branch edit is
+registered back onto the original (translation only), and only the pixels that really
+changed near the branch's target are pasted in. A fixed box around the thief was not
+enough: it cut objects in half and left seams, so the mask follows the actual change.
+Every other pixel stays identical to the original.
+"""
+from __future__ import annotations
+
+import numpy as np
+from PIL import Image, ImageDraw, ImageFilter
+from scipy.ndimage import binary_opening, find_objects, gaussian_filter, label
+
+REG_SIDE = 512      # register at this resolution
+MAX_SHIFT = 0.05    # ignore a "registration" bigger than this fraction of the frame
+GROW = 0.04         # grow the branch region by this fraction of the frame
+NEAR = 0.08         # a changed pixel counts only within this distance of the branch region
+DIFF_SIDE = 256     # resolution of the change map
+DIFF_THRESH = 24    # mean RGB difference (0..255) that counts as a real change
+BLOB_PAD = 0.015    # grow each changed blob's box by this fraction of the frame
+FEATHER = 0.012     # feather radius, fraction of the long side
+
+
+def _gray(img: Image.Image, size) -> np.ndarray:
+    return np.asarray(img.convert("L").resize(size, Image.BILINEAR), dtype=np.float64)
+
+
+def estimate_shift(orig: Image.Image, edited: Image.Image) -> tuple[float, float]:
+    """(dx, dy) in original pixels such that edited(x + dx, y + dy) ~ orig(x, y),
+    by phase correlation. Returns (0, 0) when the peak is implausibly far."""
+    w, h = orig.size
+    s = REG_SIDE / max(w, h)
+    size = (max(16, int(w * s)), max(16, int(h * s)))
+    a, b = _gray(orig, size), _gray(edited, size)
+    win = np.outer(np.hanning(size[1]), np.hanning(size[0]))
+    fa, fb = np.fft.fft2((a - a.mean()) * win), np.fft.fft2((b - b.mean()) * win)
+    r = fb * np.conj(fa)
+    corr = np.fft.ifft2(r / (np.abs(r) + 1e-9)).real
+    dy, dx = np.unravel_index(int(corr.argmax()), corr.shape)
+    if dy > size[1] // 2:
+        dy -= size[1]
+    if dx > size[0] // 2:
+        dx -= size[0]
+    if abs(dx) > MAX_SHIFT * size[0] or abs(dy) > MAX_SHIFT * size[1]:
+        return 0.0, 0.0
+    return dx / s, dy / s
+
+
+def register(orig: Image.Image, edited: Image.Image) -> Image.Image:
+    """Shift `edited` so it lines up with `orig`."""
+    edited = edited.convert("RGB").resize(orig.size, Image.LANCZOS)
+    dx, dy = estimate_shift(orig, edited)
+    if dx == 0 and dy == 0:
+        return edited
+    return edited.transform(orig.size, Image.AFFINE, (1, 0, dx, 0, 1, dy), resample=Image.BILINEAR)
+
+
+def region_mask(size, regions, grow: float = GROW, feather: float = FEATHER) -> Image.Image:
+    """Feathered mask covering the normalized [x, y, w, h] regions, each grown by `grow`."""
+    w, h = size
+    m = Image.new("L", size, 0)
+    d = ImageDraw.Draw(m)
+    for x, y, bw, bh in regions:
+        d.rectangle([(x - grow) * w, (y - grow) * h, (x + bw + grow) * w, (y + bh + grow) * h], fill=255)
+    return m.filter(ImageFilter.GaussianBlur(feather * max(w, h)))
+
+
+def change_mask(orig: Image.Image, edited: Image.Image, near, protect=None) -> Image.Image:
+    """Feathered mask of where `edited` really differs from `orig`: blurred at low
+    resolution so jitter drops out, limited to within NEAR of the `near` regions, and
+    never inside the `protect` regions (the brand's own box)."""
+    w, h = orig.size
+    s = DIFF_SIDE / max(w, h)
+    size = (max(16, int(w * s)), max(16, int(h * s)))
+    a = np.asarray(orig.convert("RGB").resize(size, Image.BILINEAR), np.float64)
+    b = np.asarray(edited.convert("RGB").resize(size, Image.BILINEAR), np.float64)
+    m = gaussian_filter(np.abs(a - b).mean(2), 1.5) > DIFF_THRESH
+    m = binary_opening(m, iterations=1)
+    allow = np.asarray(region_mask(size, near, grow=NEAR, feather=0)) > 0
+    if protect:
+        allow &= ~(np.asarray(region_mask(size, protect, grow=0.0, feather=0)) > 0)
+    m &= allow
+    # A removed object's faint parts (a phone's edge over a keyboard, the thin strokes of a
+    # sign) fall under the threshold and came back as ghosts. So every changed blob is
+    # replaced by its whole bounding box, grown a little, before feathering.
+    labels, n = label(m)
+    full = np.zeros_like(m)
+    pad = max(2, int(BLOB_PAD * max(size)))
+    for sl in find_objects(labels):
+        if sl is None:
+            continue
+        ys, xs = sl
+        full[max(0, ys.start - pad):ys.stop + pad, max(0, xs.start - pad):xs.stop + pad] = True
+    full &= allow
+    mi = Image.fromarray((full * 255).astype(np.uint8)).resize((w, h), Image.BILINEAR)
+    return mi.filter(ImageFilter.GaussianBlur(FEATHER * 0.5 * max(w, h)))
+
+
+def keep_changes(orig: Image.Image, edited: Image.Image, near, protect=None) -> Image.Image:
+    """The original, with only the real changes of the registered edit near `near` pasted in."""
+    orig = orig.convert("RGB")
+    reg = register(orig, edited)
+    return Image.composite(reg, orig, change_mask(orig, reg, near, protect))

backend/deepgaze_runner.py

@@ -252,7 +252,7 @@ def _scanpath(density: np.ndarray, n: int = 5, sigma_frac: float = 0.08) -> list
     return out
 
 
-def predict(image: Image.Image, target_box: Box | None = None) -> dict:
+def predict(image: Image.Image, target_box: Box | None = None, n_thieves: int = 2) -> dict:
     """Full /predict payload for one image. `attention_score` is the size-invariant
     prominence; `target_salience` is the absolute attention mass on target (used by the
     reward-hack guard to tell a real improvement from 'suppress everything else')."""
@@ -268,7 +268,7 @@ def predict(image: Image.Image, target_box: Box | None = None) -> dict:
         "heatmap_png": _heatmap_data_url(density),
         "target_box": [round(v, 4) for v in target],
         "scanpath": _scanpath(density),
-        "distractors": _top_distractors(density, target),
+        "distractors": _top_distractors(density, target, k=n_thieves),
         "engine": _engine,
     }
 

backend/eval_guard.py

@@ -47,6 +47,9 @@
 GLOBAL_COVERAGE = 0.55  # fraction of pixels changed above which an edit looks "global"
 GLOBAL_BBOX = 0.60      # change bounding-box covering more of the frame than this = "global"
 EPS = 1e-4
+# Smallest score rise that counts as a win. Regenerating a region with no visible change
+# moved the score by up to about half a point on the recorded edits, so smaller rises are noise.
+MIN_GAIN = 0.005
 GRID = 8                # cells per side for the rest-of-frame comparison
 OUTSIDE_MARGIN = 0.04   # grow the target box by this before measuring "the rest of the frame"
 # Floors for the rest of the frame, as after/before ratios. Below any of these, the edit
@@ -219,7 +222,7 @@ def verdict(
     """
     pc = perceptual_change(before, after)
     reasons: list[str] = []
-    ratio_gain = (ratio_after - ratio_before) > EPS
+    ratio_gain = (ratio_after - ratio_before) > MIN_GAIN
     have_abs = target_sal_before is not None and target_sal_after is not None
     real_gain = have_abs and (target_sal_after - target_sal_before) > EPS
     suppression = have_abs and ratio_gain and not real_gain