fix(trending): strip post emoji on the server only (#474)

Author: harlan-zw

layers/marketing/app/components/TrendingPostCard.vue

@@ -1,7 +1,7 @@
 <script setup lang="ts">
 import type { TrendingPost } from '#shared/trending-range'
 import { avatarProxyUrl } from '#shared/image-proxy'
-import { postExcerpt, unstyled } from '#shared/trending-post'
+import { postExcerpt } from '#shared/trending-post'
 
 /**
  * One post, laid out the way people already read posts: who said it, when,
@@ -28,8 +28,6 @@ const LINES = 4
  * mention past this moves the excerpt to it; see `postExcerpt`.
  */
 const segments = computed(() => postExcerpt({ text: post.text, names, budget: LINES * 42 }))
-/** The name in the site's type, like the post under it. A name of emoji alone shows the handle only. */
-const authorName = computed(() => (post.authorName ? unstyled(post.authorName).trim() || null : null))
 const network = computed(() => (post.platform === 'bsky' ? 'Bluesky' : 'X'))
 
 function likesLabel(count: number): string {
@@ -56,7 +54,7 @@ function likesLabel(count: number): string {
         decoding="async"
       >
       <span class="trending-post__byline">
-        <span v-if="authorName" class="trending-post__name">{{ authorName }}</span>
+        <span v-if="post.authorName" class="trending-post__name">{{ post.authorName }}</span>
         <span class="trending-post__handle">@{{ post.handle }}</span>
       </span>
       <span class="trending-post__meta">{{ post.when }}</span>

shared/server/trending-skills.ts

@@ -222,14 +222,44 @@ function compareFit(a: QuoteFit, b: QuoteFit): number {
   return Number(a.automated) - Number(b.automated) || a.breadth - b.breadth
 }
 
+/**
+ * Emoji, with the joiners, variation selectors, skin tones, flag letters and
+ * keycap marks that build them. A keycap keeps its digit.
+ */
+const EMOJI = /[\p{Extended_Pictographic}\p{Emoji_Modifier}\p{Regional_Indicator}\p{Variation_Selector}\p{Enclosing_Mark}\u200D]/gu
+
+/**
+ * Post text as plain letters, without emoji, before it leaves the server.
+ *
+ * A quote is testimony set in the site's type, and emoji rows such as
+ * `🎉 Celebrating 🎉` were the loudest thing in a muted line. `NFKC` folds
+ * the letter styles posts use for emphasis, such as mathematical bold, back
+ * to plain letters.
+ *
+ * Done here, once, and never in the page. Which characters count as emoji
+ * depends on the engine's Unicode tables: Node 24 and Chrome 145 disagree
+ * about `★`. Stripping on both sides rendered different quote text on the
+ * server and in the browser, and every board failed hydration.
+ */
+function plainText(text: string): string {
+  return text
+    .normalize('NFKC')
+    .replace(EMOJI, '')
+    .split('\n')
+    .map(line => line.replace(/[ \t\xA0]+/g, ' ').trim())
+    .join('\n')
+    .trim()
+}
+
 function toEvidence(row: MentionRow): TrendingSkillEvidence {
   return {
     postId: row.post_id,
     url: postUrl(row),
     authorHandle: row.author_handle,
-    authorName: row.author_name ?? null,
+    // A name of emoji alone becomes no name, and the card shows the handle.
+    authorName: (row.author_name && plainText(row.author_name)) || null,
     authorAvatar: row.author_avatar ?? null,
-    text: row.text_extract,
+    text: plainText(row.text_extract),
     postedAt: row.posted_at,
     favouriteCount: row.favourite_count,
     platform: row.platform,

shared/trending-post.ts

@@ -40,34 +40,17 @@ const SHORT_LINK = /\s*\(\s*https?:\/\/t\.co\/\w+\s*\)|\s*(?:[-–—:]\s*)?http
 /** Scheme and `www.` on any other link. The host and path are the useful part. */
 const LINK_PREFIX = /https?:\/\/(?:www\.)?/g
 
-/**
- * Emoji, with the joiners, variation selectors, skin tones, flag letters and
- * keycap marks that build them. A keycap keeps its digit.
- *
- * A quote is testimony set in the site's type. Emoji rows such as
- * `🎉 Celebrating 🎉 📦` read as the poster's styling, and they were the
- * loudest thing in a muted line.
- */
-const EMOJI = /[\p{Extended_Pictographic}\p{Emoji_Modifier}\p{Regional_Indicator}\p{Variation_Selector}\p{Enclosing_Mark}\u200D]/gu
-
 function escapeRegExp(value: string): string {
   return value.replace(/[.*+?^${}()|[\]\\]/g, '\\$&')
 }
 
 /**
- * Text as plain letters, without emoji.
- *
- * `NFKC` folds the Unicode letter styles posts use for emphasis, such as
- * mathematical bold `𝘀𝗵𝗼𝘄`, back to plain letters. Those rendered as bold text
- * in the quote, and a Skill name written in them never matched. Spacing is
- * left to the caller.
+ * No Unicode property escapes or normalization here. Their tables differ
+ * between engines, and this runs on the server and in the browser; emoji
+ * and letter styles leave the post on the server, in `trending-skills.ts`.
  */
-export function unstyled(text: string): string {
-  return text.normalize('NFKC').replace(EMOJI, '')
-}
-
 function tidy(text: string): string {
-  return unstyled(text)
+  return text
     .replace(/\r\n?/g, '\n')
     .replace(SHORT_LINK, '')
     .replace(LINK_PREFIX, '')

test/unit/trending-post.test.ts

@@ -1,7 +1,7 @@
 // @vitest-environment node
 import type { PostSegment } from '../../shared/trending-post'
 import { describe, expect, it } from 'vitest'
-import { postExcerpt, relativeDay, unstyled } from '../../shared/trending-post'
+import { postExcerpt, relativeDay } from '../../shared/trending-post'
 
 function plain(segments: PostSegment[]): string {
   return segments.map(s => s.value).join('')
@@ -86,28 +86,9 @@ describe('postExcerpt', () => {
     expect(text.length).toBeLessThan(80)
   })
 
-  it('drops emoji and keeps the words around them', () => {
-    const excerpt = postExcerpt({
-      text: '🎉 Celebrating 🎉 (500+ new stars) 📦 blader / humanizer ⭐️ 👨‍👩‍👧 🇦🇺',
-      names: [],
-      budget: 200,
-    })
-    expect(plain(excerpt)).toBe('Celebrating (500+ new stars) blader / humanizer')
-  })
-
-  it('keeps the digit of a keycap emoji', () => {
-    const excerpt = postExcerpt({ text: '1️⃣ Install\n2️⃣ Prompt', names: [], budget: 200 })
-    expect(plain(excerpt)).toBe('1 Install\n2 Prompt')
-  })
-
-  it('reads Unicode bold and italic letters as plain text', () => {
-    const excerpt = postExcerpt({
-      text: 'as long as you 𝘀𝗵𝗼𝘄 𝗶𝘁 𝘁𝗼𝗽-𝘁𝗶𝗲𝗿 design with 𝙝𝙪𝙢𝙖𝙣𝙞𝙯𝙚𝙧',
-      names: ['humanizer'],
-      budget: 200,
-    })
-    expect(plain(excerpt)).toBe('as long as you show it top-tier design with humanizer')
-    expect(mentions(excerpt)).toEqual(['humanizer'])
+  it('keeps every character it is given, so the server and the browser cut the same text', () => {
+    const text = '33.5k★ 🎉 𝘀𝗵𝗼𝘄 ☆☆'
+    expect(plain(postExcerpt({ text, names: [], budget: 200 }))).toBe(text)
   })
 
   it('returns the tidied text as one segment when no name matches', () => {
@@ -116,12 +97,6 @@ describe('postExcerpt', () => {
   })
 })
 
-describe('unstyled', () => {
-  it('returns a display name as plain letters without emoji', () => {
-    expect(unstyled('𝗝𝗼𝗵𝗻 ⚡️').trim()).toBe('John')
-  })
-})
-
 describe('relativeDay', () => {
   const NOW = 1_760_000_000
 

test/unit/trending-skills.test.ts

@@ -32,6 +32,7 @@ function mention(input: {
   canonical?: string
   avatar?: string
   name?: string
+  text?: string
 }): string {
   db().raw.prepare(
     `INSERT OR IGNORE INTO repos (owner, repo) VALUES (?, ?)`,
@@ -57,7 +58,7 @@ function mention(input: {
     `a-${input.handle}`,
     input.handle,
     input.name ?? null,
-    `post about /${input.slug}`,
+    input.text ?? `post about /${input.slug}`,
     postedAt,
     postedAt,
     input.likes ?? 1,
@@ -276,6 +277,30 @@ describe('loadTrendingSkills presentation', () => {
   })
 })
 
+describe('loadTrendingSkills post text', () => {
+  it('ships post text and author names as plain letters without emoji', async () => {
+    mention({
+      owner: 'a',
+      repo: 'r',
+      slug: 's',
+      handle: 'styled',
+      name: '𝗔𝗹𝗶 ⚡️',
+      text: '🎉 Celebrating 🎉 /s\n1️⃣ 𝘀𝗵𝗼𝘄 it ⭐️ 👨‍👩‍👧 🇦🇺',
+    })
+
+    const [skill] = await loadTrendingSkills({ db: db().db, now: NOW })
+    expect(skill?.evidence?.text).toBe('Celebrating /s\n1 show it')
+    expect(skill?.evidence?.authorName).toBe('Ali')
+  })
+
+  it('drops a name made of emoji alone', async () => {
+    mention({ owner: 'a', repo: 'r', slug: 's', handle: 'sparkles', name: '✨✨' })
+
+    const [skill] = await loadTrendingSkills({ db: db().db, now: NOW })
+    expect(skill?.evidence?.authorName).toBeNull()
+  })
+})
+
 describe('loadTrendingSkills posts beyond the quote', () => {
   it('carries one post per other author and never repeats the quoted one', async () => {
     mention({ owner: 'a', repo: 'r', slug: 's', handle: 'quoted', likes: 90, name: 'Quoted Dev' })