fix: extract Instagram profiles from page preload

packages/db/drizzle/PostgreSQL/meta/_journal.json

@@ -197,6 +197,13 @@
       "when": 1789997848278,
       "tag": "0027_dashboard_query_indexes",
       "breakpoints": true
+    },
+    {
+      "idx": 28,
+      "version": "7",
+      "when": 1790214324454,
+      "tag": "0028_instagram_profile_preload",
+      "breakpoints": true
     }
   ]
-}
+}

packages/db/drizzle/SQLite/meta/_journal.json

@@ -162,6 +162,13 @@
       "when": 1788683877889,
       "tag": "0022_repair_sqlite_job_billing",
       "breakpoints": true
+    },
+    {
+      "idx": 23,
+      "version": "6",
+      "when": 1790214324451,
+      "tag": "0023_instagram_profile_preload",
+      "breakpoints": true
     }
   ]
-}
+}

packages/template-client/src/__tests__/instagram-handler.test.ts

@@ -0,0 +1,43 @@
+import { describe, expect, it, jest } from '@jest/globals';
+import { readFileSync } from 'node:fs';
+import { resolve } from 'node:path';
+
+const source = readFileSync(
+  resolve(process.cwd(), '../../templates/instagram-scraper/requestHandler.js'),
+  'utf8',
+);
+const AsyncFunction = Object.getPrototypeOf(async function () {}).constructor;
+const run = new AsyncFunction('context', source) as (context: unknown) => Promise<any>;
+
+describe('published Instagram handler', () => {
+  it('extracts a public profile from its page preload without waiting for the old API', async () => {
+    const user = {
+      username: 'masonthames', full_name: 'Mason Thames',
+      follower_count: 3306694, following_count: 562,
+      is_verified: true, profile_pic_url: 'https://example.test/avatar.jpg',
+    };
+    const html = `<script type="application/json" data-sjs>${JSON.stringify({
+      require: [{ __bbox: { result: { data: { xig_user_by_username: user } } } }],
+    })}</script>`;
+    const wait = jest.fn();
+
+    const result = await run({
+      request: { url: 'https://www.instagram.com/masonthames/' },
+      html,
+      preNav: { has: async () => false, wait },
+    });
+
+    expect(wait).not.toHaveBeenCalled();
+    expect(result.jsonResult).toEqual([expect.objectContaining({
+      type: 'profile', username: 'masonthames', followers_count: 3306694,
+    })]);
+    expect(result.markdown).toContain('Mason Thames');
+  });
+
+  it('fails when a profile page has no public profile data', async () => {
+    await expect(run({
+      request: { url: 'https://www.instagram.com/masonthames/' },
+      html: '<html></html>',
+    })).rejects.toThrow('No public Instagram data was extracted');
+  });
+});

templates/instagram-scraper/requestHandler.js

@@ -72,6 +72,35 @@ const extractKeyedPayloadFromHtml = (html, targetKey = 'xdt_api__v1__profile_tim
     return timelineItems;
 };
 
+// Logged-out profile pages now preload public profile data under
+// xig_user_by_username instead of requesting web_profile_info.
+const extractProfileFromHtml = (html, username) => {
+    if (typeof html !== 'string' || !html || !username) return null;
+    const scripts = /<script(?=[^>]*type=["']application\/json["'])(?=[^>]*\bdata-sjs\b)[^>]*>([\s\S]*?)<\/script>/gi;
+    let match;
+    while ((match = scripts.exec(html)) !== null) {
+        const raw = match[1] || '';
+        if (!raw.includes('xig_user_by_username') || raw.length > 1_000_000) continue;
+        try {
+            const pending = [JSON.parse(raw)];
+            let visited = 0;
+            while (pending.length && visited++ < 30_000) {
+                const value = pending.pop();
+                if (!value || typeof value !== 'object') continue;
+                const user = value.xig_user_by_username;
+                if (user && typeof user === 'object' &&
+                    String(user.username || '').toLowerCase() === username.toLowerCase()) {
+                    return user;
+                }
+                for (const child of Object.values(value)) {
+                    if (child && typeof child === 'object') pending.push(child);
+                }
+            }
+        } catch (_) { /* Ignore unrelated preloaded scripts. */ }
+    }
+    return null;
+};
+
 const scrapeInstagram = async (context) => {
     const page = context.page;
     const preNav = context.preNav;
@@ -98,6 +127,8 @@ const scrapeInstagram = async (context) => {
     log('URL type:', isInstagramProfileUrl ? 'profile' : isInstagramReelUrl ? 'reel' : 'unknown');
 
     if (isInstagramProfileUrl) {
+        const username = url.match(/instagram\.com\/([\w.\-]+)/i)?.[1] || '';
+        const preloadedUser = extractProfileFromHtml(context.html, username);
         try {
             log('Fetching instagramProfileInfo via preNav...');
             let profileInfo;
@@ -106,7 +137,7 @@ const scrapeInstagram = async (context) => {
                 log('preNav has instagramProfileInfo:', exists);
                 if (exists) {
                     profileInfo = await (preNav.get?.('instagramProfileInfo'));
-                } else {
+                } else if (!preloadedUser) {
                     log('Waiting for instagramProfileInfo (timeout: 30s)...');
                     profileInfo = await (preNav.wait?.('instagramProfileInfo', { timeoutMs: 30000 }));
                 }
@@ -197,6 +228,26 @@ const scrapeInstagram = async (context) => {
             log('Profile extraction error:', err?.message || err);
             console.error('Instagram profile extraction failed:', err?.message || err);
         }
+
+        if (jsonResult.length === 0) {
+            const user = preloadedUser;
+            if (user) {
+                jsonResult.push({
+                    type: 'profile',
+                    username: user.username,
+                    full_name: user.full_name || '',
+                    biography: user.biography || '',
+                    profile_pic_url: user.profile_pic_url || '',
+                    followers_count: user.follower_count ?? null,
+                    following_count: user.following_count ?? null,
+                    posts_count: user.all_media_count ?? null,
+                    is_verified: !!user.is_verified,
+                    is_private: !!user.is_private,
+                    url,
+                });
+                log('Profile extracted from page preload:', user.username);
+            }
+        }
     }
 
     if (isInstagramReelUrl) {
@@ -326,13 +377,19 @@ const scrapeInstagram = async (context) => {
     if (Array.isArray(jsonResult) && jsonResult.length) {
         if (jsonResult.length === 1) {
             const it = jsonResult[0];
-            markdown = `# Instagram Reel ${it.reel_id || ''}\n\n${buildReelMarkdown(it)}`;
+            markdown = it.type === 'profile'
+                ? `# ${it.full_name || it.username} (@${it.username})\n\n${it.biography || ''}\n\n- Followers: ${it.followers_count ?? 'unknown'}\n- Following: ${it.following_count ?? 'unknown'}\n- Posts: ${it.posts_count ?? 'unknown'}\n- Verified: ${it.is_verified ? 'yes' : 'no'}\n- URL: ${url}`
+                : `# Instagram Reel ${it.reel_id || ''}\n\n${buildReelMarkdown(it)}`;
         } else {
             const itemsMd = jsonResult.map(buildReelMarkdown).join('\n\n');
             markdown = `## Reels\n\nLoaded ${jsonResult.length} reel(s) from ${url}\n\n${itemsMd}`;
         }
     }
 
+    if (jsonResult.length === 0) {
+        throw new Error(`No public Instagram data was extracted from ${url}`);
+    }
+
     log('Final result: markdown length:', markdown.length, '| jsonResult count:', jsonResult.length);
     return { markdown, jsonResult };
 };