Add tracker/company reverse lookup, methodology page, sitemap and social metadata

README.md

@@ -16,6 +16,9 @@ The website also includes jurisdiction analysis, showing which companies and cou
 - Detect embedded tracker signatures and declared tracking domains.
 - Store current and historical analysis results.
 - Show tracker, permission, and jurisdiction summaries.
+- Reverse lookup: which apps contain a given tracker or a given company's trackers.
+- An about page explaining how apps are analysed, and what a report does and does not tell you.
+- Sitemap, `robots.txt`, canonical links, and Open Graph/Twitter card metadata.
 - Run the analyser from macOS or a Raspberry Pi host.
 
 Only free App Store apps are queued for analysis. The queue prioritises apps with more stored App Store reviews, then rechecks stale analyses over time.
@@ -34,6 +37,30 @@ public/        Browser assets
 static/        Static image assets
 ```
 
+## Public Pages
+
+| Path | Purpose |
+| --- | --- |
+| `/` | Search and headline statistics |
+| `/analysis/:appId` | Per-app tracker, permission, and jurisdiction report |
+| `/statistics` | Aggregate jurisdiction statistics |
+| `/trackers`, `/companies` | Directories of every tracker and company seen in an analysed app |
+| `/tracker/:slug`, `/company/:slug` | Reverse lookup: the apps a tracker or company was found in |
+| `/about` | How apps are analysed, what a report means, jurisdiction labels, project background and contact |
+| `/sitemap.xml`, `/robots.txt` | Crawler metadata |
+
+The reverse lookup pages are served from an inverted index built by
+`lib/reverseIndex.js` and cached under `CACHE_DIR` alongside the aggregate site
+data. It is rebuilt whenever the set of stored analyses changes, so no extra
+work happens per request.
+
+`CACHE_DIR` is a persistent volume in production, so cache entries outlive the
+code that wrote them. An entry is only rebuilt when the set of stored analyses
+changes, which cannot detect a change in how the cached data is *derived*.
+After editing `buildSiteData` or `buildReverseIndex`, bump `SCHEMA_VERSION` in
+`lib/cache.js` so the deploy discards entries built by the previous logic —
+otherwise the old figures are served until the next analysis lands.
+
 ## Requirements
 
 Website:
@@ -73,6 +100,25 @@ PORT=3000
 `PUBLIC_FORM_BODY_LIMIT` is the smaller limit for the public analysis request form.
 `APP_STORE_CACHE_RETENTION_DAYS` controls how long cached App Store metadata is kept.
 
+Set `SITE_URL` in production to the public origin, for example
+`SITE_URL=https://ios.trackercontrol.org`. Canonical links, Open Graph URLs,
+`robots.txt`, and `sitemap.xml` use it. Without it, those URLs are derived from
+the request in development, which yields `http://` links when TLS is terminated
+by a proxy. It is required in production, so an untrusted Host header cannot
+become a public canonical URL — the server refuses to start without it rather
+than answering 500 on every route.
+
+Rate limits are applied per IP over a five-minute window, and published pages
+are budgeted separately from the App Store entry points because `sitemap.xml`
+points crawlers at every app, tracker and company URL. `RATE_LIMIT_BROWSE_MAX`
+(default 300) covers `GET`/`HEAD` of the published pages, which are served from
+the cached site data. `RATE_LIMIT_FORM_MAX` (default 20) covers everything else,
+including `/search` and `/request/:appId` — these are `GET`s so that Cloudflare
+can challenge them, but each one reaches the App Store, so they are budgeted as
+the form submissions they are rather than as page views. Authenticated analyser
+traffic is exempt from both. `robots.txt` disallows both paths as well, so a
+crawler neither spends App Store calls nor collects challenge interstitials.
+
 ### Bot protection
 
 Everything that costs an App Store call is protected by Cloudflare WAF rules

index.js

@@ -2,6 +2,7 @@
 require('dotenv').config();
 
 const { getOriginSecret } = require('./lib/originGate');
+const siteUrl = require('./lib/siteUrl');
 
 // Load the actual app
 const app = require('./server');
@@ -12,6 +13,15 @@ if (env == 'production') {
   app.set('trust proxy', 1);
   if (!getOriginSecret())
     console.warn('CLOUDFLARE_ORIGIN_SECRET is not set: the origin is trusted to be reachable only through Cloudflare.');
+  // Every route builds canonical/social URLs from this, so a missing value
+  // fails the request rather than degrading it. Refuse to boot instead of
+  // serving 500s from a process the platform considers healthy.
+  try {
+    siteUrl.assertSiteUrlConfiguration();
+  } catch (err) {
+    console.error(`Site URL configuration error: ${err.message}`);
+    throw err;
+  }
 }
 
 // Server express HTTP server

lib/appMetadata.js

@@ -48,4 +48,34 @@ function buildReportMetadata({
   };
 }
 
-module.exports = { buildReportMetadata };
+// Fields the listing pages render. The rest of the queue-time snapshot is
+// carried through untouched so this stays a refresh rather than a projection.
+const LISTING_FIELDS = ['title', 'icon', 'url', 'version', 'primaryGenre', 'reviews'];
+
+/**
+ * Display details for the pages that list many apps at once — the homepage,
+ * the directories, the lookup pages and the sitemap.
+ *
+ * apps.details is the snapshot taken when the app was queued and is never
+ * updated; the App Store cache row is what the metadata jobs refresh. Listing
+ * pages read the snapshot directly, which would leave them showing a title the
+ * report page contradicts. Refreshed values therefore win field by field, so a
+ * new title is not paired with an icon dropped from a partial response.
+ */
+function buildListingDetails({ queueSnapshot = null, storefront = null } = {}) {
+  const current = detailsFromStorefront(storefront) || {};
+  const merged = { ...(queueSnapshot || {}) };
+
+  for (const field of LISTING_FIELDS) {
+    // Tested directly rather than through firstNonEmpty, whose `||` maps a
+    // numeric 0 to null. An app really can hold zero reviews, and that is a
+    // refreshed value like any other.
+    const value = current[field];
+    if (value !== null && value !== undefined && value !== '')
+      merged[field] = value;
+  }
+
+  return merged;
+}
+
+module.exports = { buildReportMetadata, buildListingDetails };

lib/cache.js

@@ -14,14 +14,24 @@ function cachePath(key) {
   return path.join(CACHE_DIR, key + '.json');
 }
 
+// CACHE_DIR is a persistent volume in production, so entries outlive the code
+// that produced them. Callers detect new *data* through their own meta
+// signature, which cannot see a change in how that data is derived: after a
+// deploy that alters buildSiteData or buildReverseIndex, the old entry still
+// matches the signature and would be served indefinitely. Bump this whenever
+// the shape or the derivation of any cached payload changes.
+const SCHEMA_VERSION = 2;
+
 /**
- * Read from cache. Returns { data, meta } or null if no cache exists.
+ * Read from cache. Returns { data, meta } or null if no cache exists or the
+ * entry was written by an incompatible version of the builders.
  */
 function read(key) {
   const file = cachePath(key);
   try {
     const raw = fs.readFileSync(file, 'utf-8');
     const cached = JSON.parse(raw);
+    if (cached._schema !== SCHEMA_VERSION) return null;
     return { data: cached.data, meta: cached._meta || {} };
   } catch (err) {
     return null;
@@ -34,7 +44,7 @@ function read(key) {
 function write(key, data, meta) {
   const file = cachePath(key);
   try {
-    const payload = JSON.stringify({ data, _meta: meta || {} });
+    const payload = JSON.stringify({ data, _meta: meta || {}, _schema: SCHEMA_VERSION });
     fs.writeFileSync(file, payload, 'utf-8');
   } catch (err) {
     console.error('Cache write error:', err.message);
@@ -50,4 +60,4 @@ function invalidate(key) {
   }
 }
 
-module.exports = { read, write, invalidate };
+module.exports = { read, write, invalidate, SCHEMA_VERSION };

lib/jurisdiction.js

@@ -134,6 +134,22 @@ function getCountryName(code) {
   return countryNames[code.toUpperCase()] || code.toUpperCase();
 }
 
+/**
+ * Whether a signature name is an iOS system API rather than a third-party
+ * tracker. These are reported by the analyser but excluded from jurisdiction
+ * analysis.
+ */
+function isSystemSignature(trackerName) {
+  if (!trackerName) return false;
+  return excludedSignatures.has(String(trackerName).toLowerCase().trim());
+}
+
+// The partial match below scans every known company, and the same handful of
+// tracker names recur across thousands of apps, so results are memoised. The
+// company database is static after module load, which makes this safe.
+const resolutionCache = new Map();
+const MAX_RESOLUTION_CACHE_SIZE = 10000;
+
 /**
  * Resolve a tracker name to a company.
  * Tries exact match, then partial/substring match against Xray owner names.
@@ -142,6 +158,16 @@ function resolveTrackerName(trackerName) {
   if (!trackerName) return null;
   const key = trackerName.toLowerCase().trim();
 
+  if (resolutionCache.has(key)) return resolutionCache.get(key);
+
+  const resolved = resolveTrackerNameUncached(key);
+  if (resolutionCache.size >= MAX_RESOLUTION_CACHE_SIZE)
+    resolutionCache.delete(resolutionCache.keys().next().value);
+  resolutionCache.set(key, resolved);
+  return resolved;
+}
+
+function resolveTrackerNameUncached(key) {
   // Skip system APIs that aren't third-party trackers
   if (excludedSignatures.has(key)) return null;
 
@@ -454,6 +480,7 @@ module.exports = {
   europeanAlternatives,
   computeAggregateStats,
   resolveTrackerName,
+  isSystemSignature,
   resolveHost,
   classifyRegion,
   countryFlag,