From bda0501731bb1df3786e10051e564cffb672cb7a Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 11 Aug 2026 08:54:16 +0000 Subject: [PATCH 1/7] Add tracker/company reverse lookup, methodology page, sitemap and social metadata The site could only answer "what is in this app?". Reporters and researchers normally arrive with the opposite question, and with a need to check how a number was produced before quoting it. Reverse lookup: lib/reverseIndex.js builds an inverted tracker -> apps and company -> apps index from the stored analyses, cached and rebuilt on the same signature as the aggregate site data so requests do no extra work. New pages /trackers, /companies, /tracker/:slug and /company/:slug list every tracker and company with the apps they were found in, most reviewed first and paginated. App reports and the statistics tables now link into them. Methodology: /methodology documents the sample (free UK App Store apps, queued on demand and ordered by popularity, so not a random sample), how detection works and what a detection does and does not mean, the jurisdiction classification rules, counting rules, limitations and citation guidance. Discoverability: canonical links, Open Graph and Twitter card metadata per page, with the app icon as the card image on reports; /sitemap.xml re-enabled and extended to the lookup and reference pages with real lastmod timestamps; /robots.txt points at it. SITE_URL pins the public origin behind a TLS proxy. Also memoises jurisdiction.resolveTrackerName, whose substring scan over the company database now runs once per distinct tracker name instead of once per app occurrence. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_011VVgEEw2J1LensebivHrwW --- README.md | 26 +++ lib/jurisdiction.js | 24 +++ lib/reverseIndex.js | 342 +++++++++++++++++++++++++++++++++ public/js/filter.js | 32 ++++ routes/index.js | 394 ++++++++++++++++++++++++++++++++++---- test/lookupPages.test.js | 200 +++++++++++++++++++ test/reverseIndex.test.js | 168 ++++++++++++++++ views/about.pug | 7 +- views/directory.pug | 86 +++++++++ views/form.pug | 26 ++- views/layout.pug | 28 +++ views/lookup.pug | 105 ++++++++++ views/methodology.pug | 139 ++++++++++++++ views/statistics.pug | 26 ++- 14 files changed, 1553 insertions(+), 50 deletions(-) create mode 100644 lib/reverseIndex.js create mode 100644 public/js/filter.js create mode 100644 test/lookupPages.test.js create mode 100644 test/reverseIndex.test.js create mode 100644 views/directory.pug create mode 100644 views/lookup.pug create mode 100644 views/methodology.pug diff --git a/README.md b/README.md index 0e5004c..9d2ca57 100644 --- a/README.md +++ b/README.md @@ -16,6 +16,9 @@ The website also includes jurisdiction analysis, showing which companies and cou - Detect embedded tracker signatures and declared tracking domains. - Store current and historical analysis results. - Show tracker, permission, and jurisdiction summaries. +- Reverse lookup: which apps contain a given tracker or a given company's trackers. +- A methodology page documenting sampling, detection, counting rules, and limitations. +- Sitemap, `robots.txt`, canonical links, and Open Graph/Twitter card metadata. - Run the analyser from macOS or a Raspberry Pi host. Only free App Store apps are queued for analysis. The queue prioritises apps with more stored App Store reviews, then rechecks stale analyses over time. @@ -34,6 +37,24 @@ public/ Browser assets static/ Static image assets ``` +## Public Pages + +| Path | Purpose | +| --- | --- | +| `/` | Search and headline statistics | +| `/analysis/:appId` | Per-app tracker, permission, and jurisdiction report | +| `/statistics` | Aggregate jurisdiction statistics | +| `/trackers`, `/companies` | Directories of every tracker and company seen in an analysed app | +| `/tracker/:slug`, `/company/:slug` | Reverse lookup: the apps a tracker or company was found in | +| `/methodology` | Sampling, detection, counting rules, limitations, and citation guidance | +| `/about` | Project background and contact | +| `/sitemap.xml`, `/robots.txt` | Crawler metadata | + +The reverse lookup pages are served from an inverted index built by +`lib/reverseIndex.js` and cached under `CACHE_DIR` alongside the aggregate site +data. It is rebuilt whenever the set of stored analyses changes, so no extra +work happens per request. + ## Requirements Website: @@ -71,6 +92,11 @@ PORT=3000 `BODY_LIMIT` applies to authenticated analyser JSON and text uploads. `PUBLIC_FORM_BODY_LIMIT` is the smaller limit for the public search form. +Set `SITE_URL` in production to the public origin, for example +`SITE_URL=https://ios.trackercontrol.org`. Canonical links, Open Graph URLs, +`robots.txt`, and `sitemap.xml` use it. Without it, those URLs are derived from +the request, which yields `http://` links when TLS is terminated by a proxy. + Run migrations: ```sh diff --git a/lib/jurisdiction.js b/lib/jurisdiction.js index d82a68a..b1778d6 100644 --- a/lib/jurisdiction.js +++ b/lib/jurisdiction.js @@ -134,6 +134,21 @@ function getCountryName(code) { return countryNames[code.toUpperCase()] || code.toUpperCase(); } +/** + * Whether a signature name is an iOS system API rather than a third-party + * tracker. These are reported by the analyser but excluded from jurisdiction + * analysis. + */ +function isSystemSignature(trackerName) { + if (!trackerName) return false; + return excludedSignatures.has(String(trackerName).toLowerCase().trim()); +} + +// The partial match below scans every known company, and the same handful of +// tracker names recur across thousands of apps, so results are memoised. The +// company database is static after module load, which makes this safe. +const resolutionCache = new Map(); + /** * Resolve a tracker name to a company. * Tries exact match, then partial/substring match against Xray owner names. @@ -142,6 +157,14 @@ function resolveTrackerName(trackerName) { if (!trackerName) return null; const key = trackerName.toLowerCase().trim(); + if (resolutionCache.has(key)) return resolutionCache.get(key); + + const resolved = resolveTrackerNameUncached(key); + resolutionCache.set(key, resolved); + return resolved; +} + +function resolveTrackerNameUncached(key) { // Skip system APIs that aren't third-party trackers if (excludedSignatures.has(key)) return null; @@ -454,6 +477,7 @@ module.exports = { europeanAlternatives, computeAggregateStats, resolveTrackerName, + isSystemSignature, resolveHost, classifyRegion, countryFlag, diff --git a/lib/reverseIndex.js b/lib/reverseIndex.js new file mode 100644 index 0000000..004797c --- /dev/null +++ b/lib/reverseIndex.js @@ -0,0 +1,342 @@ +// Reverse lookup index: tracker -> apps and company -> apps. +// +// The website's per-app reports answer "what is in this app?". Journalists and +// researchers usually arrive with the opposite question: "which apps contain +// this tracker?". This module builds that inverted view once per analysis +// generation so the lookup pages can be served from a cached structure instead +// of scanning every stored analysis per request. +// +// The index is normalised: app metadata is stored once in `apps`, and tracker +// and company entries reference apps by bundle ID. That keeps the cached JSON +// small enough to read cheaply even when every app appears in several lists. +const jurisdiction = require('./jurisdiction'); + +const MAX_SLUG_LENGTH = 80; +// Allows the numeric suffix appended when two names slugify identically. +const MAX_SLUG_LENGTH_WITH_SUFFIX = MAX_SLUG_LENGTH + 8; +const SLUG_PATTERN = /^[a-z0-9][a-z0-9-]*$/; + +/** + * Build a URL-safe slug for a tracker or company name. + * Names contain spaces, dots and other punctuation ("Mob.com", "Unity3d Ads"), + * so the slug is lossy; collisions are resolved when slugs are assigned. + */ +function slugify(name) { + const slug = String(name == null ? '' : name) + .toLowerCase() + .replace(/[^a-z0-9]+/g, '-') + .replace(/^-+|-+$/g, '') + .slice(0, MAX_SLUG_LENGTH) + .replace(/-+$/, ''); + return slug || 'unnamed'; +} + +/** + * Whether a value can be a slug at all. Route handlers check this before + * touching the index so arbitrary path segments never reach a property lookup. + */ +function isValidSlug(value) { + return typeof value === 'string' + && value.length > 0 + && value.length <= MAX_SLUG_LENGTH_WITH_SUFFIX + && SLUG_PATTERN.test(value); +} + +function reviewCount(details) { + const raw = details && details.reviews; + const parsed = typeof raw === 'number' ? raw : parseInt(raw, 10); + return Number.isFinite(parsed) && parsed > 0 ? parsed : 0; +} + +function percentage(count, total) { + return total > 0 ? (count / total * 100).toFixed(1) : '0.0'; +} + +function createTrackerEntry(displayName) { + const resolved = jurisdiction.resolveTrackerName(displayName); + const company = resolved ? jurisdiction.getUltimateParent(resolved) : null; + const country = resolved ? jurisdiction.getUltimateCountry(resolved) : null; + + return { + name: displayName, + // System APIs are reported by the analyser but are not third-party + // trackers, and jurisdiction analysis excludes them. Flagging them keeps + // the directory honest rather than listing them as unattributed trackers. + system: jurisdiction.isSystemSignature(displayName), + company: company || null, + country: company ? country : null, + countryName: company ? jurisdiction.getCountryName(country) : null, + flag: company ? jurisdiction.countryFlag(country) : '', + region: company ? jurisdiction.classifyRegion(country) : 'Unresolved', + appIds: [] + }; +} + +function compareNames(a, b) { + const left = a.toLowerCase(); + const right = b.toLowerCase(); + if (left < right) return -1; + if (left > right) return 1; + if (a < b) return -1; + if (a > b) return 1; + return 0; +} + +/** + * Assign a unique slug to every entry. Entries are slugged in name order so + * that a collision resolves to the same slug on every rebuild. + */ +function assignSlugs(entries) { + const used = new Set(); + for (const entry of [...entries].sort((a, b) => compareNames(a.name, b.name))) { + const base = slugify(entry.name); + let slug = base; + let suffix = 2; + while (used.has(slug)) { + slug = `${base}-${suffix}`; + suffix++; + } + used.add(slug); + entry.slug = slug; + } +} + +/** + * Build the reverse index from the rows returned by Apps.getAllApps(). + * + * `totalApps` counts every successfully analysed app, including apps where no + * tracker was detected, so it matches the denominator used by the aggregate + * jurisdiction statistics and can be quoted as "N of M apps". + */ +function buildReverseIndex(allApps) { + const apps = {}; + const trackerEntries = new Map(); // lowercased tracker name -> entry + let totalApps = 0; + let trackedApps = 0; + let latestAnalysis = null; + + for (const app of allApps || []) { + const analysis = app && app.analysis; + if (!app.appid || !analysis || analysis.success === false) continue; + + totalApps++; + + const analysedAt = app.analysed ? new Date(app.analysed) : null; + const analysedValid = analysedAt && !Number.isNaN(analysedAt.getTime()); + if (analysedValid && (!latestAnalysis || analysedAt > latestAnalysis)) + latestAnalysis = analysedAt; + + const trackerNames = analysis.trackers ? Object.keys(analysis.trackers) : []; + if (trackerNames.length > 0) trackedApps++; + + const details = app.details || {}; + const analysed = jurisdiction.analyseApp(analysis); + + // Every analysed app enters the directory, including apps with no detected + // tracker: the lookup pages only reference the tracked ones, but the + // sitemap covers all of them. + apps[app.appid] = { + appid: app.appid, + title: details.title || app.appid, + icon: details.icon || null, + category: details.primaryGenre || null, + reviews: reviewCount(details), + trackerCount: trackerNames.length, + classification: analysed.classification, + analysed: analysedValid ? analysedAt.toISOString() : null + }; + + for (const trackerName of trackerNames) { + const key = String(trackerName).toLowerCase().trim(); + if (!key) continue; + + let entry = trackerEntries.get(key); + if (!entry) { + entry = createTrackerEntry(trackerName); + trackerEntries.set(key, entry); + } + + // Two tracker names in the same app can normalise to one key; count the + // app once. + if (entry.appIds[entry.appIds.length - 1] !== app.appid) + entry.appIds.push(app.appid); + } + } + + // Trackers resolve to a company deterministically, so companies are derived + // from the tracker entries rather than recomputed from each app. + const companyEntries = new Map(); + for (const entry of trackerEntries.values()) { + if (!entry.company) continue; + + const key = entry.company.toLowerCase(); + let company = companyEntries.get(key); + if (!company) { + company = { + name: entry.company, + country: entry.country, + countryName: entry.countryName, + flag: entry.flag, + region: entry.region, + trackers: [], + appIdSet: new Set() + }; + companyEntries.set(key, company); + } + company.trackers.push(entry); + for (const appid of entry.appIds) company.appIdSet.add(appid); + } + + const trackerArray = [...trackerEntries.values()]; + const companyArray = [...companyEntries.values()]; + assignSlugs(trackerArray); + assignSlugs(companyArray); + + const companySlugByName = new Map(); + for (const company of companyArray) + companySlugByName.set(company.name.toLowerCase(), company.slug); + + // Most popular app first: the apps a reader recognises are the ones worth + // showing on page one. + const byPopularity = (a, b) => + apps[b].reviews - apps[a].reviews + || compareNames(apps[a].title, apps[b].title) + || compareNames(a, b); + + const trackers = {}; + for (const entry of trackerArray) { + entry.appIds.sort(byPopularity); + trackers[entry.slug] = { + slug: entry.slug, + name: entry.name, + system: entry.system, + company: entry.company, + companySlug: entry.company + ? companySlugByName.get(entry.company.toLowerCase()) || null + : null, + country: entry.country, + countryName: entry.countryName, + flag: entry.flag, + region: entry.region, + appCount: entry.appIds.length, + pct: percentage(entry.appIds.length, totalApps), + appIds: entry.appIds + }; + } + + const companies = {}; + for (const company of companyArray) { + const appIds = [...company.appIdSet].sort(byPopularity); + companies[company.slug] = { + slug: company.slug, + name: company.name, + country: company.country, + countryName: company.countryName, + flag: company.flag, + region: company.region, + appCount: appIds.length, + pct: percentage(appIds.length, totalApps), + appIds, + trackers: company.trackers + .map((entry) => ({ + name: entry.name, + slug: entry.slug, + appCount: entry.appIds.length + })) + .sort((a, b) => b.appCount - a.appCount || compareNames(a.name, b.name)) + }; + } + + const byPrevalence = (source) => (a, b) => + source[b].appCount - source[a].appCount || compareNames(source[a].name, source[b].name); + + // Name -> slug maps let other pages (app reports, statistics tables) link + // into the lookup pages without re-deriving a slug that may have been + // deduplicated here. + const trackerSlugs = {}; + for (const entry of trackerArray) trackerSlugs[entry.name.toLowerCase()] = entry.slug; + const companySlugs = {}; + for (const entry of companyArray) companySlugs[entry.name.toLowerCase()] = entry.slug; + + return { + trackerSlugs, + companySlugs, + totalApps, + trackedApps, + latestAnalysis: latestAnalysis ? latestAnalysis.toISOString() : null, + apps, + trackers, + trackerList: Object.keys(trackers).sort(byPrevalence(trackers)), + companies, + companyList: Object.keys(companies).sort(byPrevalence(companies)) + }; +} + +/** + * Look up an entry by slug. Uses an own-property check so that slugs like + * "constructor" cannot reach inherited properties. + */ +function lookup(collection, slug) { + if (!collection || !isValidSlug(slug)) return null; + return Object.prototype.hasOwnProperty.call(collection, slug) + ? collection[slug] + : null; +} + +function lookupTracker(index, slug) { + return index ? lookup(index.trackers, slug) : null; +} + +function lookupCompany(index, slug) { + return index ? lookup(index.companies, slug) : null; +} + +/** + * Slug for a tracker or company name, or null when the name does not appear in + * any analysed app. + */ +function slugForName(slugMap, name) { + if (!slugMap || !name) return null; + const key = String(name).toLowerCase(); + return Object.prototype.hasOwnProperty.call(slugMap, key) ? slugMap[key] : null; +} + +/** + * Resolve a page of app IDs into app records, dropping any ID that is missing + * from the directory. + */ +function paginate(appIds, appDirectory, page, perPage) { + const items = appIds || []; + const totalPages = Math.max(1, Math.ceil(items.length / perPage)); + const currentPage = Math.min(Math.max(1, page), totalPages); + const start = (currentPage - 1) * perPage; + + return { + page: currentPage, + totalPages, + perPage, + total: items.length, + from: items.length === 0 ? 0 : start + 1, + to: Math.min(start + perPage, items.length), + apps: items + .slice(start, start + perPage) + .map((appid) => appDirectory[appid]) + .filter(Boolean) + }; +} + +function parsePage(value) { + const parsed = parseInt(value, 10); + return Number.isFinite(parsed) && parsed > 0 ? parsed : 1; +} + +module.exports = { + slugify, + isValidSlug, + buildReverseIndex, + lookupTracker, + lookupCompany, + slugForName, + paginate, + parsePage +}; diff --git a/public/js/filter.js b/public/js/filter.js new file mode 100644 index 0000000..bbd0ec5 --- /dev/null +++ b/public/js/filter.js @@ -0,0 +1,32 @@ +// Client-side filter for the tracker and company directories. The tables are +// rendered in full, so filtering is a local narrowing of what is already on the +// page and needs no requests. +(function () { + 'use strict'; + + var input = document.getElementById('directory-filter'); + if (!input) return; + + var table = document.querySelector(input.getAttribute('data-filter-target')); + if (!table) return; + + var rows = Array.prototype.slice.call(table.querySelectorAll('tbody tr')); + var empty = document.getElementById('directory-empty'); + + function apply() { + var term = input.value.trim().toLowerCase(); + var visible = 0; + + rows.forEach(function (row) { + var haystack = row.getAttribute('data-filter') || ''; + var matches = term === '' || haystack.indexOf(term) !== -1; + row.style.display = matches ? '' : 'none'; + if (matches) visible++; + }); + + if (empty) empty.classList.toggle('d-none', visible > 0); + } + + input.addEventListener('input', apply); + apply(); +})(); diff --git a/routes/index.js b/routes/index.js index 5fb6110..e5efadf 100644 --- a/routes/index.js +++ b/routes/index.js @@ -5,9 +5,14 @@ const store = require('../lib/appStore'); const Apps = require('../models/Apps'); const jurisdiction = require('../lib/jurisdiction'); const cache = require('../lib/cache'); +const reverseIndex = require('../lib/reverseIndex'); const { isValidAppId } = require('../lib/appId'); const { classifyAnalysisFailure } = require('../lib/analysisFailure'); const asyncHandler = require('../lib/asyncHandler'); +const { + CURRENT_ANALYSIS_VERSION, + STALE_ANALYSIS_DAYS +} = require('../lib/analysisPolicy'); // Taken from https://reports.exodus-privacy.eu.org/api/trackers const exodusTrackers = JSON.parse(fs.readFileSync('./exodusTrackers.json', 'utf-8')) @@ -18,6 +23,11 @@ for (const [key, value] of Object.entries(exodusTrackers.trackers)) const router = express.Router(); const COUNTRY = 'gb'; +const SITE_NAME = 'TrackerControl for iOS'; +const DEFAULT_DESCRIPTION = 'Find out which trackers are embedded in iOS apps, ' + + 'which companies control them, and under which jurisdiction that tracking falls.'; +const APPS_PER_PAGE = 50; + let lastPing = 0; // unix timestamp // ping from analyser in past hour? @@ -26,6 +36,31 @@ router.use(function (req, res, next) { next(); }); +/** + * Absolute base URL of this deployment, used for canonical links, social card + * metadata and the sitemap. SITE_URL pins it when the site runs behind a proxy + * that terminates TLS, where req.protocol would otherwise report http. + */ +function siteBaseUrl(req) { + const configured = process.env.SITE_URL; + if (configured) return configured.trim().replace(/\/+$/, ''); + return `${req.protocol}://${req.get('host')}`; +} + +// Social card and canonical link defaults. Individual routes override these +// with page-specific values by passing them to res.render. +router.use(function (req, res, next) { + const base = siteBaseUrl(req); + const path = req.path.length > 1 ? req.path.replace(/\/+$/, '') : req.path; + + res.locals.siteName = SITE_NAME; + res.locals.siteBaseUrl = base; + res.locals.canonicalUrl = base + path; + res.locals.pageDescription = DEFAULT_DESCRIPTION; + res.locals.ogImage = null; + next(); +}); + /** * Build all homepage + statistics data from DB. * Returns { homepage, statistics, appCount }. @@ -121,6 +156,16 @@ function buildSiteData(allApps) { }; } +/** + * Whether cached data was built from the same set of analyses as the database + * currently holds. + */ +function signatureMatches(meta, signature) { + return Boolean(meta) + && meta.appCount === signature.appCount + && meta.latestAnalysis === signature.latestAnalysis; +} + /** * Get site data: serve from cache if app count hasn't changed, otherwise rebuild. * Falls back to stale cache on any DB error. @@ -130,10 +175,7 @@ async function getSiteData() { try { const signature = await Apps.getSiteDataSignature(); - if (cached - && cached.meta - && cached.meta.appCount === signature.appCount - && cached.meta.latestAnalysis === signature.latestAnalysis) { + if (cached && signatureMatches(cached.meta, signature)) { return cached.data; } @@ -151,12 +193,71 @@ async function getSiteData() { } } +// The reverse index is large compared with the aggregate site data, so it is +// kept in its own cache entry and only touched by the lookup pages and the +// sitemap. The in-process copy avoids re-parsing the cache file per request. +let reverseIndexMemo = null; // { meta, index } + +/** + * Get the tracker/company reverse index, rebuilding it when new analyses have + * landed. Falls back to the last known index if the database is unavailable. + */ +async function getReverseIndex() { + try { + const signature = await Apps.getSiteDataSignature(); + + if (reverseIndexMemo && signatureMatches(reverseIndexMemo.meta, signature)) + return reverseIndexMemo.index; + + const cached = cache.read('reverseindex'); + if (cached && signatureMatches(cached.meta, signature)) { + reverseIndexMemo = { meta: cached.meta, index: cached.data }; + return cached.data; + } + + const allApps = await Apps.getAllApps(); + const index = reverseIndex.buildReverseIndex(allApps); + if (index.totalApps > 0) { + cache.write('reverseindex', index, signature); + reverseIndexMemo = { meta: signature, index }; + console.log('Reverse index rebuilt for', index.totalApps, 'apps'); + } + return index; + } catch (err) { + console.error('DB error in getReverseIndex:', err.message); + if (reverseIndexMemo) return reverseIndexMemo.index; + const cached = cache.read('reverseindex'); + if (cached) return cached.data; + throw err; + } +} + +function invalidateSiteCaches() { + cache.invalidate('sitedata'); + cache.invalidate('reverseindex'); + reverseIndexMemo = null; +} + +const EMPTY_REVERSE_INDEX = { + totalApps: 0, + trackedApps: 0, + latestAnalysis: null, + apps: {}, + trackers: {}, + trackerList: [], + companies: {}, + companyList: [] +}; + router.get('/', asyncHandler(async (req, res) => { try { const data = await getSiteData(); return res.render('form', { title: 'App Privacy Checker', data: req.body, + pageDescription: `Search ${data.headlines.totalApps} analysed iOS apps to see ` + + 'which trackers they embed, which companies control them, and under ' + + 'which jurisdiction that tracking falls.', headlines: data.headlines, appsWithMostTrackers: data.appsWithMostTrackers, jurisdictionStats: data.jurisdictionStats, @@ -175,17 +276,45 @@ router.get('/', asyncHandler(async (req, res) => { } })); +/** + * Attach reverse-lookup slugs to the statistics tables so every tracker and + * company in them links to the apps it was found in. + */ +function withLookupSlugs(data, index) { + const trackers = (data.topTrackersEnriched || []).map((tracker) => ({ + ...tracker, + slug: reverseIndex.slugForName(index.trackerSlugs, tracker.name), + companySlug: reverseIndex.slugForName(index.companySlugs, tracker.company) + })); + const companies = (data.jurisdictionStats && data.jurisdictionStats.topCompaniesSorted || []) + .map((company) => ({ + ...company, + slug: reverseIndex.slugForName(index.companySlugs, company.name) + })); + + return { + topTrackersEnriched: trackers, + jurisdictionStats: { ...data.jurisdictionStats, topCompaniesSorted: companies } + }; +} + // Statistics detail page router.get('/statistics', asyncHandler(async (req, res) => { try { const data = await getSiteData(); + const index = await getReverseIndex(); + const linked = withLookupSlugs(data, index); + return res.render('statistics', { title: 'Detailed Statistics', data: req.body, + pageDescription: `Tracking jurisdiction across ${data.headlines.totalApps} ` + + 'analysed iOS apps: the most prevalent trackers, the companies behind ' + + 'them, and how they break down by country and App Store category.', headlines: data.headlines, - jurisdictionStats: data.jurisdictionStats, + jurisdictionStats: linked.jurisdictionStats, jurisdictionMeta: jurisdiction.classificationMeta, - topTrackersEnriched: data.topTrackersEnriched, + topTrackersEnriched: linked.topTrackersEnriched, europeanAlternatives: jurisdiction.europeanAlternatives, xrayCompanyCount: jurisdiction.xrayCompanyCount }); @@ -319,22 +448,167 @@ router.get('/analysis/:appId', asyncHandler(async (req, res) => { jurisdictionData.sovereigntyNote = jurisdiction.sovereigntyNotes[jurisdictionData.classification]; } + // Slugs let each detected tracker and company link through to the other apps + // that share it. + let trackerSlugs = {}; + let companySlugs = {}; + try { + const index = await getReverseIndex(); + trackerSlugs = index.trackerSlugs || {}; + companySlugs = index.companySlugs || {}; + } catch (err) { + console.error('Tracker links unavailable:', err.message); + } + + const trackerCount = app.analysis && app.analysis.trackers && app.analysis.success !== false + ? Object.keys(app.analysis.trackers).length + : null; + const pageDescription = trackerCount === null + ? `Tracker analysis of ${app.details.title} for iOS.` + : `${trackerCount === 0 ? 'No trackers were' : `${trackerCount} tracker${trackerCount === 1 ? ' was' : 's were'}`}` + + ` detected in ${app.details.title} for iOS` + + (jurisdictionData && jurisdictionData.meta ? `: ${jurisdictionData.meta.label.toLowerCase()}.` : '.'); + res.render('form', { title: app.details.title, data: req.body, app: app, trackerNameToExodus: trackerNameToExodus, - jurisdictionData: jurisdictionData + trackerSlugs: trackerSlugs, + companySlugs: companySlugs, + jurisdictionData: jurisdictionData, + pageDescription, + ogImage: app.details.icon || null }); })); // About page router.get('/about', (req, res) => { res.render('about', { - title: 'About' + title: 'About', + pageDescription: 'How this service analyses iOS apps for embedded trackers, ' + + 'who runs it, and how to get in touch.' }); }); +// Methodology page: how the numbers on this site are produced, and what they +// do and do not support. Written for reporters and researchers who need to +// check a claim before publishing it. +router.get('/methodology', asyncHandler(async (req, res) => { + let headlines = null; + try { + headlines = (await getSiteData()).headlines; + } catch (err) { + console.error('Methodology page stats unavailable:', err.message); + } + + res.render('methodology', { + title: 'Methodology', + pageDescription: 'How TrackerControl for iOS detects trackers, how apps are ' + + 'sampled, what the jurisdiction classifications mean, and the limits of ' + + 'the data.', + headlines, + analysisVersion: CURRENT_ANALYSIS_VERSION, + staleAnalysisDays: STALE_ANALYSIS_DAYS, + xrayCompanyCount: jurisdiction.xrayCompanyCount, + jurisdictionMeta: jurisdiction.classificationMeta + }); +})); + +/** + * Render a directory of every tracker or company seen in an analysed app. + */ +function renderDirectory(kind) { + return asyncHandler(async (req, res) => { + let index; + try { + index = await getReverseIndex(); + } catch (err) { + console.error('Directory error:', err.message); + index = EMPTY_REVERSE_INDEX; + } + + const isTracker = kind === 'tracker'; + const entries = isTracker + ? index.trackerList.map((slug) => index.trackers[slug]) + : index.companyList.map((slug) => index.companies[slug]); + + res.render('directory', { + title: isTracker ? 'Tracker directory' : 'Company directory', + kind, + entries, + totalApps: index.totalApps, + trackedApps: index.trackedApps, + latestAnalysis: index.latestAnalysis, + pageDescription: isTracker + ? `Every tracker detected across ${index.totalApps} analysed iOS apps, ` + + 'with the company and country behind it.' + : `Every company whose tracking code was detected across ${index.totalApps} ` + + 'analysed iOS apps, ranked by how many apps they reach.' + }); + }); +} + +router.get('/trackers', renderDirectory('tracker')); +router.get('/companies', renderDirectory('company')); + +/** + * Reverse lookup: the apps in which a given tracker, or any tracker belonging + * to a given company, was detected. + */ +function renderLookup(kind) { + return asyncHandler(async (req, res) => { + const isTracker = kind === 'tracker'; + let index; + try { + index = await getReverseIndex(); + } catch (err) { + console.error('Lookup error:', err.message); + return res.status(503).send('Lookup data is temporarily unavailable. Please try again later.'); + } + + const entry = isTracker + ? reverseIndex.lookupTracker(index, req.params.slug) + : reverseIndex.lookupCompany(index, req.params.slug); + + if (!entry) { + return res.status(404).send(isTracker + ? 'Unknown tracker. See /trackers for the full list.' + : 'Unknown company. See /companies for the full list.'); + } + + const pagination = reverseIndex.paginate( + entry.appIds, + index.apps, + reverseIndex.parsePage(req.query.page), + APPS_PER_PAGE + ); + + const attribution = entry.company || (isTracker ? null : entry.name); + const description = `${entry.name} was detected in ${entry.appCount} of ` + + `${index.totalApps} analysed iOS apps (${entry.pct}%)` + + (attribution && entry.countryName ? `. Operated by ${attribution} (${entry.countryName}).` : '.'); + + res.render('lookup', { + title: entry.name, + kind, + entry, + pagination, + totalApps: index.totalApps, + latestAnalysis: index.latestAnalysis, + jurisdictionMeta: jurisdiction.classificationMeta, + exodus: isTracker ? trackerNameToExodus[entry.name] : null, + companySlug: isTracker ? entry.companySlug : null, + pageDescription: description, + canonicalUrl: res.locals.canonicalUrl + + (pagination.page > 1 ? `?page=${pagination.page}` : '') + }); + }); +} + +router.get('/tracker/:slug', renderLookup('tracker')); +router.get('/company/:slug', renderLookup('company')); + // serve next task to analyser router.get('/queue', asyncHandler(async (req, res) => { const requestedAppId = req.query.appId || null; @@ -384,7 +658,7 @@ router.post('/uploadAnalysis', asyncHandler(async (req, res) => { if (result.rowCount === 0) return res.status(409).send('Analysis claim is no longer active.'); - cache.invalidate('sitedata'); + invalidateSiteCaches(); res.json({ ok: true }); })); @@ -413,36 +687,82 @@ router.post('/reportAnalysisFailure', asyncHandler(async (req, res) => { if (result.rowCount === 0) return res.status(409).send('Analysis claim is no longer active.'); - cache.invalidate('sitedata'); + invalidateSiteCaches(); res.json({ ok: true }); })); -/*router.get('/sitemap.xml', async (req, res) => { - try { - const apps = await Apps.getAllApps(); - - let sitemap = ` -`; - - for (const app of apps) { - sitemap += ` - - ${req.protocol}://${req.get('host')}/analysis/${app.appid} - ${new Date().toISOString()} - daily - 0.8 - `; - } - - sitemap += ` -`; - - res.header('Content-Type', 'application/xml'); - res.send(sitemap); - } catch (err) { - console.error('Error generating sitemap:', err); - res.status(500).send('Error generating sitemap'); - } -});*/ +function escapeXml(value) { + return String(value) + .replace(/&/g, '&') + .replace(//g, '>') + .replace(/"/g, '"') + .replace(/'/g, '''); +} + +function sitemapEntry(base, path, { lastmod, changefreq, priority } = {}) { + const parts = [` ${escapeXml(base + path)}`]; + if (lastmod) parts.push(` ${escapeXml(lastmod)}`); + if (changefreq) parts.push(` ${changefreq}`); + if (priority) parts.push(` ${priority}`); + return ` \n${parts.join('\n')}\n `; +} + +// Sitemap over the report, lookup and reference pages. Built from the cached +// reverse index so a crawl does not read every stored analysis from the +// database. +router.get('/sitemap.xml', asyncHandler(async (req, res) => { + const base = siteBaseUrl(req); + + let index; + try { + index = await getReverseIndex(); + } catch (err) { + console.error('Sitemap error:', err.message); + index = EMPTY_REVERSE_INDEX; + } + + const updated = index.latestAnalysis; + const entries = [ + sitemapEntry(base, '/', { lastmod: updated, changefreq: 'daily', priority: '1.0' }), + sitemapEntry(base, '/statistics', { lastmod: updated, changefreq: 'daily', priority: '0.9' }), + sitemapEntry(base, '/trackers', { lastmod: updated, changefreq: 'daily', priority: '0.9' }), + sitemapEntry(base, '/companies', { lastmod: updated, changefreq: 'daily', priority: '0.8' }), + sitemapEntry(base, '/methodology', { changefreq: 'monthly', priority: '0.7' }), + sitemapEntry(base, '/about', { changefreq: 'monthly', priority: '0.5' }) + ]; + + for (const slug of index.trackerList) + entries.push(sitemapEntry(base, `/tracker/${slug}`, { lastmod: updated, changefreq: 'weekly', priority: '0.7' })); + + for (const slug of index.companyList) + entries.push(sitemapEntry(base, `/company/${slug}`, { lastmod: updated, changefreq: 'weekly', priority: '0.6' })); + + for (const app of Object.values(index.apps)) + entries.push(sitemapEntry(base, `/analysis/${app.appid}`, { + lastmod: app.analysed, + changefreq: 'monthly', + priority: '0.6' + })); + + res.header('Content-Type', 'application/xml'); + res.send(` + +${entries.join('\n')} +`); +})); + +router.get('/robots.txt', (req, res) => { + res.type('text/plain').send([ + 'User-agent: *', + 'Allow: /', + 'Disallow: /queue', + 'Disallow: /ping', + 'Disallow: /healthz', + '', + `Sitemap: ${siteBaseUrl(req)}/sitemap.xml`, + '' + ].join('\n')); +}); module.exports = router; // make accessible to /app.js diff --git a/test/lookupPages.test.js b/test/lookupPages.test.js new file mode 100644 index 0000000..2171b8b --- /dev/null +++ b/test/lookupPages.test.js @@ -0,0 +1,200 @@ +'use strict'; + +const assert = require('node:assert/strict'); +const test = require('node:test'); +const fs = require('node:fs'); +const os = require('node:os'); +const path = require('node:path'); + +// Must be set before the cache and server modules are loaded: the cache +// directory is resolved at require time. +process.env.CACHE_DIR = fs.mkdtempSync(path.join(os.tmpdir(), 'tc-cache-')); +process.env.UPLOAD_PASSWORD = 'test-secret'; +process.env.SITE_URL = 'https://example.test'; + +const Apps = require('../models/Apps'); +const app = require('../server'); + +const analysed = new Date('2025-01-02T03:04:05.000Z'); + +const corpus = [ + { + appid: 'com.example.two', + analysed, + details: { + appId: 'com.example.two', + title: 'Example Two', + icon: 'https://icons.test/two.png', + version: '2.0', + url: 'https://apps.apple.com/two', + reviews: 9000, + primaryGenre: 'News', + free: true + }, + analysis: { trackers: { 'Google Firebase Analytics': {} }, permissions: ['Camera'] } + }, + { + appid: 'com.example.one', + analysed, + details: { + appId: 'com.example.one', + title: 'Example One', + icon: 'https://icons.test/one.png', + version: '1.0', + url: 'https://apps.apple.com/one', + reviews: 500, + primaryGenre: 'Games', + free: true + }, + analysis: { trackers: { 'Google Firebase Analytics': {}, 'Facebook Login': {} } } + } +]; + +async function withServer(run) { + const server = await new Promise((resolve) => { + const instance = app.listen(0, '127.0.0.1', () => resolve(instance)); + }); + + try { + await run(`http://127.0.0.1:${server.address().port}`); + } finally { + await new Promise((resolve, reject) => { + server.close((err) => err ? reject(err) : resolve()); + }); + } +} + +function stubDatabase() { + const original = { + getSiteDataSignature: Apps.getSiteDataSignature, + getAllApps: Apps.getAllApps, + findApp: Apps.findApp, + log: console.log + }; + + Apps.getSiteDataSignature = async () => ({ + appCount: corpus.length, + latestAnalysis: analysed.toISOString() + }); + Apps.getAllApps = async () => corpus; + Apps.findApp = async (appId) => + corpus.find((row) => row.appid.toLowerCase() === String(appId).toLowerCase()) || null; + console.log = () => {}; + + return () => { + Apps.getSiteDataSignature = original.getSiteDataSignature; + Apps.getAllApps = original.getAllApps; + Apps.findApp = original.findApp; + console.log = original.log; + }; +} + +test('reverse lookup, methodology, sitemap and social metadata', async (t) => { + const restore = stubDatabase(); + + try { + await withServer(async (base) => { + await t.test('tracker directory lists trackers and links to lookups', async () => { + const response = await fetch(`${base}/trackers`); + const body = await response.text(); + + assert.equal(response.status, 200); + assert.match(body, /Google Firebase Analytics/); + assert.match(body, /href="\/tracker\/google-firebase-analytics"/); + assert.match(body, /href="\/company\/alphabet"/); + }); + + await t.test('company directory renders', async () => { + const response = await fetch(`${base}/companies`); + const body = await response.text(); + + assert.equal(response.status, 200); + assert.match(body, /href="\/company\/alphabet"/); + }); + + await t.test('tracker page lists the apps, most reviewed first', async () => { + const response = await fetch(`${base}/tracker/google-firebase-analytics`); + const body = await response.text(); + + assert.equal(response.status, 200); + assert.match(body, /href="\/analysis\/com.example.two"/); + assert.match(body, /href="\/analysis\/com.example.one"/); + assert.ok( + body.indexOf('Example Two') < body.indexOf('Example One'), + 'the app with more reviews should come first' + ); + assert.match(body, //); + assert.match(body, /detected in 2 of 2 analysed iOS apps/); + }); + + await t.test('company page aggregates its trackers', async () => { + const response = await fetch(`${base}/company/alphabet`); + const body = await response.text(); + + assert.equal(response.status, 200); + assert.match(body, /href="\/tracker\/google-firebase-analytics"/); + assert.match(body, /href="\/analysis\/com.example.two"/); + }); + + await t.test('unknown slugs 404 instead of erroring', async () => { + for (const url of ['/tracker/no-such-tracker', '/company/no-such-company', '/tracker/__proto__']) { + const response = await fetch(`${base}${url}`); + assert.equal(response.status, 404, url); + } + }); + + await t.test('methodology page renders with sampling caveats', async () => { + const response = await fetch(`${base}/methodology`); + const body = await response.text(); + + assert.equal(response.status, 200); + assert.match(body, /This is not a random sample of the App Store/); + assert.match(body, //); + }); + + await t.test('app report links trackers to their lookup page and sets a social image', async () => { + const response = await fetch(`${base}/analysis/com.example.one`); + const body = await response.text(); + + assert.equal(response.status, 200); + assert.match(body, /href="\/tracker\/google-firebase-analytics"/); + assert.match(body, //); + assert.match(body, /2 trackers were detected in Example One/); + }); + + await t.test('statistics page links its tables into the lookup pages', async () => { + const response = await fetch(`${base}/statistics`); + const body = await response.text(); + + assert.equal(response.status, 200); + assert.match(body, /href="\/tracker\/google-firebase-analytics"/); + assert.match(body, /href="\/company\/alphabet"/); + }); + + await t.test('sitemap covers reports, lookups and reference pages', async () => { + const response = await fetch(`${base}/sitemap.xml`); + const body = await response.text(); + + assert.equal(response.status, 200); + assert.match(response.headers.get('content-type'), /xml/); + assert.match(body, /https:\/\/example.test\/<\/loc>/); + assert.match(body, /https:\/\/example.test\/methodology<\/loc>/); + assert.match(body, /https:\/\/example.test\/tracker\/google-firebase-analytics<\/loc>/); + assert.match(body, /https:\/\/example.test\/company\/alphabet<\/loc>/); + assert.match(body, /https:\/\/example.test\/analysis\/com.example.one<\/loc>/); + assert.match(body, new RegExp(`${analysed.toISOString()}`)); + }); + + await t.test('robots.txt points at the sitemap', async () => { + const response = await fetch(`${base}/robots.txt`); + const body = await response.text(); + + assert.equal(response.status, 200); + assert.match(body, /Sitemap: https:\/\/example.test\/sitemap.xml/); + }); + }); + } finally { + restore(); + fs.rmSync(process.env.CACHE_DIR, { recursive: true, force: true }); + } +}); diff --git a/test/reverseIndex.test.js b/test/reverseIndex.test.js new file mode 100644 index 0000000..6c6d50f --- /dev/null +++ b/test/reverseIndex.test.js @@ -0,0 +1,168 @@ +'use strict'; + +const assert = require('node:assert/strict'); +const test = require('node:test'); +const reverseIndex = require('../lib/reverseIndex'); + +function app(appid, { trackers, title, reviews, analysed, success, genre } = {}) { + return { + appid, + analysed: analysed || '2025-01-01T00:00:00.000Z', + details: { + appId: appid, + title: title || appid, + icon: `https://example.test/${appid}.png`, + reviews: reviews === undefined ? 0 : reviews, + primaryGenre: genre || 'Utilities' + }, + analysis: success === false + ? { success: false, reason: 'app_not_found' } + : { trackers: (trackers || []).reduce((acc, name) => ({ ...acc, [name]: {} }), {}) } + }; +} + +const corpus = [ + app('com.example.one', { trackers: ['Google Firebase Analytics', 'Facebook Login'], reviews: 500, title: 'One' }), + app('com.example.two', { trackers: ['Google Firebase Analytics'], reviews: 9000, title: 'Two' }), + app('com.example.three', { trackers: [], reviews: 10, title: 'Three' }), + app('com.example.four', { trackers: ['Google Firebase Analytics'], success: false, title: 'Four' }) +]; + +test('index counts apps per tracker and uses all analysed apps as denominator', () => { + const index = reverseIndex.buildReverseIndex(corpus); + + // The failed analysis is excluded; the tracker-free app still counts. + assert.equal(index.totalApps, 3); + assert.equal(index.trackedApps, 2); + + const slug = index.trackerSlugs['google firebase analytics']; + const firebase = reverseIndex.lookupTracker(index, slug); + assert.equal(firebase.appCount, 2); + assert.equal(firebase.pct, '66.7'); + assert.ok(!firebase.appIds.includes('com.example.four')); +}); + +test('apps within a tracker are ordered by review count', () => { + const index = reverseIndex.buildReverseIndex(corpus); + const firebase = reverseIndex.lookupTracker(index, index.trackerSlugs['google firebase analytics']); + + assert.deepEqual(firebase.appIds, ['com.example.two', 'com.example.one']); +}); + +test('the app directory covers analysed apps without trackers', () => { + const index = reverseIndex.buildReverseIndex(corpus); + + assert.ok(index.apps['com.example.three']); + assert.equal(index.apps['com.example.three'].trackerCount, 0); + assert.equal(index.apps['com.example.three'].classification, 'no_tracking'); + assert.ok(!index.apps['com.example.four'], 'failed analyses stay out of the directory'); +}); + +test('trackers are grouped under the company that owns them', () => { + const index = reverseIndex.buildReverseIndex(corpus); + const firebase = reverseIndex.lookupTracker(index, index.trackerSlugs['google firebase analytics']); + + assert.equal(firebase.company, 'Alphabet'); + assert.equal(firebase.region, 'US'); + + const company = reverseIndex.lookupCompany(index, firebase.companySlug); + assert.equal(company.name, 'Alphabet'); + assert.equal(company.appCount, 2); + assert.ok(company.trackers.some((tracker) => tracker.name === 'Google Firebase Analytics')); +}); + +test('an app is counted once per company even with several of its trackers', () => { + const index = reverseIndex.buildReverseIndex([ + app('com.example.multi', { trackers: ['Google Firebase Analytics', 'Google AdMob'], reviews: 1 }) + ]); + + const firebase = reverseIndex.lookupTracker(index, index.trackerSlugs['google firebase analytics']); + const company = reverseIndex.lookupCompany(index, firebase.companySlug); + + assert.equal(company.appCount, 1); + assert.equal(company.trackers.length, 2); +}); + +test('system APIs are flagged rather than listed as unattributed trackers', () => { + const index = reverseIndex.buildReverseIndex([ + app('com.example.system', { trackers: ['AdID access'], reviews: 1 }) + ]); + + const entry = reverseIndex.lookupTracker(index, index.trackerSlugs['adid access']); + assert.equal(entry.system, true); + assert.equal(entry.company, null); + assert.equal(entry.region, 'Unresolved'); +}); + +test('slugs are URL-safe and collisions get distinct slugs', () => { + assert.equal(reverseIndex.slugify('Mob.com'), 'mob-com'); + assert.equal(reverseIndex.slugify('Unity3d Ads'), 'unity3d-ads'); + assert.equal(reverseIndex.slugify('!!!'), 'unnamed'); + + const index = reverseIndex.buildReverseIndex([ + app('com.example.collide', { trackers: ['Mob.com', 'Mob com'], reviews: 1 }) + ]); + + const slugs = index.trackerList.slice().sort(); + assert.deepEqual(slugs, ['mob-com', 'mob-com-2']); + for (const slug of slugs) assert.ok(reverseIndex.lookupTracker(index, slug)); +}); + +test('lookup rejects invalid slugs and inherited properties', () => { + const index = reverseIndex.buildReverseIndex(corpus); + + assert.equal(reverseIndex.lookupTracker(index, 'constructor'), null); + assert.equal(reverseIndex.lookupTracker(index, '__proto__'), null); + assert.equal(reverseIndex.lookupTracker(index, '../../etc/passwd'), null); + assert.equal(reverseIndex.lookupTracker(index, ''), null); + assert.equal(reverseIndex.isValidSlug('Mob-Com'), false); + assert.equal(reverseIndex.isValidSlug('mob-com'), true); +}); + +test('the index survives a JSON cache round-trip', () => { + const index = JSON.parse(JSON.stringify(reverseIndex.buildReverseIndex(corpus))); + const firebase = reverseIndex.lookupTracker(index, index.trackerSlugs['google firebase analytics']); + + assert.equal(firebase.appCount, 2); + assert.equal(index.trackerList[0], firebase.slug); +}); + +test('pagination clamps the page number and resolves app records', () => { + const index = reverseIndex.buildReverseIndex(corpus); + const firebase = reverseIndex.lookupTracker(index, index.trackerSlugs['google firebase analytics']); + + const first = reverseIndex.paginate(firebase.appIds, index.apps, 1, 1); + assert.equal(first.totalPages, 2); + assert.equal(first.from, 1); + assert.equal(first.to, 1); + assert.equal(first.apps[0].title, 'Two'); + + const clamped = reverseIndex.paginate(firebase.appIds, index.apps, 99, 1); + assert.equal(clamped.page, 2); + assert.equal(clamped.apps[0].title, 'One'); + + const empty = reverseIndex.paginate([], index.apps, 1, 50); + assert.equal(empty.total, 0); + assert.equal(empty.from, 0); + assert.equal(empty.apps.length, 0); +}); + +test('parsePage falls back to the first page for junk input', () => { + assert.equal(reverseIndex.parsePage('3'), 3); + assert.equal(reverseIndex.parsePage('0'), 1); + assert.equal(reverseIndex.parsePage('-2'), 1); + assert.equal(reverseIndex.parsePage('abc'), 1); + assert.equal(reverseIndex.parsePage(undefined), 1); +}); + +test('slugForName only resolves names that are present', () => { + const index = reverseIndex.buildReverseIndex(corpus); + + assert.equal( + reverseIndex.slugForName(index.trackerSlugs, 'GOOGLE FIREBASE ANALYTICS'), + index.trackerSlugs['google firebase analytics'] + ); + assert.equal(reverseIndex.slugForName(index.trackerSlugs, 'Not A Tracker'), null); + assert.equal(reverseIndex.slugForName(index.trackerSlugs, 'toString'), null); + assert.equal(reverseIndex.slugForName(index.trackerSlugs, null), null); +}); diff --git a/views/about.pug b/views/about.pug index 959b514..7876d61 100644 --- a/views/about.pug +++ b/views/about.pug @@ -8,9 +8,10 @@ block content | . A tracker is a piece of software that collects data about b you or your app usage behaviour | . - p This project was motivated by - a(target='_blank' rel='noopener noreferrer' href='https://exodus-privacy.eu.org/') Exodus Privacy - | , which is a similar project for Android apps. Some of the underlying code as well as the design of this website is based on this project. + p + | For a full account of how apps are sampled, how trackers are detected, what the results do and do not support, and how to cite them, see the #[a(href='/methodology') Methodology] page. + p + | This project was motivated by #[a(target='_blank' rel='noopener noreferrer' href='https://exodus-privacy.eu.org/') Exodus Privacy], which is a similar project for Android apps. Some of the underlying code as well as the design of this website is based on this project. p The underlying analysis technique for iOS apps was developed in the PhD research of Konrad Kollnig at the Department of Computer Science of the University of Oxford. | This research was published in a range of academic papers and is available at a(target='_blank' rel='noopener noreferrer' href='https://www.platformcontrol.org/') PlatformControl.org diff --git a/views/directory.pug b/views/directory.pug new file mode 100644 index 0000000..ba4a584 --- /dev/null +++ b/views/directory.pug @@ -0,0 +1,86 @@ +extends layout + +block content + - const isTracker = kind === 'tracker' + h2= isTracker ? 'Tracker directory' : 'Company directory' + + if entries && entries.length > 0 + p.text-muted + if isTracker + | Every tracker signature detected across #{totalApps} analysed iOS apps. Select one to see the apps it was found in. + else + | Every company whose tracking code was detected across #{totalApps} analysed iOS apps, ranked by reach. Select one to see the apps it reaches. + p.text-muted.small + | #{trackedApps} of #{totalApps} analysed apps contain at least one detected tracker. + if latestAnalysis + | Last analysis: #{new Date(latestAnalysis).toLocaleDateString('en-GB', {day:'numeric', month:'short', year:'numeric'})}. + | See #[a(href='/methodology') how these numbers are produced]. + + .form-group.mt-3 + input#directory-filter.form-control( + type='search' + placeholder=isTracker ? 'Filter trackers or companies' : 'Filter companies' + aria-label='Filter table' + data-filter-target='#directory-table') + + .table-responsive + table#directory-table.table.table-sm.table-striped + thead + tr + th # + th= isTracker ? 'Tracker' : 'Company' + if isTracker + th Company + else + th Trackers + th Country + th Region + th Apps + th % of apps + tbody + each entry, index in entries + - const haystack = `${entry.name} ${isTracker ? (entry.company || '') : ''} ${entry.countryName || ''}`.toLowerCase() + tr(data-filter=haystack) + td= index + 1 + td.font-weight-bold + a.report-link(href=`/${kind}/${entry.slug}`)= entry.name + if isTracker && entry.system + =" " + span.badge.badge-pill.region-unresolved System API + if isTracker + td + if entry.company && entry.companySlug + a.report-link(href=`/company/${entry.companySlug}`)= entry.company + else if entry.company + = entry.company + else if entry.system + span.text-muted Apple system API + else + span.text-muted Unattributed + else + td= entry.trackers.length + td + if entry.countryName + span #{entry.flag} #{entry.countryName} + else + span.text-muted Unknown + td + span.badge.badge-pill(class=`region-${entry.region.toLowerCase()}`) #{entry.region} + td= entry.appCount + td #{entry.pct}% + + p#directory-empty.text-muted.d-none No entries match that filter. + + p.text-muted.small.mt-4 + if isTracker + | A tracker is counted once per app, regardless of how many times its code appears. "System API" marks Apple platform APIs that the analyser reports but that are not third-party trackers; "Unattributed" marks signatures that could not be mapped to a company. + else + | Companies are counted at the level of the ultimate parent, so subsidiaries are grouped under their owner. An app is counted once per company even when several of that company's trackers are present. + else + .alert.alert-warning(role='alert') + p Directory data is not available right now. Please try again later. + +block app + +block scripts + script(src='/js/filter.js') diff --git a/views/form.pug b/views/form.pug index d31c1e8..86ccfc9 100644 --- a/views/form.pug +++ b/views/form.pug @@ -57,7 +57,8 @@ block content div.text-muted.small Analysed #{new Date(app.analysed).toLocaleDateString('en-GB', {day:'numeric', month:'short', year:'numeric'})} p.text-center.mt-4 - a.btn.btn-outline-primary(href='/statistics') View detailed statistics → + a.btn.btn-outline-primary.mr-2.mb-2(href='/statistics') View detailed statistics → + a.btn.btn-outline-primary.mb-2(href='/trackers') Look up a tracker → if errors && errors.length > 0 .alert.alert-warning(role='alert') @@ -125,8 +126,11 @@ block app - const jdMap = {} - if (jurisdictionData && jurisdictionData.trackerDetails) { jurisdictionData.trackerDetails.forEach(d => { jdMap[d.name] = d }) } for tracker in Object.keys(app.analysis.trackers) + - const trackerSlug = trackerSlugs ? trackerSlugs[tracker.toLowerCase()] : null p.mb-0 - if tracker in trackerNameToExodus + if trackerSlug + a.link.black(href=`/tracker/${trackerSlug}`) #{tracker} + else if tracker in trackerNameToExodus a.link.black(target='_blank' rel='noopener noreferrer' href=`https://reports.exodus-privacy.eu.org/en/trackers/${trackerNameToExodus[tracker].id}/`) #{tracker} else | #{tracker} @@ -142,8 +146,10 @@ block app =" " else span.text-muted No further information available for this tracker. - | A tracker is a piece of software meant to collect data about you or your usages. + | A tracker is a piece of software meant to collect data about you or your usages. a(target='_blank' rel='noopener noreferrer' href='https://reports.exodus-privacy.eu.org/en/info/trackers/') Learn more... + p.text-muted.small.mt-2 + | Select a tracker to see every other analysed app it was found in. Detection means the tracker's code is present in this app, not that data was observed leaving it — see #[a(href='/methodology') Methodology]. if jurisdictionData .row.justify-content-sm-center.mb-5 @@ -177,9 +183,19 @@ block app th Region tbody each detail in jurisdictionData.trackerDetails + - const detailTrackerSlug = trackerSlugs ? trackerSlugs[detail.name.toLowerCase()] : null + - const detailCompanySlug = detail.company && companySlugs ? companySlugs[detail.company.toLowerCase()] : null tr - td= detail.name - td= detail.company || 'Unknown' + td + if detailTrackerSlug + a.report-link(href=`/tracker/${detailTrackerSlug}`)= detail.name + else + = detail.name + td + if detailCompanySlug + a.report-link(href=`/company/${detailCompanySlug}`)= detail.company + else + = detail.company || 'Unknown' td if detail.flag span #{detail.flag} #{detail.countryName} diff --git a/views/layout.pug b/views/layout.pug index dc07aa6..cd20d1a 100644 --- a/views/layout.pug +++ b/views/layout.pug @@ -4,6 +4,28 @@ html meta(charset='utf-8') meta(name='viewport' content='width=device-width, initial-scale=1, shrink-to-fit=no') title= `${title} | TrackerControl for iOS` + + if pageDescription + meta(name='description' content=pageDescription) + if canonicalUrl + link(rel='canonical' href=canonicalUrl) + + meta(property='og:site_name' content=siteName || 'TrackerControl for iOS') + meta(property='og:type' content=ogType || 'website') + meta(property='og:title' content=ogTitle || title) + if pageDescription + meta(property='og:description' content=pageDescription) + if canonicalUrl + meta(property='og:url' content=canonicalUrl) + if ogImage + meta(property='og:image' content=ogImage) + meta(name='twitter:card' content='summary') + meta(name='twitter:title' content=ogTitle || title) + if pageDescription + meta(name='twitter:description' content=pageDescription) + if ogImage + meta(name='twitter:image' content=ogImage) + link(rel='stylesheet', href='/css/bootstrap.min.css') link(rel='stylesheet', href='/css/exodus.css') link(rel='stylesheet', href='/css/styles.css') @@ -25,8 +47,12 @@ html ul.navbar-nav.mr-auto.mt-2.mt-lg-0 li.nav-item.mr-xl-3.ml-xl-3.mr-lg-2.ml-lg-2 a.nav-link(href='/') Home + li.nav-item.mr-xl-3.ml-xl-3.mr-lg-2.ml-lg-2 + a.nav-link(href='/trackers') Trackers li.nav-item.mr-xl-3.ml-xl-3.mr-lg-2.ml-lg-2 a.nav-link(href='/statistics') Statistics + li.nav-item.mr-xl-3.ml-xl-3.mr-lg-2.ml-lg-2 + a.nav-link(href='/methodology') Methodology li.nav-item.mr-xl-3.ml-xl-3.mr-lg-2.ml-lg-2 a.nav-link(href='/about') About @@ -50,3 +76,5 @@ html script(src='/js/jquery-3.4.1.slim.min.js') script(src='/js/popper.min.js') script(src='/js/bootstrap.min.js') + + block scripts diff --git a/views/lookup.pug b/views/lookup.pug new file mode 100644 index 0000000..6271dac --- /dev/null +++ b/views/lookup.pug @@ -0,0 +1,105 @@ +extends layout + +block content + - const isTracker = kind === 'tracker' + + h2= entry.name + p + if entry.countryName + span.badge.badge-pill(class=`region-${entry.region.toLowerCase()}`) #{entry.region} + =" " + span #{entry.flag} #{entry.countryName} + else if isTracker && entry.system + span.badge.badge-pill.region-unresolved System API + else + span.badge.badge-pill.region-unresolved Unattributed + if isTracker && entry.company + if entry.companySlug + | Operated by #[a.report-link(href=`/company/${entry.companySlug}`)= entry.company]. + else + | Operated by #{entry.company}. + + .jumbotron-stats.mb-4 + .row.text-center + .col-md-6.mb-3 + .stat-number= entry.appCount + .stat-label= entry.appCount === 1 ? 'app contains it' : 'apps contain it' + .col-md-6.mb-3 + .stat-number #{entry.pct}% + .stat-label of analysed apps + .stat-base.text-muted of #{totalApps} apps analysed + + if isTracker && entry.system + .alert.alert-info(role='alert') + p.mb-0 + | This signature identifies an Apple system API rather than a third-party tracker. It is reported for completeness and is excluded from the jurisdiction analysis. + + if isTracker && !entry.company && !entry.system + .alert.alert-warning(role='alert') + p.mb-0 + | This tracker could not be matched to a company in the tracker database, so no jurisdiction is shown for it. + + if !isTracker && entry.trackers && entry.trackers.length > 0 + h5 Trackers operated by #{entry.name} + p + each tracker in entry.trackers + a.badge.badge-pill.badge-outline-primary.analytics.mr-1(href=`/tracker/${tracker.slug}`) + | #{tracker.name} (#{tracker.appCount}) + p.text-muted.small Numbers in brackets are the apps each tracker was detected in. + + if isTracker && exodus + p + a.link(target='_blank' rel='noopener noreferrer' href=`https://reports.exodus-privacy.eu.org/en/trackers/${exodus.id}/`) Tracker profile on Exodus Privacy + if exodus.categories && exodus.categories.length > 0 + | + each category in exodus.categories + span.badge.badge-pill.badge-outline-primary.analytics.ml-1 #{category} + + h3.mt-4 Apps + p.text-muted.small + if pagination.total > 0 + | Showing #{pagination.from}–#{pagination.to} of #{pagination.total} apps, most reviewed first. + else + | No apps to show. + + .container + each app in pagination.apps + .row.position-relative.mb-2 + .col-3.col-sm-2.col-md-2.my-auto + if app.icon + img.rounded(src=`${app.icon}` width='50' height='50' alt=`${app.title}`) + .col-9.col-sm-10.col-md-10.text-truncate.position-static + div + a.stretched-link.report-link(href=`/analysis/${app.appid}`)= app.title + div + span.badge.badge-pill.badge-danger.reports #{app.trackerCount} + | trackers + - const meta = jurisdictionMeta[app.classification] + if meta + span.badge.badge-pill.ml-1(class=meta.cssClass) !{meta.icon} #{meta.label} + .text-muted.small + if app.category + span #{app.category} + if app.analysed + | · analysed #{new Date(app.analysed).toLocaleDateString('en-GB', {day:'numeric', month:'short', year:'numeric'})} + + if pagination.totalPages > 1 + nav.mt-4(aria-label='App list pages') + ul.pagination.justify-content-center + li.page-item(class=pagination.page <= 1 ? 'disabled' : '') + a.page-link(href=`?page=${pagination.page - 1}` rel='prev') Previous + li.page-item.disabled + span.page-link Page #{pagination.page} of #{pagination.totalPages} + li.page-item(class=pagination.page >= pagination.totalPages ? 'disabled' : '') + a.page-link(href=`?page=${pagination.page + 1}` rel='next') Next + + p.text-muted.small.mt-4 + | Detection means the tracker's code signature was found in the app binary. It does not by itself prove that data was transmitted while the app was in use. Percentages are shares of the #{totalApps} apps analysed so far, which are not a random sample of the App Store — see #[a(href='/methodology') Methodology] before quoting these figures. + if latestAnalysis + | Data as of #{new Date(latestAnalysis).toLocaleDateString('en-GB', {day:'numeric', month:'short', year:'numeric'})}. + + p.mt-3 + a.btn.btn-outline-primary(href=isTracker ? '/trackers' : '/companies') + | ← All #{isTracker ? 'trackers' : 'companies'} + +block app diff --git a/views/methodology.pug b/views/methodology.pug new file mode 100644 index 0000000..ac1f633 --- /dev/null +++ b/views/methodology.pug @@ -0,0 +1,139 @@ +extends layout + +block content + h2 Methodology + p.text-muted + | How the figures on this site are produced, what they support, and where they stop. Written for journalists, researchers and anyone checking a claim before publishing it. + if headlines && headlines.latestAnalysis + | Data as of #{new Date(headlines.latestAnalysis).toLocaleDateString('en-GB', {day:'numeric', month:'short', year:'numeric'})}. + + .card.mb-4 + .card-body + h4.card-title In one paragraph + p.mb-0 + | We download free iOS apps from the UK App Store, install each one on a physical iPhone, and list the third-party tracking libraries whose code is present in the app. We then map each tracker to the company that operates it, and that company to the country whose law it answers to. The result says what tracking code an app ships with, and who controls it. It does not say what data an app sent while you were using it. + + h3.mt-5 1. Which apps are analysed + p + | The sample covers #[b free apps on the UK App Store]. Paid apps are never queued, and results reflect the UK storefront; app binaries and bundled SDKs can differ by country. + ul + li An app enters the queue when someone looks it up on this site, so the sample follows public interest rather than a sampling frame. + li The queue is ordered by the number of App Store reviews stored for an app, so widely used apps are analysed first, then by the order in which apps were added. + li Apps are re-analysed when the analysis pipeline is upgraded, or when their last result is older than #{staleAnalysisDays} days. Results currently shown come from analysis version #{analysisVersion} or earlier. + .alert.alert-warning(role='alert') + p.mb-0 + | #[b This is not a random sample of the App Store.] It is demand-driven and skewed towards popular apps. A percentage on this site describes #[i the apps analysed so far], not "iOS apps" in general. A phrase such as "of the + if headlines && headlines.totalApps + | #{headlines.totalApps} + | apps analysed by TrackerControl" keeps that distinction visible. + + h3.mt-5 2. How apps are obtained + p + | Apps are downloaded through Apple's own distribution channel with a regular App Store account and installed on a real iPhone. Apple encrypts App Store binaries with its FairPlay DRM; #[b this project does not circumvent that DRM], because doing so may be unlawful in some jurisdictions. Analysis instead runs on the device, where the app is decrypted by the operating system in the normal course of being launched. + p + | Avoiding jailbreaks and DRM circumvention is what makes this analysis repeatable by others. The technique was published in the #[a(target='_blank' rel='noopener noreferrer' href='https://petsymposium.org/popets/2022/popets-2022-0033.pdf') Proceedings on Privacy Enhancing Technologies] in 2022. + + h3.mt-5 3. How trackers are detected + p + | Detection is #[b static]: it looks at what code is inside the app, not at network traffic. On the device, the analyser enumerates the classes contained in the installed app and matches them against a curated set of tracker signatures. A signature is a set of class names specific to one vendor's SDK, for example a Firebase, Braze or PubMatic class prefix. + p + | Signatures are only added when the class names are vendor-specific and are corroborated either by tracking domains the SDK is known to contact or by repeated, low-noise evidence across the app corpus. This is deliberately conservative: it prefers missing a tracker to inventing one. + p Two further pieces of evidence are collected per app: + ul + li #[b Tracking domains]: the domains an app itself declares in Apple's privacy manifests as being used for tracking. These are the developer's own declarations, not measured traffic. + li #[b Permissions]: the permission usage descriptions in the app's Info.plist, including those of its app extensions. This shows what an app is #[i able] to request, not what it requests or receives. + + .alert.alert-info(role='alert') + p #[b What a detection means:] the tracker's code ships inside the app. + p.mb-0 + | #[b What it does not mean:] that the tracker ran, that it collected anything, or that data left the device during use. Code can sit dormant, be gated behind a consent prompt, or be reached only in a region or account state we did not exercise. Conversely, absence of a detection is not proof of no tracking: first-party and server-side tracking are invisible to this method, and an SDK with no signature yet will not be counted. + + h3.mt-5 4. How jurisdiction is determined + p + | Each detected tracker is matched to the company that operates it, using the Xray tracker database (#{xrayCompanyCount} tracker companies) together with a small set of manual aliases for iOS SDK names that do not match a company name directly. Ownership is then followed to the #[b ultimate parent], so subsidiaries are counted under the group that owns them, and the parent's home country is used. + p Countries are grouped into regions: US, European (EU, EEA and the UK), CN, and Other. Each app then receives one classification: + .table-responsive + table.table.table-sm.table-striped + thead + tr + th Classification + th Applies when + tbody + tr + td + span.badge.badge-pill(class=jurisdictionMeta.no_tracking.cssClass) !{jurisdictionMeta.no_tracking.icon} #{jurisdictionMeta.no_tracking.label} + td No third-party tracker signature was detected. + tr + td + span.badge.badge-pill(class=jurisdictionMeta.us_only.cssClass) !{jurisdictionMeta.us_only.icon} #{jurisdictionMeta.us_only.label} + td Every identified tracker company is headquartered in the US. + tr + td + span.badge.badge-pill(class=jurisdictionMeta.european_only.cssClass) !{jurisdictionMeta.european_only.icon} #{jurisdictionMeta.european_only.label} + td Every identified tracker company is headquartered in the EU, EEA or UK. + tr + td + span.badge.badge-pill(class=jurisdictionMeta.mixed_with_us_cn.cssClass) !{jurisdictionMeta.mixed_with_us_cn.icon} #{jurisdictionMeta.mixed_with_us_cn.label} + td Identified companies include both US and Chinese ones. + tr + td + span.badge.badge-pill(class=jurisdictionMeta.mixed_with_us.cssClass) !{jurisdictionMeta.mixed_with_us.icon} #{jurisdictionMeta.mixed_with_us.label} + td Identified companies include US ones alongside others, but not Chinese ones. + tr + td + span.badge.badge-pill(class=jurisdictionMeta.mixed_no_us.cssClass) !{jurisdictionMeta.mixed_no_us.icon} #{jurisdictionMeta.mixed_no_us.label} + td Identified companies are outside the US and span more than one region. + tr + td + span.badge.badge-pill(class=jurisdictionMeta.unresolved_only.cssClass) !{jurisdictionMeta.unresolved_only.icon} #{jurisdictionMeta.unresolved_only.label} + td Trackers were detected, but none could be matched to a company. + p + | Jurisdiction here means #[b legal control over the company], not the physical location of a server. A US-headquartered company can be compelled to disclose data under the US CLOUD Act wherever that data is stored, which is why the classification follows corporate control rather than hosting. + + h3.mt-5 5. Counting rules + ul + li A tracker is counted once per app, however many times its code appears. + li A company is counted once per app, even when several of its trackers are present. + li Percentages on tracker and company pages use all successfully analysed apps as the denominator, including apps where no tracker was found. + li Apple system APIs that the analyser reports, such as advertising-identifier access, are labelled as system APIs and excluded from the jurisdiction analysis. + + h3.mt-5 6. Limitations + ul + li #[b Static analysis only.] Presence of code is not evidence of transmission, and this site publishes no traffic measurements. + li #[b Incomplete signatures.] Trackers without a signature are missed entirely. New SDKs and renamed classes lag behind. + li #[b Name matching can misattribute.] Tracker names are matched to companies partly by substring, which can attach a tracker to the wrong company where names overlap. Attribution for an individual tracker should be spot-checked before it carries weight in a story. + li #[b Ownership data ages.] Acquisitions change who ultimately controls a tracker, and the company database may lag behind the latest deal. + li #[b Results age.] An app's report reflects the version analysed on the date shown, which may not be the version in the store today. Results older than #{staleAnalysisDays} days are re-queued, so a report can lag a recent app update. + li #[b Storefront and price limits.] UK storefront, free apps only. Paid apps, enterprise apps and apps unavailable in the UK are out of scope. + li #[b Unresolved trackers.] Some detected signatures cannot be mapped to any company. Apps whose trackers are all unresolved are reported separately rather than folded into a jurisdiction. + + h3.mt-5 7. Using this data in reporting + p Claims the data supports: + ul + li The code of a named tracker is present in a named app, as analysed on the date and app version shown on its report page. + li Of the apps analysed by TrackerControl for iOS, a given number contain a tracker operated by a given company. + li Every tracker company identified in a given app is headquartered in the US. + p Claims the data does not support: + ul + li "This app sent your data to company Z." Detection shows shipped code, not observed transmission. + li "N% of iOS apps do X." The sample is demand-driven and skewed towards popular apps. + li "This app does not track you", on the basis of an empty result. Undetected and first-party tracking remain possible. + p + | #[b Before publishing:] check the analysis date and app version on the report page, confirm that the version you are writing about is the one analysed, and put the finding to the developer for comment. If a result looks wrong, tell us — corrections improve the signature set for everyone. + + h3.mt-5 8. Verifying and reproducing + ul + li #[b Request an analysis.] Searching for an app on this site queues it; the report page shows its position in the queue. + li #[b Read the code.] The website and the analyser pipeline are open source under the AGPLv3 at #[a(target='_blank' rel='noopener noreferrer' href='https://github.com/TrackerControl/tracker-control-ios') github.com/TrackerControl/tracker-control-ios], including the tracker signature sets and the jurisdiction mapping. + li #[b Read the research.] The underlying technique and its validation are documented at #[a(target='_blank' rel='noopener noreferrer' href='https://www.platformcontrol.org/') PlatformControl.org]. + li #[b Ask us.] For verification requests, methodology questions or corrections, write to #[a(href='mailto:ios@trackercontrol.org') ios@trackercontrol.org]. Please include the app, the date of the report, and the claim you intend to publish. + + h3.mt-5 9. Citing this site + p Please cite the specific page and the date you consulted it, because reports change as apps are re-analysed. For example: + blockquote.blockquote + p.mb-0.small + | TrackerControl for iOS, "Tracker report: Example App", consulted #{new Date().toLocaleDateString('en-GB', {day:'numeric', month:'long', year:'numeric'})}. + p.text-muted.small + | The analysis is research output of Konrad Kollnig (Maastricht University), building on work at the University of Oxford. See #[a(href='/about') About] for the people and projects behind it. + +block app diff --git a/views/statistics.pug b/views/statistics.pug index 8756332..5efb8be 100644 --- a/views/statistics.pug +++ b/views/statistics.pug @@ -44,8 +44,16 @@ block content each tracker, index in topTrackersEnriched tr td= index + 1 - td.font-weight-bold= tracker.name - td= tracker.company + td.font-weight-bold + if tracker.slug + a.report-link(href=`/tracker/${tracker.slug}`)= tracker.name + else + = tracker.name + td + if tracker.companySlug + a.report-link(href=`/company/${tracker.companySlug}`)= tracker.company + else + = tracker.company td #{tracker.flag} #{tracker.countryName} td span.badge.badge-pill(class=`region-${tracker.region.toLowerCase()}`) #{tracker.region} @@ -70,7 +78,11 @@ block content each company, index in jurisdictionStats.topCompaniesSorted tr td= index + 1 - td.font-weight-bold= company.name + td.font-weight-bold + if company.slug + a.report-link(href=`/company/${company.slug}`)= company.name + else + = company.name td #{company.flag} #{company.countryName} td span.badge.badge-pill(class=`region-${company.region.toLowerCase()}`) #{company.region} @@ -130,10 +142,14 @@ block content else span.badge.badge-pill.jurisdiction-us None + p.mt-4 + a.btn.btn-outline-primary.mr-2(href='/trackers') Browse all trackers → + a.btn.btn-outline-primary(href='/companies') Browse all companies → + p.text-muted small | Based on Xray Tracker Database (#{xrayCompanyCount} tracker companies). - | - a(href='/about') About this analysis + | + a(href='/methodology') How these numbers are produced block app From 104811db9d51f1b3abff08ed4efd9723ed0dbf28 Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 11 Aug 2026 08:57:17 +0000 Subject: [PATCH 2/7] Bump body-parser and brace-expansion in the lockfile to clear npm audit CI runs `npm audit` as a hard gate. Two advisories published since main last ran CI now fail it on any branch, including this one, whose dependency files are otherwise identical to main: - body-parser <1.20.6 (GHSA-v422-hmwv-36x6), reachable here because the app configures body size limits from environment variables - brace-expansion 3.0.0-5.0.8 (GHSA-3jxr-9vmj-r5cp and two related), transitive Both are patch bumps within the existing semver ranges (express depends on body-parser ~1.20.5), so package.json is unchanged. `npm audit` reports no vulnerabilities afterwards and the full suite still passes, including the body limit tests that exercise the affected body-parser behaviour. Kept as a separate commit so it can be dropped if these are handled elsewhere. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_011VVgEEw2J1LensebivHrwW --- package-lock.json | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/package-lock.json b/package-lock.json index 4564c7b..f0ecca9 100644 --- a/package-lock.json +++ b/package-lock.json @@ -152,9 +152,9 @@ } }, "node_modules/body-parser": { - "version": "1.20.5", - "resolved": "https://registry.npmjs.org/body-parser/-/body-parser-1.20.5.tgz", - "integrity": "sha512-3grm+/2tUOvu2cjJkvsIxrv/wVpfXQW4PsQHYm7yk4vfpu7Ekl6nEsYBoJUL6qDwZUx8wUhQ8tR2qz+ad9c9OA==", + "version": "1.20.6", + "resolved": "https://registry.npmjs.org/body-parser/-/body-parser-1.20.6.tgz", + "integrity": "sha512-p5tAzS57i5MV9fZFDj9LeIiTZEufbSe2eDozP+ElheSUq1m74CRq1jI4mYNDdVs9vQztXFLuk/Gd6BWTdwRJ5g==", "license": "MIT", "dependencies": { "bytes": "~3.1.2", @@ -205,16 +205,16 @@ } }, "node_modules/brace-expansion": { - "version": "5.0.6", - "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-5.0.6.tgz", - "integrity": "sha512-kLpxurY4Z4r9sgMsyG0Z9uzsBlgiU/EFKhj/h91/8yHu0edo7XuixOIH3VcJ8kkxs6/jPzoI6U9Vj3WqbMQ94g==", + "version": "5.0.9", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-5.0.9.tgz", + "integrity": "sha512-ScQ4IuvIEF1TMlP7Zt+vjJ//9zlPb2SDcxWxM3bk8s6t6GGdJ7KO1dCcTidOPJKePW30LE/2cT7wCyPho9/Wxg==", "dev": true, "license": "MIT", "dependencies": { "balanced-match": "^4.0.2" }, "engines": { - "node": "18 || 20 || >=22" + "node": "20 || >=22" } }, "node_modules/braces": { @@ -1927,9 +1927,9 @@ "dev": true }, "body-parser": { - "version": "1.20.5", - "resolved": "https://registry.npmjs.org/body-parser/-/body-parser-1.20.5.tgz", - "integrity": "sha512-3grm+/2tUOvu2cjJkvsIxrv/wVpfXQW4PsQHYm7yk4vfpu7Ekl6nEsYBoJUL6qDwZUx8wUhQ8tR2qz+ad9c9OA==", + "version": "1.20.6", + "resolved": "https://registry.npmjs.org/body-parser/-/body-parser-1.20.6.tgz", + "integrity": "sha512-p5tAzS57i5MV9fZFDj9LeIiTZEufbSe2eDozP+ElheSUq1m74CRq1jI4mYNDdVs9vQztXFLuk/Gd6BWTdwRJ5g==", "requires": { "bytes": "~3.1.2", "content-type": "~1.0.5", @@ -1965,9 +1965,9 @@ } }, "brace-expansion": { - "version": "5.0.6", - "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-5.0.6.tgz", - "integrity": "sha512-kLpxurY4Z4r9sgMsyG0Z9uzsBlgiU/EFKhj/h91/8yHu0edo7XuixOIH3VcJ8kkxs6/jPzoI6U9Vj3WqbMQ94g==", + "version": "5.0.9", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-5.0.9.tgz", + "integrity": "sha512-ScQ4IuvIEF1TMlP7Zt+vjJ//9zlPb2SDcxWxM3bk8s6t6GGdJ7KO1dCcTidOPJKePW30LE/2cT7wCyPho9/Wxg==", "dev": true, "requires": { "balanced-match": "^4.0.2" From 54e225bb1fedba94441e0a54c4ddd570cb1b1c20 Mon Sep 17 00:00:00 2001 From: Konrad Kollnig <5175206+kasnder@users.noreply.github.com> Date: Tue, 11 Aug 2026 13:58:57 +0200 Subject: [PATCH 3/7] Address PR review findings --- README.md | 4 +- lib/jurisdiction.js | 3 + lib/reverseIndex.js | 77 +++++++++++---- routes/index.js | 198 +++++++++++++++++++++++++++++--------- test/reverseIndex.test.js | 38 +++++++- views/form.pug | 6 +- 6 files changed, 259 insertions(+), 67 deletions(-) diff --git a/README.md b/README.md index 9d2ca57..74cf91f 100644 --- a/README.md +++ b/README.md @@ -95,7 +95,9 @@ PORT=3000 Set `SITE_URL` in production to the public origin, for example `SITE_URL=https://ios.trackercontrol.org`. Canonical links, Open Graph URLs, `robots.txt`, and `sitemap.xml` use it. Without it, those URLs are derived from -the request, which yields `http://` links when TLS is terminated by a proxy. +the request in development, which yields `http://` links when TLS is terminated +by a proxy. Production requests fail if `SITE_URL` is not configured, +so an untrusted Host header cannot become a public canonical URL. Run migrations: diff --git a/lib/jurisdiction.js b/lib/jurisdiction.js index b1778d6..c0fdba4 100644 --- a/lib/jurisdiction.js +++ b/lib/jurisdiction.js @@ -148,6 +148,7 @@ function isSystemSignature(trackerName) { // tracker names recur across thousands of apps, so results are memoised. The // company database is static after module load, which makes this safe. const resolutionCache = new Map(); +const MAX_RESOLUTION_CACHE_SIZE = 10000; /** * Resolve a tracker name to a company. @@ -160,6 +161,8 @@ function resolveTrackerName(trackerName) { if (resolutionCache.has(key)) return resolutionCache.get(key); const resolved = resolveTrackerNameUncached(key); + if (resolutionCache.size >= MAX_RESOLUTION_CACHE_SIZE) + resolutionCache.delete(resolutionCache.keys().next().value); resolutionCache.set(key, resolved); return resolved; } diff --git a/lib/reverseIndex.js b/lib/reverseIndex.js index 004797c..860b0f5 100644 --- a/lib/reverseIndex.js +++ b/lib/reverseIndex.js @@ -10,10 +10,12 @@ // and company entries reference apps by bundle ID. That keeps the cached JSON // small enough to read cheaply even when every app appears in several lists. const jurisdiction = require('./jurisdiction'); +const crypto = require('node:crypto'); const MAX_SLUG_LENGTH = 80; -// Allows the numeric suffix appended when two names slugify identically. +// Allows persisted legacy slugs as well as deterministic hash suffixes. const MAX_SLUG_LENGTH_WITH_SUFFIX = MAX_SLUG_LENGTH + 8; +const SLUG_HASH_LENGTH = 8; const SLUG_PATTERN = /^[a-z0-9][a-z0-9-]*$/; /** @@ -83,18 +85,52 @@ function compareNames(a, b) { } /** - * Assign a unique slug to every entry. Entries are slugged in name order so - * that a collision resolves to the same slug on every rebuild. + * Add a deterministic discriminator for a new name that collides with an + * existing slug. The name is canonicalised because tracker and company maps + * are case-insensitive, so changing only the display-name casing cannot move + * a URL. */ -function assignSlugs(entries) { +function collisionSlug(name, base, hashLength = SLUG_HASH_LENGTH) { + const canonical = String(name == null ? '' : name).toLowerCase().trim(); + const hash = crypto.createHash('sha256').update(canonical).digest('hex'); + const suffix = hash.slice(0, hashLength); + const prefixLength = MAX_SLUG_LENGTH - suffix.length - 1; + const prefix = base.slice(0, prefixLength).replace(/-+$/, '') || 'unnamed'; + return `${prefix}-${suffix}`; +} + +/** + * Assign a unique slug to every entry. Previously assigned slugs win, so a + * newly discovered name cannot take over a URL that was already published. + * New collisions use a name-derived hash instead of a positional counter. + */ +function assignSlugs(entries, previousSlugs = {}) { const used = new Set(); + const pending = []; + for (const entry of [...entries].sort((a, b) => compareNames(a.name, b.name))) { + const key = String(entry.name).toLowerCase().trim(); + const previous = previousSlugs && previousSlugs[key]; + if (isValidSlug(previous) && !used.has(previous)) { + used.add(previous); + entry.slug = previous; + } else { + pending.push(entry); + } + } + + for (const entry of pending) { const base = slugify(entry.name); let slug = base; - let suffix = 2; - while (used.has(slug)) { - slug = `${base}-${suffix}`; - suffix++; + if (used.has(slug)) { + let hashLength = SLUG_HASH_LENGTH; + do { + slug = collisionSlug(entry.name, base, hashLength); + hashLength += 8; + } while (used.has(slug) && hashLength <= 64); + if (used.has(slug)) { + throw new Error(`Unable to assign a unique slug for ${entry.name}`); + } } used.add(slug); entry.slug = slug; @@ -108,7 +144,7 @@ function assignSlugs(entries) { * tracker was detected, so it matches the denominator used by the aggregate * jurisdiction statistics and can be quoted as "N of M apps". */ -function buildReverseIndex(allApps) { +function buildReverseIndex(allApps, previousIndex = null) { const apps = {}; const trackerEntries = new Map(); // lowercased tracker name -> entry let totalApps = 0; @@ -127,7 +163,10 @@ function buildReverseIndex(allApps) { latestAnalysis = analysedAt; const trackerNames = analysis.trackers ? Object.keys(analysis.trackers) : []; - if (trackerNames.length > 0) trackedApps++; + const thirdPartyTrackerNames = trackerNames.filter( + (trackerName) => !jurisdiction.isSystemSignature(trackerName) + ); + if (thirdPartyTrackerNames.length > 0) trackedApps++; const details = app.details || {}; const analysed = jurisdiction.analyseApp(analysis); @@ -141,7 +180,7 @@ function buildReverseIndex(allApps) { icon: details.icon || null, category: details.primaryGenre || null, reviews: reviewCount(details), - trackerCount: trackerNames.length, + trackerCount: thirdPartyTrackerNames.length, classification: analysed.classification, analysed: analysedValid ? analysedAt.toISOString() : null }; @@ -189,8 +228,8 @@ function buildReverseIndex(allApps) { const trackerArray = [...trackerEntries.values()]; const companyArray = [...companyEntries.values()]; - assignSlugs(trackerArray); - assignSlugs(companyArray); + assignSlugs(trackerArray, previousIndex && previousIndex.trackerSlugs); + assignSlugs(companyArray, previousIndex && previousIndex.companySlugs); const companySlugByName = new Map(); for (const company of companyArray) @@ -254,9 +293,9 @@ function buildReverseIndex(allApps) { // into the lookup pages without re-deriving a slug that may have been // deduplicated here. const trackerSlugs = {}; - for (const entry of trackerArray) trackerSlugs[entry.name.toLowerCase()] = entry.slug; + for (const entry of trackerArray) trackerSlugs[entry.name.toLowerCase().trim()] = entry.slug; const companySlugs = {}; - for (const entry of companyArray) companySlugs[entry.name.toLowerCase()] = entry.slug; + for (const entry of companyArray) companySlugs[entry.name.toLowerCase().trim()] = entry.slug; return { trackerSlugs, @@ -297,7 +336,7 @@ function lookupCompany(index, slug) { */ function slugForName(slugMap, name) { if (!slugMap || !name) return null; - const key = String(name).toLowerCase(); + const key = String(name).toLowerCase().trim(); return Object.prototype.hasOwnProperty.call(slugMap, key) ? slugMap[key] : null; } @@ -306,7 +345,11 @@ function slugForName(slugMap, name) { * from the directory. */ function paginate(appIds, appDirectory, page, perPage) { - const items = appIds || []; + const items = (appIds || []).filter((appid) => + appDirectory + && Object.prototype.hasOwnProperty.call(appDirectory, appid) + && appDirectory[appid] + ); const totalPages = Math.max(1, Math.ceil(items.length / perPage)); const currentPage = Math.min(Math.max(1, page), totalPages); const start = (currentPage - 1) * perPage; diff --git a/routes/index.js b/routes/index.js index e5efadf..705c35c 100644 --- a/routes/index.js +++ b/routes/index.js @@ -27,6 +27,8 @@ const SITE_NAME = 'TrackerControl for iOS'; const DEFAULT_DESCRIPTION = 'Find out which trackers are embedded in iOS apps, ' + 'which companies control them, and under which jurisdiction that tracking falls.'; const APPS_PER_PAGE = 50; +const MAX_SITEMAP_URLS = 50000; +const MAX_SITEMAP_BYTES = 50 * 1024 * 1024; let lastPing = 0; // unix timestamp @@ -42,11 +44,21 @@ router.use(function (req, res, next) { * that terminates TLS, where req.protocol would otherwise report http. */ function siteBaseUrl(req) { - const configured = process.env.SITE_URL; - if (configured) return configured.trim().replace(/\/+$/, ''); + const configured = (process.env.SITE_URL || '').trim(); + if (configured) return configured.replace(/\/+$/, ''); + if (process.env.NODE_ENV === 'production') + throw new Error('SITE_URL must be configured in production'); return `${req.protocol}://${req.get('host')}`; } +function thirdPartyTrackerNames(analysis) { + return analysis && analysis.trackers + ? Object.keys(analysis.trackers).filter( + (trackerName) => !jurisdiction.isSystemSignature(trackerName) + ) + : []; +} + // Social card and canonical link defaults. Individual routes override these // with page-specific values by passing them to res.render. router.use(function (req, res, next) { @@ -78,7 +90,7 @@ function buildSiteData(allApps) { // Top trackers enriched with company/country const trackerCounts = {}; for (const app of analysedApps) { - for (const tracker of Object.keys(app.analysis.trackers)) { + for (const tracker of thirdPartyTrackerNames(app.analysis)) { if (!trackerCounts[tracker]) trackerCounts[tracker] = 0; trackerCounts[tracker]++; } @@ -107,7 +119,7 @@ function buildSiteData(allApps) { const appsWithMostTrackers = analysedApps .filter(a => a.details && a.details.title) .map(a => { - const trackerCount = Object.keys(a.analysis.trackers).length; + const trackerCount = thirdPartyTrackerNames(a.analysis).length; const jd = jurisdiction.analyseApp(a.analysis); const topCountries = Object.entries(jd.countryBreakdown || {}) .sort((a, b) => b[1] - a[1]) @@ -179,7 +191,7 @@ async function getSiteData() { return cached.data; } - const allApps = await Apps.getAllApps(); + const allApps = await getAllAppsForSignature(signature); const data = buildSiteData(allApps); if (data.appCount > 0) { cache.write('sitedata', data, signature); @@ -197,6 +209,16 @@ async function getSiteData() { // kept in its own cache entry and only touched by the lookup pages and the // sitemap. The in-process copy avoids re-parsing the cache file per request. let reverseIndexMemo = null; // { meta, index } +let allAppsMemo = null; // { meta, apps } + +async function getAllAppsForSignature(signature) { + if (allAppsMemo && signatureMatches(allAppsMemo.meta, signature)) + return allAppsMemo.apps; + + const apps = await Apps.getAllApps(); + allAppsMemo = { meta: signature, apps }; + return apps; +} /** * Get the tracker/company reverse index, rebuilding it when new analyses have @@ -215,8 +237,8 @@ async function getReverseIndex() { return cached.data; } - const allApps = await Apps.getAllApps(); - const index = reverseIndex.buildReverseIndex(allApps); + const allApps = await getAllAppsForSignature(signature); + const index = reverseIndex.buildReverseIndex(allApps, cached && cached.data); if (index.totalApps > 0) { cache.write('reverseindex', index, signature); reverseIndexMemo = { meta: signature, index }; @@ -236,9 +258,12 @@ function invalidateSiteCaches() { cache.invalidate('sitedata'); cache.invalidate('reverseindex'); reverseIndexMemo = null; + allAppsMemo = null; } const EMPTY_REVERSE_INDEX = { + trackerSlugs: {}, + companySlugs: {}, totalApps: 0, trackedApps: 0, latestAnalysis: null, @@ -281,15 +306,17 @@ router.get('/', asyncHandler(async (req, res) => { * company in them links to the apps it was found in. */ function withLookupSlugs(data, index) { + const trackerSlugs = index.trackerSlugs || {}; + const companySlugs = index.companySlugs || {}; const trackers = (data.topTrackersEnriched || []).map((tracker) => ({ ...tracker, - slug: reverseIndex.slugForName(index.trackerSlugs, tracker.name), - companySlug: reverseIndex.slugForName(index.companySlugs, tracker.company) + slug: reverseIndex.slugForName(trackerSlugs, tracker.name), + companySlug: reverseIndex.slugForName(companySlugs, tracker.company) })); const companies = (data.jurisdictionStats && data.jurisdictionStats.topCompaniesSorted || []) .map((company) => ({ ...company, - slug: reverseIndex.slugForName(index.companySlugs, company.name) + slug: reverseIndex.slugForName(companySlugs, company.name) })); return { @@ -300,24 +327,9 @@ function withLookupSlugs(data, index) { // Statistics detail page router.get('/statistics', asyncHandler(async (req, res) => { + let data; try { - const data = await getSiteData(); - const index = await getReverseIndex(); - const linked = withLookupSlugs(data, index); - - return res.render('statistics', { - title: 'Detailed Statistics', - data: req.body, - pageDescription: `Tracking jurisdiction across ${data.headlines.totalApps} ` - + 'analysed iOS apps: the most prevalent trackers, the companies behind ' - + 'them, and how they break down by country and App Store category.', - headlines: data.headlines, - jurisdictionStats: linked.jurisdictionStats, - jurisdictionMeta: jurisdiction.classificationMeta, - topTrackersEnriched: linked.topTrackersEnriched, - europeanAlternatives: jurisdiction.europeanAlternatives, - xrayCompanyCount: jurisdiction.xrayCompanyCount - }); + data = await getSiteData(); } catch (err) { console.error('Statistics error:', err.message); return res.render('statistics', { @@ -331,6 +343,28 @@ router.get('/statistics', asyncHandler(async (req, res) => { xrayCompanyCount: jurisdiction.xrayCompanyCount }); } + + let index = EMPTY_REVERSE_INDEX; + try { + index = await getReverseIndex(); + } catch (err) { + console.error('Statistics lookup data unavailable:', err.message); + } + const linked = withLookupSlugs(data, index); + + return res.render('statistics', { + title: 'Detailed Statistics', + data: req.body, + pageDescription: `Tracking jurisdiction across ${data.headlines.totalApps} ` + + 'analysed iOS apps: the most prevalent trackers, the companies behind ' + + 'them, and how they break down by country and App Store category.', + headlines: data.headlines, + jurisdictionStats: linked.jurisdictionStats, + jurisdictionMeta: jurisdiction.classificationMeta, + topTrackersEnriched: linked.topTrackersEnriched, + europeanAlternatives: jurisdiction.europeanAlternatives, + xrayCompanyCount: jurisdiction.xrayCompanyCount + }); })); router.get('/healthz', asyncHandler(async (req, res) => { @@ -399,8 +433,12 @@ router.get('/analysis/:appId', asyncHandler(async (req, res) => { if (analysis.success !== undefined && analysis.success === false) app.analysisFailure = analysis.reason === 'app_not_found' ? "App not found on App Store." : "Analysis failed." else { - if (analysis.trackers) - app.trackers = "Found trackers: " + Object.keys(analysis.trackers).join(", "); + if (analysis.trackers) { + const trackerNames = thirdPartyTrackerNames(analysis); + app.trackers = trackerNames.length > 0 + ? "Found trackers: " + trackerNames.join(", ") + : "No trackers found."; + } else app.trackers = "No trackers found." @@ -461,8 +499,13 @@ router.get('/analysis/:appId', asyncHandler(async (req, res) => { } const trackerCount = app.analysis && app.analysis.trackers && app.analysis.success !== false - ? Object.keys(app.analysis.trackers).length + ? thirdPartyTrackerNames(app.analysis).length : null; + const systemTrackerNames = app.analysis && app.analysis.trackers && app.analysis.success !== false + ? Object.keys(app.analysis.trackers).filter((trackerName) => + jurisdiction.isSystemSignature(trackerName) + ) + : []; const pageDescription = trackerCount === null ? `Tracker analysis of ${app.details.title} for iOS.` : `${trackerCount === 0 ? 'No trackers were' : `${trackerCount} tracker${trackerCount === 1 ? ' was' : 's were'}`}` @@ -477,6 +520,8 @@ router.get('/analysis/:appId', asyncHandler(async (req, res) => { trackerSlugs: trackerSlugs, companySlugs: companySlugs, jurisdictionData: jurisdictionData, + trackerCount, + systemTrackerNames, pageDescription, ogImage: app.details.icon || null }); @@ -708,20 +753,49 @@ function sitemapEntry(base, path, { lastmod, changefreq, priority } = {}) { return ` \n${parts.join('\n')}\n `; } -// Sitemap over the report, lookup and reference pages. Built from the cached -// reverse index so a crawl does not read every stored analysis from the -// database. -router.get('/sitemap.xml', asyncHandler(async (req, res) => { - const base = siteBaseUrl(req); +function renderSitemap(entries) { + return ` + +${entries.join('\n')} +`; +} - let index; - try { - index = await getReverseIndex(); - } catch (err) { - console.error('Sitemap error:', err.message); - index = EMPTY_REVERSE_INDEX; +function renderSitemapIndex(base, pageCount) { + const entries = Array.from({ length: pageCount }, (_, index) => + ` \n ${escapeXml(base + `/sitemap-${index + 1}.xml`)}\n ` + ); + return ` + +${entries.join('\n')} +`; +} + +function splitSitemapEntries(entries) { + const pages = []; + let page = []; + const emptyPageBytes = Buffer.byteLength(renderSitemap([]), 'utf8'); + let pageBytes = emptyPageBytes; + + for (const entry of entries) { + const entryBytes = Buffer.byteLength(entry, 'utf8'); + const candidateBytes = pageBytes + (page.length > 0 ? 1 : 0) + entryBytes; + const tooManyUrls = page.length + 1 > MAX_SITEMAP_URLS; + const tooLarge = candidateBytes > MAX_SITEMAP_BYTES; + if (page.length > 0 && (tooManyUrls || tooLarge)) { + pages.push(page); + page = [entry]; + pageBytes = emptyPageBytes + entryBytes; + } else { + page.push(entry); + pageBytes = candidateBytes; + } } + if (page.length > 0) pages.push(page); + return pages; +} + +function sitemapEntries(base, index) { const updated = index.latestAnalysis; const entries = [ sitemapEntry(base, '/', { lastmod: updated, changefreq: 'daily', priority: '1.0' }), @@ -745,11 +819,43 @@ router.get('/sitemap.xml', asyncHandler(async (req, res) => { priority: '0.6' })); - res.header('Content-Type', 'application/xml'); - res.send(` - -${entries.join('\n')} -`); + return entries; +} + +async function getSitemapPages(base) { + let index; + try { + index = await getReverseIndex(); + } catch (err) { + console.error('Sitemap error:', err.message); + index = EMPTY_REVERSE_INDEX; + } + return splitSitemapEntries(sitemapEntries(base, index)); +} + +// Sitemap over the report, lookup and reference pages. Built from the cached +// reverse index so a crawl does not read every stored analysis from the +// database. +router.get('/sitemap.xml', asyncHandler(async (req, res) => { + const base = siteBaseUrl(req); + const pages = await getSitemapPages(base); + + res.type('application/xml').send(pages.length === 1 + ? renderSitemap(pages[0]) + : renderSitemapIndex(base, pages.length)); +})); + +router.get('/sitemap-:page.xml', asyncHandler(async (req, res) => { + const pageNumber = Number(req.params.page); + if (!Number.isInteger(pageNumber) || pageNumber < 1) + return res.status(404).send('Sitemap not found.'); + + const base = siteBaseUrl(req); + const pages = await getSitemapPages(base); + if (pageNumber > pages.length) + return res.status(404).send('Sitemap not found.'); + + res.type('application/xml').send(renderSitemap(pages[pageNumber - 1])); })); router.get('/robots.txt', (req, res) => { diff --git a/test/reverseIndex.test.js b/test/reverseIndex.test.js index 6c6d50f..240853b 100644 --- a/test/reverseIndex.test.js +++ b/test/reverseIndex.test.js @@ -85,13 +85,17 @@ test('an app is counted once per company even with several of its trackers', () test('system APIs are flagged rather than listed as unattributed trackers', () => { const index = reverseIndex.buildReverseIndex([ - app('com.example.system', { trackers: ['AdID access'], reviews: 1 }) + app('com.example.system', { trackers: ['AdID access'], reviews: 1 }), + app('com.example.empty', { trackers: [], reviews: 1 }) ]); const entry = reverseIndex.lookupTracker(index, index.trackerSlugs['adid access']); assert.equal(entry.system, true); assert.equal(entry.company, null); assert.equal(entry.region, 'Unresolved'); + assert.equal(index.totalApps, 2); + assert.equal(index.trackedApps, 0); + assert.equal(index.apps['com.example.system'].trackerCount, 0); }); test('slugs are URL-safe and collisions get distinct slugs', () => { @@ -104,10 +108,26 @@ test('slugs are URL-safe and collisions get distinct slugs', () => { ]); const slugs = index.trackerList.slice().sort(); - assert.deepEqual(slugs, ['mob-com', 'mob-com-2']); + assert.equal(slugs.length, 2); + assert.ok(slugs.includes('mob-com')); + assert.match(slugs.find((slug) => slug !== 'mob-com'), /^mob-com-[a-f0-9]{8}$/); for (const slug of slugs) assert.ok(reverseIndex.lookupTracker(index, slug)); }); +test('previous slugs stay attached to their names as the corpus grows', () => { + const before = reverseIndex.buildReverseIndex([ + app('com.example.original', { trackers: ['Mob.com'] }) + ]); + const after = reverseIndex.buildReverseIndex([ + app('com.example.original', { trackers: ['Mob.com'] }), + app('com.example.new', { trackers: ['Mob com'] }) + ], before); + + assert.equal(after.trackerSlugs['mob.com'], before.trackerSlugs['mob.com']); + assert.match(after.trackerSlugs['mob com'], /^mob-com-[a-f0-9]{8}$/); + assert.equal(reverseIndex.lookupTracker(after, before.trackerSlugs['mob.com']).name, 'Mob.com'); +}); + test('lookup rejects invalid slugs and inherited properties', () => { const index = reverseIndex.buildReverseIndex(corpus); @@ -147,6 +167,20 @@ test('pagination clamps the page number and resolves app records', () => { assert.equal(empty.apps.length, 0); }); +test('pagination counts only apps that still exist in the directory', () => { + const pagination = reverseIndex.paginate( + ['com.example.one', 'com.example.missing'], + { 'com.example.one': { title: 'One' } }, + 1, + 50 + ); + + assert.equal(pagination.total, 1); + assert.equal(pagination.from, 1); + assert.equal(pagination.to, 1); + assert.deepEqual(pagination.apps, [{ title: 'One' }]); +}); + test('parsePage falls back to the first page for junk input', () => { assert.equal(reverseIndex.parsePage('3'), 3); assert.equal(reverseIndex.parsePage('0'), 1); diff --git a/views/form.pug b/views/form.pug index 86ccfc9..bc13f61 100644 --- a/views/form.pug +++ b/views/form.pug @@ -112,7 +112,7 @@ block app .col-md-8.col-12 a#trackers.anchor h3 - span.badge.badge-pill.badge-danger.reports #{Object.keys(app.analysis.trackers).length} + span.badge.badge-pill.badge-danger.reports #{trackerCount} =" " b trackers @@ -123,6 +123,8 @@ block app | of the following trackers in the application: if app.analysis.trackers + if systemTrackerNames && systemTrackerNames.length > 0 + p.text-muted.small Apple system API signatures are shown below for transparency and are not included in the tracker count. - const jdMap = {} - if (jurisdictionData && jurisdictionData.trackerDetails) { jurisdictionData.trackerDetails.forEach(d => { jdMap[d.name] = d }) } for tracker in Object.keys(app.analysis.trackers) @@ -134,6 +136,8 @@ block app a.link.black(target='_blank' rel='noopener noreferrer' href=`https://reports.exodus-privacy.eu.org/en/trackers/${trackerNameToExodus[tracker].id}/`) #{tracker} else | #{tracker} + if systemTrackerNames && systemTrackerNames.includes(tracker) + span.badge.badge-pill.region-unresolved.ml-1 System API - const jdEntry = jdMap[tracker] if jdEntry && jdEntry.flag span.ml-2 #{jdEntry.flag} From 9f56ff063198f639a6ed8a75ce45897613132482 Mon Sep 17 00:00:00 2001 From: Konrad Kollnig <5175206+kasnder@users.noreply.github.com> Date: Tue, 11 Aug 2026 16:27:54 +0200 Subject: [PATCH 4/7] Discard cache entries built by superseded derivation logic CACHE_DIR is a persistent volume, so entries survive a deploy. Both consumers detect new data through a signature of appCount and the latest analysis timestamp, which cannot see a change in how the cached data is derived from that data. Filtering Apple system signatures out of the tracker counts in buildSiteData is exactly such a change: the signature still matches the entry written before the deploy, so the pre-filter figures would be served on the homepage and statistics page until the next analysis lands. Stamp a schema version on every entry and treat a mismatch as a miss. --- README.md | 7 ++++++ lib/cache.js | 16 +++++++++++--- test/cache.test.js | 55 ++++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 75 insertions(+), 3 deletions(-) create mode 100644 test/cache.test.js diff --git a/README.md b/README.md index 39874d1..7ee760d 100644 --- a/README.md +++ b/README.md @@ -55,6 +55,13 @@ The reverse lookup pages are served from an inverted index built by data. It is rebuilt whenever the set of stored analyses changes, so no extra work happens per request. +`CACHE_DIR` is a persistent volume in production, so cache entries outlive the +code that wrote them. An entry is only rebuilt when the set of stored analyses +changes, which cannot detect a change in how the cached data is *derived*. +After editing `buildSiteData` or `buildReverseIndex`, bump `SCHEMA_VERSION` in +`lib/cache.js` so the deploy discards entries built by the previous logic — +otherwise the old figures are served until the next analysis lands. + ## Requirements Website: diff --git a/lib/cache.js b/lib/cache.js index a4175ef..1bbd9e4 100644 --- a/lib/cache.js +++ b/lib/cache.js @@ -14,14 +14,24 @@ function cachePath(key) { return path.join(CACHE_DIR, key + '.json'); } +// CACHE_DIR is a persistent volume in production, so entries outlive the code +// that produced them. Callers detect new *data* through their own meta +// signature, which cannot see a change in how that data is derived: after a +// deploy that alters buildSiteData or buildReverseIndex, the old entry still +// matches the signature and would be served indefinitely. Bump this whenever +// the shape or the derivation of any cached payload changes. +const SCHEMA_VERSION = 2; + /** - * Read from cache. Returns { data, meta } or null if no cache exists. + * Read from cache. Returns { data, meta } or null if no cache exists or the + * entry was written by an incompatible version of the builders. */ function read(key) { const file = cachePath(key); try { const raw = fs.readFileSync(file, 'utf-8'); const cached = JSON.parse(raw); + if (cached._schema !== SCHEMA_VERSION) return null; return { data: cached.data, meta: cached._meta || {} }; } catch (err) { return null; @@ -34,7 +44,7 @@ function read(key) { function write(key, data, meta) { const file = cachePath(key); try { - const payload = JSON.stringify({ data, _meta: meta || {} }); + const payload = JSON.stringify({ data, _meta: meta || {}, _schema: SCHEMA_VERSION }); fs.writeFileSync(file, payload, 'utf-8'); } catch (err) { console.error('Cache write error:', err.message); @@ -50,4 +60,4 @@ function invalidate(key) { } } -module.exports = { read, write, invalidate }; +module.exports = { read, write, invalidate, SCHEMA_VERSION }; diff --git a/test/cache.test.js b/test/cache.test.js new file mode 100644 index 0000000..44762eb --- /dev/null +++ b/test/cache.test.js @@ -0,0 +1,55 @@ +'use strict'; + +const assert = require('node:assert/strict'); +const test = require('node:test'); +const fs = require('node:fs'); +const os = require('node:os'); +const path = require('node:path'); + +// Resolved at require time, so it must be set before lib/cache is loaded. +process.env.CACHE_DIR = fs.mkdtempSync(path.join(os.tmpdir(), 'tc-cache-schema-')); + +const cache = require('../lib/cache'); + +const cacheFile = (key) => path.join(process.env.CACHE_DIR, key + '.json'); + +test('the cache is keyed by builder schema as well as data signature', async (t) => { + t.after(() => fs.rmSync(process.env.CACHE_DIR, { recursive: true, force: true })); + + await t.test('a round-trip returns the data and its signature', () => { + const signature = { appCount: 2, latestAnalysis: '2025-01-02T03:04:05.000Z' }; + cache.write('roundtrip', { totalApps: 2 }, signature); + + const cached = cache.read('roundtrip'); + assert.deepEqual(cached.data, { totalApps: 2 }); + assert.deepEqual(cached.meta, signature); + }); + + await t.test('an entry from an older schema reads as a miss', () => { + // CACHE_DIR is a persistent volume, so entries written before a deploy + // survive it. The signature alone still matches when only the derivation + // changed, which is what the schema version exists to catch. + const signature = { appCount: 2, latestAnalysis: '2025-01-02T03:04:05.000Z' }; + fs.writeFileSync(cacheFile('stale'), JSON.stringify({ + data: { totalApps: 2 }, + _meta: signature, + _schema: cache.SCHEMA_VERSION - 1 + }), 'utf-8'); + + assert.equal(cache.read('stale'), null); + }); + + await t.test('an entry predating the schema field reads as a miss', () => { + fs.writeFileSync(cacheFile('legacy'), JSON.stringify({ + data: { totalApps: 2 }, + _meta: { appCount: 2, latestAnalysis: null } + }), 'utf-8'); + + assert.equal(cache.read('legacy'), null); + }); + + await t.test('a rewrite at the current schema is readable again', () => { + cache.write('stale', { totalApps: 3 }, { appCount: 3, latestAnalysis: null }); + assert.deepEqual(cache.read('stale').data, { totalApps: 3 }); + }); +}); From 112871ce8aa0d7b25938c16d56835eb01a51857e Mon Sep 17 00:00:00 2001 From: Konrad Kollnig <5175206+kasnder@users.noreply.github.com> Date: Tue, 11 Aug 2026 16:41:48 +0200 Subject: [PATCH 5/7] Align the new public pages with the checks added alongside them MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Four issues where this branch and the Turnstile/storefront work on main met without being reconciled: SITE_URL was read from routing middleware that runs before every route. Unset in production, the throw became a 500 on every path including /healthz, from a process that had started cleanly. Move the resolution into lib/siteUrl.js and assert it at startup next to the Turnstile check, so the deployment fails where a missing variable is visible. One rate-limit budget covered both page views and form submissions. sitemap.xml now points crawlers at every app, tracker and company URL, and 100 requests per 5 minutes is 0.33/s — low enough that a single crawler's discovery pass collects 429s. Budget them separately: cached page views get a larger allowance, the forms that reach the App Store get a smaller one than they had. getSiteDataSignature aggregates over every stored app and now runs on the report page too, since the tracker links need the reverse index — twice per /statistics request. Memoise it briefly; writes clear it. getAllApps selected apps.* only, so the directories, lookup pages, sitemap and homepage rendered the queue-time snapshot while the report page rendered the refreshed storefront row: the same app under two titles. Join the cache row, resolve display metadata once via buildListingDetails, and add the storefront generation to the cache signature — without it a metadata refresh changes nothing the signature can see, and the persisted cache would never rebuild. --- README.md | 13 +++++++-- index.js | 10 +++++++ lib/appMetadata.js | 32 +++++++++++++++++++++- lib/siteUrl.js | 42 +++++++++++++++++++++++++++++ models/Apps.js | 24 ++++++++++++++--- routes/index.js | 57 +++++++++++++++++++++++++-------------- server.js | 34 ++++++++++++++++++----- test/appMetadata.test.js | 53 +++++++++++++++++++++++++++++++++++- test/lookupPages.test.js | 21 +++++++++++++-- test/siteUrl.test.js | 58 ++++++++++++++++++++++++++++++++++++++++ 10 files changed, 309 insertions(+), 35 deletions(-) create mode 100644 lib/siteUrl.js create mode 100644 test/siteUrl.test.js diff --git a/README.md b/README.md index 7ee760d..6059250 100644 --- a/README.md +++ b/README.md @@ -108,8 +108,17 @@ Set `SITE_URL` in production to the public origin, for example `SITE_URL=https://ios.trackercontrol.org`. Canonical links, Open Graph URLs, `robots.txt`, and `sitemap.xml` use it. Without it, those URLs are derived from the request in development, which yields `http://` links when TLS is terminated -by a proxy. Production requests fail if `SITE_URL` is not configured, -so an untrusted Host header cannot become a public canonical URL. +by a proxy. It is required in production, so an untrusted Host header cannot +become a public canonical URL — the server refuses to start without it rather +than answering 500 on every route. + +Rate limits are applied per IP over a five-minute window, and page views are +budgeted separately from form submissions because `sitemap.xml` points crawlers +at every app, tracker and company URL. `RATE_LIMIT_BROWSE_MAX` (default 300) +covers `GET`/`HEAD` of the public pages, which are served from the cached site +data. `RATE_LIMIT_FORM_MAX` (default 20) covers everything else — the search +and analysis-request forms, which reach the App Store and write to the +database. Authenticated analyser traffic is exempt from both. Run migrations: diff --git a/index.js b/index.js index 377e046..30d1197 100644 --- a/index.js +++ b/index.js @@ -2,6 +2,7 @@ require('dotenv').config(); const turnstile = require('./lib/turnstile'); +const siteUrl = require('./lib/siteUrl'); // Load the actual app const app = require('./server'); @@ -16,6 +17,15 @@ if (env == 'production') { console.error(`Turnstile configuration error: ${err.message}`); throw err; } + // Every route builds canonical/social URLs from this, so a missing value + // fails the request rather than degrading it. Refuse to boot instead of + // serving 500s from a process the platform considers healthy. + try { + siteUrl.assertSiteUrlConfiguration(); + } catch (err) { + console.error(`Site URL configuration error: ${err.message}`); + throw err; + } } // Server express HTTP server diff --git a/lib/appMetadata.js b/lib/appMetadata.js index 221fdd8..575af09 100644 --- a/lib/appMetadata.js +++ b/lib/appMetadata.js @@ -48,4 +48,34 @@ function buildReportMetadata({ }; } -module.exports = { buildReportMetadata }; +// Fields the listing pages render. The rest of the queue-time snapshot is +// carried through untouched so this stays a refresh rather than a projection. +const LISTING_FIELDS = ['title', 'icon', 'url', 'version', 'primaryGenre', 'reviews']; + +/** + * Display details for the pages that list many apps at once — the homepage, + * the directories, the lookup pages and the sitemap. + * + * apps.details is the snapshot taken when the app was queued and is never + * updated; the App Store cache row is what the metadata jobs refresh. Listing + * pages read the snapshot directly, which would leave them showing a title the + * report page contradicts. Refreshed values therefore win field by field, so a + * new title is not paired with an icon dropped from a partial response. + */ +function buildListingDetails({ queueSnapshot = null, storefront = null } = {}) { + const current = detailsFromStorefront(storefront) || {}; + const merged = { ...(queueSnapshot || {}) }; + + for (const field of LISTING_FIELDS) { + // Tested directly rather than through firstNonEmpty, whose `||` maps a + // numeric 0 to null. An app really can hold zero reviews, and that is a + // refreshed value like any other. + const value = current[field]; + if (value !== null && value !== undefined && value !== '') + merged[field] = value; + } + + return merged; +} + +module.exports = { buildReportMetadata, buildListingDetails }; diff --git a/lib/siteUrl.js b/lib/siteUrl.js new file mode 100644 index 0000000..faa040e --- /dev/null +++ b/lib/siteUrl.js @@ -0,0 +1,42 @@ +'use strict'; + +// Absolute base URL of this deployment, used for canonical links, social card +// metadata, robots.txt and the sitemap. +// +// SITE_URL pins the origin when the site runs behind a proxy that terminates +// TLS, where req.protocol would otherwise report http, and so that an +// untrusted Host header cannot become a public canonical URL. It is therefore +// required in production. index.js asserts it at startup: without that, the +// first request would throw from routing middleware and every route — the +// health check included — would answer 500 on a deployment that looked +// healthy at boot. + +function configuredSiteUrl() { + return (process.env.SITE_URL || '').trim().replace(/\/+$/, ''); +} + +function getSiteUrlConfigurationError() { + if (process.env.NODE_ENV === 'production' && !configuredSiteUrl()) + return 'SITE_URL is not set'; + + return null; +} + +function assertSiteUrlConfiguration() { + const error = getSiteUrlConfigurationError(); + if (error) throw new Error(error); +} + +function siteBaseUrl(req) { + const configured = configuredSiteUrl(); + if (configured) return configured; + + assertSiteUrlConfiguration(); + return `${req.protocol}://${req.get('host')}`; +} + +module.exports = { + siteBaseUrl, + getSiteUrlConfigurationError, + assertSiteUrlConfiguration +}; diff --git a/models/Apps.js b/models/Apps.js index c087b39..7501ec0 100644 --- a/models/Apps.js +++ b/models/Apps.js @@ -447,11 +447,27 @@ const updateAnalysis = async (appId, analysis, analysisVersion, claimToken) => { } } +// apps.details is the queue-time snapshot and is never updated, so the latest +// known storefront row travels with each app for callers that display a title +// or icon. See buildListingDetails in lib/appMetadata.js. const getAllApps = async () => { - const result = await pool.query("SELECT * FROM apps WHERE status = 'analysed'"); + const result = await pool.query(` + SELECT + apps.*, + cache.details AS current_storefront_details + FROM apps + LEFT JOIN app_store_cache cache + ON cache.appid_key = lower(apps.appid) + WHERE apps.status = 'analysed' + `); return result.rows; } +// Identifies the generation of data the cached site views were built from. +// latestStorefront covers the metadata refresh jobs: they change titles, +// icons and versions without adding an app or writing an analysis, so the +// first two components alone would leave a persisted cache serving metadata +// that the report pages have already moved past. const getSiteDataSignature = async () => { const result = await pool.query(` SELECT @@ -462,14 +478,16 @@ const getSiteDataSignature = async () => { MAX(analysed) FILTER ( WHERE status = 'analysed' AND analysis->'trackers' IS NOT NULL - ) AS latest_analysis + ) AS latest_analysis, + (SELECT MAX(fetched_at) FROM app_store_cache) AS latest_storefront FROM apps `); const row = result.rows[0]; return { appCount: parseInt(row.app_count, 10), - latestAnalysis: row.latest_analysis ? new Date(row.latest_analysis).toISOString() : null + latestAnalysis: row.latest_analysis ? new Date(row.latest_analysis).toISOString() : null, + latestStorefront: row.latest_storefront ? new Date(row.latest_storefront).toISOString() : null }; } diff --git a/routes/index.js b/routes/index.js index 03fcc6a..916e83c 100644 --- a/routes/index.js +++ b/routes/index.js @@ -14,7 +14,8 @@ const { STALE_ANALYSIS_DAYS } = require('../lib/analysisPolicy'); const turnstile = require('../lib/turnstile'); -const { buildReportMetadata } = require('../lib/appMetadata'); +const { buildReportMetadata, buildListingDetails } = require('../lib/appMetadata'); +const { siteBaseUrl } = require('../lib/siteUrl'); // Taken from https://reports.exodus-privacy.eu.org/api/trackers const exodusTrackers = JSON.parse(fs.readFileSync('./exodusTrackers.json', 'utf-8')) @@ -96,19 +97,6 @@ router.use(function (req, res, next) { next(); }); -/** - * Absolute base URL of this deployment, used for canonical links, social card - * metadata and the sitemap. SITE_URL pins it when the site runs behind a proxy - * that terminates TLS, where req.protocol would otherwise report http. - */ -function siteBaseUrl(req) { - const configured = (process.env.SITE_URL || '').trim(); - if (configured) return configured.replace(/\/+$/, ''); - if (process.env.NODE_ENV === 'production') - throw new Error('SITE_URL must be configured in production'); - return `${req.protocol}://${req.get('host')}`; -} - function thirdPartyTrackerNames(analysis) { return analysis && analysis.trackers ? Object.keys(analysis.trackers).filter( @@ -227,13 +215,32 @@ function buildSiteData(allApps) { } /** - * Whether cached data was built from the same set of analyses as the database - * currently holds. + * Whether cached data was built from the same generation of stored analyses + * and App Store metadata as the database currently holds. */ function signatureMatches(meta, signature) { return Boolean(meta) && meta.appCount === signature.appCount - && meta.latestAnalysis === signature.latestAnalysis; + && meta.latestAnalysis === signature.latestAnalysis + && meta.latestStorefront === signature.latestStorefront; +} + +// The signature is an aggregate over every stored app, and each cached view +// asks for it before serving: the report page needs the reverse index for its +// tracker links, and /statistics reads both cached views. Without this it +// would run twice per statistics request and once per report view. A few +// seconds of staleness only delays a rebuild that a background metadata job +// triggered; writes from this process clear the memo outright. +const SIGNATURE_TTL_MS = 5000; +let signatureMemo = null; // { at, signature } + +async function getSiteDataSignature() { + if (signatureMemo && Date.now() - signatureMemo.at < SIGNATURE_TTL_MS) + return signatureMemo.signature; + + const signature = await Apps.getSiteDataSignature(); + signatureMemo = { at: Date.now(), signature }; + return signature; } /** @@ -244,7 +251,7 @@ async function getSiteData() { const cached = cache.read('sitedata'); try { - const signature = await Apps.getSiteDataSignature(); + const signature = await getSiteDataSignature(); if (cached && signatureMatches(cached.meta, signature)) { return cached.data; } @@ -273,7 +280,16 @@ async function getAllAppsForSignature(signature) { if (allAppsMemo && signatureMatches(allAppsMemo.meta, signature)) return allAppsMemo.apps; - const apps = await Apps.getAllApps(); + // Resolve the display metadata once here so that buildSiteData and + // buildReverseIndex both see the refreshed title and icon under `details`, + // rather than each reaching into the storefront columns themselves. + const apps = (await Apps.getAllApps()).map((row) => ({ + ...row, + details: buildListingDetails({ + queueSnapshot: row.details, + storefront: { details: row.current_storefront_details } + }) + })); allAppsMemo = { meta: signature, apps }; return apps; } @@ -284,7 +300,7 @@ async function getAllAppsForSignature(signature) { */ async function getReverseIndex() { try { - const signature = await Apps.getSiteDataSignature(); + const signature = await getSiteDataSignature(); if (reverseIndexMemo && signatureMatches(reverseIndexMemo.meta, signature)) return reverseIndexMemo.index; @@ -317,6 +333,7 @@ function invalidateSiteCaches() { cache.invalidate('reverseindex'); reverseIndexMemo = null; allAppsMemo = null; + signatureMemo = null; } const EMPTY_REVERSE_INDEX = { diff --git a/server.js b/server.js index 30c6d1b..9baa489 100644 --- a/server.js +++ b/server.js @@ -40,15 +40,37 @@ const analyserPaths = new Set([ const isAnalyserPath = (req) => analyserPaths.has(req.path.toLowerCase().replace(/\/+$/, '')); +// Reads of the public pages are served from the cached site data and reverse +// index, so they cost far less than a form submission, which reaches the App +// Store and writes to the database. They also arrive in very different +// volumes: sitemap.xml points crawlers at every app, tracker and company URL, +// and a crawler works through those from a narrow range of addresses. Sharing +// one budget between the two means either throttling a normal crawl or +// loosening the limit that actually matters, so they are budgeted separately. +const isBrowseRequest = (req) => + (req.method === 'GET' || req.method === 'HEAD') && !isAnalyserPath(req); + if(os.hostname().indexOf("local") <= -1) { // only on remote host - const limiter = rateLimit({ - windowMs: 5 * 60 * 1000, // 5 minutes - max: 100, // Limit each IP to 10 requests per `window` + const windowMs = 5 * 60 * 1000; // 5 minutes + const skipAnalyser = (req) => isAnalyserPath(req) && analyserAuthenticated(req); + + // Everything that is not a cacheable page view: the public forms, and + // analyser endpoints called without credentials. + app.use(rateLimit({ + windowMs, + max: Number(process.env.RATE_LIMIT_FORM_MAX) || 20, + standardHeaders: false, + legacyHeaders: false, + skip: (req) => skipAnalyser(req) || isBrowseRequest(req), + })) + + app.use(rateLimit({ + windowMs, + max: Number(process.env.RATE_LIMIT_BROWSE_MAX) || 300, standardHeaders: false, legacyHeaders: false, - skip: (req) => isAnalyserPath(req) && analyserAuthenticated(req), - }) - app.use(limiter) + skip: (req) => skipAnalyser(req) || !isBrowseRequest(req), + })) } const analyserBodyLimit = process.env.BODY_LIMIT || '25mb'; diff --git a/test/appMetadata.test.js b/test/appMetadata.test.js index 71aa179..2b1e9f7 100644 --- a/test/appMetadata.test.js +++ b/test/appMetadata.test.js @@ -2,7 +2,7 @@ const assert = require('node:assert/strict'); const test = require('node:test'); -const { buildReportMetadata } = require('../lib/appMetadata'); +const { buildReportMetadata, buildListingDetails } = require('../lib/appMetadata'); test('report metadata prefers current storefront and exposes version divergence', () => { const metadata = buildReportMetadata({ @@ -71,3 +71,54 @@ test('queue snapshot supplies a version when no current storefront exists', () = assert.equal(metadata.currentVersion, '1.0'); assert.equal(metadata.currentVersionFromStorefront, false); }); + +test('listing details prefer the refreshed storefront over the queue snapshot', () => { + const details = buildListingDetails({ + queueSnapshot: { + title: 'Queue title', + icon: 'queue-icon', + primaryGenre: 'Games', + reviews: 100, + free: true + }, + storefront: { + details: { title: 'Renamed', icon: 'new-icon', primaryGenre: 'News', reviews: 5000 } + } + }); + + assert.equal(details.title, 'Renamed'); + assert.equal(details.icon, 'new-icon'); + assert.equal(details.primaryGenre, 'News'); + assert.equal(details.reviews, 5000); + // Fields the storefront does not carry survive the refresh. + assert.equal(details.free, true); +}); + +test('listing details fall back field by field, not wholesale', () => { + const details = buildListingDetails({ + queueSnapshot: { title: 'Queue title', icon: 'queue-icon', url: 'queue-url' }, + // A partial refresh must not blank the fields it omits. + storefront: { details: { title: 'Renamed', icon: '' } } + }); + + assert.equal(details.title, 'Renamed'); + assert.equal(details.icon, 'queue-icon'); + assert.equal(details.url, 'queue-url'); +}); + +test('listing details keep a refreshed review count of zero', () => { + const details = buildListingDetails({ + queueSnapshot: { title: 'Queued', reviews: 4000 }, + storefront: { details: { reviews: 0 } } + }); + + assert.equal(details.reviews, 0); +}); + +test('listing details tolerate a missing storefront row and a missing snapshot', () => { + assert.deepEqual( + buildListingDetails({ queueSnapshot: { title: 'Queued' }, storefront: { details: null } }), + { title: 'Queued' } + ); + assert.deepEqual(buildListingDetails(), {}); +}); diff --git a/test/lookupPages.test.js b/test/lookupPages.test.js index 2171b8b..06cba91 100644 --- a/test/lookupPages.test.js +++ b/test/lookupPages.test.js @@ -46,6 +46,12 @@ const corpus = [ primaryGenre: 'Games', free: true }, + // The queue-time snapshot above is frozen; this is what the metadata + // refresh job keeps up to date. + current_storefront_details: { + title: 'Example One Renamed', + icon: 'https://icons.test/one-v2.png' + }, analysis: { trackers: { 'Google Firebase Analytics': {}, 'Facebook Login': {} } } } ]; @@ -127,6 +133,15 @@ test('reverse lookup, methodology, sitemap and social metadata', async (t) => { assert.match(body, /detected in 2 of 2 analysed iOS apps/); }); + await t.test('listing pages show refreshed storefront metadata, not the queue snapshot', async () => { + const response = await fetch(`${base}/tracker/google-firebase-analytics`); + const body = await response.text(); + + assert.match(body, /Example One Renamed/); + assert.match(body, /one-v2\.png/); + assert.doesNotMatch(body, /icons\.test\/one\.png/); + }); + await t.test('company page aggregates its trackers', async () => { const response = await fetch(`${base}/company/alphabet`); const body = await response.text(); @@ -158,8 +173,10 @@ test('reverse lookup, methodology, sitemap and social metadata', async (t) => { assert.equal(response.status, 200); assert.match(body, /href="\/tracker\/google-firebase-analytics"/); - assert.match(body, //); - assert.match(body, /2 trackers were detected in Example One/); + // The social card follows the same storefront precedence as the report + // body, so it cannot advertise a title or icon the page contradicts. + assert.match(body, //); + assert.match(body, /2 trackers were detected in Example One Renamed/); }); await t.test('statistics page links its tables into the lookup pages', async () => { diff --git a/test/siteUrl.test.js b/test/siteUrl.test.js new file mode 100644 index 0000000..1214e9a --- /dev/null +++ b/test/siteUrl.test.js @@ -0,0 +1,58 @@ +'use strict'; + +const assert = require('node:assert/strict'); +const test = require('node:test'); +const siteUrl = require('../lib/siteUrl'); + +function withEnv(values, run) { + const original = { SITE_URL: process.env.SITE_URL, NODE_ENV: process.env.NODE_ENV }; + Object.assign(process.env, values); + for (const [key, value] of Object.entries(values)) + if (value === undefined) delete process.env[key]; + + try { + run(); + } finally { + for (const [key, value] of Object.entries(original)) { + if (value === undefined) delete process.env[key]; + else process.env[key] = value; + } + } +} + +const request = (protocol, host) => ({ protocol, get: () => host }); + +test('production refuses to resolve a base URL from the request', () => { + withEnv({ NODE_ENV: 'production', SITE_URL: undefined }, () => { + assert.equal(siteUrl.getSiteUrlConfigurationError(), 'SITE_URL is not set'); + assert.throws(() => siteUrl.assertSiteUrlConfiguration(), /SITE_URL is not set/); + // An untrusted Host header must not become a public canonical URL. + assert.throws(() => siteUrl.siteBaseUrl(request('http', 'evil.test')), /SITE_URL is not set/); + }); +}); + +test('a blank SITE_URL counts as unset in production', () => { + withEnv({ NODE_ENV: 'production', SITE_URL: ' ' }, () => { + assert.equal(siteUrl.getSiteUrlConfigurationError(), 'SITE_URL is not set'); + }); +}); + +test('a configured SITE_URL wins over the request and loses its trailing slashes', () => { + withEnv({ NODE_ENV: 'production', SITE_URL: 'https://ios.example.org//' }, () => { + assert.equal(siteUrl.getSiteUrlConfigurationError(), null); + assert.equal( + siteUrl.siteBaseUrl(request('http', 'evil.test')), + 'https://ios.example.org' + ); + }); +}); + +test('development falls back to the request origin', () => { + withEnv({ NODE_ENV: 'development', SITE_URL: undefined }, () => { + assert.equal(siteUrl.getSiteUrlConfigurationError(), null); + assert.equal( + siteUrl.siteBaseUrl(request('http', 'localhost:3000')), + 'http://localhost:3000' + ); + }); +}); From a9489f10e07a3166754671dea219a276bdb17511 Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 12 Aug 2026 16:45:16 +0000 Subject: [PATCH 6/7] Fold the methodology page into About, with visuals MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The methodology page repeated most of About and read as technical documentation rather than an explanation for a general reader: refresh intervals, analysis versions, database sizes, counting rules, and citation guidance. It also described how the analysis reaches an app's code in more detail than we want to publish. Drop the page and keep the part About was missing — what a report does and does not mean — in plainer language, carried by visuals rather than prose so About stays short: - A four-step diagram of how an app is analysed. - Side-by-side cards for what a report can and cannot show, which is the distinction most easily lost when a result is quoted. - The jurisdiction labels as a key, rendered from the same metadata the reports use, replacing the longer jurisdiction prose. The sampling caveat moves into an alert under the diagram. Nav, the directory, lookup, report and statistics pages, the sitemap and the README now point at /about and its section anchors. --- README.md | 5 +- public/css/styles.css | 77 +++++++++++++++++++ routes/index.js | 38 ++-------- test/lookupPages.test.js | 20 +++-- views/about.pug | 157 +++++++++++++++++++++++++++------------ views/directory.pug | 2 +- views/form.pug | 2 +- views/layout.pug | 2 - views/lookup.pug | 2 +- views/methodology.pug | 139 ---------------------------------- views/statistics.pug | 2 +- 11 files changed, 214 insertions(+), 232 deletions(-) delete mode 100644 views/methodology.pug diff --git a/README.md b/README.md index 1331c1f..4d9610e 100644 --- a/README.md +++ b/README.md @@ -17,7 +17,7 @@ The website also includes jurisdiction analysis, showing which companies and cou - Store current and historical analysis results. - Show tracker, permission, and jurisdiction summaries. - Reverse lookup: which apps contain a given tracker or a given company's trackers. -- A methodology page documenting sampling, detection, counting rules, and limitations. +- An about page explaining how apps are analysed, and what a report does and does not tell you. - Sitemap, `robots.txt`, canonical links, and Open Graph/Twitter card metadata. - Run the analyser from macOS or a Raspberry Pi host. @@ -46,8 +46,7 @@ static/ Static image assets | `/statistics` | Aggregate jurisdiction statistics | | `/trackers`, `/companies` | Directories of every tracker and company seen in an analysed app | | `/tracker/:slug`, `/company/:slug` | Reverse lookup: the apps a tracker or company was found in | -| `/methodology` | Sampling, detection, counting rules, limitations, and citation guidance | -| `/about` | Project background and contact | +| `/about` | How apps are analysed, what a report means, jurisdiction labels, project background and contact | | `/sitemap.xml`, `/robots.txt` | Crawler metadata | The reverse lookup pages are served from an inverted index built by diff --git a/public/css/styles.css b/public/css/styles.css index f44b507..13ec7c1 100644 --- a/public/css/styles.css +++ b/public/css/styles.css @@ -164,3 +164,80 @@ body { .listing-table th:first-child { border-right: 1px solid #666; } + +/* About page: how-an-app-is-analysed pipeline */ +.pipeline-step { + background-color: #fff; + border-radius: 8px; + box-shadow: 0 1px 3px rgba(0,0,0,0.08); + padding: 16px 12px; + height: 100%; + text-align: center; + position: relative; +} +.pipeline-icon { + color: #007bff; + margin-bottom: 8px; +} +.pipeline-step h4 { + font-size: 1rem; + font-weight: 600; + margin-bottom: 6px; +} +.pipeline-step p { + font-size: 0.85rem; + color: #6c757d; + margin-bottom: 0; +} +/* Chevrons between the steps, on the widths where they sit in one row. */ +@media (min-width: 768px) { + .pipeline [class^='col-']:not(:last-child) .pipeline-step::after { + content: ''; + position: absolute; + top: 50%; + right: -13px; + margin-top: -7px; + border-top: 7px solid transparent; + border-bottom: 7px solid transparent; + border-left: 8px solid #adb5bd; + } +} + +/* About page: what a report does and does not tell you */ +.result-card { + background-color: #fff; + border-radius: 8px; + border-left: 4px solid #6c757d; + box-shadow: 0 1px 3px rgba(0,0,0,0.08); + padding: 16px; + height: 100%; +} +.result-card h4 { + font-size: 1rem; + font-weight: 600; + margin-bottom: 10px; +} +.result-card ul { + padding-left: 20px; + font-size: 0.9rem; +} +.result-card li + li { + margin-top: 6px; +} +.result-card-yes { + border-left-color: #28a745; +} +.result-card-yes h4 { + color: #1e7e34; +} +.result-card-no { + border-left-color: #dc3545; +} +.result-card-no h4 { + color: #b02a37; +} + +/* Keep in-page anchors clear of the sticky navbar. */ +:target { + scroll-margin-top: 90px; +} diff --git a/routes/index.js b/routes/index.js index 4eb083c..334bcef 100644 --- a/routes/index.js +++ b/routes/index.js @@ -9,10 +9,6 @@ const reverseIndex = require('../lib/reverseIndex'); const { isValidAppId } = require('../lib/appId'); const { classifyAnalysisFailure } = require('../lib/analysisFailure'); const asyncHandler = require('../lib/asyncHandler'); -const { - CURRENT_ANALYSIS_VERSION, - STALE_ANALYSIS_DAYS -} = require('../lib/analysisPolicy'); const { buildReportMetadata, buildListingDetails } = require('../lib/appMetadata'); const { siteBaseUrl } = require('../lib/siteUrl'); @@ -639,38 +635,17 @@ router.post('/analysis/:appId', return res.redirect(303, `/analysis/${details.appId}`); })); -// About page +// About page: what this service does, what a report does and does not mean, +// where the country labels come from, and who is behind it. router.get('/about', (req, res) => { res.render('about', { title: 'About', pageDescription: 'How this service analyses iOS apps for embedded trackers, ' - + 'who runs it, and how to get in touch.' - }); -}); - -// Methodology page: how the numbers on this site are produced, and what they -// do and do not support. Written for reporters and researchers who need to -// check a claim before publishing it. -router.get('/methodology', asyncHandler(async (req, res) => { - let headlines = null; - try { - headlines = (await getSiteData()).headlines; - } catch (err) { - console.error('Methodology page stats unavailable:', err.message); - } - - res.render('methodology', { - title: 'Methodology', - pageDescription: 'How TrackerControl for iOS detects trackers, how apps are ' - + 'sampled, what the jurisdiction classifications mean, and the limits of ' - + 'the data.', - headlines, - analysisVersion: CURRENT_ANALYSIS_VERSION, - staleAnalysisDays: STALE_ANALYSIS_DAYS, - xrayCompanyCount: jurisdiction.xrayCompanyCount, + + 'what a report does and does not tell you, who runs it, and how to get ' + + 'in touch.', jurisdictionMeta: jurisdiction.classificationMeta }); -})); +}); /** * Render a directory of every tracker or company seen in an analysed app. @@ -914,8 +889,7 @@ function sitemapEntries(base, index) { sitemapEntry(base, '/statistics', { lastmod: updated, changefreq: 'daily', priority: '0.9' }), sitemapEntry(base, '/trackers', { lastmod: updated, changefreq: 'daily', priority: '0.9' }), sitemapEntry(base, '/companies', { lastmod: updated, changefreq: 'daily', priority: '0.8' }), - sitemapEntry(base, '/methodology', { changefreq: 'monthly', priority: '0.7' }), - sitemapEntry(base, '/about', { changefreq: 'monthly', priority: '0.5' }) + sitemapEntry(base, '/about', { changefreq: 'monthly', priority: '0.7' }) ]; for (const slug of index.trackerList) diff --git a/test/lookupPages.test.js b/test/lookupPages.test.js index bbe6002..fae394c 100644 --- a/test/lookupPages.test.js +++ b/test/lookupPages.test.js @@ -95,7 +95,7 @@ function stubDatabase() { }; } -test('reverse lookup, methodology, sitemap and social metadata', async (t) => { +test('reverse lookup, about page, sitemap and social metadata', async (t) => { const restore = stubDatabase(); try { @@ -158,13 +158,21 @@ test('reverse lookup, methodology, sitemap and social metadata', async (t) => { } }); - await t.test('methodology page renders with sampling caveats', async () => { - const response = await fetch(`${base}/methodology`); + await t.test('about page carries the sampling caveat and the jurisdiction labels', async () => { + const response = await fetch(`${base}/about`); const body = await response.text(); assert.equal(response.status, 200); - assert.match(body, /This is not a random sample of the App Store/); - assert.match(body, //); + assert.match(body, /this is not a random sample of the App Store/i); + // The labels are rendered from the same metadata the reports use, so + // the key cannot drift out of the page without the reports changing too. + assert.match(body, /Every company we identified is based in the US/); + assert.match(body, //); + }); + + await t.test('the retired methodology page is gone', async () => { + const response = await fetch(`${base}/methodology`); + assert.equal(response.status, 404); }); await t.test('app report links trackers to their lookup page and sets a social image', async () => { @@ -195,7 +203,7 @@ test('reverse lookup, methodology, sitemap and social metadata', async (t) => { assert.equal(response.status, 200); assert.match(response.headers.get('content-type'), /xml/); assert.match(body, /https:\/\/example.test\/<\/loc>/); - assert.match(body, /https:\/\/example.test\/methodology<\/loc>/); + assert.match(body, /https:\/\/example.test\/about<\/loc>/); assert.match(body, /https:\/\/example.test\/tracker\/google-firebase-analytics<\/loc>/); assert.match(body, /https:\/\/example.test\/company\/alphabet<\/loc>/); assert.match(body, /https:\/\/example.test\/analysis\/com.example.one<\/loc>/); diff --git a/views/about.pug b/views/about.pug index 7876d61..efcc895 100644 --- a/views/about.pug +++ b/views/about.pug @@ -3,59 +3,124 @@ extends layout block content h2 About TrackerControl for iOS p - | This service analyses iOS applications in order to - b list the embedded trackers and permissions - | . A tracker is a piece of software that collects data about - b you or your app usage behaviour - | . + | This service analyses iOS applications in order to #[b list the embedded trackers and permissions]. A tracker is a piece of software that collects data about #[b you or your app usage behaviour]. + + h3.mt-5#how-it-works How an app is analysed + .row.pipeline + .col-6.col-md-3.mb-3 + .pipeline-step + svg.pipeline-icon(xmlns='http://www.w3.org/2000/svg' width='34' height='34' viewBox='0 0 24 24' fill='none' stroke='currentColor' stroke-width='1.6' stroke-linecap='round' stroke-linejoin='round' role='img' aria-hidden='true') + path(d='M21 15v4a2 2 0 0 1-2 2H5a2 2 0 0 1-2-2v-4') + polyline(points='7 10 12 15 17 10') + line(x1='12' y1='15' x2='12' y2='3') + h4 Download + p We fetch the app from the UK App Store, like anyone else would. + .col-6.col-md-3.mb-3 + .pipeline-step + svg.pipeline-icon(xmlns='http://www.w3.org/2000/svg' width='34' height='34' viewBox='0 0 24 24' fill='none' stroke='currentColor' stroke-width='1.6' stroke-linecap='round' stroke-linejoin='round' role='img' aria-hidden='true') + rect(x='6' y='2' width='12' height='20' rx='2') + line(x1='11' y1='18' x2='13' y2='18') + h4 Run + p We install it on a real iPhone and start it up. + .col-6.col-md-3.mb-3 + .pipeline-step + svg.pipeline-icon(xmlns='http://www.w3.org/2000/svg' width='34' height='34' viewBox='0 0 24 24' fill='none' stroke='currentColor' stroke-width='1.6' stroke-linecap='round' stroke-linejoin='round' role='img' aria-hidden='true') + circle(cx='11' cy='11' r='7') + line(x1='16.5' y1='16.5' x2='21' y2='21') + h4 Look inside + p We list the tracking software the app comes with. + .col-6.col-md-3.mb-3 + .pipeline-step + svg.pipeline-icon(xmlns='http://www.w3.org/2000/svg' width='34' height='34' viewBox='0 0 24 24' fill='none' stroke='currentColor' stroke-width='1.6' stroke-linecap='round' stroke-linejoin='round' role='img' aria-hidden='true') + circle(cx='12' cy='12' r='9') + line(x1='3' y1='12' x2='21' y2='12') + path(d='M12 3a15 15 0 0 1 0 18a15 15 0 0 1 0-18z') + h4 Trace it back + p Each tracker is matched to the company behind it, and its home country. + .alert.alert-warning(role='alert') + p.mb-0#sample + | We analyse free apps from the UK App Store, and an app joins the queue when somebody searches for it here. #[b So this is not a random sample of the App Store.] It leans heavily towards popular apps: a percentage on this site describes the apps analysed so far, not iOS apps in general. + + h3.mt-5#results What a report tells you + .row + .col-md-6.mb-3 + .result-card.result-card-yes + h4 + svg(xmlns='http://www.w3.org/2000/svg' width='20' height='20' viewBox='0 0 24 24' fill='none' stroke='currentColor' stroke-width='2.5' stroke-linecap='round' stroke-linejoin='round' role='img' aria-hidden='true') + polyline(points='4 12 10 18 20 6') + | What we can see + ul.mb-0 + li Which tracking software is built into an app. + li Which companies it belongs to, and where they are based. + li Which permissions the app is able to ask you for. + .col-md-6.mb-3 + .result-card.result-card-no + h4 + svg(xmlns='http://www.w3.org/2000/svg' width='20' height='20' viewBox='0 0 24 24' fill='none' stroke='currentColor' stroke-width='2.5' stroke-linecap='round' stroke-linejoin='round' role='img' aria-hidden='true') + line(x1='6' y1='6' x2='18' y2='18') + line(x1='18' y1='6' x2='6' y2='18') + | What we cannot + ul.mb-0 + li Whether a tracker ever ran, or whether anything left your phone. + li Tracking we have no way of seeing, such as an app sending data straight to its developer's own servers. An empty report is not a promise. + li What a company does with your data once it has it. + + h3.mt-5#jurisdiction Who controls the tracking + p + | Each tracker is traced to the company that runs it, up to the parent company at the top, and then to that company's home country. What counts is whose laws a company answers to, not where its servers are: a US company can be compelled to hand data over under the CLOUD Act wherever that data is stored. Every app gets one label: + .table-responsive + table.table.table-sm.table-striped + tbody + tr + td + span.badge.badge-pill(class=jurisdictionMeta.no_tracking.cssClass) !{jurisdictionMeta.no_tracking.icon} #{jurisdictionMeta.no_tracking.label} + td No third-party tracker was found. + tr + td + span.badge.badge-pill(class=jurisdictionMeta.us_only.cssClass) !{jurisdictionMeta.us_only.icon} #{jurisdictionMeta.us_only.label} + td Every company we identified is based in the US. + tr + td + span.badge.badge-pill(class=jurisdictionMeta.european_only.cssClass) !{jurisdictionMeta.european_only.icon} #{jurisdictionMeta.european_only.label} + td Every company we identified is based in the EU, EEA or UK, and subject to GDPR. This is rare. + tr + td + span.badge.badge-pill(class=jurisdictionMeta.mixed_with_us_cn.cssClass) !{jurisdictionMeta.mixed_with_us_cn.icon} #{jurisdictionMeta.mixed_with_us_cn.label} + td The companies include both US and Chinese ones. + tr + td + span.badge.badge-pill(class=jurisdictionMeta.mixed_with_us.cssClass) !{jurisdictionMeta.mixed_with_us.icon} #{jurisdictionMeta.mixed_with_us.label} + td The companies include US ones alongside others, but none from China. + tr + td + span.badge.badge-pill(class=jurisdictionMeta.mixed_no_us.cssClass) !{jurisdictionMeta.mixed_no_us.icon} #{jurisdictionMeta.mixed_no_us.label} + td The companies span several parts of the world, but none is in the US. + tr + td + span.badge.badge-pill(class=jurisdictionMeta.unresolved_only.cssClass) !{jurisdictionMeta.unresolved_only.icon} #{jurisdictionMeta.unresolved_only.label} + td Trackers were found, but we could not tell which companies run them. + + h3.mt-5#limitations Where this stops + ul + li We can only find trackers we already know about, so new or renamed ones are missed. + li A tracker occasionally ends up attributed to the wrong company, so it is worth checking an individual result before leaning on it. + li A report describes the version of the app we analysed, on the date shown. That may no longer be the version in the App Store. p - | For a full account of how apps are sampled, how trackers are detected, what the results do and do not support, and how to cite them, see the #[a(href='/methodology') Methodology] page. + | If a result looks wrong, tell us — corrections make this better for everyone. + + h3.mt-5#background Where this comes from p | This project was motivated by #[a(target='_blank' rel='noopener noreferrer' href='https://exodus-privacy.eu.org/') Exodus Privacy], which is a similar project for Android apps. Some of the underlying code as well as the design of this website is based on this project. - p The underlying analysis technique for iOS apps was developed in the PhD research of Konrad Kollnig at the Department of Computer Science of the University of Oxford. - | This research was published in a range of academic papers and is available at - a(target='_blank' rel='noopener noreferrer' href='https://www.platformcontrol.org/') PlatformControl.org - | . This research was, in turn, based - a(target='_blank' rel='noopener noreferrer' href='https://sociam.org/mobile-app-x-ray') on previous work - | by the Oxford research group led by Sir Nigel Shadbolt and led to - a(target='_blank' rel='noopener noreferrer' href='https://trackercontrol.org/') TrackerControl for Android - | . + p + | The underlying analysis technique for iOS apps was developed in the PhD research of Konrad Kollnig at the Department of Computer Science of the University of Oxford. This research was published in a range of academic papers and is available at #[a(target='_blank' rel='noopener noreferrer' href='https://www.platformcontrol.org/') PlatformControl.org]. This research was, in turn, based #[a(target='_blank' rel='noopener noreferrer' href='https://sociam.org/mobile-app-x-ray') on previous work] by the Oxford research group led by Sir Nigel Shadbolt and led to #[a(target='_blank' rel='noopener noreferrer' href='https://trackercontrol.org/') TrackerControl for Android]. p A key aim of this research and this tool is to enable anyone to analyse privacy in iOS apps without relying on jailbreaks or the circumvention of copyright protections. These were issues that held back iOS research in the past. - p Unfortunately, Apple currently encrypts every app downloaded from the App Store with its FairPlay DRM. The circumvention of this DRM might be illegal in some jurisdictions and is thus not done by this tool. This is one of the key innovations behind this work, and has been published in the highly renowed and selective - a(target='_blank' rel='noopener noreferrer' href='https://petsymposium.org/popets/2022/popets-2022-0033.pdf') Proceedings on Privacy Enhancing Technologies - | in 2022. + p + | Unfortunately, Apple currently encrypts every app downloaded from the App Store with its FairPlay DRM. The circumvention of this DRM might be illegal in some jurisdictions and is thus not done by this tool. This is one of the key innovations behind this work, and has been published in the highly renowed and selective #[a(target='_blank' rel='noopener noreferrer' href='https://petsymposium.org/popets/2022/popets-2022-0033.pdf') Proceedings on Privacy Enhancing Technologies] in 2022. p This website operates a fair use policy and restricts disproportionate access. Scraping is not permitted. p We never collect personal data. It's yours. This website is operated by Konrad Kollnig, Assistant Professor at Maastricht University. - h3.mt-5 About Jurisdiction Analysis - p - | This tool also analyses the - b jurisdictional geography - | of third-party tracking: which countries control the tracking infrastructure in each app, and what that means for data sovereignty. - p - b What this measures: - | Which companies control the tracking infrastructure in your apps, and what jurisdiction those companies fall under. - p - b Why it matters: - | Even if data is processed in compliance with GDPR, US-based companies are subject to the CLOUD Act, which allows US authorities to compel disclosure regardless of where data is stored. The Schrems II ruling invalidated the EU-US Privacy Shield for this reason. + h3.mt-5#contact Contact p - b What "US-only" means: - | Every identified tracker in the app is controlled by a US-headquartered company. This doesn't mean the data is stored in the US — it means the company - em could - | be compelled to hand it over under US law. - p - b What "European-only" means: - | All tracking uses infrastructure controlled by companies in EU/UK/EEA countries, subject to GDPR. This is rare — across a study of ~24,000 apps, only 0.1% qualified. - - h4.mt-4 Limitations - ul - li Not all trackers can be identified (some hosts remain unresolved). - li Static analysis detects code presence even if inactive; dynamic analysis only captures traffic during testing. - li Company ownership changes over time; the database may not reflect the very latest acquisitions. - - h3.mt-5 Contact - p You can report issues and ask questions at - a(href="mailto:ios@trackercontrol.org") ios@trackercontrol.org - | . + | You can report issues and ask questions at #[a(href='mailto:ios@trackercontrol.org') ios@trackercontrol.org], or read the code at #[a(target='_blank' rel='noopener noreferrer' href='https://github.com/TrackerControl/tracker-control-ios') github.com/TrackerControl/tracker-control-ios]. block app diff --git a/views/directory.pug b/views/directory.pug index ba4a584..9b072ec 100644 --- a/views/directory.pug +++ b/views/directory.pug @@ -14,7 +14,7 @@ block content | #{trackedApps} of #{totalApps} analysed apps contain at least one detected tracker. if latestAnalysis | Last analysis: #{new Date(latestAnalysis).toLocaleDateString('en-GB', {day:'numeric', month:'short', year:'numeric'})}. - | See #[a(href='/methodology') how these numbers are produced]. + | See #[a(href='/about#how-it-works') how these numbers are produced]. .form-group.mt-3 input#directory-filter.form-control( diff --git a/views/form.pug b/views/form.pug index c14adb5..d4d5dde 100644 --- a/views/form.pug +++ b/views/form.pug @@ -162,7 +162,7 @@ block app | A tracker is a piece of software meant to collect data about you or your usages. a(target='_blank' rel='noopener noreferrer' href='https://reports.exodus-privacy.eu.org/en/info/trackers/') Learn more... p.text-muted.small.mt-2 - | Select a tracker to see every other analysed app it was found in. Detection means the tracker's code is present in this app, not that data was observed leaving it — see #[a(href='/methodology') Methodology]. + | Select a tracker to see every other analysed app it was found in. Detection means the tracker's software is built into this app, not that data was observed leaving it — see #[a(href='/about#results') what a report tells you]. if jurisdictionData .row.justify-content-sm-center.mb-5 diff --git a/views/layout.pug b/views/layout.pug index 7d6cad1..597443c 100644 --- a/views/layout.pug +++ b/views/layout.pug @@ -51,8 +51,6 @@ html a.nav-link(href='/trackers') Trackers li.nav-item.mr-xl-3.ml-xl-3.mr-lg-2.ml-lg-2 a.nav-link(href='/statistics') Statistics - li.nav-item.mr-xl-3.ml-xl-3.mr-lg-2.ml-lg-2 - a.nav-link(href='/methodology') Methodology li.nav-item.mr-xl-3.ml-xl-3.mr-lg-2.ml-lg-2 a.nav-link(href='/about') About diff --git a/views/lookup.pug b/views/lookup.pug index 6271dac..468db10 100644 --- a/views/lookup.pug +++ b/views/lookup.pug @@ -94,7 +94,7 @@ block content a.page-link(href=`?page=${pagination.page + 1}` rel='next') Next p.text-muted.small.mt-4 - | Detection means the tracker's code signature was found in the app binary. It does not by itself prove that data was transmitted while the app was in use. Percentages are shares of the #{totalApps} apps analysed so far, which are not a random sample of the App Store — see #[a(href='/methodology') Methodology] before quoting these figures. + | Detection means the tracker's software is built into the app. It does not by itself prove that data was transmitted while the app was in use. Percentages are shares of the #{totalApps} apps analysed so far, which are not a random sample of the App Store — see #[a(href='/about#results') About] before quoting these figures. if latestAnalysis | Data as of #{new Date(latestAnalysis).toLocaleDateString('en-GB', {day:'numeric', month:'short', year:'numeric'})}. diff --git a/views/methodology.pug b/views/methodology.pug deleted file mode 100644 index ac1f633..0000000 --- a/views/methodology.pug +++ /dev/null @@ -1,139 +0,0 @@ -extends layout - -block content - h2 Methodology - p.text-muted - | How the figures on this site are produced, what they support, and where they stop. Written for journalists, researchers and anyone checking a claim before publishing it. - if headlines && headlines.latestAnalysis - | Data as of #{new Date(headlines.latestAnalysis).toLocaleDateString('en-GB', {day:'numeric', month:'short', year:'numeric'})}. - - .card.mb-4 - .card-body - h4.card-title In one paragraph - p.mb-0 - | We download free iOS apps from the UK App Store, install each one on a physical iPhone, and list the third-party tracking libraries whose code is present in the app. We then map each tracker to the company that operates it, and that company to the country whose law it answers to. The result says what tracking code an app ships with, and who controls it. It does not say what data an app sent while you were using it. - - h3.mt-5 1. Which apps are analysed - p - | The sample covers #[b free apps on the UK App Store]. Paid apps are never queued, and results reflect the UK storefront; app binaries and bundled SDKs can differ by country. - ul - li An app enters the queue when someone looks it up on this site, so the sample follows public interest rather than a sampling frame. - li The queue is ordered by the number of App Store reviews stored for an app, so widely used apps are analysed first, then by the order in which apps were added. - li Apps are re-analysed when the analysis pipeline is upgraded, or when their last result is older than #{staleAnalysisDays} days. Results currently shown come from analysis version #{analysisVersion} or earlier. - .alert.alert-warning(role='alert') - p.mb-0 - | #[b This is not a random sample of the App Store.] It is demand-driven and skewed towards popular apps. A percentage on this site describes #[i the apps analysed so far], not "iOS apps" in general. A phrase such as "of the - if headlines && headlines.totalApps - | #{headlines.totalApps} - | apps analysed by TrackerControl" keeps that distinction visible. - - h3.mt-5 2. How apps are obtained - p - | Apps are downloaded through Apple's own distribution channel with a regular App Store account and installed on a real iPhone. Apple encrypts App Store binaries with its FairPlay DRM; #[b this project does not circumvent that DRM], because doing so may be unlawful in some jurisdictions. Analysis instead runs on the device, where the app is decrypted by the operating system in the normal course of being launched. - p - | Avoiding jailbreaks and DRM circumvention is what makes this analysis repeatable by others. The technique was published in the #[a(target='_blank' rel='noopener noreferrer' href='https://petsymposium.org/popets/2022/popets-2022-0033.pdf') Proceedings on Privacy Enhancing Technologies] in 2022. - - h3.mt-5 3. How trackers are detected - p - | Detection is #[b static]: it looks at what code is inside the app, not at network traffic. On the device, the analyser enumerates the classes contained in the installed app and matches them against a curated set of tracker signatures. A signature is a set of class names specific to one vendor's SDK, for example a Firebase, Braze or PubMatic class prefix. - p - | Signatures are only added when the class names are vendor-specific and are corroborated either by tracking domains the SDK is known to contact or by repeated, low-noise evidence across the app corpus. This is deliberately conservative: it prefers missing a tracker to inventing one. - p Two further pieces of evidence are collected per app: - ul - li #[b Tracking domains]: the domains an app itself declares in Apple's privacy manifests as being used for tracking. These are the developer's own declarations, not measured traffic. - li #[b Permissions]: the permission usage descriptions in the app's Info.plist, including those of its app extensions. This shows what an app is #[i able] to request, not what it requests or receives. - - .alert.alert-info(role='alert') - p #[b What a detection means:] the tracker's code ships inside the app. - p.mb-0 - | #[b What it does not mean:] that the tracker ran, that it collected anything, or that data left the device during use. Code can sit dormant, be gated behind a consent prompt, or be reached only in a region or account state we did not exercise. Conversely, absence of a detection is not proof of no tracking: first-party and server-side tracking are invisible to this method, and an SDK with no signature yet will not be counted. - - h3.mt-5 4. How jurisdiction is determined - p - | Each detected tracker is matched to the company that operates it, using the Xray tracker database (#{xrayCompanyCount} tracker companies) together with a small set of manual aliases for iOS SDK names that do not match a company name directly. Ownership is then followed to the #[b ultimate parent], so subsidiaries are counted under the group that owns them, and the parent's home country is used. - p Countries are grouped into regions: US, European (EU, EEA and the UK), CN, and Other. Each app then receives one classification: - .table-responsive - table.table.table-sm.table-striped - thead - tr - th Classification - th Applies when - tbody - tr - td - span.badge.badge-pill(class=jurisdictionMeta.no_tracking.cssClass) !{jurisdictionMeta.no_tracking.icon} #{jurisdictionMeta.no_tracking.label} - td No third-party tracker signature was detected. - tr - td - span.badge.badge-pill(class=jurisdictionMeta.us_only.cssClass) !{jurisdictionMeta.us_only.icon} #{jurisdictionMeta.us_only.label} - td Every identified tracker company is headquartered in the US. - tr - td - span.badge.badge-pill(class=jurisdictionMeta.european_only.cssClass) !{jurisdictionMeta.european_only.icon} #{jurisdictionMeta.european_only.label} - td Every identified tracker company is headquartered in the EU, EEA or UK. - tr - td - span.badge.badge-pill(class=jurisdictionMeta.mixed_with_us_cn.cssClass) !{jurisdictionMeta.mixed_with_us_cn.icon} #{jurisdictionMeta.mixed_with_us_cn.label} - td Identified companies include both US and Chinese ones. - tr - td - span.badge.badge-pill(class=jurisdictionMeta.mixed_with_us.cssClass) !{jurisdictionMeta.mixed_with_us.icon} #{jurisdictionMeta.mixed_with_us.label} - td Identified companies include US ones alongside others, but not Chinese ones. - tr - td - span.badge.badge-pill(class=jurisdictionMeta.mixed_no_us.cssClass) !{jurisdictionMeta.mixed_no_us.icon} #{jurisdictionMeta.mixed_no_us.label} - td Identified companies are outside the US and span more than one region. - tr - td - span.badge.badge-pill(class=jurisdictionMeta.unresolved_only.cssClass) !{jurisdictionMeta.unresolved_only.icon} #{jurisdictionMeta.unresolved_only.label} - td Trackers were detected, but none could be matched to a company. - p - | Jurisdiction here means #[b legal control over the company], not the physical location of a server. A US-headquartered company can be compelled to disclose data under the US CLOUD Act wherever that data is stored, which is why the classification follows corporate control rather than hosting. - - h3.mt-5 5. Counting rules - ul - li A tracker is counted once per app, however many times its code appears. - li A company is counted once per app, even when several of its trackers are present. - li Percentages on tracker and company pages use all successfully analysed apps as the denominator, including apps where no tracker was found. - li Apple system APIs that the analyser reports, such as advertising-identifier access, are labelled as system APIs and excluded from the jurisdiction analysis. - - h3.mt-5 6. Limitations - ul - li #[b Static analysis only.] Presence of code is not evidence of transmission, and this site publishes no traffic measurements. - li #[b Incomplete signatures.] Trackers without a signature are missed entirely. New SDKs and renamed classes lag behind. - li #[b Name matching can misattribute.] Tracker names are matched to companies partly by substring, which can attach a tracker to the wrong company where names overlap. Attribution for an individual tracker should be spot-checked before it carries weight in a story. - li #[b Ownership data ages.] Acquisitions change who ultimately controls a tracker, and the company database may lag behind the latest deal. - li #[b Results age.] An app's report reflects the version analysed on the date shown, which may not be the version in the store today. Results older than #{staleAnalysisDays} days are re-queued, so a report can lag a recent app update. - li #[b Storefront and price limits.] UK storefront, free apps only. Paid apps, enterprise apps and apps unavailable in the UK are out of scope. - li #[b Unresolved trackers.] Some detected signatures cannot be mapped to any company. Apps whose trackers are all unresolved are reported separately rather than folded into a jurisdiction. - - h3.mt-5 7. Using this data in reporting - p Claims the data supports: - ul - li The code of a named tracker is present in a named app, as analysed on the date and app version shown on its report page. - li Of the apps analysed by TrackerControl for iOS, a given number contain a tracker operated by a given company. - li Every tracker company identified in a given app is headquartered in the US. - p Claims the data does not support: - ul - li "This app sent your data to company Z." Detection shows shipped code, not observed transmission. - li "N% of iOS apps do X." The sample is demand-driven and skewed towards popular apps. - li "This app does not track you", on the basis of an empty result. Undetected and first-party tracking remain possible. - p - | #[b Before publishing:] check the analysis date and app version on the report page, confirm that the version you are writing about is the one analysed, and put the finding to the developer for comment. If a result looks wrong, tell us — corrections improve the signature set for everyone. - - h3.mt-5 8. Verifying and reproducing - ul - li #[b Request an analysis.] Searching for an app on this site queues it; the report page shows its position in the queue. - li #[b Read the code.] The website and the analyser pipeline are open source under the AGPLv3 at #[a(target='_blank' rel='noopener noreferrer' href='https://github.com/TrackerControl/tracker-control-ios') github.com/TrackerControl/tracker-control-ios], including the tracker signature sets and the jurisdiction mapping. - li #[b Read the research.] The underlying technique and its validation are documented at #[a(target='_blank' rel='noopener noreferrer' href='https://www.platformcontrol.org/') PlatformControl.org]. - li #[b Ask us.] For verification requests, methodology questions or corrections, write to #[a(href='mailto:ios@trackercontrol.org') ios@trackercontrol.org]. Please include the app, the date of the report, and the claim you intend to publish. - - h3.mt-5 9. Citing this site - p Please cite the specific page and the date you consulted it, because reports change as apps are re-analysed. For example: - blockquote.blockquote - p.mb-0.small - | TrackerControl for iOS, "Tracker report: Example App", consulted #{new Date().toLocaleDateString('en-GB', {day:'numeric', month:'long', year:'numeric'})}. - p.text-muted.small - | The analysis is research output of Konrad Kollnig (Maastricht University), building on work at the University of Oxford. See #[a(href='/about') About] for the people and projects behind it. - -block app diff --git a/views/statistics.pug b/views/statistics.pug index 5efb8be..4e1f4dc 100644 --- a/views/statistics.pug +++ b/views/statistics.pug @@ -150,6 +150,6 @@ block content small | Based on Xray Tracker Database (#{xrayCompanyCount} tracker companies). | - a(href='/methodology') How these numbers are produced + a(href='/about#how-it-works') How these numbers are produced block app From d0e0fd087a640013eaa053abc27fa14075988eb5 Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 12 Aug 2026 16:50:40 +0000 Subject: [PATCH 7/7] Fix text glued onto inline tags on the report page Pug joins sibling piped-text lines with a newline, but concatenates a sibling tag with no whitespace at all, so a text line that has lost its trailing space renders straight into the tag that follows it. Three places on the app report did: "your usages.Learn more...", "Based onXray Tracker Database", and "on your phone.Learn more...". Use inline interpolation instead of a bare tag line, so the space is written in the template rather than carried by trailing whitespace that an editor or a reformat can quietly strip. Add a test that renders every public page and fails on text sitting directly against a single inline tag. A run of several tags is two elements next to each other, which is a layout decision, so only single tags count. --- test/lookupPages.test.js | 21 +++++++++++++++++++++ views/form.pug | 9 +++------ 2 files changed, 24 insertions(+), 6 deletions(-) diff --git a/test/lookupPages.test.js b/test/lookupPages.test.js index fae394c..f521586 100644 --- a/test/lookupPages.test.js +++ b/test/lookupPages.test.js @@ -196,6 +196,27 @@ test('reverse lookup, about page, sitemap and social metadata', async (t) => { assert.match(body, /href="\/company\/alphabet"/); }); + await t.test('no page glues a word onto an inline tag', async () => { + // Pug joins sibling piped-text lines with a newline, but concatenates a + // sibling tag with no whitespace at all, so dropping the trailing space + // from `| ...read more` above `a(href=...) here` silently renders + // "read morehere". Catch it in the output rather than in review. + // One tag only: a run of several (``) is two + // elements sitting next to each other, which is a layout decision + // rather than a sentence that lost its space. + const inline = /<\/?(?:b|i|em|strong|a|span|code|abbr|small)\b[^>]*>/; + const glued = new RegExp(`[A-Za-z0-9.,:]${inline.source}[A-Za-z0-9]`, 'g'); + + for (const page of ['/', '/about', '/analysis/com.example.one', '/statistics', + '/trackers', '/companies', '/tracker/google-firebase-analytics']) { + const body = await (await fetch(`${base}${page}`)).text(); + const hits = [...body.matchAll(glued)] + .map((m) => body.slice(Math.max(0, m.index - 40), m.index + m[0].length + 20)); + + assert.deepEqual(hits, [], `${page} renders text with no space before or after an inline tag`); + } + }); + await t.test('sitemap covers reports, lookups and reference pages', async () => { const response = await fetch(`${base}/sitemap.xml`); const body = await response.text(); diff --git a/views/form.pug b/views/form.pug index d4d5dde..844f27a 100644 --- a/views/form.pug +++ b/views/form.pug @@ -159,8 +159,7 @@ block app =" " else span.text-muted No further information available for this tracker. - | A tracker is a piece of software meant to collect data about you or your usages. - a(target='_blank' rel='noopener noreferrer' href='https://reports.exodus-privacy.eu.org/en/info/trackers/') Learn more... + | A tracker is a piece of software meant to collect data about you or your usages. #[a(target='_blank' rel='noopener noreferrer' href='https://reports.exodus-privacy.eu.org/en/info/trackers/') Learn more...] p.text-muted.small.mt-2 | Select a tracker to see every other analysed app it was found in. Detection means the tracker's software is built into this app, not that data was observed leaving it — see #[a(href='/about#results') what a report tells you]. @@ -232,8 +231,7 @@ block app .text-muted.mt-3 small - | Based on - a(href='/about') Xray Tracker Database + | Based on #[a(href='/about') Xray Tracker Database] if app.analysis.trackingDomains && app.analysis.trackingDomains.length > 0 .row.justify-content-sm-center.mb-5 @@ -264,8 +262,7 @@ block app p.text-truncate span(style='padding-left:28px') span(data-toggle='tooltip' data-placement='top' title='' data-original-title=`NS${permission}UsageDescription`) #{permission} - | Permissions are actions the application can do on your phone. - a(target='_blank' rel='noopener noreferrer' href='https://reports.exodus-privacy.eu.org/en/info/permissions/') Learn more... + | Permissions are actions the application can do on your phone. #[a(target='_blank' rel='noopener noreferrer' href='https://reports.exodus-privacy.eu.org/en/info/permissions/') Learn more...] if app.analysisFailure