Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion README.md
Original file line number Diff line number Diff line change
Expand Up @@ -44,7 +44,7 @@ File names: Latin letters, digits, `.`, `_` and `-`. Up to 8 MB per file and
catalog.lilka.dev (GitHub Pages)
site/ the website; reads the mirror below, and the API for
sign-in, publishing and moderation
apps/ wallpapers/ mods/ authors.json
apps/ wallpapers/ mods/ authors.json authors_info.json
static mirror of the API's /catalog for Keira devices

lilka-catalog-api (api/, Cloudflare)
Expand Down
3 changes: 2 additions & 1 deletion api/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -175,7 +175,8 @@ All bodies are JSON unless noted. Authenticated calls send
| GET | `/catalog/{apps,wallpapers,mods}/index_<n>.json` | mirror job | Ids of published items, 12 per page |
| GET | `/catalog/<dir>/<id>/index.json`, `index_short.json` | mirror job | Manifests in build.py's format |
| GET | `/catalog/<dir>/<id>/static/<file>`, `package.zip` | mirror job | Files |
| GET | `/catalog/authors.json` | mirror job | Items grouped by author |
| GET | `/catalog/authors.json` | mirror job | Items grouped by author line (build.py's format) |
| GET | `/catalog/authors_info.json` | mirror job | The website's Authors tab: one entry per person (GitHub logins merged ignoring case), public GitHub profile of authors who signed in, Printables links for mod authors, item counts and dates |
| GET | `/catalog/export.json` | mirror job | Everything above in one response, plus versions and a stamp |
| GET | `/apps?kind=&q=&page=` | user | Published apps, 24 per page |
| GET | `/apps/:slug` | user / owner | Detail with versions; owner also sees drafts and statuses |
Expand Down
7 changes: 7 additions & 0 deletions api/migrations/0005_profiles.sql
Original file line number Diff line number Diff line change
@@ -0,0 +1,7 @@
-- Public GitHub profile fields for the Authors page, refreshed at every
-- sign-in from the same /user response that gives the login. Only authors with
-- a published item are shown there.
ALTER TABLE users ADD COLUMN bio TEXT;
ALTER TABLE users ADD COLUMN blog TEXT;
ALTER TABLE users ADD COLUMN location TEXT;
ALTER TABLE users ADD COLUMN company TEXT;
10 changes: 9 additions & 1 deletion api/scripts/smoke.sh
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,8 @@ sha() { printf '%s' "$1" | shasum -a 256 | cut -d' ' -f1; }

npx wrangler d1 execute lilka-catalog --local --command "
DELETE FROM sessions; DELETE FROM ratings; DELETE FROM reports; UPDATE apps SET latest_version_id = NULL; DELETE FROM versions; DELETE FROM apps; DELETE FROM users WHERE id != 0;
INSERT INTO users (id, login, role) VALUES (1, 'smoke-dev', 'user'), (2, 'smoke-mod', 'moderator'), (3, 'smoke-fan', 'user');
INSERT INTO users (id, login, role, bio, blog) VALUES (1, 'smoke-dev', 'user', 'Makes <b>tests</b>', 'https://example.com/');
INSERT INTO users (id, login, role) VALUES (2, 'smoke-mod', 'moderator'), (3, 'smoke-fan', 'user');
INSERT INTO sessions VALUES ('$(sha $USER_TOKEN)', 1, datetime('now', '+1 day'));
INSERT INTO sessions VALUES ('$(sha $MOD_TOKEN)', 2, datetime('now', '+1 day'));
INSERT INTO sessions VALUES ('$(sha fan-token)', 3, datetime('now', '+1 day'));" >/dev/null
Expand Down Expand Up @@ -223,9 +224,16 @@ call 401 "anonymous user list refused" "$API/users/smoke-dev/apps"
call 401 "catalog without the mirror secret" "$API/catalog/export.json"
call 401 "catalog with a wrong secret" -H "Authorization: Bearer nope" "$API/catalog/apps/index_0.json"
call 401 "scanner route with the mirror secret" "${mirror[@]}" "$API/scan/pending"
# An author line naming someone else gets no profile: the owner's stays theirs
call 200 "author info: export with a foreign author line" "${mirror[@]}" "$API/catalog/authors_info.json"
jq -e '.authors | length == 1 and .[0].id == "historic" and .[0].registered == false and .[0].bio == null' "$work/body" >/dev/null && fa=ok || fa=bad
check "a foreign author line does not borrow the owner's profile" ok "$fa"
call 200 "moderator clears the author line" "${mod[@]}" "${json[@]}" -X PATCH "$API/apps/smoke-hello" -d '{"author":null}'
call 200 "export for the mirror job" "${mirror[@]}" "$API/catalog/export.json"
jq -e '.items["apps/smoke-hello"].version and .documents["apps/smoke-hello/index.json"]' "$work/body" >/dev/null && ex=ok || ex=bad
check "export lists the app" ok "$ex"
jq -e '.documents["authors_info.json"] | fromjson | .authors[] | select(.id == "smoke-dev") | .registered and .login == "smoke-dev" and .bio == "Makes <b>tests</b>" and .blog == "https://example.com/" and .counts.apps >= 1' "$work/body" >/dev/null && ai=ok || ai=bad
check "authors_info.json: one author per login, with the profile" ok "$ai"
latest=$(npx wrangler d1 execute lilka-catalog --local --json --command "SELECT latest_version_id FROM apps WHERE slug = 'smoke-hello'" 2>/dev/null | jq -r '.[0].results[0].latest_version_id')
check "latest_version_id is the approved version, not the infected one" "$version_id" "$latest"
location=$(curl -sS -o /dev/null -w '%{redirect_url}' "$API/auth/login?return=http://localhost:8000/")
Expand Down
42 changes: 40 additions & 2 deletions api/src/auth.ts
Original file line number Diff line number Diff line change
Expand Up @@ -57,6 +57,29 @@ interface GithubUser {
login: string;
name: string | null;
avatar_url: string | null;
bio?: string | null;
blog?: string | null;
location?: string | null;
company?: string | null;
}

/** Profile text is shown on the public Authors page: short, single-line, trimmed. */
function profileText(value: unknown, max: number): string | null {
if (typeof value !== 'string') return null;
const text = value.replace(/\s+/g, ' ').trim();
return text ? text.slice(0, max) : null;
}

/** GitHub lets "blog" be anything, often without a scheme; only http(s) links are kept. */
function profileUrl(value: unknown): string | null {
const text = profileText(value, 200);
if (!text) return null;
try {
const url = new URL(/^https?:\/\//i.test(text) ? text : `https://${text}`);
return url.protocol === 'https:' || url.protocol === 'http:' ? url.toString() : null;
} catch {
return null;
}
}

export const auth = new Hono<AppEnv>();
Expand Down Expand Up @@ -176,13 +199,28 @@ export async function createSession(env: Env, gh: GithubUser): Promise<string> {
gh.id,
),
env.DB.prepare(
`INSERT INTO users (id, login, name, avatar_url, role) VALUES (?1, ?2, ?3, ?4, ?5)
`INSERT INTO users (id, login, name, avatar_url, role, bio, blog, location, company)
VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9)
ON CONFLICT (id) DO UPDATE SET
login = excluded.login,
name = excluded.name,
avatar_url = excluded.avatar_url,
bio = excluded.bio,
blog = excluded.blog,
location = excluded.location,
company = excluded.company,
role = CASE WHEN users.role = 'banned' THEN 'banned' ELSE excluded.role END`,
).bind(gh.id, gh.login, gh.name, gh.avatar_url, role),
).bind(
gh.id,
gh.login,
profileText(gh.name, 100),
gh.avatar_url,
role,
profileText(gh.bio, 300),
profileUrl(gh.blog),
profileText(gh.location, 100),
profileText(gh.company, 100),
),
env.DB.prepare(
`INSERT INTO sessions (token_hash, user_id, expires_at) VALUES (?, ?, datetime('now', ?))`,
).bind(tokenHash, gh.id, `+${SESSION_DAYS} days`),
Expand Down
128 changes: 126 additions & 2 deletions api/src/catalog.ts
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@ import { objectKey } from './versions';
* /catalog/<dir>/<id>/index.json, index_short.json, package.zip
* /catalog/<dir>/<id>/static/<file>
* /catalog/authors.json
* /catalog/authors_info.json (website only, see authorsInfoDoc)
*
* Keira's lilcatalog requests these paths under CATALOG_BASE_URL, and the
* Pages mirror is a plain copy of this tree, so every field name and its order
Expand Down Expand Up @@ -42,6 +43,11 @@ interface CatalogRow extends AppRow {
package: string | null;
security: string | null;
storage_prefix: string;
owner_name: string | null;
owner_bio: string | null;
owner_blog: string | null;
owner_location: string | null;
owner_company: string | null;
}

/** Published apps with their latest approved version, sorted like Python's sorted(os.listdir()). */
Expand All @@ -57,7 +63,8 @@ async function published(env: Env, kind?: AppRow['kind'], slug?: string): Promis
params.push(slug);
}
const { results } = await env.DB.prepare(
`SELECT a.*, u.login AS owner_login, u.avatar_url AS owner_avatar,
`SELECT a.*, u.login AS owner_login, u.avatar_url AS owner_avatar, u.name AS owner_name,
u.bio AS owner_bio, u.blog AS owner_blog, u.location AS owner_location, u.company AS owner_company,
v.id AS v_id, v.keira_version, v.entry_type, v.entry_file, v.files, v.files_meta, v.icon, v.icon_min,
v.screenshots, v.changelog, v.changelog_en, v.package, v.security, v.storage_prefix,
v.approved_at
Expand Down Expand Up @@ -225,6 +232,119 @@ function authorsDoc(rows: CatalogRow[]): Record<string, unknown[]> {
return sorted;
}

const GITHUB_LOGIN = /^@([A-Za-z0-9](?:[A-Za-z0-9-]{0,37}[A-Za-z0-9])?)$/;
/** Mods credit their Printables author as `@Name_12345`, which is that profile's address. */
const PRINTABLES_HANDLE = /^@([A-Za-z0-9_-]+_\d+)$/;

/**
* authors_info.json: the website's Authors tab, one entry per person.
*
* authors.json groups by the exact author line, so "@ImbirWithoutSugar" from
* an old manifest and the owner's login "imbirwithoutsugar" were two authors.
* Here a line that is a GitHub login is grouped by that login, ignoring case,
* and an author who has signed in gets their public GitHub profile. Free-text
* lines (mods credit Printables names) stay as they are, without a profile.
* Keira does not read this file.
*/
function authorsInfoDoc(rows: CatalogRow[]) {
interface Author {
id: string;
name: string;
handle: string | null;
login: string | null;
github: string | null;
printables: string | null;
avatar: string | null;
bio: string | null;
location: string | null;
company: string | null;
blog: string | null;
registered: boolean;
aliases: string[];
counts: Record<'apps' | 'wallpapers' | 'mods', number>;
first_added: string | null;
last_updated: string | null;
items: Record<string, unknown>[];
}
const authors = new Map<string, Author>();
const min = (a: string | null, b: string | null) => (!a ? b : !b ? a : a < b ? a : b);
const max = (a: string | null, b: string | null) => (!a ? b : !b ? a : a > b ? a : b);

for (const row of rows) {
const line = displayAuthor(row);
const login = GITHUB_LOGIN.exec(line)?.[1] ?? null;
// The profile is the owner's only when the line names the owner
const own = !!login && login.toLowerCase() === row.owner_login.toLowerCase() && row.owner_id !== 0;
const id = login ? login.toLowerCase() : line.toLowerCase().replace(/[^\p{L}\p{N}]+/gu, '-').replace(/^-|-$/g, '') || 'unknown';
let author = authors.get(id);
if (!author) {
const printables = !login && PRINTABLES_HANDLE.test(line);
author = {
id,
// "@BFB_Workshop_1838755" reads as "BFB Workshop"; the handle stays in `handle`
name: login ?? (printables ? line.slice(1).replace(/_\d+$/, '').replace(/_/g, ' ') : line),
handle: login ? `@${login}` : printables ? line : null,
login,
github: login ? `https://github.com/${login}` : null,
printables: printables ? `https://www.printables.com/${line}` : null,
// github.com/<login>.png redirects to the avatar, also for people who never signed in here
avatar: login ? `https://github.com/${login}.png?size=160` : null,
bio: null,
location: null,
company: null,
blog: null,
registered: false,
aliases: [],
counts: { apps: 0, wallpapers: 0, mods: 0 },
first_added: null,
last_updated: null,
items: [],
};
authors.set(id, author);
}
if (own && !author.registered) {
Object.assign(author, {
// GitHub's spelling of the login, not the old manifest's
name: row.owner_name || row.owner_login,
handle: `@${row.owner_login}`,
login: row.owner_login,
github: `https://github.com/${row.owner_login}`,
avatar: row.owner_avatar ?? author.avatar,
bio: row.owner_bio,
location: row.owner_location,
company: row.owner_company,
blog: row.owner_blog,
registered: true,
});
}
if (!author.aliases.includes(line)) author.aliases.push(line);

const dir = kindDir(row.kind);
const added = row.published_at ?? row.approved_at;
author.counts[dir] += 1;
author.first_added = min(author.first_added, added);
author.last_updated = max(author.last_updated, row.approved_at);
const loc = localization(row);
const entry: Record<string, unknown> = {
name: pick(loc, 'name'),
short_description: pick(loc, 'short_description'),
icon: row.icon ?? '',
path: row.slug,
type: dir,
added,
updated: row.approved_at,
};
if (Object.keys(loc).length) {
entry.languages = Object.keys(loc);
entry.localization = onlyFields(loc, ['name', 'short_description']);
}
author.items.push(entry);
}
const list = [...authors.values()].sort((a, b) => (a.id < b.id ? -1 : a.id > b.id ? 1 : 0));
for (const author of list) author.items.sort((a, b) => String(b.updated ?? '').localeCompare(String(a.updated ?? '')));
return { authors: list };
}

/**
* listing.json: everything the website's list needs for one kind, in one file.
*
Expand Down Expand Up @@ -267,6 +387,7 @@ function indexDocs(rows: CatalogRow[]) {
}

catalog.get('/authors.json', async (c) => json(c, authorsDoc(await published(c.env))));
catalog.get('/authors_info.json', async (c) => json(c, authorsInfoDoc(await published(c.env))));

catalog.get('/:dir/listing.json', async (c) => json(c, listingDoc(await published(c.env, dirKind(c.req.param('dir'))))));

Expand All @@ -289,7 +410,10 @@ catalog.get('/:dir/:page{index_[0-9]+\\.json}', async (c) => {
*/
catalog.get('/export.json', async (c) => {
const rows = await published(c.env);
const documents: Record<string, string> = { 'authors.json': text(authorsDoc(rows)) };
const documents: Record<string, string> = {
'authors.json': text(authorsDoc(rows)),
'authors_info.json': text(authorsInfoDoc(rows)),
};
const items: Record<string, { version: number; files: string[] }> = {};
for (const dir of Object.keys(DIRS)) {
const ofKind = rows.filter((row) => kindDir(row.kind) === dir);
Expand Down
Loading
Loading