diff --git a/app/lib/apiUtils.ts b/app/lib/apiUtils.ts index 28f3fae..0d0b98b 100644 --- a/app/lib/apiUtils.ts +++ b/app/lib/apiUtils.ts @@ -4,6 +4,7 @@ import fs from 'node:fs'; import type { User } from './user'; import { getUserInfo, getUserTokenCookie } from './user'; +import { filterArray, IR } from './typescriptCommonTypes'; export async function getUser( request: NextApiRequest, @@ -56,3 +57,33 @@ export function noCaching(res: NextApiResponse): NextApiResponse { res.setHeader('Expires', '0'); return res; } + +export function formatUrl( + url: string, + parameters: IR, +): string { + const urlObject = new URL(url); + urlObject.search = new URLSearchParams({ + ...Object.fromEntries(urlObject.searchParams), + ...Object.fromEntries( + filterArray( + Object.entries(parameters).map(([key, value]) => + value === undefined || value === null + ? undefined + : [key, value.toString()] + ) + ) + ), + }).toString(); + return urlObject.toString(); +} + +export type Writable = { + -readonly [K in keyof T]: T[K]; +}; + +/** + * Cast type to writable. Equivalent to doing "as Writable", except this + * way, don't have to manually specify the generic type + */ +export const writable = (value: T): Writable => value; diff --git a/app/pages/api/dockerhub/[image].ts b/app/pages/api/dockerhub/[image].ts index caa4dbf..9b68c6c 100644 --- a/app/pages/api/dockerhub/[image].ts +++ b/app/pages/api/dockerhub/[image].ts @@ -1,6 +1,54 @@ import type { NextApiRequest, NextApiResponse } from 'next'; import type { IR, RA } from '../../../lib/typescriptCommonTypes'; +import { formatUrl, Writable } from '../../../lib/apiUtils'; + +// Docker Hub has a maximum of 10 pages for unauthenticated users +// At the 11th or higher page, Docker Hub will throw an error +const PAGE_MAX = 10; + +// The default maximum page size set by Docker +const MAX_PAGE_SIZE = 100; + +type TagOrderBy = + | "last_updated" + | "name" + | "tag_status" + | "tag_last_pulled" + | "tag_last_pushed"; + +type TagFilter = { + readonly orderBy?: TagOrderBy; + readonly name?: string; + readonly maxPages?: number; + readonly pageSize?: number; +}; + +const DEFAULT_TAG_FILTER: TagFilter = { + orderBy: "last_updated", + maxPages: PAGE_MAX, + pageSize: MAX_PAGE_SIZE, +}; + +export const SPECIAL_TAGS = { + "specify7-service": [ + { + // This is to make sure we have all of the v7 tags even if they're + // excluded from the main tag fetch + orderBy: "last_updated", + name: "v7", + }, + { + name: "main", + maxPages: 1, + }, + { + orderBy: "last_updated", + maxPages: 5, + }, + ], +} as const; + export type DockerHubTag = { readonly lastUpdated: string; @@ -8,13 +56,30 @@ export type DockerHubTag = { }; export const fetchTagsForImage = async ( - image: string + imageName: string, + options?: RA, ): Promise> => - fetchTags( - `https://hub.docker.com/v2/repositories/specifyconsortium/${image}/tags/?page_size=1000` - ).then(processTagsResponse); + Promise.allSettled( + (options ?? [DEFAULT_TAG_FILTER]).map( + async (filter) => await fetchTags(imageName, filter), + ), + ) + .then((results) => + results + .filter( + ( + result, + ): result is PromiseFulfilledResult => + result.status === "fulfilled", + ) + .map((result) => result.value), + ) + .then((results) => mergeTagResponses(results)) + .then(processTagsResponse); + + -type Response = { +type SuccessfulResponse = { readonly results: RA<{ readonly name: string; readonly last_updated: string; @@ -24,17 +89,71 @@ type Response = { }>; }>; readonly next: string | undefined; -}; +} + +type ErrorResponse = { + readonly errinfo: IR; + readonly message: string; +} + +type Response = SuccessfulResponse | ErrorResponse + +const mergeTagResponses = (responses: RA) => + responses.reduce( + (previous, current) => { + current.forEach((tag) => { + // We exclude already seen tags from the accumulated result + if (!previous.seenTags.has(tag.name)) { + previous.seenTags.add(tag.name); + previous.merged.push(tag); + } + }) + return previous; + }, + { + seenTags: new Set(), + merged: [] as Writable, + } + ).merged; -export const fetchTags = async (url: string): Promise => - fetch(url) +const urlFromFilter = ( + image: string, + filter: TagFilter, + currentPage: number = 1, +) => + formatUrl( + `https://hub.docker.com/v2/repositories/specifyconsortium/${image}/tags/`, + { + page_size: filter.pageSize ?? MAX_PAGE_SIZE, + page: currentPage, + ordering: filter.orderBy, + name: filter.name, + }, + ); + + +async function _fetchTags(url: string, filter: TagFilter, currentPage: number = 1): Promise { + return currentPage > (filter.maxPages ?? PAGE_MAX) ? Promise.resolve([]) : fetch(url) .then(async (response) => response.json()) - .then(async ({ results, next }: Response) => [ - ...results, - ...(typeof next === 'string' ? await fetchTags(next) : []), - ]); + .then(async (response: Response) => { + if ('message' in response) { + return [] + } + return [ + ...response.results, + ...(typeof response.next === 'string' ? + await _fetchTags(response.next, filter, currentPage + 1) : []) + ] + }); +} + +const fetchTags = async ( + imageName: string, + filter: TagFilter, +): Promise => + _fetchTags(urlFromFilter(imageName, filter), filter); -const processTagsResponse = (tags: Response['results']): IR => +const processTagsResponse = (tags: SuccessfulResponse['results']): IR => Object.fromEntries( tags // Latest is an unpredictable branch, thus will exclude it @@ -65,7 +184,8 @@ export default async function handler( res: NextApiResponse ) { const image = request.query.image as string; - await fetchTagsForImage(image) + const specialFilters = SPECIAL_TAGS[image as keyof typeof SPECIAL_TAGS] as RA | undefined; + await fetchTagsForImage(image, specialFilters) .then((tags) => res.status(200).json({ data: tags })) .catch((error) => res.status(500).json({ error: error.toString() })); } diff --git a/app/pages/api/state/index.ts b/app/pages/api/state/index.ts index 4b43105..5d0c8a5 100644 --- a/app/pages/api/state/index.ts +++ b/app/pages/api/state/index.ts @@ -16,7 +16,7 @@ import { createDockerConfig } from '../../../lib/dockerCompose'; import { createNginxConfig } from '../../../lib/nginx'; import type { RA } from '../../../lib/typescriptCommonTypes'; import type { User } from '../../../lib/user'; -import { fetchTagsForImage } from '../dockerhub/[image]'; +import { fetchTagsForImage, SPECIAL_TAGS } from '../dockerhub/[image]'; const configurationFile = path.resolve(stateDirectory, 'configuration.json'); const nginxConfigurationFile = path.resolve(nginxConfigDirectory, 'nginx.conf'); @@ -74,7 +74,7 @@ export async function setState( autoDeploy ); - const branches = await fetchTagsForImage('specify7-service'); + const branches = await fetchTagsForImage('specify7-service', SPECIAL_TAGS['specify7-service']); const state = await Promise.all( rawState.map(async (deployment) => { const hasInteralSp7ConfigDirectory = await branchHasConfigDirectory(deployment.branch);