From 2c3db3c4471980d4b038c1970d446e49ee663834 Mon Sep 17 00:00:00 2001 From: melton-jason Date: Mon, 10 Aug 2026 23:16:28 -0500 Subject: [PATCH 1/9] fix: handle docker pagination limits --- app/lib/apiUtils.ts | 31 ++++++++ app/pages/api/dockerhub/[image].ts | 121 +++++++++++++++++++++++++---- app/pages/api/state/index.ts | 4 +- 3 files changed, 141 insertions(+), 15 deletions(-) diff --git a/app/lib/apiUtils.ts b/app/lib/apiUtils.ts index 28f3fae..0d0b98b 100644 --- a/app/lib/apiUtils.ts +++ b/app/lib/apiUtils.ts @@ -4,6 +4,7 @@ import fs from 'node:fs'; import type { User } from './user'; import { getUserInfo, getUserTokenCookie } from './user'; +import { filterArray, IR } from './typescriptCommonTypes'; export async function getUser( request: NextApiRequest, @@ -56,3 +57,33 @@ export function noCaching(res: NextApiResponse): NextApiResponse { res.setHeader('Expires', '0'); return res; } + +export function formatUrl( + url: string, + parameters: IR, +): string { + const urlObject = new URL(url); + urlObject.search = new URLSearchParams({ + ...Object.fromEntries(urlObject.searchParams), + ...Object.fromEntries( + filterArray( + Object.entries(parameters).map(([key, value]) => + value === undefined || value === null + ? undefined + : [key, value.toString()] + ) + ) + ), + }).toString(); + return urlObject.toString(); +} + +export type Writable = { + -readonly [K in keyof T]: T[K]; +}; + +/** + * Cast type to writable. Equivalent to doing "as Writable", except this + * way, don't have to manually specify the generic type + */ +export const writable = (value: T): Writable => value; diff --git a/app/pages/api/dockerhub/[image].ts b/app/pages/api/dockerhub/[image].ts index caa4dbf..7036b5d 100644 --- a/app/pages/api/dockerhub/[image].ts +++ b/app/pages/api/dockerhub/[image].ts @@ -1,6 +1,48 @@ import type { NextApiRequest, NextApiResponse } from 'next'; import type { IR, RA } from '../../../lib/typescriptCommonTypes'; +import { formatUrl, Writable } from '../../../lib/apiUtils'; + +// Docker Hub has a maximum of 10 pages for unauthenticated users +// At the 11th or higher page, Docker Hub will throw an error +const PAGE_MAX = 10; + +// The default maximum page size set by Docker +const MAX_PAGE_SIZE = 100; + +type TagOrderBy = 'last_updated' | 'name' | 'tag_status' | 'tag_last_pulled' | 'tag_last_pushed'; + +type TagFilter = { + readonly orderBy?: TagOrderBy; + readonly name?: string; + readonly maxPages?: number; + readonly pageSize?: number; +} + +const DEFAULT_TAG_FILTER: TagFilter = { + orderBy: 'last_updated', + maxPages: PAGE_MAX, + pageSize: MAX_PAGE_SIZE +} + +export const SPECIAL_TAGS = ({ + 'specify7-service': [ + { + // This is to make sure we have all of the v7 tags even if they're + // excluded from the main tag fetch + orderBy: 'last_updated', + name: 'v7' + }, + { + name: 'main', + maxPages: 1 + }, + { + orderBy: 'last_updated', + maxPages: 5 + } + ] +} as const) export type DockerHubTag = { readonly lastUpdated: string; @@ -8,13 +50,24 @@ export type DockerHubTag = { }; export const fetchTagsForImage = async ( - image: string + imageName: string, + options?: RA, ): Promise> => - fetchTags( - `https://hub.docker.com/v2/repositories/specifyconsortium/${image}/tags/?page_size=1000` - ).then(processTagsResponse); + Promise.allSettled( + (options ?? [DEFAULT_TAG_FILTER]).map( + async (filter) => await fetchTags(imageName, filter), + ) + ) + .then((results) => + results + .filter((result) => result.status === "fulfilled") + .map((result) => result.value), + ) + .then((results) => mergeTagResponses(results)) + .then(processTagsResponse); -type Response = { + +type SuccessfulResponse = { readonly results: RA<{ readonly name: string; readonly last_updated: string; @@ -24,17 +77,59 @@ type Response = { }>; }>; readonly next: string | undefined; -}; +} + +type ErrorResponse = { + readonly errinfo: IR; + readonly message: string; +} + +type Response = SuccessfulResponse | ErrorResponse + +const mergeTagResponses = (responses: RA) => + responses.reduce( + (previous, current) => { + current.forEach((tag) => { + // We exclude already seen tags from the accumulated result + if (!previous.seenDigests.has(tag.digest)) { + previous.seenDigests.add(tag.digest); + previous.merged.push(tag); + } + }) + return previous; + }, + { + seenDigests: new Set(), + merged: [] as Writable, + } + ).merged; -export const fetchTags = async (url: string): Promise => - fetch(url) +const urlFromFilter = (image: string, filter: TagFilter, currentPage: number = 1) => formatUrl(`https://hub.docker.com/v2/repositories/specifyconsortium/${image}/tags/`, { + pageSize: filter.pageSize ?? MAX_PAGE_SIZE, + maxPages: filter.maxPages ?? PAGE_MAX, + page: currentPage, + orderby: filter.orderBy, + name: filter.name +}) + +async function _fetchTags(url: string, currentPage: number = 1): Promise { + return currentPage > PAGE_MAX ? [] : fetch(url) .then(async (response) => response.json()) - .then(async ({ results, next }: Response) => [ - ...results, - ...(typeof next === 'string' ? await fetchTags(next) : []), - ]); + .then(async (response: Response) => { + if ('message' in response) { + return [] + } + return [ + ...response.results, + ...(typeof response.next === 'string' ? await _fetchTags(response.next, currentPage + 1) : []) + ] + }); +} + +const fetchTags = async (imageName: string, filter: TagFilter): Promise => _fetchTags(urlFromFilter(imageName, filter)); + -const processTagsResponse = (tags: Response['results']): IR => +const processTagsResponse = (tags: SuccessfulResponse['results']): IR => Object.fromEntries( tags // Latest is an unpredictable branch, thus will exclude it diff --git a/app/pages/api/state/index.ts b/app/pages/api/state/index.ts index 4b43105..5d0c8a5 100644 --- a/app/pages/api/state/index.ts +++ b/app/pages/api/state/index.ts @@ -16,7 +16,7 @@ import { createDockerConfig } from '../../../lib/dockerCompose'; import { createNginxConfig } from '../../../lib/nginx'; import type { RA } from '../../../lib/typescriptCommonTypes'; import type { User } from '../../../lib/user'; -import { fetchTagsForImage } from '../dockerhub/[image]'; +import { fetchTagsForImage, SPECIAL_TAGS } from '../dockerhub/[image]'; const configurationFile = path.resolve(stateDirectory, 'configuration.json'); const nginxConfigurationFile = path.resolve(nginxConfigDirectory, 'nginx.conf'); @@ -74,7 +74,7 @@ export async function setState( autoDeploy ); - const branches = await fetchTagsForImage('specify7-service'); + const branches = await fetchTagsForImage('specify7-service', SPECIAL_TAGS['specify7-service']); const state = await Promise.all( rawState.map(async (deployment) => { const hasInteralSp7ConfigDirectory = await branchHasConfigDirectory(deployment.branch); From 2cbcbd79b1331cc4b13477116417664c1bc67532 Mon Sep 17 00:00:00 2001 From: melton-jason Date: Mon, 10 Aug 2026 23:24:17 -0500 Subject: [PATCH 2/9] chore: add bug comment --- app/pages/api/dockerhub/[image].ts | 1 + 1 file changed, 1 insertion(+) diff --git a/app/pages/api/dockerhub/[image].ts b/app/pages/api/dockerhub/[image].ts index 7036b5d..b64ac1f 100644 --- a/app/pages/api/dockerhub/[image].ts +++ b/app/pages/api/dockerhub/[image].ts @@ -121,6 +121,7 @@ async function _fetchTags(url: string, currentPage: number = 1): Promise Date: Mon, 10 Aug 2026 23:28:38 -0500 Subject: [PATCH 3/9] chore: add type guard and reformat --- app/pages/api/dockerhub/[image].ts | 78 +++++++++++++++++++----------- 1 file changed, 51 insertions(+), 27 deletions(-) diff --git a/app/pages/api/dockerhub/[image].ts b/app/pages/api/dockerhub/[image].ts index b64ac1f..934d8a8 100644 --- a/app/pages/api/dockerhub/[image].ts +++ b/app/pages/api/dockerhub/[image].ts @@ -10,39 +10,45 @@ const PAGE_MAX = 10; // The default maximum page size set by Docker const MAX_PAGE_SIZE = 100; -type TagOrderBy = 'last_updated' | 'name' | 'tag_status' | 'tag_last_pulled' | 'tag_last_pushed'; +type TagOrderBy = + | "last_updated" + | "name" + | "tag_status" + | "tag_last_pulled" + | "tag_last_pushed"; type TagFilter = { readonly orderBy?: TagOrderBy; readonly name?: string; readonly maxPages?: number; readonly pageSize?: number; -} +}; const DEFAULT_TAG_FILTER: TagFilter = { - orderBy: 'last_updated', + orderBy: "last_updated", maxPages: PAGE_MAX, - pageSize: MAX_PAGE_SIZE -} + pageSize: MAX_PAGE_SIZE, +}; -export const SPECIAL_TAGS = ({ - 'specify7-service': [ +export const SPECIAL_TAGS = { + "specify7-service": [ { // This is to make sure we have all of the v7 tags even if they're // excluded from the main tag fetch - orderBy: 'last_updated', - name: 'v7' + orderBy: "last_updated", + name: "v7", }, { - name: 'main', - maxPages: 1 + name: "main", + maxPages: 1, }, { - orderBy: 'last_updated', - maxPages: 5 - } - ] -} as const) + orderBy: "last_updated", + maxPages: 5, + }, + ], +} as const; + export type DockerHubTag = { readonly lastUpdated: string; @@ -56,17 +62,23 @@ export const fetchTagsForImage = async ( Promise.allSettled( (options ?? [DEFAULT_TAG_FILTER]).map( async (filter) => await fetchTags(imageName, filter), - ) + ), ) .then((results) => results - .filter((result) => result.status === "fulfilled") + .filter( + ( + result, + ): result is PromiseFulfilledResult => + result.status === "fulfilled", + ) .map((result) => result.value), ) .then((results) => mergeTagResponses(results)) .then(processTagsResponse); + type SuccessfulResponse = { readonly results: RA<{ readonly name: string; @@ -104,13 +116,22 @@ const mergeTagResponses = (responses: RA) => } ).merged; -const urlFromFilter = (image: string, filter: TagFilter, currentPage: number = 1) => formatUrl(`https://hub.docker.com/v2/repositories/specifyconsortium/${image}/tags/`, { - pageSize: filter.pageSize ?? MAX_PAGE_SIZE, - maxPages: filter.maxPages ?? PAGE_MAX, - page: currentPage, - orderby: filter.orderBy, - name: filter.name -}) +const urlFromFilter = ( + image: string, + filter: TagFilter, + currentPage: number = 1, +) => + formatUrl( + `https://hub.docker.com/v2/repositories/specifyconsortium/${image}/tags/`, + { + pageSize: filter.pageSize ?? MAX_PAGE_SIZE, + maxPages: filter.maxPages ?? PAGE_MAX, + page: currentPage, + orderby: filter.orderBy, + name: filter.name, + }, + ); + async function _fetchTags(url: string, currentPage: number = 1): Promise { return currentPage > PAGE_MAX ? [] : fetch(url) @@ -127,8 +148,11 @@ async function _fetchTags(url: string, currentPage: number = 1): Promise => _fetchTags(urlFromFilter(imageName, filter)); - +const fetchTags = async ( + imageName: string, + filter: TagFilter, +): Promise => + _fetchTags(urlFromFilter(imageName, filter)); const processTagsResponse = (tags: SuccessfulResponse['results']): IR => Object.fromEntries( From ac8c90bb6f0ec1879cd792738a664dc69c6d4e67 Mon Sep 17 00:00:00 2001 From: melton-jason Date: Mon, 10 Aug 2026 23:51:41 -0500 Subject: [PATCH 4/9] fix: misc fixes --- app/pages/api/dockerhub/[image].ts | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/app/pages/api/dockerhub/[image].ts b/app/pages/api/dockerhub/[image].ts index 934d8a8..9697f85 100644 --- a/app/pages/api/dockerhub/[image].ts +++ b/app/pages/api/dockerhub/[image].ts @@ -124,17 +124,16 @@ const urlFromFilter = ( formatUrl( `https://hub.docker.com/v2/repositories/specifyconsortium/${image}/tags/`, { - pageSize: filter.pageSize ?? MAX_PAGE_SIZE, - maxPages: filter.maxPages ?? PAGE_MAX, + page_size: filter.pageSize ?? MAX_PAGE_SIZE, page: currentPage, - orderby: filter.orderBy, + ordering: filter.orderBy, name: filter.name, }, ); async function _fetchTags(url: string, currentPage: number = 1): Promise { - return currentPage > PAGE_MAX ? [] : fetch(url) + return currentPage > PAGE_MAX ? Promise.resolve([]) : fetch(url) .then(async (response) => response.json()) .then(async (response: Response) => { if ('message' in response) { From 5364e902b1742e0eb53aecb5e9c5d87a1de15d85 Mon Sep 17 00:00:00 2001 From: melton-jason Date: Tue, 11 Aug 2026 00:03:26 -0500 Subject: [PATCH 5/9] fix: page max not being respected in filter --- app/pages/api/dockerhub/[image].ts | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/app/pages/api/dockerhub/[image].ts b/app/pages/api/dockerhub/[image].ts index 9697f85..d9e5f08 100644 --- a/app/pages/api/dockerhub/[image].ts +++ b/app/pages/api/dockerhub/[image].ts @@ -120,8 +120,8 @@ const urlFromFilter = ( image: string, filter: TagFilter, currentPage: number = 1, -) => - formatUrl( +) => { + const url = formatUrl( `https://hub.docker.com/v2/repositories/specifyconsortium/${image}/tags/`, { page_size: filter.pageSize ?? MAX_PAGE_SIZE, @@ -130,10 +130,13 @@ const urlFromFilter = ( name: filter.name, }, ); + console.log(`Sending URL: ${url}`); + return url; +} -async function _fetchTags(url: string, currentPage: number = 1): Promise { - return currentPage > PAGE_MAX ? Promise.resolve([]) : fetch(url) +async function _fetchTags(url: string, filter: TagFilter, currentPage: number = 1): Promise { + return currentPage > (filter.maxPages ?? PAGE_MAX) ? Promise.resolve([]) : fetch(url) .then(async (response) => response.json()) .then(async (response: Response) => { if ('message' in response) { @@ -141,8 +144,8 @@ async function _fetchTags(url: string, currentPage: number = 1): Promise => - _fetchTags(urlFromFilter(imageName, filter)); + _fetchTags(urlFromFilter(imageName, filter), filter); const processTagsResponse = (tags: SuccessfulResponse['results']): IR => Object.fromEntries( From 3a349a8d6f6ab871a9481bdf7d8283be311c0933 Mon Sep 17 00:00:00 2001 From: melton-jason Date: Tue, 11 Aug 2026 00:13:44 -0500 Subject: [PATCH 6/9] fix: use special filters in backend handler --- app/pages/api/dockerhub/[image].ts | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/app/pages/api/dockerhub/[image].ts b/app/pages/api/dockerhub/[image].ts index d9e5f08..b7b2984 100644 --- a/app/pages/api/dockerhub/[image].ts +++ b/app/pages/api/dockerhub/[image].ts @@ -120,8 +120,8 @@ const urlFromFilter = ( image: string, filter: TagFilter, currentPage: number = 1, -) => { - const url = formatUrl( +) => + formatUrl( `https://hub.docker.com/v2/repositories/specifyconsortium/${image}/tags/`, { page_size: filter.pageSize ?? MAX_PAGE_SIZE, @@ -130,9 +130,6 @@ const urlFromFilter = ( name: filter.name, }, ); - console.log(`Sending URL: ${url}`); - return url; -} async function _fetchTags(url: string, filter: TagFilter, currentPage: number = 1): Promise { @@ -187,7 +184,8 @@ export default async function handler( res: NextApiResponse ) { const image = request.query.image as string; - await fetchTagsForImage(image) + const specialFilters = SPECIAL_TAGS[image as keyof typeof SPECIAL_TAGS] as RA | undefined; + await fetchTagsForImage(image, specialFilters) .then((tags) => res.status(200).json({ data: tags })) .catch((error) => res.status(500).json({ error: error.toString() })); } From bce44c6e4726d7df97dbafc61cd111baded25589 Mon Sep 17 00:00:00 2001 From: melton-jason Date: Tue, 11 Aug 2026 00:19:20 -0500 Subject: [PATCH 7/9] chore: add debug statements --- app/pages/api/dockerhub/[image].ts | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/app/pages/api/dockerhub/[image].ts b/app/pages/api/dockerhub/[image].ts index b7b2984..a85702e 100644 --- a/app/pages/api/dockerhub/[image].ts +++ b/app/pages/api/dockerhub/[image].ts @@ -120,7 +120,7 @@ const urlFromFilter = ( image: string, filter: TagFilter, currentPage: number = 1, -) => +) => formatUrl( `https://hub.docker.com/v2/repositories/specifyconsortium/${image}/tags/`, { @@ -150,8 +150,11 @@ async function _fetchTags(url: string, filter: TagFilter, currentPage: number = const fetchTags = async ( imageName: string, filter: TagFilter, -): Promise => - _fetchTags(urlFromFilter(imageName, filter), filter); +): Promise => { + const url = urlFromFilter(imageName, filter); + console.log("fetching URL ", url, " with filter ", filter); + return _fetchTags(urlFromFilter(imageName, filter), filter); +} const processTagsResponse = (tags: SuccessfulResponse['results']): IR => Object.fromEntries( From 6b90538e3f30e852f8333a8027324f0e03b99102 Mon Sep 17 00:00:00 2001 From: melton-jason Date: Tue, 11 Aug 2026 00:30:11 -0500 Subject: [PATCH 8/9] Revert "chore: add debug statements" This reverts commit bce44c6e4726d7df97dbafc61cd111baded25589. --- app/pages/api/dockerhub/[image].ts | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/app/pages/api/dockerhub/[image].ts b/app/pages/api/dockerhub/[image].ts index a85702e..b7b2984 100644 --- a/app/pages/api/dockerhub/[image].ts +++ b/app/pages/api/dockerhub/[image].ts @@ -120,7 +120,7 @@ const urlFromFilter = ( image: string, filter: TagFilter, currentPage: number = 1, -) => +) => formatUrl( `https://hub.docker.com/v2/repositories/specifyconsortium/${image}/tags/`, { @@ -150,11 +150,8 @@ async function _fetchTags(url: string, filter: TagFilter, currentPage: number = const fetchTags = async ( imageName: string, filter: TagFilter, -): Promise => { - const url = urlFromFilter(imageName, filter); - console.log("fetching URL ", url, " with filter ", filter); - return _fetchTags(urlFromFilter(imageName, filter), filter); -} +): Promise => + _fetchTags(urlFromFilter(imageName, filter), filter); const processTagsResponse = (tags: SuccessfulResponse['results']): IR => Object.fromEntries( From e6a71619865ce2cdaf8428270bfb28aa9e4f7691 Mon Sep 17 00:00:00 2001 From: melton-jason Date: Tue, 11 Aug 2026 00:31:27 -0500 Subject: [PATCH 9/9] fix: dedupe by tage name rather than digest --- app/pages/api/dockerhub/[image].ts | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/app/pages/api/dockerhub/[image].ts b/app/pages/api/dockerhub/[image].ts index b7b2984..9b68c6c 100644 --- a/app/pages/api/dockerhub/[image].ts +++ b/app/pages/api/dockerhub/[image].ts @@ -103,15 +103,15 @@ const mergeTagResponses = (responses: RA) => (previous, current) => { current.forEach((tag) => { // We exclude already seen tags from the accumulated result - if (!previous.seenDigests.has(tag.digest)) { - previous.seenDigests.add(tag.digest); + if (!previous.seenTags.has(tag.name)) { + previous.seenTags.add(tag.name); previous.merged.push(tag); } }) return previous; }, { - seenDigests: new Set(), + seenTags: new Set(), merged: [] as Writable, } ).merged; @@ -120,7 +120,7 @@ const urlFromFilter = ( image: string, filter: TagFilter, currentPage: number = 1, -) => +) => formatUrl( `https://hub.docker.com/v2/repositories/specifyconsortium/${image}/tags/`, {