fix: handle docker pagination limits

app/lib/apiUtils.ts

@@ -4,6 +4,7 @@ import fs from 'node:fs';
 
 import type { User } from './user';
 import { getUserInfo, getUserTokenCookie } from './user';
+import { filterArray, IR } from './typescriptCommonTypes';
 
 export async function getUser(
   request: NextApiRequest,
@@ -56,3 +57,33 @@ export function noCaching(res: NextApiResponse): NextApiResponse {
   res.setHeader('Expires', '0');
   return res;
 }
+
+export function formatUrl(
+  url: string,
+  parameters: IR<number | string | null | undefined>,
+): string {
+  const urlObject = new URL(url);
+  urlObject.search = new URLSearchParams({
+    ...Object.fromEntries(urlObject.searchParams),
+    ...Object.fromEntries(
+      filterArray(
+        Object.entries(parameters).map(([key, value]) =>
+          value === undefined || value === null
+            ? undefined
+            : [key, value.toString()]
+        )
+      )
+    ),
+  }).toString();
+  return urlObject.toString();
+}
+
+export type Writable<T> = {
+  -readonly [K in keyof T]: T[K];
+};
+
+/**
+ * Cast type to writable. Equivalent to doing "as Writable<T>", except this
+ * way, don't have to manually specify the generic type
+ */
+export const writable = <T>(value: T): Writable<T> => value;

app/pages/api/dockerhub/[image].ts

@@ -1,20 +1,85 @@
 import type { NextApiRequest, NextApiResponse } from 'next';
 
 import type { IR, RA } from '../../../lib/typescriptCommonTypes';
+import { formatUrl, Writable } from '../../../lib/apiUtils';
+
+// Docker Hub has a maximum of 10 pages for unauthenticated users
+// At the 11th or higher page, Docker Hub will throw an error
+const PAGE_MAX = 10;
+
+// The default maximum page size set by Docker
+const MAX_PAGE_SIZE = 100;
+
+type TagOrderBy =
+  | "last_updated"
+  | "name"
+  | "tag_status"
+  | "tag_last_pulled"
+  | "tag_last_pushed";
+
+type TagFilter = {
+  readonly orderBy?: TagOrderBy;
+  readonly name?: string;
+  readonly maxPages?: number;
+  readonly pageSize?: number;
+};
+
+const DEFAULT_TAG_FILTER: TagFilter = {
+  orderBy: "last_updated",
+  maxPages: PAGE_MAX,
+  pageSize: MAX_PAGE_SIZE,
+};
+
+export const SPECIAL_TAGS = {
+  "specify7-service": [
+    {
+      // This is to make sure we have all of the v7 tags even if they're
+      // excluded from the main tag fetch
+      orderBy: "last_updated",
+      name: "v7",
+    },
+    {
+      name: "main",
+      maxPages: 1,
+    },
+    {
+      orderBy: "last_updated",
+      maxPages: 5,
+    },
+  ],
+} as const;
+
 
 export type DockerHubTag = {
   readonly lastUpdated: string;
   readonly digest: string;
 };
 
 export const fetchTagsForImage = async (
-  image: string
+  imageName: string,
+  options?: RA<TagFilter>,
 ): Promise<IR<DockerHubTag>> =>
-  fetchTags(
-    `https://hub.docker.com/v2/repositories/specifyconsortium/${image}/tags/?page_size=1000`
-  ).then(processTagsResponse);
+  Promise.allSettled(
+    (options ?? [DEFAULT_TAG_FILTER]).map(
+      async (filter) => await fetchTags(imageName, filter),
+    ),
+  )
+    .then((results) =>
+      results
+        .filter(
+          (
+            result,
+          ): result is PromiseFulfilledResult<SuccessfulResponse["results"]> =>
+            result.status === "fulfilled",
+        )
+        .map((result) => result.value),
+    )
+    .then((results) => mergeTagResponses(results))
+    .then(processTagsResponse);
+
+
 
-type Response = {
+type SuccessfulResponse = {
   readonly results: RA<{
     readonly name: string;
     readonly last_updated: string;
@@ -24,17 +89,71 @@ type Response = {
     }>;
   }>;
   readonly next: string | undefined;
-};
+}
+
+type ErrorResponse = {
+  readonly errinfo: IR<unknown>;
+  readonly message: string;
+}
+
+type Response = SuccessfulResponse | ErrorResponse
+
+const mergeTagResponses = (responses: RA<SuccessfulResponse["results"]>) =>
+  responses.reduce(
+    (previous, current) => {
+      current.forEach((tag) => {
+        // We exclude already seen tags from the accumulated result
+        if (!previous.seenTags.has(tag.name)) {
+          previous.seenTags.add(tag.name);
+          previous.merged.push(tag);
+        }
+      })
+      return previous;
+    },
+    {
+      seenTags: new Set<string>(),
+      merged: [] as Writable<SuccessfulResponse["results"]>,
+    }
+  ).merged;
 
-export const fetchTags = async (url: string): Promise<Response['results']> =>
-  fetch(url)
+const urlFromFilter = (
+  image: string,
+  filter: TagFilter,
+  currentPage: number = 1,
+) =>
+  formatUrl(
+    `https://hub.docker.com/v2/repositories/specifyconsortium/${image}/tags/`,
+    {
+      page_size: filter.pageSize ?? MAX_PAGE_SIZE,
+      page: currentPage,
+      ordering: filter.orderBy,
+      name: filter.name,
+    },
+  );
+
+
+async function _fetchTags(url: string, filter: TagFilter, currentPage: number = 1): Promise<SuccessfulResponse['results']> {
+  return currentPage > (filter.maxPages ?? PAGE_MAX) ? Promise.resolve([]) : fetch(url)
     .then(async (response) => response.json())
-    .then(async ({ results, next }: Response) => [
-      ...results,
-      ...(typeof next === 'string' ? await fetchTags(next) : []),
-    ]);
+    .then(async (response: Response) => {
+      if ('message' in response) {
+        return []
+      }
+      return [
+        ...response.results,
+        ...(typeof response.next === 'string' ?
+          await _fetchTags(response.next, filter, currentPage + 1) : [])
+      ]
+    });
+}
+
+const fetchTags = async (
+  imageName: string,
+  filter: TagFilter,
+): Promise<SuccessfulResponse["results"]> =>
+  _fetchTags(urlFromFilter(imageName, filter), filter);
 
-const processTagsResponse = (tags: Response['results']): IR<DockerHubTag> =>
+const processTagsResponse = (tags: SuccessfulResponse['results']): IR<DockerHubTag> =>
   Object.fromEntries(
     tags
       // Latest is an unpredictable branch, thus will exclude it
@@ -65,7 +184,8 @@ export default async function handler(
   res: NextApiResponse
 ) {
   const image = request.query.image as string;
-  await fetchTagsForImage(image)
+  const specialFilters = SPECIAL_TAGS[image as keyof typeof SPECIAL_TAGS] as RA<TagFilter> | undefined;
+  await fetchTagsForImage(image, specialFilters)
     .then((tags) => res.status(200).json({ data: tags }))
     .catch((error) => res.status(500).json({ error: error.toString() }));
 }

app/pages/api/state/index.ts

@@ -16,7 +16,7 @@ import { createDockerConfig } from '../../../lib/dockerCompose';
 import { createNginxConfig } from '../../../lib/nginx';
 import type { RA } from '../../../lib/typescriptCommonTypes';
 import type { User } from '../../../lib/user';
-import { fetchTagsForImage } from '../dockerhub/[image]';
+import { fetchTagsForImage, SPECIAL_TAGS } from '../dockerhub/[image]';
 
 const configurationFile = path.resolve(stateDirectory, 'configuration.json');
 const nginxConfigurationFile = path.resolve(nginxConfigDirectory, 'nginx.conf');
@@ -74,7 +74,7 @@ export async function setState(
     autoDeploy
   );
 
-  const branches = await fetchTagsForImage('specify7-service');
+  const branches = await fetchTagsForImage('specify7-service', SPECIAL_TAGS['specify7-service']);
   const state = await Promise.all(
     rawState.map(async (deployment) => {
       const hasInteralSp7ConfigDirectory = await branchHasConfigDirectory(deployment.branch);