From 738b1a66706f9f7444a5de62098454febb33e0cd Mon Sep 17 00:00:00 2001 From: abdellah belaid Date: Wed, 15 Jul 2026 16:37:26 +0200 Subject: [PATCH 01/11] feat: single file download API with streaming option --- .dockerignore | 2 + README.md | 2 +- config/config.json | 25 +- docker-compose.yml | 13 - public/css/index.css | 21 ++ scripts/gen_nexus.py | 49 +++ src/app.ts | 37 +++ src/auth.ts | 62 +++- src/common/scicatAPI.ts | 18 +- src/index.ts | 2 - src/routes/file.ts | 674 ++++++++++++++++++++++++++++++++++++++++ src/routes/zip.ts | 17 +- src/types/global.d.ts | 1 + src/views/error.ejs | 2 +- src/views/file_form.ejs | 78 +++++ src/views/index.ejs | 81 +---- src/views/zip_form.ejs | 97 ++++++ src/views/zipping.ejs | 11 +- 18 files changed, 1073 insertions(+), 119 deletions(-) delete mode 100644 docker-compose.yml create mode 100644 scripts/gen_nexus.py create mode 100644 src/routes/file.ts create mode 100644 src/views/file_form.ejs create mode 100644 src/views/zip_form.ejs diff --git a/.dockerignore b/.dockerignore index 77135c9..0ecb910 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1,7 +1,9 @@ +.git node_modules # testing testdata +test/* # generated zips files/* \ No newline at end of file diff --git a/README.md b/README.md index 714c32f..ed41aea 100644 --- a/README.md +++ b/README.md @@ -86,4 +86,4 @@ Example data "files": ["", "", ""] } } -``` +``` \ No newline at end of file diff --git a/config/config.json b/config/config.json index 018daf5..08aef0d 100644 --- a/config/config.json +++ b/config/config.json @@ -1,19 +1,28 @@ { "zipDir": "./files", - "jwtSecret": "fhjekrsgvhuirs", - "sessionSecret": "fj9832mnsaf3j9adsa", - "facility": "ess", + "jwtSecret": "dev_secret", + "sessionSecret": "dev_secret", + "facility": "ILL", "zipRetentionMillis": 3600000, - "dramDirectory": "/nfs/groups/beamlines/dram", + "dataDirectory": "/illdata", + "directoryPathPattern": "/illdata/*/{instrumentIds[0]}/{proposalIds[0]}/{type}", + "requiredKeywords": [ + "intrumentIds", + "proposalIds", + "type" + ], + "hdfViewServiceUrl": "http://localhost:8080", "testData": { - "jwt": "eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJncm91cHMiOlsiZXNzIiwibG9raSIsIm9kaW4iXSwiaWF0IjoxNTY3NDI3NzA5LCJleHAiOjE1Njc0MzEzMDl9.ENcuTBS67shghefL3wRCHxoyobwZGaFQIUvSZ0cywcE", + "jwt": "", "directory": "./test/", - "files": ["test_file_1.txt"] + "files": [ + "test_file_1.txt" + ] }, "graylogEnabled": false, "graylogServer": "it-graylog.esss.lu.se", "graylogPort": 12201, "environment": "development", - "basePath": "", + "basePath": "http://localhost:4200", "accessToken": "" -} +} \ No newline at end of file diff --git a/docker-compose.yml b/docker-compose.yml deleted file mode 100644 index ecbcd65..0000000 --- a/docker-compose.yml +++ /dev/null @@ -1,13 +0,0 @@ -version: "3" -services: - node: - build: . - restart: always - ports: - - "3012:3012" - volumes: - - /nfs/groups/beamlines/:/nfs/groups/beamlines - - /users/detector/experiments/:/users/detector/experiments - - /mnt/groupdata/guide_optimizations/:/mnt/groupdata/guide_optimizations - - /tmp/zip-service:/home/node/app/files #location of the zipped files - - /ess/data/:/ess/data diff --git a/public/css/index.css b/public/css/index.css index 48fc473..4f32e73 100644 --- a/public/css/index.css +++ b/public/css/index.css @@ -12,10 +12,26 @@ h2 { font-size: 1.2em; font-weight: normal; } +h3 { + font-size: 1em; + font-weight: normal; + margin: 0 0 0.5em 0; +} a { text-decoration: none; color: #769eda; } +.button-link { + display: inline-block; + padding: 0.6em 1em; + background: #eee; + border-radius: 2px; + color: #333; +} +.button-link:hover { + background: #ddd; + color: #000; +} code { font-style: normal; background: #eee; @@ -80,6 +96,11 @@ input[type="submit"]:hover { width: 50%; padding: 1em; } +.form-stack form + form { + border-top: 1px solid #eee; + margin-top: 2em; + padding-top: 2em; +} .input{ width: 25em; } diff --git a/scripts/gen_nexus.py b/scripts/gen_nexus.py new file mode 100644 index 0000000..29fad25 --- /dev/null +++ b/scripts/gen_nexus.py @@ -0,0 +1,49 @@ +import os +import sys +import h5py +import numpy as np + +out = sys.argv[1] +size_mb = int(sys.argv[2]) + +# uint16 +target_bytes = size_mb * 1024 * 1024 +n_values = target_bytes // 2 + +rows = 1024 +cols = max(1, n_values // rows) + +os.makedirs(os.path.dirname(out), exist_ok=True) + +with h5py.File(out, "w") as f: + f.attrs["NX_class"] = "NXroot" + + entry = f.create_group("entry0") + entry.attrs["NX_class"] = "NXentry" + + instrument = entry.create_group("instrument") + instrument.attrs["NX_class"] = "NXinstrument" + + detector = instrument.create_group("detector") + detector.attrs["NX_class"] = "NXdetector" + + data = detector.create_dataset( + "data", + shape=(rows, cols), + dtype="uint16", + chunks=(256, 256), + ) + + chunk_rows = 256 + for start in range(0, rows, chunk_rows): + end = min(start + chunk_rows, rows) + data[start:end, :] = np.random.randint( + 0, + 65535, + size=(end - start, cols), + dtype=np.uint16, + ) + + entry["data"] = h5py.SoftLink("/entry0/data") + +print(f"Created {out}: {os.path.getsize(out) / 1024 / 1024:.1f} MB") diff --git a/src/app.ts b/src/app.ts index a3bc602..3199507 100644 --- a/src/app.ts +++ b/src/app.ts @@ -3,12 +3,14 @@ import session from "express-session"; import fs from "fs"; import rimraf from "rimraf"; import { config } from "./common/config"; +import cors from "cors"; import express from "express"; import path from "path"; import cookieParser from "cookie-parser"; import { router as zipRouter } from "./routes/zip"; import { router as zipInPlaceRouter } from "./routes/zip_in_place"; import { router as downloadRouter } from "./routes/download"; +import { router as fileRouter } from "./routes/file"; import { router as indexRouter } from "./routes/index"; import { router as uploadRouter } from "./routes/upload"; import { logger } from "@user-office-software/duo-logger"; @@ -17,9 +19,23 @@ import { configureLogger } from "./common/configureLogger"; const app = express(); app.set("views", path.join(__dirname, "views")); app.set("view engine", "ejs"); +app.use( + cors({ + exposedHeaders: [ + "Accept-Ranges", + "Content-Disposition", + "Content-Length", + "Content-Range", + ], + }) +); app.use(express.json()); app.use(express.urlencoded({ extended: true })); app.use(cookieParser()); +app.use((req, res, next) => { + res.locals.routeBasePath = getForwardedPrefix(req); + next(); +}); app.use(express.static("public")); app.use( @@ -42,6 +58,7 @@ app.use("/", indexRouter); app.use("/zip", zipRouter); app.use("/zip_in_place", zipInPlaceRouter); app.use("/download", downloadRouter); +app.use("/file", fileRouter); app.use("/upload", uploadRouter); configureLogger( @@ -85,4 +102,24 @@ const deleteZipFiles = () => { }; setInterval(deleteZipFiles, config.zipRetentionMillis || 60 * 60 * 1000); +function getForwardedPrefix(req: express.Request) { + const forwardedPrefix = req.headers["x-forwarded-prefix"]; + const prefix = Array.isArray(forwardedPrefix) + ? forwardedPrefix[0] + : forwardedPrefix || ""; + const firstPrefix = prefix.split(",")[0].trim(); + + if (!firstPrefix || firstPrefix === "/") { + return ""; + } + + const prefixedPath = firstPrefix.startsWith("/") + ? firstPrefix + : `/${firstPrefix}`; + + return prefixedPath.endsWith("/") + ? prefixedPath.slice(0, -1) + : prefixedPath; +} + export default app; diff --git a/src/auth.ts b/src/auth.ts index 83e6718..ab37442 100644 --- a/src/auth.ts +++ b/src/auth.ts @@ -5,16 +5,17 @@ import jwtLib from "jsonwebtoken"; import { logger } from "@user-office-software/duo-logger"; import { scicatDataSetAPI } from "./common/scicatAPI"; -import { OutputDatasetObsoleteDto } from "@scicatproject/scicat-sdk-ts-fetch/dist/models"; +import { OutputDatasetDto } from "@scicatproject/scicat-sdk-ts-fetch/dist/models"; import path from "path"; export const hasFileAccess = async ( req: express.Request, absoluteFileNames: string[], - dataset: string + dataset: string | undefined, ): Promise => { const { jwtSecret } = config; const dataSetAPI = scicatDataSetAPI(); + let keywords = {}; if (!jwtSecret) { return { @@ -76,22 +77,38 @@ export const hasFileAccess = async ( fileNames: [], }; } + logger.logInfo("User Groups: ", { groups }); + + if (!dataset) { + const isAdmin = authRequest.jwt.groups.includes("admin"); + if (isAdmin) { + return { + hasAccess: true, + statusCode: 200, + error: "", + fileNames: authRequest.fileNames, + keywords: keywords, + }; + } + } const valid = await dataSetAPI - .datasetsControllerFindById({ pid: authRequest.dataset }) - .then((value: OutputDatasetObsoleteDto) => { + .datasetsV4ControllerFindById({ pid: authRequest.dataset }) + .then((value: OutputDatasetDto) => { const isPublic = value.isPublished; - const hasAccessGroup = value.accessGroups.some((item) => - new Set(authRequest.jwt.groups).has(item) + const isAdmin = authRequest.jwt.groups?.includes("admin"); + const hasAccessGroup = value.accessGroups?.some((item) => + new Set(authRequest.jwt.groups).has(item), ); - - const hasOwnerGroup = authRequest.jwt.groups.includes(value.ownerGroup); - if (isPublic || hasAccessGroup || hasOwnerGroup) { + const hasOwnerGroup = authRequest.jwt.groups?.includes(value.ownerGroup); + keywords = getDatasetKeywords(value); + if (isPublic || isAdmin || hasAccessGroup || hasOwnerGroup) { return true; } return false; }) - .catch(() => { + .catch((err) => { + logger.logError("Error caught at access validation", { err }); return false; }); @@ -100,5 +117,30 @@ export const hasFileAccess = async ( statusCode: valid ? 200 : 403, error: valid ? "" : "You do not have access to this resource", fileNames: valid ? authRequest.fileNames : [], + keywords: valid ? keywords : {}, }; }; + +function getDatasetKeywords(dataset: OutputDatasetDto) { + const requiredKeywords: string[] = Array.isArray(config.requiredKeywords) + ? config.requiredKeywords + : []; + const keywords: Record = {}; + const datasetValues = dataset as unknown as Record; + for (const keyword of requiredKeywords) { + const match = keyword.match(/^([^[\]]+)\[(\d+)\]$/); + + if (match) { + const [, property, index] = match; + const value = datasetValues[property]; + + keywords[keyword] = Array.isArray(value) + ? value[Number(index)] + : undefined; + } else { + keywords[keyword] = datasetValues[keyword]; + } + } + + return keywords; +} diff --git a/src/common/scicatAPI.ts b/src/common/scicatAPI.ts index 00d3447..4798543 100644 --- a/src/common/scicatAPI.ts +++ b/src/common/scicatAPI.ts @@ -1,23 +1,27 @@ -import { Configuration, DatasetsApi } from "@scicatproject/scicat-sdk-ts-fetch"; +import { + Configuration, + DatasetsV4Api, +} from "@scicatproject/scicat-sdk-ts-fetch"; import { config } from "./config"; -let datasetsApiInstance: DatasetsApi | null = null; +let datasetsApiInstance: DatasetsV4Api | null = null; -export function scicatDataSetAPI(): DatasetsApi { +export function scicatDataSetAPI(): DatasetsV4Api { const { basePath, accessToken } = config; if (!datasetsApiInstance) { - if (!basePath || !accessToken) { - throw new Error("SciCat API configuration is missing: Check SCICAT_API_BASE_PATH and SCICAT_API_ACCESS_TOKEN."); + throw new Error( + "SciCat API configuration is missing: Check SCICAT_API_BASE_PATH and SCICAT_API_ACCESS_TOKEN.", + ); } const apiConfig = new Configuration({ - basePath , + basePath, accessToken, }); - datasetsApiInstance = new DatasetsApi(apiConfig); + datasetsApiInstance = new DatasetsV4Api(apiConfig); } return datasetsApiInstance; diff --git a/src/index.ts b/src/index.ts index afa16b6..10653d9 100644 --- a/src/index.ts +++ b/src/index.ts @@ -1,11 +1,9 @@ import app from "./app"; -import cors from "cors"; import http from "http"; import { logger } from "@user-office-software/duo-logger"; const port = parseInt(process.env.PORT || "3011", 10); app.set("port", port); -app.use(cors()); const server = http.createServer(app); server.listen(port); diff --git a/src/routes/file.ts b/src/routes/file.ts new file mode 100644 index 0000000..74c6497 --- /dev/null +++ b/src/routes/file.ts @@ -0,0 +1,674 @@ +import express from "express"; +import * as fs from "fs"; +import { config } from "../common/config"; +import { hasFileAccess } from "../auth"; +import { logger } from "@user-office-software/duo-logger"; +import path from "path"; +import { v4 as uuidv4 } from "uuid"; + +export const router = express.Router(); + +interface FileLink { + filename: string; + filepath: string; + token: string; + expiresAt: number; + datasetId?: string; + directory?: string; +} +interface FileResolution { + statusCode: number; + filename?: string; + error?: string; + folders?: string[]; +} + +const fileLinks = new Map(); +const fileLinkRetentionMillis = + config.fileLinkRetentionMillis || 60 * 60 * 1000; + +router.get("/", (req, res) => { + res.render("file_form", getFormDefaults()); +}); + +router.post("/", async (req, res) => { + cleanupExpiredFileLinks(); + + const bodyDirectory = req.body.directory; + const bodyFileNames = getBodyFileNames(req.body); + + if (!Array.isArray(bodyFileNames) || bodyFileNames.length !== 1) { + return res.status(400).send({ + error: "Exactly one file must be specified", + }); + } + + if (typeof bodyFileNames[0] !== "string") { + return res.status(400).send({ + error: "File must be a string", + }); + } + + const bodyFileName = bodyFileNames[0]; + + if ( + !path.isAbsolute(bodyFileName) && + typeof bodyDirectory !== "string" && + !config.directoryPathPattern + ) { + return res.status(400).send({ + error: "Directory or directoryPathPattern must be specified", + }); + } + + return createFileLink(req, res); +}); + +router.head("/download", (req, res) => { + return sendFileHead(req, res, "attachment"); +}); + +router.head("/stream", (req, res) => { + return sendFileHead(req, res, "inline"); +}); + +router.get("/open", (req, res) => { + return openFile(req, res); +}); + +router.get("/download", (req, res) => { + return sendFile(req, res, "attachment"); +}); + +router.get("/stream", (req, res) => { + return sendFile(req, res, "inline"); +}); + +function sendFile( + req: express.Request, + res: express.Response, + disposition: "attachment" | "inline", +) { + const fileData = getFileLink(req); + if (!fileData) { + return res.status(403).send("This file link is no longer valid"); + } + + return sendResolvedFile( + fileData.filepath, + res, + disposition, + req.headers.range, + ); +} + +function sendFileHead( + req: express.Request, + res: express.Response, + disposition: "attachment" | "inline", +) { + const fileData = getFileLink(req); + if (!fileData) { + return res.status(403).end(); + } + const fileStat = getFileStat(fileData.filepath); + if (!fileStat || fileStat.isDirectory()) { + return res.status(404).end(); + } + + return sendResolvedFileHead(fileData.filepath, res, disposition); +} + +function openFile(req: express.Request, res: express.Response) { + const fileData = getFileLink(req); + if (!fileData) { + return res.status(403).send("This file link is no longer valid"); + } + + const streamUrl = createFileAccessUrl(req, "stream", fileData.token, true); + + const hdfUrl = createHdfViewUrl(streamUrl, fileData.filename); + if (!hdfUrl) { + return res.status(500).send("HDF View service URL is not configured"); + } + + return res.redirect(303, hdfUrl); +} + +async function createFileLink(req: express.Request, res: express.Response) { + const datasetId = req.body.dataset || req.body.datasetId; + const directory = + typeof req.body.directory === "string" ? req.body.directory : undefined; + const filename = getBodyFileNames(req.body)[0] as string; + const fileAction = + typeof req.body.fileAction === "string" ? req.body.fileAction : ""; + + if (shouldResolveFromDataset(filename, directory)) { + const authResponse = await hasFileAccess(req, [filename], datasetId); + + if (!authResponse.hasAccess) { + logger.logError("Error accessing file", { + statusCode: authResponse.statusCode, + error: authResponse.error, + }); + + return res.status(authResponse.statusCode).send({ + error: authResponse.error, + }); + } + + return createResolvedFileLink( + req, + res, + datasetId, + directory, + filename, + authResponse.keywords ?? {}, + undefined, + fileAction, + ); + } + + const resolution = resolveFilePath(filename, directory); + if (resolution.error) { + return res.status(resolution.statusCode).send({ + error: resolution.error, + folders: resolution.folders, + }); + } + const filepath = path.join(resolution.folders?.[0], filename); + if (!filepath) { + return res.status(500).send({ + error: "File path could not be resolved", + }); + } + const { hasAccess, statusCode, error } = await hasFileAccess( + req, + [filepath], + datasetId, + ); + if (!hasAccess) { + logger.logError("Error accessing file", { statusCode, error }); + return res.status(statusCode).send({ error }); + } + + return createResolvedFileLink( + req, + res, + datasetId, + directory, + filename, + undefined, + filepath, + fileAction, + ); +} + +function createResolvedFileLink( + req: express.Request, + res: express.Response, + datasetId: string, + directory: string | undefined, + filename: string, + keywords?: Record, + resolvedFilepath?: string, + fileAction = "", +) { + const resolution: FileResolution = resolvedFilepath + ? { + statusCode: 200, + filename, + folders: [path.dirname(resolvedFilepath)], + } + : resolveFilePath(filename, directory, keywords); + + if (resolution.error) { + return res.status(resolution.statusCode).send({ + error: resolution.error, + folders: resolution.folders, + }); + } + + const filepath = path.join(resolution.folders?.[0], filename); + + if (!filepath) { + return res.status(500).send({ + error: "File path could not be resolved", + }); + } + const fileStat = getFileStat(filepath); + + if (!fileStat || fileStat.isDirectory()) { + return res.status(404).send({ + error: "File not found", + }); + } + + const token = uuidv4(); + + fileLinks.set(token, { + filename, + filepath, + datasetId, + directory, + token, + expiresAt: Date.now() + fileLinkRetentionMillis, + }); + + const downloadUrl = createFileAccessUrl(req, "download", token); + const streamUrl = createFileAccessUrl(req, "stream", token); + const openUrl = createFileAccessUrl(req, "open", token); + + if (fileAction === "Download") { + return res.redirect(303, downloadUrl); + } + + if (fileAction === "Open") { + return res.redirect(303, openUrl); + } + + return res.status(201).send({ + fileName: filename, + size: fileStat.size, + expiresAt: new Date(Date.now() + fileLinkRetentionMillis).toISOString(), + downloadUrl, + streamUrl, + openUrl, + }); +} + +function sendResolvedFile( + absoluteFileName: string, + res: express.Response, + disposition: "attachment" | "inline", + rangeHeader?: string, +) { + const fileStat = getFileStat(absoluteFileName); + if (!fileStat || fileStat.isDirectory()) { + return res.status(404).send("File not found"); + } + + const fileName = path.basename(absoluteFileName); + const contentType = getContentType(fileName); + const range = parseRange(rangeHeader, fileStat.size); + + res.setHeader("Accept-Ranges", "bytes"); + res.setHeader("Content-Type", contentType); + res.setHeader( + "Content-Disposition", + getContentDisposition(disposition, fileName), + ); + + if (!range) { + res.setHeader("Content-Length", fileStat.size); + + return pipeFile(absoluteFileName, res); + } + + if (!range.valid) { + res.statusCode = 416; + res.setHeader("Content-Range", `bytes */${fileStat.size}`); + + return res.end(); + } + + const { start, end } = range; + res.statusCode = 206; + res.setHeader("Content-Range", `bytes ${start}-${end}/${fileStat.size}`); + res.setHeader("Content-Length", end - start + 1); + + return pipeFile(absoluteFileName, res, { start, end }); +} + +function sendResolvedFileHead( + absoluteFileName: string, + res: express.Response, + disposition: "attachment" | "inline", +) { + const fileStat = getFileStat(absoluteFileName); + if (!fileStat || fileStat.isDirectory()) { + return res.status(404).end(); + } + + const fileName = path.basename(absoluteFileName); + res.setHeader("Accept-Ranges", "bytes"); + res.setHeader("Content-Type", getContentType(fileName)); + res.setHeader( + "Content-Disposition", + getContentDisposition(disposition, fileName), + ); + res.setHeader("Content-Length", fileStat.size); + + return res.status(200).end(); +} + +function createFileAccessUrl( + req: express.Request, + action: "download" | "stream" | "open", + token: string, + absolute = false, +) { + const pathname = `/file/${action}?token=${token}`; + const prefixedPath = withForwardedPrefix(req, pathname); + + if (!absolute) { + return prefixedPath; + } + + return withRequestOrigin(req, prefixedPath); +} + +function createHdfViewUrl(fileUrl: string, filename?: string) { + const url = new URL(config.hdfViewServiceUrl); + if (!url) { + return null; + } + url.searchParams.set("url", encodeURIComponent(fileUrl)); + url.searchParams.set("filesSidebar", "hidden"); + if (filename) { + url.searchParams.set("label", filename); + } + return url.toString(); +} + +function withForwardedPrefix(req: express.Request, pathname: string) { + const forwardedPrefix = req.headers["x-forwarded-prefix"]; + const prefix = Array.isArray(forwardedPrefix) + ? forwardedPrefix[0] + : forwardedPrefix || ""; + const normalizedPrefix = prefix.endsWith("/") ? prefix.slice(0, -1) : prefix; + return `${normalizedPrefix}${pathname}`; +} + +function withRequestOrigin(req: express.Request, pathname: string) { + const protocol = + getFirstHeader(req, "x-forwarded-proto") || req.protocol || "http"; + const host = getFirstHeader(req, "x-forwarded-host") || req.get("host"); + if (!host) { + return pathname; + } + return `${protocol}://${host}${pathname}`; +} + +function getFirstHeader(req: express.Request, headerName: string) { + const value = req.headers[headerName]; + if (Array.isArray(value)) { + return value[0]; + } + if (typeof value === "string") { + return value.split(",")[0].trim(); + } + return ""; +} + +function getFileLink(req: express.Request) { + cleanupExpiredFileLinks(); + + const token = typeof req.query.token === "string" ? req.query.token : ""; + const fileData = fileLinks.get(token); + if ( + !fileData || + fileData.token !== token || + Date.now() > fileData.expiresAt + ) { + return null; + } + + return fileData; +} + +function cleanupExpiredFileLinks() { + const now = Date.now(); + + for (const [id, fileLink] of fileLinks) { + if (now > fileLink.expiresAt) { + fileLinks.delete(id); + } + } +} + +function resolveFilePath( + filename: string, + directory: string | undefined, + keywords: Record = {}, +): FileResolution { + // filename is an Absolute Path case + if (path.isAbsolute(filename)) { + return { + statusCode: 200, + filename, + folders: [filename], + }; + } + const dataRoot = config.dataDirectory; + const dirPattern = config.directoryPathPattern; + if (!dataRoot) { + return { + statusCode: 500, + error: "Data root directory is not configured", + }; + } + // "directory" is specified case + if (typeof directory === "string") { + return { + statusCode: 200, + filename, + folders: [path.join(dataRoot, directory)], + }; + } + // else : directoryPattern must be configurad + if (!dirPattern) { + return { + statusCode: 400, + error: "Directory or directoryPathPattern must be specified", + }; + } + + let resolvedPattern: string; + try { + resolvedPattern = resolvePattern(dirPattern, keywords); + } catch (err) { + logger.logError("Error resolving file path pattern", { err }); + return { + statusCode: 400, + error: + err instanceof Error ? err.message : "File path pattern is invalid", + }; + } + + const matchedFolders = findFile(resolvedPattern, filename); + if (matchedFolders.length === 0) { + return { + statusCode: 404, + error: "File not found", + }; + } else if (matchedFolders.length > 1) { + return { + statusCode: 409, + error: "File exists in multiple directories", + folders: matchedFolders, + }; + } + return { + statusCode: 200, + filename, + folders: matchedFolders, + }; +} + +function shouldResolveFromDataset(filename: string, directory: unknown) { + return ( + !path.isAbsolute(filename) && + typeof directory !== "string" && + Boolean(config.directoryPathPattern) + ); +} + +function resolvePattern( + pattern: string, + keywords: Record, +): string { + return pattern.replace(/\{([^}]+)\}/g, (_, key) => { + try { + const value = keywords[key]; + if (value === undefined) { + throw new Error(`Missing keyword '${key}'`); + } + if (config.facility === "ILL") { + if (key.startsWith("proposalId")) { + return value.startsWith("internalUse") ? value : "exp_" + value; + } else if (key == "type") { + return value.startsWith("raw") ? "rawdata" : value; + } else if (key.startsWith("instrumentId")) { + return value.toLowerCase(); + } + } + return value; + } catch { + return undefined; + } + }); +} + +function findFile(dirPattern: string, filename: string) { + const stars = (dirPattern.match(/\*/g) ?? []).length; + if (stars > 1) { + throw new Error("Only a single '*' is supported."); + } + try { + if (stars === 0) { + const filePath = path.join(dirPattern, filename); + return fs.existsSync(filePath) ? [filePath] : []; + } + + const [prefix, suffix] = dirPattern.split("*"); + const parentDir = path.resolve(prefix); + const normalizedSuffix = suffix.replace(/^[/\\]/, ""); + const matchedFolders: string[] = []; + + fs.readdirSync(parentDir, { withFileTypes: true }) + .filter((entry) => entry.isDirectory()) + .forEach((entry) => { + const pathFromPattern = path.join( + parentDir, + entry.name, + normalizedSuffix, + ); + + const filePath = path.join(pathFromPattern, filename); + if (fs.existsSync(filePath)) { + matchedFolders.push(path.dirname(filePath)); + } + }); + return matchedFolders; + } catch (err) { + logger.logError("Error searching file directories", { err }); + return []; + } +} + +function getBodyFileNames(body: Record) { + if (Array.isArray(body.files)) { + return body.files; + } + if (body.fileName) { + return [body.fileName]; + } + if (body.file) { + return [body.file]; + } + return []; +} + +function getFileStat(fileName: string) { + try { + return fs.statSync(fileName); + } catch (err) { + logger.logError("Error getting file stat", { err }); + + return null; + } +} + +function getContentType(fileName: string) { + const extension = path.extname(fileName).toLowerCase(); + + if ( + extension === ".h5" || + extension === ".hdf5" || + extension === ".hdf" || + extension === ".nxs" + ) { + return "application/x-hdf5"; + } + return "application/octet-stream"; +} + +function getContentDisposition( + disposition: "attachment" | "inline", + fileName: string, +) { + const safeFileName = fileName.replace(/[\r\n"\\]/g, "_"); + return `${disposition}; filename="${safeFileName}"`; +} + +function pipeFile( + fileName: string, + res: express.Response, + options?: { start: number; end: number }, +) { + const readStream = fs.createReadStream(fileName, options); + readStream.on("error", (err) => { + logger.logError("Error reading file stream", { err }); + if (!res.headersSent) { + return res.status(500).send("Unable to read file"); + } + return res.end(); + }); + return readStream.pipe(res); +} + +function parseRange(rangeHeader: string | undefined, fileSize: number) { + if (!rangeHeader) { + return null; + } + + const match = rangeHeader.match(/^bytes=(\d*)-(\d*)$/); + if (!match) { + return { valid: false as const }; + } + + const [, startValue, endValue] = match; + let start: number; + let end: number; + + if (!startValue && !endValue) { + return { valid: false as const }; + } + + if (!startValue) { + const suffixLength = parseInt(endValue, 10); + if (suffixLength <= 0) { + return { valid: false as const }; + } + start = Math.max(fileSize - suffixLength, 0); + end = fileSize - 1; + } else { + start = parseInt(startValue, 10); + end = endValue ? parseInt(endValue, 10) : fileSize - 1; + } + + if (start >= fileSize || end >= fileSize || start > end) { + return { valid: false as const }; + } + return { valid: true as const, start, end }; +} + +function getFormDefaults() { + return { + jwt: config.testData.jwt || "", + directory: config.testData.directory || "", + dataset: "", + file0: config.testData?.files[0] || "", + }; +} diff --git a/src/routes/zip.ts b/src/routes/zip.ts index e8f20c9..ad29867 100644 --- a/src/routes/zip.ts +++ b/src/routes/zip.ts @@ -9,6 +9,10 @@ import { v4 as uuidv4 } from "uuid"; export const router = express.Router(); +router.get("/", (req, res) => { + res.render("zip_form", getFormDefaults()); +}); + /* POST zip */ /** * Request zipping of files. Require directory:string and files:string[] in the request body @@ -90,7 +94,7 @@ router.post("/", async (req, res) => { }); // Polled periodically from the zipping view. Returns current progress or resulting file name if the zipping is done -router.get("/", (req, res) => { +router.get("/status", (req, res) => { const { currentFileIndex, ready, files, zipFileName, zipSizeOnLastCompletedEntry } = req.session.zipData; const zipSize = getFileSizeInBytes(config.zipDir + "/" + zipFileName); @@ -136,3 +140,14 @@ const initSession = ( ready: false, datasetId, }); + +function getFormDefaults() { + return { + jwt: config.testData.jwt || "", + directory: config.testData.directory || "", + dataset: "", + file0: config.testData?.files[0] || "", + file1: config.testData?.files[1] || "", + file2: config.testData?.files[2] || "", + }; +} diff --git a/src/types/global.d.ts b/src/types/global.d.ts index f55c2d8..0c7d9e9 100644 --- a/src/types/global.d.ts +++ b/src/types/global.d.ts @@ -33,6 +33,7 @@ declare global { statusCode: number; error?: string; fileNames: string[]; + keywords?: Record; } } } diff --git a/src/views/error.ejs b/src/views/error.ejs index c58ac00..001db37 100644 --- a/src/views/error.ejs +++ b/src/views/error.ejs @@ -1,7 +1,7 @@ - +
diff --git a/src/views/file_form.ejs b/src/views/file_form.ejs new file mode 100644 index 0000000..6d8530c --- /dev/null +++ b/src/views/file_form.ejs @@ -0,0 +1,78 @@ + + + + + + + zip-service file + + +
+

Single file

+
+
+

+ Submit a POST request to /file with a jwt, dataset, + and file name. Use directory for direct path resolution, or omit it + to resolve the path from the configured dataset keywords. +

+

Back to forms

+
+
+
+
JWT
+ +
+
+
Directory
+ +
+
+
Dataset PID
+ +
+
+
File
+ +
+
+ +
+
+ +
+
+ +
+
+
+
+ + diff --git a/src/views/index.ejs b/src/views/index.ejs index fc31ff6..90fe415 100644 --- a/src/views/index.ejs +++ b/src/views/index.ejs @@ -6,7 +6,7 @@ name="viewport" content="width=device-width,initial-scale=1,shrink-to-fit=no" /> - + zip-service @@ -20,80 +20,19 @@

UI for testing zip-service. While implementation details of file authorization may vary between facilities, a valid jwt (JSON Web - Token) is required for all of them. This can be specified either as - a cookie (all requests), a url parameter (GET) or in the body - (POST). + Token) is required. This can be specified either as a cookie, a url + parameter for download requests, or in a POST body.

- The default value of directory corresponds to the root - of the scientific data directory tree. The specified files must all be directly under this directory. -

-

- By submitting the form on the right, a http POST is made to /zip, with the specified jwt, directory and file array as its body. The jwt can be omitted if already present as a cookie. If not, one has to be generated using the same secret as the zip-service, and a JSON payload containing the keys necessary for authorization. + Use the zip form to create a zip archive from several files, or the + file form to create a temporary download/stream link for one file.

-
-
-
JWT
-
-
-
Directory
- -
-
-
Dataset PID
- -
-
-
File 1
- -
-
-
File 2
- -
-
-
File 3
- -
-
- -
-
- -
-
+
+

Forms

+

Zip files

+

Single file

+
diff --git a/src/views/zip_form.ejs b/src/views/zip_form.ejs new file mode 100644 index 0000000..f77dac2 --- /dev/null +++ b/src/views/zip_form.ejs @@ -0,0 +1,97 @@ + + + + + + + zip-service zip + + +
+

Zip files

+
+
+

+ Submit a POST request to /zip with a jwt, dataset, + directory, and file array. The result is a progress page that links + to the generated zip file when processing is complete. +

+

Back to forms

+
+
+
+
JWT
+ +
+
+
Directory
+ +
+
+
Dataset PID
+ +
+
+
File 1
+ +
+
+
File 2
+ +
+
+
File 3
+ +
+
+ +
+
+ +
+
+
+
+ + diff --git a/src/views/zipping.ejs b/src/views/zipping.ejs index 23cc975..9da83bd 100644 --- a/src/views/zipping.ejs +++ b/src/views/zipping.ejs @@ -2,9 +2,9 @@ - - - + + +