Write the track export zip in-process (UTF-8 names, no zip binary)

DockerfileAlpine

@@ -24,7 +24,7 @@ LABEL org.opencontainers.image.version=$TEDDYCLOUD_VERSION
 EXPOSE 80 443 8443
 
 # Install necessary runtime dependencies
-RUN apk --no-cache add strace ffmpeg zip curl ca-certificates bash su-exec shadow libcap
+RUN apk --no-cache add strace ffmpeg curl ca-certificates bash su-exec shadow libcap
 
 # Update CA certificates
 RUN update-ca-certificates

DockerfileUbuntu

@@ -20,7 +20,7 @@ LABEL org.opencontainers.image.version=$TEDDYCLOUD_VERSION
 EXPOSE 80 443 8443
 
 RUN apt-get update \
-    && apt-get install -y --no-install-recommends libubsan1 strace ffmpeg zip curl ca-certificates gosu libcap2-bin \
+    && apt-get install -y --no-install-recommends libubsan1 strace ffmpeg curl ca-certificates gosu libcap2-bin \
     && rm -rf /var/lib/apt/lists/*
 
 RUN update-ca-certificates

Makefile

@@ -811,7 +811,7 @@ TEST_C_SRC := $(wildcard tests/c/*.c)
 test_c:
 	$(QUIET)$(ECHO) "[ ${CYAN}TEST${NC} ] Build and run C unit tests"
 	$(QUIET)mkdir -p $(BIN_DIR)
-	$(QUIET)$(CC) -I include -I src/cyclone/common -I src/cyclone/cyclone_tcp -I cyclone/common -I cyclone/cyclone_tcp -I cyclone/cyclone_ssl -I cyclone/cyclone_crypto -D GPL_LICENSE_TERMS_ACCEPTED -ffunction-sections -Wl,--gc-sections -o $(BIN_DIR)/test_c $(TEST_C_SRC) $(SRC_DIR)/os_ext.c $(SRC_DIR)/str_ext.c $(SRC_DIR)/fs_ext.c $(SRC_DIR)/esp32_port.c $(SRC_DIR)/platform/platform_linux.c $(SRC_DIR)/cyclone/common/os_port_posix.c $(SRC_DIR)/box_cert.c
+	$(QUIET)$(CC) -I include -I src/cyclone/common -I src/cyclone/cyclone_tcp -I cyclone/common -I cyclone/cyclone_tcp -I cyclone/cyclone_ssl -I cyclone/cyclone_crypto -D GPL_LICENSE_TERMS_ACCEPTED -ffunction-sections -Wl,--gc-sections -o $(BIN_DIR)/test_c $(TEST_C_SRC) $(SRC_DIR)/os_ext.c $(SRC_DIR)/str_ext.c $(SRC_DIR)/fs_ext.c $(SRC_DIR)/esp32_port.c $(SRC_DIR)/platform/platform_linux.c $(SRC_DIR)/cyclone/common/os_port_posix.c $(SRC_DIR)/box_cert.c $(SRC_DIR)/zip_writer.c
 	$(QUIET)$(BIN_DIR)/test_c
 
 .PHONY: test

include/zip_writer.h

@@ -0,0 +1,18 @@
+#pragma once
+
+#include <stddef.h>
+#include "error.h"
+
+/**
+ * Writes a zip archive that stores the given files uncompressed (Ogg/Opus does not compress).
+ * The entry names are taken as UTF-8 and flagged as such (general purpose bit 11), so unzip
+ * tools show non-ASCII names correctly regardless of the locale of the server.
+ * Limited to the classic zip format: at most 65535 entries and 4 GiB in total.
+ *
+ * @param zip_path    archive to create
+ * @param count       number of entries
+ * @param file_paths  files to add
+ * @param entry_names names of the entries inside the archive, UTF-8, no directories
+ * @return NO_ERROR or ERROR_FAILURE (the partial archive is removed)
+ */
+error_t zipWriteStored(const char *zip_path, size_t count, const char *const *file_paths, const char *const *entry_names);

src/handler_api.c

@@ -26,6 +26,7 @@
 #include "esp32.h"
 #include "cache.h"
 #include "libraryMeta.h"
+#include "zip_writer.h"
 
 error_t parsePostData(HttpConnection *connection, char_t *post_data, size_t buffer_size)
 {
@@ -1899,16 +1900,25 @@ static void sanitize_track_title(const char *in, char *out, size_t out_len)
     {
         in = "";
     }
-    for (size_t i = 0; in[i] != '\0' && j + 1 < out_len; i++)
+    for (size_t i = 0; in[i] != '\0';)
     {
         unsigned char c = (unsigned char)in[i];
+        /* length of the UTF-8 sequence, so a truncated title never ends in half a character */
+        size_t len = (c >= 0xF0) ? 4 : (c >= 0xE0) ? 3 : (c >= 0xC0) ? 2 : 1;
+
+        if (j + len + 1 > out_len)
+        {
+            break;
+        }
         if (c < 0x20 || c == '/' || c == '\\' || c == '"' || c == '`' || c == '$')
         {
             out[j++] = '_';
+            i++;
+            continue;
         }
-        else
+        for (size_t k = 0; k < len && in[i] != '\0'; k++)
         {
-            out[j++] = (char)c;
+            out[j++] = in[i++];
         }
     }
     out[j] = '\0';
@@ -1988,13 +1998,15 @@ static error_t handleApiContentTrackExport(HttpConnection *connection, const cha
     int track_numbers[TAF_TRACK_EXPORT_MAX];
     char titles[TAF_TRACK_EXPORT_MAX][96];
     char *out_paths[TAF_TRACK_EXPORT_MAX];
+    char *entry_names[TAF_TRACK_EXPORT_MAX];
     size_t page_count = 0;
     size_t track_count = 0;
     uint64_t num_bytes = 0;
     long file_size = 0;
     error_t error = NO_ERROR;
 
     osMemset(out_paths, 0, sizeof(out_paths));
+    osMemset(entry_names, 0, sizeof(entry_names));
     osMemset(titles, 0, sizeof(titles));
 
     src = fopen(file_path, "rb");
@@ -2117,6 +2129,7 @@ static error_t handleApiContentTrackExport(HttpConnection *connection, const cha
             break;
         }
 
+        /* the title only goes into the zip entry name, the temp file name stays ASCII */
         if (titles[chapter][0] != '\0')
         {
             osSnprintf(name, sizeof(name), "%02d %s.ogg", track_numbers[i], titles[chapter]);
@@ -2125,7 +2138,7 @@ static error_t handleApiContentTrackExport(HttpConnection *connection, const cha
         {
             osSnprintf(name, sizeof(name), "%02d.ogg", track_numbers[i]);
         }
-        osSnprintf(path, sizeof(path), "%s/%s", tmpdir, name);
+        osSnprintf(path, sizeof(path), "%s/%02d.ogg", tmpdir, track_numbers[i]);
         dst = fopen(path, "wb");
         if (dst == NULL)
         {
@@ -2146,7 +2159,8 @@ static error_t handleApiContentTrackExport(HttpConnection *connection, const cha
             break;
         }
         out_paths[i] = strdup(path);
-        if (out_paths[i] == NULL)
+        entry_names[i] = strdup(name);
+        if (out_paths[i] == NULL || entry_names[i] == NULL)
         {
             error = ERROR_FAILURE;
             break;
@@ -2167,22 +2181,9 @@ static error_t handleApiContentTrackExport(HttpConnection *connection, const cha
     else
     {
         char zip_path[TAF_TEMP_FILE_LEN];
-        char *argv[5 + TAF_TRACK_EXPORT_MAX + 1];
-        size_t n = 0;
 
         osSnprintf(zip_path, sizeof(zip_path), "%s/tracks.zip", tmpdir);
-        argv[n++] = "zip";
-        argv[n++] = "-j";
-        argv[n++] = "-q";
-        argv[n++] = "-X";
-        argv[n++] = zip_path;
-        for (size_t i = 0; i < track_count; i++)
-        {
-            argv[n++] = out_paths[i];
-        }
-        argv[n] = NULL;
-
-        if (osSpawnvp("zip", argv) != 0)
+        if (zipWriteStored(zip_path, track_count, (const char *const *)out_paths, (const char *const *)entry_names) != NO_ERROR)
         {
             TRACE_ERROR("Failed to zip selected TAF tracks from %s\r\n", file_path);
             error = ERROR_FAILURE;
@@ -2198,10 +2199,8 @@ static error_t handleApiContentTrackExport(HttpConnection *connection, const cha
     }
     for (size_t i = 0; i < track_count; i++)
     {
-        if (out_paths[i] != NULL)
-        {
-            free(out_paths[i]);
-        }
+        free(out_paths[i]);
+        free(entry_names[i]);
     }
     if (have_dir)
     {

src/zip_writer.c

@@ -0,0 +1,227 @@
+#include <stdint.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <time.h>
+
+#include "zip_writer.h"
+
+#define ZIP_LOCAL_HEADER_SIG 0x04034b50u
+#define ZIP_CENTRAL_HEADER_SIG 0x02014b50u
+#define ZIP_END_OF_CENTRAL_DIR_SIG 0x06054b50u
+#define ZIP_VERSION 20
+#define ZIP_FLAG_UTF8 0x0800
+#define ZIP_METHOD_STORED 0
+
+typedef struct
+{
+    uint32_t crc;
+    uint32_t size;
+    uint32_t offset;
+    size_t name_len;
+} zip_entry_t;
+
+static uint32_t crc32_update(uint32_t crc, const uint8_t *data, size_t len)
+{
+    crc = ~crc;
+    for (size_t i = 0; i < len; i++)
+    {
+        crc ^= data[i];
+        for (int bit = 0; bit < 8; bit++)
+        {
+            crc = (crc >> 1) ^ (0xEDB88320u & (0u - (crc & 1u)));
+        }
+    }
+    return ~crc;
+}
+
+static void put16(FILE *f, uint16_t v)
+{
+    uint8_t b[2] = {(uint8_t)v, (uint8_t)(v >> 8)};
+    fwrite(b, 1, sizeof(b), f);
+}
+
+static void put32(FILE *f, uint32_t v)
+{
+    uint8_t b[4] = {(uint8_t)v, (uint8_t)(v >> 8), (uint8_t)(v >> 16), (uint8_t)(v >> 24)};
+    fwrite(b, 1, sizeof(b), f);
+}
+
+static void dos_time(uint16_t *dos_date, uint16_t *dos_time_out)
+{
+    time_t now = time(NULL);
+    struct tm *t = localtime(&now);
+    int year = (t != NULL) ? t->tm_year + 1900 : 1980;
+
+    if (year < 1980)
+    {
+        *dos_date = (1 << 5) | 1;
+        *dos_time_out = 0;
+        return;
+    }
+    *dos_date = (uint16_t)(((year - 1980) << 9) | ((t->tm_mon + 1) << 5) | t->tm_mday);
+    *dos_time_out = (uint16_t)((t->tm_hour << 11) | (t->tm_min << 5) | (t->tm_sec / 2));
+}
+
+/* the local header needs crc and size, so read the file once before writing it */
+static int measure_file(const char *path, uint32_t *crc, uint32_t *size)
+{
+    uint8_t buf[8192];
+    uint64_t total = 0;
+    size_t n;
+    FILE *f = fopen(path, "rb");
+
+    if (f == NULL)
+    {
+        return -1;
+    }
+    *crc = 0;
+    while ((n = fread(buf, 1, sizeof(buf), f)) > 0)
+    {
+        *crc = crc32_update(*crc, buf, n);
+        total += n;
+    }
+    if (ferror(f) || total > 0xFFFFFFFFu)
+    {
+        fclose(f);
+        return -1;
+    }
+    fclose(f);
+    *size = (uint32_t)total;
+    return 0;
+}
+
+static int copy_file(const char *path, FILE *out)
+{
+    uint8_t buf[8192];
+    size_t n;
+    FILE *f = fopen(path, "rb");
+
+    if (f == NULL)
+    {
+        return -1;
+    }
+    while ((n = fread(buf, 1, sizeof(buf), f)) > 0)
+    {
+        if (fwrite(buf, 1, n, out) != n)
+        {
+            fclose(f);
+            return -1;
+        }
+    }
+    fclose(f);
+    return 0;
+}
+
+error_t zipWriteStored(const char *zip_path, size_t count, const char *const *file_paths, const char *const *entry_names)
+{
+    zip_entry_t *entries = NULL;
+    uint64_t offset = 0;
+    uint64_t central_start;
+    uint64_t central_size = 0;
+    uint16_t date, time_of_day;
+    FILE *out = NULL;
+    error_t error = ERROR_FAILURE;
+
+    if (count == 0 || count > 0xFFFF)
+    {
+        return ERROR_FAILURE;
+    }
+    entries = calloc(count, sizeof(zip_entry_t));
+    out = fopen(zip_path, "wb");
+    if (entries == NULL || out == NULL)
+    {
+        goto done;
+    }
+    dos_time(&date, &time_of_day);
+
+    for (size_t i = 0; i < count; i++)
+    {
+        zip_entry_t *e = &entries[i];
+
+        e->name_len = strlen(entry_names[i]);
+        if (e->name_len == 0 || e->name_len > 0xFFFF || measure_file(file_paths[i], &e->crc, &e->size) != 0)
+        {
+            goto done;
+        }
+        e->offset = (uint32_t)offset;
+
+        put32(out, ZIP_LOCAL_HEADER_SIG);
+        put16(out, ZIP_VERSION);
+        put16(out, ZIP_FLAG_UTF8);
+        put16(out, ZIP_METHOD_STORED);
+        put16(out, time_of_day);
+        put16(out, date);
+        put32(out, e->crc);
+        put32(out, e->size);
+        put32(out, e->size);
+        put16(out, (uint16_t)e->name_len);
+        put16(out, 0);
+        fwrite(entry_names[i], 1, e->name_len, out);
+        if (copy_file(file_paths[i], out) != 0)
+        {
+            goto done;
+        }
+        offset += 30 + e->name_len + e->size;
+        if (offset > 0xFFFFFFFFu)
+        {
+            goto done;
+        }
+    }
+
+    central_start = offset;
+    for (size_t i = 0; i < count; i++)
+    {
+        const zip_entry_t *e = &entries[i];
+
+        put32(out, ZIP_CENTRAL_HEADER_SIG);
+        put16(out, ZIP_VERSION);
+        put16(out, ZIP_VERSION);
+        put16(out, ZIP_FLAG_UTF8);
+        put16(out, ZIP_METHOD_STORED);
+        put16(out, time_of_day);
+        put16(out, date);
+        put32(out, e->crc);
+        put32(out, e->size);
+        put32(out, e->size);
+        put16(out, (uint16_t)e->name_len);
+        put16(out, 0); /* extra field */
+        put16(out, 0); /* comment */
+        put16(out, 0); /* disk number */
+        put16(out, 0); /* internal attributes */
+        put32(out, 0); /* external attributes */
+        put32(out, e->offset);
+        fwrite(entry_names[i], 1, e->name_len, out);
+        central_size += 46 + e->name_len;
+    }
+    if (central_start + central_size > 0xFFFFFFFFu)
+    {
+        goto done;
+    }
+
+    put32(out, ZIP_END_OF_CENTRAL_DIR_SIG);
+    put16(out, 0);
+    put16(out, 0);
+    put16(out, (uint16_t)count);
+    put16(out, (uint16_t)count);
+    put32(out, (uint32_t)central_size);
+    put32(out, (uint32_t)central_start);
+    put16(out, 0);
+
+    if (ferror(out) == 0)
+    {
+        error = NO_ERROR;
+    }
+
+done:
+    if (out != NULL && fclose(out) != 0)
+    {
+        error = ERROR_FAILURE;
+    }
+    if (error != NO_ERROR)
+    {
+        remove(zip_path);
+    }
+    free(entries);
+    return error;
+}

tests/c/test_runner.c

@@ -28,6 +28,7 @@ static const test_case_t tests[] = {
     {"socket_receive_line_capped_at_size", test_socket_receive_line_capped_at_size},
     {"box_cert_id", test_box_cert_id},
     {"box_cert_issuer_known", test_box_cert_issuer_known},
+    {"zip_write_stored", test_zip_write_stored},
 };
 
 int main(void)