diff --git a/DockerfileAlpine b/DockerfileAlpine index 942f520e0..76ae0108e 100644 --- a/DockerfileAlpine +++ b/DockerfileAlpine @@ -24,7 +24,7 @@ LABEL org.opencontainers.image.version=$TEDDYCLOUD_VERSION EXPOSE 80 443 8443 # Install necessary runtime dependencies -RUN apk --no-cache add strace ffmpeg zip curl ca-certificates bash su-exec shadow libcap +RUN apk --no-cache add strace ffmpeg curl ca-certificates bash su-exec shadow libcap # Update CA certificates RUN update-ca-certificates diff --git a/DockerfileUbuntu b/DockerfileUbuntu index 53b0237d5..79ed38904 100644 --- a/DockerfileUbuntu +++ b/DockerfileUbuntu @@ -20,7 +20,7 @@ LABEL org.opencontainers.image.version=$TEDDYCLOUD_VERSION EXPOSE 80 443 8443 RUN apt-get update \ - && apt-get install -y --no-install-recommends libubsan1 strace ffmpeg zip curl ca-certificates gosu libcap2-bin \ + && apt-get install -y --no-install-recommends libubsan1 strace ffmpeg curl ca-certificates gosu libcap2-bin \ && rm -rf /var/lib/apt/lists/* RUN update-ca-certificates diff --git a/Makefile b/Makefile index 34c229e57..192cdae51 100644 --- a/Makefile +++ b/Makefile @@ -811,7 +811,7 @@ TEST_C_SRC := $(wildcard tests/c/*.c) test_c: $(QUIET)$(ECHO) "[ ${CYAN}TEST${NC} ] Build and run C unit tests" $(QUIET)mkdir -p $(BIN_DIR) - $(QUIET)$(CC) -I include -I src/cyclone/common -I src/cyclone/cyclone_tcp -I cyclone/common -I cyclone/cyclone_tcp -I cyclone/cyclone_ssl -I cyclone/cyclone_crypto -D GPL_LICENSE_TERMS_ACCEPTED -ffunction-sections -Wl,--gc-sections -o $(BIN_DIR)/test_c $(TEST_C_SRC) $(SRC_DIR)/os_ext.c $(SRC_DIR)/str_ext.c $(SRC_DIR)/fs_ext.c $(SRC_DIR)/esp32_port.c $(SRC_DIR)/platform/platform_linux.c $(SRC_DIR)/cyclone/common/os_port_posix.c $(SRC_DIR)/box_cert.c + $(QUIET)$(CC) -I include -I src/cyclone/common -I src/cyclone/cyclone_tcp -I cyclone/common -I cyclone/cyclone_tcp -I cyclone/cyclone_ssl -I cyclone/cyclone_crypto -D GPL_LICENSE_TERMS_ACCEPTED -ffunction-sections -Wl,--gc-sections -o $(BIN_DIR)/test_c $(TEST_C_SRC) $(SRC_DIR)/os_ext.c $(SRC_DIR)/str_ext.c $(SRC_DIR)/fs_ext.c $(SRC_DIR)/esp32_port.c $(SRC_DIR)/platform/platform_linux.c $(SRC_DIR)/cyclone/common/os_port_posix.c $(SRC_DIR)/box_cert.c $(SRC_DIR)/zip_writer.c $(QUIET)$(BIN_DIR)/test_c .PHONY: test diff --git a/include/zip_writer.h b/include/zip_writer.h new file mode 100644 index 000000000..7ede4c0c2 --- /dev/null +++ b/include/zip_writer.h @@ -0,0 +1,18 @@ +#pragma once + +#include +#include "error.h" + +/** + * Writes a zip archive that stores the given files uncompressed (Ogg/Opus does not compress). + * The entry names are taken as UTF-8 and flagged as such (general purpose bit 11), so unzip + * tools show non-ASCII names correctly regardless of the locale of the server. + * Limited to the classic zip format: at most 65535 entries and 4 GiB in total. + * + * @param zip_path archive to create + * @param count number of entries + * @param file_paths files to add + * @param entry_names names of the entries inside the archive, UTF-8, no directories + * @return NO_ERROR or ERROR_FAILURE (the partial archive is removed) + */ +error_t zipWriteStored(const char *zip_path, size_t count, const char *const *file_paths, const char *const *entry_names); diff --git a/src/handler_api.c b/src/handler_api.c index a0e4c9281..ee9fb6f2a 100644 --- a/src/handler_api.c +++ b/src/handler_api.c @@ -26,6 +26,7 @@ #include "esp32.h" #include "cache.h" #include "libraryMeta.h" +#include "zip_writer.h" error_t parsePostData(HttpConnection *connection, char_t *post_data, size_t buffer_size) { @@ -1899,16 +1900,25 @@ static void sanitize_track_title(const char *in, char *out, size_t out_len) { in = ""; } - for (size_t i = 0; in[i] != '\0' && j + 1 < out_len; i++) + for (size_t i = 0; in[i] != '\0';) { unsigned char c = (unsigned char)in[i]; + /* length of the UTF-8 sequence, so a truncated title never ends in half a character */ + size_t len = (c >= 0xF0) ? 4 : (c >= 0xE0) ? 3 : (c >= 0xC0) ? 2 : 1; + + if (j + len + 1 > out_len) + { + break; + } if (c < 0x20 || c == '/' || c == '\\' || c == '"' || c == '`' || c == '$') { out[j++] = '_'; + i++; + continue; } - else + for (size_t k = 0; k < len && in[i] != '\0'; k++) { - out[j++] = (char)c; + out[j++] = in[i++]; } } out[j] = '\0'; @@ -1988,6 +1998,7 @@ static error_t handleApiContentTrackExport(HttpConnection *connection, const cha int track_numbers[TAF_TRACK_EXPORT_MAX]; char titles[TAF_TRACK_EXPORT_MAX][96]; char *out_paths[TAF_TRACK_EXPORT_MAX]; + char *entry_names[TAF_TRACK_EXPORT_MAX]; size_t page_count = 0; size_t track_count = 0; uint64_t num_bytes = 0; @@ -1995,6 +2006,7 @@ static error_t handleApiContentTrackExport(HttpConnection *connection, const cha error_t error = NO_ERROR; osMemset(out_paths, 0, sizeof(out_paths)); + osMemset(entry_names, 0, sizeof(entry_names)); osMemset(titles, 0, sizeof(titles)); src = fopen(file_path, "rb"); @@ -2117,6 +2129,7 @@ static error_t handleApiContentTrackExport(HttpConnection *connection, const cha break; } + /* the title only goes into the zip entry name, the temp file name stays ASCII */ if (titles[chapter][0] != '\0') { osSnprintf(name, sizeof(name), "%02d %s.ogg", track_numbers[i], titles[chapter]); @@ -2125,7 +2138,7 @@ static error_t handleApiContentTrackExport(HttpConnection *connection, const cha { osSnprintf(name, sizeof(name), "%02d.ogg", track_numbers[i]); } - osSnprintf(path, sizeof(path), "%s/%s", tmpdir, name); + osSnprintf(path, sizeof(path), "%s/%02d.ogg", tmpdir, track_numbers[i]); dst = fopen(path, "wb"); if (dst == NULL) { @@ -2146,7 +2159,8 @@ static error_t handleApiContentTrackExport(HttpConnection *connection, const cha break; } out_paths[i] = strdup(path); - if (out_paths[i] == NULL) + entry_names[i] = strdup(name); + if (out_paths[i] == NULL || entry_names[i] == NULL) { error = ERROR_FAILURE; break; @@ -2167,22 +2181,9 @@ static error_t handleApiContentTrackExport(HttpConnection *connection, const cha else { char zip_path[TAF_TEMP_FILE_LEN]; - char *argv[5 + TAF_TRACK_EXPORT_MAX + 1]; - size_t n = 0; osSnprintf(zip_path, sizeof(zip_path), "%s/tracks.zip", tmpdir); - argv[n++] = "zip"; - argv[n++] = "-j"; - argv[n++] = "-q"; - argv[n++] = "-X"; - argv[n++] = zip_path; - for (size_t i = 0; i < track_count; i++) - { - argv[n++] = out_paths[i]; - } - argv[n] = NULL; - - if (osSpawnvp("zip", argv) != 0) + if (zipWriteStored(zip_path, track_count, (const char *const *)out_paths, (const char *const *)entry_names) != NO_ERROR) { TRACE_ERROR("Failed to zip selected TAF tracks from %s\r\n", file_path); error = ERROR_FAILURE; @@ -2198,10 +2199,8 @@ static error_t handleApiContentTrackExport(HttpConnection *connection, const cha } for (size_t i = 0; i < track_count; i++) { - if (out_paths[i] != NULL) - { - free(out_paths[i]); - } + free(out_paths[i]); + free(entry_names[i]); } if (have_dir) { diff --git a/src/zip_writer.c b/src/zip_writer.c new file mode 100644 index 000000000..4d36c761f --- /dev/null +++ b/src/zip_writer.c @@ -0,0 +1,227 @@ +#include +#include +#include +#include +#include + +#include "zip_writer.h" + +#define ZIP_LOCAL_HEADER_SIG 0x04034b50u +#define ZIP_CENTRAL_HEADER_SIG 0x02014b50u +#define ZIP_END_OF_CENTRAL_DIR_SIG 0x06054b50u +#define ZIP_VERSION 20 +#define ZIP_FLAG_UTF8 0x0800 +#define ZIP_METHOD_STORED 0 + +typedef struct +{ + uint32_t crc; + uint32_t size; + uint32_t offset; + size_t name_len; +} zip_entry_t; + +static uint32_t crc32_update(uint32_t crc, const uint8_t *data, size_t len) +{ + crc = ~crc; + for (size_t i = 0; i < len; i++) + { + crc ^= data[i]; + for (int bit = 0; bit < 8; bit++) + { + crc = (crc >> 1) ^ (0xEDB88320u & (0u - (crc & 1u))); + } + } + return ~crc; +} + +static void put16(FILE *f, uint16_t v) +{ + uint8_t b[2] = {(uint8_t)v, (uint8_t)(v >> 8)}; + fwrite(b, 1, sizeof(b), f); +} + +static void put32(FILE *f, uint32_t v) +{ + uint8_t b[4] = {(uint8_t)v, (uint8_t)(v >> 8), (uint8_t)(v >> 16), (uint8_t)(v >> 24)}; + fwrite(b, 1, sizeof(b), f); +} + +static void dos_time(uint16_t *dos_date, uint16_t *dos_time_out) +{ + time_t now = time(NULL); + struct tm *t = localtime(&now); + int year = (t != NULL) ? t->tm_year + 1900 : 1980; + + if (year < 1980) + { + *dos_date = (1 << 5) | 1; + *dos_time_out = 0; + return; + } + *dos_date = (uint16_t)(((year - 1980) << 9) | ((t->tm_mon + 1) << 5) | t->tm_mday); + *dos_time_out = (uint16_t)((t->tm_hour << 11) | (t->tm_min << 5) | (t->tm_sec / 2)); +} + +/* the local header needs crc and size, so read the file once before writing it */ +static int measure_file(const char *path, uint32_t *crc, uint32_t *size) +{ + uint8_t buf[8192]; + uint64_t total = 0; + size_t n; + FILE *f = fopen(path, "rb"); + + if (f == NULL) + { + return -1; + } + *crc = 0; + while ((n = fread(buf, 1, sizeof(buf), f)) > 0) + { + *crc = crc32_update(*crc, buf, n); + total += n; + } + if (ferror(f) || total > 0xFFFFFFFFu) + { + fclose(f); + return -1; + } + fclose(f); + *size = (uint32_t)total; + return 0; +} + +static int copy_file(const char *path, FILE *out) +{ + uint8_t buf[8192]; + size_t n; + FILE *f = fopen(path, "rb"); + + if (f == NULL) + { + return -1; + } + while ((n = fread(buf, 1, sizeof(buf), f)) > 0) + { + if (fwrite(buf, 1, n, out) != n) + { + fclose(f); + return -1; + } + } + fclose(f); + return 0; +} + +error_t zipWriteStored(const char *zip_path, size_t count, const char *const *file_paths, const char *const *entry_names) +{ + zip_entry_t *entries = NULL; + uint64_t offset = 0; + uint64_t central_start; + uint64_t central_size = 0; + uint16_t date, time_of_day; + FILE *out = NULL; + error_t error = ERROR_FAILURE; + + if (count == 0 || count > 0xFFFF) + { + return ERROR_FAILURE; + } + entries = calloc(count, sizeof(zip_entry_t)); + out = fopen(zip_path, "wb"); + if (entries == NULL || out == NULL) + { + goto done; + } + dos_time(&date, &time_of_day); + + for (size_t i = 0; i < count; i++) + { + zip_entry_t *e = &entries[i]; + + e->name_len = strlen(entry_names[i]); + if (e->name_len == 0 || e->name_len > 0xFFFF || measure_file(file_paths[i], &e->crc, &e->size) != 0) + { + goto done; + } + e->offset = (uint32_t)offset; + + put32(out, ZIP_LOCAL_HEADER_SIG); + put16(out, ZIP_VERSION); + put16(out, ZIP_FLAG_UTF8); + put16(out, ZIP_METHOD_STORED); + put16(out, time_of_day); + put16(out, date); + put32(out, e->crc); + put32(out, e->size); + put32(out, e->size); + put16(out, (uint16_t)e->name_len); + put16(out, 0); + fwrite(entry_names[i], 1, e->name_len, out); + if (copy_file(file_paths[i], out) != 0) + { + goto done; + } + offset += 30 + e->name_len + e->size; + if (offset > 0xFFFFFFFFu) + { + goto done; + } + } + + central_start = offset; + for (size_t i = 0; i < count; i++) + { + const zip_entry_t *e = &entries[i]; + + put32(out, ZIP_CENTRAL_HEADER_SIG); + put16(out, ZIP_VERSION); + put16(out, ZIP_VERSION); + put16(out, ZIP_FLAG_UTF8); + put16(out, ZIP_METHOD_STORED); + put16(out, time_of_day); + put16(out, date); + put32(out, e->crc); + put32(out, e->size); + put32(out, e->size); + put16(out, (uint16_t)e->name_len); + put16(out, 0); /* extra field */ + put16(out, 0); /* comment */ + put16(out, 0); /* disk number */ + put16(out, 0); /* internal attributes */ + put32(out, 0); /* external attributes */ + put32(out, e->offset); + fwrite(entry_names[i], 1, e->name_len, out); + central_size += 46 + e->name_len; + } + if (central_start + central_size > 0xFFFFFFFFu) + { + goto done; + } + + put32(out, ZIP_END_OF_CENTRAL_DIR_SIG); + put16(out, 0); + put16(out, 0); + put16(out, (uint16_t)count); + put16(out, (uint16_t)count); + put32(out, (uint32_t)central_size); + put32(out, (uint32_t)central_start); + put16(out, 0); + + if (ferror(out) == 0) + { + error = NO_ERROR; + } + +done: + if (out != NULL && fclose(out) != 0) + { + error = ERROR_FAILURE; + } + if (error != NO_ERROR) + { + remove(zip_path); + } + free(entries); + return error; +} diff --git a/tests/c/test_runner.c b/tests/c/test_runner.c index d4ae2efb9..95e96193e 100644 --- a/tests/c/test_runner.c +++ b/tests/c/test_runner.c @@ -28,6 +28,7 @@ static const test_case_t tests[] = { {"socket_receive_line_capped_at_size", test_socket_receive_line_capped_at_size}, {"box_cert_id", test_box_cert_id}, {"box_cert_issuer_known", test_box_cert_issuer_known}, + {"zip_write_stored", test_zip_write_stored}, }; int main(void) diff --git a/tests/c/test_zip_writer.c b/tests/c/test_zip_writer.c new file mode 100644 index 000000000..655e99976 --- /dev/null +++ b/tests/c/test_zip_writer.c @@ -0,0 +1,77 @@ +#include +#include +#include +#include + +#include "tests.h" +#include "zip_writer.h" + +static uint32_t rd16(const uint8_t *p) +{ + return (uint32_t)p[0] | ((uint32_t)p[1] << 8); +} + +static uint32_t rd32(const uint8_t *p) +{ + return rd16(p) | (rd16(p + 2) << 16); +} + +static void write_file(const char *path, const char *data) +{ + FILE *f = fopen(path, "wb"); + assert(f != NULL); + assert(fwrite(data, 1, strlen(data), f) == strlen(data)); + fclose(f); +} + +void test_zip_write_stored(void) +{ + const char *zip_path = "bin/test_zip_writer.zip"; + const char *files[] = {"bin/test_zip_writer_a.bin", "bin/test_zip_writer_b.bin"}; + /* typographic quotes as in teddycloud#552 */ + const char *names[] = {"01 Intro.ogg", "09 3. Satz: \xE2\x80\x9EAllegro\xE2\x80\x9D.ogg"}; + uint8_t buf[1024]; + + write_file(files[0], "123456789"); + write_file(files[1], ""); + assert(zipWriteStored(zip_path, 2, files, names) == NO_ERROR); + + FILE *f = fopen(zip_path, "rb"); + assert(f != NULL); + size_t len = fread(buf, 1, sizeof(buf), f); + fclose(f); + + /* first local header: stored, UTF-8 flag, crc32("123456789"), sizes, name, data */ + assert(rd32(buf) == 0x04034b50u); + assert(rd16(buf + 6) == 0x0800); + assert(rd16(buf + 8) == 0); + assert(rd32(buf + 14) == 0xCBF43926u); + assert(rd32(buf + 18) == 9 && rd32(buf + 22) == 9); + size_t name0 = strlen(names[0]); + assert(rd16(buf + 26) == name0); + assert(memcmp(buf + 30, names[0], name0) == 0); + assert(memcmp(buf + 30 + name0, "123456789", 9) == 0); + + /* end of central directory: two entries, UTF-8 name preserved byte for byte */ + const uint8_t *eocd = buf + len - 22; + assert(rd32(eocd) == 0x06054b50u); + assert(rd16(eocd + 10) == 2); + size_t name1 = strlen(names[1]); + int found = 0; + for (size_t i = 0; i + name1 <= len; i++) + { + found |= memcmp(buf + i, names[1], name1) == 0; + } + assert(found); + const uint8_t *central = buf + rd32(eocd + 16); + assert(rd32(central) == 0x02014b50u); + assert(rd16(central + 8) == 0x0800); + + /* a missing source file fails and leaves no partial archive */ + const char *missing[] = {"bin/does_not_exist.bin"}; + assert(zipWriteStored(zip_path, 1, missing, names) != NO_ERROR); + assert(fopen(zip_path, "rb") == NULL); + + remove(files[0]); + remove(files[1]); +} diff --git a/tests/c/tests.h b/tests/c/tests.h index f795c1c6b..13e7f8d26 100644 --- a/tests/c/tests.h +++ b/tests/c/tests.h @@ -14,3 +14,4 @@ void test_socket_receive_line_from_full_buffer(void); void test_socket_receive_line_capped_at_size(void); void test_box_cert_id(void); void test_box_cert_issuer_known(void); +void test_zip_write_stored(void); diff --git a/tests/py/test_track_export.py b/tests/py/test_track_export.py index ef3c2fc20..7b6450f77 100644 --- a/tests/py/test_track_export.py +++ b/tests/py/test_track_export.py @@ -7,7 +7,7 @@ /content/download/?...&tracks=... tonie card, tag with a lib:// source (#350) The TAF is encoded by the server from generated white noise (needs ffmpeg), -the zip is built with the external zip binary. +the zip is written by the server itself (zip_writer.c). Run via: make test_py TESTS=track_export """ @@ -20,12 +20,13 @@ import wave import zipfile -from test_hardening import BASE, BASE_DIR, Base +from test_hardening import BASE, BASE_DIR, Base, request TRACK_SECONDS = [2, 2, 2] RATE = 4000 # low rate: the test upload helper only handles small bodies, the server resamples TAF = "__trackexport.taf" TAG_DIR, TAG_FILE = "A64E581D", "500304E0" +CUSTOM_MODEL = "TC_TEST_TRACK_EXPORT" def white_noise_wav(seconds): @@ -66,8 +67,26 @@ def setUp(self): def tearDown(self): self.tag_json.unlink(missing_ok=True) + body = json.dumps({"models": [CUSTOM_MODEL]}) + request("POST", "/api/toniesCustomJsonDelete", body, {"Content-Type": "application/json"}) super().tearDown() + def name_tracks(self, titles): + """Give the TAF track titles, like a tonies.custom.json entry does for a real tonie.""" + status, text = request("GET", "/api/fileIndexV2?path=/&special=library") + self.assertEqual(status, 200, text) + header = next(f["tafHeader"] for f in json.loads(text)["files"] if f["name"] == TAF) + entry = { + "model": CUSTOM_MODEL, + "series": "Track export test", + "title": "Track export test", + "audio_id": [header["audioId"]], + "hash": [header["sha1Hash"].upper()], + "tracks": titles, + } + status, text = request("POST", "/api/toniesCustomJsonUpsert", json.dumps([entry]), {"Content-Type": "application/json"}) + self.assertEqual(status, 200, text) + def assert_zip_of_tracks(self, path, count): status, content_type, body = get(path) self.assertEqual(status, 200, body[:200]) @@ -101,6 +120,23 @@ def test_content_invalid_track(self): status, _, _ = get(f"/content/{TAF}?special=library&ogg=true&{query}") self.assertEqual(status, 400, query) + def test_zip_entry_names_are_utf8(self): + # typographic quotes as in #552, a path separator, and a title longer than the name buffer + self.name_tracks(["Käse/Mäuse – Überraschung", "3. Satz: „Allegro”", "ä" * 60 + " Ende"]) + status, content_type, body = get(f"/content/{TAF}?special=library&ogg=true&tracks=1%2C2%2C3") + self.assertEqual(status, 200, body[:200]) + self.assertEqual(content_type, "application/zip") + with zipfile.ZipFile(io.BytesIO(body)) as z: + self.assertIsNone(z.testzip()) + infos = z.infolist() + # general purpose bit 11: the names are UTF-8 whatever the locale of the server + self.assertTrue(all(i.flag_bits & 0x800 for i in infos)) + names = [i.filename for i in infos] + self.assertEqual(names[0], "01 Käse_Mäuse – Überraschung.ogg") + self.assertEqual(names[1], "02 3. Satz: „Allegro”.ogg") + # cut at a character boundary, never in the middle of a UTF-8 sequence + self.assertEqual(names[2], "03 " + "ä" * 47 + ".ogg") + def test_content_download_tag_invalid_track(self): for query in ("tracks=9", "tracks=0", "tracks=abc", "tracks=,"): status, _, _ = get(f"/content/download/{TAG_DIR}/{TAG_FILE}?skip_header=true&ogg=true&{query}")