Skip to content

Report corrupted TAFs and allow redownloading them from the cloud #565

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
weatherhog wants to merge 2 commits into
base: develop
Choose a base branch
from
Open

Report corrupted TAFs and allow redownloading them from the cloud #565

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
1c955b3
Replace a corrupted library TAF when it is cached again
weatherhog Oct 26, 2026
95978cc
Report corrupted TAFs and allow redownloading them from the cloud
weatherhog Oct 26, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions include/contentJson.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -64,6 +64,7 @@ typedef struct
char *jsonPath;
bool_t exists;
bool_t valid;
bool_t corrupt; // header is fine, but SHA1 hash or length of the audio is not
bool_t updated;
bool_t locked;
contentJson_t json;
Expand Down
2 changes: 1 addition & 1 deletion include/handler_cloud.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,7 @@ error_t handleCloudChapterV3(HttpConnection *connection, const char_t *uri, cons
error_t handleCloudContentMetaV3(HttpConnection *connection, const char_t *uri, const char_t *queryString, client_ctx_t *client_ctx);
error_t handleCloudLog(HttpConnection *connection, const char_t *uri, const char_t *queryString, client_ctx_t *client_ctx);
error_t handleCloudClaim(HttpConnection *connection, const char_t *uri, const char_t *queryString, client_ctx_t *client_ctx);
error_t handleCloudContent(HttpConnection *connection, const char_t *uri, const char_t *queryString, client_ctx_t *client_ctx, bool_t noPassword);
error_t handleCloudContent(HttpConnection *connection, const char_t *uri, const char_t *queryString, client_ctx_t *client_ctx, bool_t noPassword, bool_t forceCloud);
error_t handleCloudContentV1(HttpConnection *connection, const char_t *uri, const char_t *queryString, client_ctx_t *client_ctx);
error_t handleCloudContentV2(HttpConnection *connection, const char_t *uri, const char_t *queryString, client_ctx_t *client_ctx);
error_t handleCloudFreshnessCheck(HttpConnection *connection, const char_t *uri, const char_t *queryString, client_ctx_t *client_ctx);
Expand Down
3 changes: 2 additions & 1 deletion src/contentJson.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -100,7 +100,8 @@ error_t load_content_json(const char *content_path, contentJson_t *content_json,
if (osStrlen(content_json->source) > 0)
{
resolveSpecialPathPrefix(&content_json->_source_resolved, settings);
if (isValidTaf(content_json->_source_resolved, settings->core.full_taf_validation))
/* header only: a TAF with a wrong SHA1 hash is still a TAF (corrupted), not a stream source */
if (isValidTaf(content_json->_source_resolved, false))
{
content_json->_source_type = CT_SOURCE_TAF;
}
Expand Down
23 changes: 18 additions & 5 deletions src/handler.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -884,11 +884,12 @@ void readTrackPositions(tonie_info_t *tonieInfo, FsFile *file)
osFreeMem(trackPos->pos);
trackPos->pos = NULL;

if (get_settings()->core.track_pos_taf_validation)
if (get_settings()->core.track_pos_taf_validation && !tonieInfo->corrupt)
{
if (!isValidTaf(tonieInfo->contentPath, true))
{
TRACE_ERROR("SHA1 not valid or length different for TAF %s. File may be corrupted!\r\n", tonieInfo->contentPath);
tonieInfo->corrupt = true;
}
}
}
Expand Down Expand Up @@ -921,6 +922,7 @@ tonie_info_t *getTonieInfoV2(const char *contentPath, bool lock, bool force_taf_
tonieInfo = osAllocMem(sizeof(tonie_info_t));

tonieInfo->valid = false;
tonieInfo->corrupt = false;
tonieInfo->updated = false;
tonieInfo->tafHeader = NULL;
tonieInfo->contentPath = strdup(contentPath);
Expand Down Expand Up @@ -976,11 +978,15 @@ tonie_info_t *getTonieInfoV2(const char *contentPath, bool lock, bool force_taf_
{
if (tonieInfo->tafHeader->sha1_hash.len == 20)
{
tonieInfo->valid = isValidTaf(tonieInfo->contentPath, settings->core.full_taf_validation || force_taf_validation);
bool_t fullValidation = settings->core.full_taf_validation || force_taf_validation;
tonieInfo->valid = isValidTaf(tonieInfo->contentPath, fullValidation);
tonieInfo->corrupt = fullValidation && !tonieInfo->valid;
readTrackPositions(tonieInfo, file);
if (tonieInfo->tafHeader->num_bytes == get_settings()->encode.stream_max_size)
{
tonieInfo->json._source_type = CT_SOURCE_TAF_INCOMPLETE;
/* still being written, the hash cannot match yet */
tonieInfo->corrupt = false;
}
else if ((tonieInfo->json.tonie_model == NULL || tonieInfo->json.tonie_model[0] == '\0') && (tonieInfo->json._source_type == CT_SOURCE_NONE || tonieInfo->json._source_type == CT_SOURCE_TAF)) // TAF beside the content json
{
Expand Down Expand Up @@ -1225,10 +1231,17 @@ error_t moveTAF2Lib(tonie_info_t *tonieInfo, settings_t *settings, bool_t rootDi
}
if (libraryPath)
{
tonie_info_t *tonieInfoLib = getTonieInfo(libraryPath, false, settings);
/* full validation, a corrupted library file has the same header hash as a fresh download */
tonie_info_t *tonieInfoLib = getTonieInfoV2(libraryPath, false, true, settings);
bool moveToLibrary = true;
bool skipMove = false;
if (tonieInfoLib->valid)
bool overwrite = false;
if (tonieInfoLib->exists && !tonieInfoLib->valid)
{
TRACE_WARNING(">> Library file %s for Audio ID %" PRIu32 " is corrupted, replacing it\r\n", libraryPath, audioId);
overwrite = true;
}
else if (tonieInfoLib->valid)
{
if (!osMemcmp(tonieInfoLib->tafHeader->sha1_hash.data, tonieInfo->tafHeader->sha1_hash.data, tonieInfoLib->tafHeader->sha1_hash.len))
{
Expand All @@ -1247,7 +1260,7 @@ error_t moveTAF2Lib(tonie_info_t *tonieInfo, settings_t *settings, bool_t rootDi
{
if (!skipMove)
{
error = fsMoveFile(tonieInfo->contentPath, libraryPath, false);
error = fsMoveFile(tonieInfo->contentPath, libraryPath, overwrite);
}
if (error == NO_ERROR)
{
Expand Down
72 changes: 66 additions & 6 deletions src/handler_api.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2378,6 +2378,32 @@ error_t handleApiContent(HttpConnection *connection, const char_t *uri, const ch

return error;
}

static error_t writeApiStatusText(HttpConnection *connection, uint_t statusCode, const char *message);

/* The tag plays its own cloud content: a TAF in the content dir, or the library file it was cached to.
* A forced cloud download clears the source, so it must not be offered for any other source. */
static bool_t isCachedCloudContent(tonie_info_t *tonieInfo)
{
if (tonieInfo->json._source_type != CT_SOURCE_NONE && tonieInfo->json._source_type != CT_SOURCE_TAF)
{
return false;
}
if (tonieInfo->json.source == NULL || tonieInfo->json.source[0] == '\0')
{
return true;
}
if (tonieInfo->tafHeader == NULL)
{
return false;
}
/* same as getLibraryCachePath(settings, audioId, true), which also creates directories */
char *cachePath = custom_asprintf("lib://by/audioID/%" PRIu32 ".taf", tonieInfo->tafHeader->audio_id);
bool_t cached = osStrcmp(tonieInfo->json.source, cachePath) == 0;
osFreeMem(cachePath);
return cached;
}

error_t handleApiContentDownload(HttpConnection *connection, const char_t *uri, const char_t *queryString, client_ctx_t *client_ctx)
{
// TODO Rewrite URL
Expand Down Expand Up @@ -2431,6 +2457,30 @@ error_t handleApiContentDownload(HttpConnection *connection, const char_t *uri,
return export_error;
}

bool_t forceCloud = false;
char force_param[8];
if (queryGet(queryString, "force", force_param, sizeof(force_param)) && force_param[0] == 't')
{
tonie_info_t *tonieInfo = getTonieInfo(pathAbsolute, false, client_ctx->settings);
bool_t cached = isCachedCloudContent(tonieInfo);
freeTonieInfo(tonieInfo);
if (!cached)
{
osFreeMem(pathAbsolute);
TRACE_WARNING("Not redownloading '%s', the tag has a custom source\r\n", path);
return writeApiStatusText(connection, 409, "Tag has a custom source, not redownloading it");
}
if (!client_ctx->settings->cloud.cacheContent)
{
osFreeMem(pathAbsolute);
TRACE_WARNING("Not redownloading '%s', cloud.cacheContent is disabled\r\n", path);
return writeApiStatusText(connection, 409, "Caching cloud content is disabled, not redownloading");
}
forceCloud = true;
/* force and overlay are local parameters, the cloud gets a plain content request */
queryString = "";
}

char ruid[17];

contentJson_t contentJson;
Expand All @@ -2451,13 +2501,13 @@ error_t handleApiContentDownload(HttpConnection *connection, const char_t *uri,
if (isSys || contentJson.nocloud)
{
osSprintf((char *)uri, "/v1/content/%s", ruid);
error = handleCloudContent(connection, uri, queryString, client_ctx, true);
error = handleCloudContent(connection, uri, queryString, client_ctx, true, forceCloud);
}
else
{
osMemcpy(connection->private.authentication_token, contentJson.cloud_auth, contentJson.cloud_auth_len);
osSprintf((char *)uri, "/v2/content/%s", ruid);
error = handleCloudContent(connection, uri, queryString, client_ctx, false);
error = handleCloudContent(connection, uri, queryString, client_ctx, false, forceCloud);
}

free_content_json(&contentJson);
Expand Down Expand Up @@ -4927,7 +4977,7 @@ bool isHexString(const char *buf, size_t maxLen)
return isHex;
}

error_t getTagInfoJson(char ruid[17], cJSON *jsonTarget, client_ctx_t *client_ctx)
error_t getTagInfoJson(char ruid[17], cJSON *jsonTarget, bool_t validate, client_ctx_t *client_ctx)
{
error_t error = NO_ERROR;
/* build filename with 8 chars of the taf/json */
Expand All @@ -4944,7 +4994,7 @@ error_t getTagInfoJson(char ruid[17], cJSON *jsonTarget, client_ctx_t *client_ct
contentJson_t contentJson;
/* read TAF info - would create .json if not existing */
// tonie_info_t *tafInfo = getTonieInfoFromRuid(ruid, true, client_ctx->settings);
tonie_info_t *tafInfo = getTonieInfo(fullTagPath, false, client_ctx->settings);
tonie_info_t *tafInfo = getTonieInfoV2(fullTagPath, false, validate, client_ctx->settings);
/* now update with updated model if found. */
saveTonieInfo(tafInfo, true);
contentJson = tafInfo->json;
Expand Down Expand Up @@ -4978,6 +5028,7 @@ error_t getTagInfoJson(char ruid[17], cJSON *jsonTarget, client_ctx_t *client_ct
}
cJSON_AddBoolToObject(jsonEntry, "valid", tafInfo->valid);
cJSON_AddBoolToObject(jsonEntry, "exists", tafInfo->exists);
cJSON_AddBoolToObject(jsonEntry, "corrupt", tafInfo->corrupt);
cJSON_AddBoolToObject(jsonEntry, "live", tafInfo->json.live);
cJSON_AddBoolToObject(jsonEntry, "nocloud", tafInfo->json.nocloud);
cJSON_AddBoolToObject(jsonEntry, "hasCloudAuth", tafInfo->json._has_cloud_auth && !tafInfo->json.cloud_override);
Expand Down Expand Up @@ -5009,6 +5060,12 @@ error_t getTagInfoJson(char ruid[17], cJSON *jsonTarget, client_ctx_t *client_ct
cJSON_AddStringToObject(jsonEntry, "downloadTriggerUrl", "");
}
}
if (tafInfo->exists && !tafInfo->json.nocloud && client_ctx->settings->cloud.cacheContent && (contentJson._has_cloud_auth || isSys) && isCachedCloudContent(tafInfo))
{
char *redownloadUrl = custom_asprintf("%s&force=true", downloadUrl);
cJSON_AddStringToObject(jsonEntry, "redownloadUrl", redownloadUrl);
osFreeMem(redownloadUrl);
}
osFreeMem(audioUrl);
osFreeMem(downloadUrl);

Expand Down Expand Up @@ -5065,10 +5122,13 @@ error_t handleApiTagInfo(HttpConnection *connection, const char_t *uri, const ch
{
return ERROR_FAILURE;
}
/* validate=true reads the whole TAF to check its SHA1 hash and length */
char validate[8];
bool_t fullValidation = queryGet(queryString, "validate", validate, sizeof(validate)) && validate[0] == 't';

cJSON *json = cJSON_CreateObject();

error_t error = getTagInfoJson(ruid, json, client_ctx);
error_t error = getTagInfoJson(ruid, json, fullValidation, client_ctx);
if (error != NO_ERROR)
{
cJSON_Delete(json);
Expand Down Expand Up @@ -5160,7 +5220,7 @@ error_t handleApiTagIndex(HttpConnection *connection, const char_t *uri, const c
ruid[i] = tolower(ruid[i]);
}

if (getTagInfoJson(ruid, jsonArray, client_ctx) == NO_ERROR)
if (getTagInfoJson(ruid, jsonArray, false, client_ctx) == NO_ERROR)
{
break;
}
Expand Down
39 changes: 32 additions & 7 deletions src/handler_cloud.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -625,11 +625,27 @@ tonie_info_t *getTonieInfoForRequest(HttpConnection *connection, const char_t *u
return tonieInfo;
}

error_t handleCloudContentExt(HttpConnection *connection, const char_t *uri, const char_t *queryString, client_ctx_t *client_ctx, bool_t noPassword, int ruid_begin, cloudapi_t api)
/* A forced download clears the source before fetching from the cloud. If nothing was cached,
* the tag gets its previous (corrupted, but existing) file back instead of none. */
static void restoreSourceIfNotCached(char ruid[17], const char *previousSource, settings_t *settings)
{
tonie_info_t *tonieInfo = getTonieInfoFromRuid(ruid, true, settings);
if (!tonieInfo->exists && (tonieInfo->json.source == NULL || tonieInfo->json.source[0] == '\0'))
{
TRACE_WARNING(" >> Nothing cached from the cloud, restoring source %s\r\n", previousSource);
free(tonieInfo->json.source);
tonieInfo->json.source = strdup(previousSource);
tonieInfo->json._updated = true;
}
freeTonieInfo(tonieInfo);
}

error_t handleCloudContentExt(HttpConnection *connection, const char_t *uri, const char_t *queryString, client_ctx_t *client_ctx, bool_t noPassword, int ruid_begin, cloudapi_t api, bool_t forceCloud)
{
char ruid[17];
error_t error = NO_ERROR;
bool_t tonie_marked = false;
char *previousSource = NULL;

tonie_info_t *tonieInfo = getTonieInfoForRequest(connection, uri, ruid_begin, queryString, client_ctx, noPassword, ruid, &tonie_marked, &error);

Expand Down Expand Up @@ -753,7 +769,7 @@ error_t handleCloudContentExt(HttpConnection *connection, const char_t *uri, con
osDelayTask(100);
}
}
else if (tonieInfo->exists && tonieInfo->valid && (!tonie_marked || !can_use_cloud))
else if (tonieInfo->exists && tonieInfo->valid && !forceCloud && (!tonie_marked || !can_use_cloud))
{
TRACE_INFO("Serve local content from %s\r\n", tonieInfo->contentPath);
connection->response.keepAlive = true;
Expand Down Expand Up @@ -867,6 +883,10 @@ error_t handleCloudContentExt(HttpConnection *connection, const char_t *uri, con
if (tonieInfo->json.source != NULL)
{
TRACE_INFO(" >> Removing source %s for download\r\n", tonieInfo->json.source);
if (forceCloud && tonieInfo->json.source[0] != '\0')
{
previousSource = strdup(tonieInfo->json.source);
}
tonieInfo->json.source[0] = '\0';
tonieInfo->json._updated = true;
freeTonieInfo(tonieInfo);
Expand All @@ -890,25 +910,30 @@ error_t handleCloudContentExt(HttpConnection *connection, const char_t *uri, con
}
}
freeTonieInfo(tonieInfo);
if (previousSource != NULL)
{
restoreSourceIfNotCached(ruid, previousSource, client_ctx->settings);
free(previousSource);
}
return error;
}

error_t handleCloudContent(HttpConnection *connection, const char_t *uri, const char_t *queryString, client_ctx_t *client_ctx, bool_t noPassword)
error_t handleCloudContent(HttpConnection *connection, const char_t *uri, const char_t *queryString, client_ctx_t *client_ctx, bool_t noPassword, bool_t forceCloud)
{
return handleCloudContentExt(connection, uri, queryString, client_ctx, noPassword, 12, V2_CONTENT);
return handleCloudContentExt(connection, uri, queryString, client_ctx, noPassword, 12, V2_CONTENT, forceCloud);
}


error_t handleCloudContentV1(HttpConnection *connection, const char_t *uri, const char_t *queryString, client_ctx_t *client_ctx)
{
return handleCloudContent(connection, uri, queryString, client_ctx, TRUE);
return handleCloudContent(connection, uri, queryString, client_ctx, TRUE, FALSE);
}

error_t handleCloudContentV2(HttpConnection *connection, const char_t *uri, const char_t *queryString, client_ctx_t *client_ctx)
{
if (connection->request.auth.found && connection->request.auth.mode == HTTP_AUTH_MODE_DIGEST)
{
return handleCloudContent(connection, uri, queryString, client_ctx, FALSE);
return handleCloudContent(connection, uri, queryString, client_ctx, FALSE, FALSE);
}
else
{
Expand Down Expand Up @@ -1580,7 +1605,7 @@ error_t handleCloudChapterV3(HttpConnection *connection, const char_t *uri, cons

connection->private.client_ctx.skip_taf_header = true;

return handleCloudContentExt(connection, uri, queryString, client_ctx, false, 12 + 11 + 3, V3_CHAPTER);
return handleCloudContentExt(connection, uri, queryString, client_ctx, false, 12 + 11 + 3, V3_CHAPTER, false);
}

if (client_ctx->settings->cloud.enabled && client_ctx->settings->cloud.enableV3Chapter)
Expand Down
Loading