Skip to content

fix: Optimize GOG and EPIC download #1649

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
utkarshdalal merged 5 commits into from
Jul 7, 2026
+426 −88
Merged

fix: Optimize GOG and EPIC download #1649

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
cc61515
Optimize GOG chunk decompression and download resource usage
joshuatam Jul 1, 2026
b9b16a2
Optimize Epic download processing and monitoring
joshuatam Jul 1, 2026
a639209
only scan cache dir to show the disk usage during download
joshuatam Jul 1, 2026
25c92c9
replace epic calculateTotalSize with calculateDirectorySize
joshuatam Jul 1, 2026
69f14d0
add tests for GOG decompressChunkToFile
joshuatam Jul 1, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
66 changes: 52 additions & 14 deletions app/src/main/java/app/gamenative/service/epic/EpicDownloadManager.kt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -880,6 +880,9 @@ class EpicDownloadManager @Inject constructor(
val downloadedChunkIds = newKeySet<String>()
val pendingChunks = AtomicInteger(chunkQueue.size)

// Calculate total expected installed size once (sum of all file sizes)
val totalExpectedSize = files.sumOf { it.fileSize }

chunkQueue.forEach { chunkInfo ->
chunkUsageCounts[chunkInfo.guidStr] = AtomicInteger(
files.sumOf { file -> file.chunkParts.count { chunk -> chunk.guidStr == chunkInfo.guidStr } }
Expand Down Expand Up @@ -1027,7 +1030,15 @@ class EpicDownloadManager @Inject constructor(

val chunksAdded = mutableListOf<String>()

files.forEach { file ->
// Sort files by total chunk usage (lowest first) - files with less-shared chunks complete faster and free cache sooner
val sortedFiles = files.sortedBy { file ->
file.chunkParts.sumOf { chunk ->
chunkUsageCounts[chunk.guidStr]?.get() ?: 0
}
}
Timber.tag("EPIC").d("Processing ${sortedFiles.size} files sorted by chunk usage (least shared first)")

sortedFiles.forEach { file ->
if (!downloadInfo.isActive()) {
Timber.tag("EPIC").w("Download cancelled during file iteration")
return@launch
Expand Down Expand Up @@ -1075,7 +1086,16 @@ class EpicDownloadManager @Inject constructor(
return@withContext Result.failure(Exception("Download cancelled"))
}

Timber.tag("EPIC").d("Waiting for $currentPendingChunks pending chunks to complete")
// Calculate storage usage stats (only scan cache dir, not entire install dir)
val chunkCacheSize = calculateDirectorySize(chunkCacheDir)

Timber.tag("EPIC").d(
"""Waiting for $currentPendingChunks pending chunks
| Cache: ${chunkCacheSize / 1_000_000}MB
| Game files: ${totalExpectedSize / 1_000_000}MB
| Total disk: ${(chunkCacheSize + totalExpectedSize) / 1_000_000}MB
""".trimMargin()
)

if (currentPendingChunks == lastPendingChunks) {
samePendingChunksAttempts++
Expand All @@ -1097,8 +1117,8 @@ class EpicDownloadManager @Inject constructor(
samePendingChunksAttempts = 0
}

// Wait for 1 second to recheck
delay(1000)
// Wait for 5 seconds to recheck (not too quick due to added stats causing IO)
delay(5000)

currentPendingChunks = pendingChunks.get()
}
Expand Down Expand Up @@ -1255,7 +1275,7 @@ class EpicDownloadManager @Inject constructor(
}

if (isRoot) {
val totalSize = calculateTotalSize(dir)
val totalSize = calculateDirectorySize(dir)
val fileCount = countFiles(dir)
Timber.tag("Epic").i("=== Summary ===")
Timber.tag("Epic").i("Total files: $fileCount")
Expand All @@ -1276,15 +1296,6 @@ class EpicDownloadManager @Inject constructor(
}
}

/**
* Calculate total size of a directory recursively
*/
private fun calculateTotalSize(dir: File): Long {
if (!dir.exists()) return 0
if (dir.isFile) return dir.length()
return dir.listFiles()?.sumOf { calculateTotalSize(it) } ?: 0
}

/**
* Count total number of files in a directory recursively
*/
Expand All @@ -1293,4 +1304,31 @@ class EpicDownloadManager @Inject constructor(
if (dir.isFile) return 1
return dir.listFiles()?.sumOf { countFiles(it) } ?: 0
}

/**
* Total size under [directory]. Symlinks are skipped to avoid double-counting.
*/
private fun calculateDirectorySize(directory: File): Long {
Comment thread
cubic-dev-ai[bot] marked this conversation as resolved.
var size = 0L
try {
if (!directory.exists() || !directory.isDirectory) {
return 0L
}

val files = directory.listFiles() ?: return 0L
for (file in files) {
if (java.nio.file.Files.isSymbolicLink(file.toPath())) {
continue
}
size += if (file.isDirectory) {
calculateDirectorySize(file)
} else {
file.length()
}
}
} catch (e: Exception) {
Timber.tag("Epic").w(e, "Error calculating directory size for ${directory.name}")
}
return size
}
}
197 changes: 123 additions & 74 deletions app/src/main/java/app/gamenative/service/gog/GOGDownloadManager.kt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -816,6 +816,9 @@ class GOGDownloadManager @Inject constructor(
val chunkAttempts = ConcurrentHashMap<String, AtomicInteger>()
val pendingChunks = AtomicInteger(chunkHashes.size)

// Calculate total expected installed size once (sum of all file sizes)
val totalExpectedSize = files.sumOf { file -> file.chunks.sumOf { it.size } }

chunkHashes.forEach { chunkMd5 ->
chunkUsageCounts[chunkMd5] = AtomicInteger(
files.sumOf { file -> file.chunks.count { chunk -> chunk.compressedMd5 == chunkMd5 } }
Expand Down Expand Up @@ -984,7 +987,15 @@ class GOGDownloadManager @Inject constructor(

val chunksAdded = mutableListOf<String>()

files.forEach { file ->
// Sort files by total chunk usage (lowest first) - files with less-shared chunks complete faster and free cache sooner
val sortedFiles = files.sortedBy { file ->
file.chunks.sumOf { chunk ->
chunkUsageCounts[chunk.compressedMd5]?.get() ?: 0
}
}
Timber.tag("GOG").d("Processing ${sortedFiles.size} files sorted by chunk usage (least shared first)")

sortedFiles.forEach { file ->
if (!downloadInfo.isActive()) {
Timber.tag("GOG").w("Download cancelled during file iteration")
return@launch
Expand Down Expand Up @@ -1029,7 +1040,16 @@ class GOGDownloadManager @Inject constructor(
return@withContext Result.failure(Exception("Download cancelled"))
}

Timber.tag("GOG").d("Waiting for $currentPendingChunks pending chunks to complete")
// Calculate storage usage stats (only scan cache dir, not entire install dir)
val chunkCacheSize = calculateDirectorySize(chunkCacheDir)

Timber.tag("GOG").d(
"""Waiting for $currentPendingChunks pending chunks
| Cache: ${chunkCacheSize / 1_000_000}MB
| Game files: ${totalExpectedSize / 1_000_000}MB
| Total disk: ${(chunkCacheSize + totalExpectedSize) / 1_000_000}MB
""".trimMargin()
)

if (currentPendingChunks == lastPendingChunks) {
samePendingChunksAttempts++
Expand All @@ -1051,8 +1071,8 @@ class GOGDownloadManager @Inject constructor(
samePendingChunksAttempts = 0
}

// Wait for 1 second to recheck
delay(1000)
// Wait for 5 seconds to recheck (not too quick due to added stats causing IO)
delay(5000)
Comment on lines +1074 to +1075

@coderabbitai coderabbitai Bot Jul 2, 2026

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

🩺 Stability & Availability | 🟡 Minor | ⚡ Quick win

Recheck delay increase also slows stuck-chunk recovery ~5×.

The samePendingChunksAttempts >= 10 re-emit threshold (Line 1061) is unchanged, so bumping the loop delay from 1,000ms to 5,000ms pushes the stuck-chunk recovery point from ~10s to ~50s. A genuinely stalled download will now appear frozen for nearly a minute before missing chunks are re-emitted. Consider decoupling the stall threshold from the log/recheck cadence (e.g., recheck at 5s but re-emit after fewer stalled seconds) so recovery stays responsive.

🤖 Prompt for AI Agents 
Verify each finding against current code. Fix only still-valid issues, skip the
rest with a brief reason, keep changes minimal, and validate.

In `@app/src/main/java/app/gamenative/service/gog/GOGDownloadManager.kt` around
lines 1074 - 1075, The recheck delay in GOGDownloadManager’s stalled-chunk loop
was increased, but the stuck-chunk recovery threshold still depends on the same
attempt count, making recovery much slower. Update the logic around the
`samePendingChunksAttempts >= 10` check in `GOGDownloadManager` so the stall
detection/re-emit timing is decoupled from the `delay(5000)` cadence, keeping
the log/recheck interval at 5s if needed but triggering chunk re-emission after
fewer stalled seconds.


currentPendingChunks = pendingChunks.get()
}
Expand Down Expand Up @@ -1426,7 +1446,7 @@ class GOGDownloadManager @Inject constructor(
?: return@withContext Result.failure(Exception("Empty response for chunk $chunkMd5"))

val md = MessageDigest.getInstance("MD5")
val buffer = ByteArray(256 * 1024) // 256KB
val buffer = ByteArray(64 * 1024) // 64KB - reduced from 256KB to minimize memory pressure during parallel downloads
var lastProgressEmitAt = System.currentTimeMillis()

if (tempChunkFile.exists()) tempChunkFile.delete()
Expand Down Expand Up @@ -1513,36 +1533,17 @@ class GOGDownloadManager @Inject constructor(
)
}

// Read compressed data
val compressedBytes = chunkFile.readBytes()
val writeOffset = file.chunks.take(chunkIndex).sumOf { it.size }

// Decompress chunk
val decompressedBytes = decompressChunk(compressedBytes, chunk)
if (decompressedBytes.isFailure) {
// Decompress directly to file while calculating MD5
val decompressResult = decompressChunkToFile(chunkFile, chunk, outputFile, writeOffset)
if (decompressResult.isFailure) {
return@withContext Result.failure(
decompressedBytes.exceptionOrNull()
decompressResult.exceptionOrNull()
?: Exception("Failed to decompress chunk ${chunk.compressedMd5}"),
)
}

val data = decompressedBytes.getOrThrow()

// Verify decompressed MD5
val actualMd5 = calculateMd5(data)
if (actualMd5 != chunk.md5) {
return@withContext Result.failure(
Exception("Decompressed MD5 mismatch for chunk: expected ${chunk.md5}, got $actualMd5"),
)
}

val writeOffset = file.chunks.take(chunkIndex).sumOf { it.size }

// Write decompressed chunk at specific file offset using RandomAccessFile
RandomAccessFile(outputFile.path, "rw").use { randomAccessFile ->
randomAccessFile.seek(writeOffset)
randomAccessFile.write(data)
}

// Verify final file hash if provided
if (file.md5 != null) {
val fileMd5 = calculateMd5File(outputFile)
Expand All @@ -1554,6 +1555,9 @@ class GOGDownloadManager @Inject constructor(
// Move the log here for files finally assembled
Timber.tag("GOG").v("Assembled: ${file.path} (${outputFile.length()} bytes)")
}
} else {
// For md5 is null, likely it does not need to decompress, need to show log here
Timber.tag("GOG").v("Assembled: ${file.path} (${outputFile.length()} bytes)")
}

Result.success(outputFile)
Expand All @@ -1564,62 +1568,107 @@ class GOGDownloadManager @Inject constructor(
}

/**
* Decompress a GOG chunk using zlib
* Decompress a GOG chunk directly to file using zlib, streaming to avoid large memory allocations.
* Calculates MD5 while decompressing and writes directly to the target file at the specified offset.
*
* GOG chunks are compressed with zlib
* If chunk.compressedSize is null, data is uncompressed
*
* @param compressedBytes Compressed chunk data
* @param chunkFile Compressed chunk file
* @param chunk Chunk metadata
* @return Decompressed data
* @param outputFile Target file to write decompressed data
* @param writeOffset Offset in the output file to write at
* @return Result indicating success or failure
*/
private fun decompressChunk(compressedBytes: ByteArray, chunk: FileChunk): Result<ByteArray> {
internal fun decompressChunkToFile(
chunkFile: File,
chunk: FileChunk,
outputFile: File,
writeOffset: Long,
): Result<Unit> {
return try {
// If no compressed size specified, data is already uncompressed
if (chunk.compressedSize == null) {
return Result.success(compressedBytes)
}
val md5Digest = MessageDigest.getInstance("MD5")
var totalBytesWritten = 0L

RandomAccessFile(outputFile.path, "rw").use { raf ->
raf.seek(writeOffset)

// If no compressed size specified, data is already uncompressed
if (chunk.compressedSize == null) {
chunkFile.inputStream().use { input ->
val buffer = ByteArray(8192)
var bytesRead: Int
while (input.read(buffer).also { bytesRead = it } != -1) {
md5Digest.update(buffer, 0, bytesRead)
raf.write(buffer, 0, bytesRead)
totalBytesWritten += bytesRead
}
}
} else {
// Decompress using zlib and stream directly to file
val inflater = Inflater()
try {
chunkFile.inputStream().buffered().use { input ->
val inputBuffer = ByteArray(8192)
val outputBuffer = ByteArray(8192)
var inputBytesRead: Int

while (input.read(inputBuffer).also { inputBytesRead = it } != -1) {
inflater.setInput(inputBuffer, 0, inputBytesRead)

while (!inflater.needsInput() && !inflater.finished()) {
val count = inflater.inflate(outputBuffer)
if (count > 0) {
md5Digest.update(outputBuffer, 0, count)
raf.write(outputBuffer, 0, count)
totalBytesWritten += count
} else {
if (inflater.needsDictionary()) {
throw java.io.IOException(
"Zlib data requires a preset dictionary which is not supported"
)
}
break
}
}
}

// Decompress using zlib
val inflater = Inflater()
try {
inflater.setInput(compressedBytes)
val outputStream = ByteArrayOutputStream(chunk.size.toInt())
val buffer = ByteArray(8192)

while (!inflater.finished()) {
val count = inflater.inflate(buffer)
if (count > 0) {
outputStream.write(buffer, 0, count)
} else {
// No bytes produced - check if we need more input or a dictionary
if (inflater.needsInput()) {
throw java.io.IOException(
"Incomplete zlib data: decompression requires more input but none available"
)
} else if (inflater.needsDictionary()) {
throw java.io.IOException(
"Zlib data requires a preset dictionary which is not supported"
)
// Finish any remaining data
while (!inflater.finished()) {
val count = inflater.inflate(outputBuffer)
if (count > 0) {
md5Digest.update(outputBuffer, 0, count)
raf.write(outputBuffer, 0, count)
totalBytesWritten += count
} else {
if (inflater.needsInput()) {
throw java.io.IOException(
"Incomplete zlib data: decompression requires more input but none available"
)
}
break
}
}
}
// If neither condition is true, inflater is still processing internally
// Continue loop, but this should be rare
} finally {
inflater.end()
}
}
}

val decompressed = outputStream.toByteArray()

// Verify size matches expected
if (decompressed.size.toLong() != chunk.size) {
return Result.failure(
Exception("Decompressed size mismatch: expected ${chunk.size}, got ${decompressed.size}"),
)
}
// Verify size matches expected
if (totalBytesWritten != chunk.size) {
return Result.failure(
Exception("Decompressed size mismatch: expected ${chunk.size}, got $totalBytesWritten"),
)
}

Result.success(decompressed)
} finally {
inflater.end()
// Verify decompressed MD5
val actualMd5 = md5Digest.digest().joinToString("") { "%02x".format(it) }
if (actualMd5 != chunk.md5) {
return Result.failure(
Exception("Decompressed MD5 mismatch for chunk: expected ${chunk.md5}, got $actualMd5"),
)
}

Result.success(Unit)
} catch (e: Exception) {
Timber.tag("GOG").e(e, "Failed to decompress chunk ${chunk.compressedMd5}")
Result.failure(e)
Expand Down
Loading
Loading