diff --git a/app/src/main/java/app/gamenative/service/epic/EpicDownloadManager.kt b/app/src/main/java/app/gamenative/service/epic/EpicDownloadManager.kt index 5b992afef5..b98e7b35ab 100644 --- a/app/src/main/java/app/gamenative/service/epic/EpicDownloadManager.kt +++ b/app/src/main/java/app/gamenative/service/epic/EpicDownloadManager.kt @@ -880,6 +880,9 @@ class EpicDownloadManager @Inject constructor( val downloadedChunkIds = newKeySet() val pendingChunks = AtomicInteger(chunkQueue.size) + // Calculate total expected installed size once (sum of all file sizes) + val totalExpectedSize = files.sumOf { it.fileSize } + chunkQueue.forEach { chunkInfo -> chunkUsageCounts[chunkInfo.guidStr] = AtomicInteger( files.sumOf { file -> file.chunkParts.count { chunk -> chunk.guidStr == chunkInfo.guidStr } } @@ -1027,7 +1030,15 @@ class EpicDownloadManager @Inject constructor( val chunksAdded = mutableListOf() - files.forEach { file -> + // Sort files by total chunk usage (lowest first) - files with less-shared chunks complete faster and free cache sooner + val sortedFiles = files.sortedBy { file -> + file.chunkParts.sumOf { chunk -> + chunkUsageCounts[chunk.guidStr]?.get() ?: 0 + } + } + Timber.tag("EPIC").d("Processing ${sortedFiles.size} files sorted by chunk usage (least shared first)") + + sortedFiles.forEach { file -> if (!downloadInfo.isActive()) { Timber.tag("EPIC").w("Download cancelled during file iteration") return@launch @@ -1075,7 +1086,16 @@ class EpicDownloadManager @Inject constructor( return@withContext Result.failure(Exception("Download cancelled")) } - Timber.tag("EPIC").d("Waiting for $currentPendingChunks pending chunks to complete") + // Calculate storage usage stats (only scan cache dir, not entire install dir) + val chunkCacheSize = calculateDirectorySize(chunkCacheDir) + + Timber.tag("EPIC").d( + """Waiting for $currentPendingChunks pending chunks + | Cache: ${chunkCacheSize / 1_000_000}MB + | Game files: ${totalExpectedSize / 1_000_000}MB + | Total disk: ${(chunkCacheSize + totalExpectedSize) / 1_000_000}MB + """.trimMargin() + ) if (currentPendingChunks == lastPendingChunks) { samePendingChunksAttempts++ @@ -1097,8 +1117,8 @@ class EpicDownloadManager @Inject constructor( samePendingChunksAttempts = 0 } - // Wait for 1 second to recheck - delay(1000) + // Wait for 5 seconds to recheck (not too quick due to added stats causing IO) + delay(5000) currentPendingChunks = pendingChunks.get() } @@ -1255,7 +1275,7 @@ class EpicDownloadManager @Inject constructor( } if (isRoot) { - val totalSize = calculateTotalSize(dir) + val totalSize = calculateDirectorySize(dir) val fileCount = countFiles(dir) Timber.tag("Epic").i("=== Summary ===") Timber.tag("Epic").i("Total files: $fileCount") @@ -1276,15 +1296,6 @@ class EpicDownloadManager @Inject constructor( } } - /** - * Calculate total size of a directory recursively - */ - private fun calculateTotalSize(dir: File): Long { - if (!dir.exists()) return 0 - if (dir.isFile) return dir.length() - return dir.listFiles()?.sumOf { calculateTotalSize(it) } ?: 0 - } - /** * Count total number of files in a directory recursively */ @@ -1293,4 +1304,31 @@ class EpicDownloadManager @Inject constructor( if (dir.isFile) return 1 return dir.listFiles()?.sumOf { countFiles(it) } ?: 0 } + + /** + * Total size under [directory]. Symlinks are skipped to avoid double-counting. + */ + private fun calculateDirectorySize(directory: File): Long { + var size = 0L + try { + if (!directory.exists() || !directory.isDirectory) { + return 0L + } + + val files = directory.listFiles() ?: return 0L + for (file in files) { + if (java.nio.file.Files.isSymbolicLink(file.toPath())) { + continue + } + size += if (file.isDirectory) { + calculateDirectorySize(file) + } else { + file.length() + } + } + } catch (e: Exception) { + Timber.tag("Epic").w(e, "Error calculating directory size for ${directory.name}") + } + return size + } } diff --git a/app/src/main/java/app/gamenative/service/gog/GOGDownloadManager.kt b/app/src/main/java/app/gamenative/service/gog/GOGDownloadManager.kt index 1949c0979c..20457fd8e6 100644 --- a/app/src/main/java/app/gamenative/service/gog/GOGDownloadManager.kt +++ b/app/src/main/java/app/gamenative/service/gog/GOGDownloadManager.kt @@ -816,6 +816,9 @@ class GOGDownloadManager @Inject constructor( val chunkAttempts = ConcurrentHashMap() val pendingChunks = AtomicInteger(chunkHashes.size) + // Calculate total expected installed size once (sum of all file sizes) + val totalExpectedSize = files.sumOf { file -> file.chunks.sumOf { it.size } } + chunkHashes.forEach { chunkMd5 -> chunkUsageCounts[chunkMd5] = AtomicInteger( files.sumOf { file -> file.chunks.count { chunk -> chunk.compressedMd5 == chunkMd5 } } @@ -984,7 +987,15 @@ class GOGDownloadManager @Inject constructor( val chunksAdded = mutableListOf() - files.forEach { file -> + // Sort files by total chunk usage (lowest first) - files with less-shared chunks complete faster and free cache sooner + val sortedFiles = files.sortedBy { file -> + file.chunks.sumOf { chunk -> + chunkUsageCounts[chunk.compressedMd5]?.get() ?: 0 + } + } + Timber.tag("GOG").d("Processing ${sortedFiles.size} files sorted by chunk usage (least shared first)") + + sortedFiles.forEach { file -> if (!downloadInfo.isActive()) { Timber.tag("GOG").w("Download cancelled during file iteration") return@launch @@ -1029,7 +1040,16 @@ class GOGDownloadManager @Inject constructor( return@withContext Result.failure(Exception("Download cancelled")) } - Timber.tag("GOG").d("Waiting for $currentPendingChunks pending chunks to complete") + // Calculate storage usage stats (only scan cache dir, not entire install dir) + val chunkCacheSize = calculateDirectorySize(chunkCacheDir) + + Timber.tag("GOG").d( + """Waiting for $currentPendingChunks pending chunks + | Cache: ${chunkCacheSize / 1_000_000}MB + | Game files: ${totalExpectedSize / 1_000_000}MB + | Total disk: ${(chunkCacheSize + totalExpectedSize) / 1_000_000}MB + """.trimMargin() + ) if (currentPendingChunks == lastPendingChunks) { samePendingChunksAttempts++ @@ -1051,8 +1071,8 @@ class GOGDownloadManager @Inject constructor( samePendingChunksAttempts = 0 } - // Wait for 1 second to recheck - delay(1000) + // Wait for 5 seconds to recheck (not too quick due to added stats causing IO) + delay(5000) currentPendingChunks = pendingChunks.get() } @@ -1426,7 +1446,7 @@ class GOGDownloadManager @Inject constructor( ?: return@withContext Result.failure(Exception("Empty response for chunk $chunkMd5")) val md = MessageDigest.getInstance("MD5") - val buffer = ByteArray(256 * 1024) // 256KB + val buffer = ByteArray(64 * 1024) // 64KB - reduced from 256KB to minimize memory pressure during parallel downloads var lastProgressEmitAt = System.currentTimeMillis() if (tempChunkFile.exists()) tempChunkFile.delete() @@ -1513,36 +1533,17 @@ class GOGDownloadManager @Inject constructor( ) } - // Read compressed data - val compressedBytes = chunkFile.readBytes() + val writeOffset = file.chunks.take(chunkIndex).sumOf { it.size } - // Decompress chunk - val decompressedBytes = decompressChunk(compressedBytes, chunk) - if (decompressedBytes.isFailure) { + // Decompress directly to file while calculating MD5 + val decompressResult = decompressChunkToFile(chunkFile, chunk, outputFile, writeOffset) + if (decompressResult.isFailure) { return@withContext Result.failure( - decompressedBytes.exceptionOrNull() + decompressResult.exceptionOrNull() ?: Exception("Failed to decompress chunk ${chunk.compressedMd5}"), ) } - val data = decompressedBytes.getOrThrow() - - // Verify decompressed MD5 - val actualMd5 = calculateMd5(data) - if (actualMd5 != chunk.md5) { - return@withContext Result.failure( - Exception("Decompressed MD5 mismatch for chunk: expected ${chunk.md5}, got $actualMd5"), - ) - } - - val writeOffset = file.chunks.take(chunkIndex).sumOf { it.size } - - // Write decompressed chunk at specific file offset using RandomAccessFile - RandomAccessFile(outputFile.path, "rw").use { randomAccessFile -> - randomAccessFile.seek(writeOffset) - randomAccessFile.write(data) - } - // Verify final file hash if provided if (file.md5 != null) { val fileMd5 = calculateMd5File(outputFile) @@ -1554,6 +1555,9 @@ class GOGDownloadManager @Inject constructor( // Move the log here for files finally assembled Timber.tag("GOG").v("Assembled: ${file.path} (${outputFile.length()} bytes)") } + } else { + // For md5 is null, likely it does not need to decompress, need to show log here + Timber.tag("GOG").v("Assembled: ${file.path} (${outputFile.length()} bytes)") } Result.success(outputFile) @@ -1564,62 +1568,107 @@ class GOGDownloadManager @Inject constructor( } /** - * Decompress a GOG chunk using zlib + * Decompress a GOG chunk directly to file using zlib, streaming to avoid large memory allocations. + * Calculates MD5 while decompressing and writes directly to the target file at the specified offset. * - * GOG chunks are compressed with zlib - * If chunk.compressedSize is null, data is uncompressed - * - * @param compressedBytes Compressed chunk data + * @param chunkFile Compressed chunk file * @param chunk Chunk metadata - * @return Decompressed data + * @param outputFile Target file to write decompressed data + * @param writeOffset Offset in the output file to write at + * @return Result indicating success or failure */ - private fun decompressChunk(compressedBytes: ByteArray, chunk: FileChunk): Result { + internal fun decompressChunkToFile( + chunkFile: File, + chunk: FileChunk, + outputFile: File, + writeOffset: Long, + ): Result { return try { - // If no compressed size specified, data is already uncompressed - if (chunk.compressedSize == null) { - return Result.success(compressedBytes) - } + val md5Digest = MessageDigest.getInstance("MD5") + var totalBytesWritten = 0L + + RandomAccessFile(outputFile.path, "rw").use { raf -> + raf.seek(writeOffset) + + // If no compressed size specified, data is already uncompressed + if (chunk.compressedSize == null) { + chunkFile.inputStream().use { input -> + val buffer = ByteArray(8192) + var bytesRead: Int + while (input.read(buffer).also { bytesRead = it } != -1) { + md5Digest.update(buffer, 0, bytesRead) + raf.write(buffer, 0, bytesRead) + totalBytesWritten += bytesRead + } + } + } else { + // Decompress using zlib and stream directly to file + val inflater = Inflater() + try { + chunkFile.inputStream().buffered().use { input -> + val inputBuffer = ByteArray(8192) + val outputBuffer = ByteArray(8192) + var inputBytesRead: Int + + while (input.read(inputBuffer).also { inputBytesRead = it } != -1) { + inflater.setInput(inputBuffer, 0, inputBytesRead) + + while (!inflater.needsInput() && !inflater.finished()) { + val count = inflater.inflate(outputBuffer) + if (count > 0) { + md5Digest.update(outputBuffer, 0, count) + raf.write(outputBuffer, 0, count) + totalBytesWritten += count + } else { + if (inflater.needsDictionary()) { + throw java.io.IOException( + "Zlib data requires a preset dictionary which is not supported" + ) + } + break + } + } + } - // Decompress using zlib - val inflater = Inflater() - try { - inflater.setInput(compressedBytes) - val outputStream = ByteArrayOutputStream(chunk.size.toInt()) - val buffer = ByteArray(8192) - - while (!inflater.finished()) { - val count = inflater.inflate(buffer) - if (count > 0) { - outputStream.write(buffer, 0, count) - } else { - // No bytes produced - check if we need more input or a dictionary - if (inflater.needsInput()) { - throw java.io.IOException( - "Incomplete zlib data: decompression requires more input but none available" - ) - } else if (inflater.needsDictionary()) { - throw java.io.IOException( - "Zlib data requires a preset dictionary which is not supported" - ) + // Finish any remaining data + while (!inflater.finished()) { + val count = inflater.inflate(outputBuffer) + if (count > 0) { + md5Digest.update(outputBuffer, 0, count) + raf.write(outputBuffer, 0, count) + totalBytesWritten += count + } else { + if (inflater.needsInput()) { + throw java.io.IOException( + "Incomplete zlib data: decompression requires more input but none available" + ) + } + break + } + } } - // If neither condition is true, inflater is still processing internally - // Continue loop, but this should be rare + } finally { + inflater.end() } } + } - val decompressed = outputStream.toByteArray() - - // Verify size matches expected - if (decompressed.size.toLong() != chunk.size) { - return Result.failure( - Exception("Decompressed size mismatch: expected ${chunk.size}, got ${decompressed.size}"), - ) - } + // Verify size matches expected + if (totalBytesWritten != chunk.size) { + return Result.failure( + Exception("Decompressed size mismatch: expected ${chunk.size}, got $totalBytesWritten"), + ) + } - Result.success(decompressed) - } finally { - inflater.end() + // Verify decompressed MD5 + val actualMd5 = md5Digest.digest().joinToString("") { "%02x".format(it) } + if (actualMd5 != chunk.md5) { + return Result.failure( + Exception("Decompressed MD5 mismatch for chunk: expected ${chunk.md5}, got $actualMd5"), + ) } + + Result.success(Unit) } catch (e: Exception) { Timber.tag("GOG").e(e, "Failed to decompress chunk ${chunk.compressedMd5}") Result.failure(e) diff --git a/app/src/test/java/app/gamenative/service/gog/GOGDownloadManagerTest.kt b/app/src/test/java/app/gamenative/service/gog/GOGDownloadManagerTest.kt index 4edb95f57a..4511864350 100644 --- a/app/src/test/java/app/gamenative/service/gog/GOGDownloadManagerTest.kt +++ b/app/src/test/java/app/gamenative/service/gog/GOGDownloadManagerTest.kt @@ -314,4 +314,255 @@ class GOGDownloadManagerTest { productId = null, ) } + + // ===== Chunk Decompression Tests ===== + + @Test + fun decompressChunkToFile_compressedChunk_writesCorrectly() { + val tempDir = Files.createTempDirectory("gog-chunk-test").toFile() + + // Arrange: Create compressed chunk + val originalData = "Hello World! This is test data for compression.".toByteArray() + val compressedData = compressTestData(originalData) + val chunkFile = File(tempDir, "chunk.bin") + chunkFile.writeBytes(compressedData) + + val outputFile = File(tempDir, "output.bin") + val chunk = app.gamenative.service.gog.api.FileChunk( + compressedMd5 = "test-md5", + md5 = calculateTestMd5(originalData), + size = originalData.size.toLong(), + compressedSize = compressedData.size.toLong() + ) + + // Act + val result = testDecompressChunkToFile(chunkFile, chunk, outputFile, 0) + + // Assert + assertTrue("Decompression should succeed", result.isSuccess) + assertTrue("Output file should exist", outputFile.exists()) + assertEquals("Output size should match", originalData.size.toLong(), outputFile.length()) + assertEquals("Content should match", originalData.decodeToString(), outputFile.readBytes().decodeToString()) + + tempDir.deleteRecursively() + } + + @Test + fun decompressChunkToFile_uncompressedChunk_copiesDirectly() { + val tempDir = Files.createTempDirectory("gog-chunk-test").toFile() + + // Arrange: Create uncompressed chunk (compressedSize = null) + val originalData = "Uncompressed data".toByteArray() + val chunkFile = File(tempDir, "chunk.bin") + chunkFile.writeBytes(originalData) + + val outputFile = File(tempDir, "output.bin") + val chunk = app.gamenative.service.gog.api.FileChunk( + compressedMd5 = "test-md5", + md5 = calculateTestMd5(originalData), + size = originalData.size.toLong(), + compressedSize = null // Indicates uncompressed + ) + + // Act + val result = testDecompressChunkToFile(chunkFile, chunk, outputFile, 0) + + // Assert + assertTrue("Copy should succeed", result.isSuccess) + assertTrue("Output file should exist", outputFile.exists()) + assertEquals("Output size should match", originalData.size.toLong(), outputFile.length()) + assertEquals("Content should match", originalData.decodeToString(), outputFile.readBytes().decodeToString()) + + tempDir.deleteRecursively() + } + + @Test + fun decompressChunkToFile_nonZeroOffset_writesAtCorrectPosition() { + val tempDir = Files.createTempDirectory("gog-chunk-test").toFile() + + // Arrange: Pre-create file with some data + val existingData = "AAAA".toByteArray() + val newData = "BBBB".toByteArray() + val compressedNewData = compressTestData(newData) + + val outputFile = File(tempDir, "output.bin") + java.io.RandomAccessFile(outputFile.path, "rw").use { + it.setLength(8) // Pre-allocate 8 bytes + it.write(existingData) // Write first 4 bytes + } + + val chunkFile = File(tempDir, "chunk.bin") + chunkFile.writeBytes(compressedNewData) + + val chunk = app.gamenative.service.gog.api.FileChunk( + compressedMd5 = "test-md5", + md5 = calculateTestMd5(newData), + size = newData.size.toLong(), + compressedSize = compressedNewData.size.toLong() + ) + + // Act: Write at offset 4 + val result = testDecompressChunkToFile(chunkFile, chunk, outputFile, 4) + + // Assert + assertTrue("Decompression should succeed", result.isSuccess) + val finalContent = outputFile.readBytes() + assertEquals("First 4 bytes should be unchanged", "AAAA", finalContent.sliceArray(0..3).decodeToString()) + assertEquals("Next 4 bytes should be new data", "BBBB", finalContent.sliceArray(4..7).decodeToString()) + + tempDir.deleteRecursively() + } + + @Test + fun decompressChunkToFile_sizeMismatch_fails() { + val tempDir = Files.createTempDirectory("gog-chunk-test").toFile() + + // Arrange: Compressed data with wrong expected size + val originalData = "Short".toByteArray() + val compressedData = compressTestData(originalData) + val chunkFile = File(tempDir, "chunk.bin") + chunkFile.writeBytes(compressedData) + + val outputFile = File(tempDir, "output.bin") + val actualMd5 = calculateTestMd5(originalData) + val chunk = app.gamenative.service.gog.api.FileChunk( + compressedMd5 = "test-md5", + md5 = actualMd5, // Correct MD5 so size check is reached + size = 999L, // Wrong expected size + compressedSize = compressedData.size.toLong() + ) + + // Act + val result = testDecompressChunkToFile(chunkFile, chunk, outputFile, 0) + + // Assert + assertTrue("Should fail on size mismatch", result.isFailure) + val errorMsg = result.exceptionOrNull()?.message ?: "" + assertTrue("Error should mention size mismatch, got: $errorMsg", errorMsg.contains("size", ignoreCase = true)) + + tempDir.deleteRecursively() + } + + @Test + fun decompressChunkToFile_md5Mismatch_fails() { + val tempDir = Files.createTempDirectory("gog-chunk-test").toFile() + + // Arrange: Valid compressed data but wrong MD5 + val originalData = "Test data".toByteArray() + val compressedData = compressTestData(originalData) + val chunkFile = File(tempDir, "chunk.bin") + chunkFile.writeBytes(compressedData) + + val outputFile = File(tempDir, "output.bin") + val chunk = app.gamenative.service.gog.api.FileChunk( + compressedMd5 = "test-md5", + md5 = "wrong-md5-hash", // Wrong MD5 + size = originalData.size.toLong(), + compressedSize = compressedData.size.toLong() + ) + + // Act + val result = testDecompressChunkToFile(chunkFile, chunk, outputFile, 0) + + // Assert + assertTrue("Should fail on MD5 mismatch", result.isFailure) + assertTrue("Error should mention MD5", result.exceptionOrNull()?.message?.contains("MD5", ignoreCase = true) == true) + + tempDir.deleteRecursively() + } + + @Test + fun decompressChunkToFile_emptyMd5_failsValidation() { + val tempDir = Files.createTempDirectory("gog-chunk-test").toFile() + + // Arrange: Compressed chunk with empty MD5 string + val originalData = "Test data".toByteArray() + val compressedData = compressTestData(originalData) + val chunkFile = File(tempDir, "chunk.bin") + chunkFile.writeBytes(compressedData) + + val outputFile = File(tempDir, "output.bin") + val chunk = app.gamenative.service.gog.api.FileChunk( + compressedMd5 = "test-md5", + md5 = "", // Empty MD5 still validates (won't match actual hash) + size = originalData.size.toLong(), + compressedSize = compressedData.size.toLong() + ) + + // Act + val result = testDecompressChunkToFile(chunkFile, chunk, outputFile, 0) + + // Assert + assertTrue("Should fail because empty MD5 doesn't match actual hash", result.isFailure) + assertTrue("Error should mention MD5", result.exceptionOrNull()?.message?.contains("MD5", ignoreCase = true) == true) + + tempDir.deleteRecursively() + } + + @Test + fun decompressChunkToFile_corruptedZlibData_fails() { + val tempDir = Files.createTempDirectory("gog-chunk-test").toFile() + + // Arrange: Invalid zlib data + val corruptedData = byteArrayOf(0x78.toByte(), 0x9C.toByte(), 0xFF.toByte(), 0xFF.toByte(), 0xFF.toByte()) + val chunkFile = File(tempDir, "chunk.bin") + chunkFile.writeBytes(corruptedData) + + val outputFile = File(tempDir, "output.bin") + val chunk = app.gamenative.service.gog.api.FileChunk( + compressedMd5 = "test-md5", + md5 = "any-md5", + size = 100L, + compressedSize = corruptedData.size.toLong() + ) + + // Act + val result = testDecompressChunkToFile(chunkFile, chunk, outputFile, 0) + + // Assert + assertTrue("Should fail on corrupted data", result.isFailure) + + tempDir.deleteRecursively() + } + + // Helper functions for chunk decompression tests + + private fun compressTestData(data: ByteArray): ByteArray { + val deflater = java.util.zip.Deflater() + try { + deflater.setInput(data) + deflater.finish() + + val buffer = ByteArray(data.size * 2 + 100) + var totalCompressed = 0 + + while (!deflater.finished()) { + val count = deflater.deflate(buffer, totalCompressed, buffer.size - totalCompressed) + totalCompressed += count + if (count == 0 && !deflater.finished()) { + throw IllegalStateException("Deflater stuck") + } + } + + return buffer.copyOf(totalCompressed) + } finally { + deflater.end() + } + } + + private fun calculateTestMd5(data: ByteArray): String { + val digest = java.security.MessageDigest.getInstance("MD5") + val hash = digest.digest(data) + return hash.joinToString("") { "%02x".format(it) } + } + + private fun testDecompressChunkToFile( + chunkFile: File, + chunk: app.gamenative.service.gog.api.FileChunk, + outputFile: File, + writeOffset: Long + ): Result { + // Now that decompressChunkToFile is internal, we can call it directly + return manager.decompressChunkToFile(chunkFile, chunk, outputFile, writeOffset) + } }