Skip to content

Commit 7f52592

Browse files
committed
perf(LocalFeedRepository): speed up local feed extraction
Speeds up the local feed extraction, by fetching more feeds concurrently and only applying a large timeout after fetching the full channels. This is based on the idea that YouTube's RSS feeds (extracted by the generic `FeedInfo`) have no/much less rate-limiting, likely due to the nature of RSS readers. Ref: TeamNewPipe/NewPipe#11817 feat(LocalFeedRepository): remove RSS delay
1 parent 1ff5008 commit 7f52592

1 file changed

Lines changed: 23 additions & 13 deletions

File tree

app/src/main/java/com/github/libretube/repo/LocalFeedRepository.kt

Lines changed: 23 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -81,31 +81,32 @@ class LocalFeedRepository : FeedRepository {
8181
if (channelIds.isEmpty()) return
8282

8383
val totalExtractionCount = AtomicInteger()
84-
val chunkedExtractionCount = AtomicInteger()
84+
val channelExtractionCount = AtomicInteger()
8585
withContext(Dispatchers.Main) {
8686
onProgressUpdate(FeedProgress(0, channelIds.size))
8787
}
8888

8989
for (channelIdChunk in channelIds.chunked(CHUNK_SIZE)) {
90-
// add a delay after each BATCH_SIZE amount of visited channels
91-
val count = chunkedExtractionCount.get();
90+
val count = channelExtractionCount.get();
9291
if (count >= BATCH_SIZE) {
93-
delay(BATCH_DELAY.random())
94-
chunkedExtractionCount.set(0)
92+
// add a delay after each BATCH_SIZE amount of fully-fetched channels
93+
delay(CHANNEL_BATCH_DELAY.random())
94+
channelExtractionCount.set(0)
9595
}
9696

9797
val collectedFeedItems = channelIdChunk.parallelMap { channelId ->
9898
try {
99-
getRelatedStreams(channelId, minimumDateMillis)
99+
getRelatedStreams(channelId, minimumDateMillis).also {
100+
if (it.isNotEmpty())
101+
// increase counter if we had to fully fetch the channel
102+
channelExtractionCount.incrementAndGet()
103+
}
100104
} catch (e: Exception) {
101105
Log.e(channelId, e.stackTraceToString())
102106
null
103107
} finally {
104-
chunkedExtractionCount.incrementAndGet()
105-
val currentProgress = totalExtractionCount.incrementAndGet()
106-
107108
withContext(Dispatchers.Main) {
108-
onProgressUpdate(FeedProgress(currentProgress, channelIds.size))
109+
onProgressUpdate(FeedProgress(totalExtractionCount.incrementAndGet(), channelIds.size))
109110
}
110111
}
111112
}.filterNotNull().flatten().map(StreamItem::toFeedItem)
@@ -159,17 +160,26 @@ class LocalFeedRepository : FeedRepository {
159160
}
160161

161162
companion object {
162-
private const val CHUNK_SIZE = 2
163+
/**
164+
* Amount of feeds that are fetched concurrently.
165+
*
166+
* Should ideally be a factor of `BATCH_SIZE` to be correctly applied.
167+
*/
168+
private const val CHUNK_SIZE = 5
163169

164170
/**
165171
* Maximum amount of feeds that should be fetched together, before a delay should be applied.
166172
*/
167173
private const val BATCH_SIZE = 50
168174

169175
/**
170-
* Millisecond delay between two consecutive batches to avoid throttling.
176+
* Millisecond delay after fetching `BATCH_SIZE` channels to avoid throttling.
177+
*
178+
* A channel is only counted as fetched when it had a recent upload, requiring to fetch
179+
* the channelInfo via Innertube.
171180
*/
172-
private val BATCH_DELAY = (500L..1500L)
181+
private val CHANNEL_BATCH_DELAY = (500L..1500L)
182+
173183
private const val MAX_FEED_AGE_DAYS = 30L // 30 days
174184
}
175185
}

0 commit comments

Comments
 (0)