Skip to content

Commit 6010c4e

Browse files
StypoxAudricV
Stypox
authored and
AudricV
committed
Connect poToken generation to extractor
1 parent 690b341 commit 6010c4e

5 files changed

Lines changed: 125 additions & 59 deletions

File tree

‎app/src/main/java/org/schabi/newpipe/App.java‎

Lines changed: 3 additions & 31 deletions
Original file line numberDiff line numberDiff line change
@@ -3,15 +3,7 @@
33
import android.app.Application;
44
import android.content.Context;
55
import android.content.SharedPreferences;
6-
import android.os.Build;
7-
import android.os.UserManager;
86
import android.util.Log;
9-
import android.view.View;
10-
import android.webkit.JavascriptInterface;
11-
import android.webkit.WebSettings;
12-
import android.webkit.WebView;
13-
import android.webkit.WebViewClient;
14-
import android.widget.RelativeLayout;
157

168
import androidx.annotation.NonNull;
179
import androidx.core.app.NotificationChannelCompat;
@@ -25,8 +17,7 @@
2517
import org.schabi.newpipe.error.ReCaptchaActivity;
2618
import org.schabi.newpipe.extractor.NewPipe;
2719
import org.schabi.newpipe.extractor.downloader.Downloader;
28-
import org.schabi.newpipe.extractor.services.youtube.PoTokenResult;
29-
import org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper;
20+
import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeStreamExtractor;
3021
import org.schabi.newpipe.ktx.ExceptionUtils;
3122
import org.schabi.newpipe.settings.NewPipeSettings;
3223
import org.schabi.newpipe.util.BridgeStateSaverInitializer;
@@ -36,24 +27,20 @@
3627
import org.schabi.newpipe.util.image.ImageStrategy;
3728
import org.schabi.newpipe.util.image.PicassoHelper;
3829
import org.schabi.newpipe.util.image.PreferredImageQuality;
39-
import org.schabi.newpipe.util.potoken.PoTokenWebView;
30+
import org.schabi.newpipe.util.potoken.PoTokenProviderImpl;
4031

4132
import java.io.IOException;
4233
import java.io.InterruptedIOException;
4334
import java.net.SocketException;
4435
import java.util.List;
4536
import java.util.Objects;
4637

47-
import io.reactivex.rxjava3.android.schedulers.AndroidSchedulers;
48-
import io.reactivex.rxjava3.core.Single;
49-
import io.reactivex.rxjava3.disposables.CompositeDisposable;
5038
import io.reactivex.rxjava3.exceptions.CompositeException;
5139
import io.reactivex.rxjava3.exceptions.MissingBackpressureException;
5240
import io.reactivex.rxjava3.exceptions.OnErrorNotImplementedException;
5341
import io.reactivex.rxjava3.exceptions.UndeliverableException;
5442
import io.reactivex.rxjava3.functions.Consumer;
5543
import io.reactivex.rxjava3.plugins.RxJavaPlugins;
56-
import kotlin.Pair;
5744

5845
/*
5946
* Copyright (C) Hans-Christoph Steiner 2016 <hans@eds.org>
@@ -134,22 +121,7 @@ public void onCreate() {
134121

135122
configureRxJavaErrorHandler();
136123

137-
CompositeDisposable disposable = new CompositeDisposable();
138-
disposable.add(PoTokenWebView.Companion.newPoTokenGenerator(this)
139-
.observeOn(AndroidSchedulers.mainThread())
140-
.subscribeOn(AndroidSchedulers.mainThread())
141-
.flatMap(poTokenGenerator -> Single.zip(
142-
poTokenGenerator.generatePoToken(YoutubeParsingHelper
143-
.randomVisitorData(NewPipe.getPreferredContentCountry())),
144-
poTokenGenerator.generatePoToken("i_SsnRdgitA"),
145-
Pair::new
146-
))
147-
.subscribe(
148-
pots -> Log.e(TAG, "success! " + pots.getSecond().poToken +
149-
",web.gvs+" + pots.getFirst().poToken +
150-
";visitor_data=" + pots.getFirst().visitorData),
151-
error -> Log.e(TAG, "error", error)
152-
));
124+
YoutubeStreamExtractor.setPoTokenProvider(PoTokenProviderImpl.INSTANCE);
153125
}
154126

155127
@Override

‎app/src/main/java/org/schabi/newpipe/util/DeviceUtils.java‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,7 @@
1717
import android.view.KeyEvent;
1818
import android.view.WindowInsets;
1919
import android.view.WindowManager;
20+
import android.webkit.CookieManager;
2021

2122
import androidx.annotation.Dimension;
2223
import androidx.annotation.NonNull;
@@ -335,4 +336,17 @@ public static boolean shouldSupportMediaTunneling() {
335336
&& !TX_50JXW834
336337
&& !HMB9213NW;
337338
}
339+
340+
/**
341+
* @return whether the device has support for WebView, see
342+
* <a href="https://stackoverflow.com/a/69626735">https://stackoverflow.com/a/69626735</a>
343+
*/
344+
public static boolean supportsWebView() {
345+
try {
346+
CookieManager.getInstance();
347+
return true;
348+
} catch (Throwable ignored) {
349+
return false;
350+
}
351+
}
338352
}

‎app/src/main/java/org/schabi/newpipe/util/potoken/PoTokenGenerator.kt‎

Lines changed: 11 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -2,16 +2,25 @@ package org.schabi.newpipe.util.potoken
22

33
import android.content.Context
44
import io.reactivex.rxjava3.core.Single
5-
import org.schabi.newpipe.extractor.services.youtube.PoTokenResult
65
import java.io.Closeable
76

7+
/**
8+
* This interface was created to allow for multiple methods to generate poTokens in the future (e.g.
9+
* via WebView and via a local DOM implementation)
10+
*/
811
interface PoTokenGenerator : Closeable {
912
/**
1013
* Generates a poToken for the provided identifier, using the `integrityToken` and
1114
* `webPoSignalOutput` previously obtained in the initialization of [PoTokenWebView]. Can be
1215
* called multiple times.
1316
*/
14-
fun generatePoToken(identifier: String): Single<PoTokenResult>
17+
fun generatePoToken(identifier: String): Single<String>
18+
19+
/**
20+
* @return whether the `integrityToken` is expired, in which case all tokens generated by
21+
* [generatePoToken] will be invalid
22+
*/
23+
fun isExpired(): Boolean
1524

1625
interface Factory {
1726
/**

‎app/src/main/java/org/schabi/newpipe/util/potoken/PoTokenProviderImpl.kt‎

Lines changed: 59 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,59 @@
1+
package org.schabi.newpipe.util.potoken
2+
3+
import android.util.Log
4+
import org.schabi.newpipe.App
5+
import org.schabi.newpipe.extractor.NewPipe
6+
import org.schabi.newpipe.extractor.services.youtube.PoTokenProvider
7+
import org.schabi.newpipe.extractor.services.youtube.PoTokenResult
8+
import org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper
9+
import org.schabi.newpipe.util.DeviceUtils
10+
11+
object PoTokenProviderImpl : PoTokenProvider {
12+
val TAG = PoTokenProviderImpl::class.simpleName
13+
private val webViewSupported by lazy { DeviceUtils.supportsWebView() }
14+
15+
private object WebPoTokenGenLock
16+
private var webPoTokenVisitorData: String? = null
17+
private var webPoTokenStreamingPot: String? = null
18+
private var webPoTokenGenerator: PoTokenGenerator? = null
19+
20+
override fun getWebClientPoToken(videoId: String): PoTokenResult? {
21+
if (!webViewSupported) {
22+
return null
23+
}
24+
25+
val (poTokenGenerator, visitorData, streamingPot) = synchronized(WebPoTokenGenLock) {
26+
if (webPoTokenGenerator == null || webPoTokenGenerator!!.isExpired()) {
27+
webPoTokenGenerator = PoTokenWebView.newPoTokenGenerator(App.getApp()).blockingGet()
28+
webPoTokenVisitorData = YoutubeParsingHelper
29+
.randomVisitorData(NewPipe.getPreferredContentCountry())
30+
31+
// The streaming poToken needs to be generated exactly once before generating any
32+
// other (player) tokens.
33+
webPoTokenStreamingPot = webPoTokenGenerator!!
34+
.generatePoToken(webPoTokenVisitorData!!).blockingGet()
35+
}
36+
return@synchronized Triple(
37+
webPoTokenGenerator!!, webPoTokenVisitorData!!, webPoTokenStreamingPot!!
38+
)
39+
}
40+
41+
// Not using synchronized here, since poTokenGenerator would be able to generate multiple
42+
// poTokens in parallel if needed. The only important thing is for exactly one
43+
// visitorData/streaming poToken to be generated before anything else.
44+
val playerPot = poTokenGenerator.generatePoToken(videoId).blockingGet()
45+
Log.e(TAG, "success($videoId) $playerPot,web.gvs+$streamingPot;visitor_data=$visitorData")
46+
47+
return PoTokenResult(
48+
webPoTokenVisitorData!!,
49+
playerPot,
50+
webPoTokenStreamingPot!!,
51+
)
52+
}
53+
54+
override fun getWebEmbedClientPoToken(videoId: String): PoTokenResult? = null
55+
56+
override fun getAndroidClientPoToken(videoId: String): PoTokenResult? = null
57+
58+
override fun getIosClientPoToken(videoId: String): PoTokenResult? = null
59+
}

‎app/src/main/java/org/schabi/newpipe/util/potoken/PoTokenWebView.kt‎

Lines changed: 38 additions & 26 deletions
Original file line numberDiff line numberDiff line change
@@ -7,14 +7,13 @@ import android.os.Looper
77
import android.util.Log
88
import android.webkit.JavascriptInterface
99
import android.webkit.WebView
10-
import androidx.annotation.MainThread
1110
import io.reactivex.rxjava3.android.schedulers.AndroidSchedulers
1211
import io.reactivex.rxjava3.core.Single
1312
import io.reactivex.rxjava3.core.SingleEmitter
1413
import io.reactivex.rxjava3.disposables.CompositeDisposable
1514
import io.reactivex.rxjava3.schedulers.Schedulers
1615
import org.schabi.newpipe.DownloaderImpl
17-
import org.schabi.newpipe.extractor.services.youtube.PoTokenResult
16+
import java.time.Instant
1817

1918
class PoTokenWebView private constructor(
2019
context: Context,
@@ -23,7 +22,8 @@ class PoTokenWebView private constructor(
2322
) : PoTokenGenerator {
2423
private val webView = WebView(context)
2524
private val disposables = CompositeDisposable() // used only during initialization
26-
private val poTokenEmitters = mutableListOf<Pair<String, SingleEmitter<PoTokenResult>>>()
25+
private val poTokenEmitters = mutableListOf<Pair<String, SingleEmitter<String>>>()
26+
private lateinit var expirationInstant: Instant
2727

2828
//region Initialization
2929
init {
@@ -114,11 +114,12 @@ class PoTokenWebView private constructor(
114114
"https://jnn-pa.googleapis.com/\$rpc/google.internal.waa.v1.Waa/GenerateIT",
115115
"[ \"$REQUEST_KEY\", \"$botguardResponse\" ]",
116116
) { responseBody ->
117+
Log.e(TAG, "GenerateIT response: $responseBody")
117118
webView.evaluateJavascript(
118119
"""(async function() {
119120
try {
120121
globalThis.integrityToken = JSON.parse(String.raw`$responseBody`)
121-
PoTokenWebView.onInitializationFinished()
122+
PoTokenWebView.onInitializationFinished(integrityToken[1])
122123
} catch (error) {
123124
PoTokenWebView.onJsInitializationError(error.toString())
124125
}
@@ -130,9 +131,14 @@ class PoTokenWebView private constructor(
130131
/**
131132
* Called during initialization by the JavaScript snippet from [onRunBotguardResult] when the
132133
* `integrityToken` has been received by JavaScript.
134+
*
135+
* @param expirationTimeInSeconds in how many seconds the integrity token expires, can be found
136+
* in `integrityToken[1]`
133137
*/
134138
@JavascriptInterface
135-
fun onInitializationFinished() {
139+
fun onInitializationFinished(expirationTimeInSeconds: Long) {
140+
// leave 10 minutes of margin just to be sure
141+
expirationInstant = Instant.now().plusSeconds(expirationTimeInSeconds - 600)
136142
generatorEmitter.onSuccess(this)
137143
}
138144
//endregion
@@ -143,7 +149,7 @@ class PoTokenWebView private constructor(
143149
* multiple poToken requests can be generated invparallel, and the results will be notified to
144150
* the right emitters.
145151
*/
146-
private fun addPoTokenEmitter(identifier: String, emitter: SingleEmitter<PoTokenResult>) {
152+
private fun addPoTokenEmitter(identifier: String, emitter: SingleEmitter<String>) {
147153
synchronized(poTokenEmitters) {
148154
poTokenEmitters.add(Pair(identifier, emitter))
149155
}
@@ -154,30 +160,31 @@ class PoTokenWebView private constructor(
154160
* [identifier]. The emitter is supposed to be used immediately after to either signal a success
155161
* or an error.
156162
*/
157-
private fun popPoTokenEmitter(identifier: String): SingleEmitter<PoTokenResult>? {
163+
private fun popPoTokenEmitter(identifier: String): SingleEmitter<String>? {
158164
return synchronized(poTokenEmitters) {
159165
poTokenEmitters.indexOfFirst { it.first == identifier }.takeIf { it >= 0 }?.let {
160166
poTokenEmitters.removeAt(it).second
161167
}
162168
}
163169
}
164170

165-
@MainThread
166-
override fun generatePoToken(identifier: String): Single<PoTokenResult> =
171+
override fun generatePoToken(identifier: String): Single<String> =
167172
Single.create { emitter ->
168173
addPoTokenEmitter(identifier, emitter)
169-
170-
webView.evaluateJavascript(
171-
"""(async function() {
172-
identifier = String.raw`$identifier`
173-
try {
174-
poToken = await obtainPoToken(webPoSignalOutput, integrityToken, identifier)
175-
PoTokenWebView.onObtainPoTokenResult(identifier, poToken)
176-
} catch (error) {
177-
PoTokenWebView.onObtainPoTokenError(identifier, error.toString())
178-
}
179-
})();""",
180-
) {}
174+
Handler(Looper.getMainLooper()).post {
175+
webView.evaluateJavascript(
176+
"""(async function() {
177+
identifier = String.raw`$identifier`
178+
try {
179+
poToken = await obtainPoToken(webPoSignalOutput, integrityToken,
180+
identifier)
181+
PoTokenWebView.onObtainPoTokenResult(identifier, poToken)
182+
} catch (error) {
183+
PoTokenWebView.onObtainPoTokenError(identifier, error.toString())
184+
}
185+
})();""",
186+
) {}
187+
}
181188
}
182189

183190
/**
@@ -198,7 +205,11 @@ class PoTokenWebView private constructor(
198205
fun onObtainPoTokenResult(identifier: String, poToken: String) {
199206
Log.e(TAG, "identifier=$identifier")
200207
Log.e(TAG, "poToken=$poToken")
201-
popPoTokenEmitter(identifier)?.onSuccess(PoTokenResult(identifier, poToken))
208+
popPoTokenEmitter(identifier)?.onSuccess(poToken)
209+
}
210+
211+
override fun isExpired(): Boolean {
212+
return Instant.now().isAfter(expirationInstant)
202213
}
203214
//endregion
204215

@@ -286,12 +297,13 @@ class PoTokenWebView private constructor(
286297
private const val REQUEST_KEY = "O43z0dpjhgX20SCx4KAo"
287298
private const val USER_AGENT = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.3"
288299

289-
@MainThread
290300
override fun newPoTokenGenerator(context: Context): Single<PoTokenGenerator> =
291301
Single.create { emitter ->
292-
val potWv = PoTokenWebView(context, emitter)
293-
potWv.loadHtmlAndObtainBotguard(context)
294-
emitter.setDisposable(potWv.disposables)
302+
Handler(Looper.getMainLooper()).post {
303+
val potWv = PoTokenWebView(context, emitter)
304+
potWv.loadHtmlAndObtainBotguard(context)
305+
emitter.setDisposable(potWv.disposables)
306+
}
295307
}
296308
}
297309
}

0 commit comments

Comments
 (0)