From b5cce29d906f98433922400fe11c57b603265433 Mon Sep 17 00:00:00 2001 From: Cris Barreiro Date: Wed, 8 Jan 2025 16:55:58 +0100 Subject: [PATCH] Intercept requests and add skeleton for malicious site detection (#5369) Task/Issue URL: https://app.asana.com/0/1205008441501016/1207151848931035/f ### Description Adds logic to intercept requests using shouldOverrideUrlLoading and shouldInterceptRequest, when the request matches the following conditions * Is for mainframe * Is for iframe and the host loading it matches the webview host * The same URL hasn't been already intercepted during the same page load (used to prevent the same URL being intercepted by the 2 webview callbacks) ### Steps to test this PR _Feature 1_ - [ ] Load a site and check you get `Timber.tag("MaliciousSiteProtection").d("isMalicious $url")` for all mainframe and iframe requests (only for internal builds). - [ ] Check you should never get 2 instances of those logs for the exact same URL ### UI changes | Before | After | | ------ | ----- | !(Upload before screenshot)|(Upload after screenshot)| --- .../app/browser/BrowserWebViewClientTest.kt | 16 +- .../browser/WebViewRequestInterceptorTest.kt | 3 + .../UrlExtractingWebViewClientTest.kt | 2 + .../referencetests/DomainsReferenceTest.kt | 3 + .../referencetests/SurrogatesReferenceTest.kt | 3 + .../app/browser/BrowserWebViewClient.kt | 17 +- .../app/browser/WebViewRequestInterceptor.kt | 34 ++++ .../app/browser/di/BrowserModule.kt | 3 + .../MaliciousSiteBlockerWebViewIntegration.kt | 162 ++++++++++++++++++ .../AndroidBrowserConfigFeature.kt | 8 + ...iciousSiteBlockerWebViewIntegrationTest.kt | 143 ++++++++++++++++ .../api/MaliciousSiteProtection.kt | 13 +- .../build.gradle | 2 + .../impl/MaliciousSiteProtectionRCFeature.kt | 85 +++++++++ .../impl/RealMaliciousSiteProtection.kt | 47 +---- 15 files changed, 496 insertions(+), 45 deletions(-) create mode 100644 app/src/main/java/com/duckduckgo/app/browser/webview/MaliciousSiteBlockerWebViewIntegration.kt create mode 100644 app/src/test/java/com/duckduckgo/app/browser/webview/RealMaliciousSiteBlockerWebViewIntegrationTest.kt create mode 100644 malicious-site-protection/malicious-site-protection-impl/src/main/kotlin/com/duckduckgo/malicioussiteprotection/impl/MaliciousSiteProtectionRCFeature.kt diff --git a/app/src/androidTest/java/com/duckduckgo/app/browser/BrowserWebViewClientTest.kt b/app/src/androidTest/java/com/duckduckgo/app/browser/BrowserWebViewClientTest.kt index c96b3fd19478..3061968b86f1 100644 --- a/app/src/androidTest/java/com/duckduckgo/app/browser/BrowserWebViewClientTest.kt +++ b/app/src/androidTest/java/com/duckduckgo/app/browser/BrowserWebViewClientTest.kt @@ -59,6 +59,7 @@ import com.duckduckgo.app.browser.pageloadpixel.firstpaint.PagePaintedHandler import com.duckduckgo.app.browser.print.PrintInjector import com.duckduckgo.app.browser.trafficquality.AndroidFeaturesHeaderPlugin import com.duckduckgo.app.browser.uriloaded.UriLoadedManager +import com.duckduckgo.app.browser.webview.MaliciousSiteBlockerWebViewIntegration import com.duckduckgo.app.global.model.Site import com.duckduckgo.app.pixels.remoteconfig.AndroidBrowserConfigFeature import com.duckduckgo.app.statistics.pixels.Pixel @@ -152,10 +153,11 @@ class BrowserWebViewClientTest { private val mockUriLoadedManager: UriLoadedManager = mock() private val mockAndroidBrowserConfigFeature: AndroidBrowserConfigFeature = mock() private val mockAndroidFeaturesHeaderPlugin = AndroidFeaturesHeaderPlugin(mockDuckDuckGoUrlDetector, mockAndroidBrowserConfigFeature, mock()) + private val mockMaliciousSiteProtection: MaliciousSiteBlockerWebViewIntegration = mock() @UiThreadTest @Before - fun setup() { + fun setup() = runTest { webView = TestWebView(context) whenever(mockDuckPlayer.observeShouldOpenInNewTab()).thenReturn(openInNewTabFlow) testee = BrowserWebViewClient( @@ -196,6 +198,7 @@ class BrowserWebViewClientTest { whenever(currentTimeProvider.elapsedRealtime()).thenReturn(0) whenever(webViewVersionProvider.getMajorVersion()).thenReturn("1") whenever(deviceInfo.appVersion).thenReturn("1") + whenever(mockMaliciousSiteProtection.shouldOverrideUrlLoading(any(), any(), any())).thenReturn(false) } @UiThreadTest @@ -315,8 +318,8 @@ class BrowserWebViewClientTest { @Test fun whenOnReceivedHttpAuthRequestThenListenerNotified() { val mockHandler = mock() - val authenticationRequest = BasicAuthenticationRequest(mockHandler, EXAMPLE_URL, EXAMPLE_URL, EXAMPLE_URL) - testee.onReceivedHttpAuthRequest(webView, mockHandler, EXAMPLE_URL, EXAMPLE_URL) + val authenticationRequest = BasicAuthenticationRequest(mockHandler, "example.com", EXAMPLE_URL, EXAMPLE_URL) + testee.onReceivedHttpAuthRequest(webView, mockHandler, "example.com", EXAMPLE_URL) verify(listener).requiresAuthentication(authenticationRequest) } @@ -765,6 +768,7 @@ class BrowserWebViewClientTest { private fun getImmediatelyInvokedMockWebView(): WebView { val mockWebView = mock() whenever(mockWebView.originalUrl).thenReturn(EXAMPLE_URL) + whenever(mockWebView.url).thenReturn(EXAMPLE_URL) whenever(mockWebView.post(any())).thenAnswer { invocation -> invocation.getArgument(0, Runnable::class.java).run() null @@ -1071,6 +1075,10 @@ class BrowserWebViewClientTest { override fun getOriginalUrl(): String { return EXAMPLE_URL } + + override fun getUrl(): String { + return EXAMPLE_URL + } } private class FakePluginPoint : PluginPoint { @@ -1149,6 +1157,6 @@ class BrowserWebViewClientTest { } companion object { - const val EXAMPLE_URL = "example.com" + const val EXAMPLE_URL = "https://example.com" } } diff --git a/app/src/androidTest/java/com/duckduckgo/app/browser/WebViewRequestInterceptorTest.kt b/app/src/androidTest/java/com/duckduckgo/app/browser/WebViewRequestInterceptorTest.kt index 2753e38780ee..c1aca6bcaf18 100644 --- a/app/src/androidTest/java/com/duckduckgo/app/browser/WebViewRequestInterceptorTest.kt +++ b/app/src/androidTest/java/com/duckduckgo/app/browser/WebViewRequestInterceptorTest.kt @@ -29,6 +29,7 @@ import androidx.core.net.toUri import androidx.test.annotation.UiThreadTest import com.duckduckgo.adclick.api.AdClickManager import com.duckduckgo.app.browser.useragent.provideUserAgentOverridePluginPoint +import com.duckduckgo.app.browser.webview.MaliciousSiteBlockerWebViewIntegration import com.duckduckgo.app.fakes.FeatureToggleFake import com.duckduckgo.app.fakes.UserAgentFake import com.duckduckgo.app.fakes.UserAllowListRepositoryFake @@ -97,6 +98,7 @@ class WebViewRequestInterceptorTest { fakeToggle, fakeUserAllowListRepository, ) + private val mockMaliciousSiteProtection: MaliciousSiteBlockerWebViewIntegration = mock() private var webView: WebView = mock() @@ -117,6 +119,7 @@ class WebViewRequestInterceptorTest { cloakedCnameDetector = mockCloakedCnameDetector, requestFilterer = mockRequestFilterer, duckPlayer = mockDuckPlayer, + maliciousSiteBlockerWebViewIntegration = mockMaliciousSiteProtection, ) } diff --git a/app/src/androidTest/java/com/duckduckgo/app/browser/urlextraction/UrlExtractingWebViewClientTest.kt b/app/src/androidTest/java/com/duckduckgo/app/browser/urlextraction/UrlExtractingWebViewClientTest.kt index 52df9a0b1ecb..df4281b4a21b 100644 --- a/app/src/androidTest/java/com/duckduckgo/app/browser/urlextraction/UrlExtractingWebViewClientTest.kt +++ b/app/src/androidTest/java/com/duckduckgo/app/browser/urlextraction/UrlExtractingWebViewClientTest.kt @@ -24,6 +24,7 @@ import com.duckduckgo.app.browser.* import com.duckduckgo.app.browser.certificates.rootstore.TrustedCertificateStore import com.duckduckgo.app.browser.cookies.ThirdPartyCookieManager import com.duckduckgo.app.browser.httpauth.WebViewHttpAuthStore +import com.duckduckgo.app.browser.webview.MaliciousSiteBlockerWebViewIntegration import com.duckduckgo.common.test.CoroutineTestRule import com.duckduckgo.cookies.api.CookieManagerProvider import kotlinx.coroutines.test.TestScope @@ -50,6 +51,7 @@ class UrlExtractingWebViewClientTest { private val thirdPartyCookieManager: ThirdPartyCookieManager = mock() private val urlExtractor: DOMUrlExtractor = mock() private val mockWebView: WebView = mock() + private val mockMaliciousSiteProtection: MaliciousSiteBlockerWebViewIntegration = mock() @UiThreadTest @Before diff --git a/app/src/androidTest/java/com/duckduckgo/app/referencetests/DomainsReferenceTest.kt b/app/src/androidTest/java/com/duckduckgo/app/referencetests/DomainsReferenceTest.kt index 9bc809b2efdf..f98a8e43cce8 100644 --- a/app/src/androidTest/java/com/duckduckgo/app/referencetests/DomainsReferenceTest.kt +++ b/app/src/androidTest/java/com/duckduckgo/app/referencetests/DomainsReferenceTest.kt @@ -28,6 +28,7 @@ import androidx.test.platform.app.InstrumentationRegistry import com.duckduckgo.adclick.api.AdClickManager import com.duckduckgo.app.browser.WebViewRequestInterceptor import com.duckduckgo.app.browser.useragent.provideUserAgentOverridePluginPoint +import com.duckduckgo.app.browser.webview.MaliciousSiteBlockerWebViewIntegration import com.duckduckgo.app.fakes.FeatureToggleFake import com.duckduckgo.app.fakes.UserAgentFake import com.duckduckgo.app.fakes.UserAllowListRepositoryFake @@ -119,6 +120,7 @@ class DomainsReferenceTest(private val testCase: TestCase) { ) private val mockGpc: Gpc = mock() private val mockAdClickManager: AdClickManager = mock() + private val mockMaliciousSiteProtection: MaliciousSiteBlockerWebViewIntegration = mock() companion object { private val moshi = Moshi.Builder().add(ActionJsonAdapter()).build() @@ -174,6 +176,7 @@ class DomainsReferenceTest(private val testCase: TestCase) { adClickManager = mockAdClickManager, cloakedCnameDetector = CloakedCnameDetectorImpl(tdsCnameEntityDao, mockTrackerAllowlist, mockUserAllowListRepository), requestFilterer = mockRequestFilterer, + maliciousSiteBlockerWebViewIntegration = mockMaliciousSiteProtection, duckPlayer = mockDuckPlayer, ) } diff --git a/app/src/androidTest/java/com/duckduckgo/app/referencetests/SurrogatesReferenceTest.kt b/app/src/androidTest/java/com/duckduckgo/app/referencetests/SurrogatesReferenceTest.kt index 83fb7607d59f..98673c5f285a 100644 --- a/app/src/androidTest/java/com/duckduckgo/app/referencetests/SurrogatesReferenceTest.kt +++ b/app/src/androidTest/java/com/duckduckgo/app/referencetests/SurrogatesReferenceTest.kt @@ -27,6 +27,7 @@ import androidx.test.platform.app.InstrumentationRegistry import com.duckduckgo.adclick.api.AdClickManager import com.duckduckgo.app.browser.WebViewRequestInterceptor import com.duckduckgo.app.browser.useragent.provideUserAgentOverridePluginPoint +import com.duckduckgo.app.browser.webview.MaliciousSiteBlockerWebViewIntegration import com.duckduckgo.app.fakes.FeatureToggleFake import com.duckduckgo.app.fakes.UserAgentFake import com.duckduckgo.app.fakes.UserAllowListRepositoryFake @@ -115,6 +116,7 @@ class SurrogatesReferenceTest(private val testCase: TestCase) { private val mockGpc: Gpc = mock() private val mockAdClickManager: AdClickManager = mock() private val mockCloakedCnameDetector: CloakedCnameDetector = mock() + private val mockMaliciousSiteProtection: MaliciousSiteBlockerWebViewIntegration = mock() companion object { private val moshi = Moshi.Builder().add(ActionJsonAdapter()).build() @@ -170,6 +172,7 @@ class SurrogatesReferenceTest(private val testCase: TestCase) { adClickManager = mockAdClickManager, cloakedCnameDetector = mockCloakedCnameDetector, requestFilterer = mockRequestFilterer, + maliciousSiteBlockerWebViewIntegration = mockMaliciousSiteProtection, duckPlayer = mockDuckPlayer, ) } diff --git a/app/src/main/java/com/duckduckgo/app/browser/BrowserWebViewClient.kt b/app/src/main/java/com/duckduckgo/app/browser/BrowserWebViewClient.kt index de04c2b54cc9..a0cb6de7cda9 100644 --- a/app/src/main/java/com/duckduckgo/app/browser/BrowserWebViewClient.kt +++ b/app/src/main/java/com/duckduckgo/app/browser/BrowserWebViewClient.kt @@ -126,6 +126,10 @@ class BrowserWebViewClient @Inject constructor( private var shouldOpenDuckPlayerInNewTab: Boolean = true + private val confirmationCallback: (isMalicious: Boolean) -> Unit = { + // TODO (cbarreiro): Handle site blocked asynchronously + } + init { appCoroutineScope.launch { duckPlayer.observeShouldOpenInNewTab().collect { @@ -158,6 +162,10 @@ class BrowserWebViewClient @Inject constructor( try { Timber.v("shouldOverride webViewUrl: ${webView.url} URL: $url") webViewClientListener?.onShouldOverride() + if (requestInterceptor.shouldOverrideUrlLoading(url, isForMainFrame)) { + return true + } + if (isForMainFrame && dosDetector.isUrlGeneratingDos(url)) { webView.loadUrl(ABOUT_BLANK) webViewClientListener?.dosAttackDetected() @@ -407,11 +415,11 @@ class BrowserWebViewClient @Inject constructor( // See https://app.asana.com/0/0/1206159443951489/f (WebView limitations) if (it != ABOUT_BLANK && start == null) { start = currentTimeProvider.elapsedRealtime() + requestInterceptor.onPageStarted(url) } handleMediaPlayback(webView, it) autoconsent.injectAutoconsent(webView, url) adClickManager.detectAdDomain(url) - requestInterceptor.onPageStarted(url) appCoroutineScope.launch(dispatcherProvider.io()) { thirdPartyCookieManager.processUriForThirdPartyCookies(webView, url.toUri()) } @@ -509,7 +517,12 @@ class BrowserWebViewClient @Inject constructor( loginDetector.onEvent(WebNavigationEvent.ShouldInterceptRequest(webView, request)) } Timber.v("Intercepting resource ${request.url} type:${request.method} on page $documentUrl") - requestInterceptor.shouldIntercept(request, webView, documentUrl?.toUri(), webViewClientListener) + requestInterceptor.shouldIntercept( + request, + webView, + documentUrl?.toUri(), + webViewClientListener, + ) } } diff --git a/app/src/main/java/com/duckduckgo/app/browser/WebViewRequestInterceptor.kt b/app/src/main/java/com/duckduckgo/app/browser/WebViewRequestInterceptor.kt index 5ca57ebb0029..5c3ec224118e 100644 --- a/app/src/main/java/com/duckduckgo/app/browser/WebViewRequestInterceptor.kt +++ b/app/src/main/java/com/duckduckgo/app/browser/WebViewRequestInterceptor.kt @@ -22,6 +22,7 @@ import android.webkit.WebResourceResponse import android.webkit.WebView import androidx.annotation.WorkerThread import com.duckduckgo.adclick.api.AdClickManager +import com.duckduckgo.app.browser.webview.MaliciousSiteBlockerWebViewIntegration import com.duckduckgo.app.privacy.db.PrivacyProtectionCountDao import com.duckduckgo.app.privacy.model.TrustedSites import com.duckduckgo.app.surrogates.ResourceSurrogates @@ -58,6 +59,12 @@ interface RequestInterceptor { ): WebResourceResponse? fun onPageStarted(url: String) + + @WorkerThread + fun shouldOverrideUrlLoading( + url: Uri, + isForMainFrame: Boolean, + ): Boolean } class WebViewRequestInterceptor( @@ -71,11 +78,13 @@ class WebViewRequestInterceptor( private val cloakedCnameDetector: CloakedCnameDetector, private val requestFilterer: RequestFilterer, private val duckPlayer: DuckPlayer, + private val maliciousSiteBlockerWebViewIntegration: MaliciousSiteBlockerWebViewIntegration, private val dispatchers: DispatcherProvider = DefaultDispatcherProvider(), ) : RequestInterceptor { override fun onPageStarted(url: String) { requestFilterer.registerOnPageCreated(url) + maliciousSiteBlockerWebViewIntegration.onPageLoadStarted() } /** @@ -96,6 +105,13 @@ class WebViewRequestInterceptor( ): WebResourceResponse? { val url: Uri? = request.url + maliciousSiteBlockerWebViewIntegration.shouldIntercept(request, documentUri) { + handleSiteBlocked() + }?.let { + handleSiteBlocked() + return it + } + if (requestFilterer.shouldFilterOutRequest(request, documentUri.toString())) return WebResourceResponse(null, null, null) adClickManager.detectAdClick(url?.toString(), request.isForMainFrame) @@ -161,6 +177,24 @@ class WebViewRequestInterceptor( return getWebResourceResponse(request, documentUrl, null) } + override fun shouldOverrideUrlLoading(url: Uri, isForMainFrame: Boolean): Boolean { + if (maliciousSiteBlockerWebViewIntegration.shouldOverrideUrlLoading( + url, + isForMainFrame, + ) { + handleSiteBlocked() + } + ) { + handleSiteBlocked() + return true + } + return false + } + + private fun handleSiteBlocked() { + // TODO (cbarreiro): Handle site blocked + } + private fun getWebResourceResponse( request: WebResourceRequest, documentUrl: Uri, diff --git a/app/src/main/java/com/duckduckgo/app/browser/di/BrowserModule.kt b/app/src/main/java/com/duckduckgo/app/browser/di/BrowserModule.kt index 98876d97a533..ca07b2c52796 100644 --- a/app/src/main/java/com/duckduckgo/app/browser/di/BrowserModule.kt +++ b/app/src/main/java/com/duckduckgo/app/browser/di/BrowserModule.kt @@ -55,6 +55,7 @@ import com.duckduckgo.app.browser.tabpreview.WebViewPreviewPersister import com.duckduckgo.app.browser.urlextraction.DOMUrlExtractor import com.duckduckgo.app.browser.urlextraction.JsUrlExtractor import com.duckduckgo.app.browser.urlextraction.UrlExtractingWebViewClient +import com.duckduckgo.app.browser.webview.MaliciousSiteBlockerWebViewIntegration import com.duckduckgo.app.di.AppCoroutineScope import com.duckduckgo.app.fire.* import com.duckduckgo.app.fire.fireproofwebsite.data.FireproofWebsiteRepository @@ -201,6 +202,7 @@ class BrowserModule { cloakedCnameDetector: CloakedCnameDetector, requestFilterer: RequestFilterer, duckPlayer: DuckPlayer, + maliciousSiteBlockerWebViewIntegration: MaliciousSiteBlockerWebViewIntegration, ): RequestInterceptor = WebViewRequestInterceptor( resourceSurrogates, @@ -213,6 +215,7 @@ class BrowserModule { cloakedCnameDetector, requestFilterer, duckPlayer, + maliciousSiteBlockerWebViewIntegration, ) @Provides diff --git a/app/src/main/java/com/duckduckgo/app/browser/webview/MaliciousSiteBlockerWebViewIntegration.kt b/app/src/main/java/com/duckduckgo/app/browser/webview/MaliciousSiteBlockerWebViewIntegration.kt new file mode 100644 index 000000000000..ab00b5d78f2a --- /dev/null +++ b/app/src/main/java/com/duckduckgo/app/browser/webview/MaliciousSiteBlockerWebViewIntegration.kt @@ -0,0 +1,162 @@ +/* + * Copyright (c) 2024 DuckDuckGo + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.duckduckgo.app.browser.webview + +import android.net.Uri +import android.webkit.WebResourceRequest +import android.webkit.WebResourceResponse +import androidx.annotation.VisibleForTesting +import androidx.core.net.toUri +import com.duckduckgo.app.di.AppCoroutineScope +import com.duckduckgo.app.di.IsMainProcess +import com.duckduckgo.app.pixels.remoteconfig.AndroidBrowserConfigFeature +import com.duckduckgo.common.utils.DispatcherProvider +import com.duckduckgo.di.scopes.AppScope +import com.duckduckgo.malicioussiteprotection.api.MaliciousSiteProtection +import com.duckduckgo.malicioussiteprotection.api.MaliciousSiteProtection.IsMaliciousResult.MALICIOUS +import com.duckduckgo.privacy.config.api.PrivacyConfigCallbackPlugin +import com.squareup.anvil.annotations.ContributesBinding +import com.squareup.anvil.annotations.ContributesMultibinding +import java.net.URLDecoder +import javax.inject.Inject +import kotlinx.coroutines.CoroutineScope +import kotlinx.coroutines.launch +import kotlinx.coroutines.runBlocking +import timber.log.Timber + +interface MaliciousSiteBlockerWebViewIntegration { + + suspend fun shouldIntercept( + request: WebResourceRequest, + documentUri: Uri?, + confirmationCallback: (isMalicious: Boolean) -> Unit, + ): WebResourceResponse? + + fun shouldOverrideUrlLoading( + url: Uri, + isForMainFrame: Boolean, + confirmationCallback: (isMalicious: Boolean) -> Unit, + ): Boolean + + fun onPageLoadStarted() +} + +@ContributesMultibinding(AppScope::class, PrivacyConfigCallbackPlugin::class) +@ContributesBinding(AppScope::class, MaliciousSiteBlockerWebViewIntegration::class) +class RealMaliciousSiteBlockerWebViewIntegration @Inject constructor( + private val maliciousSiteProtection: MaliciousSiteProtection, + private val androidBrowserConfigFeature: AndroidBrowserConfigFeature, + private val dispatchers: DispatcherProvider, + @AppCoroutineScope private val appCoroutineScope: CoroutineScope, + @IsMainProcess private val isMainProcess: Boolean, +) : MaliciousSiteBlockerWebViewIntegration, PrivacyConfigCallbackPlugin { + + @VisibleForTesting(otherwise = VisibleForTesting.PRIVATE) + val processedUrls = mutableListOf() + private var isFeatureEnabled = false + + init { + if (isMainProcess) { + loadToMemory() + } + } + + private fun loadToMemory() { + appCoroutineScope.launch(dispatchers.io()) { + isFeatureEnabled = androidBrowserConfigFeature.enableMaliciousSiteProtection().isEnabled() + } + } + + override fun onPrivacyConfigDownloaded() { + loadToMemory() + } + + override suspend fun shouldIntercept( + request: WebResourceRequest, + documentUri: Uri?, + confirmationCallback: (isMalicious: Boolean) -> Unit, + ): WebResourceResponse? { + if (!isFeatureEnabled) { + return null + } + val url = request.url.let { + if (it.fragment != null) { + it.buildUpon().fragment(null).build() + } else { + it + } + } + + val decodedUrl = URLDecoder.decode(url.toString(), "UTF-8").lowercase() + + if (processedUrls.contains(decodedUrl)) { + processedUrls.remove(decodedUrl) + Timber.tag("PhishingAndMalwareDetector").d("Already intercepted, skipping $decodedUrl") + return null + } + + if (request.isForMainFrame) { + if (maliciousSiteProtection.isMalicious(decodedUrl.toUri(), confirmationCallback) == MALICIOUS) { + return WebResourceResponse(null, null, null) + } + processedUrls.add(decodedUrl) + } else if (isForIframe(request) && documentUri?.host == request.requestHeaders["Referer"]?.toUri()?.host) { + if (maliciousSiteProtection.isMalicious(decodedUrl.toUri(), confirmationCallback) == MALICIOUS) { + return WebResourceResponse(null, null, null) + } + processedUrls.add(decodedUrl) + } + return null + } + + override fun shouldOverrideUrlLoading( + url: Uri, + isForMainFrame: Boolean, + confirmationCallback: (isMalicious: Boolean) -> Unit, + ): Boolean { + return runBlocking { + if (!isFeatureEnabled) { + return@runBlocking false + } + val decodedUrl = URLDecoder.decode(url.toString(), "UTF-8").lowercase() + + if (processedUrls.contains(decodedUrl)) { + processedUrls.remove(decodedUrl) + Timber.tag("PhishingAndMalwareDetector").d("Already intercepted, skipping $decodedUrl") + return@runBlocking false + } + + // iframes always go through the shouldIntercept method, so we only need to check the main frame here + if (isForMainFrame) { + if (maliciousSiteProtection.isMalicious(decodedUrl.toUri(), confirmationCallback) == MALICIOUS) { + return@runBlocking true + } + processedUrls.add(decodedUrl) + } + false + } + } + + private fun isForIframe(request: WebResourceRequest) = request.requestHeaders["Sec-Fetch-Dest"] == "iframe" || + request.url.path?.contains("/embed/") == true || + request.url.path?.contains("/iframe/") == true || + request.requestHeaders["Accept"]?.contains("text/html") == true + + override fun onPageLoadStarted() { + processedUrls.clear() + } +} diff --git a/app/src/main/java/com/duckduckgo/app/pixels/remoteconfig/AndroidBrowserConfigFeature.kt b/app/src/main/java/com/duckduckgo/app/pixels/remoteconfig/AndroidBrowserConfigFeature.kt index f555e8ccb068..c3354e110fcb 100644 --- a/app/src/main/java/com/duckduckgo/app/pixels/remoteconfig/AndroidBrowserConfigFeature.kt +++ b/app/src/main/java/com/duckduckgo/app/pixels/remoteconfig/AndroidBrowserConfigFeature.kt @@ -91,4 +91,12 @@ interface AndroidBrowserConfigFeature { */ @Toggle.DefaultValue(false) fun webLocalStorage(): Toggle + + /** + * @return `true` when the remote config has the global "enableMaliciousSiteProtection" androidBrowserConfig + * sub-feature flag enabled + * If the remote feature is not present defaults to `false` + */ + @Toggle.DefaultValue(false) + fun enableMaliciousSiteProtection(): Toggle } diff --git a/app/src/test/java/com/duckduckgo/app/browser/webview/RealMaliciousSiteBlockerWebViewIntegrationTest.kt b/app/src/test/java/com/duckduckgo/app/browser/webview/RealMaliciousSiteBlockerWebViewIntegrationTest.kt new file mode 100644 index 000000000000..b4141551c217 --- /dev/null +++ b/app/src/test/java/com/duckduckgo/app/browser/webview/RealMaliciousSiteBlockerWebViewIntegrationTest.kt @@ -0,0 +1,143 @@ +package com.duckduckgo.app.browser.webview + +import android.webkit.WebResourceRequest +import androidx.core.net.toUri +import androidx.test.ext.junit.runners.AndroidJUnit4 +import com.duckduckgo.app.pixels.remoteconfig.AndroidBrowserConfigFeature +import com.duckduckgo.common.test.CoroutineTestRule +import com.duckduckgo.feature.toggles.api.FakeFeatureToggleFactory +import com.duckduckgo.feature.toggles.api.Toggle.State +import com.duckduckgo.malicioussiteprotection.api.MaliciousSiteProtection +import com.duckduckgo.malicioussiteprotection.api.MaliciousSiteProtection.IsMaliciousResult.MALICIOUS +import junit.framework.TestCase.assertTrue +import kotlinx.coroutines.test.runTest +import org.junit.Assert.assertFalse +import org.junit.Assert.assertNotNull +import org.junit.Assert.assertNull +import org.junit.Before +import org.junit.Rule +import org.junit.Test +import org.junit.runner.RunWith +import org.mockito.Mockito.mock +import org.mockito.kotlin.any +import org.mockito.kotlin.whenever + +@RunWith(AndroidJUnit4::class) +class RealMaliciousSiteBlockerWebViewIntegrationTest { + + @get:Rule + var coroutineRule = CoroutineTestRule() + + private val maliciousSiteProtection: MaliciousSiteProtection = mock(MaliciousSiteProtection::class.java) + private val fakeAndroidBrowserConfigFeature = FakeFeatureToggleFactory.create(AndroidBrowserConfigFeature::class.java) + private val maliciousUri = "http://malicious.com".toUri() + private val exampleUri = "http://example.com".toUri() + private val testee = RealMaliciousSiteBlockerWebViewIntegration( + maliciousSiteProtection, + androidBrowserConfigFeature = fakeAndroidBrowserConfigFeature, + dispatchers = coroutineRule.testDispatcherProvider, + appCoroutineScope = coroutineRule.testScope, + isMainProcess = true, + ) + + @Before + fun setup() { + updateFeatureEnabled(true) + } + + @Test + fun `shouldOverrideUrlLoading returns false when feature is disabled`() = runTest { + updateFeatureEnabled(false) + + val result = testee.shouldOverrideUrlLoading(exampleUri, true) {} + assertFalse(result) + } + + @Test + fun `shouldInterceptRequest returns null when feature is disabled`() = runTest { + val request = mock(WebResourceRequest::class.java) + whenever(request.url).thenReturn(exampleUri) + updateFeatureEnabled(false) + + val result = testee.shouldIntercept(request, null) {} + assertNull(result) + } + + @Test + fun `shouldOverrideUrlLoading returns false when url is already processed`() = runTest { + testee.processedUrls.add(exampleUri.toString()) + + val result = testee.shouldOverrideUrlLoading(exampleUri, true) {} + assertFalse(result) + } + + @Test + fun `shouldInterceptRequest returns result when feature is enabled, is malicious, and is mainframe`() = runTest { + val request = mock(WebResourceRequest::class.java) + whenever(request.url).thenReturn(maliciousUri) + whenever(request.isForMainFrame).thenReturn(true) + whenever(maliciousSiteProtection.isMalicious(any(), any())).thenReturn(MALICIOUS) + + val result = testee.shouldIntercept(request, maliciousUri) {} + assertNotNull(result) + } + + @Test + fun `shouldInterceptRequest returns result when feature is enabled, is malicious, and is iframe`() = runTest { + val request = mock(WebResourceRequest::class.java) + whenever(request.url).thenReturn(maliciousUri) + whenever(request.isForMainFrame).thenReturn(true) + whenever(request.requestHeaders).thenReturn(mapOf("Sec-Fetch-Dest" to "iframe")) + whenever(maliciousSiteProtection.isMalicious(any(), any())).thenReturn(MALICIOUS) + + val result = testee.shouldIntercept(request, maliciousUri) {} + assertNotNull(result) + } + + @Test + fun `shouldInterceptRequest returns null when feature is enabled, is malicious, and is not mainframe nor iframe`() = runTest { + val request = mock(WebResourceRequest::class.java) + whenever(request.url).thenReturn(maliciousUri) + whenever(request.isForMainFrame).thenReturn(false) + whenever(maliciousSiteProtection.isMalicious(any(), any())).thenReturn(MALICIOUS) + + val result = testee.shouldIntercept(request, maliciousUri) {} + assertNull(result) + } + + @Test + fun `shouldOverride returns false when feature is enabled, is malicious, and is not mainframe`() = runTest { + whenever(maliciousSiteProtection.isMalicious(any(), any())).thenReturn(MALICIOUS) + + val result = testee.shouldOverrideUrlLoading(maliciousUri, false) {} + assertFalse(result) + } + + @Test + fun `shouldOverride returns true when feature is enabled, is malicious, and is mainframe`() = runTest { + whenever(maliciousSiteProtection.isMalicious(any(), any())).thenReturn(MALICIOUS) + + val result = testee.shouldOverrideUrlLoading(maliciousUri, true) {} + assertTrue(result) + } + + @Test + fun `shouldOverride returns false when feature is enabled, is malicious, and not mainframe nor iframe`() = runTest { + whenever(maliciousSiteProtection.isMalicious(any(), any())).thenReturn(MALICIOUS) + + val result = testee.shouldOverrideUrlLoading(maliciousUri, false) {} + assertFalse(result) + } + + @Test + fun `onPageLoadStarted clears processedUrls`() = runTest { + testee.processedUrls.add(exampleUri.toString()) + testee.onPageLoadStarted() + assertTrue(testee.processedUrls.isEmpty()) + } + + private fun updateFeatureEnabled(enabled: Boolean) { + fakeAndroidBrowserConfigFeature.enableMaliciousSiteProtection().setRawStoredState(State(enabled)) + testee.onPrivacyConfigDownloaded() + } +} diff --git a/malicious-site-protection/malicious-site-protection-api/src/main/kotlin/com/duckduckgo/malicioussiteprotection/api/MaliciousSiteProtection.kt b/malicious-site-protection/malicious-site-protection-api/src/main/kotlin/com/duckduckgo/malicioussiteprotection/api/MaliciousSiteProtection.kt index ac25c78c3083..7f5ad066d72d 100644 --- a/malicious-site-protection/malicious-site-protection-api/src/main/kotlin/com/duckduckgo/malicioussiteprotection/api/MaliciousSiteProtection.kt +++ b/malicious-site-protection/malicious-site-protection-api/src/main/kotlin/com/duckduckgo/malicioussiteprotection/api/MaliciousSiteProtection.kt @@ -16,4 +16,15 @@ package com.duckduckgo.malicioussiteprotection.api -interface MaliciousSiteProtection +import android.net.Uri + +interface MaliciousSiteProtection { + + suspend fun isMalicious(url: Uri, confirmationCallback: (isMalicious: Boolean) -> Unit): IsMaliciousResult + + enum class IsMaliciousResult { + MALICIOUS, + SAFE, + WAIT_FOR_CONFIRMATION, + } +} diff --git a/malicious-site-protection/malicious-site-protection-impl/build.gradle b/malicious-site-protection/malicious-site-protection-impl/build.gradle index 8e5767047da1..c9454a21f7aa 100644 --- a/malicious-site-protection/malicious-site-protection-impl/build.gradle +++ b/malicious-site-protection/malicious-site-protection-impl/build.gradle @@ -42,11 +42,13 @@ dependencies { implementation Google.android.material + testImplementation AndroidX.test.ext.junit testImplementation Testing.junit4 testImplementation "org.mockito.kotlin:mockito-kotlin:_" testImplementation project(path: ':common-test') testImplementation CashApp.turbine testImplementation Testing.robolectric + testImplementation project(':feature-toggles-test') testImplementation(KotlinX.coroutines.test) { // https://github.com/Kotlin/kotlinx.coroutines/issues/2023 // conflicts with mockito due to direct inclusion of byte buddy diff --git a/malicious-site-protection/malicious-site-protection-impl/src/main/kotlin/com/duckduckgo/malicioussiteprotection/impl/MaliciousSiteProtectionRCFeature.kt b/malicious-site-protection/malicious-site-protection-impl/src/main/kotlin/com/duckduckgo/malicioussiteprotection/impl/MaliciousSiteProtectionRCFeature.kt new file mode 100644 index 000000000000..16db750c413d --- /dev/null +++ b/malicious-site-protection/malicious-site-protection-impl/src/main/kotlin/com/duckduckgo/malicioussiteprotection/impl/MaliciousSiteProtectionRCFeature.kt @@ -0,0 +1,85 @@ +/* + * Copyright (c) 2024 DuckDuckGo + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.duckduckgo.malicioussiteprotection.impl + +import com.duckduckgo.app.di.AppCoroutineScope +import com.duckduckgo.app.di.IsMainProcess +import com.duckduckgo.common.utils.DispatcherProvider +import com.duckduckgo.di.scopes.AppScope +import com.duckduckgo.privacy.config.api.PrivacyConfigCallbackPlugin +import com.squareup.anvil.annotations.ContributesBinding +import com.squareup.anvil.annotations.ContributesMultibinding +import dagger.SingleInstanceIn +import javax.inject.Inject +import kotlinx.coroutines.CoroutineScope +import kotlinx.coroutines.launch +import org.json.JSONObject + +interface MaliciousSiteProtectionRCFeature { + fun isFeatureEnabled(): Boolean + fun getHashPrefixUpdateFrequency(): Long + fun getFilterSetUpdateFrequency(): Long +} + +@SingleInstanceIn(AppScope::class) +@ContributesBinding(AppScope::class, MaliciousSiteProtectionRCFeature::class) +@ContributesMultibinding(AppScope::class, PrivacyConfigCallbackPlugin::class) +class RealMaliciousSiteProtectionRCFeature @Inject constructor( + private val dispatchers: DispatcherProvider, + private val maliciousSiteProtectionFeature: MaliciousSiteProtectionFeature, + @IsMainProcess private val isMainProcess: Boolean, + @AppCoroutineScope private val appCoroutineScope: CoroutineScope, +) : MaliciousSiteProtectionRCFeature, PrivacyConfigCallbackPlugin { + private var isFeatureEnabled = false + + private var hashPrefixUpdateFrequency = 20L + private var filterSetUpdateFrequency = 720L + + init { + if (isMainProcess) { + loadToMemory() + } + } + + override fun onPrivacyConfigDownloaded() { + loadToMemory() + } + + override fun getFilterSetUpdateFrequency(): Long { + return filterSetUpdateFrequency + } + + override fun getHashPrefixUpdateFrequency(): Long { + return hashPrefixUpdateFrequency + } + + override fun isFeatureEnabled(): Boolean { + return isFeatureEnabled + } + + private fun loadToMemory() { + appCoroutineScope.launch(dispatchers.io()) { + isFeatureEnabled = maliciousSiteProtectionFeature.self().isEnabled() + maliciousSiteProtectionFeature.self().getSettings()?.let { + JSONObject(it).let { settings -> + hashPrefixUpdateFrequency = settings.getLong("hashPrefixUpdateFrequency") + filterSetUpdateFrequency = settings.getLong("filterSetUpdateFrequency") + } + } + } + } +} diff --git a/malicious-site-protection/malicious-site-protection-impl/src/main/kotlin/com/duckduckgo/malicioussiteprotection/impl/RealMaliciousSiteProtection.kt b/malicious-site-protection/malicious-site-protection-impl/src/main/kotlin/com/duckduckgo/malicioussiteprotection/impl/RealMaliciousSiteProtection.kt index 7cb11d7ccb00..7ff0dd28d5d5 100644 --- a/malicious-site-protection/malicious-site-protection-impl/src/main/kotlin/com/duckduckgo/malicioussiteprotection/impl/RealMaliciousSiteProtection.kt +++ b/malicious-site-protection/malicious-site-protection-impl/src/main/kotlin/com/duckduckgo/malicioussiteprotection/impl/RealMaliciousSiteProtection.kt @@ -16,51 +16,22 @@ package com.duckduckgo.malicioussiteprotection.impl -import com.duckduckgo.app.di.AppCoroutineScope -import com.duckduckgo.app.di.IsMainProcess -import com.duckduckgo.common.utils.DispatcherProvider +import android.net.Uri import com.duckduckgo.di.scopes.AppScope import com.duckduckgo.malicioussiteprotection.api.MaliciousSiteProtection -import com.duckduckgo.privacy.config.api.PrivacyConfigCallbackPlugin +import com.duckduckgo.malicioussiteprotection.api.MaliciousSiteProtection.IsMaliciousResult import com.squareup.anvil.annotations.ContributesBinding -import com.squareup.anvil.annotations.ContributesMultibinding import javax.inject.Inject -import kotlinx.coroutines.CoroutineScope -import kotlinx.coroutines.launch -import org.json.JSONObject +import timber.log.Timber @ContributesBinding(AppScope::class, MaliciousSiteProtection::class) -@ContributesMultibinding(AppScope::class, PrivacyConfigCallbackPlugin::class) class RealMaliciousSiteProtection @Inject constructor( - private val dispatchers: DispatcherProvider, - private val maliciousSiteProtectionFeature: MaliciousSiteProtectionFeature, - @IsMainProcess private val isMainProcess: Boolean, - @AppCoroutineScope private val appCoroutineScope: CoroutineScope, -) : MaliciousSiteProtection, PrivacyConfigCallbackPlugin { + maliciousSiteProtectionRCFeature: MaliciousSiteProtectionRCFeature, +) : MaliciousSiteProtection { - private var isFeatureEnabled = false - private var hashPrefixUpdateFrequency = 20L - private var filterSetUpdateFrequency = 720L - - init { - if (isMainProcess) { - loadToMemory() - } - } - - override fun onPrivacyConfigDownloaded() { - loadToMemory() - } - - private fun loadToMemory() { - appCoroutineScope.launch(dispatchers.io()) { - isFeatureEnabled = maliciousSiteProtectionFeature.self().isEnabled() - maliciousSiteProtectionFeature.self().getSettings()?.let { - JSONObject(it).let { settings -> - hashPrefixUpdateFrequency = settings.getLong("hashPrefixUpdateFrequency") - filterSetUpdateFrequency = settings.getLong("filterSetUpdateFrequency") - } - } - } + override suspend fun isMalicious(url: Uri, confirmationCallback: (isMalicious: Boolean) -> Unit): IsMaliciousResult { + Timber.tag("MaliciousSiteProtection").d("isMalicious $url") + // TODO (cbarreiro): Implement the logic to check if the URL is malicious + return IsMaliciousResult.SAFE } }