From 8674a8bdaddb62e2c150783bfc06eb4fd33138a8 Mon Sep 17 00:00:00 2001 From: Yuri Schimke Date: Sat, 20 Jan 2024 13:48:18 +0000 Subject: [PATCH 01/37] Cache Lock --- .../okhttp3/internal/cache/CacheLock.kt | 44 +++++++++++++++++++ .../okhttp3/internal/cache/DiskLruCache.kt | 7 +++ 2 files changed, 51 insertions(+) create mode 100644 okhttp/src/main/kotlin/okhttp3/internal/cache/CacheLock.kt diff --git a/okhttp/src/main/kotlin/okhttp3/internal/cache/CacheLock.kt b/okhttp/src/main/kotlin/okhttp3/internal/cache/CacheLock.kt new file mode 100644 index 000000000000..44f06c3985df --- /dev/null +++ b/okhttp/src/main/kotlin/okhttp3/internal/cache/CacheLock.kt @@ -0,0 +1,44 @@ +package okhttp3.internal.cache + +import android.annotation.SuppressLint +import java.io.Closeable +import java.nio.channels.FileChannel +import java.nio.file.StandardOpenOption +import java.util.Collections +import okhttp3.internal.platform.Platform +import okio.FileSystem +import okio.Path + +internal object CacheLock { + private val openCaches = Collections.synchronizedMap(mutableMapOf()) + + fun openLock(fileSystem: FileSystem, directory: Path): Closeable { + return if (fileSystem == FileSystem.SYSTEM && !Platform.isAndroid) { + fileSystemLock(directory) + } else { + inMemoryLock(directory) + } + } + + @SuppressLint("NewApi") + fun inMemoryLock(directory: Path): Closeable { + // TODO solution for Android N? + val existing = openCaches.putIfAbsent(directory, Exception("CacheLock($directory)")) + if (existing != null) { + throw IllegalStateException("Cache already open at '$directory'", existing) + } + return okio.Closeable { + openCaches.remove(directory) + } + } + + @SuppressLint("NewApi") + fun fileSystemLock(directory: Path): Closeable { + val lockFile = directory / "lock" + val channel = FileChannel.open(lockFile.toNioPath(), StandardOpenOption.APPEND) + + return okio.Closeable { + channel.close() + } + } +} diff --git a/okhttp/src/main/kotlin/okhttp3/internal/cache/DiskLruCache.kt b/okhttp/src/main/kotlin/okhttp3/internal/cache/DiskLruCache.kt index e581cd068b90..605eaf233314 100644 --- a/okhttp/src/main/kotlin/okhttp3/internal/cache/DiskLruCache.kt +++ b/okhttp/src/main/kotlin/okhttp3/internal/cache/DiskLruCache.kt @@ -20,6 +20,7 @@ import java.io.EOFException import java.io.Flushable import java.io.IOException import okhttp3.internal.assertThreadHoldsLock +import okhttp3.internal.cache.CacheLock.openLock import okhttp3.internal.cache.DiskLruCache.Editor import okhttp3.internal.closeQuietly import okhttp3.internal.concurrent.Task @@ -95,6 +96,8 @@ class DiskLruCache( /** Used for asynchronous journal rebuilds. */ taskRunner: TaskRunner, ) : Closeable, Flushable { + lateinit var cacheLock: Closeable + internal val fileSystem: FileSystem = object : ForwardingFileSystem(fileSystem) { override fun sink( @@ -242,6 +245,8 @@ class DiskLruCache( civilizedFileSystem = fileSystem.isCivilized(journalFileBackup) + cacheLock = openLock(fileSystem, directory) + // Prefer to pick up where we left off. if (fileSystem.exists(journalFile)) { try { @@ -705,6 +710,8 @@ class DiskLruCache( return } + cacheLock.close() + // Copying for concurrent iteration. for (entry in lruEntries.values.toTypedArray()) { if (entry.currentEditor != null) { From b02738d07d657e54918707ce274d6d9e5e43c76d Mon Sep 17 00:00:00 2001 From: Yuri Schimke Date: Mon, 1 Apr 2024 16:59:36 +0100 Subject: [PATCH 02/37] Rework --- .../okhttp3/internal/cache/CacheLock.kt | 24 +++++++++++++++++-- 1 file changed, 22 insertions(+), 2 deletions(-) diff --git a/okhttp/src/main/kotlin/okhttp3/internal/cache/CacheLock.kt b/okhttp/src/main/kotlin/okhttp3/internal/cache/CacheLock.kt index 44f06c3985df..8924d32ee0eb 100644 --- a/okhttp/src/main/kotlin/okhttp3/internal/cache/CacheLock.kt +++ b/okhttp/src/main/kotlin/okhttp3/internal/cache/CacheLock.kt @@ -1,3 +1,18 @@ +/* + * Copyright (C) 2023 Block, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package okhttp3.internal.cache import android.annotation.SuppressLint @@ -7,6 +22,7 @@ import java.nio.file.StandardOpenOption import java.util.Collections import okhttp3.internal.platform.Platform import okio.FileSystem +import okio.IOException import okio.Path internal object CacheLock { @@ -14,7 +30,7 @@ internal object CacheLock { fun openLock(fileSystem: FileSystem, directory: Path): Closeable { return if (fileSystem == FileSystem.SYSTEM && !Platform.isAndroid) { - fileSystemLock(directory) + fileSystemLock(inMemoryLock(directory), directory) } else { inMemoryLock(directory) } @@ -33,11 +49,15 @@ internal object CacheLock { } @SuppressLint("NewApi") - fun fileSystemLock(directory: Path): Closeable { + fun fileSystemLock(memoryLock: Closeable, directory: Path): Closeable { val lockFile = directory / "lock" + lockFile.toFile().createNewFile() val channel = FileChannel.open(lockFile.toNioPath(), StandardOpenOption.APPEND) + channel.tryLock() ?: throw IOException("Cache $directory is locked by another process") + return okio.Closeable { + memoryLock.close() channel.close() } } From ac0cc7973934dd1f6e120085af4f878f8c59cd02 Mon Sep 17 00:00:00 2001 From: Yuri Schimke Date: Mon, 1 Apr 2024 17:01:40 +0100 Subject: [PATCH 03/37] Rework --- okhttp/src/main/kotlin/okhttp3/internal/cache/CacheLock.kt | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/okhttp/src/main/kotlin/okhttp3/internal/cache/CacheLock.kt b/okhttp/src/main/kotlin/okhttp3/internal/cache/CacheLock.kt index 8924d32ee0eb..42918905c9b6 100644 --- a/okhttp/src/main/kotlin/okhttp3/internal/cache/CacheLock.kt +++ b/okhttp/src/main/kotlin/okhttp3/internal/cache/CacheLock.kt @@ -37,6 +37,9 @@ internal object CacheLock { } @SuppressLint("NewApi") + /** + * Create an in-memory lock, avoiding two open Cache instances. + */ fun inMemoryLock(directory: Path): Closeable { // TODO solution for Android N? val existing = openCaches.putIfAbsent(directory, Exception("CacheLock($directory)")) @@ -49,6 +52,10 @@ internal object CacheLock { } @SuppressLint("NewApi") + /** + * Create a file system lock, that excludes other processes. However within the process a + * memory lock is also needed, since locks don't work within a single process. + */ fun fileSystemLock(memoryLock: Closeable, directory: Path): Closeable { val lockFile = directory / "lock" lockFile.toFile().createNewFile() From 2525edf09be8022e32e0215ac0bd7553881f7126 Mon Sep 17 00:00:00 2001 From: Yuri Schimke Date: Mon, 1 Apr 2024 17:30:33 +0100 Subject: [PATCH 04/37] Add test --- .../okhttp3/internal/cache/CacheLock.kt | 8 +- okhttp/src/test/java/okhttp3/CacheLockTest.kt | 84 +++++++++++++++++++ 2 files changed, 89 insertions(+), 3 deletions(-) create mode 100644 okhttp/src/test/java/okhttp3/CacheLockTest.kt diff --git a/okhttp/src/main/kotlin/okhttp3/internal/cache/CacheLock.kt b/okhttp/src/main/kotlin/okhttp3/internal/cache/CacheLock.kt index 42918905c9b6..d7e06c303762 100644 --- a/okhttp/src/main/kotlin/okhttp3/internal/cache/CacheLock.kt +++ b/okhttp/src/main/kotlin/okhttp3/internal/cache/CacheLock.kt @@ -1,5 +1,5 @@ /* - * Copyright (C) 2023 Block, Inc. + * Copyright (C) 2024 Block, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -44,7 +44,7 @@ internal object CacheLock { // TODO solution for Android N? val existing = openCaches.putIfAbsent(directory, Exception("CacheLock($directory)")) if (existing != null) { - throw IllegalStateException("Cache already open at '$directory'", existing) + throw IllegalStateException("Cache already open at '$directory' in same process", existing) } return okio.Closeable { openCaches.remove(directory) @@ -61,7 +61,9 @@ internal object CacheLock { lockFile.toFile().createNewFile() val channel = FileChannel.open(lockFile.toNioPath(), StandardOpenOption.APPEND) - channel.tryLock() ?: throw IOException("Cache $directory is locked by another process") + checkNotNull(channel.tryLock()) { + "Cache already open at '$directory' in another process" + } return okio.Closeable { memoryLock.close() diff --git a/okhttp/src/test/java/okhttp3/CacheLockTest.kt b/okhttp/src/test/java/okhttp3/CacheLockTest.kt new file mode 100644 index 000000000000..db5287c69a61 --- /dev/null +++ b/okhttp/src/test/java/okhttp3/CacheLockTest.kt @@ -0,0 +1,84 @@ +/* + * Copyright (C) 2024 Block, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package okhttp3 + +import assertk.assertThat +import assertk.assertions.isEqualTo +import java.nio.file.Path +import okio.Closeable +import okio.FileSystem +import okio.IOException +import okio.Path.Companion.toOkioPath +import org.junit.jupiter.api.AfterEach +import org.junit.jupiter.api.BeforeEach +import org.junit.jupiter.api.Test +import org.junit.jupiter.api.assertThrows +import org.junit.jupiter.api.io.TempDir + +class CacheLockTest { + + private lateinit var tempDir: okio.Path + private val toClose = mutableListOf() + + @BeforeEach + fun setup( + @TempDir tempDir: Path, + ) { + this.tempDir = tempDir.toOkioPath() + } + + @AfterEach + fun cleanup( + ) { + toClose.forEach { + it.close() + } + } + + @Test + fun testCacheLock() { + openCache(tempDir) + + val ioe = assertThrows { + openCache(tempDir) + } + assertThat(ioe.message).isEqualTo("Cache already open at '$tempDir' in same process") + } + + @Test + fun testCacheLockAfterClose() { + val cache1 = openCache(tempDir) + + cache1.close() + + openCache(tempDir) + } + + @Test + fun testCacheLockDifferentPath() { + openCache(tempDir / "a") + + openCache(tempDir / "b") + } + + private fun openCache(directory: okio.Path): Cache { + return Cache(directory, 10_000, FileSystem.SYSTEM).apply { + // force early LRU initialisation + initialize() + toClose.add(this) + } + } +} From 707235851db37457018dc805ef0ec283c7cb4a80 Mon Sep 17 00:00:00 2001 From: Yuri Schimke Date: Mon, 1 Apr 2024 17:30:56 +0100 Subject: [PATCH 05/37] Cleanup --- .../kotlin/okhttp3/internal/cache/CacheLock.kt | 15 ++++++++++----- okhttp/src/test/java/okhttp3/CacheLockTest.kt | 12 +++++------- 2 files changed, 15 insertions(+), 12 deletions(-) diff --git a/okhttp/src/main/kotlin/okhttp3/internal/cache/CacheLock.kt b/okhttp/src/main/kotlin/okhttp3/internal/cache/CacheLock.kt index d7e06c303762..364eafa4a74f 100644 --- a/okhttp/src/main/kotlin/okhttp3/internal/cache/CacheLock.kt +++ b/okhttp/src/main/kotlin/okhttp3/internal/cache/CacheLock.kt @@ -22,13 +22,15 @@ import java.nio.file.StandardOpenOption import java.util.Collections import okhttp3.internal.platform.Platform import okio.FileSystem -import okio.IOException import okio.Path internal object CacheLock { private val openCaches = Collections.synchronizedMap(mutableMapOf()) - fun openLock(fileSystem: FileSystem, directory: Path): Closeable { + fun openLock( + fileSystem: FileSystem, + directory: Path, + ): Closeable { return if (fileSystem == FileSystem.SYSTEM && !Platform.isAndroid) { fileSystemLock(inMemoryLock(directory), directory) } else { @@ -36,10 +38,10 @@ internal object CacheLock { } } - @SuppressLint("NewApi") /** * Create an in-memory lock, avoiding two open Cache instances. */ + @SuppressLint("NewApi") fun inMemoryLock(directory: Path): Closeable { // TODO solution for Android N? val existing = openCaches.putIfAbsent(directory, Exception("CacheLock($directory)")) @@ -51,12 +53,15 @@ internal object CacheLock { } } - @SuppressLint("NewApi") /** * Create a file system lock, that excludes other processes. However within the process a * memory lock is also needed, since locks don't work within a single process. */ - fun fileSystemLock(memoryLock: Closeable, directory: Path): Closeable { + @SuppressLint("NewApi") + fun fileSystemLock( + memoryLock: Closeable, + directory: Path, + ): Closeable { val lockFile = directory / "lock" lockFile.toFile().createNewFile() val channel = FileChannel.open(lockFile.toNioPath(), StandardOpenOption.APPEND) diff --git a/okhttp/src/test/java/okhttp3/CacheLockTest.kt b/okhttp/src/test/java/okhttp3/CacheLockTest.kt index db5287c69a61..71b225084969 100644 --- a/okhttp/src/test/java/okhttp3/CacheLockTest.kt +++ b/okhttp/src/test/java/okhttp3/CacheLockTest.kt @@ -20,7 +20,6 @@ import assertk.assertions.isEqualTo import java.nio.file.Path import okio.Closeable import okio.FileSystem -import okio.IOException import okio.Path.Companion.toOkioPath import org.junit.jupiter.api.AfterEach import org.junit.jupiter.api.BeforeEach @@ -29,7 +28,6 @@ import org.junit.jupiter.api.assertThrows import org.junit.jupiter.api.io.TempDir class CacheLockTest { - private lateinit var tempDir: okio.Path private val toClose = mutableListOf() @@ -41,8 +39,7 @@ class CacheLockTest { } @AfterEach - fun cleanup( - ) { + fun cleanup() { toClose.forEach { it.close() } @@ -52,9 +49,10 @@ class CacheLockTest { fun testCacheLock() { openCache(tempDir) - val ioe = assertThrows { - openCache(tempDir) - } + val ioe = + assertThrows { + openCache(tempDir) + } assertThat(ioe.message).isEqualTo("Cache already open at '$tempDir' in same process") } From 5ca131f5e9471b03a2bac235992a49fcd7e0f1c7 Mon Sep 17 00:00:00 2001 From: Yuri Schimke Date: Mon, 1 Apr 2024 18:42:53 +0100 Subject: [PATCH 06/37] Activate missed test --- .../src/androidTest/java/okhttp/android/test/OkHttpTest.kt | 1 + okhttp/src/main/kotlin/okhttp3/internal/cache/CacheLock.kt | 5 ++++- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/android-test/src/androidTest/java/okhttp/android/test/OkHttpTest.kt b/android-test/src/androidTest/java/okhttp/android/test/OkHttpTest.kt index 82625ab745ac..9f64b5fca8a6 100644 --- a/android-test/src/androidTest/java/okhttp/android/test/OkHttpTest.kt +++ b/android-test/src/androidTest/java/okhttp/android/test/OkHttpTest.kt @@ -859,6 +859,7 @@ class OkHttpTest { assertEquals(setOf(OkHttpTest::class.java.name), testHandler.calls.keys) } + @Test fun testCachedRequest() { enableTls() diff --git a/okhttp/src/main/kotlin/okhttp3/internal/cache/CacheLock.kt b/okhttp/src/main/kotlin/okhttp3/internal/cache/CacheLock.kt index 364eafa4a74f..f1c5bce9712b 100644 --- a/okhttp/src/main/kotlin/okhttp3/internal/cache/CacheLock.kt +++ b/okhttp/src/main/kotlin/okhttp3/internal/cache/CacheLock.kt @@ -16,6 +16,7 @@ package okhttp3.internal.cache import android.annotation.SuppressLint +import android.os.Build import java.io.Closeable import java.nio.channels.FileChannel import java.nio.file.StandardOpenOption @@ -23,6 +24,7 @@ import java.util.Collections import okhttp3.internal.platform.Platform import okio.FileSystem import okio.Path +import org.codehaus.mojo.animal_sniffer.IgnoreJRERequirement internal object CacheLock { private val openCaches = Collections.synchronizedMap(mutableMapOf()) @@ -42,8 +44,8 @@ internal object CacheLock { * Create an in-memory lock, avoiding two open Cache instances. */ @SuppressLint("NewApi") + @IgnoreJRERequirement // D8 supports put if absent fun inMemoryLock(directory: Path): Closeable { - // TODO solution for Android N? val existing = openCaches.putIfAbsent(directory, Exception("CacheLock($directory)")) if (existing != null) { throw IllegalStateException("Cache already open at '$directory' in same process", existing) @@ -58,6 +60,7 @@ internal object CacheLock { * memory lock is also needed, since locks don't work within a single process. */ @SuppressLint("NewApi") + @IgnoreJRERequirement // only called on JVM fun fileSystemLock( memoryLock: Closeable, directory: Path, From 0a2fbb6f0029a0123467bee694012762e03af5c0 Mon Sep 17 00:00:00 2001 From: Yuri Schimke Date: Mon, 1 Apr 2024 18:54:43 +0100 Subject: [PATCH 07/37] Cleanup --- okhttp/src/main/kotlin/okhttp3/internal/cache/CacheLock.kt | 1 - 1 file changed, 1 deletion(-) diff --git a/okhttp/src/main/kotlin/okhttp3/internal/cache/CacheLock.kt b/okhttp/src/main/kotlin/okhttp3/internal/cache/CacheLock.kt index f1c5bce9712b..c112fd4800e4 100644 --- a/okhttp/src/main/kotlin/okhttp3/internal/cache/CacheLock.kt +++ b/okhttp/src/main/kotlin/okhttp3/internal/cache/CacheLock.kt @@ -16,7 +16,6 @@ package okhttp3.internal.cache import android.annotation.SuppressLint -import android.os.Build import java.io.Closeable import java.nio.channels.FileChannel import java.nio.file.StandardOpenOption From c9db9d40946b8c6c4356c20b35a87d4ad5ee07a7 Mon Sep 17 00:00:00 2001 From: Yuri Schimke Date: Mon, 1 Apr 2024 19:25:48 +0100 Subject: [PATCH 08/37] Avoid checking on non system file systems --- okhttp/src/main/kotlin/okhttp3/internal/cache/CacheLock.kt | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/okhttp/src/main/kotlin/okhttp3/internal/cache/CacheLock.kt b/okhttp/src/main/kotlin/okhttp3/internal/cache/CacheLock.kt index c112fd4800e4..8ad1a92c527c 100644 --- a/okhttp/src/main/kotlin/okhttp3/internal/cache/CacheLock.kt +++ b/okhttp/src/main/kotlin/okhttp3/internal/cache/CacheLock.kt @@ -32,7 +32,11 @@ internal object CacheLock { fileSystem: FileSystem, directory: Path, ): Closeable { - return if (fileSystem == FileSystem.SYSTEM && !Platform.isAndroid) { + if (fileSystem != FileSystem.SYSTEM) { + return Closeable {} + } + + return if (!Platform.isAndroid) { fileSystemLock(inMemoryLock(directory), directory) } else { inMemoryLock(directory) From 97e566b94f4eaf9c6266eda0ecdef48e51834dc2 Mon Sep 17 00:00:00 2001 From: Yuri Schimke Date: Mon, 1 Apr 2024 19:27:05 +0100 Subject: [PATCH 09/37] Revert "Avoid checking on non system file systems" This reverts commit c9db9d40946b8c6c4356c20b35a87d4ad5ee07a7. --- okhttp/src/main/kotlin/okhttp3/internal/cache/CacheLock.kt | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/okhttp/src/main/kotlin/okhttp3/internal/cache/CacheLock.kt b/okhttp/src/main/kotlin/okhttp3/internal/cache/CacheLock.kt index 8ad1a92c527c..c112fd4800e4 100644 --- a/okhttp/src/main/kotlin/okhttp3/internal/cache/CacheLock.kt +++ b/okhttp/src/main/kotlin/okhttp3/internal/cache/CacheLock.kt @@ -32,11 +32,7 @@ internal object CacheLock { fileSystem: FileSystem, directory: Path, ): Closeable { - if (fileSystem != FileSystem.SYSTEM) { - return Closeable {} - } - - return if (!Platform.isAndroid) { + return if (fileSystem == FileSystem.SYSTEM && !Platform.isAndroid) { fileSystemLock(inMemoryLock(directory), directory) } else { inMemoryLock(directory) From ee1daa68d333a9ae48a9e32c1ceac878a8f359b4 Mon Sep 17 00:00:00 2001 From: Yuri Schimke Date: Mon, 1 Apr 2024 19:28:28 +0100 Subject: [PATCH 10/37] Avoid failing on dns tests --- .../test/java/okhttp3/dnsoverhttps/DnsOverHttpsTest.kt | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/okhttp-dnsoverhttps/src/test/java/okhttp3/dnsoverhttps/DnsOverHttpsTest.kt b/okhttp-dnsoverhttps/src/test/java/okhttp3/dnsoverhttps/DnsOverHttpsTest.kt index 78bbaaac78fc..7636ae8612a1 100644 --- a/okhttp-dnsoverhttps/src/test/java/okhttp3/dnsoverhttps/DnsOverHttpsTest.kt +++ b/okhttp-dnsoverhttps/src/test/java/okhttp3/dnsoverhttps/DnsOverHttpsTest.kt @@ -170,7 +170,7 @@ class DnsOverHttpsTest { // 5. unsuccessful response @Test fun usesCache() { - val cache = Cache("cache".toPath(), (100 * 1024).toLong(), cacheFs) + val cache = Cache("cache-usesCache".toPath(), (100 * 1024).toLong(), cacheFs) val cachedClient = bootstrapClient.newBuilder().cache(cache).build() val cachedDns = buildLocalhost(cachedClient, false) @@ -208,7 +208,7 @@ class DnsOverHttpsTest { @Test fun usesCacheEvenForPost() { - val cache = Cache("cache".toPath(), (100 * 1024).toLong(), cacheFs) + val cache = Cache("cache-usesCacheEvenForPost".toPath(), (100 * 1024).toLong(), cacheFs) val cachedClient = bootstrapClient.newBuilder().cache(cache).build() val cachedDns = buildLocalhost(cachedClient, false, post = true) repeat(2) { @@ -241,6 +241,8 @@ class DnsOverHttpsTest { assertThat(recordedRequest.method).isEqualTo("POST") assertThat(recordedRequest.path) .isEqualTo("/lookup?ct") + + cache.close() } @Test @@ -282,6 +284,8 @@ class DnsOverHttpsTest { assertThat(recordedRequest!!.method).isEqualTo("GET") assertThat(recordedRequest.path) .isEqualTo("/lookup?ct&dns=AAABAAABAAAAAAAABmdvb2dsZQNjb20AAAEAAQ") + + cache.close() } private fun dnsResponse(s: String): MockResponse { From 3041258e4a9c3df14d269c000ab73aef3c65d9d7 Mon Sep 17 00:00:00 2001 From: Yuri Schimke Date: Tue, 2 Apr 2024 22:08:44 +0100 Subject: [PATCH 11/37] Adding a test --- .../okhttp3/internal/cache/CacheLock.kt | 1 + okhttp/src/test/java/okhttp3/CacheLockTest.kt | 31 +++++++++++++++++++ .../test/java/okhttp3/LockTestProgram.java | 22 +++++++++++++ 3 files changed, 54 insertions(+) create mode 100644 okhttp/src/test/java/okhttp3/LockTestProgram.java diff --git a/okhttp/src/main/kotlin/okhttp3/internal/cache/CacheLock.kt b/okhttp/src/main/kotlin/okhttp3/internal/cache/CacheLock.kt index c112fd4800e4..6a01d8e4e532 100644 --- a/okhttp/src/main/kotlin/okhttp3/internal/cache/CacheLock.kt +++ b/okhttp/src/main/kotlin/okhttp3/internal/cache/CacheLock.kt @@ -65,6 +65,7 @@ internal object CacheLock { directory: Path, ): Closeable { val lockFile = directory / "lock" + println("Locking $lockFile") lockFile.toFile().createNewFile() val channel = FileChannel.open(lockFile.toNioPath(), StandardOpenOption.APPEND) diff --git a/okhttp/src/test/java/okhttp3/CacheLockTest.kt b/okhttp/src/test/java/okhttp3/CacheLockTest.kt index 71b225084969..9828511e436f 100644 --- a/okhttp/src/test/java/okhttp3/CacheLockTest.kt +++ b/okhttp/src/test/java/okhttp3/CacheLockTest.kt @@ -27,6 +27,7 @@ import org.junit.jupiter.api.Test import org.junit.jupiter.api.assertThrows import org.junit.jupiter.api.io.TempDir +@org.junit.jupiter.api.parallel.Isolated class CacheLockTest { private lateinit var tempDir: okio.Path private val toClose = mutableListOf() @@ -72,6 +73,36 @@ class CacheLockTest { openCache(tempDir / "b") } + @Test + fun testCacheLockDifferentProcess() { + val lockFile = tempDir / "lock" + lockFile.toFile().createNewFile() + + val process = + ProcessBuilder().command("java", "src/test/java/okhttp3/LockTestProgram.java", (lockFile.toString())) + .redirectErrorStream(true) + .start() + + println(1) + val output = process.inputStream.bufferedReader() + + try { + println(2) + assertThat(output.readLine()).isEqualTo("Locking $lockFile") + assertThat(output.readLine()).isEqualTo("Locked $lockFile") + + println(3) + val ioe = + assertThrows { + openCache(tempDir) + } + assertThat(ioe.message).isEqualTo("Cache already open at '$tempDir' in same process") + } finally { + process.destroy() + println(process.outputStream) + } + } + private fun openCache(directory: okio.Path): Cache { return Cache(directory, 10_000, FileSystem.SYSTEM).apply { // force early LRU initialisation diff --git a/okhttp/src/test/java/okhttp3/LockTestProgram.java b/okhttp/src/test/java/okhttp3/LockTestProgram.java new file mode 100644 index 000000000000..19f4a154e6e6 --- /dev/null +++ b/okhttp/src/test/java/okhttp3/LockTestProgram.java @@ -0,0 +1,22 @@ +package okhttp3; + +import java.io.File; +import java.io.IOException; +import java.nio.channels.FileChannel; +import java.nio.file.StandardOpenOption; + +public class LockTestProgram { + public static void main(String[] args) throws IOException { + File lockFile = new File(args[0]); + + System.out.println("Locking " + lockFile); + + FileChannel channel = FileChannel.open(lockFile.toPath(), StandardOpenOption.APPEND); + + channel.lock(); + + System.out.println("Locked " + lockFile); + + System.in.read(); + } +} From 7faba3ea5037f9f4dd48f073c2544359b3762b39 Mon Sep 17 00:00:00 2001 From: Yuri Schimke Date: Tue, 2 Apr 2024 22:33:44 +0100 Subject: [PATCH 12/37] Fixes --- .../okhttp3/internal/cache/CacheLock.kt | 2 +- okhttp/src/test/java/okhttp3/CacheLockTest.kt | 25 ++++++++++++++----- 2 files changed, 20 insertions(+), 7 deletions(-) diff --git a/okhttp/src/main/kotlin/okhttp3/internal/cache/CacheLock.kt b/okhttp/src/main/kotlin/okhttp3/internal/cache/CacheLock.kt index 6a01d8e4e532..94fe0ae996e4 100644 --- a/okhttp/src/main/kotlin/okhttp3/internal/cache/CacheLock.kt +++ b/okhttp/src/main/kotlin/okhttp3/internal/cache/CacheLock.kt @@ -32,7 +32,7 @@ internal object CacheLock { fileSystem: FileSystem, directory: Path, ): Closeable { - return if (fileSystem == FileSystem.SYSTEM && !Platform.isAndroid) { + return if (!Platform.isAndroid) { fileSystemLock(inMemoryLock(directory), directory) } else { inMemoryLock(directory) diff --git a/okhttp/src/test/java/okhttp3/CacheLockTest.kt b/okhttp/src/test/java/okhttp3/CacheLockTest.kt index 9828511e436f..119b5c3f77b0 100644 --- a/okhttp/src/test/java/okhttp3/CacheLockTest.kt +++ b/okhttp/src/test/java/okhttp3/CacheLockTest.kt @@ -18,17 +18,25 @@ package okhttp3 import assertk.assertThat import assertk.assertions.isEqualTo import java.nio.file.Path +import okhttp3.testing.PlatformRule +import okhttp3.testing.PlatformVersion import okio.Closeable import okio.FileSystem import okio.Path.Companion.toOkioPath +import okio.Path.Companion.toPath import org.junit.jupiter.api.AfterEach import org.junit.jupiter.api.BeforeEach import org.junit.jupiter.api.Test import org.junit.jupiter.api.assertThrows +import org.junit.jupiter.api.extension.RegisterExtension import org.junit.jupiter.api.io.TempDir @org.junit.jupiter.api.parallel.Isolated class CacheLockTest { + @JvmField + @RegisterExtension + val platform = PlatformRule() + private lateinit var tempDir: okio.Path private val toClose = mutableListOf() @@ -78,28 +86,33 @@ class CacheLockTest { val lockFile = tempDir / "lock" lockFile.toFile().createNewFile() + val javaExe = if (PlatformVersion.majorVersion >= 9) { + @Suppress("Since15") + ProcessHandle.current().info().command().get().toPath() + } else { + System.getenv("JAVA_HOME").toPath() / "bin/java" + } + + assertThat(FileSystem.SYSTEM.exists(javaExe)) + val process = - ProcessBuilder().command("java", "src/test/java/okhttp3/LockTestProgram.java", (lockFile.toString())) + ProcessBuilder().command(javaExe.toString(), "src/test/java/okhttp3/LockTestProgram.java", (lockFile.toString())) .redirectErrorStream(true) .start() - println(1) val output = process.inputStream.bufferedReader() try { - println(2) assertThat(output.readLine()).isEqualTo("Locking $lockFile") assertThat(output.readLine()).isEqualTo("Locked $lockFile") - println(3) val ioe = assertThrows { openCache(tempDir) } - assertThat(ioe.message).isEqualTo("Cache already open at '$tempDir' in same process") + assertThat(ioe.message).isEqualTo("Cache already open at '$tempDir' in another process") } finally { process.destroy() - println(process.outputStream) } } From 44d3640fddc8c3c180c6c9182bbb379579b7e4bc Mon Sep 17 00:00:00 2001 From: Yuri Schimke Date: Wed, 3 Apr 2024 06:57:13 +0100 Subject: [PATCH 13/37] Test on more platforms --- okhttp/build.gradle.kts | 1 + .../okhttp3/internal/cache/CacheLock.kt | 4 ++-- okhttp/src/test/java/okhttp3/CacheLockTest.kt | 19 ++++++++++++++----- 3 files changed, 17 insertions(+), 7 deletions(-) diff --git a/okhttp/build.gradle.kts b/okhttp/build.gradle.kts index 5c3f2fb4e8df..0849d248d00e 100644 --- a/okhttp/build.gradle.kts +++ b/okhttp/build.gradle.kts @@ -131,6 +131,7 @@ dependencies { testImplementation(projects.okhttpSse) testImplementation(projects.okhttpCoroutines) testImplementation(libs.kotlinx.coroutines.core) + testImplementation(libs.kotlinx.coroutines.test) testImplementation(libs.squareup.moshi) testImplementation(libs.squareup.moshi.kotlin) testImplementation(libs.squareup.okio.fakefilesystem) diff --git a/okhttp/src/main/kotlin/okhttp3/internal/cache/CacheLock.kt b/okhttp/src/main/kotlin/okhttp3/internal/cache/CacheLock.kt index 94fe0ae996e4..169ef860c9d9 100644 --- a/okhttp/src/main/kotlin/okhttp3/internal/cache/CacheLock.kt +++ b/okhttp/src/main/kotlin/okhttp3/internal/cache/CacheLock.kt @@ -32,7 +32,8 @@ internal object CacheLock { fileSystem: FileSystem, directory: Path, ): Closeable { - return if (!Platform.isAndroid) { + val useFileSystemLock = true //!Platform.isAndroid + return if (useFileSystemLock) { fileSystemLock(inMemoryLock(directory), directory) } else { inMemoryLock(directory) @@ -65,7 +66,6 @@ internal object CacheLock { directory: Path, ): Closeable { val lockFile = directory / "lock" - println("Locking $lockFile") lockFile.toFile().createNewFile() val channel = FileChannel.open(lockFile.toNioPath(), StandardOpenOption.APPEND) diff --git a/okhttp/src/test/java/okhttp3/CacheLockTest.kt b/okhttp/src/test/java/okhttp3/CacheLockTest.kt index 119b5c3f77b0..0be949f2d5f4 100644 --- a/okhttp/src/test/java/okhttp3/CacheLockTest.kt +++ b/okhttp/src/test/java/okhttp3/CacheLockTest.kt @@ -18,6 +18,9 @@ package okhttp3 import assertk.assertThat import assertk.assertions.isEqualTo import java.nio.file.Path +import kotlin.time.Duration.Companion.seconds +import kotlinx.coroutines.runBlocking +import kotlinx.coroutines.withTimeout import okhttp3.testing.PlatformRule import okhttp3.testing.PlatformVersion import okio.Closeable @@ -25,6 +28,8 @@ import okio.FileSystem import okio.Path.Companion.toOkioPath import okio.Path.Companion.toPath import org.junit.jupiter.api.AfterEach +import org.junit.jupiter.api.Assumptions +import org.junit.jupiter.api.Assumptions.assumeTrue import org.junit.jupiter.api.BeforeEach import org.junit.jupiter.api.Test import org.junit.jupiter.api.assertThrows @@ -82,7 +87,11 @@ class CacheLockTest { } @Test - fun testCacheLockDifferentProcess() { + fun testCacheLockDifferentProcess() = runBlocking { + // No java command to execute LockTestProgram.java + platform.assumeNotAndroid() + assumeTrue(PlatformVersion.majorVersion >= 11) + val lockFile = tempDir / "lock" lockFile.toFile().createNewFile() @@ -93,8 +102,6 @@ class CacheLockTest { System.getenv("JAVA_HOME").toPath() / "bin/java" } - assertThat(FileSystem.SYSTEM.exists(javaExe)) - val process = ProcessBuilder().command(javaExe.toString(), "src/test/java/okhttp3/LockTestProgram.java", (lockFile.toString())) .redirectErrorStream(true) @@ -103,8 +110,10 @@ class CacheLockTest { val output = process.inputStream.bufferedReader() try { - assertThat(output.readLine()).isEqualTo("Locking $lockFile") - assertThat(output.readLine()).isEqualTo("Locked $lockFile") + withTimeout(5.seconds) { + assertThat(output.readLine()).isEqualTo("Locking $lockFile") + assertThat(output.readLine()).isEqualTo("Locked $lockFile") + } val ioe = assertThrows { From 792b6897040f2900772398b1db84abf03b8d843d Mon Sep 17 00:00:00 2001 From: Yuri Schimke Date: Wed, 3 Apr 2024 07:00:56 +0100 Subject: [PATCH 14/37] Cleanup --- .../okhttp3/internal/cache/CacheLock.kt | 3 +- okhttp/src/test/java/okhttp3/CacheLockTest.kt | 67 ++++++++++--------- .../test/java/okhttp3/LockTestProgram.java | 18 +++++ 3 files changed, 53 insertions(+), 35 deletions(-) diff --git a/okhttp/src/main/kotlin/okhttp3/internal/cache/CacheLock.kt b/okhttp/src/main/kotlin/okhttp3/internal/cache/CacheLock.kt index 169ef860c9d9..0aaa4830aa8e 100644 --- a/okhttp/src/main/kotlin/okhttp3/internal/cache/CacheLock.kt +++ b/okhttp/src/main/kotlin/okhttp3/internal/cache/CacheLock.kt @@ -20,7 +20,6 @@ import java.io.Closeable import java.nio.channels.FileChannel import java.nio.file.StandardOpenOption import java.util.Collections -import okhttp3.internal.platform.Platform import okio.FileSystem import okio.Path import org.codehaus.mojo.animal_sniffer.IgnoreJRERequirement @@ -32,7 +31,7 @@ internal object CacheLock { fileSystem: FileSystem, directory: Path, ): Closeable { - val useFileSystemLock = true //!Platform.isAndroid + val useFileSystemLock = true // !Platform.isAndroid return if (useFileSystemLock) { fileSystemLock(inMemoryLock(directory), directory) } else { diff --git a/okhttp/src/test/java/okhttp3/CacheLockTest.kt b/okhttp/src/test/java/okhttp3/CacheLockTest.kt index 0be949f2d5f4..ccae9fb4c862 100644 --- a/okhttp/src/test/java/okhttp3/CacheLockTest.kt +++ b/okhttp/src/test/java/okhttp3/CacheLockTest.kt @@ -28,7 +28,6 @@ import okio.FileSystem import okio.Path.Companion.toOkioPath import okio.Path.Companion.toPath import org.junit.jupiter.api.AfterEach -import org.junit.jupiter.api.Assumptions import org.junit.jupiter.api.Assumptions.assumeTrue import org.junit.jupiter.api.BeforeEach import org.junit.jupiter.api.Test @@ -87,43 +86,45 @@ class CacheLockTest { } @Test - fun testCacheLockDifferentProcess() = runBlocking { - // No java command to execute LockTestProgram.java - platform.assumeNotAndroid() - assumeTrue(PlatformVersion.majorVersion >= 11) - - val lockFile = tempDir / "lock" - lockFile.toFile().createNewFile() - - val javaExe = if (PlatformVersion.majorVersion >= 9) { - @Suppress("Since15") - ProcessHandle.current().info().command().get().toPath() - } else { - System.getenv("JAVA_HOME").toPath() / "bin/java" - } - - val process = - ProcessBuilder().command(javaExe.toString(), "src/test/java/okhttp3/LockTestProgram.java", (lockFile.toString())) - .redirectErrorStream(true) - .start() + fun testCacheLockDifferentProcess() = + runBlocking { + // No java command to execute LockTestProgram.java + platform.assumeNotAndroid() + assumeTrue(PlatformVersion.majorVersion >= 11) + + val lockFile = tempDir / "lock" + lockFile.toFile().createNewFile() + + val javaExe = + if (PlatformVersion.majorVersion >= 9) { + @Suppress("Since15") + ProcessHandle.current().info().command().get().toPath() + } else { + System.getenv("JAVA_HOME").toPath() / "bin/java" + } - val output = process.inputStream.bufferedReader() + val process = + ProcessBuilder().command(javaExe.toString(), "src/test/java/okhttp3/LockTestProgram.java", (lockFile.toString())) + .redirectErrorStream(true) + .start() - try { - withTimeout(5.seconds) { - assertThat(output.readLine()).isEqualTo("Locking $lockFile") - assertThat(output.readLine()).isEqualTo("Locked $lockFile") - } + val output = process.inputStream.bufferedReader() - val ioe = - assertThrows { - openCache(tempDir) + try { + withTimeout(5.seconds) { + assertThat(output.readLine()).isEqualTo("Locking $lockFile") + assertThat(output.readLine()).isEqualTo("Locked $lockFile") } - assertThat(ioe.message).isEqualTo("Cache already open at '$tempDir' in another process") - } finally { - process.destroy() + + val ioe = + assertThrows { + openCache(tempDir) + } + assertThat(ioe.message).isEqualTo("Cache already open at '$tempDir' in another process") + } finally { + process.destroy() + } } - } private fun openCache(directory: okio.Path): Cache { return Cache(directory, 10_000, FileSystem.SYSTEM).apply { diff --git a/okhttp/src/test/java/okhttp3/LockTestProgram.java b/okhttp/src/test/java/okhttp3/LockTestProgram.java index 19f4a154e6e6..aa3f25a6e770 100644 --- a/okhttp/src/test/java/okhttp3/LockTestProgram.java +++ b/okhttp/src/test/java/okhttp3/LockTestProgram.java @@ -1,3 +1,18 @@ +/* + * Copyright (C) 2024 Block, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package okhttp3; import java.io.File; @@ -5,6 +20,9 @@ import java.nio.channels.FileChannel; import java.nio.file.StandardOpenOption; +/** + * Used in CacheLockTest via Java command. + */ public class LockTestProgram { public static void main(String[] args) throws IOException { File lockFile = new File(args[0]); From a922b1d0d61048a660f1152a9ce4d5480bb15992 Mon Sep 17 00:00:00 2001 From: Yuri Schimke Date: Wed, 3 Apr 2024 07:17:33 +0100 Subject: [PATCH 15/37] Cleanup --- okhttp/src/main/kotlin/okhttp3/internal/cache/CacheLock.kt | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/okhttp/src/main/kotlin/okhttp3/internal/cache/CacheLock.kt b/okhttp/src/main/kotlin/okhttp3/internal/cache/CacheLock.kt index 0aaa4830aa8e..f3a7cba2ceac 100644 --- a/okhttp/src/main/kotlin/okhttp3/internal/cache/CacheLock.kt +++ b/okhttp/src/main/kotlin/okhttp3/internal/cache/CacheLock.kt @@ -31,7 +31,8 @@ internal object CacheLock { fileSystem: FileSystem, directory: Path, ): Closeable { - val useFileSystemLock = true // !Platform.isAndroid + // TODO solve this + val useFileSystemLock = !fileSystem.toString().contains("FakeFileSystem") // !Platform.isAndroid return if (useFileSystemLock) { fileSystemLock(inMemoryLock(directory), directory) } else { From e24e146275e05afb476456adebebac23d9ee2694 Mon Sep 17 00:00:00 2001 From: Yuri Schimke Date: Sat, 6 Apr 2024 09:05:26 +0100 Subject: [PATCH 16/37] Fixes --- .../okhttp3/internal/cache/CacheLock.kt | 47 ++++++++++++++----- .../okhttp3/internal/cache/DiskLruCache.kt | 2 +- okhttp/src/test/java/okhttp3/CacheLockTest.kt | 30 ++++++++---- 3 files changed, 55 insertions(+), 24 deletions(-) diff --git a/okhttp/src/main/kotlin/okhttp3/internal/cache/CacheLock.kt b/okhttp/src/main/kotlin/okhttp3/internal/cache/CacheLock.kt index f3a7cba2ceac..024a45410989 100644 --- a/okhttp/src/main/kotlin/okhttp3/internal/cache/CacheLock.kt +++ b/okhttp/src/main/kotlin/okhttp3/internal/cache/CacheLock.kt @@ -17,6 +17,7 @@ package okhttp3.internal.cache import android.annotation.SuppressLint import java.io.Closeable +import java.io.File import java.nio.channels.FileChannel import java.nio.file.StandardOpenOption import java.util.Collections @@ -28,16 +29,36 @@ internal object CacheLock { private val openCaches = Collections.synchronizedMap(mutableMapOf()) fun openLock( - fileSystem: FileSystem, directory: Path, ): Closeable { - // TODO solve this - val useFileSystemLock = !fileSystem.toString().contains("FakeFileSystem") // !Platform.isAndroid - return if (useFileSystemLock) { - fileSystemLock(inMemoryLock(directory), directory) - } else { - inMemoryLock(directory) + val memoryLock = inMemoryLock(directory) + + // check if possibly a non System file system + if (FileSystem.SYSTEM.exists(directory)) { + try { + val fileSystemLock = fileSystemLock(directory) + + return Closeable { + memoryLock.close() + fileSystemLock.close() + } + } catch (e: Exception) { + if (fileSystemSupportsLock()) { + memoryLock.close() + throw e + } + } } + + return memoryLock + } + + internal fun fileSystemSupportsLock(): Boolean { + val tmpLockFile = File.createTempFile("test-", ".lock") + + val channel = FileChannel.open(tmpLockFile.toPath(), StandardOpenOption.APPEND) + + return channel.tryLock().apply { close() } != null } /** @@ -48,7 +69,7 @@ internal object CacheLock { fun inMemoryLock(directory: Path): Closeable { val existing = openCaches.putIfAbsent(directory, Exception("CacheLock($directory)")) if (existing != null) { - throw IllegalStateException("Cache already open at '$directory' in same process", existing) + throw LockException("Cache already open at '$directory' in same process", existing) } return okio.Closeable { openCaches.remove(directory) @@ -62,20 +83,20 @@ internal object CacheLock { @SuppressLint("NewApi") @IgnoreJRERequirement // only called on JVM fun fileSystemLock( - memoryLock: Closeable, directory: Path, ): Closeable { + // update once https://github.com/square/okio/issues/1464 is available + val lockFile = directory / "lock" lockFile.toFile().createNewFile() val channel = FileChannel.open(lockFile.toNioPath(), StandardOpenOption.APPEND) - checkNotNull(channel.tryLock()) { - "Cache already open at '$directory' in another process" - } + channel.tryLock() ?: throw LockException("Cache already open at '$directory' in another process") return okio.Closeable { - memoryLock.close() channel.close() } } } + +class LockException(message: String, cause: Exception? = null) : Exception(message, cause) diff --git a/okhttp/src/main/kotlin/okhttp3/internal/cache/DiskLruCache.kt b/okhttp/src/main/kotlin/okhttp3/internal/cache/DiskLruCache.kt index 605eaf233314..0e3f5c199246 100644 --- a/okhttp/src/main/kotlin/okhttp3/internal/cache/DiskLruCache.kt +++ b/okhttp/src/main/kotlin/okhttp3/internal/cache/DiskLruCache.kt @@ -245,7 +245,7 @@ class DiskLruCache( civilizedFileSystem = fileSystem.isCivilized(journalFileBackup) - cacheLock = openLock(fileSystem, directory) + cacheLock = openLock(directory) // Prefer to pick up where we left off. if (fileSystem.exists(journalFile)) { diff --git a/okhttp/src/test/java/okhttp3/CacheLockTest.kt b/okhttp/src/test/java/okhttp3/CacheLockTest.kt index ccae9fb4c862..8478e1035995 100644 --- a/okhttp/src/test/java/okhttp3/CacheLockTest.kt +++ b/okhttp/src/test/java/okhttp3/CacheLockTest.kt @@ -19,8 +19,10 @@ import assertk.assertThat import assertk.assertions.isEqualTo import java.nio.file.Path import kotlin.time.Duration.Companion.seconds +import kotlinx.coroutines.delay import kotlinx.coroutines.runBlocking import kotlinx.coroutines.withTimeout +import okhttp3.internal.cache.LockException import okhttp3.testing.PlatformRule import okhttp3.testing.PlatformVersion import okio.Closeable @@ -62,11 +64,10 @@ class CacheLockTest { fun testCacheLock() { openCache(tempDir) - val ioe = - assertThrows { + val lockException = assertThrows { openCache(tempDir) } - assertThat(ioe.message).isEqualTo("Cache already open at '$tempDir' in same process") + assertThat(lockException.message).isEqualTo("Cache already open at '$tempDir' in same process") } @Test @@ -86,7 +87,7 @@ class CacheLockTest { } @Test - fun testCacheLockDifferentProcess() = + fun testCacheLockDifferentProcess() { runBlocking { // No java command to execute LockTestProgram.java platform.assumeNotAndroid() @@ -104,7 +105,11 @@ class CacheLockTest { } val process = - ProcessBuilder().command(javaExe.toString(), "src/test/java/okhttp3/LockTestProgram.java", (lockFile.toString())) + ProcessBuilder().command( + javaExe.toString(), + "src/test/java/okhttp3/LockTestProgram.java", + (lockFile.toString()) + ) .redirectErrorStream(true) .start() @@ -116,15 +121,20 @@ class CacheLockTest { assertThat(output.readLine()).isEqualTo("Locked $lockFile") } - val ioe = - assertThrows { - openCache(tempDir) - } - assertThat(ioe.message).isEqualTo("Cache already open at '$tempDir' in another process") + val lockException = assertThrows { + openCache(tempDir) + } + assertThat(lockException.message).isEqualTo("Cache already open at '$tempDir' in another process") } finally { process.destroy() } + + delay(100) + + // Should work again once process is killed + openCache(tempDir) } + } private fun openCache(directory: okio.Path): Cache { return Cache(directory, 10_000, FileSystem.SYSTEM).apply { From 04c057ff606a6ede48c6f2faa96a323b7f0f2559 Mon Sep 17 00:00:00 2001 From: Yuri Schimke Date: Sat, 6 Apr 2024 09:13:07 +0100 Subject: [PATCH 17/37] Fixes --- .../okhttp3/internal/cache/CacheLock.kt | 10 +++---- okhttp/src/test/java/okhttp3/CacheLockTest.kt | 12 +++++---- okhttp/src/test/java/okhttp3/CacheTest.kt | 26 +++++++++---------- 3 files changed, 23 insertions(+), 25 deletions(-) diff --git a/okhttp/src/main/kotlin/okhttp3/internal/cache/CacheLock.kt b/okhttp/src/main/kotlin/okhttp3/internal/cache/CacheLock.kt index 024a45410989..60489a7f417c 100644 --- a/okhttp/src/main/kotlin/okhttp3/internal/cache/CacheLock.kt +++ b/okhttp/src/main/kotlin/okhttp3/internal/cache/CacheLock.kt @@ -28,9 +28,7 @@ import org.codehaus.mojo.animal_sniffer.IgnoreJRERequirement internal object CacheLock { private val openCaches = Collections.synchronizedMap(mutableMapOf()) - fun openLock( - directory: Path, - ): Closeable { + fun openLock(directory: Path): Closeable { val memoryLock = inMemoryLock(directory) // check if possibly a non System file system @@ -67,7 +65,7 @@ internal object CacheLock { @SuppressLint("NewApi") @IgnoreJRERequirement // D8 supports put if absent fun inMemoryLock(directory: Path): Closeable { - val existing = openCaches.putIfAbsent(directory, Exception("CacheLock($directory)")) + val existing = openCaches.putIfAbsent(directory, LockException("Existing CacheLock($directory) opened at")) if (existing != null) { throw LockException("Cache already open at '$directory' in same process", existing) } @@ -82,9 +80,7 @@ internal object CacheLock { */ @SuppressLint("NewApi") @IgnoreJRERequirement // only called on JVM - fun fileSystemLock( - directory: Path, - ): Closeable { + fun fileSystemLock(directory: Path): Closeable { // update once https://github.com/square/okio/issues/1464 is available val lockFile = directory / "lock" diff --git a/okhttp/src/test/java/okhttp3/CacheLockTest.kt b/okhttp/src/test/java/okhttp3/CacheLockTest.kt index 8478e1035995..8e826f5f7b69 100644 --- a/okhttp/src/test/java/okhttp3/CacheLockTest.kt +++ b/okhttp/src/test/java/okhttp3/CacheLockTest.kt @@ -64,7 +64,8 @@ class CacheLockTest { fun testCacheLock() { openCache(tempDir) - val lockException = assertThrows { + val lockException = + assertThrows { openCache(tempDir) } assertThat(lockException.message).isEqualTo("Cache already open at '$tempDir' in same process") @@ -108,7 +109,7 @@ class CacheLockTest { ProcessBuilder().command( javaExe.toString(), "src/test/java/okhttp3/LockTestProgram.java", - (lockFile.toString()) + (lockFile.toString()), ) .redirectErrorStream(true) .start() @@ -121,9 +122,10 @@ class CacheLockTest { assertThat(output.readLine()).isEqualTo("Locked $lockFile") } - val lockException = assertThrows { - openCache(tempDir) - } + val lockException = + assertThrows { + openCache(tempDir) + } assertThat(lockException.message).isEqualTo("Cache already open at '$tempDir' in another process") } finally { process.destroy() diff --git a/okhttp/src/test/java/okhttp3/CacheTest.kt b/okhttp/src/test/java/okhttp3/CacheTest.kt index 707f0fcebd95..21825477b36c 100644 --- a/okhttp/src/test/java/okhttp3/CacheTest.kt +++ b/okhttp/src/test/java/okhttp3/CacheTest.kt @@ -92,7 +92,7 @@ class CacheTest { platform.assumeNotOpenJSSE() server.protocolNegotiationEnabled = false fileSystem.emulateUnix() - cache = Cache("/cache/".toPath(), Long.MAX_VALUE, fileSystem) + cache = Cache("/cache".toPath(), Long.MAX_VALUE, fileSystem) client = clientTestRule.newClientBuilder() .cache(cache) @@ -3495,22 +3495,22 @@ CLEAN $urlKey ${entryMetadata.length} ${entryBody.length} return path } } - val path: Path = "/cache".toPath() + val path: Path = "/cache2".toPath() val c = Cache(path, 100000L, loggingFileSystem) assertThat(c.directoryPath).isEqualTo(path) c.size() assertThat(events).containsExactly( - "metadataOrNull:/cache/journal.bkp", - "metadataOrNull:/cache", - "sink:/cache/journal.bkp", - "delete:/cache/journal.bkp", - "metadataOrNull:/cache/journal", - "metadataOrNull:/cache", - "sink:/cache/journal.tmp", - "metadataOrNull:/cache/journal", - "atomicMove:/cache/journal.tmp", - "atomicMove:/cache/journal", - "appendingSink:/cache/journal", + "metadataOrNull:/cache2/journal.bkp", + "metadataOrNull:/cache2", + "sink:/cache2/journal.bkp", + "delete:/cache2/journal.bkp", + "metadataOrNull:/cache2/journal", + "metadataOrNull:/cache2", + "sink:/cache2/journal.tmp", + "metadataOrNull:/cache2/journal", + "atomicMove:/cache2/journal.tmp", + "atomicMove:/cache2/journal", + "appendingSink:/cache2/journal", ) events.clear() c.size() From 5c6c60da55178560c1fde9daf42d1a6f0ccb9d1e Mon Sep 17 00:00:00 2001 From: Yuri Schimke Date: Sat, 6 Apr 2024 09:36:17 +0100 Subject: [PATCH 18/37] Cache fixes --- .../okhttp3/internal/cache/CacheLock.kt | 14 ++++- .../okhttp3/internal/cache/DiskLruCache.kt | 55 +++++++++++-------- .../internal/cache/DiskLruCacheTest.kt | 30 +++++++++- 3 files changed, 71 insertions(+), 28 deletions(-) diff --git a/okhttp/src/main/kotlin/okhttp3/internal/cache/CacheLock.kt b/okhttp/src/main/kotlin/okhttp3/internal/cache/CacheLock.kt index 60489a7f417c..171de84f2886 100644 --- a/okhttp/src/main/kotlin/okhttp3/internal/cache/CacheLock.kt +++ b/okhttp/src/main/kotlin/okhttp3/internal/cache/CacheLock.kt @@ -23,12 +23,16 @@ import java.nio.file.StandardOpenOption import java.util.Collections import okio.FileSystem import okio.Path +import okio.Path.Companion.toOkioPath import org.codehaus.mojo.animal_sniffer.IgnoreJRERequirement internal object CacheLock { private val openCaches = Collections.synchronizedMap(mutableMapOf()) - fun openLock(directory: Path): Closeable { + fun openLock( + fileSystem: FileSystem, + directory: Path, + ): Closeable { val memoryLock = inMemoryLock(directory) // check if possibly a non System file system @@ -41,7 +45,7 @@ internal object CacheLock { fileSystemLock.close() } } catch (e: Exception) { - if (fileSystemSupportsLock()) { + if (fileSystemSupportsLock(fileSystem)) { memoryLock.close() throw e } @@ -51,9 +55,13 @@ internal object CacheLock { return memoryLock } - internal fun fileSystemSupportsLock(): Boolean { + internal fun fileSystemSupportsLock(fileSystem: FileSystem): Boolean { val tmpLockFile = File.createTempFile("test-", ".lock") + if (!fileSystem.exists(tmpLockFile.toOkioPath())) { + return false + } + val channel = FileChannel.open(tmpLockFile.toPath(), StandardOpenOption.APPEND) return channel.tryLock().apply { close() } != null diff --git a/okhttp/src/main/kotlin/okhttp3/internal/cache/DiskLruCache.kt b/okhttp/src/main/kotlin/okhttp3/internal/cache/DiskLruCache.kt index 0e3f5c199246..8934e9f48bf6 100644 --- a/okhttp/src/main/kotlin/okhttp3/internal/cache/DiskLruCache.kt +++ b/okhttp/src/main/kotlin/okhttp3/internal/cache/DiskLruCache.kt @@ -245,35 +245,42 @@ class DiskLruCache( civilizedFileSystem = fileSystem.isCivilized(journalFileBackup) - cacheLock = openLock(directory) + cacheLock = openLock(fileSystem, directory) - // Prefer to pick up where we left off. - if (fileSystem.exists(journalFile)) { - try { - readJournal() - processJournal() - initialized = true - return - } catch (journalIsCorrupt: IOException) { - Platform.get().log( - "DiskLruCache $directory is corrupt: ${journalIsCorrupt.message}, removing", - WARN, - journalIsCorrupt, - ) - } + try { + // Prefer to pick up where we left off. + if (fileSystem.exists(journalFile)) { + try { + readJournal() + processJournal() + initialized = true + return + } catch (journalIsCorrupt: IOException) { + Platform.get().log( + "DiskLruCache $directory is corrupt: ${journalIsCorrupt.message}, removing", + WARN, + journalIsCorrupt, + ) + } - // The cache is corrupted, attempt to delete the contents of the directory. This can throw and - // we'll let that propagate out as it likely means there is a severe filesystem problem. - try { - delete() - } finally { - closed = false + // The cache is corrupted, attempt to delete the contents of the directory. This can throw and + // we'll let that propagate out as it likely means there is a severe filesystem problem. + try { + delete() + } finally { + closed = false + } } - } - rebuildJournal() + rebuildJournal() - initialized = true + initialized = true + } finally { + // If anything failed, leave without a cache lock open + if (!initialized) { + cacheLock.close() + } + } } @Throws(IOException::class) diff --git a/okhttp/src/test/java/okhttp3/internal/cache/DiskLruCacheTest.kt b/okhttp/src/test/java/okhttp3/internal/cache/DiskLruCacheTest.kt index 2a0d537b8129..918352c37d46 100644 --- a/okhttp/src/test/java/okhttp3/internal/cache/DiskLruCacheTest.kt +++ b/okhttp/src/test/java/okhttp3/internal/cache/DiskLruCacheTest.kt @@ -41,7 +41,9 @@ import okio.buffer import okio.fakefilesystem.FakeFileSystem import org.junit.jupiter.api.AfterEach import org.junit.jupiter.api.Assumptions +import org.junit.jupiter.api.BeforeEach import org.junit.jupiter.api.Tag +import org.junit.jupiter.api.TestInfo import org.junit.jupiter.api.Timeout import org.junit.jupiter.api.io.TempDir import org.junit.jupiter.params.ParameterizedTest @@ -59,6 +61,7 @@ class FileSystemParamProvider : SimpleProvider() { @Timeout(60) @Tag("Slow") class DiskLruCacheTest { + private lateinit var testInfo: TestInfo private lateinit var filesystem: FaultyFileSystem private var windows: Boolean = false @@ -89,7 +92,7 @@ class DiskLruCacheTest { windows: Boolean, ) { this.cacheDir = - if (baseFilesystem is FakeFileSystem) "/cache".toPath() else cacheDirFile.path.toPath() + if (baseFilesystem is FakeFileSystem) "/cache-${testInfo.testMethod.get().name}".toPath() else cacheDirFile.path.toPath() this.filesystem = FaultyFileSystem(baseFilesystem) this.windows = windows @@ -101,6 +104,11 @@ class DiskLruCacheTest { createNewCache() } + @BeforeEach + fun setup(testInfo: TestInfo) { + this.testInfo = testInfo + } + @AfterEach fun tearDown() { while (!toClose.isEmpty()) { toClose.pop().close() @@ -130,6 +138,9 @@ class DiskLruCacheTest { it.writeUtf8("Hello") } + // force close to test existing behaviour + cache.cacheLock.close() + // Simulate a severe Filesystem failure on the first initialization. filesystem.setFaultyDelete(cacheDir / "k1.0.tmp", true) filesystem.setFaultyDelete(cacheDir, true) @@ -249,6 +260,9 @@ class DiskLruCacheTest { creator.setString(1, "B") creator.commit() + // force close to test existing behaviour + cache.cacheLock.close() + // Simulate a dirty close of 'cache' by opening the cache directory again. createNewCache() cache["k1"]!!.use { @@ -257,6 +271,20 @@ class DiskLruCacheTest { } } + @ParameterizedTest + @ArgumentsSource(FileSystemParamProvider::class) + fun readAndWriteEntryWithoutProperCloseStoppedByLock(parameters: Pair) { + setUp(parameters.first, parameters.second) + val creator = cache.edit("k1")!! + creator.setString(0, "A") + creator.setString(1, "B") + creator.commit() + + assertFailsWith { + createNewCache() + } + } + @ParameterizedTest @ArgumentsSource(FileSystemParamProvider::class) fun journalWithEditAndPublish(parameters: Pair) { From 12b7eb979666e8d1993c99e75bd13b3b0f7d3d84 Mon Sep 17 00:00:00 2001 From: Yuri Schimke Date: Sat, 6 Apr 2024 09:43:14 +0100 Subject: [PATCH 19/37] Cache fixes --- okhttp/src/main/kotlin/okhttp3/internal/cache/CacheLock.kt | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/okhttp/src/main/kotlin/okhttp3/internal/cache/CacheLock.kt b/okhttp/src/main/kotlin/okhttp3/internal/cache/CacheLock.kt index 171de84f2886..90fd5a42412e 100644 --- a/okhttp/src/main/kotlin/okhttp3/internal/cache/CacheLock.kt +++ b/okhttp/src/main/kotlin/okhttp3/internal/cache/CacheLock.kt @@ -55,6 +55,7 @@ internal object CacheLock { return memoryLock } + @IgnoreJRERequirement // Not called on legacy Android internal fun fileSystemSupportsLock(fileSystem: FileSystem): Boolean { val tmpLockFile = File.createTempFile("test-", ".lock") @@ -87,7 +88,7 @@ internal object CacheLock { * memory lock is also needed, since locks don't work within a single process. */ @SuppressLint("NewApi") - @IgnoreJRERequirement // only called on JVM + @IgnoreJRERequirement // Not called on legacy Android fun fileSystemLock(directory: Path): Closeable { // update once https://github.com/square/okio/issues/1464 is available From ae3830dfafc9e07bc6ad26c5c36d59e947601863 Mon Sep 17 00:00:00 2001 From: Yuri Schimke Date: Sat, 6 Apr 2024 09:48:22 +0100 Subject: [PATCH 20/37] Cache fixes --- okhttp/src/test/java/okhttp3/CacheTest.kt | 26 ++++++++++++----------- 1 file changed, 14 insertions(+), 12 deletions(-) diff --git a/okhttp/src/test/java/okhttp3/CacheTest.kt b/okhttp/src/test/java/okhttp3/CacheTest.kt index 21825477b36c..045dc1b99ff8 100644 --- a/okhttp/src/test/java/okhttp3/CacheTest.kt +++ b/okhttp/src/test/java/okhttp3/CacheTest.kt @@ -3473,6 +3473,8 @@ CLEAN $urlKey ${entryMetadata.length} ${entryBody.length} @Test fun testPublicPathConstructor() { + cache.close() + val events: MutableList = ArrayList() fileSystem.createDirectories(cache.directoryPath) fileSystem.createDirectories(cache.directoryPath) @@ -3495,22 +3497,22 @@ CLEAN $urlKey ${entryMetadata.length} ${entryBody.length} return path } } - val path: Path = "/cache2".toPath() + val path: Path = "/cache".toPath() val c = Cache(path, 100000L, loggingFileSystem) assertThat(c.directoryPath).isEqualTo(path) c.size() assertThat(events).containsExactly( - "metadataOrNull:/cache2/journal.bkp", - "metadataOrNull:/cache2", - "sink:/cache2/journal.bkp", - "delete:/cache2/journal.bkp", - "metadataOrNull:/cache2/journal", - "metadataOrNull:/cache2", - "sink:/cache2/journal.tmp", - "metadataOrNull:/cache2/journal", - "atomicMove:/cache2/journal.tmp", - "atomicMove:/cache2/journal", - "appendingSink:/cache2/journal", + "metadataOrNull:/cache/journal.bkp", + "metadataOrNull:/cache", + "sink:/cache/journal.bkp", + "delete:/cache/journal.bkp", + "metadataOrNull:/cache/journal", + "metadataOrNull:/cache", + "sink:/cache/journal.tmp", + "metadataOrNull:/cache/journal", + "atomicMove:/cache/journal.tmp", + "atomicMove:/cache/journal", + "appendingSink:/cache/journal", ) events.clear() c.size() From 99a5ee5f3ccedb93f7e7032c71ecf6958c37320f Mon Sep 17 00:00:00 2001 From: Yuri Schimke Date: Sat, 6 Apr 2024 09:58:08 +0100 Subject: [PATCH 21/37] Cache fixes --- .../okhttp3/internal/cache/CacheLock.kt | 19 ++++++++++--- okhttp/src/test/java/okhttp3/CacheLockTest.kt | 27 +++++++++++++++++-- 2 files changed, 40 insertions(+), 6 deletions(-) diff --git a/okhttp/src/main/kotlin/okhttp3/internal/cache/CacheLock.kt b/okhttp/src/main/kotlin/okhttp3/internal/cache/CacheLock.kt index 90fd5a42412e..0c36ae8887c7 100644 --- a/okhttp/src/main/kotlin/okhttp3/internal/cache/CacheLock.kt +++ b/okhttp/src/main/kotlin/okhttp3/internal/cache/CacheLock.kt @@ -26,9 +26,20 @@ import okio.Path import okio.Path.Companion.toOkioPath import org.codehaus.mojo.animal_sniffer.IgnoreJRERequirement +/** + * An implementation of directory locking to ensure exclusive access in a Cache instance. + * Always applies for the current process, and uses a file system lock if supported. + */ internal object CacheLock { private val openCaches = Collections.synchronizedMap(mutableMapOf()) + /** + * Open a lock, which if successful remains until the returned [Closeable] is closed. + * The lock file will be a file with name "lock" inside directory. + * + * @param fileSystem the file system containing the lock files. + * @param directory the cache directory. + */ fun openLock( fileSystem: FileSystem, directory: Path, @@ -44,10 +55,10 @@ internal object CacheLock { memoryLock.close() fileSystemLock.close() } - } catch (e: Exception) { + } catch (le: LockException) { if (fileSystemSupportsLock(fileSystem)) { memoryLock.close() - throw e + throw le } } } @@ -73,7 +84,7 @@ internal object CacheLock { */ @SuppressLint("NewApi") @IgnoreJRERequirement // D8 supports put if absent - fun inMemoryLock(directory: Path): Closeable { + internal fun inMemoryLock(directory: Path): Closeable { val existing = openCaches.putIfAbsent(directory, LockException("Existing CacheLock($directory) opened at")) if (existing != null) { throw LockException("Cache already open at '$directory' in same process", existing) @@ -89,7 +100,7 @@ internal object CacheLock { */ @SuppressLint("NewApi") @IgnoreJRERequirement // Not called on legacy Android - fun fileSystemLock(directory: Path): Closeable { + internal fun fileSystemLock(directory: Path): Closeable { // update once https://github.com/square/okio/issues/1464 is available val lockFile = directory / "lock" diff --git a/okhttp/src/test/java/okhttp3/CacheLockTest.kt b/okhttp/src/test/java/okhttp3/CacheLockTest.kt index 8e826f5f7b69..28c85c7b3038 100644 --- a/okhttp/src/test/java/okhttp3/CacheLockTest.kt +++ b/okhttp/src/test/java/okhttp3/CacheLockTest.kt @@ -29,6 +29,8 @@ import okio.Closeable import okio.FileSystem import okio.Path.Companion.toOkioPath import okio.Path.Companion.toPath +import okio.SYSTEM +import okio.fakefilesystem.FakeFileSystem import org.junit.jupiter.api.AfterEach import org.junit.jupiter.api.Assumptions.assumeTrue import org.junit.jupiter.api.BeforeEach @@ -138,8 +140,29 @@ class CacheLockTest { } } - private fun openCache(directory: okio.Path): Cache { - return Cache(directory, 10_000, FileSystem.SYSTEM).apply { + @Test + fun testCacheLockOnFakeFileSystem() { + val fileSystem = FakeFileSystem() + val fakeCacheDir = "/missing".toPath() + + val cache = openCache(fakeCacheDir, fileSystem) + + val lockException = + assertThrows { + openCache(fakeCacheDir, fileSystem) + } + assertThat(lockException.message).isEqualTo("Cache already open at '$fakeCacheDir' in same process") + + cache.close() + + openCache(fakeCacheDir, fileSystem) + } + + private fun openCache( + directory: okio.Path, + fileSystem: FileSystem = FileSystem.SYSTEM, + ): Cache { + return Cache(directory, 10_000, fileSystem).apply { // force early LRU initialisation initialize() toClose.add(this) From cc9776983264a4295eed1b607ce95e854b148bb5 Mon Sep 17 00:00:00 2001 From: Yuri Schimke Date: Sat, 6 Apr 2024 10:08:19 +0100 Subject: [PATCH 22/37] Cache fixes --- okhttp/src/test/java/okhttp3/CacheTest.kt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/okhttp/src/test/java/okhttp3/CacheTest.kt b/okhttp/src/test/java/okhttp3/CacheTest.kt index 045dc1b99ff8..1a548f71009f 100644 --- a/okhttp/src/test/java/okhttp3/CacheTest.kt +++ b/okhttp/src/test/java/okhttp3/CacheTest.kt @@ -81,6 +81,7 @@ class CacheTest { private lateinit var client: OkHttpClient private lateinit var cache: Cache private val cookieManager = CookieManager() + private val cachePath = "/cache-CacheTest".toPath() @BeforeEach fun setUp( @@ -92,7 +93,7 @@ class CacheTest { platform.assumeNotOpenJSSE() server.protocolNegotiationEnabled = false fileSystem.emulateUnix() - cache = Cache("/cache".toPath(), Long.MAX_VALUE, fileSystem) + cache = Cache(cachePath, Long.MAX_VALUE, fileSystem) client = clientTestRule.newClientBuilder() .cache(cache) @@ -3497,7 +3498,6 @@ CLEAN $urlKey ${entryMetadata.length} ${entryBody.length} return path } } - val path: Path = "/cache".toPath() val c = Cache(path, 100000L, loggingFileSystem) assertThat(c.directoryPath).isEqualTo(path) c.size() From 6dbb03fb47f1fcd83a3c706907dc2494157e277c Mon Sep 17 00:00:00 2001 From: Yuri Schimke Date: Sat, 6 Apr 2024 10:18:07 +0100 Subject: [PATCH 23/37] Fix test --- okhttp/src/test/java/okhttp3/CacheTest.kt | 26 +++++++++++------------ 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/okhttp/src/test/java/okhttp3/CacheTest.kt b/okhttp/src/test/java/okhttp3/CacheTest.kt index 1a548f71009f..f8b31264a777 100644 --- a/okhttp/src/test/java/okhttp3/CacheTest.kt +++ b/okhttp/src/test/java/okhttp3/CacheTest.kt @@ -3498,21 +3498,21 @@ CLEAN $urlKey ${entryMetadata.length} ${entryBody.length} return path } } - val c = Cache(path, 100000L, loggingFileSystem) - assertThat(c.directoryPath).isEqualTo(path) + val c = Cache(cachePath, 100000L, loggingFileSystem) + assertThat(c.directoryPath).isEqualTo(cachePath) c.size() assertThat(events).containsExactly( - "metadataOrNull:/cache/journal.bkp", - "metadataOrNull:/cache", - "sink:/cache/journal.bkp", - "delete:/cache/journal.bkp", - "metadataOrNull:/cache/journal", - "metadataOrNull:/cache", - "sink:/cache/journal.tmp", - "metadataOrNull:/cache/journal", - "atomicMove:/cache/journal.tmp", - "atomicMove:/cache/journal", - "appendingSink:/cache/journal", + "metadataOrNull:$cachePath/journal.bkp", + "metadataOrNull:$cachePath", + "sink:$cachePath/journal.bkp", + "delete:$cachePath/journal.bkp", + "metadataOrNull:$cachePath/journal", + "metadataOrNull:$cachePath", + "sink:$cachePath/journal.tmp", + "metadataOrNull:$cachePath/journal", + "atomicMove:$cachePath/journal.tmp", + "atomicMove:$cachePath/journal", + "appendingSink:$cachePath/journal", ) events.clear() c.size() From feafff96aec98ad85f2203f3a3204239153fc114 Mon Sep 17 00:00:00 2001 From: Yuri Schimke Date: Sat, 6 Apr 2024 10:32:39 +0100 Subject: [PATCH 24/37] Fix test --- okhttp/src/test/java/okhttp3/CacheTest.kt | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/okhttp/src/test/java/okhttp3/CacheTest.kt b/okhttp/src/test/java/okhttp3/CacheTest.kt index f8b31264a777..9bc766d18784 100644 --- a/okhttp/src/test/java/okhttp3/CacheTest.kt +++ b/okhttp/src/test/java/okhttp3/CacheTest.kt @@ -3498,9 +3498,9 @@ CLEAN $urlKey ${entryMetadata.length} ${entryBody.length} return path } } - val c = Cache(cachePath, 100000L, loggingFileSystem) - assertThat(c.directoryPath).isEqualTo(cachePath) - c.size() + cache = Cache(cachePath, 100000L, loggingFileSystem) + assertThat(cache.directoryPath).isEqualTo(cachePath) + cache.size() assertThat(events).containsExactly( "metadataOrNull:$cachePath/journal.bkp", "metadataOrNull:$cachePath", @@ -3515,7 +3515,7 @@ CLEAN $urlKey ${entryMetadata.length} ${entryBody.length} "appendingSink:$cachePath/journal", ) events.clear() - c.size() + cache.size() assertThat(events).isEmpty() } From 72359a6d44c2ea2d20a9df8b3ea0ef6861c728a7 Mon Sep 17 00:00:00 2001 From: Yuri Schimke Date: Sat, 6 Apr 2024 16:31:59 +0100 Subject: [PATCH 25/37] avoid windows for now --- okcurl/src/test/kotlin/okhttp3/curl/MainTest.kt | 9 +++++++++ okcurl/src/test/kotlin/okhttp3/curl/OkcurlTest.kt | 11 +++++++++++ 2 files changed, 20 insertions(+) diff --git a/okcurl/src/test/kotlin/okhttp3/curl/MainTest.kt b/okcurl/src/test/kotlin/okhttp3/curl/MainTest.kt index 0a2598a2e694..1d8377171f1e 100644 --- a/okcurl/src/test/kotlin/okhttp3/curl/MainTest.kt +++ b/okcurl/src/test/kotlin/okhttp3/curl/MainTest.kt @@ -22,7 +22,9 @@ import assertk.assertions.startsWith import java.io.IOException import kotlin.test.Test import okhttp3.RequestBody +import okhttp3.TestUtil import okio.Buffer +import org.junit.jupiter.api.BeforeAll class MainTest { @Test @@ -137,5 +139,12 @@ class MainTest { throw RuntimeException(e) } } + + @JvmStatic + @BeforeAll + fun beforeAll() { +// https://github.com/square/okhttp/issues/8342 + TestUtil.assumeNotWindows() + } } } diff --git a/okcurl/src/test/kotlin/okhttp3/curl/OkcurlTest.kt b/okcurl/src/test/kotlin/okhttp3/curl/OkcurlTest.kt index d632e8bae9f8..e32954811600 100644 --- a/okcurl/src/test/kotlin/okhttp3/curl/OkcurlTest.kt +++ b/okcurl/src/test/kotlin/okhttp3/curl/OkcurlTest.kt @@ -16,10 +16,21 @@ package okhttp3.curl import kotlin.test.Test +import okhttp3.TestUtil +import org.junit.jupiter.api.BeforeAll class OkcurlTest { @Test fun simple() { Main().main(listOf("--help")) } + + companion object { + @JvmStatic + @BeforeAll + fun beforeAll() { +// https://github.com/square/okhttp/issues/8342 + TestUtil.assumeNotWindows() + } + } } From b76aa7f567f9b2155dff8c740b1483175f9391fd Mon Sep 17 00:00:00 2001 From: Yuri Schimke Date: Sun, 7 Apr 2024 09:11:20 +0100 Subject: [PATCH 26/37] Fix or skip on windows --- .../kotlin/okhttp3/internal/cache2/Relay.kt | 5 ++- okhttp/src/test/java/okhttp3/CacheTest.kt | 8 ++-- .../okhttp3/ServerTruncatesRequestTest.kt | 7 +++ .../internal/cache2/FileOperatorTest.kt | 43 ++++++++++--------- .../java/okhttp3/internal/cache2/RelayTest.kt | 3 +- 5 files changed, 40 insertions(+), 26 deletions(-) diff --git a/okhttp/src/main/kotlin/okhttp3/internal/cache2/Relay.kt b/okhttp/src/main/kotlin/okhttp3/internal/cache2/Relay.kt index 6ae6e61809a1..7ae877622f3f 100644 --- a/okhttp/src/main/kotlin/okhttp3/internal/cache2/Relay.kt +++ b/okhttp/src/main/kotlin/okhttp3/internal/cache2/Relay.kt @@ -344,7 +344,10 @@ class Relay private constructor( val header = Buffer() fileOperator.read(0, header, FILE_HEADER_SIZE) val prefix = header.readByteString(PREFIX_CLEAN.size.toLong()) - if (prefix != PREFIX_CLEAN) throw IOException("unreadable cache file") + if (prefix != PREFIX_CLEAN) { + randomAccessFile.close() + throw IOException("unreadable cache file") + } val upstreamSize = header.readLong() val metadataSize = header.readLong() diff --git a/okhttp/src/test/java/okhttp3/CacheTest.kt b/okhttp/src/test/java/okhttp3/CacheTest.kt index 9bc766d18784..238adaf64761 100644 --- a/okhttp/src/test/java/okhttp3/CacheTest.kt +++ b/okhttp/src/test/java/okhttp3/CacheTest.kt @@ -2759,7 +2759,7 @@ CLEAN $urlKey ${entryMetadata.length} ${entryBody.length} writeFile(cache.directoryPath, "$urlKey.0", entryMetadata) writeFile(cache.directoryPath, "$urlKey.1", entryBody) writeFile(cache.directoryPath, "journal", journalBody) - cache = Cache(cache.directory.path.toPath(), Int.MAX_VALUE.toLong(), fileSystem) + cache = Cache(cachePath, Int.MAX_VALUE.toLong(), fileSystem) client = client.newBuilder() .cache(cache) @@ -2808,7 +2808,7 @@ CLEAN $urlKey ${entryMetadata.length} ${entryBody.length} writeFile(cache.directoryPath, "$urlKey.1", entryBody) writeFile(cache.directoryPath, "journal", journalBody) cache.close() - cache = Cache(cache.directory.path.toPath(), Int.MAX_VALUE.toLong(), fileSystem) + cache = Cache(cachePath, Int.MAX_VALUE.toLong(), fileSystem) client = client.newBuilder() .cache(cache) @@ -2861,7 +2861,7 @@ CLEAN $urlKey ${entryMetadata.length} ${entryBody.length} writeFile(cache.directoryPath, "$urlKey.1", entryBody) writeFile(cache.directoryPath, "journal", journalBody) cache.close() - cache = Cache(cache.directory.path.toPath(), Int.MAX_VALUE.toLong(), fileSystem) + cache = Cache(cachePath, Int.MAX_VALUE.toLong(), fileSystem) client = client.newBuilder() .cache(cache) @@ -2906,7 +2906,7 @@ CLEAN $urlKey ${entryMetadata.length} ${entryBody.length} writeFile(cache.directoryPath, "$urlKey.1", entryBody) writeFile(cache.directoryPath, "journal", journalBody) cache.close() - cache = Cache(cache.directory.path.toPath(), Int.MAX_VALUE.toLong(), fileSystem) + cache = Cache(cachePath, Int.MAX_VALUE.toLong(), fileSystem) client = client.newBuilder() .cache(cache) diff --git a/okhttp/src/test/java/okhttp3/ServerTruncatesRequestTest.kt b/okhttp/src/test/java/okhttp3/ServerTruncatesRequestTest.kt index 1f8a28fe67d0..199648f7e31d 100644 --- a/okhttp/src/test/java/okhttp3/ServerTruncatesRequestTest.kt +++ b/okhttp/src/test/java/okhttp3/ServerTruncatesRequestTest.kt @@ -26,6 +26,7 @@ import mockwebserver3.MockResponse import mockwebserver3.MockWebServer import mockwebserver3.SocketPolicy.DoNotReadRequestBody import okhttp3.Headers.Companion.headersOf +import okhttp3.TestUtil.assumeNotWindows import okhttp3.internal.duplex.AsyncRequestBody import okhttp3.internal.http2.ErrorCode import okhttp3.testing.PlatformRule @@ -69,6 +70,9 @@ class ServerTruncatesRequestTest { @Test fun serverTruncatesRequestOnLongPostHttp1() { + // SocketException: An established connection was aborted by the software in your host machine + assumeNotWindows() + serverTruncatesRequestOnLongPost(https = false) } @@ -171,6 +175,9 @@ class ServerTruncatesRequestTest { @Test fun serverTruncatesRequestButTrailersCanStillBeReadHttp1() { + // SocketException: An established connection was aborted by the software in your host machine + assumeNotWindows() + serverTruncatesRequestButTrailersCanStillBeRead(http2 = false) } diff --git a/okhttp/src/test/java/okhttp3/internal/cache2/FileOperatorTest.kt b/okhttp/src/test/java/okhttp3/internal/cache2/FileOperatorTest.kt index ef18c3b155d4..5faf0c6e85cb 100644 --- a/okhttp/src/test/java/okhttp3/internal/cache2/FileOperatorTest.kt +++ b/okhttp/src/test/java/okhttp3/internal/cache2/FileOperatorTest.kt @@ -27,6 +27,7 @@ import okio.ByteString.Companion.encodeUtf8 import okio.buffer import okio.sink import okio.source +import okio.use import org.junit.jupiter.api.AfterEach import org.junit.jupiter.api.BeforeEach import org.junit.jupiter.api.Test @@ -34,9 +35,9 @@ import org.junit.jupiter.api.io.TempDir class FileOperatorTest { @TempDir - var tempDir: File? = null - private var file: File? = null - private var randomAccessFile: RandomAccessFile? = null + lateinit var tempDir: File + private lateinit var file: File + private lateinit var randomAccessFile: RandomAccessFile @BeforeEach fun setUp() { @@ -46,7 +47,7 @@ class FileOperatorTest { @AfterEach fun tearDown() { - randomAccessFile!!.close() + randomAccessFile.close() } @Test @@ -54,7 +55,7 @@ class FileOperatorTest { write("Hello, World".encodeUtf8()) val operator = FileOperator( - randomAccessFile!!.getChannel(), + randomAccessFile.getChannel(), ) val buffer = Buffer() operator.read(0, buffer, 5) @@ -64,10 +65,12 @@ class FileOperatorTest { } @Test - fun write() { + fun + write() { + val fileChannel = randomAccessFile.getChannel() val operator = FileOperator( - randomAccessFile!!.getChannel(), + fileChannel, ) val buffer1 = Buffer().writeUtf8("Hello, World") operator.write(0, buffer1, 5) @@ -76,13 +79,14 @@ class FileOperatorTest { operator.write(3, buffer2, 7) assertThat(buffer2.readUtf8()).isEqualTo("!") assertThat(snapshot()).isEqualTo("Helicopter".encodeUtf8()) + fileChannel.close() } @Test fun readAndWrite() { val operator = FileOperator( - randomAccessFile!!.getChannel(), + randomAccessFile.getChannel(), ) write("woman god creates dinosaurs destroys. ".encodeUtf8()) val buffer = Buffer() @@ -120,11 +124,11 @@ class FileOperatorTest { fun multipleOperatorsShareOneFile() { val operatorA = FileOperator( - randomAccessFile!!.getChannel(), + randomAccessFile.getChannel(), ) val operatorB = FileOperator( - randomAccessFile!!.getChannel(), + randomAccessFile.getChannel(), ) val bufferA = Buffer() val bufferB = Buffer() @@ -148,7 +152,7 @@ class FileOperatorTest { write(data) val operator = FileOperator( - randomAccessFile!!.getChannel(), + randomAccessFile.getChannel(), ) val buffer = Buffer() operator.read(0, buffer, data.size.toLong()) @@ -160,7 +164,7 @@ class FileOperatorTest { val data = randomByteString(1000000) val operator = FileOperator( - randomAccessFile!!.getChannel(), + randomAccessFile.getChannel(), ) val buffer = Buffer().write(data) operator.write(0, buffer, data.size.toLong()) @@ -171,7 +175,7 @@ class FileOperatorTest { fun readBounds() { val operator = FileOperator( - randomAccessFile!!.getChannel(), + randomAccessFile.getChannel(), ) val buffer = Buffer() assertFailsWith { @@ -183,7 +187,7 @@ class FileOperatorTest { fun writeBounds() { val operator = FileOperator( - randomAccessFile!!.getChannel(), + randomAccessFile.getChannel(), ) val buffer = Buffer().writeUtf8("abc") assertFailsWith { @@ -201,14 +205,13 @@ class FileOperatorTest { } private fun snapshot(): ByteString { - randomAccessFile!!.getChannel().force(false) - val source = file!!.source().buffer() - return source.readByteString() + randomAccessFile.getChannel().force(false) + return file.source().buffer().use { it.readByteString() } } private fun write(data: ByteString) { - val sink = file!!.sink().buffer() - sink.write(data) - sink.close() + val sink = file.sink().buffer().use { + it.write(data) + } } } diff --git a/okhttp/src/test/java/okhttp3/internal/cache2/RelayTest.kt b/okhttp/src/test/java/okhttp3/internal/cache2/RelayTest.kt index 597b6c43832c..6ff0229118a5 100644 --- a/okhttp/src/test/java/okhttp3/internal/cache2/RelayTest.kt +++ b/okhttp/src/test/java/okhttp3/internal/cache2/RelayTest.kt @@ -38,11 +38,12 @@ import org.junit.jupiter.api.AfterEach import org.junit.jupiter.api.BeforeEach import org.junit.jupiter.api.Tag import org.junit.jupiter.api.Test +import org.junit.jupiter.api.io.CleanupMode import org.junit.jupiter.api.io.TempDir @Tag("Slowish") class RelayTest { - @TempDir + @TempDir(cleanup = CleanupMode.ON_SUCCESS) var tempDir: File? = null private val executor = Executors.newCachedThreadPool() private val metadata: ByteString = "great metadata!".encodeUtf8() From 89e9d800a396e8ad282896f0fe1823764038ee11 Mon Sep 17 00:00:00 2001 From: Yuri Schimke Date: Sun, 7 Apr 2024 09:20:51 +0100 Subject: [PATCH 27/37] cleanup --- .../java/okhttp3/internal/cache2/FileOperatorTest.kt | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/okhttp/src/test/java/okhttp3/internal/cache2/FileOperatorTest.kt b/okhttp/src/test/java/okhttp3/internal/cache2/FileOperatorTest.kt index 5faf0c6e85cb..331e64f80cef 100644 --- a/okhttp/src/test/java/okhttp3/internal/cache2/FileOperatorTest.kt +++ b/okhttp/src/test/java/okhttp3/internal/cache2/FileOperatorTest.kt @@ -65,8 +65,7 @@ class FileOperatorTest { } @Test - fun - write() { + fun write() { val fileChannel = randomAccessFile.getChannel() val operator = FileOperator( @@ -210,8 +209,9 @@ class FileOperatorTest { } private fun write(data: ByteString) { - val sink = file.sink().buffer().use { - it.write(data) - } + val sink = + file.sink().buffer().use { + it.write(data) + } } } From 28025e20429887e6d713fc7205cc9039e4d12cba Mon Sep 17 00:00:00 2001 From: Yuri Schimke Date: Sun, 7 Apr 2024 10:47:48 +0100 Subject: [PATCH 28/37] skip on windows --- okhttp/src/test/java/okhttp3/DuplexTest.kt | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/okhttp/src/test/java/okhttp3/DuplexTest.kt b/okhttp/src/test/java/okhttp3/DuplexTest.kt index 6f7f3384e9d0..6b9bdaa8cd4c 100644 --- a/okhttp/src/test/java/okhttp3/DuplexTest.kt +++ b/okhttp/src/test/java/okhttp3/DuplexTest.kt @@ -21,6 +21,7 @@ import assertk.assertions.containsExactly import assertk.assertions.isEqualTo import assertk.assertions.isNull import assertk.assertions.isTrue +import com.sun.tools.javac.platform.PlatformUtils import java.io.IOException import java.net.HttpURLConnection import java.net.ProtocolException @@ -412,6 +413,9 @@ class DuplexTest { */ @Test fun duplexWithAuthChallenge() { + // TODO https://github.com/square/okhttp/issues/8342 + assumeNotWindows() + enableProtocol(Protocol.HTTP_2) val credential = basic("jesse", "secret") client = From fa707bcf05bbe8bc80f59f40da620f7731a769c3 Mon Sep 17 00:00:00 2001 From: Yuri Schimke Date: Sun, 7 Apr 2024 11:09:00 +0100 Subject: [PATCH 29/37] Update DuplexTest.kt --- okhttp/src/test/java/okhttp3/DuplexTest.kt | 1 - 1 file changed, 1 deletion(-) diff --git a/okhttp/src/test/java/okhttp3/DuplexTest.kt b/okhttp/src/test/java/okhttp3/DuplexTest.kt index 6b9bdaa8cd4c..6a63a63d1289 100644 --- a/okhttp/src/test/java/okhttp3/DuplexTest.kt +++ b/okhttp/src/test/java/okhttp3/DuplexTest.kt @@ -21,7 +21,6 @@ import assertk.assertions.containsExactly import assertk.assertions.isEqualTo import assertk.assertions.isNull import assertk.assertions.isTrue -import com.sun.tools.javac.platform.PlatformUtils import java.io.IOException import java.net.HttpURLConnection import java.net.ProtocolException From bb0d4b0eff213e8675de5e39ff9caa41185c1401 Mon Sep 17 00:00:00 2001 From: Yuri Schimke Date: Sat, 4 Jan 2025 16:06:05 +0000 Subject: [PATCH 30/37] Fixes --- .../kotlin/okhttp3/internal/cache/CacheLock.kt | 3 --- 1 file changed, 3 deletions(-) diff --git a/okhttp/src/commonJvmAndroid/kotlin/okhttp3/internal/cache/CacheLock.kt b/okhttp/src/commonJvmAndroid/kotlin/okhttp3/internal/cache/CacheLock.kt index 0c36ae8887c7..fbfb63f4adfd 100644 --- a/okhttp/src/commonJvmAndroid/kotlin/okhttp3/internal/cache/CacheLock.kt +++ b/okhttp/src/commonJvmAndroid/kotlin/okhttp3/internal/cache/CacheLock.kt @@ -15,7 +15,6 @@ */ package okhttp3.internal.cache -import android.annotation.SuppressLint import java.io.Closeable import java.io.File import java.nio.channels.FileChannel @@ -82,7 +81,6 @@ internal object CacheLock { /** * Create an in-memory lock, avoiding two open Cache instances. */ - @SuppressLint("NewApi") @IgnoreJRERequirement // D8 supports put if absent internal fun inMemoryLock(directory: Path): Closeable { val existing = openCaches.putIfAbsent(directory, LockException("Existing CacheLock($directory) opened at")) @@ -98,7 +96,6 @@ internal object CacheLock { * Create a file system lock, that excludes other processes. However within the process a * memory lock is also needed, since locks don't work within a single process. */ - @SuppressLint("NewApi") @IgnoreJRERequirement // Not called on legacy Android internal fun fileSystemLock(directory: Path): Closeable { // update once https://github.com/square/okio/issues/1464 is available From 192e93b1e0fc4a56bbfee6be5ddf442a533c690d Mon Sep 17 00:00:00 2001 From: Yuri Schimke Date: Sat, 4 Jan 2025 16:09:02 +0000 Subject: [PATCH 31/37] Fixes --- okhttp/src/jvmTest/kotlin/okhttp3/CacheLockTest.kt | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/okhttp/src/jvmTest/kotlin/okhttp3/CacheLockTest.kt b/okhttp/src/jvmTest/kotlin/okhttp3/CacheLockTest.kt index 28c85c7b3038..4951de09b55f 100644 --- a/okhttp/src/jvmTest/kotlin/okhttp3/CacheLockTest.kt +++ b/okhttp/src/jvmTest/kotlin/okhttp3/CacheLockTest.kt @@ -29,7 +29,6 @@ import okio.Closeable import okio.FileSystem import okio.Path.Companion.toOkioPath import okio.Path.Companion.toPath -import okio.SYSTEM import okio.fakefilesystem.FakeFileSystem import org.junit.jupiter.api.AfterEach import org.junit.jupiter.api.Assumptions.assumeTrue @@ -162,7 +161,7 @@ class CacheLockTest { directory: okio.Path, fileSystem: FileSystem = FileSystem.SYSTEM, ): Cache { - return Cache(directory, 10_000, fileSystem).apply { + return Cache(fileSystem, directory, 10_000).apply { // force early LRU initialisation initialize() toClose.add(this) From b4115b22f0ae497c2c49a12711cb45caa75582a0 Mon Sep 17 00:00:00 2001 From: Yuri Schimke Date: Sat, 4 Jan 2025 16:11:53 +0000 Subject: [PATCH 32/37] Fixes --- okhttp/src/jvmTest/kotlin/okhttp3/CacheLockTest.kt | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/okhttp/src/jvmTest/kotlin/okhttp3/CacheLockTest.kt b/okhttp/src/jvmTest/kotlin/okhttp3/CacheLockTest.kt index 4951de09b55f..51a3203607f0 100644 --- a/okhttp/src/jvmTest/kotlin/okhttp3/CacheLockTest.kt +++ b/okhttp/src/jvmTest/kotlin/okhttp3/CacheLockTest.kt @@ -99,12 +99,12 @@ class CacheLockTest { lockFile.toFile().createNewFile() val javaExe = - if (PlatformVersion.majorVersion >= 9) { - @Suppress("Since15") - ProcessHandle.current().info().command().get().toPath() - } else { +// if (PlatformVersion.majorVersion >= 9) { +// @Suppress("Since15") +// ProcessHandle.current().info().command().get().toPath() +// } else { System.getenv("JAVA_HOME").toPath() / "bin/java" - } +// } val process = ProcessBuilder().command( From 2f959afc12ada7cc1ad568b75d6bb9204ec286ca Mon Sep 17 00:00:00 2001 From: Yuri Schimke Date: Sat, 4 Jan 2025 16:15:34 +0000 Subject: [PATCH 33/37] Fixes --- okhttp/src/jvmTest/kotlin/okhttp3/CacheLockTest.kt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/okhttp/src/jvmTest/kotlin/okhttp3/CacheLockTest.kt b/okhttp/src/jvmTest/kotlin/okhttp3/CacheLockTest.kt index 51a3203607f0..d36470f25e06 100644 --- a/okhttp/src/jvmTest/kotlin/okhttp3/CacheLockTest.kt +++ b/okhttp/src/jvmTest/kotlin/okhttp3/CacheLockTest.kt @@ -103,7 +103,7 @@ class CacheLockTest { // @Suppress("Since15") // ProcessHandle.current().info().command().get().toPath() // } else { - System.getenv("JAVA_HOME").toPath() / "bin/java" + System.getenv("JAVA_HOME").toPath() / "bin/java" // } val process = From 7af570573890bc05d07a3e8eda2405550dee5296 Mon Sep 17 00:00:00 2001 From: Yuri Schimke Date: Sat, 4 Jan 2025 16:38:23 +0000 Subject: [PATCH 34/37] Fixes --- okhttp/src/jvmTest/kotlin/okhttp3/CacheTest.kt | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/okhttp/src/jvmTest/kotlin/okhttp3/CacheTest.kt b/okhttp/src/jvmTest/kotlin/okhttp3/CacheTest.kt index a211610388fc..9d36373a359f 100644 --- a/okhttp/src/jvmTest/kotlin/okhttp3/CacheTest.kt +++ b/okhttp/src/jvmTest/kotlin/okhttp3/CacheTest.kt @@ -104,6 +104,7 @@ class CacheTest { @AfterEach fun tearDown() { ResponseCache.setDefault(null) + cache.close() cache.delete() } @@ -3498,9 +3499,9 @@ CLEAN $urlKey ${entryMetadata.length} ${entryBody.length} return path } } - val c = Cache(loggingFileSystem, cachePath, 100000L) - assertThat(c.directoryPath).isEqualTo(cachePath) - c.size() + cache = Cache(loggingFileSystem, cachePath, 100000L) + assertThat(cache.directoryPath).isEqualTo(cachePath) + cache.size() assertThat(events).containsExactly( "metadataOrNull:$cachePath/journal.bkp", "metadataOrNull:$cachePath", From 4e9d4b3e5f5aff153f6abc0a03ed50918f686af2 Mon Sep 17 00:00:00 2001 From: Yuri Schimke Date: Sat, 11 Jan 2025 11:01:53 +0000 Subject: [PATCH 35/37] Fix test --- .../src/jvmTest/kotlin/okhttp3/CacheLockTest.kt | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/okhttp/src/jvmTest/kotlin/okhttp3/CacheLockTest.kt b/okhttp/src/jvmTest/kotlin/okhttp3/CacheLockTest.kt index d36470f25e06..9c0413314271 100644 --- a/okhttp/src/jvmTest/kotlin/okhttp3/CacheLockTest.kt +++ b/okhttp/src/jvmTest/kotlin/okhttp3/CacheLockTest.kt @@ -37,6 +37,7 @@ import org.junit.jupiter.api.Test import org.junit.jupiter.api.assertThrows import org.junit.jupiter.api.extension.RegisterExtension import org.junit.jupiter.api.io.TempDir +import java.util.Optional @org.junit.jupiter.api.parallel.Isolated class CacheLockTest { @@ -99,17 +100,21 @@ class CacheLockTest { lockFile.toFile().createNewFile() val javaExe = -// if (PlatformVersion.majorVersion >= 9) { -// @Suppress("Since15") -// ProcessHandle.current().info().command().get().toPath() -// } else { + if (PlatformVersion.majorVersion >= 9) { + val info = Class.forName("java.lang.ProcessHandle").run { + val handle = getMethod("current").invoke(null) + getMethod("info").invoke(handle) + } + val command = Class.forName("java.lang.ProcessHandle\$Info").getMethod("command").invoke(info) as Optional<*> + (command.get() as String).toPath() + } else { System.getenv("JAVA_HOME").toPath() / "bin/java" -// } + } val process = ProcessBuilder().command( javaExe.toString(), - "src/test/java/okhttp3/LockTestProgram.java", + "src/jvmTest/kotlin/okhttp3/LockTestProgram.java", (lockFile.toString()), ) .redirectErrorStream(true) From 019560c5c0093003e7667cdb70394e85b13371c5 Mon Sep 17 00:00:00 2001 From: Yuri Schimke Date: Sat, 11 Jan 2025 11:10:02 +0000 Subject: [PATCH 36/37] reformat --- okhttp/src/jvmTest/kotlin/okhttp3/CacheLockTest.kt | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/okhttp/src/jvmTest/kotlin/okhttp3/CacheLockTest.kt b/okhttp/src/jvmTest/kotlin/okhttp3/CacheLockTest.kt index 9c0413314271..cb50d051b370 100644 --- a/okhttp/src/jvmTest/kotlin/okhttp3/CacheLockTest.kt +++ b/okhttp/src/jvmTest/kotlin/okhttp3/CacheLockTest.kt @@ -18,6 +18,7 @@ package okhttp3 import assertk.assertThat import assertk.assertions.isEqualTo import java.nio.file.Path +import java.util.Optional import kotlin.time.Duration.Companion.seconds import kotlinx.coroutines.delay import kotlinx.coroutines.runBlocking @@ -37,7 +38,6 @@ import org.junit.jupiter.api.Test import org.junit.jupiter.api.assertThrows import org.junit.jupiter.api.extension.RegisterExtension import org.junit.jupiter.api.io.TempDir -import java.util.Optional @org.junit.jupiter.api.parallel.Isolated class CacheLockTest { @@ -101,14 +101,15 @@ class CacheLockTest { val javaExe = if (PlatformVersion.majorVersion >= 9) { - val info = Class.forName("java.lang.ProcessHandle").run { - val handle = getMethod("current").invoke(null) - getMethod("info").invoke(handle) - } + val info = + Class.forName("java.lang.ProcessHandle").run { + val handle = getMethod("current").invoke(null) + getMethod("info").invoke(handle) + } val command = Class.forName("java.lang.ProcessHandle\$Info").getMethod("command").invoke(info) as Optional<*> (command.get() as String).toPath() } else { - System.getenv("JAVA_HOME").toPath() / "bin/java" + System.getenv("JAVA_HOME").toPath() / "bin/java" } val process = From 433f6f7bc469ab46106750975674cf133c52c234 Mon Sep 17 00:00:00 2001 From: Yuri Schimke Date: Sat, 11 Jan 2025 11:24:07 +0000 Subject: [PATCH 37/37] Make it optional --- .../commonJvmAndroid/kotlin/okhttp3/Cache.kt | 11 +- .../okhttp3/internal/cache/DiskLruCache.kt | 3 +- .../internal/cache/DiskLruCacheTest.kt | 100 ++++++++++++++++-- 3 files changed, 99 insertions(+), 15 deletions(-) diff --git a/okhttp/src/commonJvmAndroid/kotlin/okhttp3/Cache.kt b/okhttp/src/commonJvmAndroid/kotlin/okhttp3/Cache.kt index 69d79cd9394f..f5489b9bca08 100644 --- a/okhttp/src/commonJvmAndroid/kotlin/okhttp3/Cache.kt +++ b/okhttp/src/commonJvmAndroid/kotlin/okhttp3/Cache.kt @@ -150,6 +150,7 @@ class Cache internal constructor( maxSize: Long, fileSystem: FileSystem, taskRunner: TaskRunner, + useCacheLock: Boolean, ) : Closeable, Flushable { /** Create a cache of at most [maxSize] bytes in [directory]. */ constructor( @@ -157,10 +158,11 @@ class Cache internal constructor( directory: Path, maxSize: Long, ) : this( - directory, - maxSize, - fileSystem, - TaskRunner.INSTANCE, + directory = directory, + maxSize = maxSize, + fileSystem = fileSystem, + taskRunner = TaskRunner.INSTANCE, + useCacheLock = true, ) /** Create a cache of at most [maxSize] bytes in [directory]. */ @@ -178,6 +180,7 @@ class Cache internal constructor( valueCount = ENTRY_COUNT, maxSize = maxSize, taskRunner = taskRunner, + useCacheLock = useCacheLock, ) // read and write statistics, all guarded by 'this'. diff --git a/okhttp/src/commonJvmAndroid/kotlin/okhttp3/internal/cache/DiskLruCache.kt b/okhttp/src/commonJvmAndroid/kotlin/okhttp3/internal/cache/DiskLruCache.kt index 8934e9f48bf6..a5d374d1a891 100644 --- a/okhttp/src/commonJvmAndroid/kotlin/okhttp3/internal/cache/DiskLruCache.kt +++ b/okhttp/src/commonJvmAndroid/kotlin/okhttp3/internal/cache/DiskLruCache.kt @@ -95,6 +95,7 @@ class DiskLruCache( maxSize: Long, /** Used for asynchronous journal rebuilds. */ taskRunner: TaskRunner, + private val useCacheLock: Boolean, ) : Closeable, Flushable { lateinit var cacheLock: Closeable @@ -245,7 +246,7 @@ class DiskLruCache( civilizedFileSystem = fileSystem.isCivilized(journalFileBackup) - cacheLock = openLock(fileSystem, directory) + cacheLock = if (useCacheLock) openLock(fileSystem, directory) else Closeable {} try { // Prefer to pick up where we left off. diff --git a/okhttp/src/jvmTest/kotlin/okhttp3/internal/cache/DiskLruCacheTest.kt b/okhttp/src/jvmTest/kotlin/okhttp3/internal/cache/DiskLruCacheTest.kt index 62eb776f0acd..7e1f31d902ae 100644 --- a/okhttp/src/jvmTest/kotlin/okhttp3/internal/cache/DiskLruCacheTest.kt +++ b/okhttp/src/jvmTest/kotlin/okhttp3/internal/cache/DiskLruCacheTest.kt @@ -81,7 +81,15 @@ class DiskLruCacheTest { private fun createNewCacheWithSize(maxSize: Int) { cache = - DiskLruCache(filesystem, cacheDir, appVersion, 2, maxSize.toLong(), taskRunner).also { + DiskLruCache( + fileSystem = filesystem, + directory = cacheDir, + appVersion = appVersion, + valueCount = 2, + maxSize = maxSize.toLong(), + taskRunner = taskRunner, + useCacheLock = true, + ).also { toClose.add(it) } synchronized(cache) { cache.initialize() } @@ -145,7 +153,15 @@ class DiskLruCacheTest { filesystem.setFaultyDelete(cacheDir / "k1.0.tmp", true) filesystem.setFaultyDelete(cacheDir, true) cache = - DiskLruCache(filesystem, cacheDir, appVersion, 2, Int.MAX_VALUE.toLong(), taskRunner).also { + DiskLruCache( + fileSystem = filesystem, + directory = cacheDir, + appVersion = appVersion, + valueCount = 2, + maxSize = Int.MAX_VALUE.toLong(), + taskRunner = taskRunner, + useCacheLock = true, + ).also { toClose.add(it) } assertFailsWith { @@ -816,7 +832,15 @@ class DiskLruCacheTest { fun constructorDoesNotAllowZeroCacheSize(parameters: Pair) { setUp(parameters.first, parameters.second) assertFailsWith { - DiskLruCache(filesystem, cacheDir, appVersion, 2, 0, taskRunner) + DiskLruCache( + fileSystem = filesystem, + directory = cacheDir, + appVersion = appVersion, + valueCount = 2, + maxSize = 0, + taskRunner = taskRunner, + useCacheLock = true, + ) } } @@ -825,7 +849,15 @@ class DiskLruCacheTest { fun constructorDoesNotAllowZeroValuesPerEntry(parameters: Pair) { setUp(parameters.first, parameters.second) assertFailsWith { - DiskLruCache(filesystem, cacheDir, appVersion, 0, 10, taskRunner) + DiskLruCache( + fileSystem = filesystem, + directory = cacheDir, + appVersion = appVersion, + valueCount = 0, + maxSize = 10, + taskRunner = taskRunner, + useCacheLock = true, + ) } } @@ -1163,7 +1195,15 @@ class DiskLruCacheTest { cache.close() val dir = (cacheDir / "testOpenCreatesDirectoryIfNecessary").also { filesystem.createDirectories(it) } cache = - DiskLruCache(filesystem, dir, appVersion, 2, Int.MAX_VALUE.toLong(), taskRunner).also { + DiskLruCache( + fileSystem = filesystem, + directory = dir, + appVersion = appVersion, + valueCount = 2, + maxSize = Int.MAX_VALUE.toLong(), + taskRunner = taskRunner, + useCacheLock = true, + ).also { toClose.add(it) } set("a", "a", "a") @@ -1557,7 +1597,15 @@ class DiskLruCacheTest { setUp(parameters.first, parameters.second) // Create an uninitialized cache. cache = - DiskLruCache(filesystem, cacheDir, appVersion, 2, Int.MAX_VALUE.toLong(), taskRunner).also { + DiskLruCache( + fileSystem = filesystem, + directory = cacheDir, + appVersion = appVersion, + valueCount = 2, + maxSize = Int.MAX_VALUE.toLong(), + taskRunner = taskRunner, + useCacheLock = true, + ).also { toClose.add(it) } assertThat(cache.isClosed()).isFalse() @@ -1583,7 +1631,15 @@ class DiskLruCacheTest { // Confirm that the fault didn't corrupt entries stored before the fault was introduced. cache.close() cache = - DiskLruCache(filesystem, cacheDir, appVersion, 2, Int.MAX_VALUE.toLong(), taskRunner).also { + DiskLruCache( + fileSystem = filesystem, + directory = cacheDir, + appVersion = appVersion, + valueCount = 2, + maxSize = Int.MAX_VALUE.toLong(), + taskRunner = taskRunner, + useCacheLock = true, + ).also { toClose.add(it) } assertValue("a", "a", "a") @@ -1617,7 +1673,15 @@ class DiskLruCacheTest { // Confirm that the fault didn't corrupt entries stored before the fault was introduced. cache.close() cache = - DiskLruCache(filesystem, cacheDir, appVersion, 2, Int.MAX_VALUE.toLong(), taskRunner).also { + DiskLruCache( + fileSystem = filesystem, + directory = cacheDir, + appVersion = appVersion, + valueCount = 2, + maxSize = Int.MAX_VALUE.toLong(), + taskRunner = taskRunner, + useCacheLock = true, + ).also { toClose.add(it) } assertValue("a", "a", "a") @@ -1647,7 +1711,15 @@ class DiskLruCacheTest { // Confirm that the fault didn't corrupt entries stored before the fault was introduced. cache.close() cache = - DiskLruCache(filesystem, cacheDir, appVersion, 2, Int.MAX_VALUE.toLong(), taskRunner).also { + DiskLruCache( + fileSystem = filesystem, + directory = cacheDir, + appVersion = appVersion, + valueCount = 2, + maxSize = Int.MAX_VALUE.toLong(), + taskRunner = taskRunner, + useCacheLock = true, + ).also { toClose.add(it) } assertValue("a", "a", "a") @@ -1671,7 +1743,15 @@ class DiskLruCacheTest { filesystem.setFaultyWrite(journalFile, false) cache.close() cache = - DiskLruCache(filesystem, cacheDir, appVersion, 2, Int.MAX_VALUE.toLong(), taskRunner).also { + DiskLruCache( + fileSystem = filesystem, + directory = cacheDir, + appVersion = appVersion, + valueCount = 2, + maxSize = Int.MAX_VALUE.toLong(), + taskRunner = taskRunner, + useCacheLock = true, + ).also { toClose.add(it) } assertAbsent("a")