From 39b8a9f287d1a09fe9c3bbc6ec8de73506094175 Mon Sep 17 00:00:00 2001 From: Kouji Matsui Date: Tue, 28 May 2024 17:58:16 +0900 Subject: [PATCH 1/2] Extracted file system abstraction interface. --- .../Primitive/RepositoryFactoryExtension.fs | 6 +- .../Structures/RepositoryFactoryExtension.fs | 6 +- GitReader.Core/IO/DeltaDecodedStream.cs | 8 +- GitReader.Core/IO/FileStreamCache.cs | 27 ++- GitReader.Core/IO/IFileSystem.cs | 191 ++++++++++++++++++ GitReader.Core/IO/MemoizedStream.cs | 14 +- GitReader.Core/IO/TemporaryFile.cs | 23 ++- GitReader.Core/Internal/IndexReader.cs | 8 +- GitReader.Core/Internal/ObjectAccessor.cs | 52 +++-- GitReader.Core/Internal/RepositoryAccessor.cs | 104 +++------- GitReader.Core/Internal/Utilities.cs | 48 ----- .../Primitive/PrimitiveRepository.cs | 7 +- .../Primitive/PrimitiveRepositoryFacade.cs | 27 ++- GitReader.Core/Repository.cs | 10 +- .../Structures/StructuredRepository.cs | 6 +- .../Structures/StructuredRepositoryFacade.cs | 27 ++- .../Internal/RepositoryAccessorTests.cs | 11 +- .../Primitive/RepositoryFactoryExtension.cs | 10 +- .../Structures/RepositoryFactoryExtension.cs | 8 +- 19 files changed, 391 insertions(+), 202 deletions(-) create mode 100644 GitReader.Core/IO/IFileSystem.cs diff --git a/FSharp.GitReader/Primitive/RepositoryFactoryExtension.fs b/FSharp.GitReader/Primitive/RepositoryFactoryExtension.fs index 42c416e..8ec911f 100644 --- a/FSharp.GitReader/Primitive/RepositoryFactoryExtension.fs +++ b/FSharp.GitReader/Primitive/RepositoryFactoryExtension.fs @@ -10,6 +10,7 @@ namespace GitReader.Primitive open GitReader +open GitReader.IO open System.Threading [] @@ -18,4 +19,7 @@ module public RepositoryFactoryExtension = type RepositoryFactory with member _.openPrimitive(path: string, ?ct: CancellationToken) = PrimitiveRepositoryFacade.OpenPrimitiveAsync( - path, unwrapCT ct) |> Async.AwaitTask + path, new StandardFileSystem(65536), unwrapCT ct) |> Async.AwaitTask + member _.openPrimitive(path: string, fileSystem: IFileSystem, ?ct: CancellationToken) = + PrimitiveRepositoryFacade.OpenPrimitiveAsync( + path, fileSystem, unwrapCT ct) |> Async.AwaitTask diff --git a/FSharp.GitReader/Structures/RepositoryFactoryExtension.fs b/FSharp.GitReader/Structures/RepositoryFactoryExtension.fs index 5866208..177c46d 100644 --- a/FSharp.GitReader/Structures/RepositoryFactoryExtension.fs +++ b/FSharp.GitReader/Structures/RepositoryFactoryExtension.fs @@ -10,6 +10,7 @@ namespace GitReader.Structures open GitReader +open GitReader.IO open System.Threading [] @@ -18,4 +19,7 @@ module public RepositoryFactoryExtension = type RepositoryFactory with member _.openStructured(path: string, ?ct: CancellationToken) = StructuredRepositoryFacade.OpenStructuredAsync( - path, unwrapCT ct) |> Async.AwaitTask + path, new StandardFileSystem(65536), unwrapCT ct) |> Async.AwaitTask + member _.openStructured(path: string, fileSystem: IFileSystem, ?ct: CancellationToken) = + StructuredRepositoryFacade.OpenStructuredAsync( + path, fileSystem, unwrapCT ct) |> Async.AwaitTask diff --git a/GitReader.Core/IO/DeltaDecodedStream.cs b/GitReader.Core/IO/DeltaDecodedStream.cs index 8dc8add..22f1680 100644 --- a/GitReader.Core/IO/DeltaDecodedStream.cs +++ b/GitReader.Core/IO/DeltaDecodedStream.cs @@ -612,11 +612,13 @@ public override void Flush() => #if NET45_OR_GREATER || NETSTANDARD || NETCOREAPP public static async ValueTask CreateAsync( Stream baseObjectStream, Stream deltaStream, - BufferPool pool, CancellationToken ct) + BufferPool pool, IFileSystem fileSystem, + CancellationToken ct) #else public static async Task CreateAsync( Stream baseObjectStream, Stream deltaStream, - BufferPool pool, CancellationToken ct) + BufferPool pool, IFileSystem fileSystem, + CancellationToken ct) #endif { void Throw(int step) => @@ -653,7 +655,7 @@ void Throw(int step) => } return new( - await MemoizedStream.CreateAsync(baseObjectStream, (long)baseObjectLength, pool, ct), + await MemoizedStream.CreateAsync(baseObjectStream, (long)baseObjectLength, pool, fileSystem, ct), deltaStream, preloadBuffer.Detach(), preloadIndex, read, (long)decodedObjectLength); } diff --git a/GitReader.Core/IO/FileStreamCache.cs b/GitReader.Core/IO/FileStreamCache.cs index ade7f83..591d0c3 100644 --- a/GitReader.Core/IO/FileStreamCache.cs +++ b/GitReader.Core/IO/FileStreamCache.cs @@ -11,6 +11,7 @@ using System.Collections.Generic; using System.Diagnostics; using System.IO; +using System.Runtime.InteropServices.ComTypes; using System.Threading; using System.Threading.Tasks; @@ -25,15 +26,14 @@ private sealed class CachedStream : Stream // and the timing is managed by FileStreamCache. private FileStreamCache parent; - private FileStream rawStream; + private Stream rawStream; internal readonly string path; - public CachedStream(FileStreamCache parent, string path) + public CachedStream(FileStreamCache parent, string path, Stream rawStream) { this.parent = parent; this.path = path; - this.rawStream = new( - path, FileMode.Open, FileAccess.Read, FileShare.Read, 65536, true); + this.rawStream = rawStream; } public override bool CanRead => @@ -118,9 +118,13 @@ public override void Flush() => internal static readonly int MaxReservedStreams = Environment.ProcessorCount * 2; + private readonly IFileSystem fileSystem; private readonly Dictionary> reserved = new(); private readonly LinkedList streamsLRU = new(); + public FileStreamCache(IFileSystem fileSystem) => + this.fileSystem = fileSystem; + public void Dispose() { lock (this.reserved) @@ -172,9 +176,15 @@ private void Return(CachedStream stream) } } - public Stream Open(string path) +#if NET45_OR_GREATER || NETSTANDARD || NETCOREAPP + public async ValueTask OpenAsync( + string path, CancellationToken ct) +#else + public async Task OpenAsync( + string path, CancellationToken ct) +#endif { - var fullPath = Path.GetFullPath(path); + var fullPath = this.fileSystem.GetFullPath(path); lock (this.reserved) { @@ -202,8 +212,11 @@ public Stream Open(string path) { this.RemoveLastReserved(); } - return new CachedStream(this, fullPath); } } + + var stream2 = await this.fileSystem.OpenAsync(path, true, ct); + + return new CachedStream(this, path, stream2); } } diff --git a/GitReader.Core/IO/IFileSystem.cs b/GitReader.Core/IO/IFileSystem.cs new file mode 100644 index 0000000..35eba91 --- /dev/null +++ b/GitReader.Core/IO/IFileSystem.cs @@ -0,0 +1,191 @@ +//////////////////////////////////////////////////////////////////////////// +// +// GitReader - Lightweight Git local repository traversal library. +// Copyright (c) Kouji Matsui (@kozy_kekyo, @kekyo@mastodon.cloud) +// +// Licensed under Apache-v2: https://opensource.org/licenses/Apache-2.0 +// +//////////////////////////////////////////////////////////////////////////// + +using GitReader.Internal; +using System; +using System.IO; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; + +namespace GitReader.IO; + +public readonly struct TemporaryFileDescriptor +{ + public readonly string Path; + public readonly Stream Stream; + + public TemporaryFileDescriptor(string path, Stream stream) + { + this.Path = path; + this.Stream = stream; + } + + public void Deconstruct(out string path, out Stream stream) + { + path = this.Path; + stream = this.Stream; + } +} + +public interface IFileSystem +{ + string Combine(params string[] paths); + + string GetDirectoryPath(string path); + + string GetFullPath(string path); + + bool IsPathRooted(string path); + + string ResolveRelativePath(string basePath, string path); + + Task IsFileExistsAsync( + string path, CancellationToken ct); + + Task GetFilesAsync( + string basePath, string match, CancellationToken ct); + + Task OpenAsync( + string path, bool isSeekable, CancellationToken ct); + + Task CreateTemporaryAsync( + CancellationToken ct); +} + +public sealed class StandardFileSystem : IFileSystem +{ + private static readonly bool isWindows = +#if NETSTANDARD1_6 + !string.IsNullOrWhiteSpace(Environment.GetEnvironmentVariable("HOMEDRIVE")); +#else + Environment.OSVersion.Platform.ToString().Contains("Win"); +#endif + + private static readonly string homePath = + Path.GetFullPath(isWindows ? + $"{Environment.GetEnvironmentVariable("HOMEDRIVE") ?? "C:"}{Environment.GetEnvironmentVariable("HOMEPATH") ?? "\\"}" : + (Environment.GetEnvironmentVariable("HOME") ?? "/")); + + private readonly int bufferSize; + + public StandardFileSystem(int bufferSize) => + this.bufferSize = bufferSize; + +#if NET35 + public string Combine(params string[] paths) => + paths.Aggregate(Path.Combine); +#else + public string Combine(params string[] paths) => + Path.Combine(paths); +#endif + + public string GetDirectoryPath(string path) => + Path.GetDirectoryName(path) switch + { + // Not accurate in Windows, but a compromise... + null => Path.DirectorySeparatorChar.ToString(), + "" => string.Empty, + var dp => dp, + }; + + public string GetFullPath(string path) => + Path.GetFullPath(path); + + public bool IsPathRooted(string path) => + Path.IsPathRooted(path); + + public string ResolveRelativePath(string basePath, string path) => + Path.GetFullPath(Path.IsPathRooted(path) ? + path : + path.StartsWith("~/") ? + Combine(homePath, path.Substring(2)) : + Combine(basePath, path)); + + public Task IsFileExistsAsync(string path, CancellationToken ct) => + Utilities.FromResult(File.Exists(path)); + + public Task GetFilesAsync( + string basePath, string match, CancellationToken ct) => + Utilities.FromResult(Directory.Exists(basePath) ? + Directory.GetFiles(basePath, match, SearchOption.AllDirectories) : + Utilities.Empty()); + + public async Task OpenAsync( + string path, bool isSeekable, CancellationToken ct) + { + // Many Git clients are supposed to be OK to use at the same time. + // If we try to open a file with the FileShare.Read share flag (i.e., write-protected), + // an error will occur when another Git client is opening (with non-read-sharable) the file. + // Retry here as this situation is expected to take a short time to complete. + // However, if multiple files are opened sequentially, + // a deadlock may occur depending on the order in which they are opened. + // Because they are not processed as transactions. + // If a constraint is imposed by the number of open attempts, + // and if the file cannot be opened by any means, + // degrade to FileShare.ReadWrite and attempt to open it. + // (In this case it might read the wrong, that is the value in the process of writing...) + + Random? r = null; + + for (var count = 0; count < 20; count++) + { + try + { + return new FileStream( + path, + FileMode.Open, + FileAccess.Read, + FileShare.Read, + this.bufferSize, true); + } + catch (FileNotFoundException) + { + throw; + } + catch (IOException) + { + } + + if (r == null) + { + r = new Random(); + } + + await Utilities.Delay(TimeSpan.FromMilliseconds(r.Next(10, 500)), ct); + } + + // Gave up and will try to open with read-write... + return new FileStream( + path, + FileMode.Open, + FileAccess.Read, + FileShare.ReadWrite, + 65536, + true); + } + + public Task CreateTemporaryAsync( + CancellationToken ct) + { + var path = this.Combine( + Path.GetTempPath(), + Path.GetTempFileName()); + + var stream = new FileStream( + path, + FileMode.Create, + FileAccess.ReadWrite, + FileShare.None, + this.bufferSize, + true); + + return Utilities.FromResult(new TemporaryFileDescriptor(path, stream)); + } +} diff --git a/GitReader.Core/IO/MemoizedStream.cs b/GitReader.Core/IO/MemoizedStream.cs index 58cbb17..373469c 100644 --- a/GitReader.Core/IO/MemoizedStream.cs +++ b/GitReader.Core/IO/MemoizedStream.cs @@ -309,15 +309,23 @@ public override void Write(byte[] buffer, int offset, int count) => #if NET45_OR_GREATER || NETSTANDARD || NETCOREAPP public static async ValueTask CreateAsync( - Stream parent, long parentLength, BufferPool pool, CancellationToken ct) + Stream parent, + long parentLength, + BufferPool pool, + IFileSystem fileSystem, + CancellationToken ct) #else public static async Task CreateAsync( - Stream parent, long parentLength, BufferPool pool, CancellationToken ct) + Stream parent, + long parentLength, + BufferPool pool, + IFileSystem fileSystem, + CancellationToken ct) #endif { if (parentLength >= memoizeToFileSize) { - var temporaryFile = TemporaryFile.CreateFile(); + var temporaryFile = await TemporaryFile.CreateFileAsync(fileSystem, ct); return new(parent, parentLength, temporaryFile, temporaryFile.Stream, pool); } else if (parentLength < 0) diff --git a/GitReader.Core/IO/TemporaryFile.cs b/GitReader.Core/IO/TemporaryFile.cs index 9b88fba..022318b 100644 --- a/GitReader.Core/IO/TemporaryFile.cs +++ b/GitReader.Core/IO/TemporaryFile.cs @@ -12,6 +12,10 @@ using System.Diagnostics; using System.IO; using System.Runtime.InteropServices; +using System.Threading; +using System.Threading.Tasks; + + #if !NETSTANDARD1_6 using System.Runtime.ConstrainedExecution; @@ -72,18 +76,15 @@ public void Dispose() public string Path => (string)this.pathHandle.Target!; - public static TemporaryFile CreateFile() +#if NET45_OR_GREATER || NETSTANDARD || NETCOREAPP + public static async ValueTask CreateFileAsync( + IFileSystem fileSystem, CancellationToken ct) +#else + public static async Task CreateFileAsync( + IFileSystem fileSystem, CancellationToken ct) +#endif { - var path = Utilities.Combine( - System.IO.Path.GetTempPath(), - System.IO.Path.GetTempFileName()); - - var stream = new FileStream( - path, - FileMode.Create, - FileAccess.ReadWrite, - FileShare.None); - + var (path, stream) = await fileSystem.CreateTemporaryAsync(ct); return new TemporaryFile(path, stream); } } diff --git a/GitReader.Core/Internal/IndexReader.cs b/GitReader.Core/Internal/IndexReader.cs index 1a5d321..4bea0fa 100644 --- a/GitReader.Core/Internal/IndexReader.cs +++ b/GitReader.Core/Internal/IndexReader.cs @@ -7,6 +7,7 @@ // //////////////////////////////////////////////////////////////////////////// +using GitReader.IO; using System; using System.Collections.Generic; using System.IO; @@ -33,18 +34,17 @@ internal static class IndexReader #if NET45_OR_GREATER || NETSTANDARD || NETCOREAPP public static async ValueTask> ReadIndexAsync( - string indexPath, BufferPool pool, CancellationToken ct) + string indexPath, IFileSystem fileSystem, BufferPool pool, CancellationToken ct) #else public static async Task> ReadIndexAsync( - string indexPath, BufferPool pool, CancellationToken ct) + string indexPath, IFileSystem fileSystem, BufferPool pool, CancellationToken ct) #endif { void Throw(int step) => throw new InvalidDataException( $"Broken index file: File={indexPath}, Step={step}"); - using var fs = new FileStream( - indexPath, FileMode.Open, FileAccess.Read, FileShare.Read, 65536, true); + using var fs = await fileSystem.OpenAsync(indexPath, false, ct); using var header = pool.Take(8); var read = await fs.ReadAsync(header, 0, header.Length, ct); diff --git a/GitReader.Core/Internal/ObjectAccessor.cs b/GitReader.Core/Internal/ObjectAccessor.cs index 1f495f0..f80df69 100644 --- a/GitReader.Core/Internal/ObjectAccessor.cs +++ b/GitReader.Core/Internal/ObjectAccessor.cs @@ -47,6 +47,7 @@ public ObjectStreamCacheHolder( } private readonly BufferPool pool; + private readonly IFileSystem fileSystem; private readonly FileStreamCache fileStreamCache; private readonly string objectsBasePath; private readonly string packedBasePath; @@ -60,14 +61,18 @@ public ObjectStreamCacheHolder( #endif public ObjectAccessor( - BufferPool pool, FileStreamCache fileStreamCache, string gitPath) + BufferPool pool, + IFileSystem fileSystem, + FileStreamCache fileStreamCache, + string gitPath) { this.pool = pool; + this.fileSystem = fileSystem; this.fileStreamCache = fileStreamCache; - this.objectsBasePath = Utilities.Combine( + this.objectsBasePath = fileSystem.Combine( gitPath, "objects"); - this.packedBasePath = Utilities.Combine( + this.packedBasePath = fileSystem.Combine( this.objectsBasePath, "pack"); this.streamLRUCacheExhaustTimer = @@ -119,7 +124,7 @@ void Throw(int step) => throw new InvalidDataException( $"Could not parse the object. Hash={hash}, Step={step}"); - var fs = this.fileStreamCache.Open(objectPath); + var fs = await this.fileStreamCache.OpenAsync(objectPath, ct); try { @@ -220,12 +225,13 @@ private async Task GetOrCacheIndexEntryAsync( } var dict = await IndexReader.ReadIndexAsync( - Utilities.Combine(this.packedBasePath, indexFileRelativePath), + this.fileSystem.Combine(this.packedBasePath, indexFileRelativePath), + this.fileSystem, this.pool, ct); cachedEntry = new( - Utilities.Combine( - Utilities.GetDirectoryPath(indexFileRelativePath), + this.fileSystem.Combine( + this.fileSystem.GetDirectoryPath(indexFileRelativePath), Path.GetFileNameWithoutExtension(indexFileRelativePath)), dict); @@ -450,7 +456,7 @@ void Throw(int step) => throw new InvalidDataException( $"Could not parse the object. File={packedFilePath}, Offset={offset}, Step={step}"); - var fs = this.fileStreamCache.Open(packedFilePath); + var fs = await this.fileStreamCache.OpenAsync(packedFilePath, ct); try { @@ -507,11 +513,12 @@ void Throw(int step) => objectEntry.Stream, new RangedStream(zlibStream, (long)objectSize), this.pool, + this.fileSystem, ct); var wrappedStream = this.AddToCache( packedFilePath, offset, objectEntry.Type, - await MemoizedStream.CreateAsync(deltaDecodedStream, -1, this.pool, ct), + await MemoizedStream.CreateAsync(deltaDecodedStream, -1, this.pool, this.fileSystem, ct), disableCaching); return new(wrappedStream, objectEntry.Type); @@ -555,11 +562,12 @@ await MemoizedStream.CreateAsync(deltaDecodedStream, -1, this.pool, ct), oe.Stream, new RangedStream(zlibStream, (long)objectSize), this.pool, + this.fileSystem, ct); var wrappedStream = this.AddToCache( packedFilePath, offset, oe.Type, - await MemoizedStream.CreateAsync(deltaDecodedStream, -1, this.pool, ct), + await MemoizedStream.CreateAsync(deltaDecodedStream, -1, this.pool, this.fileSystem, ct), disableCaching); return new(wrappedStream, oe.Type); @@ -586,7 +594,7 @@ await MemoizedStream.CreateAsync(deltaDecodedStream, -1, this.pool, ct), var wrappedStream = this.AddToCache( packedFilePath, offset, objectType, - await MemoizedStream.CreateAsync(zlibStream, (long)objectSize, this.pool, ct), + await MemoizedStream.CreateAsync(zlibStream, (long)objectSize, this.pool, this.fileSystem, ct), disableCaching); return new(wrappedStream, objectType); @@ -612,10 +620,10 @@ await MemoizedStream.CreateAsync(zlibStream, (long)objectSize, this.pool, ct), Hash hash, bool disableCaching, CancellationToken ct) #endif { + var files = await this.fileSystem.GetFilesAsync(this.packedBasePath, "pack-*.idx", ct); var entries = await Utilities.WhenAll( - Utilities.EnumerateFiles(this.packedBasePath, "pack-*.idx"). - Select(indexFilePath => this.GetOrCacheIndexEntryAsync( - indexFilePath.Substring(this.packedBasePath.Length + 1), ct))); + files.Select(indexFilePath => + this.GetOrCacheIndexEntryAsync(indexFilePath.Substring(this.packedBasePath.Length + 1), ct))); if (entries.Select(indexEntry => indexEntry.ObjectEntries.TryGetValue(hash, out var objectEntry) ? @@ -625,7 +633,7 @@ await MemoizedStream.CreateAsync(zlibStream, (long)objectSize, this.pool, ct), return null; } - var packedFilePath = Utilities.Combine( + var packedFilePath = this.fileSystem.Combine( this.packedBasePath, entry.BaseFileName + ".pack"); @@ -638,7 +646,7 @@ void Throw(int step) => Throw(1); } - if (!File.Exists(packedFilePath)) + if (!await this.fileSystem.IsFileExistsAsync(packedFilePath, ct)) { Throw(2); } @@ -650,25 +658,25 @@ void Throw(int step) => ////////////////////////////////////////////////////////////////////////// #if NET45_OR_GREATER || NETSTANDARD || NETCOREAPP - public ValueTask OpenAsync( + public async ValueTask OpenAsync( Hash hash, bool disableCaching, CancellationToken ct) #else - public Task OpenAsync( + public async Task OpenAsync( Hash hash, bool disableCaching, CancellationToken ct) #endif { - var objectPath = Utilities.Combine( + var objectPath = this.fileSystem.Combine( this.objectsBasePath, BitConverter.ToString(hash.HashCode, 0, 1).ToLowerInvariant(), BitConverter.ToString(hash.HashCode, 1).Replace("-", string.Empty).ToLowerInvariant()); - if (File.Exists(objectPath)) + if (await this.fileSystem.IsFileExistsAsync(objectPath, ct)) { - return this.OpenFromObjectFileAsync(objectPath, hash, ct); + return await this.OpenFromObjectFileAsync(objectPath, hash, ct); } else { - return this.OpenFromPackedAsync(hash, disableCaching, ct); + return await this.OpenFromPackedAsync(hash, disableCaching, ct); } } } diff --git a/GitReader.Core/Internal/RepositoryAccessor.cs b/GitReader.Core/Internal/RepositoryAccessor.cs index 582a886..64845d8 100644 --- a/GitReader.Core/Internal/RepositoryAccessor.cs +++ b/GitReader.Core/Internal/RepositoryAccessor.cs @@ -92,73 +92,32 @@ public HashResults(Hash hash, string[] names) internal static class RepositoryAccessor { -#if NET45_OR_GREATER || NETSTANDARD || NETCOREAPP - private static async ValueTask OpenFileAsync( - string path, CancellationToken ct) -#else - private static async Task OpenFileAsync( - string path, CancellationToken ct) -#endif - { - // Many Git clients are supposed to be OK to use at the same time. - // If we try to open a file with the FileShare.Read share flag (i.e., write-protected), - // an error will occur when another Git client is opening (with non-read-sharable) the file. - // Retry here as this situation is expected to take a short time to complete. - // However, if multiple files are opened sequentially, - // a deadlock may occur depending on the order in which they are opened. - // Because they are not processed as transactions. - // If a constraint is imposed by the number of open attempts, - // and if the file cannot be opened by any means, - // degrade to FileShare.ReadWrite and attempt to open it. - // (In this case it might read the wrong, that is the value in the process of writing...) - for (var count = 0; count < 10; count++) - { - try - { - return new FileStream(path, FileMode.Open, FileAccess.Read, FileShare.Read, 65536, true); - } - catch (FileNotFoundException) - { - throw; - } - catch (IOException) - { - } - - await Utilities.Delay(TimeSpan.FromMilliseconds(500), ct); - } - - // Gave up and will try to open with read-write... - return new FileStream(path, FileMode.Open, FileAccess.Read, FileShare.ReadWrite, 65536, true); - } - #if NET45_OR_GREATER || NETSTANDARD || NETCOREAPP public static async ValueTask DetectLocalRepositoryPathAsync( - string startPath, CancellationToken ct) + string startPath, IFileSystem fileSystem, CancellationToken ct) #else public static async Task DetectLocalRepositoryPathAsync( - string startPath, CancellationToken ct) + string startPath, IFileSystem fileSystem, CancellationToken ct) #endif { - var currentPath = Path.GetFullPath(startPath); + var currentPath = fileSystem.GetFullPath(startPath); while (true) { ct.ThrowIfCancellationRequested(); var candidatePath = Path.GetFileName(currentPath) != ".git" ? - Utilities.Combine(currentPath, ".git") : currentPath; + fileSystem.Combine(currentPath, ".git") : currentPath; - if (Directory.Exists(candidatePath) && - File.Exists(Path.Combine(candidatePath, "config"))) + if (await fileSystem.IsFileExistsAsync(fileSystem.Combine(candidatePath, "config"), ct)) { return candidatePath; } // Issue #11 - if (File.Exists(candidatePath)) + if (await fileSystem.IsFileExistsAsync(candidatePath, ct)) { - using var fs = await OpenFileAsync(candidatePath, ct); + using var fs = await fileSystem.OpenAsync(candidatePath, false, ct); var tr = new AsyncTextReader(fs); while (true) @@ -174,10 +133,9 @@ public static async Task DetectLocalRepositoryPathAsync( { // Resolve to full path (And normalize path directory separators) var gitDirPath = line.Substring(7).TrimStart(); - candidatePath = Utilities.ResolveRelativePath(currentPath, gitDirPath); + candidatePath = fileSystem.ResolveRelativePath(currentPath, gitDirPath); - if (Directory.Exists(candidatePath) && - File.Exists(Path.Combine(candidatePath, "config"))) + if (await fileSystem.IsFileExistsAsync(fileSystem.Combine(candidatePath, "config"), ct)) { return candidatePath; } @@ -195,7 +153,7 @@ public static async Task DetectLocalRepositoryPathAsync( throw new ArgumentException("Repository does not exist."); } - currentPath = Utilities.GetDirectoryPath(currentPath); + currentPath = fileSystem.GetDirectoryPath(currentPath); } } @@ -242,13 +200,13 @@ public static async Task> ReadRemoteReference Repository repository, CancellationToken ct) { - var path = Utilities.Combine(repository.GitPath, "config"); - if (!File.Exists(path)) + var path = repository.fileSystem.Combine(repository.GitPath, "config"); + if (!await repository.fileSystem.IsFileExistsAsync(path, ct)) { return new(new()); } - using var fs = await OpenFileAsync(path, ct); + using var fs = await repository.fileSystem.OpenAsync(path, false, ct); var tr = new AsyncTextReader(fs); var remotes = new Dictionary(); @@ -290,13 +248,13 @@ public static async Task ReadFetchHeadsAsync( var remoteNameByUrl = repository.remoteUrls. ToDictionary(entry => entry.Value, entry => entry.Key); - var path = Utilities.Combine(repository.GitPath, "FETCH_HEAD"); - if (!File.Exists(path)) + var path = repository.fileSystem.Combine(repository.GitPath, "FETCH_HEAD"); + if (!await repository.fileSystem.IsFileExistsAsync(path, ct)) { return new(new(new()), new(new()), new(new())); } - using var fs = await OpenFileAsync(path, ct); + using var fs = await repository.fileSystem.OpenAsync(path, false, ct); var tr = new AsyncTextReader(fs); var remoteBranches = new Dictionary(); @@ -382,13 +340,13 @@ public static async Task ReadPackedRefsAsync( Repository repository, CancellationToken ct) { - var path = Utilities.Combine(repository.GitPath, "packed-refs"); - if (!File.Exists(path)) + var path = repository.fileSystem.Combine(repository.GitPath, "packed-refs"); + if (!await repository.fileSystem.IsFileExistsAsync(path, ct)) { return new(new(new()), new(new()), new(new())); } - using var fs = await OpenFileAsync(path, ct); + using var fs = await repository.fileSystem.OpenAsync(path, false, ct); var tr = new AsyncTextReader(fs); var branches = new Dictionary(); @@ -495,10 +453,10 @@ public static async Task ReadPackedRefsAsync( Replace("refs/tags/", string.Empty); names.Add(name); - var path = Utilities.Combine( + var path = repository.fileSystem.Combine( repository.GitPath, currentLocation.Replace('/', Path.DirectorySeparatorChar)); - if (!File.Exists(path)) + if (!await repository.fileSystem.IsFileExistsAsync(path, ct)) { if (currentLocation.StartsWith("refs/heads/")) { @@ -524,7 +482,7 @@ public static async Task ReadPackedRefsAsync( return null; } - using var fs = await OpenFileAsync(path, ct); + using var fs = await repository.fileSystem.OpenAsync(path, false, ct); var tr = new AsyncTextReader(fs); var line = await tr.ReadLineAsync(ct); @@ -554,14 +512,14 @@ public static Task ReadReflogEntriesAsync( public static async Task ReadReflogEntriesAsync( Repository repository, string refRelativePath, CancellationToken ct) { - var path = Utilities.Combine( + var path = repository.fileSystem.Combine( repository.GitPath, "logs", refRelativePath); - if (!File.Exists(path)) + if (!await repository.fileSystem.IsFileExistsAsync(path, ct)) { return new PrimitiveReflogEntry[]{}; } - using var fs = await OpenFileAsync(path, ct); + using var fs = await repository.fileSystem.OpenAsync(path, false, ct); var tr = new AsyncTextReader(fs); var entries = new List(); @@ -587,10 +545,12 @@ public static async Task ReadReferencesAsync( ReferenceTypes type, CancellationToken ct) { - var headsPath = Utilities.Combine( + var headsPath = repository.fileSystem.Combine( repository.GitPath, "refs", GetReferenceTypeName(type)); + var files = await repository.fileSystem.GetFilesAsync( + headsPath, "*", ct); var references = (await Utilities.WhenAll( - Utilities.EnumerateFiles(headsPath, "*"). + files. #if NET45_OR_GREATER || NETSTANDARD || NETCOREAPP Select((Func>)(async path => #else @@ -656,10 +616,12 @@ public static async Task ReadTagReferencesAsync( Repository repository, CancellationToken ct) { - var headsPath = Utilities.Combine( + var headsPath = repository.fileSystem.Combine( repository.GitPath, "refs", "tags"); + var files = await repository.fileSystem.GetFilesAsync( + headsPath, "*", ct); var references = (await Utilities.WhenAll( - Utilities.EnumerateFiles(headsPath, "*"). + files. #if NET45_OR_GREATER || NETSTANDARD || NETCOREAPP Select((Func>)(async path => #else diff --git a/GitReader.Core/Internal/Utilities.cs b/GitReader.Core/Internal/Utilities.cs index db155a2..d05866b 100644 --- a/GitReader.Core/Internal/Utilities.cs +++ b/GitReader.Core/Internal/Utilities.cs @@ -135,18 +135,6 @@ internal static class Utilities private const long UnixEpochTicks = DaysTo1970 * TicksPerDay; private const long UnixEpochSeconds = UnixEpochTicks / TimeSpan.TicksPerSecond; - private static readonly bool isWindows = -#if NETSTANDARD1_6 - !string.IsNullOrWhiteSpace(Environment.GetEnvironmentVariable("HOMEDRIVE")); -#else - Environment.OSVersion.Platform.ToString().Contains("Win"); -#endif - - private static readonly string homePath = - Path.GetFullPath(isWindows ? - $"{Environment.GetEnvironmentVariable("HOMEDRIVE") ?? "C:"}{Environment.GetEnvironmentVariable("HOMEPATH") ?? "\\"}" : - (Environment.GetEnvironmentVariable("HOME") ?? "/")); - public static DateTimeOffset FromUnixTimeSeconds(long seconds, TimeSpan offset) { var ticks = seconds * TimeSpan.TicksPerSecond + UnixEpochTicks; @@ -229,35 +217,6 @@ public static unsafe bool HasFlag(this TEnum enumValue, TEnum flags) EnumHelper.HasFlag(enumValue, flags); #endif -#if NET35 - public static IEnumerable EnumerateFiles(string basePath, string match) => - Directory.Exists(basePath) ? - Directory.GetFiles(basePath, match, SearchOption.AllDirectories) : - Empty(); -#else - public static IEnumerable EnumerateFiles(string basePath, string match) => - Directory.Exists(basePath) ? - Directory.EnumerateFiles(basePath, match, SearchOption.AllDirectories) : - Empty(); -#endif - -#if NET35 - public static string Combine(params string[] paths) => - paths.Aggregate(Path.Combine); -#else - public static string Combine(params string[] paths) => - Path.Combine(paths); -#endif - - public static string GetDirectoryPath(string path) => - Path.GetDirectoryName(path) switch - { - // Not accurate in Windows, but a compromise... - null => Path.DirectorySeparatorChar.ToString(), - "" => string.Empty, - var dp => dp, - }; - public static void MakeBigEndian( byte[] buffer, int index, int size) { @@ -704,11 +663,4 @@ public static string ToGitAuthorString( Signature signature) => signature.MailAddress is { } mailAddress ? $"{signature.Name} <{mailAddress}>" : signature.Name; - - public static string ResolveRelativePath(string basePath, string path) => - Path.GetFullPath(Path.IsPathRooted(path) ? - path : - path.StartsWith("~/") ? - Combine(homePath, path.Substring(2)) : - Combine(basePath, path)); } diff --git a/GitReader.Core/Primitive/PrimitiveRepository.cs b/GitReader.Core/Primitive/PrimitiveRepository.cs index ea0e678..b86934f 100644 --- a/GitReader.Core/Primitive/PrimitiveRepository.cs +++ b/GitReader.Core/Primitive/PrimitiveRepository.cs @@ -7,13 +7,16 @@ // //////////////////////////////////////////////////////////////////////////// +using GitReader.IO; + namespace GitReader.Primitive; public sealed class PrimitiveRepository : Repository { internal PrimitiveRepository( - string repositoryPath) : - base(repositoryPath) + string repositoryPath, + IFileSystem fileSystem) : + base(repositoryPath, fileSystem) { } } diff --git a/GitReader.Core/Primitive/PrimitiveRepositoryFacade.cs b/GitReader.Core/Primitive/PrimitiveRepositoryFacade.cs index eb09f9d..600f974 100644 --- a/GitReader.Core/Primitive/PrimitiveRepositoryFacade.cs +++ b/GitReader.Core/Primitive/PrimitiveRepositoryFacade.cs @@ -8,6 +8,7 @@ //////////////////////////////////////////////////////////////////////////// using GitReader.Internal; +using GitReader.IO; using System; using System.IO; using System.Linq; @@ -19,9 +20,11 @@ namespace GitReader.Primitive; internal static class PrimitiveRepositoryFacade { private static async Task InternalOpenPrimitiveAsync( - string repositoryPath, CancellationToken ct) + string repositoryPath, + IFileSystem fileSystem, + CancellationToken ct) { - var repository = new PrimitiveRepository(repositoryPath); + var repository = new PrimitiveRepository(repositoryPath, fileSystem); try { @@ -45,10 +48,13 @@ private static async Task InternalOpenPrimitiveAsync( } public static async Task OpenPrimitiveAsync( - string path, CancellationToken ct) + string path, + IFileSystem fileSystem, + CancellationToken ct) { - var repositoryPath = await RepositoryAccessor.DetectLocalRepositoryPathAsync(path, ct); - return await InternalOpenPrimitiveAsync(repositoryPath, ct); + var repositoryPath = await RepositoryAccessor.DetectLocalRepositoryPathAsync( + path, fileSystem, ct); + return await InternalOpenPrimitiveAsync(repositoryPath, fileSystem, ct); } ////////////////////////////////////////////////////////////////////////// @@ -94,7 +100,7 @@ public static async Task GetTagReferenceAsync( throw new ArgumentException($"Could not find a tag: {tagName}"); } - public static Task OpenSubModuleAsync( + public static async Task OpenSubModuleAsync( Repository repository, PrimitiveTreeEntry[] treePath, CancellationToken ct) { @@ -107,15 +113,16 @@ public static Task OpenSubModuleAsync( throw new ArgumentException($"Could not use non-submodule entry: {treePath[treePath.Length - 1]}"); } - var repositoryPath = Utilities.Combine( + var repositoryPath = repository.fileSystem.Combine( repository.GitPath, "modules", - Utilities.Combine(treePath.Select(tree => tree.Name).ToArray())); + repository.fileSystem.Combine(treePath.Select(tree => tree.Name).ToArray())); - if (!Directory.Exists(repositoryPath)) + if (!await repository.fileSystem.IsFileExistsAsync( + repository.fileSystem.Combine(repositoryPath, "config"), ct)) { throw new ArgumentException("Submodule repository does not exist."); } - return InternalOpenPrimitiveAsync(repositoryPath, ct); + return await InternalOpenPrimitiveAsync(repositoryPath, repository.fileSystem, ct); } } diff --git a/GitReader.Core/Repository.cs b/GitReader.Core/Repository.cs index 244f0e7..8a82e20 100644 --- a/GitReader.Core/Repository.cs +++ b/GitReader.Core/Repository.cs @@ -18,16 +18,20 @@ namespace GitReader; public abstract class Repository : IDisposable { internal BufferPool pool = new(); - internal FileStreamCache fileStreamCache = new(); + internal IFileSystem fileSystem; + internal FileStreamCache fileStreamCache; internal ObjectAccessor objectAccessor; internal ReadOnlyDictionary remoteUrls = null!; internal ReferenceCache referenceCache; private protected Repository( - string gitPath) + string gitPath, + IFileSystem fileSystem) { this.GitPath = gitPath; - this.objectAccessor = new(this.pool, this.fileStreamCache, gitPath); + this.fileSystem = fileSystem; + this.fileStreamCache = new(this.fileSystem); + this.objectAccessor = new(this.pool, this.fileSystem, this.fileStreamCache, gitPath); } public void Dispose() diff --git a/GitReader.Core/Structures/StructuredRepository.cs b/GitReader.Core/Structures/StructuredRepository.cs index 25ad6e2..0c096df 100644 --- a/GitReader.Core/Structures/StructuredRepository.cs +++ b/GitReader.Core/Structures/StructuredRepository.cs @@ -8,6 +8,7 @@ //////////////////////////////////////////////////////////////////////////// using GitReader.Collections; +using GitReader.IO; using System; using System.ComponentModel; @@ -27,8 +28,9 @@ public sealed class StructuredRepository : Repository internal ReadOnlyArray stashes = null!; internal StructuredRepository( - string repositoryPath) : - base(repositoryPath) + string repositoryPath, + IFileSystem fileSystem) : + base(repositoryPath, fileSystem) { } diff --git a/GitReader.Core/Structures/StructuredRepositoryFacade.cs b/GitReader.Core/Structures/StructuredRepositoryFacade.cs index 876793f..dc8e180 100644 --- a/GitReader.Core/Structures/StructuredRepositoryFacade.cs +++ b/GitReader.Core/Structures/StructuredRepositoryFacade.cs @@ -9,6 +9,7 @@ using GitReader.Collections; using GitReader.Internal; +using GitReader.IO; using GitReader.Primitive; using System; using System.Diagnostics; @@ -167,9 +168,11 @@ public static async Task GetHeadReflogsAsync( ////////////////////////////////////////////////////////////////////////// private static async Task InternalOpenStructuredAsync( - string repositoryPath, CancellationToken ct) + string repositoryPath, + IFileSystem fileSystem, + CancellationToken ct) { - var repository = new StructuredRepository(repositoryPath); + var repository = new StructuredRepository(repositoryPath, fileSystem); try { @@ -206,10 +209,13 @@ private static async Task InternalOpenStructuredAsync( } public static async Task OpenStructuredAsync( - string path, CancellationToken ct) + string path, + IFileSystem fileSystem, + CancellationToken ct) { - var repositoryPath = await RepositoryAccessor.DetectLocalRepositoryPathAsync(path, ct); - return await InternalOpenStructuredAsync(repositoryPath, ct); + var repositoryPath = await RepositoryAccessor.DetectLocalRepositoryPathAsync( + path, fileSystem, ct); + return await InternalOpenStructuredAsync(repositoryPath, fileSystem, ct); } ////////////////////////////////////////////////////////////////////////// @@ -410,27 +416,28 @@ async Task GetChildrenAsync( return treeRoot; } - public static Task OpenSubModuleAsync( + public static async Task OpenSubModuleAsync( TreeSubModuleEntry subModule, CancellationToken ct) { var (repository, _) = GetRelatedRepository(subModule); - var repositoryPath = Utilities.Combine( + var repositoryPath = repository.fileSystem.Combine( repository.GitPath, "modules", - Utilities.Combine(subModule. + repository.fileSystem.Combine(subModule. Traverse(tree => tree.Parent as TreeEntry). Select(tree => tree.Name). Reverse(). ToArray())); - if (!Directory.Exists(repositoryPath)) + if (!await repository.fileSystem.IsFileExistsAsync( + repository.fileSystem.Combine(repositoryPath, "config"), ct)) { throw new ArgumentException("Submodule repository does not exist."); } - return InternalOpenStructuredAsync(repositoryPath, ct); + return await InternalOpenStructuredAsync(repositoryPath, repository.fileSystem, ct); } public static Task OpenBlobAsync( diff --git a/GitReader.Tests/Internal/RepositoryAccessorTests.cs b/GitReader.Tests/Internal/RepositoryAccessorTests.cs index 35565ef..ab1ebfb 100644 --- a/GitReader.Tests/Internal/RepositoryAccessorTests.cs +++ b/GitReader.Tests/Internal/RepositoryAccessorTests.cs @@ -7,6 +7,7 @@ // //////////////////////////////////////////////////////////////////////////// +using GitReader.IO; using NUnit.Framework; using System.IO; using System.IO.Compression; @@ -34,7 +35,10 @@ public async Task DetectLocalRepositoryPath(int depth) Path.Combine("artifacts", "test1.zip"), basePath); - var actual = await RepositoryAccessor.DetectLocalRepositoryPathAsync(innerPath, default); + var actual = await RepositoryAccessor.DetectLocalRepositoryPathAsync( + innerPath, + new StandardFileSystem(65536), + default); Assert.AreEqual(Path.Combine(basePath, ".git"), actual); } @@ -52,7 +56,10 @@ public async Task DetectLocalRepositoryPathFromDotGitFile() var innerPath = Path.Combine(basePath, "GitReader"); - var actual = await RepositoryAccessor.DetectLocalRepositoryPathAsync(innerPath, default); + var actual = await RepositoryAccessor.DetectLocalRepositoryPathAsync( + innerPath, + new StandardFileSystem(65536), + default); Assert.AreEqual(Path.Combine(basePath, ".git", "modules", "GitReader"), actual); } diff --git a/GitReader/Primitive/RepositoryFactoryExtension.cs b/GitReader/Primitive/RepositoryFactoryExtension.cs index fb4a494..87721dd 100644 --- a/GitReader/Primitive/RepositoryFactoryExtension.cs +++ b/GitReader/Primitive/RepositoryFactoryExtension.cs @@ -7,6 +7,7 @@ // //////////////////////////////////////////////////////////////////////////// +using GitReader.IO; using System.Threading; using System.Threading.Tasks; @@ -17,5 +18,12 @@ public static class RepositoryFactoryExtension public static Task OpenPrimitiveAsync( this RepositoryFactory _, string path, CancellationToken ct = default) => - PrimitiveRepositoryFacade.OpenPrimitiveAsync(path, ct); + PrimitiveRepositoryFacade.OpenPrimitiveAsync( + path, new StandardFileSystem(65536), ct); + + public static Task OpenPrimitiveAsync( + this RepositoryFactory _, + string path, IFileSystem fileSystem, CancellationToken ct = default) => + PrimitiveRepositoryFacade.OpenPrimitiveAsync( + path, fileSystem, ct); } diff --git a/GitReader/Structures/RepositoryFactoryExtension.cs b/GitReader/Structures/RepositoryFactoryExtension.cs index 0b23e43..ee28108 100644 --- a/GitReader/Structures/RepositoryFactoryExtension.cs +++ b/GitReader/Structures/RepositoryFactoryExtension.cs @@ -7,6 +7,7 @@ // //////////////////////////////////////////////////////////////////////////// +using GitReader.IO; using System.Threading; using System.Threading.Tasks; @@ -17,5 +18,10 @@ public static class RepositoryFactoryExtension public static Task OpenStructureAsync( this RepositoryFactory _, string path, CancellationToken ct = default) => - StructuredRepositoryFacade.OpenStructuredAsync(path, ct); + StructuredRepositoryFacade.OpenStructuredAsync(path, new StandardFileSystem(65536), ct); + + public static Task OpenStructureAsync( + this RepositoryFactory _, + string path, IFileSystem fileSystem, CancellationToken ct = default) => + StructuredRepositoryFacade.OpenStructuredAsync(path, fileSystem,ct); } From a2eb69e07b25f580e5e6647b3ba706cd6f5f6ed0 Mon Sep 17 00:00:00 2001 From: Kouji Matsui Date: Tue, 28 May 2024 19:32:46 +0900 Subject: [PATCH 2/2] Updated readme. --- README.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/README.md b/README.md index 844d600..0235edb 100644 --- a/README.md +++ b/README.md @@ -635,6 +635,10 @@ Apache-v2 ## History +* 1.8.0: + * Added file system abstraction interface called `IFileSystem`. + * This interface allows repository access independent of the local file system. + * Currently undocumented, but there is a `StandardFileSystem` class that uses `System.IO` as its default implementation, so you may want to refer to this class. * 1.7.0: * Rebuilt on .NET 8.0 SDK. * 1.6.0: