using System.Runtime.CompilerServices; using System.Security.Cryptography; using System.Text; using AvParser.Core.Collecting; using AvParser.Core.Parsing; using AvParser.Core.Settings; using AvParser.Infrastructure.Collecting; using AvParser.Infrastructure.Media; using AvParser.Infrastructure.Storage; using Microsoft.Extensions.Logging.Abstractions; using ReactiveUI.Primitives.Signals; namespace AvParser.Infrastructure.Tests.Collecting; /// A source that hands back exactly the addresses the test names. internal sealed class ListSource(params string[] urls) : IMediaSource { public string Id => "test-source"; public string DisplayName => "Test source"; public string Description => string.Empty; public int Listings { get; private set; } public bool CanParse(MediaQuery input) => true; public async IAsyncEnumerable> ParseAsync( MediaQuery input, IProgress? progress, [EnumeratorCancellation] CancellationToken cancellationToken ) { Listings++; for (var index = 0; index < urls.Length; index++) { cancellationToken.ThrowIfCancellationRequested(); await Task.Yield(); yield return ParseOutcome.Success( new MediaCandidate(new Uri(urls[index])) { SourceId = Id, Ordinal = index + 1 } ); } } } /// A fetcher that stages bytes from a table instead of using a network. internal sealed class ScriptedFetcher(BlobStore blobs) : IMediaFetcher { public Dictionary Content { get; } = new(StringComparer.Ordinal); public HashSet Failing { get; } = new(StringComparer.Ordinal); public List Fetched { get; } = []; public TimeSpan Delay { get; set; } public async Task FetchAsync( MediaCandidate candidate, FetchOptions options, CancellationToken cancellationToken = default ) { var url = candidate.Url.AbsoluteUri; lock (Fetched) { Fetched.Add(url); } if (Delay > TimeSpan.Zero) { await Task.Delay(Delay, cancellationToken); } cancellationToken.ThrowIfCancellationRequested(); if (Failing.Contains(url) || !Content.TryGetValue(url, out var bytes)) { return new FetchResult(SeenOutcome.Failed, candidate.Url) { ErrorCode = "RequestFailed", HttpStatus = 500 }; } var temp = blobs.CreateTempPath(); await File.WriteAllBytesAsync(temp, bytes, cancellationToken); var hash = Convert.ToHexStringLower(SHA256.HashData(bytes)); return new FetchResult(SeenOutcome.Stored, candidate.Url) { Blob = MediaBlob.Create(hash, MediaKind.Png, bytes.Length), TempPath = temp, HttpStatus = 200, }; } } /// In-memory settings, so the runner never reads the developer's real profile. internal sealed class FixedSettings(AppSettings? initial = null) : ISettingsService, IDisposable { private readonly BehaviorSignal _current = new(initial ?? new AppSettings()); public AppSettings Current => _current.Value; public IObservable Changes => _current; public void Update(Func mutate) => _current.OnNext(mutate(_current.Value)); public Task FlushAsync(CancellationToken cancellationToken = default) => Task.CompletedTask; public void Dispose() => _current.Dispose(); } public sealed class CollectRunnerTests : IAsyncLifetime { private readonly string _root = Path.Combine(Path.GetTempPath(), "AvParserTests", Guid.NewGuid().ToString("N")); private AppPaths _paths = null!; private SqliteMediaIndex _index = null!; private BlobStore _blobs = null!; private MediaStore _store = null!; private ScriptedFetcher _fetcher = null!; private FixedSettings _settings = null!; private CollectRunner _runner = null!; public async ValueTask InitializeAsync() { _paths = new AppPaths(_root); _paths.EnsureCreated(); _index = new SqliteMediaIndex(_paths, NullLogger.Instance); _blobs = new BlobStore(_paths, NullLogger.Instance); _store = new MediaStore( _index, _blobs, new ShowcaseLinker(_paths, _blobs, NullLogger.Instance), NullLogger.Instance ); // Direct is allowed here: these tests are about the runner, not the proxy gate. _settings = new FixedSettings(new AppSettings { AllowDirectConnection = true }); _fetcher = new ScriptedFetcher(_blobs); _runner = new CollectRunner(_fetcher, _store, _settings, NullLogger.Instance); await _store.InitialiseAsync(TestContext.Current.CancellationToken); } public ValueTask DisposeAsync() { _settings.Dispose(); _index.Dispose(); Microsoft.Data.Sqlite.SqliteConnection.ClearAllPools(); if (Directory.Exists(_root)) { try { Directory.Delete(_root, recursive: true); } catch (IOException) { // Not worth failing a green test over. } } return ValueTask.CompletedTask; } private static byte[] Image(string seed) => [.. Samples.Png(), .. Encoding.UTF8.GetBytes(seed)]; private async Task>> RunAsync(IMediaSource source, CollectOptions? options = null) { var results = new List>(); await foreach ( var outcome in _runner.RunAsync( source, new MediaQuery(), options ?? new CollectOptions(), null, TestContext.Current.CancellationToken ) ) { results.Add(outcome); } return results; } [Fact] public async Task Everything_the_source_finds_is_downloaded_and_stored() { var source = new ListSource("https://a.test/1.png", "https://a.test/2.png"); _fetcher.Content["https://a.test/1.png"] = Image("one"); _fetcher.Content["https://a.test/2.png"] = Image("two"); var results = await RunAsync(source); results.Count.ShouldBe(2); results.ShouldAllBe(r => r.IsSuccess); results.ShouldAllBe(r => r.Value!.Status == CollectStatus.Stored); var stats = await _store.GetStatsAsync(TestContext.Current.CancellationToken); stats.BlobCount.ShouldBe(2); } [Fact] public async Task A_second_run_over_the_same_list_fetches_nothing() { // The journal exists precisely so that re-running a list costs no requests and no proxy. var source = new ListSource("https://a.test/1.png", "https://a.test/2.png"); _fetcher.Content["https://a.test/1.png"] = Image("one"); _fetcher.Content["https://a.test/2.png"] = Image("two"); await RunAsync(source); _fetcher.Fetched.Clear(); var second = await RunAsync(source); _fetcher.Fetched.ShouldBeEmpty(); second.Count.ShouldBe(2); second.ShouldAllBe(r => r.Value!.Status == CollectStatus.Skipped); } [Fact] public async Task Forcing_a_refetch_ignores_the_journal() { var source = new ListSource("https://a.test/1.png"); _fetcher.Content["https://a.test/1.png"] = Image("one"); await RunAsync(source); _fetcher.Fetched.Clear(); await RunAsync(source, new CollectOptions { ForceRefetch = true }); _fetcher.Fetched.ShouldHaveSingleItem(); } [Fact] public async Task A_failure_is_reported_and_retried_next_time() { // Failures describe the moment, not the resource; a flaky network must not lose content. var source = new ListSource("https://a.test/1.png"); _fetcher.Failing.Add("https://a.test/1.png"); var first = await RunAsync(source); first.ShouldHaveSingleItem().IsSuccess.ShouldBeFalse(); _fetcher.Failing.Clear(); _fetcher.Content["https://a.test/1.png"] = Image("recovered"); _fetcher.Fetched.Clear(); var second = await RunAsync(source); _fetcher.Fetched.ShouldHaveSingleItem(); second.ShouldHaveSingleItem().Value!.Status.ShouldBe(CollectStatus.Stored); } [Fact] public async Task The_same_bytes_at_two_addresses_are_stored_once() { var source = new ListSource("https://a.test/1.png", "https://a.test/2.png"); var shared = Image("identical"); _fetcher.Content["https://a.test/1.png"] = shared; _fetcher.Content["https://a.test/2.png"] = shared; var results = await RunAsync(source, new CollectOptions { MaxConcurrentDownloads = 1 }); results.Count(r => r.Value!.Status == CollectStatus.Stored).ShouldBe(1); results.Count(r => r.Value!.Status == CollectStatus.Duplicate).ShouldBe(1); (await _store.GetStatsAsync(TestContext.Current.CancellationToken)).BlobCount.ShouldBe(1); } [Fact] public async Task A_source_that_throws_does_not_take_the_run_down() { var results = await RunAsync(new ThrowingSource()); results.ShouldHaveSingleItem().Error!.Code.ShouldBe("SourceFailed"); } [Fact] public async Task Cancelling_stops_the_run_and_nothing_writes_afterwards() { // The runner owns its workers and waits for them; otherwise a stopped run keeps writing to // the store after the page has already said it stopped. var urls = Enumerable.Range(0, 40).Select(i => $"https://a.test/{i}.png").ToArray(); var source = new ListSource(urls); foreach (var url in urls) { _fetcher.Content[url] = Image(url); } _fetcher.Delay = TimeSpan.FromMilliseconds(50); using var cancellation = new CancellationTokenSource(); var results = new List>(); await Should.ThrowAsync(async () => { await foreach ( var outcome in _runner.RunAsync( source, new MediaQuery(), new CollectOptions(), null, cancellation.Token ) ) { results.Add(outcome); if (results.Count == 3) { await cancellation.CancelAsync(); } } }); var afterReturn = await _store.GetStatsAsync(TestContext.Current.CancellationToken); await Task.Delay(200, TestContext.Current.CancellationToken); var later = await _store.GetStatsAsync(TestContext.Current.CancellationToken); later.ItemCount.ShouldBe(afterReturn.ItemCount); Directory.EnumerateFiles(_paths.MediaTempDirectory).ShouldBeEmpty(); } private sealed class ThrowingSource : IMediaSource { public string Id => "throwing"; public string DisplayName => "Throwing source"; public string Description => string.Empty; public bool CanParse(MediaQuery input) => true; public async IAsyncEnumerable> ParseAsync( MediaQuery input, IProgress? progress, [EnumeratorCancellation] CancellationToken cancellationToken ) { await Task.Yield(); throw new InvalidOperationException("the listing endpoint moved"); #pragma warning disable CS0162 yield break; #pragma warning restore CS0162 } } } public class OwnServiceListingTests { private static readonly Uri Endpoint = new("https://own.test/api/list"); [Fact] public void An_object_with_items_is_read() { var page = OwnServiceSource.ReadPage( """ { "items": [ { "url": "https://own.test/a.png", "id": "42", "name": "kitten", "published": "2026-08-13T10:00:00Z", "size": 4096, "tags": ["cats"] } ], "next": "page2" } """, Endpoint ); var item = page.Items.ShouldHaveSingleItem(); item.Url.AbsoluteUri.ShouldBe("https://own.test/a.png"); item.ExternalId.ShouldBe("42"); item.SuggestedName.ShouldBe("kitten"); item.ExpectedLength.ShouldBe(4096); item.Tags.ShouldBe(["cats"]); page.Next.ShouldBe("page2"); } [Fact] public void A_bare_array_of_addresses_is_read() { // The service on the other end is the user's own; it should not have to be rewritten to // match a schema we invented. var page = OwnServiceSource.ReadPage("""["https://own.test/a.png", "https://own.test/b.gif"]""", Endpoint); page.Items.Count.ShouldBe(2); page.Next.ShouldBeNull(); } [Fact] public void Relative_addresses_resolve_against_the_endpoint() { var page = OwnServiceSource.ReadPage("""{"items":[{"url":"/files/a.png"}]}""", Endpoint); page.Items.ShouldHaveSingleItem().Url.AbsoluteUri.ShouldBe("https://own.test/files/a.png"); } [Fact] public void Entries_without_a_usable_address_are_dropped() { var page = OwnServiceSource.ReadPage( """{"items":[{"name":"no url"}, {"url":"data:image/png;base64,AA"}, {"url":"https://own.test/ok.png"}]}""", Endpoint ); page.Items.ShouldHaveSingleItem().Url.AbsoluteUri.ShouldBe("https://own.test/ok.png"); } [Fact] public void An_empty_listing_is_not_an_error() { OwnServiceSource.ReadPage("""{"items":[]}""", Endpoint).Items.ShouldBeEmpty(); OwnServiceSource.ReadPage("[]", Endpoint).Items.ShouldBeEmpty(); } }