Implement media rule management and enhance proxy handling

- Added methods to `IMediaStore` for loading, saving, and removing media rules, allowing users to manage rules for media items effectively.
- Updated `MediaFetcher` to utilize the new rule management system, integrating rule checks into the fetching process to handle geo-blocks and previously ruled items.
- Enhanced `ProxyPool` to support exclusion of proxies from specific countries during acquisition, improving the handling of geo-blocked content.
- Adjusted `FetchOptions` to include rules instead of tombstones, streamlining the decision-making process during media fetching.
- Updated UI components to support rule editing, providing users with a more interactive experience when managing media rules.

These changes improve the overall media collection process by allowing users to define rules for handling media items and enhancing the proxy management system for better content accessibility.

tests/AvParser.Core.Tests/Collecting/PatternMediaSourceTests.cs

@@ -76,6 +76,38 @@ public class PatternMediaSourceTests
         candidate.Url.AbsoluteUri.ShouldBe($"https://imgtest.example/test1/{candidate.ExternalId}.jpg");
     }
 
+    [Fact]
+    public async Task Several_extensions_become_one_candidate_with_fallbacks_in_order()
+    {
+        // One id is one picture: three candidates for three suffixes would fetch the hit and then
+        // hunt for its imaginary twins, and count one find as three attempts.
+        var source = new PatternMediaSource(
+            Config(min: 6, max: 6, alphabet: IdAlphabet.HexLower, extension: ".jpg, png;.gif")
+        );
+
+        var candidate = (await Collect(source, 1)).ShouldHaveSingleItem();
+        var id = candidate.ExternalId;
+
+        candidate.Url.AbsoluteUri.ShouldBe($"https://imgtest.example/test1/{id}.jpg");
+        candidate
+            .Alternatives.Select(url => url.AbsoluteUri)
+            .ShouldBe([$"https://imgtest.example/test1/{id}.png", $"https://imgtest.example/test1/{id}.gif"]);
+
+        candidate.Addresses.First().ShouldBe(candidate.Url);
+        candidate.Addresses.Count().ShouldBe(3);
+    }
+
+    [Fact]
+    public async Task A_single_extension_leaves_the_candidate_without_fallbacks()
+    {
+        var source = new PatternMediaSource(Config(min: 6, max: 6, extension: ".png"));
+
+        var candidate = (await Collect(source, 1)).ShouldHaveSingleItem();
+
+        candidate.Alternatives.ShouldBeEmpty();
+        candidate.Addresses.ShouldHaveSingleItem().ShouldBe(candidate.Url);
+    }
+
     [Fact]
     public async Task The_attempt_budget_bounds_how_many_are_generated()
     {
@@ -208,6 +240,76 @@ public class PatternSourceConfigTests
         config.Id.ShouldNotBeNullOrWhiteSpace();
     }
 
+    [Theory]
+    [InlineData(".jpg,.png", ".jpg,.png")]
+    [InlineData("jpg png gif", ".jpg,.png,.gif")]
+    [InlineData(".jpg; .PNG ;jpg", ".jpg,.PNG")]
+    [InlineData("  ", ".jpg")]
+    public void Extensions_are_normalised_into_an_ordered_list(string typed, string expected)
+    {
+        // Order is an instruction, not a detail: it is the order the fetcher will try. Repeats are
+        // dropped because a repeat means fetching the same address twice before giving up on the id.
+        PatternSourceConfig.TryCreate(
+            "N",
+            "https://h/x/",
+            6,
+            8,
+            IdAlphabet.Digits,
+            null,
+            typed,
+            allowDirectConnection: false,
+            out var config
+        );
+
+        config.ShouldNotBeNull().Extension.ShouldBe(expected);
+        config.Extensions.ShouldBe(expected.Split(','));
+    }
+
+    [Fact]
+    public void A_doubled_trailing_slash_is_collapsed()
+    {
+        // "host//" is not "host/": ids resolve against it as //{id}, which most servers answer with
+        // a 404 for every attempt in the run. Easy to paste, impossible to spot in the field.
+        var error = PatternSourceConfig.TryCreate(
+            "N",
+            "https://imgtest.example/test1//",
+            6,
+            8,
+            IdAlphabet.Digits,
+            null,
+            ".jpg",
+            allowDirectConnection: false,
+            out var config
+        );
+
+        error.ShouldBe(PatternConfigError.None);
+        config.ShouldNotBeNull().BaseUrl.AbsoluteUri.ShouldBe("https://imgtest.example/test1/");
+    }
+
+    [Fact]
+    public async Task Ids_hang_off_a_collapsed_base_url_without_a_doubled_slash()
+    {
+        PatternSourceConfig.TryCreate(
+            "N",
+            "https://imgtest.example/test1//",
+            6,
+            6,
+            IdAlphabet.Digits,
+            null,
+            ".jpg",
+            allowDirectConnection: false,
+            out var config
+        );
+
+        var source = new PatternMediaSource(config!);
+
+        await foreach (var outcome in source.ParseAsync(new MediaQuery(Limit: 1), null, CancellationToken.None))
+        {
+            var candidate = outcome.Value.ShouldNotBeNull();
+            candidate.Url.AbsoluteUri.ShouldBe($"https://imgtest.example/test1/{candidate.ExternalId}.jpg");
+        }
+    }
+
     [Fact]
     public void A_blank_extension_becomes_the_default_one()
     {

tests/AvParser.Infrastructure.Tests/Collecting/MediaFetcherTests.cs

@@ -44,6 +44,7 @@ internal sealed class DirectHttpClientFactory : IProxiedHttpClientFactory
     public Task<LeasedHttpClient> LeaseAsync(
         HttpClientTimeouts timeouts,
         bool requireProxy,
+        IReadOnlySet<string>? excludedCountries = null,
         CancellationToken cancellationToken = default
     ) =>
         requireProxy
@@ -230,6 +231,88 @@ public sealed class MediaFetcherTests : IAsyncLifetime
         _throttle.CooldownRemaining(url).ShouldBeGreaterThan(TimeSpan.Zero);
     }
 
+    [Fact]
+    public async Task Several_extensions_are_tried_in_order_until_one_answers()
+    {
+        var jpg = _server.Map("/id.jpg", new Reply { Status = 404 });
+        var png = _server.MapBody("/id.png", Samples.Png(), "image/png");
+        var gif = _server.MapBody("/id.gif", Samples.Gif(), "image/gif");
+
+        var result = await _fetcher.FetchAsync(
+            new MediaCandidate(jpg) { SourceId = "test", Alternatives = [png, gif] },
+            Options(),
+            TestContext.Current.CancellationToken
+        );
+
+        result.Outcome.ShouldBe(SeenOutcome.Stored);
+        result.FinalUrl.ShouldBe(png);
+        result.Blob!.Kind.ShouldBe(MediaKind.Png);
+
+        // The gif exists too, and must not have been fetched: an id holds one picture, and going on
+        // after a hit would download its imaginary twin and count the id twice.
+        _server.Requests.ShouldNotContain(request => request.Path == "/id.gif");
+    }
+
+    [Fact]
+    public async Task An_id_that_is_at_none_of_the_extensions_reports_the_last_miss()
+    {
+        var jpg = _server.Map("/none.jpg", new Reply { Status = 404 });
+        var png = _server.Map("/none.png", new Reply { Status = 404 });
+
+        var result = await _fetcher.FetchAsync(
+            new MediaCandidate(jpg) { SourceId = "test", Alternatives = [png] },
+            Options(),
+            TestContext.Current.CancellationToken
+        );
+
+        result.Outcome.ShouldBe(SeenOutcome.Gone);
+        result.FinalUrl.ShouldBe(png);
+    }
+
+    [Fact]
+    public async Task A_placeholder_under_the_size_floor_counts_as_not_there_and_moves_on()
+    {
+        // What a service actually does with a missing id: 200 with a tiny "removed" image. That is
+        // "not there" wearing a success, so the next extension still has to be tried.
+        var jpg = _server.MapBody("/tiny.jpg", Samples.Png(), "image/png");
+        var png = _server.Map("/tiny.png", new Reply { Status = 404 });
+
+        var result = await _fetcher.FetchAsync(
+            new MediaCandidate(jpg) { SourceId = "test", Alternatives = [png] },
+            Options() with
+            {
+                MinItemBytes = Samples.Png().Length + 1,
+            },
+            TestContext.Current.CancellationToken
+        );
+
+        result.FinalUrl.ShouldBe(png);
+        result.Outcome.ShouldBe(SeenOutcome.Gone);
+    }
+
+    [Fact]
+    public async Task A_timeout_does_not_burn_the_remaining_extensions()
+    {
+        // "Not there" is a verdict about the address; a timeout is the absence of one. Moving on
+        // would report an id as missing everywhere on the strength of one broken connection.
+        var jpg = _server.Map("/slow.jpg", new Reply { Body = Samples.Png(), HeaderDelay = TimeSpan.FromSeconds(3) });
+        var png = _server.MapBody("/slow.png", Samples.Png(), "image/png");
+
+        var options = Options() with
+        {
+            Timeouts = HttpClientTimeouts.Default with { Headers = TimeSpan.FromMilliseconds(300) },
+        };
+
+        var result = await _fetcher.FetchAsync(
+            new MediaCandidate(jpg) { SourceId = "test", Alternatives = [png] },
+            options,
+            TestContext.Current.CancellationToken
+        );
+
+        result.Outcome.ShouldBe(SeenOutcome.Timeout);
+        _server.Requests.ShouldNotContain(request => request.Path == "/slow.png");
+    }
+
     [Fact]
     public async Task Headers_that_never_arrive_time_out()
     {
@@ -434,7 +517,10 @@ public sealed class MediaFetcherTests : IAsyncLifetime
 
         var options = Options() with
         {
-            Tombstones = new HashSet<string>(StringComparer.Ordinal) { probe.Blob!.Sha256 },
+            Rules = new Dictionary<string, MediaRuleAction>(StringComparer.Ordinal)
+            {
+                [probe.Blob!.Sha256] = MediaRuleAction.Skip,
+            },
         };
 
         var result = await FetchAsync(url, options);
@@ -443,6 +529,31 @@ public sealed class MediaFetcherTests : IAsyncLifetime
         StagedFileCount().ShouldBe(0);
     }
 
+    [Fact]
+    public async Task A_geo_block_rule_with_nowhere_else_to_go_stores_nothing_and_stays_retryable()
+    {
+        // Direct connections here, so there is no country to move away from. The picture must not be
+        // stored — it is a banner, not the thing — and the address must stay worth trying again with
+        // a different pool tomorrow.
+        var url = _server.MapBody("/blocked.png", Samples.Png());
+        var probe = await FetchAsync(url);
+        File.Delete(probe.TempPath!);
+
+        var options = Options() with
+        {
+            Rules = new Dictionary<string, MediaRuleAction>(StringComparer.Ordinal)
+            {
+                [probe.Blob!.Sha256] = MediaRuleAction.RetryElsewhere,
+            },
+        };
+
+        var result = await FetchAsync(url, options);
+
+        result.ErrorCode.ShouldBe("GeoBlocked");
+        SeenOutcomes.IsTerminal(result.Outcome).ShouldBeFalse();
+        StagedFileCount().ShouldBe(0);
+    }
+
     [Fact]
     public async Task Animation_is_detected_end_to_end()
     {

tests/AvParser.Infrastructure.Tests/JsonSettingsServiceTests.cs

@@ -201,6 +201,34 @@ public sealed class JsonSettingsServiceTests : IDisposable
         options.ProbeUrl.ShouldBe(new ProxyOptions().ProbeUrl);
     }
 
+    [Fact]
+    public void The_probe_opens_a_tunnel_the_way_the_collector_does()
+    {
+        // A plain-HTTP probe is a forwarded GET; an https one is a CONNECT. Free proxies routinely
+        // do the first and refuse the second, and every address this app collects is https — so an
+        // http probe marked them live and they then failed every real request.
+        new ProxyOptions().ProbeUrl.Scheme.ShouldBe("https");
+        new AppSettings().ToProxyOptions().ProbeUrl.Scheme.ShouldBe("https");
+    }
+
+    [Fact]
+    public void A_settings_file_still_holding_the_old_http_probe_is_upgraded()
+    {
+        // Nobody would think to edit this by hand, and leaving it would keep confirming proxies that
+        // cannot do the one thing the collector needs.
+        var options = new AppSettings(ProxyProbeUrl: "http://www.gstatic.com/generate_204").ToProxyOptions();
+
+        options.ProbeUrl.ShouldBe(new ProxyOptions().ProbeUrl);
+    }
+
+    [Fact]
+    public void A_probe_url_the_user_chose_is_left_alone()
+    {
+        var options = new AppSettings(ProxyProbeUrl: "http://example.test/ping").ToProxyOptions();
+
+        options.ProbeUrl.AbsoluteUri.ShouldBe("http://example.test/ping");
+    }
+
     [Fact]
     public void A_corrupt_file_falls_back_to_the_defaults_instead_of_failing_to_start()
     {

tests/AvParser.UI.HeadlessTests/FakeMediaStore.cs

@@ -57,6 +57,23 @@ internal sealed class FakeMediaStore : IMediaStore
     public Task<int> TombstoneAsync(string sha256, string? reason, CancellationToken cancellationToken = default) =>
         Task.FromResult(0);
 
+    /// <summary>Rules the page under test has saved.</summary>
+    public List<MediaRule> Rules { get; } = [];
+
+    public Task<IReadOnlyList<MediaRule>> LoadRulesAsync(CancellationToken cancellationToken = default) =>
+        Task.FromResult<IReadOnlyList<MediaRule>>([.. Rules]);
+
+    public Task<int> SaveRuleAsync(MediaRule rule, CancellationToken cancellationToken = default)
+    {
+        Rules.RemoveAll(existing => existing.Sha256 == rule.Sha256);
+        Rules.Add(rule);
+
+        return Task.FromResult(rule.Action == MediaRuleAction.Skip ? 1 : 0);
+    }
+
+    public Task<bool> RemoveRuleAsync(string sha256, CancellationToken cancellationToken = default) =>
+        Task.FromResult(Rules.RemoveAll(existing => existing.Sha256 == sha256) > 0);
+
     public Task<CollectedItem> StoreAsync(MediaStoreRequest request, CancellationToken cancellationToken = default) =>
         Task.FromResult(new CollectedItem(request.Candidate, request.Blob, CollectStatus.Stored));
 

tests/AvParser.UI.Tests/Fakes/FakeMediaStore.cs

@@ -57,6 +57,23 @@ internal sealed class FakeMediaStore : IMediaStore
     public Task<int> TombstoneAsync(string sha256, string? reason, CancellationToken cancellationToken = default) =>
         Task.FromResult(0);
 
+    /// <summary>Rules the page under test has saved.</summary>
+    public List<MediaRule> Rules { get; } = [];
+
+    public Task<IReadOnlyList<MediaRule>> LoadRulesAsync(CancellationToken cancellationToken = default) =>
+        Task.FromResult<IReadOnlyList<MediaRule>>([.. Rules]);
+
+    public Task<int> SaveRuleAsync(MediaRule rule, CancellationToken cancellationToken = default)
+    {
+        Rules.RemoveAll(existing => existing.Sha256 == rule.Sha256);
+        Rules.Add(rule);
+
+        return Task.FromResult(rule.Action == MediaRuleAction.Skip ? 1 : 0);
+    }
+
+    public Task<bool> RemoveRuleAsync(string sha256, CancellationToken cancellationToken = default) =>
+        Task.FromResult(Rules.RemoveAll(existing => existing.Sha256 == sha256) > 0);
+
     public Task<CollectedItem> StoreAsync(MediaStoreRequest request, CancellationToken cancellationToken = default) =>
         Task.FromResult(new CollectedItem(request.Candidate, request.Blob, CollectStatus.Stored));