Refactor media source handling and update collection options

- Updated `IMediaSourceCatalog` to support user-added media sources, allowing dynamic editing and management of sources.
- Removed the `UrlListSource` class as its functionality is now integrated into the new catalog structure.
- Enhanced `CollectOptions` to default `RequireProxy` to true, ensuring stricter handling of proxy requirements.
- Improved error handling in `ParseError` to include a `Subject` field for better context on failures.
- Adjusted dependency injection to reflect changes in media source management, removing old source registrations.
- Introduced background proxy checks to ensure a more robust proxy pool management during collection processes.

These changes streamline the media collection process and improve the overall user experience by providing clearer error reporting and more flexible source management.

test-service/Config.cs

@@ -0,0 +1,164 @@
+using System.Security.Cryptography;
+
+namespace AvParser.ImgTestService;
+
+/// <summary>Top-level service options, bound from the <c>ImgTest</c> configuration section.</summary>
+public sealed class ImgTestOptions
+{
+    /// <summary>Fraction of otherwise-valid ids that answer 404, to emulate a sparse site. 0 disables.</summary>
+    public double MissRate { get; set; }
+
+    /// <summary>The endpoints to expose, each with its own id shape.</summary>
+    public List<EndpointOptions> Endpoints { get; set; } = [];
+
+    /// <summary>Resolves every endpoint and indexes it by path segment.</summary>
+    public Dictionary<string, ResolvedEndpoint> BuildLookup()
+    {
+        var lookup = new Dictionary<string, ResolvedEndpoint>(StringComparer.OrdinalIgnoreCase);
+        foreach (var endpoint in Endpoints)
+        {
+            var resolved = endpoint.Resolve();
+            if (resolved is not null)
+            {
+                lookup[resolved.Path] = resolved;
+            }
+        }
+
+        return lookup;
+    }
+
+    /// <summary>
+    /// Whether an id should 404 despite being well-formed, deterministically by id.
+    /// </summary>
+    /// <remarks>
+    /// Deterministic on purpose: a given id is either always present or always absent, so a
+    /// collector's "seen this url" journal and its duplicate detection stay meaningful across runs.
+    /// </remarks>
+    public static bool IsMiss(string core, double missRate)
+    {
+        if (missRate <= 0)
+        {
+            return false;
+        }
+
+        if (missRate >= 1)
+        {
+            return true;
+        }
+
+        var hash = SHA256.HashData(System.Text.Encoding.UTF8.GetBytes("miss:" + core));
+        var value = ((uint)hash[0] << 24) | ((uint)hash[1] << 16) | ((uint)hash[2] << 8) | hash[3];
+        return value / (double)uint.MaxValue < missRate;
+    }
+}
+
+/// <summary>One endpoint's configuration, as written in appsettings.</summary>
+public sealed class EndpointOptions
+{
+    /// <summary>First path segment, e.g. <c>test1</c> for <c>/test1/{id}</c>.</summary>
+    public string Path { get; set; } = string.Empty;
+
+    /// <summary>Shortest id core (excluding any extension).</summary>
+    public int MinLength { get; set; }
+
+    /// <summary>Longest id core (excluding any extension).</summary>
+    public int MaxLength { get; set; }
+
+    /// <summary>Named character set: Letters, LettersLower, LettersUpper, Digits, Alphanumeric, HexLower, or Custom.</summary>
+    public string Alphabet { get; set; } = "Alphanumeric";
+
+    /// <summary>Exact characters, used when <see cref="Alphabet"/> is Custom.</summary>
+    public string? Chars { get; set; }
+
+    /// <summary>Accepted trailing extensions, e.g. <c>.jpg</c>; empty means none is expected.</summary>
+    public List<string> Extensions { get; set; } = [];
+
+    /// <summary>Turns this into a resolved endpoint, or null when unusable.</summary>
+    public ResolvedEndpoint? Resolve()
+    {
+        var allowed = ResolveAlphabet(Alphabet, Chars);
+        if (string.IsNullOrWhiteSpace(Path) || MinLength < 1 || MaxLength < MinLength || allowed.Count == 0)
+        {
+            return null;
+        }
+
+        return new ResolvedEndpoint(
+            Path.Trim('/'),
+            MinLength,
+            MaxLength,
+            allowed,
+            [.. Extensions.Where(e => !string.IsNullOrWhiteSpace(e)).Select(Normalise)]
+        );
+    }
+
+    private static string Normalise(string extension)
+    {
+        var trimmed = extension.Trim();
+        return trimmed.StartsWith('.') ? trimmed : "." + trimmed;
+    }
+
+    private static HashSet<char> ResolveAlphabet(string name, string? chars)
+    {
+        const string lower = "abcdefghijklmnopqrstuvwxyz";
+        const string upper = "ABCDEFGHIJKLMNOPQRSTUVWXYZ";
+        const string digit = "0123456789";
+        const string hex = "0123456789abcdef";
+
+        var set = name.Trim().ToLowerInvariant() switch
+        {
+            "letterslower" or "lower" => lower,
+            "lettersupper" or "upper" => upper,
+            "letters" => lower + upper,
+            "digits" or "digit" => digit,
+            "alphanumeric" or "alnum" => lower + upper + digit,
+            "hexlower" or "hex" => hex,
+            "custom" => chars ?? string.Empty,
+            _ => string.Empty,
+        };
+
+        return [.. set];
+    }
+}
+
+/// <summary>A validated endpoint ready to match requests.</summary>
+public sealed record ResolvedEndpoint(
+    string Path,
+    int MinLength,
+    int MaxLength,
+    HashSet<char> Allowed,
+    string[] Extensions
+)
+{
+    /// <summary>Removes a recognised trailing extension from the raw path segment.</summary>
+    public string StripExtension(string id)
+    {
+        foreach (var extension in Extensions)
+        {
+            if (id.EndsWith(extension, StringComparison.OrdinalIgnoreCase))
+            {
+                return id[..^extension.Length];
+            }
+        }
+
+        return id;
+    }
+
+    /// <summary>Whether an id core is the right length and made only of allowed characters.</summary>
+    public bool Matches(string core)
+    {
+        if (core.Length < MinLength || core.Length > MaxLength)
+        {
+            return false;
+        }
+
+        foreach (var c in core)
+        {
+            if (!Allowed.Contains(c))
+            {
+                return false;
+            }
+        }
+
+        return true;
+    }
+}