Refactor media source handling and update collection options

- Updated `IMediaSourceCatalog` to support user-added media sources, allowing dynamic editing and management of sources.
- Removed the `UrlListSource` class as its functionality is now integrated into the new catalog structure.
- Enhanced `CollectOptions` to default `RequireProxy` to true, ensuring stricter handling of proxy requirements.
- Improved error handling in `ParseError` to include a `Subject` field for better context on failures.
- Adjusted dependency injection to reflect changes in media source management, removing old source registrations.
- Introduced background proxy checks to ensure a more robust proxy pool management during collection processes.

These changes streamline the media collection process and improve the overall user experience by providing clearer error reporting and more flexible source management.

test-service/.dockerignore

@@ -0,0 +1,3 @@
+bin/
+obj/
+Properties/launchSettings.json

test-service/AvParser.ImgTestService.csproj

@@ -0,0 +1,10 @@
+<Project Sdk="Microsoft.NET.Sdk.Web">
+  <PropertyGroup>
+    <TargetFramework>net10.0</TargetFramework>
+    <Nullable>enable</Nullable>
+    <ImplicitUsings>enable</ImplicitUsings>
+    <RootNamespace>AvParser.ImgTestService</RootNamespace>
+    <InvariantGlobalization>true</InvariantGlobalization>
+    <!-- No PackageReference: the Web SDK pulls ASP.NET Core in through the shared framework. -->
+  </PropertyGroup>
+</Project>

test-service/Config.cs

@@ -0,0 +1,164 @@
+using System.Security.Cryptography;
+
+namespace AvParser.ImgTestService;
+
+/// <summary>Top-level service options, bound from the <c>ImgTest</c> configuration section.</summary>
+public sealed class ImgTestOptions
+{
+    /// <summary>Fraction of otherwise-valid ids that answer 404, to emulate a sparse site. 0 disables.</summary>
+    public double MissRate { get; set; }
+
+    /// <summary>The endpoints to expose, each with its own id shape.</summary>
+    public List<EndpointOptions> Endpoints { get; set; } = [];
+
+    /// <summary>Resolves every endpoint and indexes it by path segment.</summary>
+    public Dictionary<string, ResolvedEndpoint> BuildLookup()
+    {
+        var lookup = new Dictionary<string, ResolvedEndpoint>(StringComparer.OrdinalIgnoreCase);
+        foreach (var endpoint in Endpoints)
+        {
+            var resolved = endpoint.Resolve();
+            if (resolved is not null)
+            {
+                lookup[resolved.Path] = resolved;
+            }
+        }
+
+        return lookup;
+    }
+
+    /// <summary>
+    /// Whether an id should 404 despite being well-formed, deterministically by id.
+    /// </summary>
+    /// <remarks>
+    /// Deterministic on purpose: a given id is either always present or always absent, so a
+    /// collector's "seen this url" journal and its duplicate detection stay meaningful across runs.
+    /// </remarks>
+    public static bool IsMiss(string core, double missRate)
+    {
+        if (missRate <= 0)
+        {
+            return false;
+        }
+
+        if (missRate >= 1)
+        {
+            return true;
+        }
+
+        var hash = SHA256.HashData(System.Text.Encoding.UTF8.GetBytes("miss:" + core));
+        var value = ((uint)hash[0] << 24) | ((uint)hash[1] << 16) | ((uint)hash[2] << 8) | hash[3];
+        return value / (double)uint.MaxValue < missRate;
+    }
+}
+
+/// <summary>One endpoint's configuration, as written in appsettings.</summary>
+public sealed class EndpointOptions
+{
+    /// <summary>First path segment, e.g. <c>test1</c> for <c>/test1/{id}</c>.</summary>
+    public string Path { get; set; } = string.Empty;
+
+    /// <summary>Shortest id core (excluding any extension).</summary>
+    public int MinLength { get; set; }
+
+    /// <summary>Longest id core (excluding any extension).</summary>
+    public int MaxLength { get; set; }
+
+    /// <summary>Named character set: Letters, LettersLower, LettersUpper, Digits, Alphanumeric, HexLower, or Custom.</summary>
+    public string Alphabet { get; set; } = "Alphanumeric";
+
+    /// <summary>Exact characters, used when <see cref="Alphabet"/> is Custom.</summary>
+    public string? Chars { get; set; }
+
+    /// <summary>Accepted trailing extensions, e.g. <c>.jpg</c>; empty means none is expected.</summary>
+    public List<string> Extensions { get; set; } = [];
+
+    /// <summary>Turns this into a resolved endpoint, or null when unusable.</summary>
+    public ResolvedEndpoint? Resolve()
+    {
+        var allowed = ResolveAlphabet(Alphabet, Chars);
+        if (string.IsNullOrWhiteSpace(Path) || MinLength < 1 || MaxLength < MinLength || allowed.Count == 0)
+        {
+            return null;
+        }
+
+        return new ResolvedEndpoint(
+            Path.Trim('/'),
+            MinLength,
+            MaxLength,
+            allowed,
+            [.. Extensions.Where(e => !string.IsNullOrWhiteSpace(e)).Select(Normalise)]
+        );
+    }
+
+    private static string Normalise(string extension)
+    {
+        var trimmed = extension.Trim();
+        return trimmed.StartsWith('.') ? trimmed : "." + trimmed;
+    }
+
+    private static HashSet<char> ResolveAlphabet(string name, string? chars)
+    {
+        const string lower = "abcdefghijklmnopqrstuvwxyz";
+        const string upper = "ABCDEFGHIJKLMNOPQRSTUVWXYZ";
+        const string digit = "0123456789";
+        const string hex = "0123456789abcdef";
+
+        var set = name.Trim().ToLowerInvariant() switch
+        {
+            "letterslower" or "lower" => lower,
+            "lettersupper" or "upper" => upper,
+            "letters" => lower + upper,
+            "digits" or "digit" => digit,
+            "alphanumeric" or "alnum" => lower + upper + digit,
+            "hexlower" or "hex" => hex,
+            "custom" => chars ?? string.Empty,
+            _ => string.Empty,
+        };
+
+        return [.. set];
+    }
+}
+
+/// <summary>A validated endpoint ready to match requests.</summary>
+public sealed record ResolvedEndpoint(
+    string Path,
+    int MinLength,
+    int MaxLength,
+    HashSet<char> Allowed,
+    string[] Extensions
+)
+{
+    /// <summary>Removes a recognised trailing extension from the raw path segment.</summary>
+    public string StripExtension(string id)
+    {
+        foreach (var extension in Extensions)
+        {
+            if (id.EndsWith(extension, StringComparison.OrdinalIgnoreCase))
+            {
+                return id[..^extension.Length];
+            }
+        }
+
+        return id;
+    }
+
+    /// <summary>Whether an id core is the right length and made only of allowed characters.</summary>
+    public bool Matches(string core)
+    {
+        if (core.Length < MinLength || core.Length > MaxLength)
+        {
+            return false;
+        }
+
+        foreach (var c in core)
+        {
+            if (!Allowed.Contains(c))
+            {
+                return false;
+            }
+        }
+
+        return true;
+    }
+}

test-service/Directory.Build.props

@@ -0,0 +1,10 @@
+<Project>
+  <!--
+    Stops the upward walk so the test service does not inherit the main repo's build props
+    (central package versions, TreatWarningsAsErrors, analyzers). It is a standalone image built
+    only inside its own Docker context, never as part of AvParser.slnx.
+  -->
+  <PropertyGroup>
+    <ManagePackageVersionsCentrally>false</ManagePackageVersionsCentrally>
+  </PropertyGroup>
+</Project>

test-service/Dockerfile

@@ -0,0 +1,20 @@
+# Build the service, then run it on the smaller ASP.NET runtime image.
+FROM mcr.microsoft.com/dotnet/sdk:10.0 AS build
+WORKDIR /src
+
+COPY Directory.Build.props ./
+COPY AvParser.ImgTestService.csproj ./
+RUN dotnet restore AvParser.ImgTestService.csproj
+
+COPY . ./
+RUN dotnet publish AvParser.ImgTestService.csproj -c Release -o /app --no-restore
+
+FROM mcr.microsoft.com/dotnet/aspnet:10.0 AS runtime
+WORKDIR /app
+COPY --from=build /app ./
+
+# Listen on 8080 inside the container; a reverse proxy terminates TLS for the public domain.
+ENV ASPNETCORE_URLS=http://+:8080
+EXPOSE 8080
+
+ENTRYPOINT ["dotnet", "AvParser.ImgTestService.dll"]

test-service/Identicon.cs

@@ -0,0 +1,198 @@
+using System.IO.Compression;
+using System.Security.Cryptography;
+
+namespace AvParser.ImgTestService;
+
+/// <summary>
+/// Renders a deterministic identicon PNG from an id, with a hand-written encoder.
+/// </summary>
+/// <remarks>
+/// No image library and no native dependency on purpose: the container stays tiny and runs the same
+/// on any architecture. The same id always yields the same bytes, so a collector that fetches an id
+/// twice gets one blob, and a duplicate-detection test has something stable to assert against.
+/// </remarks>
+public static class Identicon
+{
+    private const int Cells = 5;
+    private const int CellSize = 48;
+    private const int Margin = 20;
+    private const int Size = (Cells * CellSize) + (2 * Margin);
+
+    /// <summary>Builds the PNG bytes for an id.</summary>
+    public static byte[] Render(string id)
+    {
+        var hash = SHA256.HashData(System.Text.Encoding.UTF8.GetBytes(id));
+
+        // A saturated foreground from the first bytes, on a near-white background.
+        var (fr, fg, fb) = Foreground(hash);
+        byte br = 0xF2,
+            bg = 0xF2,
+            bb = 0xF4;
+
+        var pixels = new byte[Size * Size * 3];
+        FillBackground(pixels, br, bg, bb);
+
+        // A 5x5 grid, mirrored left-to-right, so the icon reads as a single symmetric shape. Bit
+        // source is the tail of the hash, one bit per cell in the left three columns.
+        for (var col = 0; col < (Cells + 1) / 2; col++)
+        {
+            for (var row = 0; row < Cells; row++)
+            {
+                var bit = hash[(col * Cells) + row] & 1;
+                if (bit == 0)
+                {
+                    continue;
+                }
+
+                PaintCell(pixels, col, row, fr, fg, fb);
+                PaintCell(pixels, Cells - 1 - col, row, fr, fg, fb);
+            }
+        }
+
+        return Encode(pixels, Size, Size);
+    }
+
+    private static (byte R, byte G, byte B) Foreground(byte[] hash)
+    {
+        // Pick a hue-ish colour that is never too pale: keep at least one channel low and one high.
+        var r = (byte)(60 + (hash[0] % 160));
+        var g = (byte)(60 + (hash[1] % 160));
+        var b = (byte)(60 + (hash[2] % 160));
+        return (r, g, b);
+    }
+
+    private static void FillBackground(byte[] pixels, byte r, byte g, byte b)
+    {
+        for (var i = 0; i < pixels.Length; i += 3)
+        {
+            pixels[i] = r;
+            pixels[i + 1] = g;
+            pixels[i + 2] = b;
+        }
+    }
+
+    private static void PaintCell(byte[] pixels, int col, int row, byte r, byte g, byte b)
+    {
+        var x0 = Margin + (col * CellSize);
+        var y0 = Margin + (row * CellSize);
+
+        for (var y = y0; y < y0 + CellSize; y++)
+        {
+            var rowStart = y * Size * 3;
+            for (var x = x0; x < x0 + CellSize; x++)
+            {
+                var i = rowStart + (x * 3);
+                pixels[i] = r;
+                pixels[i + 1] = g;
+                pixels[i + 2] = b;
+            }
+        }
+    }
+
+    /// <summary>Encodes RGB pixels as a PNG (colour type 2, 8-bit).</summary>
+    private static byte[] Encode(byte[] rgb, int width, int height)
+    {
+        using var output = new MemoryStream();
+
+        // Signature.
+        output.Write([0x89, 0x50, 0x4E, 0x47, 0x0D, 0x0A, 0x1A, 0x0A]);
+
+        Span<byte> ihdr = stackalloc byte[13];
+        WriteBigEndian(ihdr[..4], (uint)width);
+        WriteBigEndian(ihdr.Slice(4, 4), (uint)height);
+        ihdr[8] = 8; // bit depth
+        ihdr[9] = 2; // colour type: truecolour RGB
+        ihdr[10] = 0; // compression
+        ihdr[11] = 0; // filter
+        ihdr[12] = 0; // interlace
+        WriteChunk(output, "IHDR"u8, ihdr);
+
+        // Raw scanlines: a leading filter byte (0 = none) then the row's RGB bytes.
+        var stride = width * 3;
+        var raw = new byte[height * (stride + 1)];
+        for (var y = 0; y < height; y++)
+        {
+            var src = y * stride;
+            var dst = y * (stride + 1);
+            raw[dst] = 0;
+            Array.Copy(rgb, src, raw, dst + 1, stride);
+        }
+
+        using (var compressed = new MemoryStream())
+        {
+            using (var zlib = new ZLibStream(compressed, CompressionLevel.Optimal, leaveOpen: true))
+            {
+                zlib.Write(raw, 0, raw.Length);
+            }
+
+            WriteChunk(output, "IDAT"u8, compressed.ToArray());
+        }
+
+        WriteChunk(output, "IEND"u8, []);
+
+        return output.ToArray();
+    }
+
+    private static void WriteChunk(Stream stream, ReadOnlySpan<byte> type, ReadOnlySpan<byte> data)
+    {
+        Span<byte> length = stackalloc byte[4];
+        WriteBigEndian(length, (uint)data.Length);
+        stream.Write(length);
+        stream.Write(type);
+        stream.Write(data);
+
+        var crc = Crc32.Compute(type, data);
+        Span<byte> crcBytes = stackalloc byte[4];
+        WriteBigEndian(crcBytes, crc);
+        stream.Write(crcBytes);
+    }
+
+    private static void WriteBigEndian(Span<byte> destination, uint value)
+    {
+        destination[0] = (byte)(value >> 24);
+        destination[1] = (byte)(value >> 16);
+        destination[2] = (byte)(value >> 8);
+        destination[3] = (byte)value;
+    }
+}
+
+/// <summary>Minimal CRC-32 (PNG polynomial), enough to seal chunks.</summary>
+internal static class Crc32
+{
+    private static readonly uint[] Table = BuildTable();
+
+    public static uint Compute(ReadOnlySpan<byte> type, ReadOnlySpan<byte> data)
+    {
+        var crc = 0xFFFFFFFFu;
+        crc = Update(crc, type);
+        crc = Update(crc, data);
+        return crc ^ 0xFFFFFFFFu;
+    }
+
+    private static uint Update(uint crc, ReadOnlySpan<byte> bytes)
+    {
+        foreach (var b in bytes)
+        {
+            crc = Table[(crc ^ b) & 0xFF] ^ (crc >> 8);
+        }
+
+        return crc;
+    }
+
+    private static uint[] BuildTable()
+    {
+        var table = new uint[256];
+        for (var n = 0u; n < 256; n++)
+        {
+            var c = n;
+            for (var k = 0; k < 8; k++)
+            {
+                c = (c & 1) != 0 ? 0xEDB88320u ^ (c >> 1) : c >> 1;
+            }
+
+            table[n] = c;
+        }
+
+        return table;
+    }
+}

test-service/Program.cs

@@ -0,0 +1,57 @@
+using System.Text;
+using AvParser.ImgTestService;
+
+var builder = WebApplication.CreateBuilder(args);
+
+var options = builder.Configuration.GetSection("ImgTest").Get<ImgTestOptions>() ?? new ImgTestOptions();
+var endpoints = options.BuildLookup();
+
+var app = builder.Build();
+
+// A plain index so opening the host in a browser explains what is here.
+app.MapGet(
+    "/",
+    () =>
+    {
+        var text = new StringBuilder();
+        text.AppendLine("AvParser image test service");
+        text.AppendLine();
+        text.AppendLine($"miss rate: {options.MissRate:0.###}");
+        text.AppendLine("endpoints:");
+        foreach (var endpoint in endpoints.Values)
+        {
+            var ext = endpoint.Extensions.Length == 0 ? "(none)" : string.Join(" ", endpoint.Extensions);
+            text.AppendLine(
+                $"  /{endpoint.Path}/{{id}}  length {endpoint.MinLength}-{endpoint.MaxLength}, "
+                    + $"{endpoint.Allowed.Count} chars, ext {ext}"
+            );
+        }
+
+        return Results.Text(text.ToString(), "text/plain; charset=utf-8");
+    }
+);
+
+app.MapGet("/healthz", () => Results.Text("ok", "text/plain"));
+
+// One handler for every configured endpoint: the first segment selects the config, the second is
+// the id (optionally with an extension). A miss and an unknown endpoint are both a plain 404.
+app.MapGet(
+    "/{endpoint}/{id}",
+    (string endpoint, string id) =>
+    {
+        if (!endpoints.TryGetValue(endpoint, out var config))
+        {
+            return Results.NotFound();
+        }
+
+        var core = config.StripExtension(id);
+        if (!config.Matches(core) || ImgTestOptions.IsMiss(core, options.MissRate))
+        {
+            return Results.NotFound();
+        }
+
+        return Results.Bytes(Identicon.Render(core), "image/png");
+    }
+);
+
+app.Run();

test-service/Properties/launchSettings.json

@@ -0,0 +1,12 @@
+{
+  "profiles": {
+    "http": {
+      "commandName": "Project",
+      "dotnetRunMessages": true,
+      "applicationUrl": "http://localhost:8080",
+      "environmentVariables": {
+        "ASPNETCORE_ENVIRONMENT": "Development"
+      }
+    }
+  }
+}

test-service/README.md

@@ -0,0 +1,66 @@
+# AvParser image test service
+
+A tiny, dependency-free web service that serves a deterministic image for any id matching a
+configured pattern. It exists as a **target** for the app's pattern sources: point a source at one of
+its endpoints and let the collector generate ids.
+
+Each valid id returns a unique identicon PNG (derived from `SHA-256(id)`), so fetching the same id
+twice yields the same bytes — which is what makes the collector's duplicate detection observable.
+
+## Endpoints
+
+Endpoints are declared in [`appsettings.json`](appsettings.json) under `ImgTest:Endpoints`. The
+defaults:
+
+| Path            | Id length | Characters        | Extension |
+| --------------- | --------- | ----------------- | --------- |
+| `/test1/{id}`   | 6–8       | letters + digits  | `.jpg`    |
+| `/test2/{id}`   | 8–12      | digits only       | none      |
+| `/test3/{id}`   | 8         | hex (`0-9a-f`)    | `.png`    |
+
+Anything not matching a pattern (`/test1/xx`, `/test2/abcd1234`) returns `404`. `GET /` lists the
+endpoints; `GET /healthz` returns `ok`.
+
+Example: `GET /test1/2dfhyuj.jpg` → `200 image/png`.
+
+### Adding an endpoint
+
+Add an object to `ImgTest:Endpoints`:
+
+```json
+{ "Path": "test4", "MinLength": 4, "MaxLength": 6, "Alphabet": "Custom", "Chars": "abcdef012", "Extensions": [".png"] }
+```
+
+`Alphabet` is one of `LettersLower`, `LettersUpper`, `Letters`, `Digits`, `Alphanumeric`, `HexLower`,
+or `Custom` (which uses `Chars`).
+
+## Miss rate
+
+Set `ImgTest:MissRate` (0..1) to make a deterministic fraction of otherwise-valid ids answer `404`,
+emulating a site where most guessed ids do not exist. It is stable per id, so a run is repeatable.
+Override at deploy time with the `ImgTest__MissRate` environment variable.
+
+## Run
+
+```bash
+docker compose up --build
+```
+
+The container listens on `http://localhost:8080`. It speaks plain HTTP — **TLS for a public domain
+such as `https://imgtest.example` is terminated by your own reverse proxy**, which forwards to this
+container. A Caddy example:
+
+```
+imgtest.example {
+    reverse_proxy localhost:8080
+}
+```
+
+Or run it directly without Docker:
+
+```bash
+dotnet run
+```
+
+> This service is intentionally **not** part of `AvParser.slnx` and is not built by `./build.ps1`.
+> It is a standalone support tool with its own Docker context.

test-service/appsettings.json

@@ -0,0 +1,34 @@
+{
+  "Logging": {
+    "LogLevel": {
+      "Default": "Information",
+      "Microsoft.AspNetCore": "Warning"
+    }
+  },
+  "ImgTest": {
+    "MissRate": 0.0,
+    "Endpoints": [
+      {
+        "Path": "test1",
+        "MinLength": 6,
+        "MaxLength": 8,
+        "Alphabet": "Alphanumeric",
+        "Extensions": [".jpg"]
+      },
+      {
+        "Path": "test2",
+        "MinLength": 8,
+        "MaxLength": 12,
+        "Alphabet": "Digits",
+        "Extensions": []
+      },
+      {
+        "Path": "test3",
+        "MinLength": 8,
+        "MaxLength": 8,
+        "Alphabet": "HexLower",
+        "Extensions": [".png"]
+      }
+    ]
+  }
+}

test-service/docker-compose.yml

@@ -0,0 +1,12 @@
+services:
+  imgtest:
+    build: .
+    image: avparser-imgtest
+    container_name: avparser-imgtest
+    restart: unless-stopped
+    ports:
+      # host:container — change the host port to suit; the reverse proxy points at it.
+      - "8080:8080"
+    environment:
+      # Raise this (0..1) to make a fraction of valid ids answer 404, emulating a sparse site.
+      - ImgTest__MissRate=0.0