Add a proxy pool with rotation, liveness checks and a management page

The parser will need to move between proxies, so this adds the module it will sit on: pluggable sources, a pool that hands proxies out and learns from the outcome, three rotation strategies, and a page to drive it. Sources are IProxySource implementations. The public proxifly/free-proxy-list feed is fetched as the combined all/data.json through jsDelivr and filtered locally — one conditional request beats four per-protocol ones that can disagree mid-publish — and cached for the five minutes upstream takes to regenerate. A feed that is down keeps serving its last payload rather than emptying the pool. The user's own list lives in proxies.custom.json beside the settings, takes a pasted blob, and names the lines it could not parse instead of quietly dropping them. Both knobs the pool exposes are settings, as asked: rotation is Sticky (default, the only one that keeps site sessions coherent), RoundRobin or WeightedRandom; liveness is either a parallel sweep of the whole pool or a probe at hand-out time. Free lists are a few percent alive, so skipping verification entirely means mostly waiting on timeouts. Two invariants worth keeping, both of which cost a bug to find: Availability is decided by the quarantine, never by Health. Excluding everything that has ever failed made the quarantine window dead code and discarded proxies permanently on their first hiccup, which is exactly wrong for addresses that flap constantly. Health only orders the candidates now. A probe verdict does not touch the success/failure counters. Those are about real requests, and letting a sweep over a few thousand proxies rewrite them would drown the evidence weighted selection reads. SOCKS needs no extra package — .NET resolves socks4/socks4a/socks5 in WebProxy — but a proxifly record with "protocol": "https" is still an HTTP proxy reached over http:// with CONNECT, not an https:// scheme. 115 new tests. Also fixes a pre-existing flake: a command gated on another command's IsExecuting cannot be driven straight after its Execute() completes, because IsExecuting is published on the output scheduler. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
2026-08-13 17:22:30 +03:00
parent aeafe0af36
commit 9bf2ea5532
48 changed files with 4422 additions and 5 deletions
@@ -0,0 +1,93 @@
+using AvParser.Core.Proxies;
+
+namespace AvParser.Core.Tests.Proxies;
+
+/// <summary>A clock the tests move by hand, so quarantine expiry is deterministic.</summary>
+internal sealed class FakeTimeProvider(DateTimeOffset start) : TimeProvider
+{
+    private DateTimeOffset _now = start;
+
+    public FakeTimeProvider()
+        : this(new DateTimeOffset(2026, 1, 1, 0, 0, 0, TimeSpan.Zero)) { }
+
+    public override DateTimeOffset GetUtcNow() => _now;
+
+    public void Advance(TimeSpan delta) => _now += delta;
+}
+
+/// <summary>A source that returns whatever the test handed it.</summary>
+internal sealed class FakeProxySource(ProxySourceKind kind = ProxySourceKind.Feed, params ProxyEndpoint[] endpoints)
+    : IProxySource
+{
+    public string Id { get; init; } = "fake";
+
+    public string DisplayName => "Fake source";
+
+    public ProxySourceKind Kind { get; } = kind;
+
+    public List<ProxyEndpoint> Endpoints { get; } = [.. endpoints];
+
+    public int GetCallCount { get; private set; }
+
+    public Task<IReadOnlyList<ProxyEndpoint>> GetProxiesAsync(CancellationToken cancellationToken = default)
+    {
+        GetCallCount++;
+        return Task.FromResult<IReadOnlyList<ProxyEndpoint>>(Endpoints.ToArray());
+    }
+}
+
+/// <summary>A probe whose verdict the test decides, per address.</summary>
+internal sealed class FakeProxyProbe : IProxyProbe
+{
+    private readonly Dictionary<string, bool> _verdicts = new(StringComparer.Ordinal);
+
+    /// <summary>Verdict for addresses with no explicit entry.</summary>
+    public bool DefaultAlive { get; set; }
+
+    /// <summary>How many probes were requested.</summary>
+    public int ProbeCount { get; private set; }
+
+    /// <summary>Addresses probed, in order.</summary>
+    public List<string> Probed { get; } = [];
+
+    public FakeProxyProbe Set(ProxyEndpoint endpoint, bool alive)
+    {
+        _verdicts[endpoint.Key] = alive;
+        return this;
+    }
+
+    public Task<ProxyProbeResult> ProbeAsync(
+        ProxyEndpoint endpoint,
+        ProxyOptions options,
+        CancellationToken cancellationToken = default
+    )
+    {
+        ProbeCount++;
+        Probed.Add(endpoint.Key);
+
+        var alive = _verdicts.TryGetValue(endpoint.Key, out var verdict) ? verdict : DefaultAlive;
+
+        return Task.FromResult(
+            alive ? ProxyProbeResult.Success(TimeSpan.FromMilliseconds(42)) : ProxyProbeResult.Failure("dead")
+        );
+    }
+}
+
+/// <summary>Shorthand builders for the proxy tests.</summary>
+internal static class ProxyFactory
+{
+    public static ProxyEndpoint Endpoint(
+        string host,
+        int port = 8080,
+        ProxyProtocol protocol = ProxyProtocol.Http,
+        int score = 0
+    ) => new(protocol, host, port) { Score = score };
+
+    public static ProxyEntry Entry(
+        string host,
+        int port = 8080,
+        ProxyProtocol protocol = ProxyProtocol.Http,
+        int score = 0,
+        ProxySourceKind source = ProxySourceKind.Feed
+    ) => new(Endpoint(host, port, protocol, score), source);
+}