Add a proxy pool with rotation, liveness checks and a management page
The parser will need to move between proxies, so this adds the module it will sit on: pluggable sources, a pool that hands proxies out and learns from the outcome, three rotation strategies, and a page to drive it. Sources are IProxySource implementations. The public proxifly/free-proxy-list feed is fetched as the combined all/data.json through jsDelivr and filtered locally — one conditional request beats four per-protocol ones that can disagree mid-publish — and cached for the five minutes upstream takes to regenerate. A feed that is down keeps serving its last payload rather than emptying the pool. The user's own list lives in proxies.custom.json beside the settings, takes a pasted blob, and names the lines it could not parse instead of quietly dropping them. Both knobs the pool exposes are settings, as asked: rotation is Sticky (default, the only one that keeps site sessions coherent), RoundRobin or WeightedRandom; liveness is either a parallel sweep of the whole pool or a probe at hand-out time. Free lists are a few percent alive, so skipping verification entirely means mostly waiting on timeouts. Two invariants worth keeping, both of which cost a bug to find: Availability is decided by the quarantine, never by Health. Excluding everything that has ever failed made the quarantine window dead code and discarded proxies permanently on their first hiccup, which is exactly wrong for addresses that flap constantly. Health only orders the candidates now. A probe verdict does not touch the success/failure counters. Those are about real requests, and letting a sweep over a few thousand proxies rewrite them would drown the evidence weighted selection reads. SOCKS needs no extra package — .NET resolves socks4/socks4a/socks5 in WebProxy — but a proxifly record with "protocol": "https" is still an HTTP proxy reached over http:// with CONNECT, not an https:// scheme. 115 new tests. Also fixes a pre-existing flake: a command gated on another command's IsExecuting cannot be driven straight after its Execute() completes, because IsExecuting is published on the output scheduler. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
Leonid Pershin
co-authored by
Claude Opus 5
Claude Opus 5
parent
aeafe0af36
commit
9bf2ea5532
@@ -0,0 +1,106 @@
|
||||
using AvParser.Core.Proxies;
|
||||
|
||||
namespace AvParser.Core.Tests.Proxies;
|
||||
|
||||
public class ProxyEndpointTests
|
||||
{
|
||||
[Theory]
|
||||
[InlineData("1.2.3.4:8080", ProxyProtocol.Http, "1.2.3.4", 8080)]
|
||||
[InlineData("http://1.2.3.4:8080", ProxyProtocol.Http, "1.2.3.4", 8080)]
|
||||
[InlineData("https://proxy.example.com:3128", ProxyProtocol.Https, "proxy.example.com", 3128)]
|
||||
[InlineData("socks4://1.2.3.4:1080", ProxyProtocol.Socks4, "1.2.3.4", 1080)]
|
||||
[InlineData("socks5://1.2.3.4:1080", ProxyProtocol.Socks5, "1.2.3.4", 1080)]
|
||||
[InlineData(" socks5h://1.2.3.4:1080 ", ProxyProtocol.Socks5, "1.2.3.4", 1080)]
|
||||
public void Parses_the_forms_public_lists_actually_use(string text, ProxyProtocol protocol, string host, int port)
|
||||
{
|
||||
ProxyEndpoint.TryParse(text, out var endpoint).ShouldBeTrue();
|
||||
|
||||
endpoint!.Protocol.ShouldBe(protocol);
|
||||
endpoint.Host.ShouldBe(host);
|
||||
endpoint.Port.ShouldBe(port);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Parses_credentials()
|
||||
{
|
||||
ProxyEndpoint.TryParse("http://alice:s3cret@1.2.3.4:8080", out var endpoint).ShouldBeTrue();
|
||||
|
||||
endpoint!.Username.ShouldBe("alice");
|
||||
endpoint.Password.ShouldBe("s3cret");
|
||||
endpoint.HasCredentials.ShouldBeTrue();
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Takes_the_last_at_sign_so_a_password_may_contain_one()
|
||||
{
|
||||
ProxyEndpoint.TryParse("alice:p@ss@1.2.3.4:8080", out var endpoint).ShouldBeTrue();
|
||||
|
||||
endpoint!.Username.ShouldBe("alice");
|
||||
endpoint.Password.ShouldBe("p@ss");
|
||||
endpoint.Host.ShouldBe("1.2.3.4");
|
||||
}
|
||||
|
||||
[Theory]
|
||||
[InlineData(null)]
|
||||
[InlineData("")]
|
||||
[InlineData(" ")]
|
||||
[InlineData("1.2.3.4")]
|
||||
[InlineData("1.2.3.4:")]
|
||||
[InlineData("1.2.3.4:0")]
|
||||
[InlineData("1.2.3.4:70000")]
|
||||
[InlineData("1.2.3.4:notaport")]
|
||||
[InlineData(":8080")]
|
||||
[InlineData("gopher://1.2.3.4:8080")]
|
||||
[InlineData("@1.2.3.4:8080")]
|
||||
public void Rejects_malformed_input(string? text) => ProxyEndpoint.TryParse(text, out _).ShouldBeFalse();
|
||||
|
||||
[Fact]
|
||||
public void Key_is_case_insensitive_and_identifies_the_address()
|
||||
{
|
||||
ProxyEndpoint.TryParse("SOCKS5://Proxy.Example.COM:1080", out var upper).ShouldBeTrue();
|
||||
ProxyEndpoint.TryParse("socks5://proxy.example.com:1080", out var lower).ShouldBeTrue();
|
||||
|
||||
upper!.Key.ShouldBe(lower!.Key);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Key_separates_the_same_host_on_different_protocols()
|
||||
{
|
||||
ProxyEndpoint.TryParse("http://1.2.3.4:1080", out var http).ShouldBeTrue();
|
||||
ProxyEndpoint.TryParse("socks5://1.2.3.4:1080", out var socks).ShouldBeTrue();
|
||||
|
||||
http!.Key.ShouldNotBe(socks!.Key);
|
||||
}
|
||||
|
||||
[Theory]
|
||||
[InlineData(ProxyProtocol.Http, "http")]
|
||||
[InlineData(ProxyProtocol.Https, "http")]
|
||||
[InlineData(ProxyProtocol.Socks4, "socks4")]
|
||||
[InlineData(ProxyProtocol.Socks5, "socks5")]
|
||||
public void Https_proxies_are_still_reached_over_the_http_scheme(ProxyProtocol protocol, string scheme)
|
||||
{
|
||||
// .NET has no "https" proxy scheme — an HTTPS-capable proxy tunnels TLS with CONNECT
|
||||
// over a plain http:// proxy URI. Getting this wrong makes every such proxy unusable.
|
||||
new ProxyEndpoint(protocol, "1.2.3.4", 8080).Scheme.ShouldBe(scheme);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Round_trips_through_its_own_string_form()
|
||||
{
|
||||
var original = new ProxyEndpoint(ProxyProtocol.Socks5, "1.2.3.4", 1080);
|
||||
|
||||
ProxyEndpoint.TryParse(original.ToString(), out var parsed).ShouldBeTrue();
|
||||
|
||||
parsed!.Key.ShouldBe(original.Key);
|
||||
}
|
||||
|
||||
[Theory]
|
||||
[InlineData("transparent", ProxyAnonymity.Transparent)]
|
||||
[InlineData("anonymous", ProxyAnonymity.Anonymous)]
|
||||
[InlineData("elite", ProxyAnonymity.Elite)]
|
||||
[InlineData("high", ProxyAnonymity.Elite)]
|
||||
[InlineData("nonsense", ProxyAnonymity.Unknown)]
|
||||
[InlineData(null, ProxyAnonymity.Unknown)]
|
||||
public void Parses_anonymity(string? text, ProxyAnonymity expected) =>
|
||||
ProxyEndpoint.ParseAnonymity(text).ShouldBe(expected);
|
||||
}
|
||||
@@ -0,0 +1,370 @@
|
||||
using AvParser.Core.Proxies;
|
||||
|
||||
namespace AvParser.Core.Tests.Proxies;
|
||||
|
||||
public class ProxyPoolTests
|
||||
{
|
||||
private static readonly ProxyOptions PoolMode = new()
|
||||
{
|
||||
HealthCheck = ProxyHealthCheck.Pool,
|
||||
Rotation = ProxyRotation.Sticky,
|
||||
};
|
||||
|
||||
private static ProxyPool Build(
|
||||
out FakeProxySource source,
|
||||
out FakeProxyProbe probe,
|
||||
out FakeTimeProvider clock,
|
||||
ProxyOptions? options = null,
|
||||
params string[] hosts
|
||||
)
|
||||
{
|
||||
source = new FakeProxySource(ProxySourceKind.Feed);
|
||||
source.Endpoints.AddRange(hosts.Select(host => ProxyFactory.Endpoint(host)));
|
||||
|
||||
probe = new FakeProxyProbe();
|
||||
clock = new FakeTimeProvider();
|
||||
|
||||
return new ProxyPool([source], probe, options ?? PoolMode, clock);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Refresh_loads_every_source()
|
||||
{
|
||||
var pool = Build(out _, out _, out _, hosts: ["a", "b", "c"]);
|
||||
|
||||
var count = await pool.RefreshAsync(TestContext.Current.CancellationToken);
|
||||
|
||||
count.ShouldBe(3);
|
||||
pool.Entries.Count.ShouldBe(3);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Refresh_drops_duplicates_across_sources()
|
||||
{
|
||||
var feed = new FakeProxySource(ProxySourceKind.Feed);
|
||||
feed.Endpoints.Add(ProxyFactory.Endpoint("shared"));
|
||||
|
||||
var custom = new FakeProxySource(ProxySourceKind.Custom) { Id = "custom" };
|
||||
custom.Endpoints.Add(ProxyFactory.Endpoint("shared"));
|
||||
|
||||
var pool = new ProxyPool([feed, custom], new FakeProxyProbe(), PoolMode, new FakeTimeProvider());
|
||||
|
||||
(await pool.RefreshAsync(TestContext.Current.CancellationToken)).ShouldBe(1);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Refresh_keeps_what_the_pool_already_learned()
|
||||
{
|
||||
var pool = Build(out _, out _, out var clock, hosts: ["a"]);
|
||||
await pool.RefreshAsync(TestContext.Current.CancellationToken);
|
||||
|
||||
pool.Entries[0].RecordSuccess(clock.GetUtcNow(), TimeSpan.FromMilliseconds(120));
|
||||
|
||||
// Free lists are republished every few minutes; a reload that reset every counter would
|
||||
// throw away the only real evidence the app has.
|
||||
await pool.RefreshAsync(TestContext.Current.CancellationToken);
|
||||
|
||||
pool.Entries[0].SuccessCount.ShouldBe(1);
|
||||
pool.Entries[0].Latency!.Value.TotalMilliseconds.ShouldBe(120);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Refresh_forgets_addresses_that_left_the_feed()
|
||||
{
|
||||
var pool = Build(out var source, out _, out _, hosts: ["a", "b"]);
|
||||
await pool.RefreshAsync(TestContext.Current.CancellationToken);
|
||||
|
||||
source.Endpoints.RemoveAll(endpoint => endpoint.Host == "b");
|
||||
await pool.RefreshAsync(TestContext.Current.CancellationToken);
|
||||
|
||||
pool.Entries.Select(entry => entry.Endpoint.Host).ShouldBe(["a"]);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Refresh_applies_the_protocol_filter()
|
||||
{
|
||||
var source = new FakeProxySource(ProxySourceKind.Feed);
|
||||
source.Endpoints.Add(ProxyFactory.Endpoint("http", protocol: ProxyProtocol.Http));
|
||||
source.Endpoints.Add(ProxyFactory.Endpoint("socks", protocol: ProxyProtocol.Socks5));
|
||||
|
||||
var options = PoolMode with { Protocols = ProxyProtocolFilter.Socks5 };
|
||||
var pool = new ProxyPool([source], new FakeProxyProbe(), options, new FakeTimeProvider());
|
||||
|
||||
await pool.RefreshAsync(TestContext.Current.CancellationToken);
|
||||
|
||||
pool.Entries.ShouldHaveSingleItem().Endpoint.Protocol.ShouldBe(ProxyProtocol.Socks5);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Refresh_skips_the_feed_when_it_is_switched_off()
|
||||
{
|
||||
var pool = Build(out var source, out _, out _, PoolMode with { UseFeed = false }, "a");
|
||||
|
||||
(await pool.RefreshAsync(TestContext.Current.CancellationToken)).ShouldBe(0);
|
||||
source.GetCallCount.ShouldBe(0);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Acquire_returns_null_for_an_empty_pool() =>
|
||||
(await Build(out _, out _, out _).AcquireAsync(TestContext.Current.CancellationToken)).ShouldBeNull();
|
||||
|
||||
[Fact]
|
||||
public async Task Pool_mode_hands_out_without_probing()
|
||||
{
|
||||
var pool = Build(out _, out var probe, out _, hosts: ["a"]);
|
||||
await pool.RefreshAsync(TestContext.Current.CancellationToken);
|
||||
|
||||
var lease = await pool.AcquireAsync(TestContext.Current.CancellationToken);
|
||||
|
||||
lease.ShouldNotBeNull();
|
||||
probe.ProbeCount.ShouldBe(0);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Lazy_mode_skips_past_dead_proxies()
|
||||
{
|
||||
var options = PoolMode with { HealthCheck = ProxyHealthCheck.Lazy, Rotation = ProxyRotation.RoundRobin };
|
||||
var pool = Build(out _, out var probe, out _, options, "dead1", "dead2", "alive");
|
||||
|
||||
probe.Set(ProxyFactory.Endpoint("alive"), alive: true);
|
||||
await pool.RefreshAsync(TestContext.Current.CancellationToken);
|
||||
|
||||
var lease = await pool.AcquireAsync(TestContext.Current.CancellationToken);
|
||||
|
||||
lease.ShouldNotBeNull();
|
||||
lease.Endpoint.Host.ShouldBe("alive");
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Lazy_mode_gives_up_after_the_configured_number_of_attempts()
|
||||
{
|
||||
var options = PoolMode with
|
||||
{
|
||||
HealthCheck = ProxyHealthCheck.Lazy,
|
||||
Rotation = ProxyRotation.RoundRobin,
|
||||
LazyProbeAttempts = 2,
|
||||
};
|
||||
var pool = Build(out _, out var probe, out _, options, "a", "b", "c", "d", "e");
|
||||
await pool.RefreshAsync(TestContext.Current.CancellationToken);
|
||||
|
||||
(await pool.AcquireAsync(TestContext.Current.CancellationToken)).ShouldBeNull();
|
||||
probe.ProbeCount.ShouldBe(2);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Lazy_mode_trusts_a_proxy_already_known_to_be_alive()
|
||||
{
|
||||
var options = PoolMode with { HealthCheck = ProxyHealthCheck.Lazy };
|
||||
var pool = Build(out _, out var probe, out var clock, options, "a");
|
||||
await pool.RefreshAsync(TestContext.Current.CancellationToken);
|
||||
|
||||
pool.Entries[0].RecordProbe(clock.GetUtcNow(), alive: true, TimeSpan.FromMilliseconds(10), null);
|
||||
|
||||
await pool.AcquireAsync(TestContext.Current.CancellationToken);
|
||||
|
||||
probe.ProbeCount.ShouldBe(0);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task A_failing_proxy_is_quarantined_and_comes_back_later()
|
||||
{
|
||||
var options = PoolMode with
|
||||
{
|
||||
FailuresBeforeQuarantine = 1,
|
||||
BaseQuarantine = TimeSpan.FromSeconds(30),
|
||||
MaxQuarantine = TimeSpan.FromMinutes(15),
|
||||
};
|
||||
var pool = Build(out _, out _, out var clock, options, "a");
|
||||
await pool.RefreshAsync(TestContext.Current.CancellationToken);
|
||||
|
||||
var lease = await pool.AcquireAsync(TestContext.Current.CancellationToken);
|
||||
lease.ShouldNotBeNull();
|
||||
lease.ReportFailure("boom");
|
||||
|
||||
// Sidelined immediately...
|
||||
(await pool.AcquireAsync(TestContext.Current.CancellationToken)).ShouldBeNull();
|
||||
|
||||
clock.Advance(TimeSpan.FromSeconds(31));
|
||||
|
||||
// ...and available again once the window expires.
|
||||
(await pool.AcquireAsync(TestContext.Current.CancellationToken)).ShouldNotBeNull();
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task The_quarantine_window_grows_with_repeated_failures()
|
||||
{
|
||||
var options = PoolMode with
|
||||
{
|
||||
FailuresBeforeQuarantine = 1,
|
||||
BaseQuarantine = TimeSpan.FromSeconds(10),
|
||||
MaxQuarantine = TimeSpan.FromHours(1),
|
||||
};
|
||||
var pool = Build(out _, out _, out var clock, options, "a");
|
||||
await pool.RefreshAsync(TestContext.Current.CancellationToken);
|
||||
var entry = pool.Entries[0];
|
||||
|
||||
entry.RecordFailure(clock.GetUtcNow(), options.BaseQuarantine, options.MaxQuarantine, 1);
|
||||
var first = entry.QuarantinedUntilUtc!.Value - clock.GetUtcNow();
|
||||
|
||||
entry.RecordFailure(clock.GetUtcNow(), options.BaseQuarantine, options.MaxQuarantine, 1);
|
||||
var second = entry.QuarantinedUntilUtc!.Value - clock.GetUtcNow();
|
||||
|
||||
second.ShouldBeGreaterThan(first);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task The_quarantine_window_is_capped()
|
||||
{
|
||||
var pool = Build(out _, out _, out var clock, hosts: ["a"]);
|
||||
await pool.RefreshAsync(TestContext.Current.CancellationToken);
|
||||
var entry = pool.Entries[0];
|
||||
|
||||
for (var i = 0; i < 40; i++)
|
||||
{
|
||||
entry.RecordFailure(clock.GetUtcNow(), TimeSpan.FromSeconds(10), TimeSpan.FromMinutes(5), 1);
|
||||
}
|
||||
|
||||
(entry.QuarantinedUntilUtc!.Value - clock.GetUtcNow()).ShouldBeLessThanOrEqualTo(TimeSpan.FromMinutes(5));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task A_success_clears_the_quarantine()
|
||||
{
|
||||
var pool = Build(out _, out _, out var clock, PoolMode with { FailuresBeforeQuarantine = 1 }, "a");
|
||||
await pool.RefreshAsync(TestContext.Current.CancellationToken);
|
||||
|
||||
var first = await pool.AcquireAsync(TestContext.Current.CancellationToken);
|
||||
first!.ReportFailure();
|
||||
clock.Advance(TimeSpan.FromMinutes(1));
|
||||
|
||||
var second = await pool.AcquireAsync(TestContext.Current.CancellationToken);
|
||||
second!.ReportSuccess(TimeSpan.FromMilliseconds(50));
|
||||
|
||||
pool.Entries[0].QuarantinedUntilUtc.ShouldBeNull();
|
||||
pool.Entries[0].ConsecutiveFailures.ShouldBe(0);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Disposing_a_lease_without_a_verdict_says_nothing_about_the_proxy()
|
||||
{
|
||||
var pool = Build(out _, out _, out _, hosts: ["a"]);
|
||||
await pool.RefreshAsync(TestContext.Current.CancellationToken);
|
||||
|
||||
using (await pool.AcquireAsync(TestContext.Current.CancellationToken))
|
||||
{
|
||||
// A cancelled operation is not the proxy's fault.
|
||||
}
|
||||
|
||||
pool.Entries[0].FailureCount.ShouldBe(0);
|
||||
pool.Entries[0].SuccessCount.ShouldBe(0);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task A_lease_reports_only_once()
|
||||
{
|
||||
var pool = Build(out _, out _, out _, hosts: ["a"]);
|
||||
await pool.RefreshAsync(TestContext.Current.CancellationToken);
|
||||
|
||||
var lease = await pool.AcquireAsync(TestContext.Current.CancellationToken);
|
||||
lease!.ReportSuccess();
|
||||
lease.ReportFailure("late");
|
||||
|
||||
pool.Entries[0].SuccessCount.ShouldBe(1);
|
||||
pool.Entries[0].FailureCount.ShouldBe(0);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Sweep_probes_everything_and_counts_the_survivors()
|
||||
{
|
||||
var pool = Build(out _, out var probe, out _, hosts: ["a", "b", "c"]);
|
||||
probe.Set(ProxyFactory.Endpoint("a"), alive: true);
|
||||
probe.Set(ProxyFactory.Endpoint("c"), alive: true);
|
||||
await pool.RefreshAsync(TestContext.Current.CancellationToken);
|
||||
|
||||
var reports = new List<ProxySweepProgress>();
|
||||
var alive = await pool.SweepAsync(
|
||||
new SynchronousProgress<ProxySweepProgress>(reports.Add),
|
||||
TestContext.Current.CancellationToken
|
||||
);
|
||||
|
||||
alive.ShouldBe(2);
|
||||
probe.ProbeCount.ShouldBe(3);
|
||||
reports.Count.ShouldBe(3);
|
||||
reports[^1].Fraction.ShouldBe(1d);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Sweep_on_an_empty_pool_reports_completion_rather_than_hanging()
|
||||
{
|
||||
var pool = Build(out _, out _, out _);
|
||||
var reports = new List<ProxySweepProgress>();
|
||||
|
||||
var alive = await pool.SweepAsync(
|
||||
new SynchronousProgress<ProxySweepProgress>(reports.Add),
|
||||
TestContext.Current.CancellationToken
|
||||
);
|
||||
|
||||
alive.ShouldBe(0);
|
||||
reports.ShouldHaveSingleItem().Total.ShouldBe(0);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Changing_the_rotation_strategy_takes_effect()
|
||||
{
|
||||
var pool = Build(out _, out _, out _, hosts: ["a", "b", "c"]);
|
||||
await pool.RefreshAsync(TestContext.Current.CancellationToken);
|
||||
|
||||
var sticky = await pool.AcquireAsync(TestContext.Current.CancellationToken);
|
||||
var stickyAgain = await pool.AcquireAsync(TestContext.Current.CancellationToken);
|
||||
stickyAgain!.Endpoint.Key.ShouldBe(sticky!.Endpoint.Key);
|
||||
|
||||
pool.Configure(PoolMode with { Rotation = ProxyRotation.RoundRobin });
|
||||
|
||||
var first = await pool.AcquireAsync(TestContext.Current.CancellationToken);
|
||||
var second = await pool.AcquireAsync(TestContext.Current.CancellationToken);
|
||||
second!.Endpoint.Key.ShouldNotBe(first!.Endpoint.Key);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Selection_prefers_proxies_known_to_be_alive()
|
||||
{
|
||||
var pool = Build(out _, out _, out var clock, hosts: ["slow", "fast"]);
|
||||
await pool.RefreshAsync(TestContext.Current.CancellationToken);
|
||||
|
||||
var slow = pool.Entries.Single(entry => entry.Endpoint.Host == "slow");
|
||||
var fast = pool.Entries.Single(entry => entry.Endpoint.Host == "fast");
|
||||
slow.RecordProbe(clock.GetUtcNow(), alive: true, TimeSpan.FromMilliseconds(900), null);
|
||||
fast.RecordProbe(clock.GetUtcNow(), alive: true, TimeSpan.FromMilliseconds(30), null);
|
||||
|
||||
var lease = await pool.AcquireAsync(TestContext.Current.CancellationToken);
|
||||
|
||||
lease!.Endpoint.Host.ShouldBe("fast");
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Changed_fires_when_the_pool_moves()
|
||||
{
|
||||
var pool = Build(out _, out _, out _, hosts: ["a"]);
|
||||
var fired = 0;
|
||||
pool.Changed += (_, _) => fired++;
|
||||
|
||||
await pool.RefreshAsync(TestContext.Current.CancellationToken);
|
||||
|
||||
fired.ShouldBeGreaterThan(0);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Invalid_options_are_rejected_rather_than_misbehaving_quietly()
|
||||
{
|
||||
Should.Throw<ArgumentOutOfRangeException>(() => new ProxyOptions { ProbeConcurrency = 0 }.Validated());
|
||||
Should.Throw<ArgumentOutOfRangeException>(() => new ProxyOptions { LazyProbeAttempts = 0 }.Validated());
|
||||
Should.Throw<ArgumentOutOfRangeException>(() => new ProxyOptions { ProbeTimeout = TimeSpan.Zero }.Validated());
|
||||
Should.Throw<ArgumentOutOfRangeException>(() =>
|
||||
new ProxyOptions
|
||||
{
|
||||
BaseQuarantine = TimeSpan.FromHours(2),
|
||||
MaxQuarantine = TimeSpan.FromMinutes(1),
|
||||
}.Validated()
|
||||
);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,142 @@
|
||||
using AvParser.Core.Proxies;
|
||||
using AvParser.Core.Proxies.Selection;
|
||||
|
||||
namespace AvParser.Core.Tests.Proxies;
|
||||
|
||||
public class ProxySelectionTests
|
||||
{
|
||||
private static readonly DateTimeOffset Now = new(2026, 1, 1, 0, 0, 0, TimeSpan.Zero);
|
||||
|
||||
[Fact]
|
||||
public void Sticky_keeps_returning_the_same_proxy()
|
||||
{
|
||||
var strategy = new StickyProxySelection();
|
||||
ProxyEntry[] candidates = [ProxyFactory.Entry("a"), ProxyFactory.Entry("b")];
|
||||
|
||||
var first = strategy.Pick(candidates);
|
||||
|
||||
strategy.Pick(candidates).ShouldBeSameAs(first);
|
||||
strategy.Pick(candidates).ShouldBeSameAs(first);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Sticky_moves_on_after_a_failure()
|
||||
{
|
||||
var strategy = new StickyProxySelection();
|
||||
ProxyEntry[] candidates = [ProxyFactory.Entry("a"), ProxyFactory.Entry("b")];
|
||||
|
||||
var first = strategy.Pick(candidates)!;
|
||||
strategy.Report(first, success: false);
|
||||
|
||||
var remaining = candidates.Where(entry => !ReferenceEquals(entry, first)).ToArray();
|
||||
strategy.Pick(remaining).ShouldNotBeSameAs(first);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Sticky_lets_go_when_its_pick_leaves_the_candidate_set()
|
||||
{
|
||||
var strategy = new StickyProxySelection();
|
||||
var a = ProxyFactory.Entry("a");
|
||||
var b = ProxyFactory.Entry("b");
|
||||
|
||||
strategy.Pick([a, b]).ShouldBeSameAs(a);
|
||||
|
||||
// A refresh or a quarantine can drop the current pick under us.
|
||||
strategy.Pick([b]).ShouldBeSameAs(b);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Sticky_survives_a_success_report()
|
||||
{
|
||||
var strategy = new StickyProxySelection();
|
||||
ProxyEntry[] candidates = [ProxyFactory.Entry("a"), ProxyFactory.Entry("b")];
|
||||
|
||||
var first = strategy.Pick(candidates)!;
|
||||
strategy.Report(first, success: true);
|
||||
|
||||
strategy.Pick(candidates).ShouldBeSameAs(first);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void RoundRobin_cycles_through_every_candidate()
|
||||
{
|
||||
var strategy = new RoundRobinProxySelection();
|
||||
ProxyEntry[] candidates = [ProxyFactory.Entry("a"), ProxyFactory.Entry("b"), ProxyFactory.Entry("c")];
|
||||
|
||||
var picked = Enumerable.Range(0, 6).Select(_ => strategy.Pick(candidates)!.Endpoint.Host).ToArray();
|
||||
|
||||
picked.ShouldBe(["a", "b", "c", "a", "b", "c"]);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void RoundRobin_handles_a_shrinking_candidate_list()
|
||||
{
|
||||
var strategy = new RoundRobinProxySelection();
|
||||
ProxyEntry[] three = [ProxyFactory.Entry("a"), ProxyFactory.Entry("b"), ProxyFactory.Entry("c")];
|
||||
|
||||
strategy.Pick(three);
|
||||
strategy.Pick(three);
|
||||
strategy.Pick(three);
|
||||
|
||||
// The cursor is now past the end of the smaller list; this must not throw.
|
||||
Should.NotThrow(() => strategy.Pick([three[0]]));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Every_strategy_returns_null_for_an_empty_pool()
|
||||
{
|
||||
foreach (var rotation in Enum.GetValues<ProxyRotation>())
|
||||
{
|
||||
ProxySelectionStrategyFactory.Create(rotation, new Random(1)).Pick([]).ShouldBeNull();
|
||||
}
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Weighted_selection_favours_proxies_that_have_worked()
|
||||
{
|
||||
var good = ProxyFactory.Entry("good", score: 5);
|
||||
var bad = ProxyFactory.Entry("bad", score: 5);
|
||||
|
||||
for (var i = 0; i < 20; i++)
|
||||
{
|
||||
good.RecordSuccess(Now);
|
||||
bad.RecordFailure(Now, TimeSpan.Zero, TimeSpan.Zero, failuresBeforeQuarantine: int.MaxValue);
|
||||
}
|
||||
|
||||
var strategy = new WeightedRandomProxySelection(new Random(20260813));
|
||||
var picks = Enumerable.Range(0, 400).Count(_ => strategy.Pick([good, bad])!.Endpoint.Host == "good");
|
||||
|
||||
// Not asserting an exact split — this is a random draw. The point is the bias exists and
|
||||
// is decisive, not that it hits a particular number.
|
||||
picks.ShouldBeGreaterThan(340);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Weight_never_drops_to_zero_so_a_bad_proxy_can_recover()
|
||||
{
|
||||
var hopeless = ProxyFactory.Entry("hopeless");
|
||||
for (var i = 0; i < 50; i++)
|
||||
{
|
||||
hopeless.RecordFailure(Now, TimeSpan.Zero, TimeSpan.Zero, failuresBeforeQuarantine: int.MaxValue);
|
||||
}
|
||||
|
||||
WeightedRandomProxySelection.WeightOf(hopeless).ShouldBeGreaterThan(0d);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void An_unproven_proxy_starts_at_even_odds()
|
||||
{
|
||||
// Before any evidence, success rate is 0.5 rather than 0 — otherwise a freshly loaded
|
||||
// pool would have every weight pinned to the floor and selection would be arbitrary.
|
||||
ProxyFactory.Entry("fresh").SuccessRate.ShouldBe(0.5d);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void The_factory_builds_what_it_was_asked_for()
|
||||
{
|
||||
foreach (var rotation in Enum.GetValues<ProxyRotation>())
|
||||
{
|
||||
ProxySelectionStrategyFactory.Create(rotation).Kind.ShouldBe(rotation);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,93 @@
|
||||
using AvParser.Core.Proxies;
|
||||
|
||||
namespace AvParser.Core.Tests.Proxies;
|
||||
|
||||
/// <summary>A clock the tests move by hand, so quarantine expiry is deterministic.</summary>
|
||||
internal sealed class FakeTimeProvider(DateTimeOffset start) : TimeProvider
|
||||
{
|
||||
private DateTimeOffset _now = start;
|
||||
|
||||
public FakeTimeProvider()
|
||||
: this(new DateTimeOffset(2026, 1, 1, 0, 0, 0, TimeSpan.Zero)) { }
|
||||
|
||||
public override DateTimeOffset GetUtcNow() => _now;
|
||||
|
||||
public void Advance(TimeSpan delta) => _now += delta;
|
||||
}
|
||||
|
||||
/// <summary>A source that returns whatever the test handed it.</summary>
|
||||
internal sealed class FakeProxySource(ProxySourceKind kind = ProxySourceKind.Feed, params ProxyEndpoint[] endpoints)
|
||||
: IProxySource
|
||||
{
|
||||
public string Id { get; init; } = "fake";
|
||||
|
||||
public string DisplayName => "Fake source";
|
||||
|
||||
public ProxySourceKind Kind { get; } = kind;
|
||||
|
||||
public List<ProxyEndpoint> Endpoints { get; } = [.. endpoints];
|
||||
|
||||
public int GetCallCount { get; private set; }
|
||||
|
||||
public Task<IReadOnlyList<ProxyEndpoint>> GetProxiesAsync(CancellationToken cancellationToken = default)
|
||||
{
|
||||
GetCallCount++;
|
||||
return Task.FromResult<IReadOnlyList<ProxyEndpoint>>(Endpoints.ToArray());
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>A probe whose verdict the test decides, per address.</summary>
|
||||
internal sealed class FakeProxyProbe : IProxyProbe
|
||||
{
|
||||
private readonly Dictionary<string, bool> _verdicts = new(StringComparer.Ordinal);
|
||||
|
||||
/// <summary>Verdict for addresses with no explicit entry.</summary>
|
||||
public bool DefaultAlive { get; set; }
|
||||
|
||||
/// <summary>How many probes were requested.</summary>
|
||||
public int ProbeCount { get; private set; }
|
||||
|
||||
/// <summary>Addresses probed, in order.</summary>
|
||||
public List<string> Probed { get; } = [];
|
||||
|
||||
public FakeProxyProbe Set(ProxyEndpoint endpoint, bool alive)
|
||||
{
|
||||
_verdicts[endpoint.Key] = alive;
|
||||
return this;
|
||||
}
|
||||
|
||||
public Task<ProxyProbeResult> ProbeAsync(
|
||||
ProxyEndpoint endpoint,
|
||||
ProxyOptions options,
|
||||
CancellationToken cancellationToken = default
|
||||
)
|
||||
{
|
||||
ProbeCount++;
|
||||
Probed.Add(endpoint.Key);
|
||||
|
||||
var alive = _verdicts.TryGetValue(endpoint.Key, out var verdict) ? verdict : DefaultAlive;
|
||||
|
||||
return Task.FromResult(
|
||||
alive ? ProxyProbeResult.Success(TimeSpan.FromMilliseconds(42)) : ProxyProbeResult.Failure("dead")
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>Shorthand builders for the proxy tests.</summary>
|
||||
internal static class ProxyFactory
|
||||
{
|
||||
public static ProxyEndpoint Endpoint(
|
||||
string host,
|
||||
int port = 8080,
|
||||
ProxyProtocol protocol = ProxyProtocol.Http,
|
||||
int score = 0
|
||||
) => new(protocol, host, port) { Score = score };
|
||||
|
||||
public static ProxyEntry Entry(
|
||||
string host,
|
||||
int port = 8080,
|
||||
ProxyProtocol protocol = ProxyProtocol.Http,
|
||||
int score = 0,
|
||||
ProxySourceKind source = ProxySourceKind.Feed
|
||||
) => new(Endpoint(host, port, protocol, score), source);
|
||||
}
|
||||
Reference in New Issue
Block a user