Add a proxy pool with rotation, liveness checks and a management page
The parser will need to move between proxies, so this adds the module it will sit on: pluggable sources, a pool that hands proxies out and learns from the outcome, three rotation strategies, and a page to drive it. Sources are IProxySource implementations. The public proxifly/free-proxy-list feed is fetched as the combined all/data.json through jsDelivr and filtered locally — one conditional request beats four per-protocol ones that can disagree mid-publish — and cached for the five minutes upstream takes to regenerate. A feed that is down keeps serving its last payload rather than emptying the pool. The user's own list lives in proxies.custom.json beside the settings, takes a pasted blob, and names the lines it could not parse instead of quietly dropping them. Both knobs the pool exposes are settings, as asked: rotation is Sticky (default, the only one that keeps site sessions coherent), RoundRobin or WeightedRandom; liveness is either a parallel sweep of the whole pool or a probe at hand-out time. Free lists are a few percent alive, so skipping verification entirely means mostly waiting on timeouts. Two invariants worth keeping, both of which cost a bug to find: Availability is decided by the quarantine, never by Health. Excluding everything that has ever failed made the quarantine window dead code and discarded proxies permanently on their first hiccup, which is exactly wrong for addresses that flap constantly. Health only orders the candidates now. A probe verdict does not touch the success/failure counters. Those are about real requests, and letting a sweep over a few thousand proxies rewrite them would drown the evidence weighted selection reads. SOCKS needs no extra package — .NET resolves socks4/socks4a/socks5 in WebProxy — but a proxifly record with "protocol": "https" is still an HTTP proxy reached over http:// with CONNECT, not an https:// scheme. 115 new tests. Also fixes a pre-existing flake: a command gated on another command's IsExecuting cannot be driven straight after its Execute() completes, because IsExecuting is published on the output scheduler. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
Leonid Pershin
co-authored by
Claude Opus 5
Claude Opus 5
parent
aeafe0af36
commit
9bf2ea5532
@@ -0,0 +1,89 @@
|
||||
namespace AvParser.Core.Proxies;
|
||||
|
||||
/// <summary>The pool of known proxies and the thing that hands them out.</summary>
|
||||
public interface IProxyPool
|
||||
{
|
||||
/// <summary>Snapshot of every known proxy, feed and custom alike.</summary>
|
||||
IReadOnlyList<ProxyEntry> Entries { get; }
|
||||
|
||||
/// <summary>Options currently in force.</summary>
|
||||
ProxyOptions Options { get; }
|
||||
|
||||
/// <summary>Raised after the set of entries or their health changes.</summary>
|
||||
/// <remarks>
|
||||
/// A plain event rather than <c>IObservable</c> so the domain keeps no reactive dependency;
|
||||
/// the UI layer bridges it to an observable where that is convenient.
|
||||
/// </remarks>
|
||||
event EventHandler? Changed;
|
||||
|
||||
/// <summary>Applies new options. Resets selection state when the rotation strategy changes.</summary>
|
||||
void Configure(ProxyOptions options);
|
||||
|
||||
/// <summary>Reloads from every source, preserving health statistics for addresses that survive.</summary>
|
||||
/// <returns>Number of entries in the pool afterwards.</returns>
|
||||
Task<int> RefreshAsync(CancellationToken cancellationToken = default);
|
||||
|
||||
/// <summary>
|
||||
/// Takes a proxy out of the pool for one unit of work, or <see langword="null"/> when nothing
|
||||
/// usable is left.
|
||||
/// </summary>
|
||||
/// <remarks>Report the outcome on the lease, otherwise the pool never learns anything.</remarks>
|
||||
Task<ProxyLease?> AcquireAsync(CancellationToken cancellationToken = default);
|
||||
|
||||
/// <summary>Probes every entry in parallel and updates their health.</summary>
|
||||
/// <returns>How many answered.</returns>
|
||||
Task<int> SweepAsync(IProgress<ProxySweepProgress>? progress = null, CancellationToken cancellationToken = default);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// A proxy checked out of the pool.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// Disposing without a verdict is deliberately neutral: an operation that was cancelled says
|
||||
/// nothing about the proxy, and counting that as a failure would quarantine healthy entries
|
||||
/// every time the user hits Cancel.
|
||||
/// </remarks>
|
||||
public sealed class ProxyLease : IDisposable
|
||||
{
|
||||
private readonly ProxyPool _pool;
|
||||
private bool _reported;
|
||||
|
||||
internal ProxyLease(ProxyPool pool, ProxyEntry entry)
|
||||
{
|
||||
_pool = pool;
|
||||
Entry = entry;
|
||||
}
|
||||
|
||||
/// <summary>The pool entry backing this lease.</summary>
|
||||
public ProxyEntry Entry { get; }
|
||||
|
||||
/// <summary>The address to send traffic through.</summary>
|
||||
public ProxyEndpoint Endpoint => Entry.Endpoint;
|
||||
|
||||
/// <summary>Records that the work succeeded.</summary>
|
||||
public void ReportSuccess(TimeSpan? latency = null)
|
||||
{
|
||||
if (_reported)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
_reported = true;
|
||||
_pool.ReportOutcome(Entry, success: true, latency, error: null);
|
||||
}
|
||||
|
||||
/// <summary>Records that the work failed, which may quarantine the proxy.</summary>
|
||||
public void ReportFailure(string? error = null)
|
||||
{
|
||||
if (_reported)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
_reported = true;
|
||||
_pool.ReportOutcome(Entry, success: false, latency: null, error);
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
public void Dispose() => _reported = true;
|
||||
}
|
||||
@@ -0,0 +1,65 @@
|
||||
namespace AvParser.Core.Proxies;
|
||||
|
||||
/// <summary>Supplies proxy addresses. One per list the app knows about.</summary>
|
||||
public interface IProxySource
|
||||
{
|
||||
/// <summary>Stable identifier used in settings and logs.</summary>
|
||||
string Id { get; }
|
||||
|
||||
/// <summary>Human-readable name for the UI.</summary>
|
||||
string DisplayName { get; }
|
||||
|
||||
/// <summary>Whether entries from this source are feed-provided or user-entered.</summary>
|
||||
ProxySourceKind Kind { get; }
|
||||
|
||||
/// <summary>Fetches the current list. Implementations may cache.</summary>
|
||||
Task<IReadOnlyList<ProxyEndpoint>> GetProxiesAsync(CancellationToken cancellationToken = default);
|
||||
}
|
||||
|
||||
/// <summary>A source the user can edit.</summary>
|
||||
public interface IMutableProxySource : IProxySource
|
||||
{
|
||||
/// <summary>Adds addresses, ignoring duplicates. Returns how many were actually new.</summary>
|
||||
Task<int> AddAsync(IEnumerable<ProxyEndpoint> endpoints, CancellationToken cancellationToken = default);
|
||||
|
||||
/// <summary>Removes an address. Returns whether it was present.</summary>
|
||||
Task<bool> RemoveAsync(ProxyEndpoint endpoint, CancellationToken cancellationToken = default);
|
||||
|
||||
/// <summary>Removes every address.</summary>
|
||||
Task ClearAsync(CancellationToken cancellationToken = default);
|
||||
}
|
||||
|
||||
/// <summary>Outcome of a single liveness probe.</summary>
|
||||
/// <param name="Alive">Whether the proxy answered acceptably.</param>
|
||||
/// <param name="Latency">Round-trip time when alive.</param>
|
||||
/// <param name="Error">Short failure reason when not alive.</param>
|
||||
public readonly record struct ProxyProbeResult(bool Alive, TimeSpan? Latency, string? Error)
|
||||
{
|
||||
/// <summary>A successful probe.</summary>
|
||||
public static ProxyProbeResult Success(TimeSpan latency) => new(true, latency, null);
|
||||
|
||||
/// <summary>A failed probe.</summary>
|
||||
public static ProxyProbeResult Failure(string error) => new(false, null, error);
|
||||
}
|
||||
|
||||
/// <summary>Checks whether a proxy actually works.</summary>
|
||||
public interface IProxyProbe
|
||||
{
|
||||
/// <summary>Sends one request through <paramref name="endpoint"/> and reports the outcome.</summary>
|
||||
/// <remarks>Must not throw for an unreachable proxy — that is a <see cref="ProxyProbeResult"/>, not an error.</remarks>
|
||||
Task<ProxyProbeResult> ProbeAsync(
|
||||
ProxyEndpoint endpoint,
|
||||
ProxyOptions options,
|
||||
CancellationToken cancellationToken = default
|
||||
);
|
||||
}
|
||||
|
||||
/// <summary>Progress of a pool-wide probe sweep.</summary>
|
||||
/// <param name="Checked">Proxies probed so far.</param>
|
||||
/// <param name="Total">Proxies in the sweep.</param>
|
||||
/// <param name="Alive">How many answered.</param>
|
||||
public readonly record struct ProxySweepProgress(int Checked, int Total, int Alive)
|
||||
{
|
||||
/// <summary>Completion in the range 0..1.</summary>
|
||||
public double Fraction => Total <= 0 ? 0d : Math.Clamp((double)Checked / Total, 0d, 1d);
|
||||
}
|
||||
@@ -0,0 +1,210 @@
|
||||
using System.Diagnostics.CodeAnalysis;
|
||||
using System.Globalization;
|
||||
|
||||
namespace AvParser.Core.Proxies;
|
||||
|
||||
/// <summary>Wire protocol a proxy speaks.</summary>
|
||||
public enum ProxyProtocol
|
||||
{
|
||||
/// <summary>Plain HTTP proxy.</summary>
|
||||
Http,
|
||||
|
||||
/// <summary>HTTP proxy that also handles CONNECT for TLS.</summary>
|
||||
Https,
|
||||
|
||||
/// <summary>SOCKS4.</summary>
|
||||
Socks4,
|
||||
|
||||
/// <summary>SOCKS5.</summary>
|
||||
Socks5,
|
||||
}
|
||||
|
||||
/// <summary>How much of the caller the proxy passes through.</summary>
|
||||
public enum ProxyAnonymity
|
||||
{
|
||||
/// <summary>Not reported by the source.</summary>
|
||||
Unknown,
|
||||
|
||||
/// <summary>Forwards the original address — offers no anonymity at all.</summary>
|
||||
Transparent,
|
||||
|
||||
/// <summary>Hides the original address but announces itself as a proxy.</summary>
|
||||
Anonymous,
|
||||
|
||||
/// <summary>Neither forwards the address nor announces itself.</summary>
|
||||
Elite,
|
||||
}
|
||||
|
||||
/// <summary>Where an entry came from.</summary>
|
||||
public enum ProxySourceKind
|
||||
{
|
||||
/// <summary>Downloaded from a remote list.</summary>
|
||||
Feed,
|
||||
|
||||
/// <summary>Entered by the user and stored locally.</summary>
|
||||
Custom,
|
||||
}
|
||||
|
||||
/// <summary>A single proxy address, with whatever metadata its source supplied.</summary>
|
||||
/// <param name="Protocol">Wire protocol.</param>
|
||||
/// <param name="Host">Hostname or IP literal.</param>
|
||||
/// <param name="Port">TCP port.</param>
|
||||
public sealed record ProxyEndpoint(ProxyProtocol Protocol, string Host, int Port)
|
||||
{
|
||||
/// <summary>ISO country code reported by the source, if any.</summary>
|
||||
public string? Country { get; init; }
|
||||
|
||||
/// <summary>City reported by the source, if any.</summary>
|
||||
public string? City { get; init; }
|
||||
|
||||
/// <summary>Anonymity level reported by the source.</summary>
|
||||
public ProxyAnonymity Anonymity { get; init; } = ProxyAnonymity.Unknown;
|
||||
|
||||
/// <summary>Quality score reported by the source; higher is better. 0 when unknown.</summary>
|
||||
public int Score { get; init; }
|
||||
|
||||
/// <summary>Username for proxies that need authentication.</summary>
|
||||
public string? Username { get; init; }
|
||||
|
||||
/// <summary>Password for proxies that need authentication.</summary>
|
||||
public string? Password { get; init; }
|
||||
|
||||
/// <summary>Scheme as <c>System.Net.WebProxy</c> expects it.</summary>
|
||||
public string Scheme =>
|
||||
Protocol switch
|
||||
{
|
||||
ProxyProtocol.Socks4 => "socks4",
|
||||
ProxyProtocol.Socks5 => "socks5",
|
||||
// .NET has no "https" proxy scheme: an HTTPS-capable proxy is still reached over
|
||||
// http:// and tunnels TLS with CONNECT.
|
||||
_ => "http",
|
||||
};
|
||||
|
||||
/// <summary>Address in <c>scheme://host:port</c> form.</summary>
|
||||
public Uri Uri => new($"{Scheme}://{Host}:{Port.ToString(CultureInfo.InvariantCulture)}");
|
||||
|
||||
/// <summary>Stable identity: two entries for the same address are the same proxy.</summary>
|
||||
public string Key => $"{Protocol}://{Host}:{Port.ToString(CultureInfo.InvariantCulture)}".ToLowerInvariant();
|
||||
|
||||
/// <summary>Whether credentials were supplied.</summary>
|
||||
public bool HasCredentials => !string.IsNullOrEmpty(Username);
|
||||
|
||||
/// <inheritdoc />
|
||||
public override string ToString() =>
|
||||
$"{Protocol.ToString().ToLowerInvariant()}://{Host}:{Port.ToString(CultureInfo.InvariantCulture)}";
|
||||
|
||||
/// <summary>
|
||||
/// Parses <c>[scheme://][user:pass@]host:port</c>. Missing scheme is treated as HTTP.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// Hand-rolled rather than delegating to <see cref="Uri"/>: the socks schemes and the
|
||||
/// bare <c>host:port</c> form that every proxy list uses are not valid absolute URIs.
|
||||
/// </remarks>
|
||||
public static bool TryParse(string? text, [NotNullWhen(true)] out ProxyEndpoint? endpoint)
|
||||
{
|
||||
endpoint = null;
|
||||
|
||||
if (string.IsNullOrWhiteSpace(text))
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
var value = text.Trim();
|
||||
var protocol = ProxyProtocol.Http;
|
||||
|
||||
var schemeEnd = value.IndexOf("://", StringComparison.Ordinal);
|
||||
if (schemeEnd >= 0)
|
||||
{
|
||||
if (!TryParseProtocol(value[..schemeEnd], out protocol))
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
value = value[(schemeEnd + 3)..];
|
||||
}
|
||||
|
||||
string? username = null;
|
||||
string? password = null;
|
||||
|
||||
// Rightmost '@' wins: a password may legitimately contain one.
|
||||
var credentialsEnd = value.LastIndexOf('@');
|
||||
if (credentialsEnd >= 0)
|
||||
{
|
||||
var credentials = value[..credentialsEnd];
|
||||
value = value[(credentialsEnd + 1)..];
|
||||
|
||||
var separator = credentials.IndexOf(':', StringComparison.Ordinal);
|
||||
if (separator < 0)
|
||||
{
|
||||
username = credentials;
|
||||
}
|
||||
else
|
||||
{
|
||||
username = credentials[..separator];
|
||||
password = credentials[(separator + 1)..];
|
||||
}
|
||||
|
||||
if (username.Length == 0)
|
||||
{
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
var portStart = value.LastIndexOf(':');
|
||||
if (portStart <= 0 || portStart == value.Length - 1)
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
var host = value[..portStart].Trim();
|
||||
var portText = value[(portStart + 1)..].Trim();
|
||||
|
||||
if (
|
||||
host.Length == 0
|
||||
|| !int.TryParse(portText, NumberStyles.None, CultureInfo.InvariantCulture, out var port)
|
||||
|| port is < 1 or > 65535
|
||||
)
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
endpoint = new ProxyEndpoint(protocol, host, port) { Username = username, Password = password };
|
||||
return true;
|
||||
}
|
||||
|
||||
/// <summary>Parses a protocol name; accepts the spellings the public lists use.</summary>
|
||||
public static bool TryParseProtocol(string? text, out ProxyProtocol protocol)
|
||||
{
|
||||
switch (text?.Trim().ToLowerInvariant())
|
||||
{
|
||||
case "http":
|
||||
protocol = ProxyProtocol.Http;
|
||||
return true;
|
||||
case "https":
|
||||
case "ssl":
|
||||
protocol = ProxyProtocol.Https;
|
||||
return true;
|
||||
case "socks4":
|
||||
case "socks4a":
|
||||
protocol = ProxyProtocol.Socks4;
|
||||
return true;
|
||||
case "socks5":
|
||||
case "socks5h":
|
||||
protocol = ProxyProtocol.Socks5;
|
||||
return true;
|
||||
default:
|
||||
protocol = ProxyProtocol.Http;
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>Parses an anonymity level; unrecognised values become <see cref="ProxyAnonymity.Unknown"/>.</summary>
|
||||
public static ProxyAnonymity ParseAnonymity(string? text) =>
|
||||
text?.Trim().ToLowerInvariant() switch
|
||||
{
|
||||
"transparent" => ProxyAnonymity.Transparent,
|
||||
"anonymous" => ProxyAnonymity.Anonymous,
|
||||
"elite" or "high" => ProxyAnonymity.Elite,
|
||||
_ => ProxyAnonymity.Unknown,
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,186 @@
|
||||
namespace AvParser.Core.Proxies;
|
||||
|
||||
/// <summary>What the last check or use said about a proxy.</summary>
|
||||
public enum ProxyHealthState
|
||||
{
|
||||
/// <summary>Never checked and never used.</summary>
|
||||
Unknown,
|
||||
|
||||
/// <summary>A probe or a real request succeeded.</summary>
|
||||
Alive,
|
||||
|
||||
/// <summary>A probe or a real request failed.</summary>
|
||||
Dead,
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// A proxy plus everything the pool has learned about it.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// Mutable and guarded by <see cref="ProxyPool"/>'s lock rather than being a record: the pool
|
||||
/// updates counters on every request, and reallocating an immutable entry per outcome would
|
||||
/// churn hard on a list of a few thousand proxies.
|
||||
/// </remarks>
|
||||
public sealed class ProxyEntry
|
||||
{
|
||||
private readonly Lock _gate = new();
|
||||
|
||||
/// <summary>Creates an entry in the <see cref="ProxyHealthState.Unknown"/> state.</summary>
|
||||
public ProxyEntry(ProxyEndpoint endpoint, ProxySourceKind source)
|
||||
{
|
||||
Endpoint = endpoint ?? throw new ArgumentNullException(nameof(endpoint));
|
||||
Source = source;
|
||||
}
|
||||
|
||||
/// <summary>The address.</summary>
|
||||
public ProxyEndpoint Endpoint { get; }
|
||||
|
||||
/// <summary>Whether this came from a feed or from the user.</summary>
|
||||
public ProxySourceKind Source { get; }
|
||||
|
||||
/// <summary>Latest known state.</summary>
|
||||
public ProxyHealthState Health { get; private set; }
|
||||
|
||||
/// <summary>Round-trip time of the last successful probe or request.</summary>
|
||||
public TimeSpan? Latency { get; private set; }
|
||||
|
||||
/// <summary>When the state was last updated.</summary>
|
||||
public DateTimeOffset? LastCheckedUtc { get; private set; }
|
||||
|
||||
/// <summary>Successful uses since the entry was created.</summary>
|
||||
public int SuccessCount { get; private set; }
|
||||
|
||||
/// <summary>Failed uses since the entry was created.</summary>
|
||||
public int FailureCount { get; private set; }
|
||||
|
||||
/// <summary>Failures since the last success. Drives the quarantine backoff.</summary>
|
||||
public int ConsecutiveFailures { get; private set; }
|
||||
|
||||
/// <summary>While set and in the future, the entry is skipped by selection.</summary>
|
||||
public DateTimeOffset? QuarantinedUntilUtc { get; private set; }
|
||||
|
||||
/// <summary>Reason recorded with the last failure, for the UI.</summary>
|
||||
public string? LastError { get; private set; }
|
||||
|
||||
/// <summary>Share of successful uses, 0..1. Returns 0.5 before any evidence exists.</summary>
|
||||
public double SuccessRate
|
||||
{
|
||||
get
|
||||
{
|
||||
var total = SuccessCount + FailureCount;
|
||||
return total == 0 ? 0.5d : (double)SuccessCount / total;
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>Whether selection may hand this entry out at <paramref name="now"/>.</summary>
|
||||
/// <remarks>
|
||||
/// Governed by the quarantine alone, deliberately not by <see cref="Health"/>. Health is the
|
||||
/// last thing observed; excluding every entry that has ever failed would make the quarantine
|
||||
/// window meaningless and would permanently discard proxies on their first hiccup — and free
|
||||
/// proxies flap constantly. Health still decides ordering, so dead entries sink to the back.
|
||||
/// </remarks>
|
||||
public bool IsAvailable(DateTimeOffset now) => QuarantinedUntilUtc is null || QuarantinedUntilUtc <= now;
|
||||
|
||||
/// <summary>Whether the entry is currently serving a quarantine.</summary>
|
||||
public bool IsQuarantined(DateTimeOffset now) => QuarantinedUntilUtc is { } until && until > now;
|
||||
|
||||
/// <summary>Records a successful probe or request.</summary>
|
||||
public void RecordSuccess(DateTimeOffset now, TimeSpan? latency = null)
|
||||
{
|
||||
lock (_gate)
|
||||
{
|
||||
Health = ProxyHealthState.Alive;
|
||||
SuccessCount++;
|
||||
ConsecutiveFailures = 0;
|
||||
QuarantinedUntilUtc = null;
|
||||
LastError = null;
|
||||
LastCheckedUtc = now;
|
||||
|
||||
if (latency is not null)
|
||||
{
|
||||
Latency = latency;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Records a failure and, once <paramref name="failuresBeforeQuarantine"/> pile up, sidelines
|
||||
/// the entry for an exponentially growing window capped at <paramref name="maxQuarantine"/>.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// Exponential rather than fixed: a proxy that fails once may just have hit a flaky moment,
|
||||
/// while one that has failed six times in a row should not be retried every few seconds for
|
||||
/// the rest of the session.
|
||||
/// </remarks>
|
||||
public void RecordFailure(
|
||||
DateTimeOffset now,
|
||||
TimeSpan baseQuarantine,
|
||||
TimeSpan maxQuarantine,
|
||||
int failuresBeforeQuarantine,
|
||||
string? error = null
|
||||
)
|
||||
{
|
||||
lock (_gate)
|
||||
{
|
||||
FailureCount++;
|
||||
ConsecutiveFailures++;
|
||||
LastCheckedUtc = now;
|
||||
LastError = error;
|
||||
Health = ProxyHealthState.Dead;
|
||||
|
||||
if (ConsecutiveFailures < failuresBeforeQuarantine)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
var exponent = Math.Min(ConsecutiveFailures - failuresBeforeQuarantine, 16);
|
||||
var ticks = baseQuarantine.Ticks * Math.Pow(2, exponent);
|
||||
var window = ticks >= maxQuarantine.Ticks ? maxQuarantine : TimeSpan.FromTicks((long)ticks);
|
||||
|
||||
QuarantinedUntilUtc = now + window;
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>Records the outcome of a liveness probe.</summary>
|
||||
/// <param name="now">Current time.</param>
|
||||
/// <param name="alive">Whether the probe succeeded.</param>
|
||||
/// <param name="latency">Round-trip time when alive.</param>
|
||||
/// <param name="error">Failure reason when not alive.</param>
|
||||
/// <param name="quarantineOnFailure">
|
||||
/// How long to sideline the entry if the probe failed. Leave null to only record the state.
|
||||
/// </param>
|
||||
/// <remarks>
|
||||
/// Does not touch <see cref="SuccessCount"/> or <see cref="FailureCount"/>: those track real
|
||||
/// requests, and letting a sweep of a few thousand proxies rewrite them would drown the
|
||||
/// evidence that weighted selection depends on.
|
||||
/// </remarks>
|
||||
public void RecordProbe(
|
||||
DateTimeOffset now,
|
||||
bool alive,
|
||||
TimeSpan? latency,
|
||||
string? error,
|
||||
TimeSpan? quarantineOnFailure = null
|
||||
)
|
||||
{
|
||||
lock (_gate)
|
||||
{
|
||||
Health = alive ? ProxyHealthState.Alive : ProxyHealthState.Dead;
|
||||
LastCheckedUtc = now;
|
||||
Latency = alive ? latency : null;
|
||||
LastError = alive ? null : error;
|
||||
|
||||
if (alive)
|
||||
{
|
||||
ConsecutiveFailures = 0;
|
||||
QuarantinedUntilUtc = null;
|
||||
}
|
||||
else if (quarantineOnFailure is { } window)
|
||||
{
|
||||
QuarantinedUntilUtc = now + window;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
public override string ToString() => $"{Endpoint} [{Health}]";
|
||||
}
|
||||
@@ -0,0 +1,129 @@
|
||||
namespace AvParser.Core.Proxies;
|
||||
|
||||
/// <summary>How the pool picks the next proxy.</summary>
|
||||
public enum ProxyRotation
|
||||
{
|
||||
/// <summary>Keep one proxy until it fails. Least disruptive to session cookies.</summary>
|
||||
Sticky,
|
||||
|
||||
/// <summary>Advance through the pool on every acquisition. Spreads rate limits.</summary>
|
||||
RoundRobin,
|
||||
|
||||
/// <summary>Pick at random, weighted by feed score and observed success rate.</summary>
|
||||
WeightedRandom,
|
||||
}
|
||||
|
||||
/// <summary>When liveness is verified.</summary>
|
||||
public enum ProxyHealthCheck
|
||||
{
|
||||
/// <summary>
|
||||
/// Probe the whole pool up front, in parallel. Costs one sweep, then hands out proxies with
|
||||
/// no extra latency — the right default when most of a free list is dead.
|
||||
/// </summary>
|
||||
Pool,
|
||||
|
||||
/// <summary>
|
||||
/// Probe a single proxy at the moment it is handed out, skipping to the next if it fails.
|
||||
/// No sweep, but every acquisition pays a round trip.
|
||||
/// </summary>
|
||||
Lazy,
|
||||
}
|
||||
|
||||
/// <summary>Protocols to accept when loading sources.</summary>
|
||||
/// <remarks>
|
||||
/// A flags enum rather than a list so that <see cref="ProxyOptions"/> and the persisted settings
|
||||
/// keep value equality — a record holding a collection compares by reference, which would make
|
||||
/// every "did anything change?" check say yes.
|
||||
/// </remarks>
|
||||
[Flags]
|
||||
public enum ProxyProtocolFilter
|
||||
{
|
||||
/// <summary>No filter — accept everything.</summary>
|
||||
None = 0,
|
||||
|
||||
/// <summary>Plain HTTP proxies.</summary>
|
||||
Http = 1,
|
||||
|
||||
/// <summary>HTTPS-capable HTTP proxies.</summary>
|
||||
Https = 2,
|
||||
|
||||
/// <summary>SOCKS4.</summary>
|
||||
Socks4 = 4,
|
||||
|
||||
/// <summary>SOCKS5.</summary>
|
||||
Socks5 = 8,
|
||||
|
||||
/// <summary>Every protocol.</summary>
|
||||
All = Http | Https | Socks4 | Socks5,
|
||||
}
|
||||
|
||||
/// <summary>Tuning for <see cref="ProxyPool"/>. Mirrors what the Settings page exposes.</summary>
|
||||
public sealed record ProxyOptions
|
||||
{
|
||||
/// <summary>Selection strategy.</summary>
|
||||
public ProxyRotation Rotation { get; init; } = ProxyRotation.Sticky;
|
||||
|
||||
/// <summary>Liveness policy.</summary>
|
||||
public ProxyHealthCheck HealthCheck { get; init; } = ProxyHealthCheck.Pool;
|
||||
|
||||
/// <summary>Protocols to keep when loading sources.</summary>
|
||||
public ProxyProtocolFilter Protocols { get; init; } = ProxyProtocolFilter.All;
|
||||
|
||||
/// <summary>ISO country codes to keep. Empty means "all".</summary>
|
||||
public IReadOnlyList<string> Countries { get; init; } = [];
|
||||
|
||||
/// <summary>URL fetched to decide whether a proxy works.</summary>
|
||||
/// <remarks>
|
||||
/// Defaults to a plain-HTTP 204 endpoint: it is tiny, and requiring TLS would fail every
|
||||
/// proxy that cannot do CONNECT rather than every proxy that is actually dead.
|
||||
/// </remarks>
|
||||
public Uri ProbeUrl { get; init; } = new("http://www.gstatic.com/generate_204");
|
||||
|
||||
/// <summary>Per-proxy probe timeout.</summary>
|
||||
public TimeSpan ProbeTimeout { get; init; } = TimeSpan.FromSeconds(8);
|
||||
|
||||
/// <summary>How many probes run at once during a pool sweep.</summary>
|
||||
public int ProbeConcurrency { get; init; } = 64;
|
||||
|
||||
/// <summary>Consecutive failures tolerated before an entry is quarantined.</summary>
|
||||
public int FailuresBeforeQuarantine { get; init; } = 2;
|
||||
|
||||
/// <summary>First quarantine window; doubles with each further consecutive failure.</summary>
|
||||
public TimeSpan BaseQuarantine { get; init; } = TimeSpan.FromSeconds(30);
|
||||
|
||||
/// <summary>Ceiling for the quarantine window.</summary>
|
||||
public TimeSpan MaxQuarantine { get; init; } = TimeSpan.FromMinutes(15);
|
||||
|
||||
/// <summary>
|
||||
/// Proxies tried per <see cref="IProxyPool.AcquireAsync"/> call under
|
||||
/// <see cref="ProxyHealthCheck.Lazy"/> before giving up.
|
||||
/// </summary>
|
||||
public int LazyProbeAttempts { get; init; } = 5;
|
||||
|
||||
/// <summary>Whether the remote feed is consulted at all.</summary>
|
||||
public bool UseFeed { get; init; } = true;
|
||||
|
||||
/// <summary>Validates the options, throwing on values that would misbehave silently.</summary>
|
||||
/// <exception cref="ArgumentOutOfRangeException">A numeric option is out of range.</exception>
|
||||
public ProxyOptions Validated()
|
||||
{
|
||||
ArgumentOutOfRangeException.ThrowIfLessThan(ProbeConcurrency, 1);
|
||||
ArgumentOutOfRangeException.ThrowIfLessThan(LazyProbeAttempts, 1);
|
||||
ArgumentOutOfRangeException.ThrowIfLessThan(FailuresBeforeQuarantine, 1);
|
||||
ArgumentOutOfRangeException.ThrowIfLessThanOrEqual(ProbeTimeout, TimeSpan.Zero);
|
||||
ArgumentOutOfRangeException.ThrowIfGreaterThan(BaseQuarantine, MaxQuarantine);
|
||||
|
||||
return this;
|
||||
}
|
||||
|
||||
/// <summary>Maps a protocol onto its filter flag.</summary>
|
||||
public static ProxyProtocolFilter ToFlag(ProxyProtocol protocol) =>
|
||||
protocol switch
|
||||
{
|
||||
ProxyProtocol.Http => ProxyProtocolFilter.Http,
|
||||
ProxyProtocol.Https => ProxyProtocolFilter.Https,
|
||||
ProxyProtocol.Socks4 => ProxyProtocolFilter.Socks4,
|
||||
ProxyProtocol.Socks5 => ProxyProtocolFilter.Socks5,
|
||||
_ => ProxyProtocolFilter.None,
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,340 @@
|
||||
using AvParser.Core.Proxies.Selection;
|
||||
|
||||
namespace AvParser.Core.Proxies;
|
||||
|
||||
/// <inheritdoc cref="IProxyPool" />
|
||||
public sealed class ProxyPool : IProxyPool
|
||||
{
|
||||
private readonly IReadOnlyList<IProxySource> _sources;
|
||||
private readonly IProxyProbe _probe;
|
||||
private readonly TimeProvider _time;
|
||||
private readonly Random? _random;
|
||||
private readonly Lock _gate = new();
|
||||
|
||||
private readonly Dictionary<string, ProxyEntry> _byKey = new(StringComparer.Ordinal);
|
||||
private List<ProxyEntry> _entries = [];
|
||||
private IProxySelectionStrategy _strategy;
|
||||
private ProxyOptions _options;
|
||||
|
||||
/// <summary>Creates a pool over the given sources.</summary>
|
||||
/// <param name="sources">Every list the app knows about; order decides nothing.</param>
|
||||
/// <param name="probe">Liveness checker.</param>
|
||||
/// <param name="options">Initial options.</param>
|
||||
/// <param name="timeProvider">Clock; tests inject a fake one to exercise quarantine expiry.</param>
|
||||
/// <param name="random">Randomness for weighted selection; tests seed it.</param>
|
||||
public ProxyPool(
|
||||
IEnumerable<IProxySource> sources,
|
||||
IProxyProbe probe,
|
||||
ProxyOptions? options = null,
|
||||
TimeProvider? timeProvider = null,
|
||||
Random? random = null
|
||||
)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(sources);
|
||||
|
||||
_sources = sources.ToArray();
|
||||
_probe = probe ?? throw new ArgumentNullException(nameof(probe));
|
||||
_time = timeProvider ?? TimeProvider.System;
|
||||
_random = random;
|
||||
_options = (options ?? new ProxyOptions()).Validated();
|
||||
_strategy = ProxySelectionStrategyFactory.Create(_options.Rotation, random);
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
public IReadOnlyList<ProxyEntry> Entries
|
||||
{
|
||||
get
|
||||
{
|
||||
lock (_gate)
|
||||
{
|
||||
return _entries;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
public ProxyOptions Options
|
||||
{
|
||||
get
|
||||
{
|
||||
lock (_gate)
|
||||
{
|
||||
return _options;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
public event EventHandler? Changed;
|
||||
|
||||
/// <inheritdoc />
|
||||
public void Configure(ProxyOptions options)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(options);
|
||||
|
||||
var validated = options.Validated();
|
||||
|
||||
lock (_gate)
|
||||
{
|
||||
var rotationChanged = validated.Rotation != _options.Rotation;
|
||||
_options = validated;
|
||||
|
||||
if (rotationChanged)
|
||||
{
|
||||
_strategy = ProxySelectionStrategyFactory.Create(validated.Rotation, _random);
|
||||
}
|
||||
}
|
||||
|
||||
RaiseChanged();
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
public async Task<int> RefreshAsync(CancellationToken cancellationToken = default)
|
||||
{
|
||||
var options = Options;
|
||||
var collected = new List<(ProxyEndpoint Endpoint, ProxySourceKind Kind)>();
|
||||
|
||||
foreach (var source in _sources)
|
||||
{
|
||||
if (source.Kind == ProxySourceKind.Feed && !options.UseFeed)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
var proxies = await source.GetProxiesAsync(cancellationToken).ConfigureAwait(false);
|
||||
foreach (var endpoint in proxies)
|
||||
{
|
||||
collected.Add((endpoint, source.Kind));
|
||||
}
|
||||
}
|
||||
|
||||
int count;
|
||||
lock (_gate)
|
||||
{
|
||||
var next = new List<ProxyEntry>(collected.Count);
|
||||
var seen = new HashSet<string>(StringComparer.Ordinal);
|
||||
|
||||
foreach (var (endpoint, kind) in collected)
|
||||
{
|
||||
if (!Matches(endpoint, options) || !seen.Add(endpoint.Key))
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
// Reuse the existing entry so a refresh does not wipe out everything the pool
|
||||
// has learned — free lists are republished every few minutes.
|
||||
if (_byKey.TryGetValue(endpoint.Key, out var existing))
|
||||
{
|
||||
next.Add(existing);
|
||||
}
|
||||
else
|
||||
{
|
||||
var created = new ProxyEntry(endpoint, kind);
|
||||
_byKey[endpoint.Key] = created;
|
||||
next.Add(created);
|
||||
}
|
||||
}
|
||||
|
||||
foreach (var key in _byKey.Keys.Where(key => !seen.Contains(key)).ToArray())
|
||||
{
|
||||
_byKey.Remove(key);
|
||||
}
|
||||
|
||||
_entries = next;
|
||||
_strategy.Reset();
|
||||
count = next.Count;
|
||||
}
|
||||
|
||||
RaiseChanged();
|
||||
return count;
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
public async Task<ProxyLease?> AcquireAsync(CancellationToken cancellationToken = default)
|
||||
{
|
||||
var options = Options;
|
||||
|
||||
if (options.HealthCheck == ProxyHealthCheck.Pool)
|
||||
{
|
||||
var entry = SelectAvailable();
|
||||
return entry is null ? null : new ProxyLease(this, entry);
|
||||
}
|
||||
|
||||
// Lazy: verify the pick before handing it over, stepping past dead ones.
|
||||
for (var attempt = 0; attempt < options.LazyProbeAttempts; attempt++)
|
||||
{
|
||||
cancellationToken.ThrowIfCancellationRequested();
|
||||
|
||||
var entry = SelectAvailable();
|
||||
if (entry is null)
|
||||
{
|
||||
return null;
|
||||
}
|
||||
|
||||
if (entry.Health == ProxyHealthState.Alive)
|
||||
{
|
||||
return new ProxyLease(this, entry);
|
||||
}
|
||||
|
||||
var result = await _probe.ProbeAsync(entry.Endpoint, options, cancellationToken).ConfigureAwait(false);
|
||||
var now = _time.GetUtcNow();
|
||||
|
||||
if (result.Alive)
|
||||
{
|
||||
entry.RecordProbe(now, alive: true, result.Latency, error: null);
|
||||
RaiseChanged();
|
||||
return new ProxyLease(this, entry);
|
||||
}
|
||||
|
||||
// Same reasoning as the sweep: this is a probe verdict, so it sidelines the entry
|
||||
// without inflating the request counters that weighted selection reads.
|
||||
entry.RecordProbe(now, alive: false, latency: null, result.Error, options.BaseQuarantine);
|
||||
|
||||
lock (_gate)
|
||||
{
|
||||
_strategy.Report(entry, success: false);
|
||||
}
|
||||
|
||||
RaiseChanged();
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
public async Task<int> SweepAsync(
|
||||
IProgress<ProxySweepProgress>? progress = null,
|
||||
CancellationToken cancellationToken = default
|
||||
)
|
||||
{
|
||||
var options = Options;
|
||||
var targets = Entries;
|
||||
var total = targets.Count;
|
||||
|
||||
if (total == 0)
|
||||
{
|
||||
progress?.Report(new ProxySweepProgress(0, 0, 0));
|
||||
return 0;
|
||||
}
|
||||
|
||||
using var limiter = new SemaphoreSlim(options.ProbeConcurrency, options.ProbeConcurrency);
|
||||
var checkedCount = 0;
|
||||
var aliveCount = 0;
|
||||
|
||||
var work = targets.Select(async entry =>
|
||||
{
|
||||
await limiter.WaitAsync(cancellationToken).ConfigureAwait(false);
|
||||
try
|
||||
{
|
||||
var result = await _probe.ProbeAsync(entry.Endpoint, options, cancellationToken).ConfigureAwait(false);
|
||||
|
||||
// A failed sweep probe sidelines the entry for one base window rather than
|
||||
// counting as a request failure: it is evidence about the proxy, not usage of it.
|
||||
entry.RecordProbe(
|
||||
_time.GetUtcNow(),
|
||||
result.Alive,
|
||||
result.Latency,
|
||||
result.Error,
|
||||
quarantineOnFailure: options.BaseQuarantine
|
||||
);
|
||||
|
||||
if (result.Alive)
|
||||
{
|
||||
Interlocked.Increment(ref aliveCount);
|
||||
}
|
||||
}
|
||||
finally
|
||||
{
|
||||
limiter.Release();
|
||||
progress?.Report(
|
||||
new ProxySweepProgress(
|
||||
Interlocked.Increment(ref checkedCount),
|
||||
total,
|
||||
Volatile.Read(ref aliveCount)
|
||||
)
|
||||
);
|
||||
}
|
||||
});
|
||||
|
||||
await Task.WhenAll(work).ConfigureAwait(false);
|
||||
|
||||
lock (_gate)
|
||||
{
|
||||
// A sweep can invalidate a sticky pick, so make the strategy re-choose.
|
||||
_strategy.Reset();
|
||||
}
|
||||
|
||||
RaiseChanged();
|
||||
return aliveCount;
|
||||
}
|
||||
|
||||
/// <summary>Records the outcome of a lease. Called by <see cref="ProxyLease"/>.</summary>
|
||||
internal void ReportOutcome(ProxyEntry entry, bool success, TimeSpan? latency, string? error)
|
||||
{
|
||||
var options = Options;
|
||||
var now = _time.GetUtcNow();
|
||||
|
||||
if (success)
|
||||
{
|
||||
entry.RecordSuccess(now, latency);
|
||||
}
|
||||
else
|
||||
{
|
||||
entry.RecordFailure(
|
||||
now,
|
||||
options.BaseQuarantine,
|
||||
options.MaxQuarantine,
|
||||
options.FailuresBeforeQuarantine,
|
||||
error
|
||||
);
|
||||
}
|
||||
|
||||
lock (_gate)
|
||||
{
|
||||
_strategy.Report(entry, success);
|
||||
}
|
||||
|
||||
RaiseChanged();
|
||||
}
|
||||
|
||||
/// <summary>Whether an endpoint passes the protocol and country filters.</summary>
|
||||
public static bool Matches(ProxyEndpoint endpoint, ProxyOptions options)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(endpoint);
|
||||
ArgumentNullException.ThrowIfNull(options);
|
||||
|
||||
if (
|
||||
options.Protocols != ProxyProtocolFilter.None
|
||||
&& !options.Protocols.HasFlag(ProxyOptions.ToFlag(endpoint.Protocol))
|
||||
)
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
return options.Countries.Count == 0
|
||||
|| (
|
||||
endpoint.Country is { } country && options.Countries.Contains(country, StringComparer.OrdinalIgnoreCase)
|
||||
);
|
||||
}
|
||||
|
||||
private ProxyEntry? SelectAvailable()
|
||||
{
|
||||
lock (_gate)
|
||||
{
|
||||
var now = _time.GetUtcNow();
|
||||
|
||||
// Best-first ordering: sticky selection takes the head of this list, and the other
|
||||
// strategies benefit from healthy proxies being clustered at the front.
|
||||
var candidates = _entries
|
||||
.Where(entry => entry.IsAvailable(now))
|
||||
.OrderByDescending(entry => entry.Health == ProxyHealthState.Alive)
|
||||
.ThenBy(entry => entry.Latency ?? TimeSpan.MaxValue)
|
||||
.ThenByDescending(entry => entry.Endpoint.Score)
|
||||
.ToArray();
|
||||
|
||||
return _strategy.Pick(candidates);
|
||||
}
|
||||
}
|
||||
|
||||
private void RaiseChanged() => Changed?.Invoke(this, EventArgs.Empty);
|
||||
}
|
||||
@@ -0,0 +1,38 @@
|
||||
namespace AvParser.Core.Proxies.Selection;
|
||||
|
||||
/// <summary>Decides which proxy to hand out next.</summary>
|
||||
/// <remarks>
|
||||
/// Stateful by design — sticky selection has to remember its pick, and round-robin its cursor.
|
||||
/// Implementations are called under the pool's lock and need no locking of their own.
|
||||
/// </remarks>
|
||||
public interface IProxySelectionStrategy
|
||||
{
|
||||
/// <summary>Which setting this strategy implements.</summary>
|
||||
ProxyRotation Kind { get; }
|
||||
|
||||
/// <summary>Picks from the already-filtered available candidates, or <see langword="null"/> if none.</summary>
|
||||
ProxyEntry? Pick(IReadOnlyList<ProxyEntry> candidates);
|
||||
|
||||
/// <summary>Tells the strategy how the handed-out proxy fared.</summary>
|
||||
void Report(ProxyEntry entry, bool success);
|
||||
|
||||
/// <summary>Drops any remembered state, e.g. after the pool is reloaded.</summary>
|
||||
void Reset();
|
||||
}
|
||||
|
||||
/// <summary>Builds the strategy named by <see cref="ProxyOptions.Rotation"/>.</summary>
|
||||
public static class ProxySelectionStrategyFactory
|
||||
{
|
||||
/// <summary>Creates a strategy instance.</summary>
|
||||
/// <param name="rotation">Which strategy to build.</param>
|
||||
/// <param name="random">Randomness for <see cref="ProxyRotation.WeightedRandom"/>; tests pass a seeded instance.</param>
|
||||
/// <exception cref="ArgumentOutOfRangeException">Unknown rotation value.</exception>
|
||||
public static IProxySelectionStrategy Create(ProxyRotation rotation, Random? random = null) =>
|
||||
rotation switch
|
||||
{
|
||||
ProxyRotation.Sticky => new StickyProxySelection(),
|
||||
ProxyRotation.RoundRobin => new RoundRobinProxySelection(),
|
||||
ProxyRotation.WeightedRandom => new WeightedRandomProxySelection(random),
|
||||
_ => throw new ArgumentOutOfRangeException(nameof(rotation), rotation, "Unknown rotation strategy."),
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,43 @@
|
||||
namespace AvParser.Core.Proxies.Selection;
|
||||
|
||||
/// <summary>
|
||||
/// Advances one position through the candidates on every acquisition.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// Spreads requests evenly, which is what rate limits care about. The cursor is kept as a
|
||||
/// monotonic counter reduced modulo the candidate count rather than as an index into the list,
|
||||
/// so a list that shrinks between calls cannot throw or silently skip entries.
|
||||
/// </remarks>
|
||||
public sealed class RoundRobinProxySelection : IProxySelectionStrategy
|
||||
{
|
||||
private int _cursor;
|
||||
|
||||
/// <inheritdoc />
|
||||
public ProxyRotation Kind => ProxyRotation.RoundRobin;
|
||||
|
||||
/// <inheritdoc />
|
||||
public ProxyEntry? Pick(IReadOnlyList<ProxyEntry> candidates)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(candidates);
|
||||
|
||||
if (candidates.Count == 0)
|
||||
{
|
||||
return null;
|
||||
}
|
||||
|
||||
var index = (int)((uint)_cursor % (uint)candidates.Count);
|
||||
_cursor = _cursor == int.MaxValue ? 0 : _cursor + 1;
|
||||
|
||||
return candidates[index];
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
public void Report(ProxyEntry entry, bool success)
|
||||
{
|
||||
// Position is independent of outcome: a failing proxy is dropped by the pool's own
|
||||
// quarantine, not by rewinding the cursor.
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
public void Reset() => _cursor = 0;
|
||||
}
|
||||
@@ -0,0 +1,45 @@
|
||||
namespace AvParser.Core.Proxies.Selection;
|
||||
|
||||
/// <summary>
|
||||
/// Holds one proxy until it fails.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// The default because it is the only strategy that keeps a site's session coherent: rotating on
|
||||
/// every request changes the apparent client mid-session, which reliably triggers re-logins and
|
||||
/// captchas on anything that tracks cookies.
|
||||
/// </remarks>
|
||||
public sealed class StickyProxySelection : IProxySelectionStrategy
|
||||
{
|
||||
private ProxyEntry? _current;
|
||||
|
||||
/// <inheritdoc />
|
||||
public ProxyRotation Kind => ProxyRotation.Sticky;
|
||||
|
||||
/// <inheritdoc />
|
||||
public ProxyEntry? Pick(IReadOnlyList<ProxyEntry> candidates)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(candidates);
|
||||
|
||||
// Keep the current pick only while it is still among the available candidates: a refresh
|
||||
// or a quarantine may have taken it out from under us.
|
||||
if (_current is not null && candidates.Contains(_current))
|
||||
{
|
||||
return _current;
|
||||
}
|
||||
|
||||
_current = candidates.Count > 0 ? candidates[0] : null;
|
||||
return _current;
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
public void Report(ProxyEntry entry, bool success)
|
||||
{
|
||||
if (!success && ReferenceEquals(entry, _current))
|
||||
{
|
||||
_current = null;
|
||||
}
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
public void Reset() => _current = null;
|
||||
}
|
||||
@@ -0,0 +1,83 @@
|
||||
namespace AvParser.Core.Proxies.Selection;
|
||||
|
||||
/// <summary>
|
||||
/// Picks at random, biased towards proxies that have actually worked.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// Weight is the feed's score multiplied by the observed success rate, so a proxy the feed likes
|
||||
/// but that keeps failing here drifts to the bottom without ever being excluded outright — free
|
||||
/// lists recover, and a hard ban would lose them permanently.
|
||||
/// </remarks>
|
||||
public sealed class WeightedRandomProxySelection(Random? random = null) : IProxySelectionStrategy
|
||||
{
|
||||
/// <summary>Floor on the weight so an unproven proxy still gets picked occasionally.</summary>
|
||||
private const double MinimumWeight = 0.05d;
|
||||
|
||||
private readonly Random _random = random ?? Random.Shared;
|
||||
|
||||
/// <inheritdoc />
|
||||
public ProxyRotation Kind => ProxyRotation.WeightedRandom;
|
||||
|
||||
/// <inheritdoc />
|
||||
public ProxyEntry? Pick(IReadOnlyList<ProxyEntry> candidates)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(candidates);
|
||||
|
||||
if (candidates.Count == 0)
|
||||
{
|
||||
return null;
|
||||
}
|
||||
|
||||
if (candidates.Count == 1)
|
||||
{
|
||||
return candidates[0];
|
||||
}
|
||||
|
||||
var weights = new double[candidates.Count];
|
||||
var total = 0d;
|
||||
|
||||
for (var i = 0; i < candidates.Count; i++)
|
||||
{
|
||||
var weight = WeightOf(candidates[i]);
|
||||
weights[i] = weight;
|
||||
total += weight;
|
||||
}
|
||||
|
||||
var target = _random.NextDouble() * total;
|
||||
var running = 0d;
|
||||
|
||||
for (var i = 0; i < weights.Length; i++)
|
||||
{
|
||||
running += weights[i];
|
||||
if (target < running)
|
||||
{
|
||||
return candidates[i];
|
||||
}
|
||||
}
|
||||
|
||||
// Floating-point drift can leave `target` a hair past the final boundary.
|
||||
return candidates[^1];
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
public void Report(ProxyEntry entry, bool success)
|
||||
{
|
||||
// The weight is derived from the entry's own counters, which the pool already updated.
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
public void Reset()
|
||||
{
|
||||
// Stateless beyond the RNG.
|
||||
}
|
||||
|
||||
/// <summary>Weight of a single entry. Exposed so the tests can assert the ordering.</summary>
|
||||
public static double WeightOf(ProxyEntry entry)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(entry);
|
||||
|
||||
// Feed scores are small integers and frequently zero; +1 keeps an unscored proxy in play.
|
||||
var score = Math.Max(0, entry.Endpoint.Score) + 1d;
|
||||
return Math.Max(MinimumWeight, score * entry.SuccessRate);
|
||||
}
|
||||
}
|
||||
@@ -1,3 +1,5 @@
|
||||
using AvParser.Core.Proxies;
|
||||
|
||||
namespace AvParser.Core.Settings;
|
||||
|
||||
/// <summary>Theme preference. <see cref="System"/> follows the OS setting.</summary>
|
||||
@@ -39,4 +41,48 @@ public sealed record AppSettings
|
||||
|
||||
/// <summary>Minimum Serilog level, as a Serilog level name.</summary>
|
||||
public string MinimumLogLevel { get; init; } = "Information";
|
||||
|
||||
/// <summary>How the pool picks the next proxy.</summary>
|
||||
public ProxyRotation ProxyRotation { get; init; } = ProxyRotation.Sticky;
|
||||
|
||||
/// <summary>When proxy liveness is verified.</summary>
|
||||
public ProxyHealthCheck ProxyHealthCheck { get; init; } = ProxyHealthCheck.Pool;
|
||||
|
||||
/// <summary>Whether the remote proxy feed is consulted.</summary>
|
||||
public bool ProxyUseFeed { get; init; } = true;
|
||||
|
||||
/// <summary>Protocols accepted when loading proxy sources.</summary>
|
||||
public ProxyProtocolFilter ProxyProtocols { get; init; } = ProxyProtocolFilter.All;
|
||||
|
||||
/// <summary>URL fetched to decide whether a proxy works.</summary>
|
||||
public string ProxyProbeUrl { get; init; } = "http://www.gstatic.com/generate_204";
|
||||
|
||||
/// <summary>Per-proxy probe timeout, in seconds.</summary>
|
||||
public int ProxyProbeTimeoutSeconds { get; init; } = 8;
|
||||
|
||||
/// <summary>How many probes run at once during a pool sweep.</summary>
|
||||
public int ProxyProbeConcurrency { get; init; } = 64;
|
||||
|
||||
/// <summary>Projects the proxy-related settings onto <see cref="ProxyOptions"/>.</summary>
|
||||
/// <remarks>
|
||||
/// Settings are persisted as primitives so an old file still deserialises; the pool wants a
|
||||
/// validated options object. This is the single place that bridges the two.
|
||||
/// </remarks>
|
||||
public ProxyOptions ToProxyOptions()
|
||||
{
|
||||
var probeUrl = Uri.TryCreate(ProxyProbeUrl, UriKind.Absolute, out var parsed)
|
||||
? parsed
|
||||
: new ProxyOptions().ProbeUrl;
|
||||
|
||||
return new ProxyOptions
|
||||
{
|
||||
Rotation = ProxyRotation,
|
||||
HealthCheck = ProxyHealthCheck,
|
||||
UseFeed = ProxyUseFeed,
|
||||
Protocols = ProxyProtocols,
|
||||
ProbeUrl = probeUrl,
|
||||
ProbeTimeout = TimeSpan.FromSeconds(Math.Clamp(ProxyProbeTimeoutSeconds, 1, 120)),
|
||||
ProbeConcurrency = Math.Clamp(ProxyProbeConcurrency, 1, 512),
|
||||
}.Validated();
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user