Add a proxy pool with rotation, liveness checks and a management page
The parser will need to move between proxies, so this adds the module it will sit on: pluggable sources, a pool that hands proxies out and learns from the outcome, three rotation strategies, and a page to drive it. Sources are IProxySource implementations. The public proxifly/free-proxy-list feed is fetched as the combined all/data.json through jsDelivr and filtered locally — one conditional request beats four per-protocol ones that can disagree mid-publish — and cached for the five minutes upstream takes to regenerate. A feed that is down keeps serving its last payload rather than emptying the pool. The user's own list lives in proxies.custom.json beside the settings, takes a pasted blob, and names the lines it could not parse instead of quietly dropping them. Both knobs the pool exposes are settings, as asked: rotation is Sticky (default, the only one that keeps site sessions coherent), RoundRobin or WeightedRandom; liveness is either a parallel sweep of the whole pool or a probe at hand-out time. Free lists are a few percent alive, so skipping verification entirely means mostly waiting on timeouts. Two invariants worth keeping, both of which cost a bug to find: Availability is decided by the quarantine, never by Health. Excluding everything that has ever failed made the quarantine window dead code and discarded proxies permanently on their first hiccup, which is exactly wrong for addresses that flap constantly. Health only orders the candidates now. A probe verdict does not touch the success/failure counters. Those are about real requests, and letting a sweep over a few thousand proxies rewrite them would drown the evidence weighted selection reads. SOCKS needs no extra package — .NET resolves socks4/socks4a/socks5 in WebProxy — but a proxifly record with "protocol": "https" is still an HTTP proxy reached over http:// with CONNECT, not an https:// scheme. 115 new tests. Also fixes a pre-existing flake: a command gated on another command's IsExecuting cannot be driven straight after its Execute() completes, because IsExecuting is published on the output scheduler. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
Leonid Pershin
co-authored by
Claude Opus 5
Claude Opus 5
parent
aeafe0af36
commit
9bf2ea5532
@@ -0,0 +1,363 @@
|
||||
using System.Collections.ObjectModel;
|
||||
using System.Globalization;
|
||||
using AvParser.Core.Proxies;
|
||||
using AvParser.Infrastructure.Proxies;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using ReactiveUI;
|
||||
using ReactiveUI.Primitives;
|
||||
using ReactiveUI.Primitives.Concurrency;
|
||||
using ReactiveUI.Primitives.Signals;
|
||||
using ReactiveUI.SourceGenerators;
|
||||
|
||||
namespace AvParser.UI.ViewModels;
|
||||
|
||||
/// <summary>Health filter offered by the proxy list.</summary>
|
||||
public enum ProxyHealthFilter
|
||||
{
|
||||
/// <summary>No filtering.</summary>
|
||||
All,
|
||||
|
||||
/// <summary>Only proxies whose last check succeeded.</summary>
|
||||
Alive,
|
||||
|
||||
/// <summary>Only proxies whose last check failed.</summary>
|
||||
Dead,
|
||||
|
||||
/// <summary>Only proxies that were never checked.</summary>
|
||||
Unchecked,
|
||||
}
|
||||
|
||||
/// <summary>Manages the proxy pool: refresh from the feed, probe, and edit the custom list.</summary>
|
||||
public partial class ProxiesViewModel : PageViewModel, IDisposable
|
||||
{
|
||||
/// <summary>
|
||||
/// Rows rendered at once. A free feed carries a few thousand proxies, and a virtualising list
|
||||
/// still pays to build every row view model, so the view is capped and says so.
|
||||
/// </summary>
|
||||
private const int MaxDisplayedRows = 500;
|
||||
|
||||
private readonly IProxyPool _pool;
|
||||
private readonly IMutableProxySource _customSource;
|
||||
private readonly ILogger<ProxiesViewModel> _logger;
|
||||
private readonly ISequencer _mainThread;
|
||||
private readonly Dictionary<string, ProxyRowViewModel> _rows = new(StringComparer.Ordinal);
|
||||
private readonly Signal<RxVoid> _poolChanged = new();
|
||||
|
||||
/// <summary>Free-text filter over address and country.</summary>
|
||||
[Reactive]
|
||||
public partial string SearchText { get; set; }
|
||||
|
||||
/// <summary>Protocol filter; <see cref="ProxyProtocolFilter.All"/> means no filtering.</summary>
|
||||
[Reactive]
|
||||
public partial ProxyProtocolFilter ProtocolFilter { get; set; }
|
||||
|
||||
/// <summary>Health filter.</summary>
|
||||
[Reactive]
|
||||
public partial ProxyHealthFilter HealthFilter { get; set; }
|
||||
|
||||
/// <summary>Text box contents for adding custom proxies.</summary>
|
||||
[Reactive]
|
||||
public partial string NewProxies { get; set; }
|
||||
|
||||
/// <summary>Currently selected row.</summary>
|
||||
[Reactive]
|
||||
public partial ProxyRowViewModel? SelectedProxy { get; set; }
|
||||
|
||||
/// <summary>Outcome of the last action; <see langword="null"/> when idle.</summary>
|
||||
[Reactive]
|
||||
public partial string? StatusMessage { get; set; }
|
||||
|
||||
/// <summary>Progress of the running sweep, 0..1.</summary>
|
||||
[Reactive]
|
||||
public partial double SweepProgress { get; set; }
|
||||
|
||||
/// <summary>Whether a sweep is running.</summary>
|
||||
[Reactive]
|
||||
public partial bool IsSweeping { get; set; }
|
||||
|
||||
/// <summary>How many rows the filter matched before the display cap.</summary>
|
||||
[Reactive]
|
||||
public partial int MatchedCount { get; set; }
|
||||
|
||||
/// <summary>Total entries in the pool.</summary>
|
||||
[Reactive]
|
||||
public partial int TotalCount { get; set; }
|
||||
|
||||
/// <summary>How many entries are currently alive.</summary>
|
||||
[Reactive]
|
||||
public partial int AliveCount { get; set; }
|
||||
|
||||
/// <summary>Creates the page.</summary>
|
||||
/// <param name="pool">The pool being managed.</param>
|
||||
/// <param name="customSource">The user's editable list.</param>
|
||||
/// <param name="logger">Diagnostics.</param>
|
||||
/// <param name="mainThread">Scheduler for UI-affine updates; tests pass an immediate one.</param>
|
||||
public ProxiesViewModel(
|
||||
IProxyPool pool,
|
||||
IMutableProxySource customSource,
|
||||
ILogger<ProxiesViewModel> logger,
|
||||
ISequencer? mainThread = null
|
||||
)
|
||||
{
|
||||
_pool = pool ?? throw new ArgumentNullException(nameof(pool));
|
||||
_customSource = customSource ?? throw new ArgumentNullException(nameof(customSource));
|
||||
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
|
||||
_mainThread = mainThread ?? RxSchedulers.MainThreadScheduler;
|
||||
|
||||
SearchText = string.Empty;
|
||||
NewProxies = string.Empty;
|
||||
ProtocolFilter = ProxyProtocolFilter.All;
|
||||
HealthFilter = ProxyHealthFilter.All;
|
||||
|
||||
var idle = this.WhenAnyValue(x => x.IsSweeping).Select(static sweeping => !sweeping);
|
||||
|
||||
RefreshCommand = ReactiveCommand.CreateFromTask(RefreshAsync, idle, _mainThread);
|
||||
SweepCommand = ReactiveCommand.CreateFromTask(SweepAsync, idle, _mainThread);
|
||||
AddCustomCommand = ReactiveCommand.CreateFromTask(AddCustomAsync, idle, _mainThread);
|
||||
ClearCustomCommand = ReactiveCommand.CreateFromTask(ClearCustomAsync, idle, _mainThread);
|
||||
|
||||
RemoveSelectedCommand = ReactiveCommand.CreateFromTask(
|
||||
RemoveSelectedAsync,
|
||||
this.WhenAnyValue(x => x.SelectedProxy).Select(static row => row is { IsCustom: true }),
|
||||
_mainThread
|
||||
);
|
||||
|
||||
// The pool raises Changed on every single lease outcome; rebuilding the view that often
|
||||
// would make a running parse unusable, so coalesce into one refresh per burst.
|
||||
_poolChanged
|
||||
.Throttle(TimeSpan.FromMilliseconds(250), _mainThread)
|
||||
.ObserveOn(_mainThread)
|
||||
.Subscribe(_ => Rebuild());
|
||||
|
||||
_pool.Changed += OnPoolChanged;
|
||||
|
||||
this.WhenAnyValue(x => x.SearchText, x => x.ProtocolFilter, x => x.HealthFilter, (_, _, _) => RxVoid.Default)
|
||||
.Throttle(TimeSpan.FromMilliseconds(150), _mainThread)
|
||||
.ObserveOn(_mainThread)
|
||||
.Subscribe(_ => Rebuild());
|
||||
|
||||
foreach (var command in new[] { RefreshCommand, SweepCommand, AddCustomCommand, ClearCustomCommand })
|
||||
{
|
||||
command.ThrownExceptions.Subscribe(OnCommandFailed);
|
||||
}
|
||||
|
||||
RemoveSelectedCommand.ThrownExceptions.Subscribe(OnCommandFailed);
|
||||
|
||||
Rebuild();
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
public override string Title => "Proxies";
|
||||
|
||||
/// <inheritdoc />
|
||||
public override string IconKey => "IconShield";
|
||||
|
||||
/// <summary>Rows currently shown, already filtered and capped.</summary>
|
||||
public ObservableCollection<ProxyRowViewModel> Proxies { get; } = [];
|
||||
|
||||
/// <summary>Protocol filter options.</summary>
|
||||
public IReadOnlyList<ProxyProtocolFilter> ProtocolFilters { get; } =
|
||||
[
|
||||
ProxyProtocolFilter.All,
|
||||
ProxyProtocolFilter.Http,
|
||||
ProxyProtocolFilter.Https,
|
||||
ProxyProtocolFilter.Socks4,
|
||||
ProxyProtocolFilter.Socks5,
|
||||
];
|
||||
|
||||
/// <summary>Health filter options.</summary>
|
||||
public IReadOnlyList<ProxyHealthFilter> HealthFilters { get; } =
|
||||
[ProxyHealthFilter.All, ProxyHealthFilter.Alive, ProxyHealthFilter.Dead, ProxyHealthFilter.Unchecked];
|
||||
|
||||
/// <summary>Reloads every source into the pool.</summary>
|
||||
public ReactiveCommand<RxVoid, RxVoid> RefreshCommand { get; }
|
||||
|
||||
/// <summary>Probes the whole pool.</summary>
|
||||
public ReactiveCommand<RxVoid, RxVoid> SweepCommand { get; }
|
||||
|
||||
/// <summary>Adds the addresses typed into <see cref="NewProxies"/>.</summary>
|
||||
public ReactiveCommand<RxVoid, RxVoid> AddCustomCommand { get; }
|
||||
|
||||
/// <summary>Removes the selected custom proxy.</summary>
|
||||
public ReactiveCommand<RxVoid, RxVoid> RemoveSelectedCommand { get; }
|
||||
|
||||
/// <summary>Empties the custom list.</summary>
|
||||
public ReactiveCommand<RxVoid, RxVoid> ClearCustomCommand { get; }
|
||||
|
||||
private async Task RefreshAsync(CancellationToken cancellationToken)
|
||||
{
|
||||
var count = await _pool.RefreshAsync(cancellationToken).ConfigureAwait(false);
|
||||
OnUi(() => StatusMessage = $"Pool holds {Format(count)} prox{(count == 1 ? "y" : "ies")}.");
|
||||
}
|
||||
|
||||
private async Task SweepAsync(CancellationToken cancellationToken)
|
||||
{
|
||||
OnUi(() =>
|
||||
{
|
||||
IsSweeping = true;
|
||||
SweepProgress = 0d;
|
||||
StatusMessage = null;
|
||||
});
|
||||
|
||||
try
|
||||
{
|
||||
var progress = new Progress<ProxySweepProgress>(value => OnUi(() => SweepProgress = value.Fraction));
|
||||
var alive = await _pool.SweepAsync(progress, cancellationToken).ConfigureAwait(false);
|
||||
var total = _pool.Entries.Count;
|
||||
|
||||
OnUi(() => StatusMessage = $"{Format(alive)} of {Format(total)} answered.");
|
||||
}
|
||||
catch (OperationCanceledException)
|
||||
{
|
||||
OnUi(() => StatusMessage = "Check cancelled.");
|
||||
}
|
||||
finally
|
||||
{
|
||||
OnUi(() =>
|
||||
{
|
||||
IsSweeping = false;
|
||||
SweepProgress = 0d;
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
private async Task AddCustomAsync(CancellationToken cancellationToken)
|
||||
{
|
||||
var (parsed, rejected) = CustomProxySource.ParseList(NewProxies);
|
||||
|
||||
if (parsed.Count == 0 && rejected.Count == 0)
|
||||
{
|
||||
OnUi(() => StatusMessage = "Nothing to add.");
|
||||
return;
|
||||
}
|
||||
|
||||
var added = await _customSource.AddAsync(parsed, cancellationToken).ConfigureAwait(false);
|
||||
await _pool.RefreshAsync(cancellationToken).ConfigureAwait(false);
|
||||
|
||||
var message = $"Added {Format(added)} of {Format(parsed.Count)}.";
|
||||
if (rejected.Count > 0)
|
||||
{
|
||||
// Naming the first few beats "3 lines were invalid" when a paste is hundreds long.
|
||||
message += $" Could not parse: {string.Join(", ", rejected.Take(3))}";
|
||||
if (rejected.Count > 3)
|
||||
{
|
||||
message += $" and {Format(rejected.Count - 3)} more";
|
||||
}
|
||||
}
|
||||
|
||||
OnUi(() =>
|
||||
{
|
||||
StatusMessage = message;
|
||||
if (added > 0)
|
||||
{
|
||||
NewProxies = string.Empty;
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
private async Task RemoveSelectedAsync(CancellationToken cancellationToken)
|
||||
{
|
||||
if (SelectedProxy is not { IsCustom: true } row)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
await _customSource.RemoveAsync(row.Entry.Endpoint, cancellationToken).ConfigureAwait(false);
|
||||
await _pool.RefreshAsync(cancellationToken).ConfigureAwait(false);
|
||||
|
||||
OnUi(() => StatusMessage = $"Removed {row.Address}.");
|
||||
}
|
||||
|
||||
private async Task ClearCustomAsync(CancellationToken cancellationToken)
|
||||
{
|
||||
await _customSource.ClearAsync(cancellationToken).ConfigureAwait(false);
|
||||
await _pool.RefreshAsync(cancellationToken).ConfigureAwait(false);
|
||||
|
||||
OnUi(() => StatusMessage = "Custom list cleared.");
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
/// <remarks>
|
||||
/// Detaching from the pool matters: the pool is a singleton and would otherwise keep this
|
||||
/// page — and every row in it — alive for the process lifetime.
|
||||
/// </remarks>
|
||||
public void Dispose()
|
||||
{
|
||||
_pool.Changed -= OnPoolChanged;
|
||||
_poolChanged.Dispose();
|
||||
GC.SuppressFinalize(this);
|
||||
}
|
||||
|
||||
private void OnPoolChanged(object? sender, EventArgs e) => _poolChanged.OnNext(RxVoid.Default);
|
||||
|
||||
private void Rebuild()
|
||||
{
|
||||
var entries = _pool.Entries;
|
||||
|
||||
// Reconcile rows rather than recreating them: a row rebuilt on every pool event would
|
||||
// drop the user's selection mid-sweep.
|
||||
var seen = new HashSet<string>(StringComparer.Ordinal);
|
||||
foreach (var entry in entries)
|
||||
{
|
||||
var key = entry.Endpoint.Key;
|
||||
seen.Add(key);
|
||||
|
||||
if (_rows.TryGetValue(key, out var existing))
|
||||
{
|
||||
existing.Refresh();
|
||||
}
|
||||
else
|
||||
{
|
||||
_rows[key] = new ProxyRowViewModel(entry);
|
||||
}
|
||||
}
|
||||
|
||||
foreach (var stale in _rows.Keys.Where(key => !seen.Contains(key)).ToArray())
|
||||
{
|
||||
_rows.Remove(stale);
|
||||
}
|
||||
|
||||
var matched = _rows.Values.Where(PassesFilters).OrderBy(row => row.Address, StringComparer.Ordinal).ToArray();
|
||||
|
||||
Proxies.Clear();
|
||||
foreach (var row in matched.Take(MaxDisplayedRows))
|
||||
{
|
||||
Proxies.Add(row);
|
||||
}
|
||||
|
||||
MatchedCount = matched.Length;
|
||||
TotalCount = entries.Count;
|
||||
AliveCount = entries.Count(entry => entry.Health == ProxyHealthState.Alive);
|
||||
}
|
||||
|
||||
private bool PassesFilters(ProxyRowViewModel row)
|
||||
{
|
||||
if (
|
||||
ProtocolFilter != ProxyProtocolFilter.All
|
||||
&& !ProtocolFilter.HasFlag(ProxyOptions.ToFlag(row.Entry.Endpoint.Protocol))
|
||||
)
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
var healthOk = HealthFilter switch
|
||||
{
|
||||
ProxyHealthFilter.Alive => row.Entry.Health == ProxyHealthState.Alive,
|
||||
ProxyHealthFilter.Dead => row.Entry.Health == ProxyHealthState.Dead,
|
||||
ProxyHealthFilter.Unchecked => row.Entry.Health == ProxyHealthState.Unknown,
|
||||
_ => true,
|
||||
};
|
||||
|
||||
return healthOk && row.Matches(SearchText);
|
||||
}
|
||||
|
||||
private void OnCommandFailed(Exception exception)
|
||||
{
|
||||
_logger.LogError(exception, "Proxy action failed");
|
||||
OnUi(() => StatusMessage = $"Failed: {exception.Message}");
|
||||
}
|
||||
|
||||
private static string Format(int value) => value.ToString("N0", CultureInfo.CurrentCulture);
|
||||
|
||||
private void OnUi(Action action) => _mainThread.Schedule(action);
|
||||
}
|
||||
Reference in New Issue
Block a user