Add a proxy pool with rotation, liveness checks and a management page
The parser will need to move between proxies, so this adds the module it will sit on: pluggable sources, a pool that hands proxies out and learns from the outcome, three rotation strategies, and a page to drive it. Sources are IProxySource implementations. The public proxifly/free-proxy-list feed is fetched as the combined all/data.json through jsDelivr and filtered locally — one conditional request beats four per-protocol ones that can disagree mid-publish — and cached for the five minutes upstream takes to regenerate. A feed that is down keeps serving its last payload rather than emptying the pool. The user's own list lives in proxies.custom.json beside the settings, takes a pasted blob, and names the lines it could not parse instead of quietly dropping them. Both knobs the pool exposes are settings, as asked: rotation is Sticky (default, the only one that keeps site sessions coherent), RoundRobin or WeightedRandom; liveness is either a parallel sweep of the whole pool or a probe at hand-out time. Free lists are a few percent alive, so skipping verification entirely means mostly waiting on timeouts. Two invariants worth keeping, both of which cost a bug to find: Availability is decided by the quarantine, never by Health. Excluding everything that has ever failed made the quarantine window dead code and discarded proxies permanently on their first hiccup, which is exactly wrong for addresses that flap constantly. Health only orders the candidates now. A probe verdict does not touch the success/failure counters. Those are about real requests, and letting a sweep over a few thousand proxies rewrite them would drown the evidence weighted selection reads. SOCKS needs no extra package — .NET resolves socks4/socks4a/socks5 in WebProxy — but a proxifly record with "protocol": "https" is still an HTTP proxy reached over http:// with CONNECT, not an https:// scheme. 115 new tests. Also fixes a pre-existing flake: a command gated on another command's IsExecuting cannot be driven straight after its Execute() completes, because IsExecuting is published on the output scheduler. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 5
parent
aeafe0af36
commit
9bf2ea5532
@@ -1,3 +1,4 @@
|
||||
using AvParser.Core.Proxies;
|
||||
using AvParser.Core.Settings;
|
||||
using AvParser.Infrastructure.Logging;
|
||||
using AvParser.Infrastructure.Storage;
|
||||
@@ -17,6 +18,7 @@ public partial class SettingsViewModel : PageViewModel
|
||||
private readonly ISettingsService _settings;
|
||||
private readonly IThemeService _theme;
|
||||
private readonly LoggingLevelSwitch _levelSwitch;
|
||||
private readonly IProxyPool _proxyPool;
|
||||
|
||||
/// <summary>Selected theme. Applied immediately, not on an OK button.</summary>
|
||||
[Reactive]
|
||||
@@ -26,18 +28,44 @@ public partial class SettingsViewModel : PageViewModel
|
||||
[Reactive]
|
||||
public partial string SelectedLogLevel { get; set; }
|
||||
|
||||
/// <summary>How the pool picks the next proxy.</summary>
|
||||
[Reactive]
|
||||
public partial ProxyRotation SelectedRotation { get; set; }
|
||||
|
||||
/// <summary>When proxy liveness is verified.</summary>
|
||||
[Reactive]
|
||||
public partial ProxyHealthCheck SelectedHealthCheck { get; set; }
|
||||
|
||||
/// <summary>Whether the remote proxy feed is consulted.</summary>
|
||||
[Reactive]
|
||||
public partial bool UseProxyFeed { get; set; }
|
||||
|
||||
/// <summary>URL fetched to decide whether a proxy works.</summary>
|
||||
[Reactive]
|
||||
public partial string ProxyProbeUrl { get; set; }
|
||||
|
||||
/// <summary>Per-proxy probe timeout, in seconds.</summary>
|
||||
[Reactive]
|
||||
public partial int ProxyProbeTimeoutSeconds { get; set; }
|
||||
|
||||
/// <summary>How many probes run at once during a sweep.</summary>
|
||||
[Reactive]
|
||||
public partial int ProxyProbeConcurrency { get; set; }
|
||||
|
||||
/// <summary>Creates the page.</summary>
|
||||
public SettingsViewModel(
|
||||
ISettingsService settings,
|
||||
IThemeService theme,
|
||||
IAppPaths paths,
|
||||
LoggingLevelSwitch levelSwitch,
|
||||
IProxyPool proxyPool,
|
||||
ISequencer? mainThread = null
|
||||
)
|
||||
{
|
||||
_settings = settings ?? throw new ArgumentNullException(nameof(settings));
|
||||
_theme = theme ?? throw new ArgumentNullException(nameof(theme));
|
||||
_levelSwitch = levelSwitch ?? throw new ArgumentNullException(nameof(levelSwitch));
|
||||
_proxyPool = proxyPool ?? throw new ArgumentNullException(nameof(proxyPool));
|
||||
ArgumentNullException.ThrowIfNull(paths);
|
||||
|
||||
var scheduler = mainThread ?? RxSchedulers.MainThreadScheduler;
|
||||
@@ -48,6 +76,14 @@ public partial class SettingsViewModel : PageViewModel
|
||||
SelectedTheme = theme.Current;
|
||||
SelectedLogLevel = settings.Current.MinimumLogLevel;
|
||||
|
||||
var current = settings.Current;
|
||||
SelectedRotation = current.ProxyRotation;
|
||||
SelectedHealthCheck = current.ProxyHealthCheck;
|
||||
UseProxyFeed = current.ProxyUseFeed;
|
||||
ProxyProbeUrl = current.ProxyProbeUrl;
|
||||
ProxyProbeTimeoutSeconds = current.ProxyProbeTimeoutSeconds;
|
||||
ProxyProbeConcurrency = current.ProxyProbeConcurrency;
|
||||
|
||||
this.WhenAnyValue(x => x.SelectedTheme).ObserveOn(scheduler).Subscribe(_theme.Apply);
|
||||
|
||||
this.WhenAnyValue(x => x.SelectedLogLevel)
|
||||
@@ -57,6 +93,27 @@ public partial class SettingsViewModel : PageViewModel
|
||||
|
||||
// Keep the radio group honest when the theme is flipped from the title-bar button.
|
||||
theme.Changes.ObserveOn(scheduler).Subscribe(value => SelectedTheme = value);
|
||||
|
||||
// Every proxy knob funnels through one handler: they all end up rebuilding the same
|
||||
// ProxyOptions, and applying them one at a time would reconfigure the pool six times.
|
||||
this.WhenAnyValue(
|
||||
x => x.SelectedRotation,
|
||||
x => x.SelectedHealthCheck,
|
||||
x => x.UseProxyFeed,
|
||||
x => x.ProxyProbeUrl,
|
||||
x => x.ProxyProbeTimeoutSeconds,
|
||||
x => x.ProxyProbeConcurrency,
|
||||
(_, _, _, _, _, _) => RxVoid.Default
|
||||
)
|
||||
.Throttle(TimeSpan.FromMilliseconds(200), scheduler)
|
||||
.ObserveOn(scheduler)
|
||||
.Subscribe(_ => ApplyProxySettings());
|
||||
|
||||
this.WhenAnyValue(x => x.SelectedRotation)
|
||||
.Subscribe(_ => this.RaisePropertyChanged(nameof(RotationDescription)));
|
||||
|
||||
this.WhenAnyValue(x => x.SelectedHealthCheck)
|
||||
.Subscribe(_ => this.RaisePropertyChanged(nameof(HealthCheckDescription)));
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
@@ -71,6 +128,29 @@ public partial class SettingsViewModel : PageViewModel
|
||||
/// <summary>Serilog level names, most to least verbose.</summary>
|
||||
public IReadOnlyList<string> LogLevels => AppLogging.AvailableLevels;
|
||||
|
||||
/// <summary>Rotation strategies offered by the picker.</summary>
|
||||
public IReadOnlyList<ProxyRotation> Rotations { get; } =
|
||||
[ProxyRotation.Sticky, ProxyRotation.RoundRobin, ProxyRotation.WeightedRandom];
|
||||
|
||||
/// <summary>Liveness policies offered by the picker.</summary>
|
||||
public IReadOnlyList<ProxyHealthCheck> HealthChecks { get; } = [ProxyHealthCheck.Pool, ProxyHealthCheck.Lazy];
|
||||
|
||||
/// <summary>Explains the selected rotation in one line.</summary>
|
||||
public string RotationDescription =>
|
||||
SelectedRotation switch
|
||||
{
|
||||
ProxyRotation.Sticky => "One proxy per session, replaced only when it fails. Keeps site sessions intact.",
|
||||
ProxyRotation.RoundRobin =>
|
||||
"A different proxy on every request. Spreads rate limits, but breaks session cookies.",
|
||||
_ => "Random, weighted by feed score and how often the proxy has actually worked here.",
|
||||
};
|
||||
|
||||
/// <summary>Explains the selected health-check policy in one line.</summary>
|
||||
public string HealthCheckDescription =>
|
||||
SelectedHealthCheck == ProxyHealthCheck.Pool
|
||||
? "Probe the whole pool up front, in parallel. One sweep, then no per-request delay."
|
||||
: "Probe each proxy as it is handed out. No sweep, but every acquisition pays a round trip.";
|
||||
|
||||
/// <summary>Full path of the settings file.</summary>
|
||||
public string SettingsFile { get; }
|
||||
|
||||
@@ -88,4 +168,28 @@ public partial class SettingsViewModel : PageViewModel
|
||||
_levelSwitch.MinimumLevel = AppLogging.ParseLevel(level);
|
||||
_settings.Update(current => current with { MinimumLogLevel = level });
|
||||
}
|
||||
|
||||
private void ApplyProxySettings()
|
||||
{
|
||||
AppSettings? applied = null;
|
||||
|
||||
_settings.Update(current =>
|
||||
{
|
||||
applied = current with
|
||||
{
|
||||
ProxyRotation = SelectedRotation,
|
||||
ProxyHealthCheck = SelectedHealthCheck,
|
||||
ProxyUseFeed = UseProxyFeed,
|
||||
ProxyProbeUrl = ProxyProbeUrl,
|
||||
ProxyProbeTimeoutSeconds = ProxyProbeTimeoutSeconds,
|
||||
ProxyProbeConcurrency = ProxyProbeConcurrency,
|
||||
};
|
||||
|
||||
return applied;
|
||||
});
|
||||
|
||||
// Update() short-circuits a no-op change, so fall back to the stored value: the pool must
|
||||
// still be configured on the very first pass, when nothing has changed yet.
|
||||
_proxyPool.Configure((applied ?? _settings.Current).ToProxyOptions());
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user