diff --git a/AvParser.slnx b/AvParser.slnx index a7b276c..eb758da 100644 --- a/AvParser.slnx +++ b/AvParser.slnx @@ -23,6 +23,7 @@ + diff --git a/CLAUDE.md b/CLAUDE.md index b5e095b..60ac4ae 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -84,6 +84,29 @@ dotnet csharpier check . выбор конструктора контейнером зависит от порядка регистраций. - `[Reactive]` из `ReactiveUI.SourceGenerators` на partial-свойствах; класс — `partial`. +## Добавить источник прокси + +1. Реализовать `IProxySource` (или `IMutableProxySource`, если список редактируемый). +2. Зарегистрировать как `IProxySource` в `AddAvParserProxies()`. Порядок регистрации = порядок + слияния; свой список идёт последним, чтобы пользовательский адрес перебивал фидовый. + +`ProxyPool` при обновлении **переиспользует существующие `ProxyEntry`** по `Endpoint.Key` — иначе +перезагрузка списка стирала бы всю накопленную статистику, а публичные фиды переиздаются каждые +несколько минут. + +## Инварианты прокси-пула + +- **Доступность определяется карантином, а не `Health`.** `Health` — это «что видели в последний + раз». Если исключать всё, что когда-либо падало, окно карантина становится бессмысленным, а + прокси теряется навсегда после первой же осечки. Это уже был баг, его ловит + `A_failing_proxy_is_quarantined_and_comes_back_later`. +- **Проба не трогает `SuccessCount`/`FailureCount`.** Эти счётчики про реальные запросы; свип по + паре тысяч прокси перезаписал бы всё, на чём держится взвешенный выбор. Провалившаяся проба + выставляет карантин через `RecordProbe(..., quarantineOnFailure:)`. +- **Лиза без вердикта нейтральна.** Отменённая операция — не вина прокси; считать это отказом + значит карантинить здоровые прокси на каждый Cancel. +- **`Select` и `Next` — зарезервированные слова для CA1716.** Метод стратегии называется `Pick`. + ## Грабли, уже оплаченные - **Селектор типа в Avalonia матчит точный тип.** `UserControl.shell` не матчит `ShellView` @@ -110,6 +133,11 @@ dotnet csharpier check . резолвиться, даже если эту конфигурацию никто не собирает. Так тут проехал мёртвый `Avalonia.Diagnostics` (его нет под Avalonia 12): `dotnet build -c Release` работал, а голый `dotnet restore` падал. +- **`Execute()` завершился ≠ `IsExecuting` уже false.** Второе публикуется на выходном + планировщике. Тест, который сразу после `await` дёргает команду, закрытую по чужому + `IsExecuting`, будет мигать под нагрузкой — ждите `CanExecute`, а не предполагайте. +- **Проект VM-тестов не параллелится.** `ReactiveUiBootstrap` ставит глобальные планировщики + ReactiveUI, то есть тесты делят изменяемое состояние независимо от их желания. - **Инспектора в Avalonia 12 нет из коробки.** `Avalonia.Diagnostics` закончился на 11.3.x; DevTools живут отдельно (`AvaloniaUI.DiagnosticsSupport` + `.WithDeveloperTools()`), со своей установкой. Зависимость намеренно не добавлена. diff --git a/Directory.Packages.props b/Directory.Packages.props index 743dc8e..4f63d97 100644 --- a/Directory.Packages.props +++ b/Directory.Packages.props @@ -41,6 +41,7 @@ Include="Microsoft.Extensions.DependencyInjection.Abstractions" Version="$(MicrosoftExtensionsVersion)" /> + diff --git a/README.md b/README.md index 9f86207..a89a491 100644 --- a/README.md +++ b/README.md @@ -61,9 +61,10 @@ src/ ResponsiveLayout, дизайн-токены, навигация AvParser.Desktop WinExe-хост: Program.cs, App.axaml, composition root tests/ - AvParser.Core.Tests парсеры, реестр, отмена, прогресс - AvParser.UI.Tests ViewModel'и без Avalonia - AvParser.UI.HeadlessTests реальное дерево контролов через [AvaloniaFact] + AvParser.Core.Tests парсеры, реестр, отмена, прогресс, пул прокси и стратегии + AvParser.Infrastructure.Tests разбор фида прокси, локальный список, маппинг на WebProxy + AvParser.UI.Tests ViewModel'и без Avalonia + AvParser.UI.HeadlessTests реальное дерево контролов через [AvaloniaFact] ``` Ссылки идут строго в одну сторону: `Core ← Infrastructure ← UI ← Desktop`. @@ -102,6 +103,55 @@ Avalonia матчит **точный** тип, а `ShellView` наследует --- +## Прокси + +Пул прокси с ротацией — `AvParser.Core/Proxies`, источники и сетевая часть — +`AvParser.Infrastructure/Proxies`, управление — страница **Proxies**. + +Источники: + +- **[proxifly/free-proxy-list](https://github.com/proxifly/free-proxy-list)** — публичный список, + обновляется каждые 5 минут. Тянем сводный `all/data.json` через jsDelivr и фильтруем локально: + один условный запрос за весь список надёжнее четырёх по протоколам, которые могут разъехаться + между собой в момент публикации. Ответ кэшируется на 5 минут, недоступность фида не роняет + приложение — остаётся прошлый список. +- **Свой список** — `proxies.custom.json` рядом с настройками. Вставляется пачкой, по одной на + строку; поддерживаются `scheme://host:port`, голый `host:port` и `user:pass@`. Непонятые строки + не проглатываются молча, а называются в статусе. + +Ротация выбирается в настройках: + +| Стратегия | Поведение | Когда | +|---|---|---| +| Sticky | одна прокси, смена только по отказу | по умолчанию: не рвёт сессии и cookie | +| RoundRobin | новая на каждый запрос | размазывает рейт-лимиты, но ломает сессии | +| WeightedRandom | случайно, с весом по score и доле успехов | при сильном разбросе качества | + +Проверка живости — тоже настройка, два режима: **Pool** прогоняет весь список параллельно один +раз, **Lazy** проверяет прокси в момент выдачи и перескакивает на следующую. У бесплатных списков +рабочих обычно единицы процентов, поэтому без проверки парсер будет в основном ждать таймауты. + +Упавшая прокси уходит в карантин с экспоненциальным окном (30 с → 15 мин), но **не** удаляется +навсегда: бесплатные прокси постоянно мигают, и жёсткий бан терял бы их безвозвратно. + +Использование из кода: + +```csharp +var (http, lease) = await clientFactory.CreateFromPoolAsync(); +using (http) +using (lease) +{ + try { var response = await http.GetAsync(url); lease?.ReportSuccess(); } + catch { lease?.ReportFailure("request failed"); throw; } +} +``` + +Отчёт об исходе — не формальность: без него пул ничего не узнаёт о том, какие прокси работают. +Освобождение лизы без вердикта нейтрально — отменённая операция не вина прокси. + +SOCKS работает штатно: .NET понимает схемы `socks4/socks4a/socks5` в `WebProxy`. Учтите, что +proxifly-запись с `"protocol": "https"` — это всё равно HTTP-прокси с CONNECT, а не схема `https://`. + ## Дизайн-токены Все цвета, отступы, радиусы и типографика — в `Styles/Tokens.axaml`, с отдельными словарями diff --git a/src/AvParser.Core/Proxies/IProxyPool.cs b/src/AvParser.Core/Proxies/IProxyPool.cs new file mode 100644 index 0000000..2a1bdaf --- /dev/null +++ b/src/AvParser.Core/Proxies/IProxyPool.cs @@ -0,0 +1,89 @@ +namespace AvParser.Core.Proxies; + +/// The pool of known proxies and the thing that hands them out. +public interface IProxyPool +{ + /// Snapshot of every known proxy, feed and custom alike. + IReadOnlyList Entries { get; } + + /// Options currently in force. + ProxyOptions Options { get; } + + /// Raised after the set of entries or their health changes. + /// + /// A plain event rather than IObservable so the domain keeps no reactive dependency; + /// the UI layer bridges it to an observable where that is convenient. + /// + event EventHandler? Changed; + + /// Applies new options. Resets selection state when the rotation strategy changes. + void Configure(ProxyOptions options); + + /// Reloads from every source, preserving health statistics for addresses that survive. + /// Number of entries in the pool afterwards. + Task RefreshAsync(CancellationToken cancellationToken = default); + + /// + /// Takes a proxy out of the pool for one unit of work, or when nothing + /// usable is left. + /// + /// Report the outcome on the lease, otherwise the pool never learns anything. + Task AcquireAsync(CancellationToken cancellationToken = default); + + /// Probes every entry in parallel and updates their health. + /// How many answered. + Task SweepAsync(IProgress? progress = null, CancellationToken cancellationToken = default); +} + +/// +/// A proxy checked out of the pool. +/// +/// +/// Disposing without a verdict is deliberately neutral: an operation that was cancelled says +/// nothing about the proxy, and counting that as a failure would quarantine healthy entries +/// every time the user hits Cancel. +/// +public sealed class ProxyLease : IDisposable +{ + private readonly ProxyPool _pool; + private bool _reported; + + internal ProxyLease(ProxyPool pool, ProxyEntry entry) + { + _pool = pool; + Entry = entry; + } + + /// The pool entry backing this lease. + public ProxyEntry Entry { get; } + + /// The address to send traffic through. + public ProxyEndpoint Endpoint => Entry.Endpoint; + + /// Records that the work succeeded. + public void ReportSuccess(TimeSpan? latency = null) + { + if (_reported) + { + return; + } + + _reported = true; + _pool.ReportOutcome(Entry, success: true, latency, error: null); + } + + /// Records that the work failed, which may quarantine the proxy. + public void ReportFailure(string? error = null) + { + if (_reported) + { + return; + } + + _reported = true; + _pool.ReportOutcome(Entry, success: false, latency: null, error); + } + + /// + public void Dispose() => _reported = true; +} diff --git a/src/AvParser.Core/Proxies/IProxySource.cs b/src/AvParser.Core/Proxies/IProxySource.cs new file mode 100644 index 0000000..879b766 --- /dev/null +++ b/src/AvParser.Core/Proxies/IProxySource.cs @@ -0,0 +1,65 @@ +namespace AvParser.Core.Proxies; + +/// Supplies proxy addresses. One per list the app knows about. +public interface IProxySource +{ + /// Stable identifier used in settings and logs. + string Id { get; } + + /// Human-readable name for the UI. + string DisplayName { get; } + + /// Whether entries from this source are feed-provided or user-entered. + ProxySourceKind Kind { get; } + + /// Fetches the current list. Implementations may cache. + Task> GetProxiesAsync(CancellationToken cancellationToken = default); +} + +/// A source the user can edit. +public interface IMutableProxySource : IProxySource +{ + /// Adds addresses, ignoring duplicates. Returns how many were actually new. + Task AddAsync(IEnumerable endpoints, CancellationToken cancellationToken = default); + + /// Removes an address. Returns whether it was present. + Task RemoveAsync(ProxyEndpoint endpoint, CancellationToken cancellationToken = default); + + /// Removes every address. + Task ClearAsync(CancellationToken cancellationToken = default); +} + +/// Outcome of a single liveness probe. +/// Whether the proxy answered acceptably. +/// Round-trip time when alive. +/// Short failure reason when not alive. +public readonly record struct ProxyProbeResult(bool Alive, TimeSpan? Latency, string? Error) +{ + /// A successful probe. + public static ProxyProbeResult Success(TimeSpan latency) => new(true, latency, null); + + /// A failed probe. + public static ProxyProbeResult Failure(string error) => new(false, null, error); +} + +/// Checks whether a proxy actually works. +public interface IProxyProbe +{ + /// Sends one request through and reports the outcome. + /// Must not throw for an unreachable proxy — that is a , not an error. + Task ProbeAsync( + ProxyEndpoint endpoint, + ProxyOptions options, + CancellationToken cancellationToken = default + ); +} + +/// Progress of a pool-wide probe sweep. +/// Proxies probed so far. +/// Proxies in the sweep. +/// How many answered. +public readonly record struct ProxySweepProgress(int Checked, int Total, int Alive) +{ + /// Completion in the range 0..1. + public double Fraction => Total <= 0 ? 0d : Math.Clamp((double)Checked / Total, 0d, 1d); +} diff --git a/src/AvParser.Core/Proxies/ProxyEndpoint.cs b/src/AvParser.Core/Proxies/ProxyEndpoint.cs new file mode 100644 index 0000000..9b3c428 --- /dev/null +++ b/src/AvParser.Core/Proxies/ProxyEndpoint.cs @@ -0,0 +1,210 @@ +using System.Diagnostics.CodeAnalysis; +using System.Globalization; + +namespace AvParser.Core.Proxies; + +/// Wire protocol a proxy speaks. +public enum ProxyProtocol +{ + /// Plain HTTP proxy. + Http, + + /// HTTP proxy that also handles CONNECT for TLS. + Https, + + /// SOCKS4. + Socks4, + + /// SOCKS5. + Socks5, +} + +/// How much of the caller the proxy passes through. +public enum ProxyAnonymity +{ + /// Not reported by the source. + Unknown, + + /// Forwards the original address — offers no anonymity at all. + Transparent, + + /// Hides the original address but announces itself as a proxy. + Anonymous, + + /// Neither forwards the address nor announces itself. + Elite, +} + +/// Where an entry came from. +public enum ProxySourceKind +{ + /// Downloaded from a remote list. + Feed, + + /// Entered by the user and stored locally. + Custom, +} + +/// A single proxy address, with whatever metadata its source supplied. +/// Wire protocol. +/// Hostname or IP literal. +/// TCP port. +public sealed record ProxyEndpoint(ProxyProtocol Protocol, string Host, int Port) +{ + /// ISO country code reported by the source, if any. + public string? Country { get; init; } + + /// City reported by the source, if any. + public string? City { get; init; } + + /// Anonymity level reported by the source. + public ProxyAnonymity Anonymity { get; init; } = ProxyAnonymity.Unknown; + + /// Quality score reported by the source; higher is better. 0 when unknown. + public int Score { get; init; } + + /// Username for proxies that need authentication. + public string? Username { get; init; } + + /// Password for proxies that need authentication. + public string? Password { get; init; } + + /// Scheme as System.Net.WebProxy expects it. + public string Scheme => + Protocol switch + { + ProxyProtocol.Socks4 => "socks4", + ProxyProtocol.Socks5 => "socks5", + // .NET has no "https" proxy scheme: an HTTPS-capable proxy is still reached over + // http:// and tunnels TLS with CONNECT. + _ => "http", + }; + + /// Address in scheme://host:port form. + public Uri Uri => new($"{Scheme}://{Host}:{Port.ToString(CultureInfo.InvariantCulture)}"); + + /// Stable identity: two entries for the same address are the same proxy. + public string Key => $"{Protocol}://{Host}:{Port.ToString(CultureInfo.InvariantCulture)}".ToLowerInvariant(); + + /// Whether credentials were supplied. + public bool HasCredentials => !string.IsNullOrEmpty(Username); + + /// + public override string ToString() => + $"{Protocol.ToString().ToLowerInvariant()}://{Host}:{Port.ToString(CultureInfo.InvariantCulture)}"; + + /// + /// Parses [scheme://][user:pass@]host:port. Missing scheme is treated as HTTP. + /// + /// + /// Hand-rolled rather than delegating to : the socks schemes and the + /// bare host:port form that every proxy list uses are not valid absolute URIs. + /// + public static bool TryParse(string? text, [NotNullWhen(true)] out ProxyEndpoint? endpoint) + { + endpoint = null; + + if (string.IsNullOrWhiteSpace(text)) + { + return false; + } + + var value = text.Trim(); + var protocol = ProxyProtocol.Http; + + var schemeEnd = value.IndexOf("://", StringComparison.Ordinal); + if (schemeEnd >= 0) + { + if (!TryParseProtocol(value[..schemeEnd], out protocol)) + { + return false; + } + + value = value[(schemeEnd + 3)..]; + } + + string? username = null; + string? password = null; + + // Rightmost '@' wins: a password may legitimately contain one. + var credentialsEnd = value.LastIndexOf('@'); + if (credentialsEnd >= 0) + { + var credentials = value[..credentialsEnd]; + value = value[(credentialsEnd + 1)..]; + + var separator = credentials.IndexOf(':', StringComparison.Ordinal); + if (separator < 0) + { + username = credentials; + } + else + { + username = credentials[..separator]; + password = credentials[(separator + 1)..]; + } + + if (username.Length == 0) + { + return false; + } + } + + var portStart = value.LastIndexOf(':'); + if (portStart <= 0 || portStart == value.Length - 1) + { + return false; + } + + var host = value[..portStart].Trim(); + var portText = value[(portStart + 1)..].Trim(); + + if ( + host.Length == 0 + || !int.TryParse(portText, NumberStyles.None, CultureInfo.InvariantCulture, out var port) + || port is < 1 or > 65535 + ) + { + return false; + } + + endpoint = new ProxyEndpoint(protocol, host, port) { Username = username, Password = password }; + return true; + } + + /// Parses a protocol name; accepts the spellings the public lists use. + public static bool TryParseProtocol(string? text, out ProxyProtocol protocol) + { + switch (text?.Trim().ToLowerInvariant()) + { + case "http": + protocol = ProxyProtocol.Http; + return true; + case "https": + case "ssl": + protocol = ProxyProtocol.Https; + return true; + case "socks4": + case "socks4a": + protocol = ProxyProtocol.Socks4; + return true; + case "socks5": + case "socks5h": + protocol = ProxyProtocol.Socks5; + return true; + default: + protocol = ProxyProtocol.Http; + return false; + } + } + + /// Parses an anonymity level; unrecognised values become . + public static ProxyAnonymity ParseAnonymity(string? text) => + text?.Trim().ToLowerInvariant() switch + { + "transparent" => ProxyAnonymity.Transparent, + "anonymous" => ProxyAnonymity.Anonymous, + "elite" or "high" => ProxyAnonymity.Elite, + _ => ProxyAnonymity.Unknown, + }; +} diff --git a/src/AvParser.Core/Proxies/ProxyEntry.cs b/src/AvParser.Core/Proxies/ProxyEntry.cs new file mode 100644 index 0000000..d534d20 --- /dev/null +++ b/src/AvParser.Core/Proxies/ProxyEntry.cs @@ -0,0 +1,186 @@ +namespace AvParser.Core.Proxies; + +/// What the last check or use said about a proxy. +public enum ProxyHealthState +{ + /// Never checked and never used. + Unknown, + + /// A probe or a real request succeeded. + Alive, + + /// A probe or a real request failed. + Dead, +} + +/// +/// A proxy plus everything the pool has learned about it. +/// +/// +/// Mutable and guarded by 's lock rather than being a record: the pool +/// updates counters on every request, and reallocating an immutable entry per outcome would +/// churn hard on a list of a few thousand proxies. +/// +public sealed class ProxyEntry +{ + private readonly Lock _gate = new(); + + /// Creates an entry in the state. + public ProxyEntry(ProxyEndpoint endpoint, ProxySourceKind source) + { + Endpoint = endpoint ?? throw new ArgumentNullException(nameof(endpoint)); + Source = source; + } + + /// The address. + public ProxyEndpoint Endpoint { get; } + + /// Whether this came from a feed or from the user. + public ProxySourceKind Source { get; } + + /// Latest known state. + public ProxyHealthState Health { get; private set; } + + /// Round-trip time of the last successful probe or request. + public TimeSpan? Latency { get; private set; } + + /// When the state was last updated. + public DateTimeOffset? LastCheckedUtc { get; private set; } + + /// Successful uses since the entry was created. + public int SuccessCount { get; private set; } + + /// Failed uses since the entry was created. + public int FailureCount { get; private set; } + + /// Failures since the last success. Drives the quarantine backoff. + public int ConsecutiveFailures { get; private set; } + + /// While set and in the future, the entry is skipped by selection. + public DateTimeOffset? QuarantinedUntilUtc { get; private set; } + + /// Reason recorded with the last failure, for the UI. + public string? LastError { get; private set; } + + /// Share of successful uses, 0..1. Returns 0.5 before any evidence exists. + public double SuccessRate + { + get + { + var total = SuccessCount + FailureCount; + return total == 0 ? 0.5d : (double)SuccessCount / total; + } + } + + /// Whether selection may hand this entry out at . + /// + /// Governed by the quarantine alone, deliberately not by . Health is the + /// last thing observed; excluding every entry that has ever failed would make the quarantine + /// window meaningless and would permanently discard proxies on their first hiccup — and free + /// proxies flap constantly. Health still decides ordering, so dead entries sink to the back. + /// + public bool IsAvailable(DateTimeOffset now) => QuarantinedUntilUtc is null || QuarantinedUntilUtc <= now; + + /// Whether the entry is currently serving a quarantine. + public bool IsQuarantined(DateTimeOffset now) => QuarantinedUntilUtc is { } until && until > now; + + /// Records a successful probe or request. + public void RecordSuccess(DateTimeOffset now, TimeSpan? latency = null) + { + lock (_gate) + { + Health = ProxyHealthState.Alive; + SuccessCount++; + ConsecutiveFailures = 0; + QuarantinedUntilUtc = null; + LastError = null; + LastCheckedUtc = now; + + if (latency is not null) + { + Latency = latency; + } + } + } + + /// + /// Records a failure and, once pile up, sidelines + /// the entry for an exponentially growing window capped at . + /// + /// + /// Exponential rather than fixed: a proxy that fails once may just have hit a flaky moment, + /// while one that has failed six times in a row should not be retried every few seconds for + /// the rest of the session. + /// + public void RecordFailure( + DateTimeOffset now, + TimeSpan baseQuarantine, + TimeSpan maxQuarantine, + int failuresBeforeQuarantine, + string? error = null + ) + { + lock (_gate) + { + FailureCount++; + ConsecutiveFailures++; + LastCheckedUtc = now; + LastError = error; + Health = ProxyHealthState.Dead; + + if (ConsecutiveFailures < failuresBeforeQuarantine) + { + return; + } + + var exponent = Math.Min(ConsecutiveFailures - failuresBeforeQuarantine, 16); + var ticks = baseQuarantine.Ticks * Math.Pow(2, exponent); + var window = ticks >= maxQuarantine.Ticks ? maxQuarantine : TimeSpan.FromTicks((long)ticks); + + QuarantinedUntilUtc = now + window; + } + } + + /// Records the outcome of a liveness probe. + /// Current time. + /// Whether the probe succeeded. + /// Round-trip time when alive. + /// Failure reason when not alive. + /// + /// How long to sideline the entry if the probe failed. Leave null to only record the state. + /// + /// + /// Does not touch or : those track real + /// requests, and letting a sweep of a few thousand proxies rewrite them would drown the + /// evidence that weighted selection depends on. + /// + public void RecordProbe( + DateTimeOffset now, + bool alive, + TimeSpan? latency, + string? error, + TimeSpan? quarantineOnFailure = null + ) + { + lock (_gate) + { + Health = alive ? ProxyHealthState.Alive : ProxyHealthState.Dead; + LastCheckedUtc = now; + Latency = alive ? latency : null; + LastError = alive ? null : error; + + if (alive) + { + ConsecutiveFailures = 0; + QuarantinedUntilUtc = null; + } + else if (quarantineOnFailure is { } window) + { + QuarantinedUntilUtc = now + window; + } + } + } + + /// + public override string ToString() => $"{Endpoint} [{Health}]"; +} diff --git a/src/AvParser.Core/Proxies/ProxyOptions.cs b/src/AvParser.Core/Proxies/ProxyOptions.cs new file mode 100644 index 0000000..a892348 --- /dev/null +++ b/src/AvParser.Core/Proxies/ProxyOptions.cs @@ -0,0 +1,129 @@ +namespace AvParser.Core.Proxies; + +/// How the pool picks the next proxy. +public enum ProxyRotation +{ + /// Keep one proxy until it fails. Least disruptive to session cookies. + Sticky, + + /// Advance through the pool on every acquisition. Spreads rate limits. + RoundRobin, + + /// Pick at random, weighted by feed score and observed success rate. + WeightedRandom, +} + +/// When liveness is verified. +public enum ProxyHealthCheck +{ + /// + /// Probe the whole pool up front, in parallel. Costs one sweep, then hands out proxies with + /// no extra latency — the right default when most of a free list is dead. + /// + Pool, + + /// + /// Probe a single proxy at the moment it is handed out, skipping to the next if it fails. + /// No sweep, but every acquisition pays a round trip. + /// + Lazy, +} + +/// Protocols to accept when loading sources. +/// +/// A flags enum rather than a list so that and the persisted settings +/// keep value equality — a record holding a collection compares by reference, which would make +/// every "did anything change?" check say yes. +/// +[Flags] +public enum ProxyProtocolFilter +{ + /// No filter — accept everything. + None = 0, + + /// Plain HTTP proxies. + Http = 1, + + /// HTTPS-capable HTTP proxies. + Https = 2, + + /// SOCKS4. + Socks4 = 4, + + /// SOCKS5. + Socks5 = 8, + + /// Every protocol. + All = Http | Https | Socks4 | Socks5, +} + +/// Tuning for . Mirrors what the Settings page exposes. +public sealed record ProxyOptions +{ + /// Selection strategy. + public ProxyRotation Rotation { get; init; } = ProxyRotation.Sticky; + + /// Liveness policy. + public ProxyHealthCheck HealthCheck { get; init; } = ProxyHealthCheck.Pool; + + /// Protocols to keep when loading sources. + public ProxyProtocolFilter Protocols { get; init; } = ProxyProtocolFilter.All; + + /// ISO country codes to keep. Empty means "all". + public IReadOnlyList Countries { get; init; } = []; + + /// URL fetched to decide whether a proxy works. + /// + /// Defaults to a plain-HTTP 204 endpoint: it is tiny, and requiring TLS would fail every + /// proxy that cannot do CONNECT rather than every proxy that is actually dead. + /// + public Uri ProbeUrl { get; init; } = new("http://www.gstatic.com/generate_204"); + + /// Per-proxy probe timeout. + public TimeSpan ProbeTimeout { get; init; } = TimeSpan.FromSeconds(8); + + /// How many probes run at once during a pool sweep. + public int ProbeConcurrency { get; init; } = 64; + + /// Consecutive failures tolerated before an entry is quarantined. + public int FailuresBeforeQuarantine { get; init; } = 2; + + /// First quarantine window; doubles with each further consecutive failure. + public TimeSpan BaseQuarantine { get; init; } = TimeSpan.FromSeconds(30); + + /// Ceiling for the quarantine window. + public TimeSpan MaxQuarantine { get; init; } = TimeSpan.FromMinutes(15); + + /// + /// Proxies tried per call under + /// before giving up. + /// + public int LazyProbeAttempts { get; init; } = 5; + + /// Whether the remote feed is consulted at all. + public bool UseFeed { get; init; } = true; + + /// Validates the options, throwing on values that would misbehave silently. + /// A numeric option is out of range. + public ProxyOptions Validated() + { + ArgumentOutOfRangeException.ThrowIfLessThan(ProbeConcurrency, 1); + ArgumentOutOfRangeException.ThrowIfLessThan(LazyProbeAttempts, 1); + ArgumentOutOfRangeException.ThrowIfLessThan(FailuresBeforeQuarantine, 1); + ArgumentOutOfRangeException.ThrowIfLessThanOrEqual(ProbeTimeout, TimeSpan.Zero); + ArgumentOutOfRangeException.ThrowIfGreaterThan(BaseQuarantine, MaxQuarantine); + + return this; + } + + /// Maps a protocol onto its filter flag. + public static ProxyProtocolFilter ToFlag(ProxyProtocol protocol) => + protocol switch + { + ProxyProtocol.Http => ProxyProtocolFilter.Http, + ProxyProtocol.Https => ProxyProtocolFilter.Https, + ProxyProtocol.Socks4 => ProxyProtocolFilter.Socks4, + ProxyProtocol.Socks5 => ProxyProtocolFilter.Socks5, + _ => ProxyProtocolFilter.None, + }; +} diff --git a/src/AvParser.Core/Proxies/ProxyPool.cs b/src/AvParser.Core/Proxies/ProxyPool.cs new file mode 100644 index 0000000..7fca817 --- /dev/null +++ b/src/AvParser.Core/Proxies/ProxyPool.cs @@ -0,0 +1,340 @@ +using AvParser.Core.Proxies.Selection; + +namespace AvParser.Core.Proxies; + +/// +public sealed class ProxyPool : IProxyPool +{ + private readonly IReadOnlyList _sources; + private readonly IProxyProbe _probe; + private readonly TimeProvider _time; + private readonly Random? _random; + private readonly Lock _gate = new(); + + private readonly Dictionary _byKey = new(StringComparer.Ordinal); + private List _entries = []; + private IProxySelectionStrategy _strategy; + private ProxyOptions _options; + + /// Creates a pool over the given sources. + /// Every list the app knows about; order decides nothing. + /// Liveness checker. + /// Initial options. + /// Clock; tests inject a fake one to exercise quarantine expiry. + /// Randomness for weighted selection; tests seed it. + public ProxyPool( + IEnumerable sources, + IProxyProbe probe, + ProxyOptions? options = null, + TimeProvider? timeProvider = null, + Random? random = null + ) + { + ArgumentNullException.ThrowIfNull(sources); + + _sources = sources.ToArray(); + _probe = probe ?? throw new ArgumentNullException(nameof(probe)); + _time = timeProvider ?? TimeProvider.System; + _random = random; + _options = (options ?? new ProxyOptions()).Validated(); + _strategy = ProxySelectionStrategyFactory.Create(_options.Rotation, random); + } + + /// + public IReadOnlyList Entries + { + get + { + lock (_gate) + { + return _entries; + } + } + } + + /// + public ProxyOptions Options + { + get + { + lock (_gate) + { + return _options; + } + } + } + + /// + public event EventHandler? Changed; + + /// + public void Configure(ProxyOptions options) + { + ArgumentNullException.ThrowIfNull(options); + + var validated = options.Validated(); + + lock (_gate) + { + var rotationChanged = validated.Rotation != _options.Rotation; + _options = validated; + + if (rotationChanged) + { + _strategy = ProxySelectionStrategyFactory.Create(validated.Rotation, _random); + } + } + + RaiseChanged(); + } + + /// + public async Task RefreshAsync(CancellationToken cancellationToken = default) + { + var options = Options; + var collected = new List<(ProxyEndpoint Endpoint, ProxySourceKind Kind)>(); + + foreach (var source in _sources) + { + if (source.Kind == ProxySourceKind.Feed && !options.UseFeed) + { + continue; + } + + var proxies = await source.GetProxiesAsync(cancellationToken).ConfigureAwait(false); + foreach (var endpoint in proxies) + { + collected.Add((endpoint, source.Kind)); + } + } + + int count; + lock (_gate) + { + var next = new List(collected.Count); + var seen = new HashSet(StringComparer.Ordinal); + + foreach (var (endpoint, kind) in collected) + { + if (!Matches(endpoint, options) || !seen.Add(endpoint.Key)) + { + continue; + } + + // Reuse the existing entry so a refresh does not wipe out everything the pool + // has learned — free lists are republished every few minutes. + if (_byKey.TryGetValue(endpoint.Key, out var existing)) + { + next.Add(existing); + } + else + { + var created = new ProxyEntry(endpoint, kind); + _byKey[endpoint.Key] = created; + next.Add(created); + } + } + + foreach (var key in _byKey.Keys.Where(key => !seen.Contains(key)).ToArray()) + { + _byKey.Remove(key); + } + + _entries = next; + _strategy.Reset(); + count = next.Count; + } + + RaiseChanged(); + return count; + } + + /// + public async Task AcquireAsync(CancellationToken cancellationToken = default) + { + var options = Options; + + if (options.HealthCheck == ProxyHealthCheck.Pool) + { + var entry = SelectAvailable(); + return entry is null ? null : new ProxyLease(this, entry); + } + + // Lazy: verify the pick before handing it over, stepping past dead ones. + for (var attempt = 0; attempt < options.LazyProbeAttempts; attempt++) + { + cancellationToken.ThrowIfCancellationRequested(); + + var entry = SelectAvailable(); + if (entry is null) + { + return null; + } + + if (entry.Health == ProxyHealthState.Alive) + { + return new ProxyLease(this, entry); + } + + var result = await _probe.ProbeAsync(entry.Endpoint, options, cancellationToken).ConfigureAwait(false); + var now = _time.GetUtcNow(); + + if (result.Alive) + { + entry.RecordProbe(now, alive: true, result.Latency, error: null); + RaiseChanged(); + return new ProxyLease(this, entry); + } + + // Same reasoning as the sweep: this is a probe verdict, so it sidelines the entry + // without inflating the request counters that weighted selection reads. + entry.RecordProbe(now, alive: false, latency: null, result.Error, options.BaseQuarantine); + + lock (_gate) + { + _strategy.Report(entry, success: false); + } + + RaiseChanged(); + } + + return null; + } + + /// + public async Task SweepAsync( + IProgress? progress = null, + CancellationToken cancellationToken = default + ) + { + var options = Options; + var targets = Entries; + var total = targets.Count; + + if (total == 0) + { + progress?.Report(new ProxySweepProgress(0, 0, 0)); + return 0; + } + + using var limiter = new SemaphoreSlim(options.ProbeConcurrency, options.ProbeConcurrency); + var checkedCount = 0; + var aliveCount = 0; + + var work = targets.Select(async entry => + { + await limiter.WaitAsync(cancellationToken).ConfigureAwait(false); + try + { + var result = await _probe.ProbeAsync(entry.Endpoint, options, cancellationToken).ConfigureAwait(false); + + // A failed sweep probe sidelines the entry for one base window rather than + // counting as a request failure: it is evidence about the proxy, not usage of it. + entry.RecordProbe( + _time.GetUtcNow(), + result.Alive, + result.Latency, + result.Error, + quarantineOnFailure: options.BaseQuarantine + ); + + if (result.Alive) + { + Interlocked.Increment(ref aliveCount); + } + } + finally + { + limiter.Release(); + progress?.Report( + new ProxySweepProgress( + Interlocked.Increment(ref checkedCount), + total, + Volatile.Read(ref aliveCount) + ) + ); + } + }); + + await Task.WhenAll(work).ConfigureAwait(false); + + lock (_gate) + { + // A sweep can invalidate a sticky pick, so make the strategy re-choose. + _strategy.Reset(); + } + + RaiseChanged(); + return aliveCount; + } + + /// Records the outcome of a lease. Called by . + internal void ReportOutcome(ProxyEntry entry, bool success, TimeSpan? latency, string? error) + { + var options = Options; + var now = _time.GetUtcNow(); + + if (success) + { + entry.RecordSuccess(now, latency); + } + else + { + entry.RecordFailure( + now, + options.BaseQuarantine, + options.MaxQuarantine, + options.FailuresBeforeQuarantine, + error + ); + } + + lock (_gate) + { + _strategy.Report(entry, success); + } + + RaiseChanged(); + } + + /// Whether an endpoint passes the protocol and country filters. + public static bool Matches(ProxyEndpoint endpoint, ProxyOptions options) + { + ArgumentNullException.ThrowIfNull(endpoint); + ArgumentNullException.ThrowIfNull(options); + + if ( + options.Protocols != ProxyProtocolFilter.None + && !options.Protocols.HasFlag(ProxyOptions.ToFlag(endpoint.Protocol)) + ) + { + return false; + } + + return options.Countries.Count == 0 + || ( + endpoint.Country is { } country && options.Countries.Contains(country, StringComparer.OrdinalIgnoreCase) + ); + } + + private ProxyEntry? SelectAvailable() + { + lock (_gate) + { + var now = _time.GetUtcNow(); + + // Best-first ordering: sticky selection takes the head of this list, and the other + // strategies benefit from healthy proxies being clustered at the front. + var candidates = _entries + .Where(entry => entry.IsAvailable(now)) + .OrderByDescending(entry => entry.Health == ProxyHealthState.Alive) + .ThenBy(entry => entry.Latency ?? TimeSpan.MaxValue) + .ThenByDescending(entry => entry.Endpoint.Score) + .ToArray(); + + return _strategy.Pick(candidates); + } + } + + private void RaiseChanged() => Changed?.Invoke(this, EventArgs.Empty); +} diff --git a/src/AvParser.Core/Proxies/Selection/IProxySelectionStrategy.cs b/src/AvParser.Core/Proxies/Selection/IProxySelectionStrategy.cs new file mode 100644 index 0000000..225313d --- /dev/null +++ b/src/AvParser.Core/Proxies/Selection/IProxySelectionStrategy.cs @@ -0,0 +1,38 @@ +namespace AvParser.Core.Proxies.Selection; + +/// Decides which proxy to hand out next. +/// +/// Stateful by design — sticky selection has to remember its pick, and round-robin its cursor. +/// Implementations are called under the pool's lock and need no locking of their own. +/// +public interface IProxySelectionStrategy +{ + /// Which setting this strategy implements. + ProxyRotation Kind { get; } + + /// Picks from the already-filtered available candidates, or if none. + ProxyEntry? Pick(IReadOnlyList candidates); + + /// Tells the strategy how the handed-out proxy fared. + void Report(ProxyEntry entry, bool success); + + /// Drops any remembered state, e.g. after the pool is reloaded. + void Reset(); +} + +/// Builds the strategy named by . +public static class ProxySelectionStrategyFactory +{ + /// Creates a strategy instance. + /// Which strategy to build. + /// Randomness for ; tests pass a seeded instance. + /// Unknown rotation value. + public static IProxySelectionStrategy Create(ProxyRotation rotation, Random? random = null) => + rotation switch + { + ProxyRotation.Sticky => new StickyProxySelection(), + ProxyRotation.RoundRobin => new RoundRobinProxySelection(), + ProxyRotation.WeightedRandom => new WeightedRandomProxySelection(random), + _ => throw new ArgumentOutOfRangeException(nameof(rotation), rotation, "Unknown rotation strategy."), + }; +} diff --git a/src/AvParser.Core/Proxies/Selection/RoundRobinProxySelection.cs b/src/AvParser.Core/Proxies/Selection/RoundRobinProxySelection.cs new file mode 100644 index 0000000..e4ff441 --- /dev/null +++ b/src/AvParser.Core/Proxies/Selection/RoundRobinProxySelection.cs @@ -0,0 +1,43 @@ +namespace AvParser.Core.Proxies.Selection; + +/// +/// Advances one position through the candidates on every acquisition. +/// +/// +/// Spreads requests evenly, which is what rate limits care about. The cursor is kept as a +/// monotonic counter reduced modulo the candidate count rather than as an index into the list, +/// so a list that shrinks between calls cannot throw or silently skip entries. +/// +public sealed class RoundRobinProxySelection : IProxySelectionStrategy +{ + private int _cursor; + + /// + public ProxyRotation Kind => ProxyRotation.RoundRobin; + + /// + public ProxyEntry? Pick(IReadOnlyList candidates) + { + ArgumentNullException.ThrowIfNull(candidates); + + if (candidates.Count == 0) + { + return null; + } + + var index = (int)((uint)_cursor % (uint)candidates.Count); + _cursor = _cursor == int.MaxValue ? 0 : _cursor + 1; + + return candidates[index]; + } + + /// + public void Report(ProxyEntry entry, bool success) + { + // Position is independent of outcome: a failing proxy is dropped by the pool's own + // quarantine, not by rewinding the cursor. + } + + /// + public void Reset() => _cursor = 0; +} diff --git a/src/AvParser.Core/Proxies/Selection/StickyProxySelection.cs b/src/AvParser.Core/Proxies/Selection/StickyProxySelection.cs new file mode 100644 index 0000000..c7db604 --- /dev/null +++ b/src/AvParser.Core/Proxies/Selection/StickyProxySelection.cs @@ -0,0 +1,45 @@ +namespace AvParser.Core.Proxies.Selection; + +/// +/// Holds one proxy until it fails. +/// +/// +/// The default because it is the only strategy that keeps a site's session coherent: rotating on +/// every request changes the apparent client mid-session, which reliably triggers re-logins and +/// captchas on anything that tracks cookies. +/// +public sealed class StickyProxySelection : IProxySelectionStrategy +{ + private ProxyEntry? _current; + + /// + public ProxyRotation Kind => ProxyRotation.Sticky; + + /// + public ProxyEntry? Pick(IReadOnlyList candidates) + { + ArgumentNullException.ThrowIfNull(candidates); + + // Keep the current pick only while it is still among the available candidates: a refresh + // or a quarantine may have taken it out from under us. + if (_current is not null && candidates.Contains(_current)) + { + return _current; + } + + _current = candidates.Count > 0 ? candidates[0] : null; + return _current; + } + + /// + public void Report(ProxyEntry entry, bool success) + { + if (!success && ReferenceEquals(entry, _current)) + { + _current = null; + } + } + + /// + public void Reset() => _current = null; +} diff --git a/src/AvParser.Core/Proxies/Selection/WeightedRandomProxySelection.cs b/src/AvParser.Core/Proxies/Selection/WeightedRandomProxySelection.cs new file mode 100644 index 0000000..0d95a2d --- /dev/null +++ b/src/AvParser.Core/Proxies/Selection/WeightedRandomProxySelection.cs @@ -0,0 +1,83 @@ +namespace AvParser.Core.Proxies.Selection; + +/// +/// Picks at random, biased towards proxies that have actually worked. +/// +/// +/// Weight is the feed's score multiplied by the observed success rate, so a proxy the feed likes +/// but that keeps failing here drifts to the bottom without ever being excluded outright — free +/// lists recover, and a hard ban would lose them permanently. +/// +public sealed class WeightedRandomProxySelection(Random? random = null) : IProxySelectionStrategy +{ + /// Floor on the weight so an unproven proxy still gets picked occasionally. + private const double MinimumWeight = 0.05d; + + private readonly Random _random = random ?? Random.Shared; + + /// + public ProxyRotation Kind => ProxyRotation.WeightedRandom; + + /// + public ProxyEntry? Pick(IReadOnlyList candidates) + { + ArgumentNullException.ThrowIfNull(candidates); + + if (candidates.Count == 0) + { + return null; + } + + if (candidates.Count == 1) + { + return candidates[0]; + } + + var weights = new double[candidates.Count]; + var total = 0d; + + for (var i = 0; i < candidates.Count; i++) + { + var weight = WeightOf(candidates[i]); + weights[i] = weight; + total += weight; + } + + var target = _random.NextDouble() * total; + var running = 0d; + + for (var i = 0; i < weights.Length; i++) + { + running += weights[i]; + if (target < running) + { + return candidates[i]; + } + } + + // Floating-point drift can leave `target` a hair past the final boundary. + return candidates[^1]; + } + + /// + public void Report(ProxyEntry entry, bool success) + { + // The weight is derived from the entry's own counters, which the pool already updated. + } + + /// + public void Reset() + { + // Stateless beyond the RNG. + } + + /// Weight of a single entry. Exposed so the tests can assert the ordering. + public static double WeightOf(ProxyEntry entry) + { + ArgumentNullException.ThrowIfNull(entry); + + // Feed scores are small integers and frequently zero; +1 keeps an unscored proxy in play. + var score = Math.Max(0, entry.Endpoint.Score) + 1d; + return Math.Max(MinimumWeight, score * entry.SuccessRate); + } +} diff --git a/src/AvParser.Core/Settings/AppSettings.cs b/src/AvParser.Core/Settings/AppSettings.cs index 1be5300..39227ac 100644 --- a/src/AvParser.Core/Settings/AppSettings.cs +++ b/src/AvParser.Core/Settings/AppSettings.cs @@ -1,3 +1,5 @@ +using AvParser.Core.Proxies; + namespace AvParser.Core.Settings; /// Theme preference. follows the OS setting. @@ -39,4 +41,48 @@ public sealed record AppSettings /// Minimum Serilog level, as a Serilog level name. public string MinimumLogLevel { get; init; } = "Information"; + + /// How the pool picks the next proxy. + public ProxyRotation ProxyRotation { get; init; } = ProxyRotation.Sticky; + + /// When proxy liveness is verified. + public ProxyHealthCheck ProxyHealthCheck { get; init; } = ProxyHealthCheck.Pool; + + /// Whether the remote proxy feed is consulted. + public bool ProxyUseFeed { get; init; } = true; + + /// Protocols accepted when loading proxy sources. + public ProxyProtocolFilter ProxyProtocols { get; init; } = ProxyProtocolFilter.All; + + /// URL fetched to decide whether a proxy works. + public string ProxyProbeUrl { get; init; } = "http://www.gstatic.com/generate_204"; + + /// Per-proxy probe timeout, in seconds. + public int ProxyProbeTimeoutSeconds { get; init; } = 8; + + /// How many probes run at once during a pool sweep. + public int ProxyProbeConcurrency { get; init; } = 64; + + /// Projects the proxy-related settings onto . + /// + /// Settings are persisted as primitives so an old file still deserialises; the pool wants a + /// validated options object. This is the single place that bridges the two. + /// + public ProxyOptions ToProxyOptions() + { + var probeUrl = Uri.TryCreate(ProxyProbeUrl, UriKind.Absolute, out var parsed) + ? parsed + : new ProxyOptions().ProbeUrl; + + return new ProxyOptions + { + Rotation = ProxyRotation, + HealthCheck = ProxyHealthCheck, + UseFeed = ProxyUseFeed, + Protocols = ProxyProtocols, + ProbeUrl = probeUrl, + ProbeTimeout = TimeSpan.FromSeconds(Math.Clamp(ProxyProbeTimeoutSeconds, 1, 120)), + ProbeConcurrency = Math.Clamp(ProxyProbeConcurrency, 1, 512), + }.Validated(); + } } diff --git a/src/AvParser.Infrastructure/AvParser.Infrastructure.csproj b/src/AvParser.Infrastructure/AvParser.Infrastructure.csproj index 2ea6d1b..2414d0d 100644 --- a/src/AvParser.Infrastructure/AvParser.Infrastructure.csproj +++ b/src/AvParser.Infrastructure/AvParser.Infrastructure.csproj @@ -7,8 +7,15 @@ + + + + + + diff --git a/src/AvParser.Infrastructure/DependencyInjection/InfrastructureServiceCollectionExtensions.cs b/src/AvParser.Infrastructure/DependencyInjection/InfrastructureServiceCollectionExtensions.cs index b1fba01..926851d 100644 --- a/src/AvParser.Infrastructure/DependencyInjection/InfrastructureServiceCollectionExtensions.cs +++ b/src/AvParser.Infrastructure/DependencyInjection/InfrastructureServiceCollectionExtensions.cs @@ -1,4 +1,6 @@ +using AvParser.Core.Proxies; using AvParser.Core.Settings; +using AvParser.Infrastructure.Proxies; using AvParser.Infrastructure.Settings; using AvParser.Infrastructure.Storage; using Microsoft.Extensions.DependencyInjection; @@ -31,6 +33,49 @@ public static class InfrastructureServiceCollectionExtensions sp.GetRequiredService>() )); + services.AddAvParserProxies(); + + return services; + } + + /// Registers the proxy sources, the probe, the pool and the proxied client factory. + /// + /// The feed source gets a pooled because it always talks to the same + /// CDN host. The probe deliberately does not: its handler carries the proxy, so it has to + /// build one per check. + /// + public static IServiceCollection AddAvParserProxies(this IServiceCollection services) + { + ArgumentNullException.ThrowIfNull(services); + + services + .AddHttpClient(client => + { + client.Timeout = TimeSpan.FromSeconds(30); + client.DefaultRequestHeaders.UserAgent.ParseAdd("AvParser/0.1"); + }) + .ConfigurePrimaryHttpMessageHandler(() => + new SocketsHttpHandler { AutomaticDecompression = System.Net.DecompressionMethods.All } + ); + + services.AddSingleton(); + services.AddSingleton(sp => sp.GetRequiredService()); + + // Registration order here is the order the pool merges sources; the custom list comes + // last so a user-entered address wins over a feed entry for the same host and port. + services.AddSingleton(sp => sp.GetRequiredService()); + services.AddSingleton(sp => sp.GetRequiredService()); + + services.AddSingleton(); + + services.AddSingleton(sp => new ProxyPool( + sp.GetServices(), + sp.GetRequiredService(), + sp.GetRequiredService().Current.ToProxyOptions() + )); + + services.AddSingleton(); + return services; } } diff --git a/src/AvParser.Infrastructure/Proxies/CustomProxySource.cs b/src/AvParser.Infrastructure/Proxies/CustomProxySource.cs new file mode 100644 index 0000000..d59edcc --- /dev/null +++ b/src/AvParser.Infrastructure/Proxies/CustomProxySource.cs @@ -0,0 +1,273 @@ +using System.Text.Json; +using System.Text.Json.Serialization; +using AvParser.Core.Proxies; +using AvParser.Infrastructure.Storage; +using Microsoft.Extensions.Logging; + +namespace AvParser.Infrastructure.Proxies; + +/// One user-entered proxy, as persisted. +/// Address in scheme://host:port form. +/// Optional username. +/// Optional password. +/// Free-text note shown in the UI. +public sealed record CustomProxyRecord( + string Address, + string? Username = null, + string? Password = null, + string? Note = null +); + +/// Source-generated serialiser metadata for the user's proxy list. +[JsonSourceGenerationOptions(WriteIndented = true, PropertyNamingPolicy = JsonKnownNamingPolicy.CamelCase)] +[JsonSerializable(typeof(List))] +internal sealed partial class CustomProxyJsonContext : JsonSerializerContext; + +/// +/// The user's own proxy list, persisted next to the settings. +/// +/// +/// A separate file rather than a field inside settings.json: it is a list people paste into, +/// back up and share, and burying it in the settings blob makes all three awkward. Credentials +/// are stored in clear text — the same as every other proxy client, and worth knowing. +/// +public sealed class CustomProxySource : IMutableProxySource, IDisposable +{ + private readonly IAppPaths _paths; + private readonly ILogger _logger; + private readonly SemaphoreSlim _gate = new(1, 1); + + private List? _records; + + /// Creates the source. + public CustomProxySource(IAppPaths paths, ILogger logger) + { + _paths = paths ?? throw new ArgumentNullException(nameof(paths)); + _logger = logger ?? throw new ArgumentNullException(nameof(logger)); + } + + /// + public string Id => "custom"; + + /// + public string DisplayName => "Custom list"; + + /// + public ProxySourceKind Kind => ProxySourceKind.Custom; + + /// + public async Task> GetProxiesAsync(CancellationToken cancellationToken = default) + { + await _gate.WaitAsync(cancellationToken).ConfigureAwait(false); + try + { + var records = Load(); + var endpoints = new List(records.Count); + + foreach (var record in records) + { + if (ProxyEndpoint.TryParse(record.Address, out var endpoint)) + { + endpoints.Add( + endpoint with + { + Username = record.Username ?? endpoint.Username, + Password = record.Password ?? endpoint.Password, + } + ); + } + else + { + _logger.LogWarning("Skipping unparsable custom proxy {Address}", record.Address); + } + } + + return endpoints; + } + finally + { + _gate.Release(); + } + } + + /// + public async Task AddAsync(IEnumerable endpoints, CancellationToken cancellationToken = default) + { + ArgumentNullException.ThrowIfNull(endpoints); + + await _gate.WaitAsync(cancellationToken).ConfigureAwait(false); + try + { + var records = Load(); + var known = new HashSet( + records.Where(r => ProxyEndpoint.TryParse(r.Address, out _)).Select(KeyOf), + StringComparer.Ordinal + ); + + var added = 0; + foreach (var endpoint in endpoints) + { + if (!known.Add(endpoint.Key)) + { + continue; + } + + records.Add(new CustomProxyRecord(endpoint.ToString(), endpoint.Username, endpoint.Password)); + added++; + } + + if (added > 0) + { + await SaveAsync(records, cancellationToken).ConfigureAwait(false); + } + + return added; + } + finally + { + _gate.Release(); + } + } + + /// + public async Task RemoveAsync(ProxyEndpoint endpoint, CancellationToken cancellationToken = default) + { + ArgumentNullException.ThrowIfNull(endpoint); + + await _gate.WaitAsync(cancellationToken).ConfigureAwait(false); + try + { + var records = Load(); + var removed = records.RemoveAll(record => + ProxyEndpoint.TryParse(record.Address, out var parsed) + && string.Equals(parsed.Key, endpoint.Key, StringComparison.Ordinal) + ); + + if (removed > 0) + { + await SaveAsync(records, cancellationToken).ConfigureAwait(false); + } + + return removed > 0; + } + finally + { + _gate.Release(); + } + } + + /// + public async Task ClearAsync(CancellationToken cancellationToken = default) + { + await _gate.WaitAsync(cancellationToken).ConfigureAwait(false); + try + { + await SaveAsync([], cancellationToken).ConfigureAwait(false); + } + finally + { + _gate.Release(); + } + } + + /// + public void Dispose() => _gate.Dispose(); + + /// + /// Parses a pasted blob into endpoints, one per line. + /// + /// The endpoints that parsed, and the lines that did not. + /// + /// Reports the bad lines instead of dropping them: people paste lists of hundreds, and a + /// silent "added 97 of 100" is impossible to act on. + /// + public static (IReadOnlyList Parsed, IReadOnlyList Rejected) ParseList(string? text) + { + var parsed = new List(); + var rejected = new List(); + + if (string.IsNullOrWhiteSpace(text)) + { + return (parsed, rejected); + } + + // Lines first, then separators within a line. Doing it the other way round splits + // "# my proxies" into three tokens and reports two of them as invalid addresses. + foreach (var rawLine in text.Split(['\n', '\r'], StringSplitOptions.RemoveEmptyEntries)) + { + var line = rawLine.Trim(); + if (line.Length == 0 || line.StartsWith('#') || line.StartsWith("//", StringComparison.Ordinal)) + { + continue; + } + + foreach (var token in line.Split([',', ';', ' ', '\t'], StringSplitOptions.RemoveEmptyEntries)) + { + if (ProxyEndpoint.TryParse(token, out var endpoint)) + { + parsed.Add(endpoint); + } + else + { + rejected.Add(token); + } + } + } + + return (parsed, rejected); + } + + private static string KeyOf(CustomProxyRecord record) => + ProxyEndpoint.TryParse(record.Address, out var endpoint) ? endpoint.Key : record.Address; + + private List Load() + { + if (_records is not null) + { + return _records; + } + + try + { + if (!File.Exists(_paths.CustomProxiesFile)) + { + _records = []; + return _records; + } + + var json = File.ReadAllText(_paths.CustomProxiesFile); + _records = JsonSerializer.Deserialize(json, CustomProxyJsonContext.Default.ListCustomProxyRecord) ?? []; + } + catch (Exception ex) when (ex is IOException or UnauthorizedAccessException or JsonException) + { + _logger.LogWarning( + ex, + "Could not read {Path}; starting with an empty custom list", + _paths.CustomProxiesFile + ); + _records = []; + } + + return _records; + } + + private async Task SaveAsync(List records, CancellationToken cancellationToken) + { + _records = records; + + try + { + Directory.CreateDirectory(Path.GetDirectoryName(_paths.CustomProxiesFile)!); + + var temp = _paths.CustomProxiesFile + ".tmp"; + var json = JsonSerializer.Serialize(records, CustomProxyJsonContext.Default.ListCustomProxyRecord); + + await File.WriteAllTextAsync(temp, json, cancellationToken).ConfigureAwait(false); + File.Move(temp, _paths.CustomProxiesFile, overwrite: true); + } + catch (Exception ex) when (ex is IOException or UnauthorizedAccessException) + { + _logger.LogWarning(ex, "Could not write {Path}", _paths.CustomProxiesFile); + } + } +} diff --git a/src/AvParser.Infrastructure/Proxies/HttpProxyProbe.cs b/src/AvParser.Infrastructure/Proxies/HttpProxyProbe.cs new file mode 100644 index 0000000..20424b5 --- /dev/null +++ b/src/AvParser.Infrastructure/Proxies/HttpProxyProbe.cs @@ -0,0 +1,113 @@ +using System.Diagnostics; +using System.Net; +using AvParser.Core.Proxies; + +namespace AvParser.Infrastructure.Proxies; + +/// Builds the .NET plumbing needed to route a request through a proxy. +public static class ProxyHandlerFactory +{ + /// Wraps an endpoint in a , attaching credentials if present. + public static WebProxy CreateWebProxy(ProxyEndpoint endpoint) + { + ArgumentNullException.ThrowIfNull(endpoint); + + var proxy = new WebProxy(endpoint.Uri); + + if (endpoint.HasCredentials) + { + proxy.Credentials = new NetworkCredential(endpoint.Username, endpoint.Password); + } + + return proxy; + } + + /// + /// Creates a handler bound to one proxy, or a direct handler when is null. + /// + /// + /// A fresh handler per proxy is unavoidable — the proxy is a property of the handler, not of + /// the request — so connection pooling cannot be shared across proxies. + /// SOCKS is handled natively: .NET understands the socks4/socks4a/socks5 schemes. + /// + public static SocketsHttpHandler CreateHandler(ProxyEndpoint? endpoint, TimeSpan connectTimeout) + { + var handler = new SocketsHttpHandler + { + AllowAutoRedirect = false, + ConnectTimeout = connectTimeout, + PooledConnectionLifetime = TimeSpan.FromMinutes(2), + AutomaticDecompression = DecompressionMethods.All, + }; + + if (endpoint is null) + { + handler.UseProxy = false; + return handler; + } + + handler.UseProxy = true; + handler.Proxy = CreateWebProxy(endpoint); + return handler; + } +} + +/// Checks a proxy by fetching through it. +public sealed class HttpProxyProbe : IProxyProbe +{ + /// + public async Task ProbeAsync( + ProxyEndpoint endpoint, + ProxyOptions options, + CancellationToken cancellationToken = default + ) + { + ArgumentNullException.ThrowIfNull(endpoint); + ArgumentNullException.ThrowIfNull(options); + + using var handler = ProxyHandlerFactory.CreateHandler(endpoint, options.ProbeTimeout); + using var client = new HttpClient(handler, disposeHandler: false) { Timeout = options.ProbeTimeout }; + + var stopwatch = Stopwatch.StartNew(); + + try + { + using var response = await client + .GetAsync(options.ProbeUrl, HttpCompletionOption.ResponseHeadersRead, cancellationToken) + .ConfigureAwait(false); + + stopwatch.Stop(); + + return response.IsSuccessStatusCode + ? ProxyProbeResult.Success(stopwatch.Elapsed) + : ProxyProbeResult.Failure($"HTTP {(int)response.StatusCode}"); + } + catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) + { + // The caller pulled the plug — that is not a verdict on the proxy. + throw; + } + catch (OperationCanceledException) + { + return ProxyProbeResult.Failure("timeout"); + } + catch (HttpRequestException ex) + { + return ProxyProbeResult.Failure(Describe(ex)); + } + catch (Exception ex) when (ex is IOException or InvalidOperationException or NotSupportedException) + { + return ProxyProbeResult.Failure(ex.Message); + } + } + + private static string Describe(HttpRequestException exception) => + exception.HttpRequestError switch + { + HttpRequestError.ProxyTunnelError => "proxy refused CONNECT", + HttpRequestError.ConnectionError => "connection failed", + HttpRequestError.NameResolutionError => "DNS failed", + HttpRequestError.SecureConnectionError => "TLS failed", + _ => exception.Message, + }; +} diff --git a/src/AvParser.Infrastructure/Proxies/ProxiedHttpClientFactory.cs b/src/AvParser.Infrastructure/Proxies/ProxiedHttpClientFactory.cs new file mode 100644 index 0000000..c6e34d4 --- /dev/null +++ b/src/AvParser.Infrastructure/Proxies/ProxiedHttpClientFactory.cs @@ -0,0 +1,56 @@ +using AvParser.Core.Proxies; + +namespace AvParser.Infrastructure.Proxies; + +/// Creates instances bound to a specific proxy. +/// +/// Not IHttpClientFactory: that exists to share handlers across requests, and a per-proxy +/// handler is the opposite of shareable. This is the seam the parser layer will use once it +/// starts making real requests. +/// +public interface IProxiedHttpClientFactory +{ + /// Creates a client routed through , or direct when null. + /// The caller owns the returned client and must dispose it. + HttpClient Create(ProxyEndpoint? endpoint, TimeSpan? timeout = null); + + /// + /// Takes a proxy from the pool and returns a client bound to it, or a direct client when the + /// pool has nothing usable. + /// + /// + /// The lease comes back with the client so the caller can report the outcome — without that + /// the pool never learns which proxies work. + /// + Task<(HttpClient Client, ProxyLease? Lease)> CreateFromPoolAsync( + TimeSpan? timeout = null, + CancellationToken cancellationToken = default + ); +} + +/// +public sealed class ProxiedHttpClientFactory(IProxyPool pool) : IProxiedHttpClientFactory +{ + private static readonly TimeSpan DefaultTimeout = TimeSpan.FromSeconds(30); + + private readonly IProxyPool _pool = pool ?? throw new ArgumentNullException(nameof(pool)); + + /// + public HttpClient Create(ProxyEndpoint? endpoint, TimeSpan? timeout = null) + { + var effective = timeout ?? DefaultTimeout; + var handler = ProxyHandlerFactory.CreateHandler(endpoint, effective); + + return new HttpClient(handler, disposeHandler: true) { Timeout = effective }; + } + + /// + public async Task<(HttpClient Client, ProxyLease? Lease)> CreateFromPoolAsync( + TimeSpan? timeout = null, + CancellationToken cancellationToken = default + ) + { + var lease = await _pool.AcquireAsync(cancellationToken).ConfigureAwait(false); + return (Create(lease?.Endpoint, timeout), lease); + } +} diff --git a/src/AvParser.Infrastructure/Proxies/ProxiflyProxySource.cs b/src/AvParser.Infrastructure/Proxies/ProxiflyProxySource.cs new file mode 100644 index 0000000..a833542 --- /dev/null +++ b/src/AvParser.Infrastructure/Proxies/ProxiflyProxySource.cs @@ -0,0 +1,186 @@ +using System.Text.Json; +using System.Text.Json.Serialization; +using AvParser.Core.Proxies; +using Microsoft.Extensions.Logging; + +namespace AvParser.Infrastructure.Proxies; + +/// One record of the proxifly feed. +internal sealed record ProxiflyRecord +{ + public string? Proxy { get; init; } + public string? Protocol { get; init; } + public string? Ip { get; init; } + public int Port { get; init; } + public bool Https { get; init; } + public string? Anonymity { get; init; } + public int Score { get; init; } + public ProxiflyGeolocation? Geolocation { get; init; } +} + +/// Geolocation block of a proxifly record. +internal sealed record ProxiflyGeolocation +{ + public string? Country { get; init; } + public string? City { get; init; } +} + +/// Source-generated serialiser metadata for the proxifly feed. +[JsonSourceGenerationOptions(PropertyNamingPolicy = JsonKnownNamingPolicy.CamelCase)] +[JsonSerializable(typeof(ProxiflyRecord[]))] +internal sealed partial class ProxiflyJsonContext : JsonSerializerContext; + +/// +/// Reads the public list published at github.com/proxifly/free-proxy-list. +/// +/// +/// Fetches the combined all/data.json and filters locally rather than requesting a +/// per-protocol slice: the feed is regenerated every few minutes, so one conditional request for +/// the whole list beats four that can disagree with each other mid-publish. +/// +public sealed class ProxiflyProxySource : IProxySource, IDisposable +{ + /// Served from jsDelivr rather than raw.githubusercontent.com — a CDN meant for this. + public static readonly Uri FeedUrl = new( + "https://cdn.jsdelivr.net/gh/proxifly/free-proxy-list@main/proxies/all/data.json" + ); + + /// Upstream refreshes every five minutes; asking more often only burns bandwidth. + public static readonly TimeSpan CacheTtl = TimeSpan.FromMinutes(5); + + private readonly HttpClient _http; + private readonly ILogger _logger; + private readonly TimeProvider _time; + private readonly SemaphoreSlim _fetchLock = new(1, 1); + + private IReadOnlyList _cached = []; + private DateTimeOffset? _fetchedAtUtc; + + /// Creates the source. + public ProxiflyProxySource(HttpClient http, ILogger logger, TimeProvider? timeProvider = null) + { + _http = http ?? throw new ArgumentNullException(nameof(http)); + _logger = logger ?? throw new ArgumentNullException(nameof(logger)); + _time = timeProvider ?? TimeProvider.System; + } + + /// + public string Id => "proxifly"; + + /// + public string DisplayName => "Proxifly free-proxy-list"; + + /// + public ProxySourceKind Kind => ProxySourceKind.Feed; + + /// When the feed was last downloaded, or if never. + public DateTimeOffset? FetchedAtUtc => _fetchedAtUtc; + + /// + public async Task> GetProxiesAsync(CancellationToken cancellationToken = default) + { + if (IsFresh()) + { + return _cached; + } + + await _fetchLock.WaitAsync(cancellationToken).ConfigureAwait(false); + try + { + // Another caller may have refreshed while we waited for the lock. + if (IsFresh()) + { + return _cached; + } + + var endpoints = await FetchAsync(cancellationToken).ConfigureAwait(false); + _cached = endpoints; + _fetchedAtUtc = _time.GetUtcNow(); + return endpoints; + } + finally + { + _fetchLock.Release(); + } + } + + /// Drops the cache so the next call re-downloads. + public void Invalidate() => _fetchedAtUtc = null; + + /// + /// The is owned by the container, so only the lock is released here. + public void Dispose() => _fetchLock.Dispose(); + + /// Converts one feed record into an endpoint, or if unusable. + /// Internal so the parsing can be tested against captured feed samples without HTTP. + internal static ProxyEndpoint? ToEndpoint(ProxiflyRecord record) + { + if (record.Ip is not { Length: > 0 } host || record.Port is < 1 or > 65535) + { + // The "proxy" field carries the same address; fall back to it when the parts are absent. + return ProxyEndpoint.TryParse(record.Proxy, out var parsed) ? Decorate(parsed, record) : null; + } + + if (!ProxyEndpoint.TryParseProtocol(record.Protocol, out var protocol)) + { + return null; + } + + return Decorate(new ProxyEndpoint(protocol, host, record.Port), record); + } + + /// Parses a whole feed payload. + internal static IReadOnlyList ParseFeed(string json) + { + var records = JsonSerializer.Deserialize(json, ProxiflyJsonContext.Default.ProxiflyRecordArray); + if (records is null) + { + return []; + } + + var endpoints = new List(records.Length); + foreach (var record in records) + { + if (ToEndpoint(record) is { } endpoint) + { + endpoints.Add(endpoint); + } + } + + return endpoints; + } + + private static ProxyEndpoint Decorate(ProxyEndpoint endpoint, ProxiflyRecord record) => + endpoint with + { + Country = record.Geolocation?.Country, + City = string.Equals(record.Geolocation?.City, "Unknown", StringComparison.OrdinalIgnoreCase) + ? null + : record.Geolocation?.City, + Anonymity = ProxyEndpoint.ParseAnonymity(record.Anonymity), + Score = record.Score, + }; + + private bool IsFresh() => _fetchedAtUtc is { } at && _time.GetUtcNow() - at < CacheTtl; + + private async Task> FetchAsync(CancellationToken cancellationToken) + { + try + { + using var response = await _http.GetAsync(FeedUrl, cancellationToken).ConfigureAwait(false); + response.EnsureSuccessStatusCode(); + + var json = await response.Content.ReadAsStringAsync(cancellationToken).ConfigureAwait(false); + var endpoints = ParseFeed(json); + + _logger.LogInformation("Fetched {Count} proxies from {Source}", endpoints.Count, DisplayName); + return endpoints; + } + catch (Exception ex) when (ex is HttpRequestException or JsonException or TaskCanceledException) + { + // A feed that is down must not take the app with it: keep serving whatever we had. + _logger.LogWarning(ex, "Could not fetch {Source}; keeping {Count} cached", DisplayName, _cached.Count); + return _cached; + } + } +} diff --git a/src/AvParser.Infrastructure/Storage/AppPaths.cs b/src/AvParser.Infrastructure/Storage/AppPaths.cs index 7fdddef..3eee02c 100644 --- a/src/AvParser.Infrastructure/Storage/AppPaths.cs +++ b/src/AvParser.Infrastructure/Storage/AppPaths.cs @@ -13,6 +13,10 @@ public interface IAppPaths /// Full path of the settings file. string SettingsFile { get; } + /// Full path of the user's own proxy list. + /// Kept separate from the settings file: it is a list the user edits and may want to back up or share. + string CustomProxiesFile { get; } + /// Directory holding rolling log files. string LogDirectory { get; } } @@ -45,6 +49,7 @@ public sealed class AppPaths : IAppPaths DataDirectory = dataDirectory; SettingsFile = Path.Combine(dataDirectory, "settings.json"); + CustomProxiesFile = Path.Combine(dataDirectory, "proxies.custom.json"); LogDirectory = Path.Combine(dataDirectory, "logs"); } @@ -54,6 +59,9 @@ public sealed class AppPaths : IAppPaths /// public string SettingsFile { get; } + /// + public string CustomProxiesFile { get; } + /// public string LogDirectory { get; } diff --git a/src/AvParser.UI/DependencyInjection/UiServiceCollectionExtensions.cs b/src/AvParser.UI/DependencyInjection/UiServiceCollectionExtensions.cs index e82260b..51867b6 100644 --- a/src/AvParser.UI/DependencyInjection/UiServiceCollectionExtensions.cs +++ b/src/AvParser.UI/DependencyInjection/UiServiceCollectionExtensions.cs @@ -1,4 +1,5 @@ using AvParser.Core.Parsing; +using AvParser.Core.Proxies; using AvParser.Core.Settings; using AvParser.Infrastructure.Storage; using AvParser.UI.Navigation; @@ -37,13 +38,20 @@ public static class UiServiceCollectionExtensions sp.GetRequiredService(), sp.GetRequiredService(), sp.GetRequiredService(), - sp.GetRequiredService() + sp.GetRequiredService(), + sp.GetRequiredService() + )); + services.AddSingleton(static sp => new ProxiesViewModel( + sp.GetRequiredService(), + sp.GetRequiredService(), + sp.GetRequiredService>() )); services.AddSingleton(); // Order here is the order of the navigation rail; the first entry is the landing page. services.AddSingleton(static sp => sp.GetRequiredService()); services.AddSingleton(static sp => sp.GetRequiredService()); + services.AddSingleton(static sp => sp.GetRequiredService()); services.AddSingleton(static sp => sp.GetRequiredService()); services.AddSingleton(static sp => sp.GetRequiredService()); diff --git a/src/AvParser.UI/Styles/Controls.axaml b/src/AvParser.UI/Styles/Controls.axaml index 32bfae9..aec0b2b 100644 --- a/src/AvParser.UI/Styles/Controls.axaml +++ b/src/AvParser.UI/Styles/Controls.axaml @@ -71,6 +71,23 @@ + + + + + + + + +