From 9bf2ea5532e7d6fc948c4a9717f1ecf4bce54746 Mon Sep 17 00:00:00 2001 From: Leonid Pershin Date: Thu, 13 Aug 2026 17:22:30 +0300 Subject: [PATCH] Add a proxy pool with rotation, liveness checks and a management page MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The parser will need to move between proxies, so this adds the module it will sit on: pluggable sources, a pool that hands proxies out and learns from the outcome, three rotation strategies, and a page to drive it. Sources are IProxySource implementations. The public proxifly/free-proxy-list feed is fetched as the combined all/data.json through jsDelivr and filtered locally — one conditional request beats four per-protocol ones that can disagree mid-publish — and cached for the five minutes upstream takes to regenerate. A feed that is down keeps serving its last payload rather than emptying the pool. The user's own list lives in proxies.custom.json beside the settings, takes a pasted blob, and names the lines it could not parse instead of quietly dropping them. Both knobs the pool exposes are settings, as asked: rotation is Sticky (default, the only one that keeps site sessions coherent), RoundRobin or WeightedRandom; liveness is either a parallel sweep of the whole pool or a probe at hand-out time. Free lists are a few percent alive, so skipping verification entirely means mostly waiting on timeouts. Two invariants worth keeping, both of which cost a bug to find: Availability is decided by the quarantine, never by Health. Excluding everything that has ever failed made the quarantine window dead code and discarded proxies permanently on their first hiccup, which is exactly wrong for addresses that flap constantly. Health only orders the candidates now. A probe verdict does not touch the success/failure counters. Those are about real requests, and letting a sweep over a few thousand proxies rewrite them would drown the evidence weighted selection reads. SOCKS needs no extra package — .NET resolves socks4/socks4a/socks5 in WebProxy — but a proxifly record with "protocol": "https" is still an HTTP proxy reached over http:// with CONNECT, not an https:// scheme. 115 new tests. Also fixes a pre-existing flake: a command gated on another command's IsExecuting cannot be driven straight after its Execute() completes, because IsExecuting is published on the output scheduler. Co-Authored-By: Claude Opus 5 --- AvParser.slnx | 1 + CLAUDE.md | 28 ++ Directory.Packages.props | 1 + README.md | 56 ++- src/AvParser.Core/Proxies/IProxyPool.cs | 89 +++++ src/AvParser.Core/Proxies/IProxySource.cs | 65 +++ src/AvParser.Core/Proxies/ProxyEndpoint.cs | 210 ++++++++++ src/AvParser.Core/Proxies/ProxyEntry.cs | 186 +++++++++ src/AvParser.Core/Proxies/ProxyOptions.cs | 129 ++++++ src/AvParser.Core/Proxies/ProxyPool.cs | 340 ++++++++++++++++ .../Selection/IProxySelectionStrategy.cs | 38 ++ .../Selection/RoundRobinProxySelection.cs | 43 ++ .../Proxies/Selection/StickyProxySelection.cs | 45 +++ .../Selection/WeightedRandomProxySelection.cs | 83 ++++ src/AvParser.Core/Settings/AppSettings.cs | 46 +++ .../AvParser.Infrastructure.csproj | 7 + ...frastructureServiceCollectionExtensions.cs | 45 +++ .../Proxies/CustomProxySource.cs | 273 +++++++++++++ .../Proxies/HttpProxyProbe.cs | 113 ++++++ .../Proxies/ProxiedHttpClientFactory.cs | 56 +++ .../Proxies/ProxiflyProxySource.cs | 186 +++++++++ .../Storage/AppPaths.cs | 8 + .../UiServiceCollectionExtensions.cs | 10 +- src/AvParser.UI/Styles/Controls.axaml | 17 + src/AvParser.UI/Styles/Icons.axaml | 10 + src/AvParser.UI/Styles/Tokens.axaml | 2 + .../ViewModels/ProxiesViewModel.cs | 363 +++++++++++++++++ .../ViewModels/ProxyRowViewModel.cs | 88 +++++ .../ViewModels/SettingsViewModel.cs | 104 +++++ src/AvParser.UI/Views/ParseView.axaml | 1 + src/AvParser.UI/Views/ProxiesView.axaml | 221 +++++++++++ src/AvParser.UI/Views/ProxiesView.axaml.cs | 13 + src/AvParser.UI/Views/SettingsView.axaml | 60 +++ .../Proxies/ProxyEndpointTests.cs | 106 +++++ .../Proxies/ProxyPoolTests.cs | 370 ++++++++++++++++++ .../Proxies/ProxySelectionTests.cs | 142 +++++++ .../Proxies/ProxyTestDoubles.cs | 93 +++++ .../AvParser.Infrastructure.Tests.csproj | 14 + .../CustomProxySourceTests.cs | 141 +++++++ .../ProxiflyFeedTests.cs | 115 ++++++ .../ProxyHandlerFactoryTests.cs | 82 ++++ tests/AvParser.UI.HeadlessTests/Fakes.cs | 41 ++ .../ProxiesViewTests.cs | 81 ++++ .../ViewLocatorTests.cs | 2 + tests/AvParser.UI.Tests/Fakes/FakeProxies.cs | 84 ++++ .../AvParser.UI.Tests/ParseViewModelTests.cs | 15 +- .../ProxiesViewModelTests.cs | 199 ++++++++++ .../AvParser.UI.Tests/ReactiveUiBootstrap.cs | 5 + 48 files changed, 4422 insertions(+), 5 deletions(-) create mode 100644 src/AvParser.Core/Proxies/IProxyPool.cs create mode 100644 src/AvParser.Core/Proxies/IProxySource.cs create mode 100644 src/AvParser.Core/Proxies/ProxyEndpoint.cs create mode 100644 src/AvParser.Core/Proxies/ProxyEntry.cs create mode 100644 src/AvParser.Core/Proxies/ProxyOptions.cs create mode 100644 src/AvParser.Core/Proxies/ProxyPool.cs create mode 100644 src/AvParser.Core/Proxies/Selection/IProxySelectionStrategy.cs create mode 100644 src/AvParser.Core/Proxies/Selection/RoundRobinProxySelection.cs create mode 100644 src/AvParser.Core/Proxies/Selection/StickyProxySelection.cs create mode 100644 src/AvParser.Core/Proxies/Selection/WeightedRandomProxySelection.cs create mode 100644 src/AvParser.Infrastructure/Proxies/CustomProxySource.cs create mode 100644 src/AvParser.Infrastructure/Proxies/HttpProxyProbe.cs create mode 100644 src/AvParser.Infrastructure/Proxies/ProxiedHttpClientFactory.cs create mode 100644 src/AvParser.Infrastructure/Proxies/ProxiflyProxySource.cs create mode 100644 src/AvParser.UI/ViewModels/ProxiesViewModel.cs create mode 100644 src/AvParser.UI/ViewModels/ProxyRowViewModel.cs create mode 100644 src/AvParser.UI/Views/ProxiesView.axaml create mode 100644 src/AvParser.UI/Views/ProxiesView.axaml.cs create mode 100644 tests/AvParser.Core.Tests/Proxies/ProxyEndpointTests.cs create mode 100644 tests/AvParser.Core.Tests/Proxies/ProxyPoolTests.cs create mode 100644 tests/AvParser.Core.Tests/Proxies/ProxySelectionTests.cs create mode 100644 tests/AvParser.Core.Tests/Proxies/ProxyTestDoubles.cs create mode 100644 tests/AvParser.Infrastructure.Tests/AvParser.Infrastructure.Tests.csproj create mode 100644 tests/AvParser.Infrastructure.Tests/CustomProxySourceTests.cs create mode 100644 tests/AvParser.Infrastructure.Tests/ProxiflyFeedTests.cs create mode 100644 tests/AvParser.Infrastructure.Tests/ProxyHandlerFactoryTests.cs create mode 100644 tests/AvParser.UI.HeadlessTests/ProxiesViewTests.cs create mode 100644 tests/AvParser.UI.Tests/Fakes/FakeProxies.cs create mode 100644 tests/AvParser.UI.Tests/ProxiesViewModelTests.cs diff --git a/AvParser.slnx b/AvParser.slnx index a7b276c..eb758da 100644 --- a/AvParser.slnx +++ b/AvParser.slnx @@ -23,6 +23,7 @@ + diff --git a/CLAUDE.md b/CLAUDE.md index b5e095b..60ac4ae 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -84,6 +84,29 @@ dotnet csharpier check . выбор конструктора контейнером зависит от порядка регистраций. - `[Reactive]` из `ReactiveUI.SourceGenerators` на partial-свойствах; класс — `partial`. +## Добавить источник прокси + +1. Реализовать `IProxySource` (или `IMutableProxySource`, если список редактируемый). +2. Зарегистрировать как `IProxySource` в `AddAvParserProxies()`. Порядок регистрации = порядок + слияния; свой список идёт последним, чтобы пользовательский адрес перебивал фидовый. + +`ProxyPool` при обновлении **переиспользует существующие `ProxyEntry`** по `Endpoint.Key` — иначе +перезагрузка списка стирала бы всю накопленную статистику, а публичные фиды переиздаются каждые +несколько минут. + +## Инварианты прокси-пула + +- **Доступность определяется карантином, а не `Health`.** `Health` — это «что видели в последний + раз». Если исключать всё, что когда-либо падало, окно карантина становится бессмысленным, а + прокси теряется навсегда после первой же осечки. Это уже был баг, его ловит + `A_failing_proxy_is_quarantined_and_comes_back_later`. +- **Проба не трогает `SuccessCount`/`FailureCount`.** Эти счётчики про реальные запросы; свип по + паре тысяч прокси перезаписал бы всё, на чём держится взвешенный выбор. Провалившаяся проба + выставляет карантин через `RecordProbe(..., quarantineOnFailure:)`. +- **Лиза без вердикта нейтральна.** Отменённая операция — не вина прокси; считать это отказом + значит карантинить здоровые прокси на каждый Cancel. +- **`Select` и `Next` — зарезервированные слова для CA1716.** Метод стратегии называется `Pick`. + ## Грабли, уже оплаченные - **Селектор типа в Avalonia матчит точный тип.** `UserControl.shell` не матчит `ShellView` @@ -110,6 +133,11 @@ dotnet csharpier check . резолвиться, даже если эту конфигурацию никто не собирает. Так тут проехал мёртвый `Avalonia.Diagnostics` (его нет под Avalonia 12): `dotnet build -c Release` работал, а голый `dotnet restore` падал. +- **`Execute()` завершился ≠ `IsExecuting` уже false.** Второе публикуется на выходном + планировщике. Тест, который сразу после `await` дёргает команду, закрытую по чужому + `IsExecuting`, будет мигать под нагрузкой — ждите `CanExecute`, а не предполагайте. +- **Проект VM-тестов не параллелится.** `ReactiveUiBootstrap` ставит глобальные планировщики + ReactiveUI, то есть тесты делят изменяемое состояние независимо от их желания. - **Инспектора в Avalonia 12 нет из коробки.** `Avalonia.Diagnostics` закончился на 11.3.x; DevTools живут отдельно (`AvaloniaUI.DiagnosticsSupport` + `.WithDeveloperTools()`), со своей установкой. Зависимость намеренно не добавлена. diff --git a/Directory.Packages.props b/Directory.Packages.props index 743dc8e..4f63d97 100644 --- a/Directory.Packages.props +++ b/Directory.Packages.props @@ -41,6 +41,7 @@ Include="Microsoft.Extensions.DependencyInjection.Abstractions" Version="$(MicrosoftExtensionsVersion)" /> + diff --git a/README.md b/README.md index 9f86207..a89a491 100644 --- a/README.md +++ b/README.md @@ -61,9 +61,10 @@ src/ ResponsiveLayout, дизайн-токены, навигация AvParser.Desktop WinExe-хост: Program.cs, App.axaml, composition root tests/ - AvParser.Core.Tests парсеры, реестр, отмена, прогресс - AvParser.UI.Tests ViewModel'и без Avalonia - AvParser.UI.HeadlessTests реальное дерево контролов через [AvaloniaFact] + AvParser.Core.Tests парсеры, реестр, отмена, прогресс, пул прокси и стратегии + AvParser.Infrastructure.Tests разбор фида прокси, локальный список, маппинг на WebProxy + AvParser.UI.Tests ViewModel'и без Avalonia + AvParser.UI.HeadlessTests реальное дерево контролов через [AvaloniaFact] ``` Ссылки идут строго в одну сторону: `Core ← Infrastructure ← UI ← Desktop`. @@ -102,6 +103,55 @@ Avalonia матчит **точный** тип, а `ShellView` наследует --- +## Прокси + +Пул прокси с ротацией — `AvParser.Core/Proxies`, источники и сетевая часть — +`AvParser.Infrastructure/Proxies`, управление — страница **Proxies**. + +Источники: + +- **[proxifly/free-proxy-list](https://github.com/proxifly/free-proxy-list)** — публичный список, + обновляется каждые 5 минут. Тянем сводный `all/data.json` через jsDelivr и фильтруем локально: + один условный запрос за весь список надёжнее четырёх по протоколам, которые могут разъехаться + между собой в момент публикации. Ответ кэшируется на 5 минут, недоступность фида не роняет + приложение — остаётся прошлый список. +- **Свой список** — `proxies.custom.json` рядом с настройками. Вставляется пачкой, по одной на + строку; поддерживаются `scheme://host:port`, голый `host:port` и `user:pass@`. Непонятые строки + не проглатываются молча, а называются в статусе. + +Ротация выбирается в настройках: + +| Стратегия | Поведение | Когда | +|---|---|---| +| Sticky | одна прокси, смена только по отказу | по умолчанию: не рвёт сессии и cookie | +| RoundRobin | новая на каждый запрос | размазывает рейт-лимиты, но ломает сессии | +| WeightedRandom | случайно, с весом по score и доле успехов | при сильном разбросе качества | + +Проверка живости — тоже настройка, два режима: **Pool** прогоняет весь список параллельно один +раз, **Lazy** проверяет прокси в момент выдачи и перескакивает на следующую. У бесплатных списков +рабочих обычно единицы процентов, поэтому без проверки парсер будет в основном ждать таймауты. + +Упавшая прокси уходит в карантин с экспоненциальным окном (30 с → 15 мин), но **не** удаляется +навсегда: бесплатные прокси постоянно мигают, и жёсткий бан терял бы их безвозвратно. + +Использование из кода: + +```csharp +var (http, lease) = await clientFactory.CreateFromPoolAsync(); +using (http) +using (lease) +{ + try { var response = await http.GetAsync(url); lease?.ReportSuccess(); } + catch { lease?.ReportFailure("request failed"); throw; } +} +``` + +Отчёт об исходе — не формальность: без него пул ничего не узнаёт о том, какие прокси работают. +Освобождение лизы без вердикта нейтрально — отменённая операция не вина прокси. + +SOCKS работает штатно: .NET понимает схемы `socks4/socks4a/socks5` в `WebProxy`. Учтите, что +proxifly-запись с `"protocol": "https"` — это всё равно HTTP-прокси с CONNECT, а не схема `https://`. + ## Дизайн-токены Все цвета, отступы, радиусы и типографика — в `Styles/Tokens.axaml`, с отдельными словарями diff --git a/src/AvParser.Core/Proxies/IProxyPool.cs b/src/AvParser.Core/Proxies/IProxyPool.cs new file mode 100644 index 0000000..2a1bdaf --- /dev/null +++ b/src/AvParser.Core/Proxies/IProxyPool.cs @@ -0,0 +1,89 @@ +namespace AvParser.Core.Proxies; + +/// The pool of known proxies and the thing that hands them out. +public interface IProxyPool +{ + /// Snapshot of every known proxy, feed and custom alike. + IReadOnlyList Entries { get; } + + /// Options currently in force. + ProxyOptions Options { get; } + + /// Raised after the set of entries or their health changes. + /// + /// A plain event rather than IObservable so the domain keeps no reactive dependency; + /// the UI layer bridges it to an observable where that is convenient. + /// + event EventHandler? Changed; + + /// Applies new options. Resets selection state when the rotation strategy changes. + void Configure(ProxyOptions options); + + /// Reloads from every source, preserving health statistics for addresses that survive. + /// Number of entries in the pool afterwards. + Task RefreshAsync(CancellationToken cancellationToken = default); + + /// + /// Takes a proxy out of the pool for one unit of work, or when nothing + /// usable is left. + /// + /// Report the outcome on the lease, otherwise the pool never learns anything. + Task AcquireAsync(CancellationToken cancellationToken = default); + + /// Probes every entry in parallel and updates their health. + /// How many answered. + Task SweepAsync(IProgress? progress = null, CancellationToken cancellationToken = default); +} + +/// +/// A proxy checked out of the pool. +/// +/// +/// Disposing without a verdict is deliberately neutral: an operation that was cancelled says +/// nothing about the proxy, and counting that as a failure would quarantine healthy entries +/// every time the user hits Cancel. +/// +public sealed class ProxyLease : IDisposable +{ + private readonly ProxyPool _pool; + private bool _reported; + + internal ProxyLease(ProxyPool pool, ProxyEntry entry) + { + _pool = pool; + Entry = entry; + } + + /// The pool entry backing this lease. + public ProxyEntry Entry { get; } + + /// The address to send traffic through. + public ProxyEndpoint Endpoint => Entry.Endpoint; + + /// Records that the work succeeded. + public void ReportSuccess(TimeSpan? latency = null) + { + if (_reported) + { + return; + } + + _reported = true; + _pool.ReportOutcome(Entry, success: true, latency, error: null); + } + + /// Records that the work failed, which may quarantine the proxy. + public void ReportFailure(string? error = null) + { + if (_reported) + { + return; + } + + _reported = true; + _pool.ReportOutcome(Entry, success: false, latency: null, error); + } + + /// + public void Dispose() => _reported = true; +} diff --git a/src/AvParser.Core/Proxies/IProxySource.cs b/src/AvParser.Core/Proxies/IProxySource.cs new file mode 100644 index 0000000..879b766 --- /dev/null +++ b/src/AvParser.Core/Proxies/IProxySource.cs @@ -0,0 +1,65 @@ +namespace AvParser.Core.Proxies; + +/// Supplies proxy addresses. One per list the app knows about. +public interface IProxySource +{ + /// Stable identifier used in settings and logs. + string Id { get; } + + /// Human-readable name for the UI. + string DisplayName { get; } + + /// Whether entries from this source are feed-provided or user-entered. + ProxySourceKind Kind { get; } + + /// Fetches the current list. Implementations may cache. + Task> GetProxiesAsync(CancellationToken cancellationToken = default); +} + +/// A source the user can edit. +public interface IMutableProxySource : IProxySource +{ + /// Adds addresses, ignoring duplicates. Returns how many were actually new. + Task AddAsync(IEnumerable endpoints, CancellationToken cancellationToken = default); + + /// Removes an address. Returns whether it was present. + Task RemoveAsync(ProxyEndpoint endpoint, CancellationToken cancellationToken = default); + + /// Removes every address. + Task ClearAsync(CancellationToken cancellationToken = default); +} + +/// Outcome of a single liveness probe. +/// Whether the proxy answered acceptably. +/// Round-trip time when alive. +/// Short failure reason when not alive. +public readonly record struct ProxyProbeResult(bool Alive, TimeSpan? Latency, string? Error) +{ + /// A successful probe. + public static ProxyProbeResult Success(TimeSpan latency) => new(true, latency, null); + + /// A failed probe. + public static ProxyProbeResult Failure(string error) => new(false, null, error); +} + +/// Checks whether a proxy actually works. +public interface IProxyProbe +{ + /// Sends one request through and reports the outcome. + /// Must not throw for an unreachable proxy — that is a , not an error. + Task ProbeAsync( + ProxyEndpoint endpoint, + ProxyOptions options, + CancellationToken cancellationToken = default + ); +} + +/// Progress of a pool-wide probe sweep. +/// Proxies probed so far. +/// Proxies in the sweep. +/// How many answered. +public readonly record struct ProxySweepProgress(int Checked, int Total, int Alive) +{ + /// Completion in the range 0..1. + public double Fraction => Total <= 0 ? 0d : Math.Clamp((double)Checked / Total, 0d, 1d); +} diff --git a/src/AvParser.Core/Proxies/ProxyEndpoint.cs b/src/AvParser.Core/Proxies/ProxyEndpoint.cs new file mode 100644 index 0000000..9b3c428 --- /dev/null +++ b/src/AvParser.Core/Proxies/ProxyEndpoint.cs @@ -0,0 +1,210 @@ +using System.Diagnostics.CodeAnalysis; +using System.Globalization; + +namespace AvParser.Core.Proxies; + +/// Wire protocol a proxy speaks. +public enum ProxyProtocol +{ + /// Plain HTTP proxy. + Http, + + /// HTTP proxy that also handles CONNECT for TLS. + Https, + + /// SOCKS4. + Socks4, + + /// SOCKS5. + Socks5, +} + +/// How much of the caller the proxy passes through. +public enum ProxyAnonymity +{ + /// Not reported by the source. + Unknown, + + /// Forwards the original address — offers no anonymity at all. + Transparent, + + /// Hides the original address but announces itself as a proxy. + Anonymous, + + /// Neither forwards the address nor announces itself. + Elite, +} + +/// Where an entry came from. +public enum ProxySourceKind +{ + /// Downloaded from a remote list. + Feed, + + /// Entered by the user and stored locally. + Custom, +} + +/// A single proxy address, with whatever metadata its source supplied. +/// Wire protocol. +/// Hostname or IP literal. +/// TCP port. +public sealed record ProxyEndpoint(ProxyProtocol Protocol, string Host, int Port) +{ + /// ISO country code reported by the source, if any. + public string? Country { get; init; } + + /// City reported by the source, if any. + public string? City { get; init; } + + /// Anonymity level reported by the source. + public ProxyAnonymity Anonymity { get; init; } = ProxyAnonymity.Unknown; + + /// Quality score reported by the source; higher is better. 0 when unknown. + public int Score { get; init; } + + /// Username for proxies that need authentication. + public string? Username { get; init; } + + /// Password for proxies that need authentication. + public string? Password { get; init; } + + /// Scheme as System.Net.WebProxy expects it. + public string Scheme => + Protocol switch + { + ProxyProtocol.Socks4 => "socks4", + ProxyProtocol.Socks5 => "socks5", + // .NET has no "https" proxy scheme: an HTTPS-capable proxy is still reached over + // http:// and tunnels TLS with CONNECT. + _ => "http", + }; + + /// Address in scheme://host:port form. + public Uri Uri => new($"{Scheme}://{Host}:{Port.ToString(CultureInfo.InvariantCulture)}"); + + /// Stable identity: two entries for the same address are the same proxy. + public string Key => $"{Protocol}://{Host}:{Port.ToString(CultureInfo.InvariantCulture)}".ToLowerInvariant(); + + /// Whether credentials were supplied. + public bool HasCredentials => !string.IsNullOrEmpty(Username); + + /// + public override string ToString() => + $"{Protocol.ToString().ToLowerInvariant()}://{Host}:{Port.ToString(CultureInfo.InvariantCulture)}"; + + /// + /// Parses [scheme://][user:pass@]host:port. Missing scheme is treated as HTTP. + /// + /// + /// Hand-rolled rather than delegating to : the socks schemes and the + /// bare host:port form that every proxy list uses are not valid absolute URIs. + /// + public static bool TryParse(string? text, [NotNullWhen(true)] out ProxyEndpoint? endpoint) + { + endpoint = null; + + if (string.IsNullOrWhiteSpace(text)) + { + return false; + } + + var value = text.Trim(); + var protocol = ProxyProtocol.Http; + + var schemeEnd = value.IndexOf("://", StringComparison.Ordinal); + if (schemeEnd >= 0) + { + if (!TryParseProtocol(value[..schemeEnd], out protocol)) + { + return false; + } + + value = value[(schemeEnd + 3)..]; + } + + string? username = null; + string? password = null; + + // Rightmost '@' wins: a password may legitimately contain one. + var credentialsEnd = value.LastIndexOf('@'); + if (credentialsEnd >= 0) + { + var credentials = value[..credentialsEnd]; + value = value[(credentialsEnd + 1)..]; + + var separator = credentials.IndexOf(':', StringComparison.Ordinal); + if (separator < 0) + { + username = credentials; + } + else + { + username = credentials[..separator]; + password = credentials[(separator + 1)..]; + } + + if (username.Length == 0) + { + return false; + } + } + + var portStart = value.LastIndexOf(':'); + if (portStart <= 0 || portStart == value.Length - 1) + { + return false; + } + + var host = value[..portStart].Trim(); + var portText = value[(portStart + 1)..].Trim(); + + if ( + host.Length == 0 + || !int.TryParse(portText, NumberStyles.None, CultureInfo.InvariantCulture, out var port) + || port is < 1 or > 65535 + ) + { + return false; + } + + endpoint = new ProxyEndpoint(protocol, host, port) { Username = username, Password = password }; + return true; + } + + /// Parses a protocol name; accepts the spellings the public lists use. + public static bool TryParseProtocol(string? text, out ProxyProtocol protocol) + { + switch (text?.Trim().ToLowerInvariant()) + { + case "http": + protocol = ProxyProtocol.Http; + return true; + case "https": + case "ssl": + protocol = ProxyProtocol.Https; + return true; + case "socks4": + case "socks4a": + protocol = ProxyProtocol.Socks4; + return true; + case "socks5": + case "socks5h": + protocol = ProxyProtocol.Socks5; + return true; + default: + protocol = ProxyProtocol.Http; + return false; + } + } + + /// Parses an anonymity level; unrecognised values become . + public static ProxyAnonymity ParseAnonymity(string? text) => + text?.Trim().ToLowerInvariant() switch + { + "transparent" => ProxyAnonymity.Transparent, + "anonymous" => ProxyAnonymity.Anonymous, + "elite" or "high" => ProxyAnonymity.Elite, + _ => ProxyAnonymity.Unknown, + }; +} diff --git a/src/AvParser.Core/Proxies/ProxyEntry.cs b/src/AvParser.Core/Proxies/ProxyEntry.cs new file mode 100644 index 0000000..d534d20 --- /dev/null +++ b/src/AvParser.Core/Proxies/ProxyEntry.cs @@ -0,0 +1,186 @@ +namespace AvParser.Core.Proxies; + +/// What the last check or use said about a proxy. +public enum ProxyHealthState +{ + /// Never checked and never used. + Unknown, + + /// A probe or a real request succeeded. + Alive, + + /// A probe or a real request failed. + Dead, +} + +/// +/// A proxy plus everything the pool has learned about it. +/// +/// +/// Mutable and guarded by 's lock rather than being a record: the pool +/// updates counters on every request, and reallocating an immutable entry per outcome would +/// churn hard on a list of a few thousand proxies. +/// +public sealed class ProxyEntry +{ + private readonly Lock _gate = new(); + + /// Creates an entry in the state. + public ProxyEntry(ProxyEndpoint endpoint, ProxySourceKind source) + { + Endpoint = endpoint ?? throw new ArgumentNullException(nameof(endpoint)); + Source = source; + } + + /// The address. + public ProxyEndpoint Endpoint { get; } + + /// Whether this came from a feed or from the user. + public ProxySourceKind Source { get; } + + /// Latest known state. + public ProxyHealthState Health { get; private set; } + + /// Round-trip time of the last successful probe or request. + public TimeSpan? Latency { get; private set; } + + /// When the state was last updated. + public DateTimeOffset? LastCheckedUtc { get; private set; } + + /// Successful uses since the entry was created. + public int SuccessCount { get; private set; } + + /// Failed uses since the entry was created. + public int FailureCount { get; private set; } + + /// Failures since the last success. Drives the quarantine backoff. + public int ConsecutiveFailures { get; private set; } + + /// While set and in the future, the entry is skipped by selection. + public DateTimeOffset? QuarantinedUntilUtc { get; private set; } + + /// Reason recorded with the last failure, for the UI. + public string? LastError { get; private set; } + + /// Share of successful uses, 0..1. Returns 0.5 before any evidence exists. + public double SuccessRate + { + get + { + var total = SuccessCount + FailureCount; + return total == 0 ? 0.5d : (double)SuccessCount / total; + } + } + + /// Whether selection may hand this entry out at . + /// + /// Governed by the quarantine alone, deliberately not by . Health is the + /// last thing observed; excluding every entry that has ever failed would make the quarantine + /// window meaningless and would permanently discard proxies on their first hiccup — and free + /// proxies flap constantly. Health still decides ordering, so dead entries sink to the back. + /// + public bool IsAvailable(DateTimeOffset now) => QuarantinedUntilUtc is null || QuarantinedUntilUtc <= now; + + /// Whether the entry is currently serving a quarantine. + public bool IsQuarantined(DateTimeOffset now) => QuarantinedUntilUtc is { } until && until > now; + + /// Records a successful probe or request. + public void RecordSuccess(DateTimeOffset now, TimeSpan? latency = null) + { + lock (_gate) + { + Health = ProxyHealthState.Alive; + SuccessCount++; + ConsecutiveFailures = 0; + QuarantinedUntilUtc = null; + LastError = null; + LastCheckedUtc = now; + + if (latency is not null) + { + Latency = latency; + } + } + } + + /// + /// Records a failure and, once pile up, sidelines + /// the entry for an exponentially growing window capped at . + /// + /// + /// Exponential rather than fixed: a proxy that fails once may just have hit a flaky moment, + /// while one that has failed six times in a row should not be retried every few seconds for + /// the rest of the session. + /// + public void RecordFailure( + DateTimeOffset now, + TimeSpan baseQuarantine, + TimeSpan maxQuarantine, + int failuresBeforeQuarantine, + string? error = null + ) + { + lock (_gate) + { + FailureCount++; + ConsecutiveFailures++; + LastCheckedUtc = now; + LastError = error; + Health = ProxyHealthState.Dead; + + if (ConsecutiveFailures < failuresBeforeQuarantine) + { + return; + } + + var exponent = Math.Min(ConsecutiveFailures - failuresBeforeQuarantine, 16); + var ticks = baseQuarantine.Ticks * Math.Pow(2, exponent); + var window = ticks >= maxQuarantine.Ticks ? maxQuarantine : TimeSpan.FromTicks((long)ticks); + + QuarantinedUntilUtc = now + window; + } + } + + /// Records the outcome of a liveness probe. + /// Current time. + /// Whether the probe succeeded. + /// Round-trip time when alive. + /// Failure reason when not alive. + /// + /// How long to sideline the entry if the probe failed. Leave null to only record the state. + /// + /// + /// Does not touch or : those track real + /// requests, and letting a sweep of a few thousand proxies rewrite them would drown the + /// evidence that weighted selection depends on. + /// + public void RecordProbe( + DateTimeOffset now, + bool alive, + TimeSpan? latency, + string? error, + TimeSpan? quarantineOnFailure = null + ) + { + lock (_gate) + { + Health = alive ? ProxyHealthState.Alive : ProxyHealthState.Dead; + LastCheckedUtc = now; + Latency = alive ? latency : null; + LastError = alive ? null : error; + + if (alive) + { + ConsecutiveFailures = 0; + QuarantinedUntilUtc = null; + } + else if (quarantineOnFailure is { } window) + { + QuarantinedUntilUtc = now + window; + } + } + } + + /// + public override string ToString() => $"{Endpoint} [{Health}]"; +} diff --git a/src/AvParser.Core/Proxies/ProxyOptions.cs b/src/AvParser.Core/Proxies/ProxyOptions.cs new file mode 100644 index 0000000..a892348 --- /dev/null +++ b/src/AvParser.Core/Proxies/ProxyOptions.cs @@ -0,0 +1,129 @@ +namespace AvParser.Core.Proxies; + +/// How the pool picks the next proxy. +public enum ProxyRotation +{ + /// Keep one proxy until it fails. Least disruptive to session cookies. + Sticky, + + /// Advance through the pool on every acquisition. Spreads rate limits. + RoundRobin, + + /// Pick at random, weighted by feed score and observed success rate. + WeightedRandom, +} + +/// When liveness is verified. +public enum ProxyHealthCheck +{ + /// + /// Probe the whole pool up front, in parallel. Costs one sweep, then hands out proxies with + /// no extra latency — the right default when most of a free list is dead. + /// + Pool, + + /// + /// Probe a single proxy at the moment it is handed out, skipping to the next if it fails. + /// No sweep, but every acquisition pays a round trip. + /// + Lazy, +} + +/// Protocols to accept when loading sources. +/// +/// A flags enum rather than a list so that and the persisted settings +/// keep value equality — a record holding a collection compares by reference, which would make +/// every "did anything change?" check say yes. +/// +[Flags] +public enum ProxyProtocolFilter +{ + /// No filter — accept everything. + None = 0, + + /// Plain HTTP proxies. + Http = 1, + + /// HTTPS-capable HTTP proxies. + Https = 2, + + /// SOCKS4. + Socks4 = 4, + + /// SOCKS5. + Socks5 = 8, + + /// Every protocol. + All = Http | Https | Socks4 | Socks5, +} + +/// Tuning for . Mirrors what the Settings page exposes. +public sealed record ProxyOptions +{ + /// Selection strategy. + public ProxyRotation Rotation { get; init; } = ProxyRotation.Sticky; + + /// Liveness policy. + public ProxyHealthCheck HealthCheck { get; init; } = ProxyHealthCheck.Pool; + + /// Protocols to keep when loading sources. + public ProxyProtocolFilter Protocols { get; init; } = ProxyProtocolFilter.All; + + /// ISO country codes to keep. Empty means "all". + public IReadOnlyList Countries { get; init; } = []; + + /// URL fetched to decide whether a proxy works. + /// + /// Defaults to a plain-HTTP 204 endpoint: it is tiny, and requiring TLS would fail every + /// proxy that cannot do CONNECT rather than every proxy that is actually dead. + /// + public Uri ProbeUrl { get; init; } = new("http://www.gstatic.com/generate_204"); + + /// Per-proxy probe timeout. + public TimeSpan ProbeTimeout { get; init; } = TimeSpan.FromSeconds(8); + + /// How many probes run at once during a pool sweep. + public int ProbeConcurrency { get; init; } = 64; + + /// Consecutive failures tolerated before an entry is quarantined. + public int FailuresBeforeQuarantine { get; init; } = 2; + + /// First quarantine window; doubles with each further consecutive failure. + public TimeSpan BaseQuarantine { get; init; } = TimeSpan.FromSeconds(30); + + /// Ceiling for the quarantine window. + public TimeSpan MaxQuarantine { get; init; } = TimeSpan.FromMinutes(15); + + /// + /// Proxies tried per call under + /// before giving up. + /// + public int LazyProbeAttempts { get; init; } = 5; + + /// Whether the remote feed is consulted at all. + public bool UseFeed { get; init; } = true; + + /// Validates the options, throwing on values that would misbehave silently. + /// A numeric option is out of range. + public ProxyOptions Validated() + { + ArgumentOutOfRangeException.ThrowIfLessThan(ProbeConcurrency, 1); + ArgumentOutOfRangeException.ThrowIfLessThan(LazyProbeAttempts, 1); + ArgumentOutOfRangeException.ThrowIfLessThan(FailuresBeforeQuarantine, 1); + ArgumentOutOfRangeException.ThrowIfLessThanOrEqual(ProbeTimeout, TimeSpan.Zero); + ArgumentOutOfRangeException.ThrowIfGreaterThan(BaseQuarantine, MaxQuarantine); + + return this; + } + + /// Maps a protocol onto its filter flag. + public static ProxyProtocolFilter ToFlag(ProxyProtocol protocol) => + protocol switch + { + ProxyProtocol.Http => ProxyProtocolFilter.Http, + ProxyProtocol.Https => ProxyProtocolFilter.Https, + ProxyProtocol.Socks4 => ProxyProtocolFilter.Socks4, + ProxyProtocol.Socks5 => ProxyProtocolFilter.Socks5, + _ => ProxyProtocolFilter.None, + }; +} diff --git a/src/AvParser.Core/Proxies/ProxyPool.cs b/src/AvParser.Core/Proxies/ProxyPool.cs new file mode 100644 index 0000000..7fca817 --- /dev/null +++ b/src/AvParser.Core/Proxies/ProxyPool.cs @@ -0,0 +1,340 @@ +using AvParser.Core.Proxies.Selection; + +namespace AvParser.Core.Proxies; + +/// +public sealed class ProxyPool : IProxyPool +{ + private readonly IReadOnlyList _sources; + private readonly IProxyProbe _probe; + private readonly TimeProvider _time; + private readonly Random? _random; + private readonly Lock _gate = new(); + + private readonly Dictionary _byKey = new(StringComparer.Ordinal); + private List _entries = []; + private IProxySelectionStrategy _strategy; + private ProxyOptions _options; + + /// Creates a pool over the given sources. + /// Every list the app knows about; order decides nothing. + /// Liveness checker. + /// Initial options. + /// Clock; tests inject a fake one to exercise quarantine expiry. + /// Randomness for weighted selection; tests seed it. + public ProxyPool( + IEnumerable sources, + IProxyProbe probe, + ProxyOptions? options = null, + TimeProvider? timeProvider = null, + Random? random = null + ) + { + ArgumentNullException.ThrowIfNull(sources); + + _sources = sources.ToArray(); + _probe = probe ?? throw new ArgumentNullException(nameof(probe)); + _time = timeProvider ?? TimeProvider.System; + _random = random; + _options = (options ?? new ProxyOptions()).Validated(); + _strategy = ProxySelectionStrategyFactory.Create(_options.Rotation, random); + } + + /// + public IReadOnlyList Entries + { + get + { + lock (_gate) + { + return _entries; + } + } + } + + /// + public ProxyOptions Options + { + get + { + lock (_gate) + { + return _options; + } + } + } + + /// + public event EventHandler? Changed; + + /// + public void Configure(ProxyOptions options) + { + ArgumentNullException.ThrowIfNull(options); + + var validated = options.Validated(); + + lock (_gate) + { + var rotationChanged = validated.Rotation != _options.Rotation; + _options = validated; + + if (rotationChanged) + { + _strategy = ProxySelectionStrategyFactory.Create(validated.Rotation, _random); + } + } + + RaiseChanged(); + } + + /// + public async Task RefreshAsync(CancellationToken cancellationToken = default) + { + var options = Options; + var collected = new List<(ProxyEndpoint Endpoint, ProxySourceKind Kind)>(); + + foreach (var source in _sources) + { + if (source.Kind == ProxySourceKind.Feed && !options.UseFeed) + { + continue; + } + + var proxies = await source.GetProxiesAsync(cancellationToken).ConfigureAwait(false); + foreach (var endpoint in proxies) + { + collected.Add((endpoint, source.Kind)); + } + } + + int count; + lock (_gate) + { + var next = new List(collected.Count); + var seen = new HashSet(StringComparer.Ordinal); + + foreach (var (endpoint, kind) in collected) + { + if (!Matches(endpoint, options) || !seen.Add(endpoint.Key)) + { + continue; + } + + // Reuse the existing entry so a refresh does not wipe out everything the pool + // has learned — free lists are republished every few minutes. + if (_byKey.TryGetValue(endpoint.Key, out var existing)) + { + next.Add(existing); + } + else + { + var created = new ProxyEntry(endpoint, kind); + _byKey[endpoint.Key] = created; + next.Add(created); + } + } + + foreach (var key in _byKey.Keys.Where(key => !seen.Contains(key)).ToArray()) + { + _byKey.Remove(key); + } + + _entries = next; + _strategy.Reset(); + count = next.Count; + } + + RaiseChanged(); + return count; + } + + /// + public async Task AcquireAsync(CancellationToken cancellationToken = default) + { + var options = Options; + + if (options.HealthCheck == ProxyHealthCheck.Pool) + { + var entry = SelectAvailable(); + return entry is null ? null : new ProxyLease(this, entry); + } + + // Lazy: verify the pick before handing it over, stepping past dead ones. + for (var attempt = 0; attempt < options.LazyProbeAttempts; attempt++) + { + cancellationToken.ThrowIfCancellationRequested(); + + var entry = SelectAvailable(); + if (entry is null) + { + return null; + } + + if (entry.Health == ProxyHealthState.Alive) + { + return new ProxyLease(this, entry); + } + + var result = await _probe.ProbeAsync(entry.Endpoint, options, cancellationToken).ConfigureAwait(false); + var now = _time.GetUtcNow(); + + if (result.Alive) + { + entry.RecordProbe(now, alive: true, result.Latency, error: null); + RaiseChanged(); + return new ProxyLease(this, entry); + } + + // Same reasoning as the sweep: this is a probe verdict, so it sidelines the entry + // without inflating the request counters that weighted selection reads. + entry.RecordProbe(now, alive: false, latency: null, result.Error, options.BaseQuarantine); + + lock (_gate) + { + _strategy.Report(entry, success: false); + } + + RaiseChanged(); + } + + return null; + } + + /// + public async Task SweepAsync( + IProgress? progress = null, + CancellationToken cancellationToken = default + ) + { + var options = Options; + var targets = Entries; + var total = targets.Count; + + if (total == 0) + { + progress?.Report(new ProxySweepProgress(0, 0, 0)); + return 0; + } + + using var limiter = new SemaphoreSlim(options.ProbeConcurrency, options.ProbeConcurrency); + var checkedCount = 0; + var aliveCount = 0; + + var work = targets.Select(async entry => + { + await limiter.WaitAsync(cancellationToken).ConfigureAwait(false); + try + { + var result = await _probe.ProbeAsync(entry.Endpoint, options, cancellationToken).ConfigureAwait(false); + + // A failed sweep probe sidelines the entry for one base window rather than + // counting as a request failure: it is evidence about the proxy, not usage of it. + entry.RecordProbe( + _time.GetUtcNow(), + result.Alive, + result.Latency, + result.Error, + quarantineOnFailure: options.BaseQuarantine + ); + + if (result.Alive) + { + Interlocked.Increment(ref aliveCount); + } + } + finally + { + limiter.Release(); + progress?.Report( + new ProxySweepProgress( + Interlocked.Increment(ref checkedCount), + total, + Volatile.Read(ref aliveCount) + ) + ); + } + }); + + await Task.WhenAll(work).ConfigureAwait(false); + + lock (_gate) + { + // A sweep can invalidate a sticky pick, so make the strategy re-choose. + _strategy.Reset(); + } + + RaiseChanged(); + return aliveCount; + } + + /// Records the outcome of a lease. Called by . + internal void ReportOutcome(ProxyEntry entry, bool success, TimeSpan? latency, string? error) + { + var options = Options; + var now = _time.GetUtcNow(); + + if (success) + { + entry.RecordSuccess(now, latency); + } + else + { + entry.RecordFailure( + now, + options.BaseQuarantine, + options.MaxQuarantine, + options.FailuresBeforeQuarantine, + error + ); + } + + lock (_gate) + { + _strategy.Report(entry, success); + } + + RaiseChanged(); + } + + /// Whether an endpoint passes the protocol and country filters. + public static bool Matches(ProxyEndpoint endpoint, ProxyOptions options) + { + ArgumentNullException.ThrowIfNull(endpoint); + ArgumentNullException.ThrowIfNull(options); + + if ( + options.Protocols != ProxyProtocolFilter.None + && !options.Protocols.HasFlag(ProxyOptions.ToFlag(endpoint.Protocol)) + ) + { + return false; + } + + return options.Countries.Count == 0 + || ( + endpoint.Country is { } country && options.Countries.Contains(country, StringComparer.OrdinalIgnoreCase) + ); + } + + private ProxyEntry? SelectAvailable() + { + lock (_gate) + { + var now = _time.GetUtcNow(); + + // Best-first ordering: sticky selection takes the head of this list, and the other + // strategies benefit from healthy proxies being clustered at the front. + var candidates = _entries + .Where(entry => entry.IsAvailable(now)) + .OrderByDescending(entry => entry.Health == ProxyHealthState.Alive) + .ThenBy(entry => entry.Latency ?? TimeSpan.MaxValue) + .ThenByDescending(entry => entry.Endpoint.Score) + .ToArray(); + + return _strategy.Pick(candidates); + } + } + + private void RaiseChanged() => Changed?.Invoke(this, EventArgs.Empty); +} diff --git a/src/AvParser.Core/Proxies/Selection/IProxySelectionStrategy.cs b/src/AvParser.Core/Proxies/Selection/IProxySelectionStrategy.cs new file mode 100644 index 0000000..225313d --- /dev/null +++ b/src/AvParser.Core/Proxies/Selection/IProxySelectionStrategy.cs @@ -0,0 +1,38 @@ +namespace AvParser.Core.Proxies.Selection; + +/// Decides which proxy to hand out next. +/// +/// Stateful by design — sticky selection has to remember its pick, and round-robin its cursor. +/// Implementations are called under the pool's lock and need no locking of their own. +/// +public interface IProxySelectionStrategy +{ + /// Which setting this strategy implements. + ProxyRotation Kind { get; } + + /// Picks from the already-filtered available candidates, or if none. + ProxyEntry? Pick(IReadOnlyList candidates); + + /// Tells the strategy how the handed-out proxy fared. + void Report(ProxyEntry entry, bool success); + + /// Drops any remembered state, e.g. after the pool is reloaded. + void Reset(); +} + +/// Builds the strategy named by . +public static class ProxySelectionStrategyFactory +{ + /// Creates a strategy instance. + /// Which strategy to build. + /// Randomness for ; tests pass a seeded instance. + /// Unknown rotation value. + public static IProxySelectionStrategy Create(ProxyRotation rotation, Random? random = null) => + rotation switch + { + ProxyRotation.Sticky => new StickyProxySelection(), + ProxyRotation.RoundRobin => new RoundRobinProxySelection(), + ProxyRotation.WeightedRandom => new WeightedRandomProxySelection(random), + _ => throw new ArgumentOutOfRangeException(nameof(rotation), rotation, "Unknown rotation strategy."), + }; +} diff --git a/src/AvParser.Core/Proxies/Selection/RoundRobinProxySelection.cs b/src/AvParser.Core/Proxies/Selection/RoundRobinProxySelection.cs new file mode 100644 index 0000000..e4ff441 --- /dev/null +++ b/src/AvParser.Core/Proxies/Selection/RoundRobinProxySelection.cs @@ -0,0 +1,43 @@ +namespace AvParser.Core.Proxies.Selection; + +/// +/// Advances one position through the candidates on every acquisition. +/// +/// +/// Spreads requests evenly, which is what rate limits care about. The cursor is kept as a +/// monotonic counter reduced modulo the candidate count rather than as an index into the list, +/// so a list that shrinks between calls cannot throw or silently skip entries. +/// +public sealed class RoundRobinProxySelection : IProxySelectionStrategy +{ + private int _cursor; + + /// + public ProxyRotation Kind => ProxyRotation.RoundRobin; + + /// + public ProxyEntry? Pick(IReadOnlyList candidates) + { + ArgumentNullException.ThrowIfNull(candidates); + + if (candidates.Count == 0) + { + return null; + } + + var index = (int)((uint)_cursor % (uint)candidates.Count); + _cursor = _cursor == int.MaxValue ? 0 : _cursor + 1; + + return candidates[index]; + } + + /// + public void Report(ProxyEntry entry, bool success) + { + // Position is independent of outcome: a failing proxy is dropped by the pool's own + // quarantine, not by rewinding the cursor. + } + + /// + public void Reset() => _cursor = 0; +} diff --git a/src/AvParser.Core/Proxies/Selection/StickyProxySelection.cs b/src/AvParser.Core/Proxies/Selection/StickyProxySelection.cs new file mode 100644 index 0000000..c7db604 --- /dev/null +++ b/src/AvParser.Core/Proxies/Selection/StickyProxySelection.cs @@ -0,0 +1,45 @@ +namespace AvParser.Core.Proxies.Selection; + +/// +/// Holds one proxy until it fails. +/// +/// +/// The default because it is the only strategy that keeps a site's session coherent: rotating on +/// every request changes the apparent client mid-session, which reliably triggers re-logins and +/// captchas on anything that tracks cookies. +/// +public sealed class StickyProxySelection : IProxySelectionStrategy +{ + private ProxyEntry? _current; + + /// + public ProxyRotation Kind => ProxyRotation.Sticky; + + /// + public ProxyEntry? Pick(IReadOnlyList candidates) + { + ArgumentNullException.ThrowIfNull(candidates); + + // Keep the current pick only while it is still among the available candidates: a refresh + // or a quarantine may have taken it out from under us. + if (_current is not null && candidates.Contains(_current)) + { + return _current; + } + + _current = candidates.Count > 0 ? candidates[0] : null; + return _current; + } + + /// + public void Report(ProxyEntry entry, bool success) + { + if (!success && ReferenceEquals(entry, _current)) + { + _current = null; + } + } + + /// + public void Reset() => _current = null; +} diff --git a/src/AvParser.Core/Proxies/Selection/WeightedRandomProxySelection.cs b/src/AvParser.Core/Proxies/Selection/WeightedRandomProxySelection.cs new file mode 100644 index 0000000..0d95a2d --- /dev/null +++ b/src/AvParser.Core/Proxies/Selection/WeightedRandomProxySelection.cs @@ -0,0 +1,83 @@ +namespace AvParser.Core.Proxies.Selection; + +/// +/// Picks at random, biased towards proxies that have actually worked. +/// +/// +/// Weight is the feed's score multiplied by the observed success rate, so a proxy the feed likes +/// but that keeps failing here drifts to the bottom without ever being excluded outright — free +/// lists recover, and a hard ban would lose them permanently. +/// +public sealed class WeightedRandomProxySelection(Random? random = null) : IProxySelectionStrategy +{ + /// Floor on the weight so an unproven proxy still gets picked occasionally. + private const double MinimumWeight = 0.05d; + + private readonly Random _random = random ?? Random.Shared; + + /// + public ProxyRotation Kind => ProxyRotation.WeightedRandom; + + /// + public ProxyEntry? Pick(IReadOnlyList candidates) + { + ArgumentNullException.ThrowIfNull(candidates); + + if (candidates.Count == 0) + { + return null; + } + + if (candidates.Count == 1) + { + return candidates[0]; + } + + var weights = new double[candidates.Count]; + var total = 0d; + + for (var i = 0; i < candidates.Count; i++) + { + var weight = WeightOf(candidates[i]); + weights[i] = weight; + total += weight; + } + + var target = _random.NextDouble() * total; + var running = 0d; + + for (var i = 0; i < weights.Length; i++) + { + running += weights[i]; + if (target < running) + { + return candidates[i]; + } + } + + // Floating-point drift can leave `target` a hair past the final boundary. + return candidates[^1]; + } + + /// + public void Report(ProxyEntry entry, bool success) + { + // The weight is derived from the entry's own counters, which the pool already updated. + } + + /// + public void Reset() + { + // Stateless beyond the RNG. + } + + /// Weight of a single entry. Exposed so the tests can assert the ordering. + public static double WeightOf(ProxyEntry entry) + { + ArgumentNullException.ThrowIfNull(entry); + + // Feed scores are small integers and frequently zero; +1 keeps an unscored proxy in play. + var score = Math.Max(0, entry.Endpoint.Score) + 1d; + return Math.Max(MinimumWeight, score * entry.SuccessRate); + } +} diff --git a/src/AvParser.Core/Settings/AppSettings.cs b/src/AvParser.Core/Settings/AppSettings.cs index 1be5300..39227ac 100644 --- a/src/AvParser.Core/Settings/AppSettings.cs +++ b/src/AvParser.Core/Settings/AppSettings.cs @@ -1,3 +1,5 @@ +using AvParser.Core.Proxies; + namespace AvParser.Core.Settings; /// Theme preference. follows the OS setting. @@ -39,4 +41,48 @@ public sealed record AppSettings /// Minimum Serilog level, as a Serilog level name. public string MinimumLogLevel { get; init; } = "Information"; + + /// How the pool picks the next proxy. + public ProxyRotation ProxyRotation { get; init; } = ProxyRotation.Sticky; + + /// When proxy liveness is verified. + public ProxyHealthCheck ProxyHealthCheck { get; init; } = ProxyHealthCheck.Pool; + + /// Whether the remote proxy feed is consulted. + public bool ProxyUseFeed { get; init; } = true; + + /// Protocols accepted when loading proxy sources. + public ProxyProtocolFilter ProxyProtocols { get; init; } = ProxyProtocolFilter.All; + + /// URL fetched to decide whether a proxy works. + public string ProxyProbeUrl { get; init; } = "http://www.gstatic.com/generate_204"; + + /// Per-proxy probe timeout, in seconds. + public int ProxyProbeTimeoutSeconds { get; init; } = 8; + + /// How many probes run at once during a pool sweep. + public int ProxyProbeConcurrency { get; init; } = 64; + + /// Projects the proxy-related settings onto . + /// + /// Settings are persisted as primitives so an old file still deserialises; the pool wants a + /// validated options object. This is the single place that bridges the two. + /// + public ProxyOptions ToProxyOptions() + { + var probeUrl = Uri.TryCreate(ProxyProbeUrl, UriKind.Absolute, out var parsed) + ? parsed + : new ProxyOptions().ProbeUrl; + + return new ProxyOptions + { + Rotation = ProxyRotation, + HealthCheck = ProxyHealthCheck, + UseFeed = ProxyUseFeed, + Protocols = ProxyProtocols, + ProbeUrl = probeUrl, + ProbeTimeout = TimeSpan.FromSeconds(Math.Clamp(ProxyProbeTimeoutSeconds, 1, 120)), + ProbeConcurrency = Math.Clamp(ProxyProbeConcurrency, 1, 512), + }.Validated(); + } } diff --git a/src/AvParser.Infrastructure/AvParser.Infrastructure.csproj b/src/AvParser.Infrastructure/AvParser.Infrastructure.csproj index 2ea6d1b..2414d0d 100644 --- a/src/AvParser.Infrastructure/AvParser.Infrastructure.csproj +++ b/src/AvParser.Infrastructure/AvParser.Infrastructure.csproj @@ -7,8 +7,15 @@ + + + + + + diff --git a/src/AvParser.Infrastructure/DependencyInjection/InfrastructureServiceCollectionExtensions.cs b/src/AvParser.Infrastructure/DependencyInjection/InfrastructureServiceCollectionExtensions.cs index b1fba01..926851d 100644 --- a/src/AvParser.Infrastructure/DependencyInjection/InfrastructureServiceCollectionExtensions.cs +++ b/src/AvParser.Infrastructure/DependencyInjection/InfrastructureServiceCollectionExtensions.cs @@ -1,4 +1,6 @@ +using AvParser.Core.Proxies; using AvParser.Core.Settings; +using AvParser.Infrastructure.Proxies; using AvParser.Infrastructure.Settings; using AvParser.Infrastructure.Storage; using Microsoft.Extensions.DependencyInjection; @@ -31,6 +33,49 @@ public static class InfrastructureServiceCollectionExtensions sp.GetRequiredService>() )); + services.AddAvParserProxies(); + + return services; + } + + /// Registers the proxy sources, the probe, the pool and the proxied client factory. + /// + /// The feed source gets a pooled because it always talks to the same + /// CDN host. The probe deliberately does not: its handler carries the proxy, so it has to + /// build one per check. + /// + public static IServiceCollection AddAvParserProxies(this IServiceCollection services) + { + ArgumentNullException.ThrowIfNull(services); + + services + .AddHttpClient(client => + { + client.Timeout = TimeSpan.FromSeconds(30); + client.DefaultRequestHeaders.UserAgent.ParseAdd("AvParser/0.1"); + }) + .ConfigurePrimaryHttpMessageHandler(() => + new SocketsHttpHandler { AutomaticDecompression = System.Net.DecompressionMethods.All } + ); + + services.AddSingleton(); + services.AddSingleton(sp => sp.GetRequiredService()); + + // Registration order here is the order the pool merges sources; the custom list comes + // last so a user-entered address wins over a feed entry for the same host and port. + services.AddSingleton(sp => sp.GetRequiredService()); + services.AddSingleton(sp => sp.GetRequiredService()); + + services.AddSingleton(); + + services.AddSingleton(sp => new ProxyPool( + sp.GetServices(), + sp.GetRequiredService(), + sp.GetRequiredService().Current.ToProxyOptions() + )); + + services.AddSingleton(); + return services; } } diff --git a/src/AvParser.Infrastructure/Proxies/CustomProxySource.cs b/src/AvParser.Infrastructure/Proxies/CustomProxySource.cs new file mode 100644 index 0000000..d59edcc --- /dev/null +++ b/src/AvParser.Infrastructure/Proxies/CustomProxySource.cs @@ -0,0 +1,273 @@ +using System.Text.Json; +using System.Text.Json.Serialization; +using AvParser.Core.Proxies; +using AvParser.Infrastructure.Storage; +using Microsoft.Extensions.Logging; + +namespace AvParser.Infrastructure.Proxies; + +/// One user-entered proxy, as persisted. +/// Address in scheme://host:port form. +/// Optional username. +/// Optional password. +/// Free-text note shown in the UI. +public sealed record CustomProxyRecord( + string Address, + string? Username = null, + string? Password = null, + string? Note = null +); + +/// Source-generated serialiser metadata for the user's proxy list. +[JsonSourceGenerationOptions(WriteIndented = true, PropertyNamingPolicy = JsonKnownNamingPolicy.CamelCase)] +[JsonSerializable(typeof(List))] +internal sealed partial class CustomProxyJsonContext : JsonSerializerContext; + +/// +/// The user's own proxy list, persisted next to the settings. +/// +/// +/// A separate file rather than a field inside settings.json: it is a list people paste into, +/// back up and share, and burying it in the settings blob makes all three awkward. Credentials +/// are stored in clear text — the same as every other proxy client, and worth knowing. +/// +public sealed class CustomProxySource : IMutableProxySource, IDisposable +{ + private readonly IAppPaths _paths; + private readonly ILogger _logger; + private readonly SemaphoreSlim _gate = new(1, 1); + + private List? _records; + + /// Creates the source. + public CustomProxySource(IAppPaths paths, ILogger logger) + { + _paths = paths ?? throw new ArgumentNullException(nameof(paths)); + _logger = logger ?? throw new ArgumentNullException(nameof(logger)); + } + + /// + public string Id => "custom"; + + /// + public string DisplayName => "Custom list"; + + /// + public ProxySourceKind Kind => ProxySourceKind.Custom; + + /// + public async Task> GetProxiesAsync(CancellationToken cancellationToken = default) + { + await _gate.WaitAsync(cancellationToken).ConfigureAwait(false); + try + { + var records = Load(); + var endpoints = new List(records.Count); + + foreach (var record in records) + { + if (ProxyEndpoint.TryParse(record.Address, out var endpoint)) + { + endpoints.Add( + endpoint with + { + Username = record.Username ?? endpoint.Username, + Password = record.Password ?? endpoint.Password, + } + ); + } + else + { + _logger.LogWarning("Skipping unparsable custom proxy {Address}", record.Address); + } + } + + return endpoints; + } + finally + { + _gate.Release(); + } + } + + /// + public async Task AddAsync(IEnumerable endpoints, CancellationToken cancellationToken = default) + { + ArgumentNullException.ThrowIfNull(endpoints); + + await _gate.WaitAsync(cancellationToken).ConfigureAwait(false); + try + { + var records = Load(); + var known = new HashSet( + records.Where(r => ProxyEndpoint.TryParse(r.Address, out _)).Select(KeyOf), + StringComparer.Ordinal + ); + + var added = 0; + foreach (var endpoint in endpoints) + { + if (!known.Add(endpoint.Key)) + { + continue; + } + + records.Add(new CustomProxyRecord(endpoint.ToString(), endpoint.Username, endpoint.Password)); + added++; + } + + if (added > 0) + { + await SaveAsync(records, cancellationToken).ConfigureAwait(false); + } + + return added; + } + finally + { + _gate.Release(); + } + } + + /// + public async Task RemoveAsync(ProxyEndpoint endpoint, CancellationToken cancellationToken = default) + { + ArgumentNullException.ThrowIfNull(endpoint); + + await _gate.WaitAsync(cancellationToken).ConfigureAwait(false); + try + { + var records = Load(); + var removed = records.RemoveAll(record => + ProxyEndpoint.TryParse(record.Address, out var parsed) + && string.Equals(parsed.Key, endpoint.Key, StringComparison.Ordinal) + ); + + if (removed > 0) + { + await SaveAsync(records, cancellationToken).ConfigureAwait(false); + } + + return removed > 0; + } + finally + { + _gate.Release(); + } + } + + /// + public async Task ClearAsync(CancellationToken cancellationToken = default) + { + await _gate.WaitAsync(cancellationToken).ConfigureAwait(false); + try + { + await SaveAsync([], cancellationToken).ConfigureAwait(false); + } + finally + { + _gate.Release(); + } + } + + /// + public void Dispose() => _gate.Dispose(); + + /// + /// Parses a pasted blob into endpoints, one per line. + /// + /// The endpoints that parsed, and the lines that did not. + /// + /// Reports the bad lines instead of dropping them: people paste lists of hundreds, and a + /// silent "added 97 of 100" is impossible to act on. + /// + public static (IReadOnlyList Parsed, IReadOnlyList Rejected) ParseList(string? text) + { + var parsed = new List(); + var rejected = new List(); + + if (string.IsNullOrWhiteSpace(text)) + { + return (parsed, rejected); + } + + // Lines first, then separators within a line. Doing it the other way round splits + // "# my proxies" into three tokens and reports two of them as invalid addresses. + foreach (var rawLine in text.Split(['\n', '\r'], StringSplitOptions.RemoveEmptyEntries)) + { + var line = rawLine.Trim(); + if (line.Length == 0 || line.StartsWith('#') || line.StartsWith("//", StringComparison.Ordinal)) + { + continue; + } + + foreach (var token in line.Split([',', ';', ' ', '\t'], StringSplitOptions.RemoveEmptyEntries)) + { + if (ProxyEndpoint.TryParse(token, out var endpoint)) + { + parsed.Add(endpoint); + } + else + { + rejected.Add(token); + } + } + } + + return (parsed, rejected); + } + + private static string KeyOf(CustomProxyRecord record) => + ProxyEndpoint.TryParse(record.Address, out var endpoint) ? endpoint.Key : record.Address; + + private List Load() + { + if (_records is not null) + { + return _records; + } + + try + { + if (!File.Exists(_paths.CustomProxiesFile)) + { + _records = []; + return _records; + } + + var json = File.ReadAllText(_paths.CustomProxiesFile); + _records = JsonSerializer.Deserialize(json, CustomProxyJsonContext.Default.ListCustomProxyRecord) ?? []; + } + catch (Exception ex) when (ex is IOException or UnauthorizedAccessException or JsonException) + { + _logger.LogWarning( + ex, + "Could not read {Path}; starting with an empty custom list", + _paths.CustomProxiesFile + ); + _records = []; + } + + return _records; + } + + private async Task SaveAsync(List records, CancellationToken cancellationToken) + { + _records = records; + + try + { + Directory.CreateDirectory(Path.GetDirectoryName(_paths.CustomProxiesFile)!); + + var temp = _paths.CustomProxiesFile + ".tmp"; + var json = JsonSerializer.Serialize(records, CustomProxyJsonContext.Default.ListCustomProxyRecord); + + await File.WriteAllTextAsync(temp, json, cancellationToken).ConfigureAwait(false); + File.Move(temp, _paths.CustomProxiesFile, overwrite: true); + } + catch (Exception ex) when (ex is IOException or UnauthorizedAccessException) + { + _logger.LogWarning(ex, "Could not write {Path}", _paths.CustomProxiesFile); + } + } +} diff --git a/src/AvParser.Infrastructure/Proxies/HttpProxyProbe.cs b/src/AvParser.Infrastructure/Proxies/HttpProxyProbe.cs new file mode 100644 index 0000000..20424b5 --- /dev/null +++ b/src/AvParser.Infrastructure/Proxies/HttpProxyProbe.cs @@ -0,0 +1,113 @@ +using System.Diagnostics; +using System.Net; +using AvParser.Core.Proxies; + +namespace AvParser.Infrastructure.Proxies; + +/// Builds the .NET plumbing needed to route a request through a proxy. +public static class ProxyHandlerFactory +{ + /// Wraps an endpoint in a , attaching credentials if present. + public static WebProxy CreateWebProxy(ProxyEndpoint endpoint) + { + ArgumentNullException.ThrowIfNull(endpoint); + + var proxy = new WebProxy(endpoint.Uri); + + if (endpoint.HasCredentials) + { + proxy.Credentials = new NetworkCredential(endpoint.Username, endpoint.Password); + } + + return proxy; + } + + /// + /// Creates a handler bound to one proxy, or a direct handler when is null. + /// + /// + /// A fresh handler per proxy is unavoidable — the proxy is a property of the handler, not of + /// the request — so connection pooling cannot be shared across proxies. + /// SOCKS is handled natively: .NET understands the socks4/socks4a/socks5 schemes. + /// + public static SocketsHttpHandler CreateHandler(ProxyEndpoint? endpoint, TimeSpan connectTimeout) + { + var handler = new SocketsHttpHandler + { + AllowAutoRedirect = false, + ConnectTimeout = connectTimeout, + PooledConnectionLifetime = TimeSpan.FromMinutes(2), + AutomaticDecompression = DecompressionMethods.All, + }; + + if (endpoint is null) + { + handler.UseProxy = false; + return handler; + } + + handler.UseProxy = true; + handler.Proxy = CreateWebProxy(endpoint); + return handler; + } +} + +/// Checks a proxy by fetching through it. +public sealed class HttpProxyProbe : IProxyProbe +{ + /// + public async Task ProbeAsync( + ProxyEndpoint endpoint, + ProxyOptions options, + CancellationToken cancellationToken = default + ) + { + ArgumentNullException.ThrowIfNull(endpoint); + ArgumentNullException.ThrowIfNull(options); + + using var handler = ProxyHandlerFactory.CreateHandler(endpoint, options.ProbeTimeout); + using var client = new HttpClient(handler, disposeHandler: false) { Timeout = options.ProbeTimeout }; + + var stopwatch = Stopwatch.StartNew(); + + try + { + using var response = await client + .GetAsync(options.ProbeUrl, HttpCompletionOption.ResponseHeadersRead, cancellationToken) + .ConfigureAwait(false); + + stopwatch.Stop(); + + return response.IsSuccessStatusCode + ? ProxyProbeResult.Success(stopwatch.Elapsed) + : ProxyProbeResult.Failure($"HTTP {(int)response.StatusCode}"); + } + catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) + { + // The caller pulled the plug — that is not a verdict on the proxy. + throw; + } + catch (OperationCanceledException) + { + return ProxyProbeResult.Failure("timeout"); + } + catch (HttpRequestException ex) + { + return ProxyProbeResult.Failure(Describe(ex)); + } + catch (Exception ex) when (ex is IOException or InvalidOperationException or NotSupportedException) + { + return ProxyProbeResult.Failure(ex.Message); + } + } + + private static string Describe(HttpRequestException exception) => + exception.HttpRequestError switch + { + HttpRequestError.ProxyTunnelError => "proxy refused CONNECT", + HttpRequestError.ConnectionError => "connection failed", + HttpRequestError.NameResolutionError => "DNS failed", + HttpRequestError.SecureConnectionError => "TLS failed", + _ => exception.Message, + }; +} diff --git a/src/AvParser.Infrastructure/Proxies/ProxiedHttpClientFactory.cs b/src/AvParser.Infrastructure/Proxies/ProxiedHttpClientFactory.cs new file mode 100644 index 0000000..c6e34d4 --- /dev/null +++ b/src/AvParser.Infrastructure/Proxies/ProxiedHttpClientFactory.cs @@ -0,0 +1,56 @@ +using AvParser.Core.Proxies; + +namespace AvParser.Infrastructure.Proxies; + +/// Creates instances bound to a specific proxy. +/// +/// Not IHttpClientFactory: that exists to share handlers across requests, and a per-proxy +/// handler is the opposite of shareable. This is the seam the parser layer will use once it +/// starts making real requests. +/// +public interface IProxiedHttpClientFactory +{ + /// Creates a client routed through , or direct when null. + /// The caller owns the returned client and must dispose it. + HttpClient Create(ProxyEndpoint? endpoint, TimeSpan? timeout = null); + + /// + /// Takes a proxy from the pool and returns a client bound to it, or a direct client when the + /// pool has nothing usable. + /// + /// + /// The lease comes back with the client so the caller can report the outcome — without that + /// the pool never learns which proxies work. + /// + Task<(HttpClient Client, ProxyLease? Lease)> CreateFromPoolAsync( + TimeSpan? timeout = null, + CancellationToken cancellationToken = default + ); +} + +/// +public sealed class ProxiedHttpClientFactory(IProxyPool pool) : IProxiedHttpClientFactory +{ + private static readonly TimeSpan DefaultTimeout = TimeSpan.FromSeconds(30); + + private readonly IProxyPool _pool = pool ?? throw new ArgumentNullException(nameof(pool)); + + /// + public HttpClient Create(ProxyEndpoint? endpoint, TimeSpan? timeout = null) + { + var effective = timeout ?? DefaultTimeout; + var handler = ProxyHandlerFactory.CreateHandler(endpoint, effective); + + return new HttpClient(handler, disposeHandler: true) { Timeout = effective }; + } + + /// + public async Task<(HttpClient Client, ProxyLease? Lease)> CreateFromPoolAsync( + TimeSpan? timeout = null, + CancellationToken cancellationToken = default + ) + { + var lease = await _pool.AcquireAsync(cancellationToken).ConfigureAwait(false); + return (Create(lease?.Endpoint, timeout), lease); + } +} diff --git a/src/AvParser.Infrastructure/Proxies/ProxiflyProxySource.cs b/src/AvParser.Infrastructure/Proxies/ProxiflyProxySource.cs new file mode 100644 index 0000000..a833542 --- /dev/null +++ b/src/AvParser.Infrastructure/Proxies/ProxiflyProxySource.cs @@ -0,0 +1,186 @@ +using System.Text.Json; +using System.Text.Json.Serialization; +using AvParser.Core.Proxies; +using Microsoft.Extensions.Logging; + +namespace AvParser.Infrastructure.Proxies; + +/// One record of the proxifly feed. +internal sealed record ProxiflyRecord +{ + public string? Proxy { get; init; } + public string? Protocol { get; init; } + public string? Ip { get; init; } + public int Port { get; init; } + public bool Https { get; init; } + public string? Anonymity { get; init; } + public int Score { get; init; } + public ProxiflyGeolocation? Geolocation { get; init; } +} + +/// Geolocation block of a proxifly record. +internal sealed record ProxiflyGeolocation +{ + public string? Country { get; init; } + public string? City { get; init; } +} + +/// Source-generated serialiser metadata for the proxifly feed. +[JsonSourceGenerationOptions(PropertyNamingPolicy = JsonKnownNamingPolicy.CamelCase)] +[JsonSerializable(typeof(ProxiflyRecord[]))] +internal sealed partial class ProxiflyJsonContext : JsonSerializerContext; + +/// +/// Reads the public list published at github.com/proxifly/free-proxy-list. +/// +/// +/// Fetches the combined all/data.json and filters locally rather than requesting a +/// per-protocol slice: the feed is regenerated every few minutes, so one conditional request for +/// the whole list beats four that can disagree with each other mid-publish. +/// +public sealed class ProxiflyProxySource : IProxySource, IDisposable +{ + /// Served from jsDelivr rather than raw.githubusercontent.com — a CDN meant for this. + public static readonly Uri FeedUrl = new( + "https://cdn.jsdelivr.net/gh/proxifly/free-proxy-list@main/proxies/all/data.json" + ); + + /// Upstream refreshes every five minutes; asking more often only burns bandwidth. + public static readonly TimeSpan CacheTtl = TimeSpan.FromMinutes(5); + + private readonly HttpClient _http; + private readonly ILogger _logger; + private readonly TimeProvider _time; + private readonly SemaphoreSlim _fetchLock = new(1, 1); + + private IReadOnlyList _cached = []; + private DateTimeOffset? _fetchedAtUtc; + + /// Creates the source. + public ProxiflyProxySource(HttpClient http, ILogger logger, TimeProvider? timeProvider = null) + { + _http = http ?? throw new ArgumentNullException(nameof(http)); + _logger = logger ?? throw new ArgumentNullException(nameof(logger)); + _time = timeProvider ?? TimeProvider.System; + } + + /// + public string Id => "proxifly"; + + /// + public string DisplayName => "Proxifly free-proxy-list"; + + /// + public ProxySourceKind Kind => ProxySourceKind.Feed; + + /// When the feed was last downloaded, or if never. + public DateTimeOffset? FetchedAtUtc => _fetchedAtUtc; + + /// + public async Task> GetProxiesAsync(CancellationToken cancellationToken = default) + { + if (IsFresh()) + { + return _cached; + } + + await _fetchLock.WaitAsync(cancellationToken).ConfigureAwait(false); + try + { + // Another caller may have refreshed while we waited for the lock. + if (IsFresh()) + { + return _cached; + } + + var endpoints = await FetchAsync(cancellationToken).ConfigureAwait(false); + _cached = endpoints; + _fetchedAtUtc = _time.GetUtcNow(); + return endpoints; + } + finally + { + _fetchLock.Release(); + } + } + + /// Drops the cache so the next call re-downloads. + public void Invalidate() => _fetchedAtUtc = null; + + /// + /// The is owned by the container, so only the lock is released here. + public void Dispose() => _fetchLock.Dispose(); + + /// Converts one feed record into an endpoint, or if unusable. + /// Internal so the parsing can be tested against captured feed samples without HTTP. + internal static ProxyEndpoint? ToEndpoint(ProxiflyRecord record) + { + if (record.Ip is not { Length: > 0 } host || record.Port is < 1 or > 65535) + { + // The "proxy" field carries the same address; fall back to it when the parts are absent. + return ProxyEndpoint.TryParse(record.Proxy, out var parsed) ? Decorate(parsed, record) : null; + } + + if (!ProxyEndpoint.TryParseProtocol(record.Protocol, out var protocol)) + { + return null; + } + + return Decorate(new ProxyEndpoint(protocol, host, record.Port), record); + } + + /// Parses a whole feed payload. + internal static IReadOnlyList ParseFeed(string json) + { + var records = JsonSerializer.Deserialize(json, ProxiflyJsonContext.Default.ProxiflyRecordArray); + if (records is null) + { + return []; + } + + var endpoints = new List(records.Length); + foreach (var record in records) + { + if (ToEndpoint(record) is { } endpoint) + { + endpoints.Add(endpoint); + } + } + + return endpoints; + } + + private static ProxyEndpoint Decorate(ProxyEndpoint endpoint, ProxiflyRecord record) => + endpoint with + { + Country = record.Geolocation?.Country, + City = string.Equals(record.Geolocation?.City, "Unknown", StringComparison.OrdinalIgnoreCase) + ? null + : record.Geolocation?.City, + Anonymity = ProxyEndpoint.ParseAnonymity(record.Anonymity), + Score = record.Score, + }; + + private bool IsFresh() => _fetchedAtUtc is { } at && _time.GetUtcNow() - at < CacheTtl; + + private async Task> FetchAsync(CancellationToken cancellationToken) + { + try + { + using var response = await _http.GetAsync(FeedUrl, cancellationToken).ConfigureAwait(false); + response.EnsureSuccessStatusCode(); + + var json = await response.Content.ReadAsStringAsync(cancellationToken).ConfigureAwait(false); + var endpoints = ParseFeed(json); + + _logger.LogInformation("Fetched {Count} proxies from {Source}", endpoints.Count, DisplayName); + return endpoints; + } + catch (Exception ex) when (ex is HttpRequestException or JsonException or TaskCanceledException) + { + // A feed that is down must not take the app with it: keep serving whatever we had. + _logger.LogWarning(ex, "Could not fetch {Source}; keeping {Count} cached", DisplayName, _cached.Count); + return _cached; + } + } +} diff --git a/src/AvParser.Infrastructure/Storage/AppPaths.cs b/src/AvParser.Infrastructure/Storage/AppPaths.cs index 7fdddef..3eee02c 100644 --- a/src/AvParser.Infrastructure/Storage/AppPaths.cs +++ b/src/AvParser.Infrastructure/Storage/AppPaths.cs @@ -13,6 +13,10 @@ public interface IAppPaths /// Full path of the settings file. string SettingsFile { get; } + /// Full path of the user's own proxy list. + /// Kept separate from the settings file: it is a list the user edits and may want to back up or share. + string CustomProxiesFile { get; } + /// Directory holding rolling log files. string LogDirectory { get; } } @@ -45,6 +49,7 @@ public sealed class AppPaths : IAppPaths DataDirectory = dataDirectory; SettingsFile = Path.Combine(dataDirectory, "settings.json"); + CustomProxiesFile = Path.Combine(dataDirectory, "proxies.custom.json"); LogDirectory = Path.Combine(dataDirectory, "logs"); } @@ -54,6 +59,9 @@ public sealed class AppPaths : IAppPaths /// public string SettingsFile { get; } + /// + public string CustomProxiesFile { get; } + /// public string LogDirectory { get; } diff --git a/src/AvParser.UI/DependencyInjection/UiServiceCollectionExtensions.cs b/src/AvParser.UI/DependencyInjection/UiServiceCollectionExtensions.cs index e82260b..51867b6 100644 --- a/src/AvParser.UI/DependencyInjection/UiServiceCollectionExtensions.cs +++ b/src/AvParser.UI/DependencyInjection/UiServiceCollectionExtensions.cs @@ -1,4 +1,5 @@ using AvParser.Core.Parsing; +using AvParser.Core.Proxies; using AvParser.Core.Settings; using AvParser.Infrastructure.Storage; using AvParser.UI.Navigation; @@ -37,13 +38,20 @@ public static class UiServiceCollectionExtensions sp.GetRequiredService(), sp.GetRequiredService(), sp.GetRequiredService(), - sp.GetRequiredService() + sp.GetRequiredService(), + sp.GetRequiredService() + )); + services.AddSingleton(static sp => new ProxiesViewModel( + sp.GetRequiredService(), + sp.GetRequiredService(), + sp.GetRequiredService>() )); services.AddSingleton(); // Order here is the order of the navigation rail; the first entry is the landing page. services.AddSingleton(static sp => sp.GetRequiredService()); services.AddSingleton(static sp => sp.GetRequiredService()); + services.AddSingleton(static sp => sp.GetRequiredService()); services.AddSingleton(static sp => sp.GetRequiredService()); services.AddSingleton(static sp => sp.GetRequiredService()); diff --git a/src/AvParser.UI/Styles/Controls.axaml b/src/AvParser.UI/Styles/Controls.axaml index 32bfae9..aec0b2b 100644 --- a/src/AvParser.UI/Styles/Controls.axaml +++ b/src/AvParser.UI/Styles/Controls.axaml @@ -71,6 +71,23 @@ + + + + + + + + +