diff --git a/CLAUDE.md b/CLAUDE.md index 138a8b9..2057afd 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -106,6 +106,37 @@ dotnet csharpier check . - **Лиза без вердикта нейтральна.** Отменённая операция — не вина прокси; считать это отказом значит карантинить здоровые прокси на каждый Cancel. - **`Select` и `Next` — зарезервированные слова для CA1716.** Метод стратегии называется `Pick`. +- **`LiveCount` считает только `Alive` и не в карантине.** На нём висит гейт парсера, поэтому + «доступна» (карантин истёк) и «живая» здесь намеренно расходятся: гейт не должен открываться + от одного лишь истечения окна. +- **Прогрев обрывается по достижении цели, а не проходит список до конца.** `WarmUpAsync` линкует + CTS и гасит остаток, как только набралось `MinimumLiveProxies`. Порядок кандидатов — + `WarmUpOrder()`, он публичный ровно затем, чтобы порядок проверялся без прогона проб. +- **`RestoreState` не выставляет `Health = Alive`.** «Работала вчера» живёт в отдельном + `WasAliveOnLastRun` и влияет только на порядок прогрева. Если восстанавливать как `Alive`, пул + отрапортует живыми тех, с кем не разговаривал: прогрев сочтёт цель достигнутой и не проверит + никого, а гейт парсера откроется по данным недельной давности. Ловит + `A_remembered_proxy_is_not_reported_live_until_it_answers_again`. +- **`RestoreState` не восстанавливает карантин.** Окно — стенные часы, между запусками могли пройти + сутки; перенос окна сажал бы прокси за то, что давно истекло. +- **Сохраняются только `HasEverAnswered`** = `SuccessCount > 0 || IsBelievedAlive`, где + «believed» = вердикт этой сессии, а без него — прошлой. Мёртвые в фиде исчисляются тысячами и + переиздаются каждые пять минут. Важен именно перенос: прогрев обрывается рано, поэтому + большинство запомненных заканчивают сессию непроверенными — строгое `Health == Alive` стирало бы + накопленный список за пару запусков. + +## Гейт парсера + +`IParser.RequiresNetwork` — дефолтная реализация возвращает `false`, поэтому добавление парсера +остаётся однострочным. Парсер, который куда-то ходит, **обязан** её переопределить, иначе поедет +напрямую в обход настройки. + +Гейт живёт в `ParseViewModel.RefreshProxyGate()` и складывается из трёх условий: парсер сетевой, +`AllowDirectConnection` выключен, `LiveCount == 0`. Он пересчитывается по событию пула (с +throttle 250 мс — пул дёргается на каждый исход лизы), при смене парсера и при смене настроек. + +`ParseView.axaml` держит баннер под `x:Name="ProxyGateBanner"`; `ParseViewTests` рендерит его +по-настоящему, потому что мёртвый биндинг `IsVisible` не ломает ни одного VM-теста. ## Добавить строку в UI diff --git a/README.md b/README.md index e94e972..4181007 100644 --- a/README.md +++ b/README.md @@ -134,6 +134,32 @@ Avalonia матчит **точный** тип, а `ShellView` наследует Упавшая прокси уходит в карантин с экспоненциальным окном (30 с → 15 мин), но **не** удаляется навсегда: бесплатные прокси постоянно мигают, и жёсткий бан терял бы их безвозвратно. +### Что запоминается между запусками + +Пул грузится и прогревается сам при старте, нажимать «Обновить» не нужно. Прогрев идёт +**от известного хорошего**: сначала пробуются те, что отвечали в прошлый раз, затем самые быстрые +из них, и проверка обрывается, как только набралось `ProxyMinimumLive` живых (по умолчанию 10). +Иначе каждый запуск был бы полным свипом по паре тысяч адресов ради десятка рабочих. + +Запомненное — это подсказка, а не зачёт: восстановленная прокси идёт первой в очередь на проверку, +но живой не считается, пока не ответит в этом запуске. Иначе запуск через неделю открывал бы гейт +парсера по недельной давности данным, а прогрев пропускал бы ровно те прокси, ради которых он есть. + +Состояние лежит в `proxies.state.json` рядом с настройками и пишется после прогрева и на выходе. +Сохраняются **только те прокси, что когда-либо отвечали**: мёртвых в фиде тысячи, они переиздаются +каждые пять минут, и «было мертво час назад» не говорит почти ничего. Карантин не восстанавливается +— окно отсчитывается по стенным часам, а между запусками могли пройти сутки. + +### Гейт «без прокси не работаем» + +Парсер, который объявил `RequiresNetwork`, не запустится, пока в пуле нет ни одной живой прокси: +кнопка «Разобрать» гаснет, а на странице появляется баннер с переходом на страницу Proxies. +Гейт снимается настройкой **«Разрешить сетевым парсерам работать без прокси»**. + +Парсеры, читающие вставленный пользователем текст, не блокируются никогда — им нечего +маршрутизировать, и блокировка делала бы приложение бесполезным всякий раз, когда публичные списки +лежат. + Использование из кода: ```csharp diff --git a/src/AvParser.Core/Parsing/IParser.cs b/src/AvParser.Core/Parsing/IParser.cs index 3eba734..3e65737 100644 --- a/src/AvParser.Core/Parsing/IParser.cs +++ b/src/AvParser.Core/Parsing/IParser.cs @@ -21,6 +21,17 @@ public interface IParser /// Cheap structural check — must not throw and must not do IO. bool CanParse(TInput input); + /// + /// Whether this parser makes network requests and therefore needs a working proxy. + /// + /// + /// A default implementation rather than an abstract member, so adding a parser stays a + /// one-line change: a parser that works on text the user pasted opts out by saying nothing. + /// Parsers that fetch anything must set this, or they will run direct even when the user has + /// asked for proxy-only operation. + /// + bool RequiresNetwork => false; + /// Streams one outcome per logical record. IAsyncEnumerable> ParseAsync( TInput input, diff --git a/src/AvParser.Core/Proxies/IProxyPool.cs b/src/AvParser.Core/Proxies/IProxyPool.cs index 2a1bdaf..b14c789 100644 --- a/src/AvParser.Core/Proxies/IProxyPool.cs +++ b/src/AvParser.Core/Proxies/IProxyPool.cs @@ -9,6 +9,9 @@ public interface IProxyPool /// Options currently in force. ProxyOptions Options { get; } + /// How many entries are known to work and are not sidelined right now. + int LiveCount { get; } + /// Raised after the set of entries or their health changes. /// /// A plain event rather than IObservable so the domain keeps no reactive dependency; @@ -33,6 +36,21 @@ public interface IProxyPool /// Probes every entry in parallel and updates their health. /// How many answered. Task SweepAsync(IProgress? progress = null, CancellationToken cancellationToken = default); + + /// + /// Probes best-known entries first and stops once of them work. + /// + /// How many are live afterwards. + /// + /// The startup counterpart to : proxies that answered on a previous + /// run are tried first, so a second launch usually confirms enough of them in a handful of + /// requests instead of re-probing a few thousand addresses. + /// + Task WarmUpAsync( + int targetLive, + IProgress? progress = null, + CancellationToken cancellationToken = default + ); } /// diff --git a/src/AvParser.Core/Proxies/ProxyEntry.cs b/src/AvParser.Core/Proxies/ProxyEntry.cs index d534d20..59b6017 100644 --- a/src/AvParser.Core/Proxies/ProxyEntry.cs +++ b/src/AvParser.Core/Proxies/ProxyEntry.cs @@ -181,6 +181,65 @@ public sealed class ProxyEntry } } + /// Whether a previous run saw this proxy answer. + /// + /// Kept apart from so that "worked yesterday" can order the warm-up + /// without being mistaken for "works now". Only this session's verdict counts as live. + /// + public bool WasAliveOnLastRun { get; private set; } + + /// + /// Reinstates what a previous run learned about this proxy. + /// + /// + /// + /// Health comes back as even for a proxy that answered + /// last time. Restoring it as would make the pool report + /// live proxies it has not spoken to — the warm-up would then skip them as already-confirmed, + /// and a launch a week later would open the parser gate on week-old evidence. + /// + /// + /// Restores no quarantine either: the window is wall-clock and a restart may be days later, so + /// carrying it over would sideline proxies for reasons that have long expired. + /// + /// + public void RestoreState( + bool wasAlive, + TimeSpan? latency, + int successCount, + int failureCount, + DateTimeOffset? lastCheckedUtc + ) + { + lock (_gate) + { + Health = ProxyHealthState.Unknown; + WasAliveOnLastRun = wasAlive; + Latency = latency; + SuccessCount = Math.Max(0, successCount); + FailureCount = Math.Max(0, failureCount); + LastCheckedUtc = lastCheckedUtc; + ConsecutiveFailures = 0; + QuarantinedUntilUtc = null; + } + } + + /// Best current belief about whether this proxy answers. + /// + /// This session's verdict wins; with none, what the previous run saw carries forward. Unlike + /// this is a belief, not a confirmation — never gate on it. + /// + public bool IsBelievedAlive => + Health switch + { + ProxyHealthState.Alive => true, + ProxyHealthState.Dead => false, + _ => WasAliveOnLastRun, + }; + + /// Whether this proxy ever answered, and is therefore worth remembering. + public bool HasEverAnswered => SuccessCount > 0 || IsBelievedAlive; + /// public override string ToString() => $"{Endpoint} [{Health}]"; } diff --git a/src/AvParser.Core/Proxies/ProxyOptions.cs b/src/AvParser.Core/Proxies/ProxyOptions.cs index a892348..afb9024 100644 --- a/src/AvParser.Core/Proxies/ProxyOptions.cs +++ b/src/AvParser.Core/Proxies/ProxyOptions.cs @@ -103,6 +103,25 @@ public sealed record ProxyOptions /// Whether the remote feed is consulted at all. public bool UseFeed { get; init; } = true; + /// + /// How many working proxies a warm-up aims for before it stops probing. + /// + /// + /// A free list holds thousands of addresses of which a few percent work. Probing all of them + /// at every launch costs thousands of requests for information that goes stale in minutes; + /// stopping once there are enough to rotate through is the useful part of that work. + /// + public int MinimumLiveProxies { get; init; } = 10; + + /// + /// Whether work may run without a proxy when none is available. + /// + /// + /// Only parsers that declare RequiresNetwork are affected; a parser working on pasted + /// text is never blocked. + /// + public bool AllowDirectConnection { get; init; } + /// Validates the options, throwing on values that would misbehave silently. /// A numeric option is out of range. public ProxyOptions Validated() diff --git a/src/AvParser.Core/Proxies/ProxyPool.cs b/src/AvParser.Core/Proxies/ProxyPool.cs index 7fca817..3226c8a 100644 --- a/src/AvParser.Core/Proxies/ProxyPool.cs +++ b/src/AvParser.Core/Proxies/ProxyPool.cs @@ -64,6 +64,20 @@ public sealed class ProxyPool : IProxyPool } } + /// + public int LiveCount + { + get + { + var now = _time.GetUtcNow(); + + lock (_gate) + { + return _entries.Count(entry => entry.Health == ProxyHealthState.Alive && entry.IsAvailable(now)); + } + } + } + /// public event EventHandler? Changed; @@ -268,6 +282,115 @@ public sealed class ProxyPool : IProxyPool return aliveCount; } + /// + public async Task WarmUpAsync( + int targetLive, + IProgress? progress = null, + CancellationToken cancellationToken = default + ) + { + ArgumentOutOfRangeException.ThrowIfLessThan(targetLive, 1); + + var options = Options; + var candidates = WarmUpOrder(); + var total = candidates.Count; + + if (total == 0 || LiveCount >= targetLive) + { + progress?.Report(new ProxySweepProgress(0, 0, LiveCount)); + return LiveCount; + } + + // Stops the remaining probes the moment the target is met. Linked so the caller's own + // cancellation still wins. + using var stop = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken); + using var limiter = new SemaphoreSlim(options.ProbeConcurrency, options.ProbeConcurrency); + + var checkedCount = 0; + + var work = candidates.Select(async entry => + { + if (stop.IsCancellationRequested) + { + return; + } + + try + { + await limiter.WaitAsync(stop.Token).ConfigureAwait(false); + } + catch (OperationCanceledException) + { + return; + } + + try + { + var result = await _probe.ProbeAsync(entry.Endpoint, options, stop.Token).ConfigureAwait(false); + entry.RecordProbe( + _time.GetUtcNow(), + result.Alive, + result.Latency, + result.Error, + quarantineOnFailure: options.BaseQuarantine + ); + } + catch (OperationCanceledException) + { + // Either the target was reached or the caller gave up; neither is the proxy's fault. + return; + } + finally + { + limiter.Release(); + } + + var live = LiveCount; + progress?.Report(new ProxySweepProgress(Interlocked.Increment(ref checkedCount), total, live)); + + if (live >= targetLive) + { + await stop.CancelAsync().ConfigureAwait(false); + } + }); + + await Task.WhenAll(work).ConfigureAwait(false); + + cancellationToken.ThrowIfCancellationRequested(); + + lock (_gate) + { + _strategy.Reset(); + } + + RaiseChanged(); + return LiveCount; + } + + /// + /// Orders candidates for a warm-up: what worked last time, then what looks most promising. + /// + /// Public so the ordering can be asserted without running probes. + public IReadOnlyList WarmUpOrder() + { + var now = _time.GetUtcNow(); + + lock (_gate) + { + return + [ + .. _entries + .Where(entry => entry.IsAvailable(now)) + .OrderByDescending(entry => entry.Health == ProxyHealthState.Alive) + .ThenByDescending(entry => entry.WasAliveOnLastRun) + .ThenByDescending(entry => entry.SuccessCount > 0) + .ThenBy(entry => entry.Latency ?? TimeSpan.MaxValue) + .ThenByDescending(entry => entry.SuccessRate) + .ThenByDescending(entry => entry.Endpoint.Score), + ]; + } + } + /// Records the outcome of a lease. Called by . internal void ReportOutcome(ProxyEntry entry, bool success, TimeSpan? latency, string? error) { diff --git a/src/AvParser.Core/Settings/AppSettings.cs b/src/AvParser.Core/Settings/AppSettings.cs index 2a4ae7a..65b7236 100644 --- a/src/AvParser.Core/Settings/AppSettings.cs +++ b/src/AvParser.Core/Settings/AppSettings.cs @@ -45,6 +45,8 @@ public enum AppTheme /// URL fetched to decide whether a proxy works. /// Per-proxy probe timeout, in seconds. /// How many probes run at once during a pool sweep. +/// How many working proxies a startup warm-up aims for. +/// Whether network parsers may run without a proxy. public sealed record AppSettings( AppTheme Theme = AppTheme.System, AppLanguage Language = AppLanguage.System, @@ -59,7 +61,9 @@ public sealed record AppSettings( ProxyProtocolFilter ProxyProtocols = ProxyProtocolFilter.All, string ProxyProbeUrl = "http://www.gstatic.com/generate_204", int ProxyProbeTimeoutSeconds = 8, - int ProxyProbeConcurrency = 64 + int ProxyProbeConcurrency = 64, + int ProxyMinimumLive = 10, + bool AllowDirectConnection = false ) { /// Projects the proxy-related settings onto . @@ -83,6 +87,8 @@ public sealed record AppSettings( ProbeUrl = probeUrl, ProbeTimeout = TimeSpan.FromSeconds(Math.Clamp(ProxyProbeTimeoutSeconds, 1, 120)), ProbeConcurrency = Math.Clamp(ProxyProbeConcurrency, 1, 512), + MinimumLiveProxies = Math.Clamp(ProxyMinimumLive, 1, 500), + AllowDirectConnection = AllowDirectConnection, }.Validated(); } } diff --git a/src/AvParser.Desktop/App.axaml.cs b/src/AvParser.Desktop/App.axaml.cs index 08adb36..dc7af12 100644 --- a/src/AvParser.Desktop/App.axaml.cs +++ b/src/AvParser.Desktop/App.axaml.cs @@ -49,19 +49,29 @@ public partial class App : Application _ = _services.GetRequiredService(); _ = _services.GetRequiredService(); + var proxies = _services.GetRequiredService(); + if (ApplicationLifetime is IClassicDesktopStyleApplicationLifetime desktop) { var settings = _services.GetRequiredService(); var window = CreateMainWindow(settings); desktop.MainWindow = window; - desktop.ShutdownRequested += (_, _) => settings.FlushAsync().GetAwaiter().GetResult(); + desktop.ShutdownRequested += (_, _) => + { + settings.FlushAsync().GetAwaiter().GetResult(); + + // Everything learned during the session — which proxies answered, how fast — is + // only in memory until now. Writing it here is what makes the next launch start + // from the ones that worked instead of re-probing a few thousand addresses. + proxies.SaveStateAsync().GetAwaiter().GetResult(); + }; } // Start filling the proxy pool as soon as the window is up. Not awaited on purpose: the // load is a network round trip, and blocking startup on a public list being reachable // would be the wrong trade. It never throws, so there is nothing to observe. - _ = _services.GetRequiredService().EnsureLoadedAsync(); + _ = proxies.EnsureLoadedAsync(); base.OnFrameworkInitializationCompleted(); } diff --git a/src/AvParser.Infrastructure/DependencyInjection/InfrastructureServiceCollectionExtensions.cs b/src/AvParser.Infrastructure/DependencyInjection/InfrastructureServiceCollectionExtensions.cs index b0dee02..4f623a4 100644 --- a/src/AvParser.Infrastructure/DependencyInjection/InfrastructureServiceCollectionExtensions.cs +++ b/src/AvParser.Infrastructure/DependencyInjection/InfrastructureServiceCollectionExtensions.cs @@ -75,6 +75,7 @@ public static class InfrastructureServiceCollectionExtensions )); services.AddSingleton(); + services.AddSingleton(); services.AddSingleton(); return services; diff --git a/src/AvParser.Infrastructure/Proxies/ProxyPoolLoader.cs b/src/AvParser.Infrastructure/Proxies/ProxyPoolLoader.cs index e838361..11d0b6a 100644 --- a/src/AvParser.Infrastructure/Proxies/ProxyPoolLoader.cs +++ b/src/AvParser.Infrastructure/Proxies/ProxyPoolLoader.cs @@ -3,35 +3,45 @@ using Microsoft.Extensions.Logging; namespace AvParser.Infrastructure.Proxies; -/// Fills the pool once at startup, so nothing has to be loaded by hand. +/// Outcome of the startup load. +/// Entries in the pool. +/// Entries confirmed to work. +/// Entries recognised from the previous run. +public readonly record struct ProxyPoolLoadResult(int Total, int Live, int Restored); + +/// Fills and warms the pool once at startup, so nothing has to be loaded by hand. public interface IProxyPoolLoader { /// - /// Loads every source, once per process. Later callers get the same operation rather than a - /// second download. + /// Loads every source, restores what the previous run learned, and probes until there are + /// enough working proxies. Runs once per process; later callers get the same operation. /// - /// How many proxies the pool holds afterwards. /// Never throws: a source that is down leaves the pool as it was. - Task EnsureLoadedAsync(); + Task EnsureLoadedAsync(); /// Whether the initial load has finished. bool IsLoaded { get; } + + /// Writes the current pool state so the next launch can start from it. + Task SaveStateAsync(CancellationToken cancellationToken = default); } /// -public sealed class ProxyPoolLoader(IProxyPool pool, ILogger logger) : IProxyPoolLoader +public sealed class ProxyPoolLoader(IProxyPool pool, IProxyStateStore stateStore, ILogger logger) + : IProxyPoolLoader { private readonly IProxyPool _pool = pool ?? throw new ArgumentNullException(nameof(pool)); + private readonly IProxyStateStore _stateStore = stateStore ?? throw new ArgumentNullException(nameof(stateStore)); private readonly ILogger _logger = logger ?? throw new ArgumentNullException(nameof(logger)); private readonly Lock _gate = new(); - private Task? _load; + private Task? _load; /// public bool IsLoaded => _load is { IsCompleted: true }; /// - public Task EnsureLoadedAsync() + public Task EnsureLoadedAsync() { if (_load is { } started) { @@ -48,28 +58,53 @@ public sealed class ProxyPoolLoader(IProxyPool pool, ILogger lo } } - private async Task LoadAsync() + /// + public Task SaveStateAsync(CancellationToken cancellationToken = default) => + _stateStore.SaveAsync(_pool.Entries, cancellationToken); + + private async Task LoadAsync() { + var options = _pool.Options; + try { // Worth logging: an empty pool is almost always a filter or a switched-off feed // rather than a network problem, and without this it looks identical to both. _logger.LogInformation( - "Loading proxy pool (feed: {UseFeed}, protocols: {Protocols})", - _pool.Options.UseFeed, - _pool.Options.Protocols + "Loading proxy pool (feed: {UseFeed}, protocols: {Protocols}, target live: {Target})", + options.UseFeed, + options.Protocols, + options.MinimumLiveProxies ); - var count = await _pool.RefreshAsync(CancellationToken.None).ConfigureAwait(false); - _logger.LogInformation("Proxy pool loaded with {Count} entries", count); - return count; + var total = await _pool.RefreshAsync(CancellationToken.None).ConfigureAwait(false); + + var state = await _stateStore.LoadAsync(CancellationToken.None).ConfigureAwait(false); + var restored = ProxyStateStore.Apply(_pool.Entries, state); + + // Warming up before anything asks for a proxy is the whole point: the first request + // should not be the thing that discovers the list is 97% dead. + var live = await _pool + .WarmUpAsync(options.MinimumLiveProxies, null, CancellationToken.None) + .ConfigureAwait(false); + + _logger.LogInformation( + "Proxy pool ready: {Live} live of {Total} ({Restored} remembered)", + live, + total, + restored + ); + + await SaveStateAsync(CancellationToken.None).ConfigureAwait(false); + + return new ProxyPoolLoadResult(total, live, restored); } catch (Exception ex) { // Startup must not fail because a public list is unreachable; the user can retry from // the Proxies page, and the app works without proxies in the meantime. _logger.LogWarning(ex, "Could not load the proxy pool at startup"); - return _pool.Entries.Count; + return new ProxyPoolLoadResult(_pool.Entries.Count, _pool.LiveCount, 0); } } } diff --git a/src/AvParser.Infrastructure/Proxies/ProxyStateStore.cs b/src/AvParser.Infrastructure/Proxies/ProxyStateStore.cs new file mode 100644 index 0000000..a4095c5 --- /dev/null +++ b/src/AvParser.Infrastructure/Proxies/ProxyStateStore.cs @@ -0,0 +1,149 @@ +using System.Text.Json; +using System.Text.Json.Serialization; +using AvParser.Core.Proxies; +using AvParser.Infrastructure.Storage; +using Microsoft.Extensions.Logging; + +namespace AvParser.Infrastructure.Proxies; + +/// What a previous run learned about one proxy. +/// Address in scheme://host:port form. +/// Whether the last check succeeded. +/// Round-trip time of the last successful check. +/// Successful uses. +/// Failed uses. +/// When the state was last updated. +public sealed record ProxyStateRecord( + string Address, + bool Alive = false, + double? LatencyMs = null, + int SuccessCount = 0, + int FailureCount = 0, + DateTimeOffset? LastCheckedUtc = null +); + +/// Source-generated serialiser metadata for the remembered pool state. +[JsonSourceGenerationOptions(WriteIndented = true, PropertyNamingPolicy = JsonKnownNamingPolicy.CamelCase)] +[JsonSerializable(typeof(List))] +internal sealed partial class ProxyStateJsonContext : JsonSerializerContext; + +/// Remembers which proxies worked, so the next launch does not start from nothing. +public interface IProxyStateStore +{ + /// Reads the remembered state, keyed by . + Task> LoadAsync(CancellationToken cancellationToken = default); + + /// Writes the entries worth remembering. + Task SaveAsync(IEnumerable entries, CancellationToken cancellationToken = default); +} + +/// +/// +/// Only entries that have ever answered are stored. Remembering the dead ones would mean carrying +/// a few thousand records to save re-testing addresses that the feed republishes every five +/// minutes anyway — and a proxy that was dead an hour ago tells you very little about now. +/// +public sealed class ProxyStateStore(IAppPaths paths, ILogger logger) : IProxyStateStore +{ + private readonly IAppPaths _paths = paths ?? throw new ArgumentNullException(nameof(paths)); + private readonly ILogger _logger = logger ?? throw new ArgumentNullException(nameof(logger)); + + /// + public async Task> LoadAsync( + CancellationToken cancellationToken = default + ) + { + try + { + if (!File.Exists(_paths.ProxyStateFile)) + { + return new Dictionary(StringComparer.Ordinal); + } + + var json = await File.ReadAllTextAsync(_paths.ProxyStateFile, cancellationToken).ConfigureAwait(false); + var records = JsonSerializer.Deserialize(json, ProxyStateJsonContext.Default.ListProxyStateRecord) ?? []; + + var byKey = new Dictionary(records.Count, StringComparer.Ordinal); + foreach (var record in records) + { + if (ProxyEndpoint.TryParse(record.Address, out var endpoint)) + { + byKey[endpoint.Key] = record; + } + } + + _logger.LogInformation("Restored {Count} remembered proxies", byKey.Count); + return byKey; + } + catch (Exception ex) when (ex is IOException or UnauthorizedAccessException or JsonException) + { + _logger.LogWarning(ex, "Could not read {Path}; starting without remembered proxies", _paths.ProxyStateFile); + return new Dictionary(StringComparer.Ordinal); + } + } + + /// + public async Task SaveAsync(IEnumerable entries, CancellationToken cancellationToken = default) + { + ArgumentNullException.ThrowIfNull(entries); + + var records = entries + .Where(entry => entry.HasEverAnswered) + .Select(entry => new ProxyStateRecord( + entry.Endpoint.ToString(), + entry.IsBelievedAlive, + entry.Latency?.TotalMilliseconds, + entry.SuccessCount, + entry.FailureCount, + entry.LastCheckedUtc + )) + .ToList(); + + try + { + Directory.CreateDirectory(Path.GetDirectoryName(_paths.ProxyStateFile)!); + + var temp = _paths.ProxyStateFile + ".tmp"; + var json = JsonSerializer.Serialize(records, ProxyStateJsonContext.Default.ListProxyStateRecord); + + await File.WriteAllTextAsync(temp, json, cancellationToken).ConfigureAwait(false); + File.Move(temp, _paths.ProxyStateFile, overwrite: true); + + _logger.LogDebug("Remembered {Count} proxies", records.Count); + } + catch (Exception ex) when (ex is IOException or UnauthorizedAccessException) + { + _logger.LogWarning(ex, "Could not write {Path}", _paths.ProxyStateFile); + } + } + + /// Applies remembered state onto the pool's entries. + /// How many entries were recognised. + public static int Apply(IEnumerable entries, IReadOnlyDictionary state) + { + ArgumentNullException.ThrowIfNull(entries); + ArgumentNullException.ThrowIfNull(state); + + var applied = 0; + + foreach (var entry in entries) + { + if (!state.TryGetValue(entry.Endpoint.Key, out var record)) + { + continue; + } + + entry.RestoreState( + record.Alive, + record.LatencyMs is { } ms ? TimeSpan.FromMilliseconds(ms) : null, + record.SuccessCount, + record.FailureCount, + record.LastCheckedUtc + ); + + applied++; + } + + return applied; + } +} diff --git a/src/AvParser.Infrastructure/Storage/AppPaths.cs b/src/AvParser.Infrastructure/Storage/AppPaths.cs index 3eee02c..2fe3618 100644 --- a/src/AvParser.Infrastructure/Storage/AppPaths.cs +++ b/src/AvParser.Infrastructure/Storage/AppPaths.cs @@ -17,6 +17,10 @@ public interface IAppPaths /// Kept separate from the settings file: it is a list the user edits and may want to back up or share. string CustomProxiesFile { get; } + /// Full path of the remembered proxy state. + /// Separate from the custom list: this one is derived data the app rewrites itself. + string ProxyStateFile { get; } + /// Directory holding rolling log files. string LogDirectory { get; } } @@ -50,6 +54,7 @@ public sealed class AppPaths : IAppPaths DataDirectory = dataDirectory; SettingsFile = Path.Combine(dataDirectory, "settings.json"); CustomProxiesFile = Path.Combine(dataDirectory, "proxies.custom.json"); + ProxyStateFile = Path.Combine(dataDirectory, "proxies.state.json"); LogDirectory = Path.Combine(dataDirectory, "logs"); } @@ -62,6 +67,9 @@ public sealed class AppPaths : IAppPaths /// public string CustomProxiesFile { get; } + /// + public string ProxyStateFile { get; } + /// public string LogDirectory { get; } diff --git a/src/AvParser.UI/DependencyInjection/UiServiceCollectionExtensions.cs b/src/AvParser.UI/DependencyInjection/UiServiceCollectionExtensions.cs index 0acf5a8..9f3ae4f 100644 --- a/src/AvParser.UI/DependencyInjection/UiServiceCollectionExtensions.cs +++ b/src/AvParser.UI/DependencyInjection/UiServiceCollectionExtensions.cs @@ -34,6 +34,8 @@ public static class UiServiceCollectionExtensions services.AddSingleton(static sp => new ParseViewModel( sp.GetRequiredService(), sp.GetRequiredService(), + sp.GetRequiredService(), + sp, sp.GetRequiredService>() )); services.AddSingleton(static sp => new SettingsViewModel( diff --git a/src/AvParser.UI/Localization/Strings.resx b/src/AvParser.UI/Localization/Strings.resx index affb37c..1746f3e 100644 --- a/src/AvParser.UI/Localization/Strings.resx +++ b/src/AvParser.UI/Localization/Strings.resx @@ -487,4 +487,25 @@ Key is empty. + + {0} in the pool, {1} live. + + + This parser needs a working proxy, and none is available. Check the proxy list, or allow direct connections in settings. + + + Open proxies + + + Allow network parsers to run without a proxy + + + When off, a parser that fetches anything refuses to start until at least one proxy answers. Parsers that only read pasted text are never blocked. + + + LIVE PROXIES TO FIND AT STARTUP + + + Startup checks the ones that worked last time first and stops as soon as it has this many. Raising it makes the first launch slower. + diff --git a/src/AvParser.UI/Localization/Strings.ru.resx b/src/AvParser.UI/Localization/Strings.ru.resx index e63b44a..c3c4c19 100644 --- a/src/AvParser.UI/Localization/Strings.ru.resx +++ b/src/AvParser.UI/Localization/Strings.ru.resx @@ -487,4 +487,25 @@ Пустой ключ. + + В пуле {0}, живых {1}. + + + Этому парсеру нужна рабочая прокси, а живых нет. Проверьте список прокси или разрешите прямое подключение в настройках. + + + Открыть прокси + + + Разрешить сетевым парсерам работать без прокси + + + Когда выключено, парсер, который куда-то ходит, не запустится, пока не ответит хотя бы одна прокси. Парсеры, читающие вставленный текст, не блокируются никогда. + + + СКОЛЬКО ЖИВЫХ ИСКАТЬ ПРИ СТАРТЕ + + + При старте сначала проверяются те, что работали в прошлый раз, и проверка прекращается, как только набралось столько. Больше значение — дольше первый запуск. + diff --git a/src/AvParser.UI/ViewModels/ParseViewModel.cs b/src/AvParser.UI/ViewModels/ParseViewModel.cs index 1e3c235..a9b5027 100644 --- a/src/AvParser.UI/ViewModels/ParseViewModel.cs +++ b/src/AvParser.UI/ViewModels/ParseViewModel.cs @@ -3,12 +3,16 @@ using System.Diagnostics; using System.Globalization; using System.Text; using AvParser.Core.Parsing; +using AvParser.Core.Proxies; using AvParser.Core.Settings; using AvParser.UI.Localization; +using AvParser.UI.Navigation; +using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Logging; using ReactiveUI; using ReactiveUI.Primitives; using ReactiveUI.Primitives.Concurrency; +using ReactiveUI.Primitives.Signals; using ReactiveUI.SourceGenerators; namespace AvParser.UI.ViewModels; @@ -18,7 +22,7 @@ namespace AvParser.UI.ViewModels; /// This page exists to exercise the whole contract — /// streaming, progress and cancellation — rather than to be a finished feature. /// -public partial class ParseViewModel : PageViewModel +public partial class ParseViewModel : PageViewModel, IDisposable { /// Records buffered before being pushed to the UI collection in one go. private const int BatchSize = 512; @@ -31,10 +35,14 @@ public partial class ParseViewModel : PageViewModel private readonly IParserCatalog _catalog; private readonly ISettingsService _settings; + private readonly IProxyPool _proxyPool; + private readonly IServiceProvider _services; private readonly ILogger _logger; private readonly ISequencer _mainThread; private readonly ObservableAsPropertyHelper _isBusy; + private readonly Signal _proxyChanged = new(); + private CancellationTokenSource? _cancellation; /// Text to parse. @@ -53,9 +61,21 @@ public partial class ParseViewModel : PageViewModel [Reactive] public partial string? StatusMessage { get; set; } + /// + /// Whether the selected parser needs the network but has no working proxy to use. + /// + /// + /// Only network parsers are gated. A parser that works on text the user pasted has nothing to + /// route, and blocking it would make the app unusable whenever the public lists are down. + /// + [Reactive] + public partial bool IsBlockedWithoutProxy { get; set; } + /// Creates the page. /// Available parsers. /// Used to remember the selected parser. + /// Consulted for the live count that gates network parsers. + /// Resolves the navigation service lazily, to keep pages acyclic. /// Diagnostics. /// /// Scheduler used to marshal collection and progress updates back to the UI thread. Tests @@ -64,12 +84,16 @@ public partial class ParseViewModel : PageViewModel public ParseViewModel( IParserCatalog catalog, ISettingsService settings, + IProxyPool proxyPool, + IServiceProvider services, ILogger logger, ISequencer? mainThread = null ) { _catalog = catalog ?? throw new ArgumentNullException(nameof(catalog)); _settings = settings ?? throw new ArgumentNullException(nameof(settings)); + _proxyPool = proxyPool ?? throw new ArgumentNullException(nameof(proxyPool)); + _services = services ?? throw new ArgumentNullException(nameof(services)); _logger = logger ?? throw new ArgumentNullException(nameof(logger)); _mainThread = mainThread ?? RxSchedulers.MainThreadScheduler; @@ -77,11 +101,30 @@ public partial class ParseViewModel : PageViewModel Parsers = [.. catalog.Parsers.Select(parser => new ParserViewModel(parser))]; SelectedParser = Parsers.First(parser => parser.Id == catalog.FindOrDefault(settings.Current.LastParserId).Id); - var canParse = this.WhenAnyValue(x => x.InputText) - .Select(static text => !string.IsNullOrWhiteSpace(text)) + // The pool changes on every lease outcome and on every probe, so coalesce before + // re-evaluating whether the parser is allowed to run. + _proxyPool.Changed += OnProxyPoolChanged; + _proxyChanged + .Throttle(TimeSpan.FromMilliseconds(250), _mainThread) + .ObserveOn(_mainThread) + .Subscribe(_ => RefreshProxyGate()); + + RefreshProxyGate(); + + var canParse = this.WhenAnyValue( + x => x.InputText, + x => x.IsBlockedWithoutProxy, + static (text, blocked) => (text, blocked) + ) + .Select(static state => !string.IsNullOrWhiteSpace(state.text) && !state.blocked) .DistinctUntilChanged(); ParseCommand = ReactiveCommand.CreateFromTask(RunParseAsync, canParse, _mainThread); + + GoToProxiesCommand = ReactiveCommand.Create( + () => _services.GetRequiredService().NavigateTo(), + outputScheduler: _mainThread + ); _isBusy = ParseCommand.IsExecuting.ToProperty(this, nameof(IsBusy), false, _mainThread); CancelCommand = ReactiveCommand.Create(() => _cancellation?.Cancel(), ParseCommand.IsExecuting, _mainThread); @@ -111,9 +154,17 @@ public partial class ParseViewModel : PageViewModel ); // Remember the parser choice; the debounced settings service coalesces the writes. + // Switching parser can also change whether the gate applies, since only network parsers + // are gated. this.WhenAnyValue(x => x.SelectedParser) .Where(static parser => parser is not null) - .Subscribe(parser => _settings.Update(current => current with { LastParserId = parser.Id })); + .Subscribe(parser => + { + _settings.Update(current => current with { LastParserId = parser.Id }); + RefreshProxyGate(); + }); + + _settings.Changes.Subscribe(_ => RefreshProxyGate()); // Errors surfacing from any command must not tear the process down. ParseCommand.ThrownExceptions.Subscribe(OnCommandFailed); @@ -152,6 +203,23 @@ public partial class ParseViewModel : PageViewModel /// Fills the input with 50 000 rows, so progress and cancellation are observable. public ReactiveCommand GenerateLargeSampleCommand { get; } + /// Takes the user to the page where the proxy problem can be fixed. + public ReactiveCommand GoToProxiesCommand { get; } + + /// Explains why parsing is blocked. + public string ProxyRequiredMessage => Localizer.Instance["Parse.ProxyRequired"]; + + /// Re-evaluates the proxy gate. Exposed so tests can drive it without waiting. + public void RefreshProxyGate() + { + var settings = _settings.Current; + + IsBlockedWithoutProxy = + SelectedParser.Parser.RequiresNetwork && !settings.AllowDirectConnection && _proxyPool.LiveCount == 0; + } + + private void OnProxyPoolChanged(object? sender, EventArgs e) => _proxyChanged.OnNext(RxVoid.Default); + private async Task RunParseAsync(CancellationToken commandToken) { using var cancellation = CancellationTokenSource.CreateLinkedTokenSource(commandToken); @@ -286,11 +354,22 @@ public partial class ParseViewModel : PageViewModel }); } + /// + /// The pool is a singleton and would otherwise keep this page alive for the process. + public void Dispose() + { + _proxyPool.Changed -= OnProxyPoolChanged; + _proxyChanged.Dispose(); + GC.SuppressFinalize(this); + } + /// protected override void OnLanguageChanged() { base.OnLanguageChanged(); + this.RaisePropertyChanged(nameof(ProxyRequiredMessage)); + // The summary and the listed errors were both rendered in the previous language. StatusMessage = null; diff --git a/src/AvParser.UI/ViewModels/ProxiesViewModel.cs b/src/AvParser.UI/ViewModels/ProxiesViewModel.cs index 518ae33..f2dd306 100644 --- a/src/AvParser.UI/ViewModels/ProxiesViewModel.cs +++ b/src/AvParser.UI/ViewModels/ProxiesViewModel.cs @@ -195,10 +195,11 @@ public partial class ProxiesViewModel : PageViewModel, IDisposable private async Task ReportInitialLoadAsync(IProxyPoolLoader loader) { - var count = await loader.EnsureLoadedAsync().ConfigureAwait(false); + var result = await loader.EnsureLoadedAsync().ConfigureAwait(false); var message = Localizer.Instance.Format( - "Proxies.Status.PoolHolds", - Localizer.Instance.Plural("Proxies.Count.Proxies", count) + "Proxies.Status.Ready", + Localizer.Instance.Plural("Proxies.Count.Proxies", result.Total), + result.Live ); // Anything the user has done since — an add, a check — is more interesting than the diff --git a/src/AvParser.UI/ViewModels/SettingsViewModel.cs b/src/AvParser.UI/ViewModels/SettingsViewModel.cs index 2fd20f3..03a460d 100644 --- a/src/AvParser.UI/ViewModels/SettingsViewModel.cs +++ b/src/AvParser.UI/ViewModels/SettingsViewModel.cs @@ -58,6 +58,14 @@ public partial class SettingsViewModel : PageViewModel [Reactive] public partial int ProxyProbeConcurrency { get; set; } + /// How many working proxies the startup warm-up aims for. + [Reactive] + public partial int ProxyMinimumLive { get; set; } + + /// Whether network parsers may run with no proxy available. + [Reactive] + public partial bool AllowDirectConnection { get; set; } + /// Creates the page. public SettingsViewModel( ISettingsService settings, @@ -92,6 +100,8 @@ public partial class SettingsViewModel : PageViewModel ProxyProbeUrl = current.ProxyProbeUrl; ProxyProbeTimeoutSeconds = current.ProxyProbeTimeoutSeconds; ProxyProbeConcurrency = current.ProxyProbeConcurrency; + ProxyMinimumLive = current.ProxyMinimumLive; + AllowDirectConnection = current.AllowDirectConnection; this.WhenAnyValue(x => x.SelectedTheme).ObserveOn(scheduler).Subscribe(option => _theme.Apply(option.Value)); @@ -116,7 +126,9 @@ public partial class SettingsViewModel : PageViewModel x => x.ProxyProbeUrl, x => x.ProxyProbeTimeoutSeconds, x => x.ProxyProbeConcurrency, - (_, _, _, _, _, _) => RxVoid.Default + x => x.ProxyMinimumLive, + x => x.AllowDirectConnection, + (_, _, _, _, _, _, _, _) => RxVoid.Default ) .Throttle(TimeSpan.FromMilliseconds(200), scheduler) .ObserveOn(scheduler) @@ -222,6 +234,8 @@ public partial class SettingsViewModel : PageViewModel ProxyProbeUrl = ProxyProbeUrl, ProxyProbeTimeoutSeconds = ProxyProbeTimeoutSeconds, ProxyProbeConcurrency = ProxyProbeConcurrency, + ProxyMinimumLive = ProxyMinimumLive, + AllowDirectConnection = AllowDirectConnection, }; return applied; diff --git a/src/AvParser.UI/Views/ParseView.axaml b/src/AvParser.UI/Views/ParseView.axaml index cc8197a..1e27ff8 100644 --- a/src/AvParser.UI/Views/ParseView.axaml +++ b/src/AvParser.UI/Views/ParseView.axaml @@ -8,7 +8,7 @@ x:Class="AvParser.UI.Views.ParseView" x:DataType="vm:ParseViewModel" > - + @@ -65,8 +65,44 @@ + + + + + + + + + + + + - + - + + + + + + + + + + + + diff --git a/tests/AvParser.Core.Tests/Proxies/ProxyPoolWarmUpTests.cs b/tests/AvParser.Core.Tests/Proxies/ProxyPoolWarmUpTests.cs new file mode 100644 index 0000000..eb80e8b --- /dev/null +++ b/tests/AvParser.Core.Tests/Proxies/ProxyPoolWarmUpTests.cs @@ -0,0 +1,105 @@ +using AvParser.Core.Proxies; + +namespace AvParser.Core.Tests.Proxies; + +public class ProxyPoolWarmUpTests +{ + private static readonly ProxyOptions Options = new() { ProbeConcurrency = 1 }; + + private static ProxyPool Build(out FakeProxyProbe probe, out FakeTimeProvider clock, params string[] hosts) + { + var source = new FakeProxySource(ProxySourceKind.Feed); + source.Endpoints.AddRange(hosts.Select(host => ProxyFactory.Endpoint(host))); + + probe = new FakeProxyProbe(); + clock = new FakeTimeProvider(); + + return new ProxyPool([source], probe, Options, clock); + } + + [Fact] + public async Task What_worked_last_time_is_tried_first() + { + // The whole point of remembering: a second launch should confirm the known-good ones + // rather than walking a list of a few thousand addresses from the top. + var pool = Build(out var probe, out var clock, "cold-a", "known-good", "cold-b"); + await pool.RefreshAsync(TestContext.Current.CancellationToken); + + pool.Entries.Single(entry => entry.Endpoint.Host == "known-good") + .RestoreState(wasAlive: true, TimeSpan.FromMilliseconds(30), 12, 1, clock.GetUtcNow()); + + pool.WarmUpOrder()[0].Endpoint.Host.ShouldBe("known-good"); + + // Asking for two: one is already known live, so the warm-up has a reason to probe at all. + probe.DefaultAlive = true; + await pool.WarmUpAsync(2, cancellationToken: TestContext.Current.CancellationToken); + + probe.Probed[0].ShouldContain("known-good"); + } + + [Fact] + public async Task The_faster_of_two_remembered_proxies_goes_first() + { + var pool = Build(out _, out var clock, "slow", "fast"); + await pool.RefreshAsync(TestContext.Current.CancellationToken); + + pool.Entries.Single(entry => entry.Endpoint.Host == "slow") + .RestoreState(wasAlive: true, TimeSpan.FromMilliseconds(900), 3, 0, clock.GetUtcNow()); + pool.Entries.Single(entry => entry.Endpoint.Host == "fast") + .RestoreState(wasAlive: true, TimeSpan.FromMilliseconds(40), 3, 0, clock.GetUtcNow()); + + pool.WarmUpOrder().Select(entry => entry.Endpoint.Host).ShouldBe(["fast", "slow"]); + } + + [Fact] + public async Task Warming_up_stops_once_the_target_is_met() + { + // Probing all 200 when 2 were asked for would turn every launch into a full sweep. + var pool = Build(out var probe, out _, [.. Enumerable.Range(0, 200).Select(i => $"h{i}")]); + await pool.RefreshAsync(TestContext.Current.CancellationToken); + probe.DefaultAlive = true; + + var live = await pool.WarmUpAsync(2, cancellationToken: TestContext.Current.CancellationToken); + + live.ShouldBeGreaterThanOrEqualTo(2); + probe.ProbeCount.ShouldBeLessThan(200); + } + + [Fact] + public async Task Warming_up_probes_nothing_when_enough_are_already_live() + { + var pool = Build(out var probe, out var clock, "a", "b"); + await pool.RefreshAsync(TestContext.Current.CancellationToken); + + foreach (var entry in pool.Entries) + { + entry.RecordSuccess(clock.GetUtcNow(), TimeSpan.FromMilliseconds(10)); + } + + (await pool.WarmUpAsync(2, cancellationToken: TestContext.Current.CancellationToken)).ShouldBe(2); + probe.ProbeCount.ShouldBe(0); + } + + [Fact] + public async Task A_quarantined_proxy_is_not_warmed_up() + { + var pool = Build(out _, out var clock, "a", "b"); + await pool.RefreshAsync(TestContext.Current.CancellationToken); + + pool.Entries.Single(entry => entry.Endpoint.Host == "a") + .RecordFailure(clock.GetUtcNow(), TimeSpan.FromMinutes(5), TimeSpan.FromHours(1), 1); + + pool.WarmUpOrder().Select(entry => entry.Endpoint.Host).ShouldBe(["b"]); + } + + [Fact] + public async Task Nothing_live_reads_as_nothing_live() + { + var pool = Build(out var probe, out _, "a", "b"); + await pool.RefreshAsync(TestContext.Current.CancellationToken); + probe.DefaultAlive = false; + + (await pool.WarmUpAsync(1, cancellationToken: TestContext.Current.CancellationToken)).ShouldBe(0); + pool.LiveCount.ShouldBe(0); + } +} diff --git a/tests/AvParser.Infrastructure.Tests/ProxyPoolLoaderTests.cs b/tests/AvParser.Infrastructure.Tests/ProxyPoolLoaderTests.cs index f73a123..7045d07 100644 --- a/tests/AvParser.Infrastructure.Tests/ProxyPoolLoaderTests.cs +++ b/tests/AvParser.Infrastructure.Tests/ProxyPoolLoaderTests.cs @@ -6,12 +6,15 @@ namespace AvParser.Infrastructure.Tests; public class ProxyPoolLoaderTests { - private static ProxyPoolLoader Build(out CountingSource source, out ProxyPool pool) + private static ProxyPoolLoader Build(out CountingSource source, out ProxyPool pool) => + Build(out source, out pool, new MemoryStateStore()); + + private static ProxyPoolLoader Build(out CountingSource source, out ProxyPool pool, IProxyStateStore stateStore) { source = new CountingSource(); pool = new ProxyPool([source], new NeverProbe(), new ProxyOptions()); - return new ProxyPoolLoader(pool, NullLogger.Instance); + return new ProxyPoolLoader(pool, stateStore, NullLogger.Instance); } [Fact] @@ -19,11 +22,42 @@ public class ProxyPoolLoaderTests { var loader = Build(out _, out var pool); - (await loader.EnsureLoadedAsync()).ShouldBe(2); + (await loader.EnsureLoadedAsync()).Total.ShouldBe(2); pool.Entries.Count.ShouldBe(2); loader.IsLoaded.ShouldBeTrue(); } + [Fact] + public async Task What_the_previous_run_learned_is_restored_onto_the_fresh_list() + { + // The feed republishes the same addresses every few minutes; the point of remembering is + // that a proxy known to work is not re-discovered from scratch on every launch. + var stateStore = new MemoryStateStore + { + State = + { + ["http://1.2.3.4:8080"] = new ProxyStateRecord("http://1.2.3.4:8080", Alive: true, LatencyMs: 120), + }, + }; + + var loader = Build(out _, out _, stateStore); + + var result = await loader.EnsureLoadedAsync(); + + result.Restored.ShouldBe(1); + } + + [Fact] + public async Task The_pool_state_is_written_back_after_a_load() + { + var stateStore = new MemoryStateStore(); + var loader = Build(out _, out _, stateStore); + + await loader.EnsureLoadedAsync(); + + stateStore.Saves.ShouldBeGreaterThan(0); + } + [Fact] public async Task The_feed_is_fetched_once_however_many_callers_there_are() { @@ -52,10 +86,27 @@ public class ProxyPoolLoaderTests public async Task A_source_that_throws_does_not_take_startup_down() { var pool = new ProxyPool([new ThrowingSource()], new NeverProbe(), new ProxyOptions()); - var loader = new ProxyPoolLoader(pool, NullLogger.Instance); + var loader = new ProxyPoolLoader(pool, new MemoryStateStore(), NullLogger.Instance); // The app has to start whether or not a public list is reachable. - (await loader.EnsureLoadedAsync()).ShouldBe(0); + (await loader.EnsureLoadedAsync()).Total.ShouldBe(0); + } + + private sealed class MemoryStateStore : IProxyStateStore + { + public Dictionary State { get; } = new(StringComparer.Ordinal); + + public int Saves { get; private set; } + + public Task> LoadAsync( + CancellationToken cancellationToken = default + ) => Task.FromResult>(State); + + public Task SaveAsync(IEnumerable entries, CancellationToken cancellationToken = default) + { + Saves++; + return Task.CompletedTask; + } } private sealed class CountingSource : IProxySource diff --git a/tests/AvParser.Infrastructure.Tests/ProxyStateStoreTests.cs b/tests/AvParser.Infrastructure.Tests/ProxyStateStoreTests.cs new file mode 100644 index 0000000..061cd6c --- /dev/null +++ b/tests/AvParser.Infrastructure.Tests/ProxyStateStoreTests.cs @@ -0,0 +1,166 @@ +using AvParser.Core.Proxies; +using AvParser.Infrastructure.Proxies; +using AvParser.Infrastructure.Storage; +using Microsoft.Extensions.Logging.Abstractions; + +namespace AvParser.Infrastructure.Tests; + +public sealed class ProxyStateStoreTests : IDisposable +{ + private readonly string _directory = Path.Combine( + Path.GetTempPath(), + "AvParserTests", + Guid.NewGuid().ToString("N") + ); + + private ProxyStateStore Create() => new(new AppPaths(_directory), NullLogger.Instance); + + public void Dispose() + { + if (Directory.Exists(_directory)) + { + Directory.Delete(_directory, recursive: true); + } + } + + private static ProxyEntry Entry(string address, ProxySourceKind kind = ProxySourceKind.Feed) + { + ProxyEndpoint.TryParse(address, out var endpoint).ShouldBeTrue(); + + return new ProxyEntry(endpoint!, kind); + } + + [Fact] + public async Task An_absent_file_reads_as_nothing_remembered() + { + var store = Create(); + + (await store.LoadAsync(TestContext.Current.CancellationToken)).ShouldBeEmpty(); + } + + [Fact] + public async Task A_working_proxy_survives_a_round_trip() + { + var alive = Entry("http://1.2.3.4:8080"); + alive.RecordSuccess(DateTimeOffset.UtcNow, TimeSpan.FromMilliseconds(250)); + + await Create().SaveAsync([alive], TestContext.Current.CancellationToken); + var state = await Create().LoadAsync(TestContext.Current.CancellationToken); + + state.ShouldContainKey(alive.Endpoint.Key); + state[alive.Endpoint.Key].Alive.ShouldBeTrue(); + state[alive.Endpoint.Key].LatencyMs.ShouldBe(250d); + } + + [Fact] + public async Task Proxies_that_never_answered_are_not_remembered() + { + // The feed republishes a few thousand dead addresses every few minutes; carrying them over + // would bloat the file to save re-testing entries whose staleness tells us nothing. + var untried = Entry("http://1.2.3.4:8080"); + var dead = Entry("http://5.6.7.8:3128"); + dead.RecordProbe(DateTimeOffset.UtcNow, alive: false, latency: null, error: "timeout"); + + await Create().SaveAsync([untried, dead], TestContext.Current.CancellationToken); + + (await Create().LoadAsync(TestContext.Current.CancellationToken)).ShouldBeEmpty(); + } + + [Fact] + public void Applying_remembered_state_restores_the_matching_entries_only() + { + var known = Entry("http://1.2.3.4:8080"); + var unknown = Entry("http://9.9.9.9:8080"); + var state = new Dictionary(StringComparer.Ordinal) + { + [known.Endpoint.Key] = new( + known.Endpoint.ToString(), + Alive: true, + LatencyMs: 120, + SuccessCount: 7, + FailureCount: 2 + ), + }; + + ProxyStateStore.Apply([known, unknown], state).ShouldBe(1); + + known.WasAliveOnLastRun.ShouldBeTrue(); + known.Latency.ShouldBe(TimeSpan.FromMilliseconds(120)); + known.SuccessCount.ShouldBe(7); + unknown.WasAliveOnLastRun.ShouldBeFalse(); + } + + [Fact] + public void A_remembered_proxy_is_not_reported_live_until_it_answers_again() + { + // Otherwise a launch a week later would open the parser gate on week-old evidence, and the + // warm-up would skip the very proxies it was supposed to re-check. + var entry = Entry("http://1.2.3.4:8080"); + var state = new Dictionary(StringComparer.Ordinal) + { + [entry.Endpoint.Key] = new(entry.Endpoint.ToString(), Alive: true), + }; + + ProxyStateStore.Apply([entry], state); + + entry.Health.ShouldBe(ProxyHealthState.Unknown); + entry.IsBelievedAlive.ShouldBeTrue(); + } + + [Fact] + public async Task A_remembered_proxy_that_was_never_re_checked_is_still_remembered() + { + // The warm-up stops early, so most remembered entries end a session unprobed. Dropping + // them on save would erode the remembered set to nothing over a few launches. + var entry = Entry("http://1.2.3.4:8080"); + ProxyStateStore.Apply( + [entry], + new Dictionary(StringComparer.Ordinal) + { + [entry.Endpoint.Key] = new(entry.Endpoint.ToString(), Alive: true), + } + ); + + await Create().SaveAsync([entry], TestContext.Current.CancellationToken); + + (await Create().LoadAsync(TestContext.Current.CancellationToken)).ShouldContainKey(entry.Endpoint.Key); + } + + [Fact] + public async Task A_remembered_proxy_that_fails_its_re_check_is_forgotten() + { + var entry = Entry("http://1.2.3.4:8080"); + ProxyStateStore.Apply( + [entry], + new Dictionary(StringComparer.Ordinal) + { + [entry.Endpoint.Key] = new(entry.Endpoint.ToString(), Alive: true), + } + ); + + entry.RecordProbe(DateTimeOffset.UtcNow, alive: false, latency: null, error: "timeout"); + + await Create().SaveAsync([entry], TestContext.Current.CancellationToken); + + (await Create().LoadAsync(TestContext.Current.CancellationToken)).ShouldBeEmpty(); + } + + [Fact] + public void A_remembered_proxy_is_never_restored_into_a_quarantine() + { + // The window is wall-clock; a restart may be days later, so an expired sideline must not + // be resurrected — the whole point of remembering is to start from the good ones. + var entry = Entry("http://1.2.3.4:8080"); + entry.RecordFailure(DateTimeOffset.UtcNow, TimeSpan.FromMinutes(5), TimeSpan.FromHours(1), 1); + entry.IsQuarantined(DateTimeOffset.UtcNow).ShouldBeTrue(); + + var restored = new Dictionary(StringComparer.Ordinal) + { + [entry.Endpoint.Key] = new(entry.Endpoint.ToString(), Alive: true), + }; + + ProxyStateStore.Apply([entry], restored); + + entry.IsQuarantined(DateTimeOffset.UtcNow).ShouldBeFalse(); + } +} diff --git a/tests/AvParser.UI.HeadlessTests/Fakes.cs b/tests/AvParser.UI.HeadlessTests/Fakes.cs index 9a09a1f..011886b 100644 --- a/tests/AvParser.UI.HeadlessTests/Fakes.cs +++ b/tests/AvParser.UI.HeadlessTests/Fakes.cs @@ -1,5 +1,6 @@ using AvParser.Core.Proxies; using AvParser.Core.Settings; +using AvParser.Infrastructure.Proxies; using AvParser.UI.Services; using AvParser.UI.ViewModels; using ReactiveUI.Primitives.Signals; @@ -41,6 +42,22 @@ internal sealed class FakeThemeService(AppTheme initial = AppTheme.System) : ITh public void Dispose() => _current.Dispose(); } +/// In-memory settings, so tests never touch the developer's real profile. +internal sealed class FakeSettingsService(AppSettings? initial = null) : ISettingsService, IDisposable +{ + private readonly BehaviorSignal _current = new(initial ?? new AppSettings()); + + public AppSettings Current => _current.Value; + + public IObservable Changes => _current; + + public void Update(Func mutate) => _current.OnNext(mutate(_current.Value)); + + public Task FlushAsync(CancellationToken cancellationToken = default) => Task.CompletedTask; + + public void Dispose() => _current.Dispose(); +} + /// An editable proxy list held in memory. internal sealed class FakeMutableProxySource : IMutableProxySource { @@ -80,3 +97,32 @@ internal sealed class FakeProxyProbe : IProxyProbe CancellationToken cancellationToken = default ) => Task.FromResult(ProxyProbeResult.Failure("not probed in tests")); } + +/// A remembered-state store that keeps everything in memory. +internal sealed class FakeProxyStateStore : IProxyStateStore +{ + public Dictionary State { get; } = new(StringComparer.Ordinal); + + public Task> LoadAsync( + CancellationToken cancellationToken = default + ) => Task.FromResult>(State); + + public Task SaveAsync(IEnumerable entries, CancellationToken cancellationToken = default) => + Task.CompletedTask; +} + +/// +/// A loader that does nothing. +/// +/// +/// The real one probes on startup, which would race these view tests: the load is fired from the +/// page constructor and would mark every fake proxy dead partway through an assertion. +/// +internal sealed class FakeProxyPoolLoader : IProxyPoolLoader +{ + public bool IsLoaded => true; + + public Task EnsureLoadedAsync() => Task.FromResult(new ProxyPoolLoadResult(0, 0, 0)); + + public Task SaveStateAsync(CancellationToken cancellationToken = default) => Task.CompletedTask; +} diff --git a/tests/AvParser.UI.HeadlessTests/ParseViewTests.cs b/tests/AvParser.UI.HeadlessTests/ParseViewTests.cs new file mode 100644 index 0000000..0956270 --- /dev/null +++ b/tests/AvParser.UI.HeadlessTests/ParseViewTests.cs @@ -0,0 +1,121 @@ +using Avalonia.Controls; +using Avalonia.Headless.XUnit; +using Avalonia.Threading; +using Avalonia.VisualTree; +using AvParser.Core.Parsing; +using AvParser.Core.Parsing.Samples; +using AvParser.Core.Proxies; +using AvParser.Core.Settings; +using AvParser.UI.ViewModels; +using AvParser.UI.Views; +using Microsoft.Extensions.Logging.Abstractions; +using ReactiveUI.Primitives.Concurrency; + +namespace AvParser.UI.HeadlessTests; + +public class ParseViewTests +{ + private static (ParseView View, ParseViewModel ViewModel, Window Window) ShowPage(params ITextParser[] extra) + { + var catalog = new ParserCatalog([new DelimitedTextParser(), new KeyValueTextParser(), .. extra]); + var lastParser = extra.Length > 0 ? extra[0].Id : null; + + var viewModel = new ParseViewModel( + catalog, + new FakeSettingsService(new AppSettings { LastParserId = lastParser }), + new ProxyPool([], new FakeProxyProbe(), new ProxyOptions()), + new EmptyServiceProvider(), + NullLogger.Instance, + ImmediateSequencer.Instance + ); + + var view = new ParseView { DataContext = viewModel }; + var window = new Window + { + Width = 1400, + Height = 900, + Content = view, + }; + + window.Show(); + Dispatcher.UIThread.RunJobs(); + + return (view, viewModel, window); + } + + private static Border Banner(ParseView view) => view.FindControl("ProxyGateBanner").ShouldNotBeNull(); + + [AvaloniaFact] + public void No_banner_is_shown_for_a_parser_that_needs_no_network() + { + var (view, viewModel, _) = ShowPage(); + + viewModel.IsBlockedWithoutProxy.ShouldBeFalse(); + Banner(view).IsVisible.ShouldBeFalse(); + } + + [AvaloniaFact] + public void A_blocked_network_parser_puts_the_banner_on_screen() + { + // Rendered rather than asserted on the view model: an IsVisible binding that never fires + // leaves the page silently unhelpful, which is exactly the failure this guards. + var (view, viewModel, _) = ShowPage(new NetworkParser()); + Dispatcher.UIThread.RunJobs(); + + viewModel.IsBlockedWithoutProxy.ShouldBeTrue(); + Banner(view).IsEffectivelyVisible.ShouldBeTrue(); + } + + [AvaloniaFact] + public void The_banner_offers_a_way_to_the_proxies_page() + { + var (view, viewModel, _) = ShowPage(new NetworkParser()); + Dispatcher.UIThread.RunJobs(); + + var button = Banner(view).GetVisualDescendants().OfType