diff --git a/av-parser.sln b/av-parser.sln new file mode 100644 index 0000000..ea6fe9e --- /dev/null +++ b/av-parser.sln @@ -0,0 +1,86 @@ +Microsoft Visual Studio Solution File, Format Version 12.00 +# Visual Studio Version 17 +VisualStudioVersion = 17.5.2.0 +MinimumVisualStudioVersion = 10.0.40219.1 +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "AvParser.ImgTestService", "test-service\AvParser.ImgTestService.csproj", "{4BECAB72-2AD3-334F-64E9-3DF43533A0F0}" +EndProject +Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "src", "src", "{827E0CD3-B72D-47B6-A68D-7590B98EB39B}" +EndProject +Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "tests", "tests", "{0AB3BF05-4346-4AA6-1389-037BE0695223}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "AvParser.Core", "src\AvParser.Core\AvParser.Core.csproj", "{B7028FF0-1304-5BC5-3203-0FC9073194AB}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "AvParser.Desktop", "src\AvParser.Desktop\AvParser.Desktop.csproj", "{8E2E63A9-23C4-98AB-CE2E-43A5189D8521}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "AvParser.Infrastructure", "src\AvParser.Infrastructure\AvParser.Infrastructure.csproj", "{68E63369-51CA-4210-B6BB-7C91620DCDAB}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "AvParser.UI", "src\AvParser.UI\AvParser.UI.csproj", "{09558738-571C-66F2-2CFB-185FD664B97A}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "AvParser.Core.Tests", "tests\AvParser.Core.Tests\AvParser.Core.Tests.csproj", "{FC2F99A4-60EA-A477-0C24-C3E2EEE0111C}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "AvParser.Infrastructure.Tests", "tests\AvParser.Infrastructure.Tests\AvParser.Infrastructure.Tests.csproj", "{FBDCADBE-E06E-4FC6-E99E-C871F5908052}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "AvParser.UI.HeadlessTests", "tests\AvParser.UI.HeadlessTests\AvParser.UI.HeadlessTests.csproj", "{92DA8927-D3BD-695A-BD6D-EB6F06290D35}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "AvParser.UI.Tests", "tests\AvParser.UI.Tests\AvParser.UI.Tests.csproj", "{710D3D45-195B-729D-3B2D-2371A9F34C8C}" +EndProject +Global + GlobalSection(SolutionConfigurationPlatforms) = preSolution + Debug|Any CPU = Debug|Any CPU + Release|Any CPU = Release|Any CPU + EndGlobalSection + GlobalSection(ProjectConfigurationPlatforms) = postSolution + {4BECAB72-2AD3-334F-64E9-3DF43533A0F0}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {4BECAB72-2AD3-334F-64E9-3DF43533A0F0}.Debug|Any CPU.Build.0 = Debug|Any CPU + {4BECAB72-2AD3-334F-64E9-3DF43533A0F0}.Release|Any CPU.ActiveCfg = Release|Any CPU + {4BECAB72-2AD3-334F-64E9-3DF43533A0F0}.Release|Any CPU.Build.0 = Release|Any CPU + {B7028FF0-1304-5BC5-3203-0FC9073194AB}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {B7028FF0-1304-5BC5-3203-0FC9073194AB}.Debug|Any CPU.Build.0 = Debug|Any CPU + {B7028FF0-1304-5BC5-3203-0FC9073194AB}.Release|Any CPU.ActiveCfg = Release|Any CPU + {B7028FF0-1304-5BC5-3203-0FC9073194AB}.Release|Any CPU.Build.0 = Release|Any CPU + {8E2E63A9-23C4-98AB-CE2E-43A5189D8521}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {8E2E63A9-23C4-98AB-CE2E-43A5189D8521}.Debug|Any CPU.Build.0 = Debug|Any CPU + {8E2E63A9-23C4-98AB-CE2E-43A5189D8521}.Release|Any CPU.ActiveCfg = Release|Any CPU + {8E2E63A9-23C4-98AB-CE2E-43A5189D8521}.Release|Any CPU.Build.0 = Release|Any CPU + {68E63369-51CA-4210-B6BB-7C91620DCDAB}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {68E63369-51CA-4210-B6BB-7C91620DCDAB}.Debug|Any CPU.Build.0 = Debug|Any CPU + {68E63369-51CA-4210-B6BB-7C91620DCDAB}.Release|Any CPU.ActiveCfg = Release|Any CPU + {68E63369-51CA-4210-B6BB-7C91620DCDAB}.Release|Any CPU.Build.0 = Release|Any CPU + {09558738-571C-66F2-2CFB-185FD664B97A}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {09558738-571C-66F2-2CFB-185FD664B97A}.Debug|Any CPU.Build.0 = Debug|Any CPU + {09558738-571C-66F2-2CFB-185FD664B97A}.Release|Any CPU.ActiveCfg = Release|Any CPU + {09558738-571C-66F2-2CFB-185FD664B97A}.Release|Any CPU.Build.0 = Release|Any CPU + {FC2F99A4-60EA-A477-0C24-C3E2EEE0111C}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {FC2F99A4-60EA-A477-0C24-C3E2EEE0111C}.Debug|Any CPU.Build.0 = Debug|Any CPU + {FC2F99A4-60EA-A477-0C24-C3E2EEE0111C}.Release|Any CPU.ActiveCfg = Release|Any CPU + {FC2F99A4-60EA-A477-0C24-C3E2EEE0111C}.Release|Any CPU.Build.0 = Release|Any CPU + {FBDCADBE-E06E-4FC6-E99E-C871F5908052}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {FBDCADBE-E06E-4FC6-E99E-C871F5908052}.Debug|Any CPU.Build.0 = Debug|Any CPU + {FBDCADBE-E06E-4FC6-E99E-C871F5908052}.Release|Any CPU.ActiveCfg = Release|Any CPU + {FBDCADBE-E06E-4FC6-E99E-C871F5908052}.Release|Any CPU.Build.0 = Release|Any CPU + {92DA8927-D3BD-695A-BD6D-EB6F06290D35}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {92DA8927-D3BD-695A-BD6D-EB6F06290D35}.Debug|Any CPU.Build.0 = Debug|Any CPU + {92DA8927-D3BD-695A-BD6D-EB6F06290D35}.Release|Any CPU.ActiveCfg = Release|Any CPU + {92DA8927-D3BD-695A-BD6D-EB6F06290D35}.Release|Any CPU.Build.0 = Release|Any CPU + {710D3D45-195B-729D-3B2D-2371A9F34C8C}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {710D3D45-195B-729D-3B2D-2371A9F34C8C}.Debug|Any CPU.Build.0 = Debug|Any CPU + {710D3D45-195B-729D-3B2D-2371A9F34C8C}.Release|Any CPU.ActiveCfg = Release|Any CPU + {710D3D45-195B-729D-3B2D-2371A9F34C8C}.Release|Any CPU.Build.0 = Release|Any CPU + EndGlobalSection + GlobalSection(SolutionProperties) = preSolution + HideSolutionNode = FALSE + EndGlobalSection + GlobalSection(NestedProjects) = preSolution + {B7028FF0-1304-5BC5-3203-0FC9073194AB} = {827E0CD3-B72D-47B6-A68D-7590B98EB39B} + {8E2E63A9-23C4-98AB-CE2E-43A5189D8521} = {827E0CD3-B72D-47B6-A68D-7590B98EB39B} + {68E63369-51CA-4210-B6BB-7C91620DCDAB} = {827E0CD3-B72D-47B6-A68D-7590B98EB39B} + {09558738-571C-66F2-2CFB-185FD664B97A} = {827E0CD3-B72D-47B6-A68D-7590B98EB39B} + {FC2F99A4-60EA-A477-0C24-C3E2EEE0111C} = {0AB3BF05-4346-4AA6-1389-037BE0695223} + {FBDCADBE-E06E-4FC6-E99E-C871F5908052} = {0AB3BF05-4346-4AA6-1389-037BE0695223} + {92DA8927-D3BD-695A-BD6D-EB6F06290D35} = {0AB3BF05-4346-4AA6-1389-037BE0695223} + {710D3D45-195B-729D-3B2D-2371A9F34C8C} = {0AB3BF05-4346-4AA6-1389-037BE0695223} + EndGlobalSection + GlobalSection(ExtensibilityGlobals) = postSolution + SolutionGuid = {464F43C0-BBD1-48AC-9C7E-383262B8A830} + EndGlobalSection +EndGlobal diff --git a/docs/collecting.md b/docs/collecting.md index 573dc8e..d18b512 100644 --- a/docs/collecting.md +++ b/docs/collecting.md @@ -56,6 +56,9 @@ `ConcurrentQueue`, слив под `_flushGate`, иначе две гонки-выгрузки перемешают строки местами. - **Строка хранит ключ и аргументы, а не готовое предложение** — смена языка посреди прогона иначе оставит половину журнала по-английски. Литеральная половина (адрес, размер) не переводится никогда. +- **Прокси попытки живёт в `ParseError.Via`** и печатается в строке как `via {адрес}`: «сайт ответил + 404» через подтверждённую прокси и через ту, с которой никто не разговаривал, — разные диагнозы, а + без этого поля разница невидима. - **Адрес неудачи живёт в `ParseError.Subject`.** В `Message` его нет и быть не может: у шаблона перевода фиксированные подстановки. Без него журнал говорит «сайт ответил 404» и не говорит, на каком из десяти тысяч id. diff --git a/docs/proxies.md b/docs/proxies.md index 4001832..a40fc91 100644 --- a/docs/proxies.md +++ b/docs/proxies.md @@ -12,9 +12,17 @@ - **`ProxyPool` переиспользует существующие `ProxyEntry` по `Endpoint.Key`** — иначе перезагрузка списка стирала бы статистику, а публичные фиды переиздаются каждые несколько минут. -- **Доступность определяется карантином, а не `Health`.** `Health` — «что видели в последний раз»; - исключать всё когда-либо упавшее значит потерять прокси после первой осечки. Уже было багом, ловит - `A_failing_proxy_is_quarantined_and_comes_back_later`. +- **Выдаются только подтверждённо живые, пока такие есть.** Сортировки «живые вперёд» не хватало: + её видит только липкая стратегия, берущая голову списка, а round-robin и взвешенный выбор тянут из + всего набора — то есть из пары тысяч непроверенных адресов, и почти каждый запрос платил полный + connect-таймаут, чтобы это выяснить. Теперь «живых N» на странице и то, куда реально ходит сбор, — + одно и то же число. +- **Фолбэк на всё доступное, когда живых нет.** Непроверенная ≠ мёртвая: на холодном пуле строгий + фильтр не выдал бы ничего и сбор просто встал бы, не попробовав. Ответившая на этом пути прокси + сама себя переводит в живые. +- **Карантин остаётся отдельным механизмом от `Health`.** `Health` — «что видели в последний раз»; + исключать навсегда всё когда-либо упавшее значит потерять прокси после первой осечки. Уже было + багом, ловит `A_failing_proxy_is_quarantined_and_comes_back_later`. - **`LiveCount` считает только `Alive` и не в карантине.** На нём висит гейт сбора, поэтому «доступна» (окно истекло) и «живая» намеренно расходятся: гейт не должен открываться от одного лишь истечения окна. diff --git a/src/AvParser.Core/Parsing/ParseError.cs b/src/AvParser.Core/Parsing/ParseError.cs index 5d9457c..dd66ca8 100644 --- a/src/AvParser.Core/Parsing/ParseError.cs +++ b/src/AvParser.Core/Parsing/ParseError.cs @@ -29,6 +29,14 @@ public sealed record ParseError(int Index, string Message) /// public string? Subject { get; init; } + /// What the attempt went through — a proxy address — when it went through anything. + /// + /// Worth carrying for the same reason as : "the site answered 404" and "the + /// request timed out" mean very different things depending on whether the pool handed out a proxy + /// it had confirmed or one it had never spoken to, and without this the difference is invisible. + /// + public string? Via { get; init; } + /// Creates an error carrying a translation code. public static ParseError Create(int index, string code, string message, params object?[] arguments) => new(index, message) { Code = code, Arguments = arguments }; diff --git a/src/AvParser.Core/Proxies/ProxyPool.cs b/src/AvParser.Core/Proxies/ProxyPool.cs index 446f7f7..85d05f8 100644 --- a/src/AvParser.Core/Proxies/ProxyPool.cs +++ b/src/AvParser.Core/Proxies/ProxyPool.cs @@ -514,18 +514,36 @@ public sealed class ProxyPool : IProxyPool ); } + /// + /// Chooses the entry to hand out: confirmed live ones while any exist, everything available + /// otherwise. + /// + /// + /// + /// Ordering is not enough. Sorting live entries to the front only helps sticky selection, which + /// takes the head; round-robin and weighted-random draw from the whole list, and on a free feed + /// that list is a few thousand addresses nobody has spoken to, so almost every pick was a + /// stranger and almost every request paid a full connect timeout to find that out. What the page + /// calls "live" and what the collector actually dials now mean the same thing. + /// + /// + /// The fallback matters as much as the filter: with nothing confirmed live — a cold start, or a + /// run that has just burned through the confirmed ones — a strict filter would hand out nothing + /// and the collector would stop rather than try. Unknown is not dead, and a proxy that answers + /// on that fallback is marked live by its own success. + /// + /// private ProxyEntry? SelectAvailable() { lock (_gate) { var now = _time.GetUtcNow(); - // Best-first ordering: sticky selection takes the head of this list, and the other - // strategies benefit from healthy proxies being clustered at the front. - var candidates = _entries - .Where(entry => entry.IsAvailable(now)) - .OrderByDescending(entry => entry.Health == ProxyHealthState.Alive) - .ThenBy(entry => entry.Latency ?? TimeSpan.MaxValue) + var available = _entries.Where(entry => entry.IsAvailable(now)).ToArray(); + var live = available.Where(entry => entry.Health == ProxyHealthState.Alive).ToArray(); + + var candidates = (live.Length > 0 ? live : available) + .OrderBy(entry => entry.Latency ?? TimeSpan.MaxValue) .ThenByDescending(entry => entry.Endpoint.Score) .ToArray(); diff --git a/src/AvParser.Infrastructure/Collecting/CollectRunner.cs b/src/AvParser.Infrastructure/Collecting/CollectRunner.cs index b328bcb..13d6f4e 100644 --- a/src/AvParser.Infrastructure/Collecting/CollectRunner.cs +++ b/src/AvParser.Infrastructure/Collecting/CollectRunner.cs @@ -314,6 +314,7 @@ public sealed class CollectRunner( ) with { Subject = candidate.Url.AbsoluteUri, + Via = result.ProxyKey, } ); } diff --git a/src/AvParser.UI/ViewModels/CollectLogEntryViewModel.cs b/src/AvParser.UI/ViewModels/CollectLogEntryViewModel.cs index aaba383..64a8266 100644 --- a/src/AvParser.UI/ViewModels/CollectLogEntryViewModel.cs +++ b/src/AvParser.UI/ViewModels/CollectLogEntryViewModel.cs @@ -159,12 +159,16 @@ public sealed class CollectLogEntryViewModel : ReactiveObject { ArgumentNullException.ThrowIfNull(error); + // The proxy belongs on the line: a wall of timeouts through addresses nobody has confirmed + // reads very differently from the same wall through the ones the page calls live. + var detail = error.Via is { Length: > 0 } via ? $"{error.Subject} via {via}" : error.Subject; + return new CollectLogEntryViewModel( CollectLogLevel.Error, $"Collect.Error.{error.Code}", error.Message, [.. error.Arguments], - error.Subject, + detail, source ); } diff --git a/tests/AvParser.Core.Tests/Proxies/ProxyPoolTests.cs b/tests/AvParser.Core.Tests/Proxies/ProxyPoolTests.cs index 5027a5d..34b1c89 100644 --- a/tests/AvParser.Core.Tests/Proxies/ProxyPoolTests.cs +++ b/tests/AvParser.Core.Tests/Proxies/ProxyPoolTests.cs @@ -341,6 +341,59 @@ public class ProxyPoolTests lease!.Endpoint.Host.ShouldBe("fast"); } + [Fact] + public async Task Only_confirmed_proxies_are_handed_out_while_any_are_confirmed() + { + // Ordering was not enough: round-robin and weighted-random draw from the whole list, so on a + // feed of a few thousand unchecked addresses nearly every pick was a stranger. + var pool = Build( + out _, + out _, + out var clock, + new ProxyOptions { Rotation = ProxyRotation.RoundRobin }, + hosts: ["unchecked-a", "known-good", "unchecked-b", "unchecked-c"] + ); + await pool.RefreshAsync(TestContext.Current.CancellationToken); + + pool.Entries.Single(entry => entry.Endpoint.Host == "known-good") + .RecordProbe(clock.GetUtcNow(), alive: true, TimeSpan.FromMilliseconds(30), null); + + for (var attempt = 0; attempt < 8; attempt++) + { + var lease = await pool.AcquireAsync(TestContext.Current.CancellationToken); + lease.ShouldNotBeNull().Endpoint.Host.ShouldBe("known-good"); + } + } + + [Fact] + public async Task With_nothing_confirmed_the_pool_still_hands_something_out() + { + // Unknown is not dead. A strict filter on a cold pool would stop the collector rather than + // let it try, and a proxy that answers on this path marks itself live by succeeding. + var pool = Build(out _, out _, out _, hosts: ["a", "b"]); + await pool.RefreshAsync(TestContext.Current.CancellationToken); + + pool.LiveCount.ShouldBe(0); + (await pool.AcquireAsync(TestContext.Current.CancellationToken)).ShouldNotBeNull(); + } + + [Fact] + public async Task A_proxy_that_has_just_failed_is_not_preferred_over_a_confirmed_one() + { + var pool = Build(out _, out _, out var clock, hosts: ["failed", "known-good"]); + await pool.RefreshAsync(TestContext.Current.CancellationToken); + + // Dead but not yet quarantined: it used to stay in the draw at full weight. + pool.Entries.Single(entry => entry.Endpoint.Host == "failed") + .RecordProbe(clock.GetUtcNow(), alive: false, null, "no route"); + pool.Entries.Single(entry => entry.Endpoint.Host == "known-good") + .RecordProbe(clock.GetUtcNow(), alive: true, TimeSpan.FromMilliseconds(50), null); + + var lease = await pool.AcquireAsync(TestContext.Current.CancellationToken); + + lease.ShouldNotBeNull().Endpoint.Host.ShouldBe("known-good"); + } + [Fact] public async Task Changed_fires_when_the_pool_moves() {