Enhance proxy handling and error reporting in collection process
- Introduced a `Via` field in `ParseError` to indicate the proxy address used during requests, improving clarity on error contexts. - Updated `ProxyPool` to prioritize confirmed live proxies while available, ensuring more reliable proxy selection and reducing connection timeouts. - Implemented fallback logic to allow the use of unconfirmed proxies when no confirmed ones are available, preventing collection stalls. - Adjusted logging in `CollectLogEntryViewModel` to include proxy details, enhancing error visibility for users. - Added unit tests to verify new proxy selection logic and ensure correct behavior under various conditions. These changes improve the robustness of the proxy management system and enhance the overall user experience by providing clearer error messages and more efficient proxy usage.
This commit is contained in:
Leonid Pershin
@@ -0,0 +1,86 @@
|
||||
Microsoft Visual Studio Solution File, Format Version 12.00
|
||||
# Visual Studio Version 17
|
||||
VisualStudioVersion = 17.5.2.0
|
||||
MinimumVisualStudioVersion = 10.0.40219.1
|
||||
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "AvParser.ImgTestService", "test-service\AvParser.ImgTestService.csproj", "{4BECAB72-2AD3-334F-64E9-3DF43533A0F0}"
|
||||
EndProject
|
||||
Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "src", "src", "{827E0CD3-B72D-47B6-A68D-7590B98EB39B}"
|
||||
EndProject
|
||||
Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "tests", "tests", "{0AB3BF05-4346-4AA6-1389-037BE0695223}"
|
||||
EndProject
|
||||
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "AvParser.Core", "src\AvParser.Core\AvParser.Core.csproj", "{B7028FF0-1304-5BC5-3203-0FC9073194AB}"
|
||||
EndProject
|
||||
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "AvParser.Desktop", "src\AvParser.Desktop\AvParser.Desktop.csproj", "{8E2E63A9-23C4-98AB-CE2E-43A5189D8521}"
|
||||
EndProject
|
||||
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "AvParser.Infrastructure", "src\AvParser.Infrastructure\AvParser.Infrastructure.csproj", "{68E63369-51CA-4210-B6BB-7C91620DCDAB}"
|
||||
EndProject
|
||||
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "AvParser.UI", "src\AvParser.UI\AvParser.UI.csproj", "{09558738-571C-66F2-2CFB-185FD664B97A}"
|
||||
EndProject
|
||||
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "AvParser.Core.Tests", "tests\AvParser.Core.Tests\AvParser.Core.Tests.csproj", "{FC2F99A4-60EA-A477-0C24-C3E2EEE0111C}"
|
||||
EndProject
|
||||
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "AvParser.Infrastructure.Tests", "tests\AvParser.Infrastructure.Tests\AvParser.Infrastructure.Tests.csproj", "{FBDCADBE-E06E-4FC6-E99E-C871F5908052}"
|
||||
EndProject
|
||||
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "AvParser.UI.HeadlessTests", "tests\AvParser.UI.HeadlessTests\AvParser.UI.HeadlessTests.csproj", "{92DA8927-D3BD-695A-BD6D-EB6F06290D35}"
|
||||
EndProject
|
||||
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "AvParser.UI.Tests", "tests\AvParser.UI.Tests\AvParser.UI.Tests.csproj", "{710D3D45-195B-729D-3B2D-2371A9F34C8C}"
|
||||
EndProject
|
||||
Global
|
||||
GlobalSection(SolutionConfigurationPlatforms) = preSolution
|
||||
Debug|Any CPU = Debug|Any CPU
|
||||
Release|Any CPU = Release|Any CPU
|
||||
EndGlobalSection
|
||||
GlobalSection(ProjectConfigurationPlatforms) = postSolution
|
||||
{4BECAB72-2AD3-334F-64E9-3DF43533A0F0}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
|
||||
{4BECAB72-2AD3-334F-64E9-3DF43533A0F0}.Debug|Any CPU.Build.0 = Debug|Any CPU
|
||||
{4BECAB72-2AD3-334F-64E9-3DF43533A0F0}.Release|Any CPU.ActiveCfg = Release|Any CPU
|
||||
{4BECAB72-2AD3-334F-64E9-3DF43533A0F0}.Release|Any CPU.Build.0 = Release|Any CPU
|
||||
{B7028FF0-1304-5BC5-3203-0FC9073194AB}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
|
||||
{B7028FF0-1304-5BC5-3203-0FC9073194AB}.Debug|Any CPU.Build.0 = Debug|Any CPU
|
||||
{B7028FF0-1304-5BC5-3203-0FC9073194AB}.Release|Any CPU.ActiveCfg = Release|Any CPU
|
||||
{B7028FF0-1304-5BC5-3203-0FC9073194AB}.Release|Any CPU.Build.0 = Release|Any CPU
|
||||
{8E2E63A9-23C4-98AB-CE2E-43A5189D8521}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
|
||||
{8E2E63A9-23C4-98AB-CE2E-43A5189D8521}.Debug|Any CPU.Build.0 = Debug|Any CPU
|
||||
{8E2E63A9-23C4-98AB-CE2E-43A5189D8521}.Release|Any CPU.ActiveCfg = Release|Any CPU
|
||||
{8E2E63A9-23C4-98AB-CE2E-43A5189D8521}.Release|Any CPU.Build.0 = Release|Any CPU
|
||||
{68E63369-51CA-4210-B6BB-7C91620DCDAB}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
|
||||
{68E63369-51CA-4210-B6BB-7C91620DCDAB}.Debug|Any CPU.Build.0 = Debug|Any CPU
|
||||
{68E63369-51CA-4210-B6BB-7C91620DCDAB}.Release|Any CPU.ActiveCfg = Release|Any CPU
|
||||
{68E63369-51CA-4210-B6BB-7C91620DCDAB}.Release|Any CPU.Build.0 = Release|Any CPU
|
||||
{09558738-571C-66F2-2CFB-185FD664B97A}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
|
||||
{09558738-571C-66F2-2CFB-185FD664B97A}.Debug|Any CPU.Build.0 = Debug|Any CPU
|
||||
{09558738-571C-66F2-2CFB-185FD664B97A}.Release|Any CPU.ActiveCfg = Release|Any CPU
|
||||
{09558738-571C-66F2-2CFB-185FD664B97A}.Release|Any CPU.Build.0 = Release|Any CPU
|
||||
{FC2F99A4-60EA-A477-0C24-C3E2EEE0111C}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
|
||||
{FC2F99A4-60EA-A477-0C24-C3E2EEE0111C}.Debug|Any CPU.Build.0 = Debug|Any CPU
|
||||
{FC2F99A4-60EA-A477-0C24-C3E2EEE0111C}.Release|Any CPU.ActiveCfg = Release|Any CPU
|
||||
{FC2F99A4-60EA-A477-0C24-C3E2EEE0111C}.Release|Any CPU.Build.0 = Release|Any CPU
|
||||
{FBDCADBE-E06E-4FC6-E99E-C871F5908052}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
|
||||
{FBDCADBE-E06E-4FC6-E99E-C871F5908052}.Debug|Any CPU.Build.0 = Debug|Any CPU
|
||||
{FBDCADBE-E06E-4FC6-E99E-C871F5908052}.Release|Any CPU.ActiveCfg = Release|Any CPU
|
||||
{FBDCADBE-E06E-4FC6-E99E-C871F5908052}.Release|Any CPU.Build.0 = Release|Any CPU
|
||||
{92DA8927-D3BD-695A-BD6D-EB6F06290D35}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
|
||||
{92DA8927-D3BD-695A-BD6D-EB6F06290D35}.Debug|Any CPU.Build.0 = Debug|Any CPU
|
||||
{92DA8927-D3BD-695A-BD6D-EB6F06290D35}.Release|Any CPU.ActiveCfg = Release|Any CPU
|
||||
{92DA8927-D3BD-695A-BD6D-EB6F06290D35}.Release|Any CPU.Build.0 = Release|Any CPU
|
||||
{710D3D45-195B-729D-3B2D-2371A9F34C8C}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
|
||||
{710D3D45-195B-729D-3B2D-2371A9F34C8C}.Debug|Any CPU.Build.0 = Debug|Any CPU
|
||||
{710D3D45-195B-729D-3B2D-2371A9F34C8C}.Release|Any CPU.ActiveCfg = Release|Any CPU
|
||||
{710D3D45-195B-729D-3B2D-2371A9F34C8C}.Release|Any CPU.Build.0 = Release|Any CPU
|
||||
EndGlobalSection
|
||||
GlobalSection(SolutionProperties) = preSolution
|
||||
HideSolutionNode = FALSE
|
||||
EndGlobalSection
|
||||
GlobalSection(NestedProjects) = preSolution
|
||||
{B7028FF0-1304-5BC5-3203-0FC9073194AB} = {827E0CD3-B72D-47B6-A68D-7590B98EB39B}
|
||||
{8E2E63A9-23C4-98AB-CE2E-43A5189D8521} = {827E0CD3-B72D-47B6-A68D-7590B98EB39B}
|
||||
{68E63369-51CA-4210-B6BB-7C91620DCDAB} = {827E0CD3-B72D-47B6-A68D-7590B98EB39B}
|
||||
{09558738-571C-66F2-2CFB-185FD664B97A} = {827E0CD3-B72D-47B6-A68D-7590B98EB39B}
|
||||
{FC2F99A4-60EA-A477-0C24-C3E2EEE0111C} = {0AB3BF05-4346-4AA6-1389-037BE0695223}
|
||||
{FBDCADBE-E06E-4FC6-E99E-C871F5908052} = {0AB3BF05-4346-4AA6-1389-037BE0695223}
|
||||
{92DA8927-D3BD-695A-BD6D-EB6F06290D35} = {0AB3BF05-4346-4AA6-1389-037BE0695223}
|
||||
{710D3D45-195B-729D-3B2D-2371A9F34C8C} = {0AB3BF05-4346-4AA6-1389-037BE0695223}
|
||||
EndGlobalSection
|
||||
GlobalSection(ExtensibilityGlobals) = postSolution
|
||||
SolutionGuid = {464F43C0-BBD1-48AC-9C7E-383262B8A830}
|
||||
EndGlobalSection
|
||||
EndGlobal
|
||||
@@ -56,6 +56,9 @@
|
||||
`ConcurrentQueue`, слив под `_flushGate`, иначе две гонки-выгрузки перемешают строки местами.
|
||||
- **Строка хранит ключ и аргументы, а не готовое предложение** — смена языка посреди прогона иначе
|
||||
оставит половину журнала по-английски. Литеральная половина (адрес, размер) не переводится никогда.
|
||||
- **Прокси попытки живёт в `ParseError.Via`** и печатается в строке как `via {адрес}`: «сайт ответил
|
||||
404» через подтверждённую прокси и через ту, с которой никто не разговаривал, — разные диагнозы, а
|
||||
без этого поля разница невидима.
|
||||
- **Адрес неудачи живёт в `ParseError.Subject`.** В `Message` его нет и быть не может: у шаблона
|
||||
перевода фиксированные подстановки. Без него журнал говорит «сайт ответил 404» и не говорит, на
|
||||
каком из десяти тысяч id.
|
||||
|
||||
+11
-3
@@ -12,9 +12,17 @@
|
||||
|
||||
- **`ProxyPool` переиспользует существующие `ProxyEntry` по `Endpoint.Key`** — иначе перезагрузка
|
||||
списка стирала бы статистику, а публичные фиды переиздаются каждые несколько минут.
|
||||
- **Доступность определяется карантином, а не `Health`.** `Health` — «что видели в последний раз»;
|
||||
исключать всё когда-либо упавшее значит потерять прокси после первой осечки. Уже было багом, ловит
|
||||
`A_failing_proxy_is_quarantined_and_comes_back_later`.
|
||||
- **Выдаются только подтверждённо живые, пока такие есть.** Сортировки «живые вперёд» не хватало:
|
||||
её видит только липкая стратегия, берущая голову списка, а round-robin и взвешенный выбор тянут из
|
||||
всего набора — то есть из пары тысяч непроверенных адресов, и почти каждый запрос платил полный
|
||||
connect-таймаут, чтобы это выяснить. Теперь «живых N» на странице и то, куда реально ходит сбор, —
|
||||
одно и то же число.
|
||||
- **Фолбэк на всё доступное, когда живых нет.** Непроверенная ≠ мёртвая: на холодном пуле строгий
|
||||
фильтр не выдал бы ничего и сбор просто встал бы, не попробовав. Ответившая на этом пути прокси
|
||||
сама себя переводит в живые.
|
||||
- **Карантин остаётся отдельным механизмом от `Health`.** `Health` — «что видели в последний раз»;
|
||||
исключать навсегда всё когда-либо упавшее значит потерять прокси после первой осечки. Уже было
|
||||
багом, ловит `A_failing_proxy_is_quarantined_and_comes_back_later`.
|
||||
- **`LiveCount` считает только `Alive` и не в карантине.** На нём висит гейт сбора, поэтому
|
||||
«доступна» (окно истекло) и «живая» намеренно расходятся: гейт не должен открываться от одного лишь
|
||||
истечения окна.
|
||||
|
||||
@@ -29,6 +29,14 @@ public sealed record ParseError(int Index, string Message)
|
||||
/// </remarks>
|
||||
public string? Subject { get; init; }
|
||||
|
||||
/// <summary>What the attempt went through — a proxy address — when it went through anything.</summary>
|
||||
/// <remarks>
|
||||
/// Worth carrying for the same reason as <see cref="Subject"/>: "the site answered 404" and "the
|
||||
/// request timed out" mean very different things depending on whether the pool handed out a proxy
|
||||
/// it had confirmed or one it had never spoken to, and without this the difference is invisible.
|
||||
/// </remarks>
|
||||
public string? Via { get; init; }
|
||||
|
||||
/// <summary>Creates an error carrying a translation code.</summary>
|
||||
public static ParseError Create(int index, string code, string message, params object?[] arguments) =>
|
||||
new(index, message) { Code = code, Arguments = arguments };
|
||||
|
||||
@@ -514,18 +514,36 @@ public sealed class ProxyPool : IProxyPool
|
||||
);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Chooses the entry to hand out: confirmed live ones while any exist, everything available
|
||||
/// otherwise.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// <para>
|
||||
/// Ordering is not enough. Sorting live entries to the front only helps sticky selection, which
|
||||
/// takes the head; round-robin and weighted-random draw from the whole list, and on a free feed
|
||||
/// that list is a few thousand addresses nobody has spoken to, so almost every pick was a
|
||||
/// stranger and almost every request paid a full connect timeout to find that out. What the page
|
||||
/// calls "live" and what the collector actually dials now mean the same thing.
|
||||
/// </para>
|
||||
/// <para>
|
||||
/// The fallback matters as much as the filter: with nothing confirmed live — a cold start, or a
|
||||
/// run that has just burned through the confirmed ones — a strict filter would hand out nothing
|
||||
/// and the collector would stop rather than try. Unknown is not dead, and a proxy that answers
|
||||
/// on that fallback is marked live by its own success.
|
||||
/// </para>
|
||||
/// </remarks>
|
||||
private ProxyEntry? SelectAvailable()
|
||||
{
|
||||
lock (_gate)
|
||||
{
|
||||
var now = _time.GetUtcNow();
|
||||
|
||||
// Best-first ordering: sticky selection takes the head of this list, and the other
|
||||
// strategies benefit from healthy proxies being clustered at the front.
|
||||
var candidates = _entries
|
||||
.Where(entry => entry.IsAvailable(now))
|
||||
.OrderByDescending(entry => entry.Health == ProxyHealthState.Alive)
|
||||
.ThenBy(entry => entry.Latency ?? TimeSpan.MaxValue)
|
||||
var available = _entries.Where(entry => entry.IsAvailable(now)).ToArray();
|
||||
var live = available.Where(entry => entry.Health == ProxyHealthState.Alive).ToArray();
|
||||
|
||||
var candidates = (live.Length > 0 ? live : available)
|
||||
.OrderBy(entry => entry.Latency ?? TimeSpan.MaxValue)
|
||||
.ThenByDescending(entry => entry.Endpoint.Score)
|
||||
.ToArray();
|
||||
|
||||
|
||||
@@ -314,6 +314,7 @@ public sealed class CollectRunner(
|
||||
) with
|
||||
{
|
||||
Subject = candidate.Url.AbsoluteUri,
|
||||
Via = result.ProxyKey,
|
||||
}
|
||||
);
|
||||
}
|
||||
|
||||
@@ -159,12 +159,16 @@ public sealed class CollectLogEntryViewModel : ReactiveObject
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(error);
|
||||
|
||||
// The proxy belongs on the line: a wall of timeouts through addresses nobody has confirmed
|
||||
// reads very differently from the same wall through the ones the page calls live.
|
||||
var detail = error.Via is { Length: > 0 } via ? $"{error.Subject} via {via}" : error.Subject;
|
||||
|
||||
return new CollectLogEntryViewModel(
|
||||
CollectLogLevel.Error,
|
||||
$"Collect.Error.{error.Code}",
|
||||
error.Message,
|
||||
[.. error.Arguments],
|
||||
error.Subject,
|
||||
detail,
|
||||
source
|
||||
);
|
||||
}
|
||||
|
||||
@@ -341,6 +341,59 @@ public class ProxyPoolTests
|
||||
lease!.Endpoint.Host.ShouldBe("fast");
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Only_confirmed_proxies_are_handed_out_while_any_are_confirmed()
|
||||
{
|
||||
// Ordering was not enough: round-robin and weighted-random draw from the whole list, so on a
|
||||
// feed of a few thousand unchecked addresses nearly every pick was a stranger.
|
||||
var pool = Build(
|
||||
out _,
|
||||
out _,
|
||||
out var clock,
|
||||
new ProxyOptions { Rotation = ProxyRotation.RoundRobin },
|
||||
hosts: ["unchecked-a", "known-good", "unchecked-b", "unchecked-c"]
|
||||
);
|
||||
await pool.RefreshAsync(TestContext.Current.CancellationToken);
|
||||
|
||||
pool.Entries.Single(entry => entry.Endpoint.Host == "known-good")
|
||||
.RecordProbe(clock.GetUtcNow(), alive: true, TimeSpan.FromMilliseconds(30), null);
|
||||
|
||||
for (var attempt = 0; attempt < 8; attempt++)
|
||||
{
|
||||
var lease = await pool.AcquireAsync(TestContext.Current.CancellationToken);
|
||||
lease.ShouldNotBeNull().Endpoint.Host.ShouldBe("known-good");
|
||||
}
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task With_nothing_confirmed_the_pool_still_hands_something_out()
|
||||
{
|
||||
// Unknown is not dead. A strict filter on a cold pool would stop the collector rather than
|
||||
// let it try, and a proxy that answers on this path marks itself live by succeeding.
|
||||
var pool = Build(out _, out _, out _, hosts: ["a", "b"]);
|
||||
await pool.RefreshAsync(TestContext.Current.CancellationToken);
|
||||
|
||||
pool.LiveCount.ShouldBe(0);
|
||||
(await pool.AcquireAsync(TestContext.Current.CancellationToken)).ShouldNotBeNull();
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task A_proxy_that_has_just_failed_is_not_preferred_over_a_confirmed_one()
|
||||
{
|
||||
var pool = Build(out _, out _, out var clock, hosts: ["failed", "known-good"]);
|
||||
await pool.RefreshAsync(TestContext.Current.CancellationToken);
|
||||
|
||||
// Dead but not yet quarantined: it used to stay in the draw at full weight.
|
||||
pool.Entries.Single(entry => entry.Endpoint.Host == "failed")
|
||||
.RecordProbe(clock.GetUtcNow(), alive: false, null, "no route");
|
||||
pool.Entries.Single(entry => entry.Endpoint.Host == "known-good")
|
||||
.RecordProbe(clock.GetUtcNow(), alive: true, TimeSpan.FromMilliseconds(50), null);
|
||||
|
||||
var lease = await pool.AcquireAsync(TestContext.Current.CancellationToken);
|
||||
|
||||
lease.ShouldNotBeNull().Endpoint.Host.ShouldBe("known-good");
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Changed_fires_when_the_pool_moves()
|
||||
{
|
||||
|
||||
Reference in New Issue
Block a user