Refactor media source handling and update collection options

- Updated `IMediaSourceCatalog` to support user-added media sources, allowing dynamic editing and management of sources.
- Removed the `UrlListSource` class as its functionality is now integrated into the new catalog structure.
- Enhanced `CollectOptions` to default `RequireProxy` to true, ensuring stricter handling of proxy requirements.
- Improved error handling in `ParseError` to include a `Subject` field for better context on failures.
- Adjusted dependency injection to reflect changes in media source management, removing old source registrations.
- Introduced background proxy checks to ensure a more robust proxy pool management during collection processes.

These changes streamline the media collection process and improve the overall user experience by providing clearer error reporting and more flexible source management.
This commit is contained in:
Leonid Pershin
2026-08-15 14:20:06 +03:00
parent a4a0ea9a6b
commit eb5061ee23
63 changed files with 5165 additions and 1557 deletions
@@ -102,4 +102,57 @@ public class ProxyPoolWarmUpTests
(await pool.WarmUpAsync(1, cancellationToken: TestContext.Current.CancellationToken)).ShouldBe(0);
pool.LiveCount.ShouldBe(0);
}
[Fact]
public async Task The_top_up_checks_what_the_warm_up_skipped()
{
// The warm-up leaves almost everything unknown by design; without this pass the pool looks
// — and behaves — as if it held two proxies rather than the fifty that answer.
var pool = Build(out var probe, out _, [.. Enumerable.Range(0, 50).Select(index => $"h{index}")]);
await pool.RefreshAsync(TestContext.Current.CancellationToken);
probe.DefaultAlive = true;
await pool.WarmUpAsync(2, cancellationToken: TestContext.Current.CancellationToken);
var afterWarmUp = probe.ProbeCount;
afterWarmUp.ShouldBeLessThan(50);
var found = await pool.TopUpAsync(4, cancellationToken: TestContext.Current.CancellationToken);
found.ShouldBe(50 - afterWarmUp);
pool.LiveCount.ShouldBe(50);
pool.Entries.ShouldAllBe(entry => entry.Health == ProxyHealthState.Alive);
}
[Fact]
public async Task The_top_up_does_not_re_probe_what_is_already_known()
{
var pool = Build(out var probe, out _, "a", "b");
await pool.RefreshAsync(TestContext.Current.CancellationToken);
probe.DefaultAlive = true;
await pool.TopUpAsync(2, cancellationToken: TestContext.Current.CancellationToken);
var first = probe.ProbeCount;
// Everything has a verdict now, so a second pass has nothing to do — re-probing would just
// be a sweep, and a sweep is something the user asks for.
(await pool.TopUpAsync(2, cancellationToken: TestContext.Current.CancellationToken)).ShouldBe(0);
probe.ProbeCount.ShouldBe(first);
}
[Fact]
public async Task A_stopped_top_up_keeps_what_it_learned()
{
var pool = Build(out var probe, out _, [.. Enumerable.Range(0, 20).Select(index => $"h{index}")]);
await pool.RefreshAsync(TestContext.Current.CancellationToken);
probe.DefaultAlive = true;
using var stop = new CancellationTokenSource();
await stop.CancelAsync();
await Should.ThrowAsync<OperationCanceledException>(() => pool.TopUpAsync(2, null, stop.Token));
// Cancelled before anything was probed, so nothing is claimed to be live either.
pool.LiveCount.ShouldBe(0);
}
}