Refactor media source handling and update collection options
- Updated `IMediaSourceCatalog` to support user-added media sources, allowing dynamic editing and management of sources. - Removed the `UrlListSource` class as its functionality is now integrated into the new catalog structure. - Enhanced `CollectOptions` to default `RequireProxy` to true, ensuring stricter handling of proxy requirements. - Improved error handling in `ParseError` to include a `Subject` field for better context on failures. - Adjusted dependency injection to reflect changes in media source management, removing old source registrations. - Introduced background proxy checks to ensure a more robust proxy pool management during collection processes. These changes streamline the media collection process and improve the overall user experience by providing clearer error reporting and more flexible source management.
This commit is contained in:
@@ -92,6 +92,58 @@ public class ProxyPoolLoaderTests
|
||||
(await loader.EnsureLoadedAsync()).Total.ShouldBe(0);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task What_the_warm_up_skipped_is_checked_in_the_background()
|
||||
{
|
||||
// The warm-up stops at the target, which on a real feed leaves thousands unknown. Without
|
||||
// this pass the pool reports ten live out of a few thousand and the rest were never asked.
|
||||
var source = new CountingSource();
|
||||
var pool = new ProxyPool(
|
||||
[source],
|
||||
new AliveProbe(),
|
||||
new ProxyOptions { MinimumLiveProxies = 1, ProbeConcurrency = 1 }
|
||||
);
|
||||
using var loader = new ProxyPoolLoader(pool, new MemoryStateStore(), NullLogger<ProxyPoolLoader>.Instance);
|
||||
|
||||
var result = await loader.EnsureLoadedAsync();
|
||||
|
||||
// One live was enough to finish starting up; the other is still unknown at this point.
|
||||
result.Live.ShouldBe(1);
|
||||
|
||||
await loader.TopUp.ShouldNotBeNull();
|
||||
|
||||
pool.LiveCount.ShouldBe(2);
|
||||
loader.IsToppingUp.ShouldBeFalse();
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task The_background_check_is_skipped_when_the_user_asked_for_lazy_probing()
|
||||
{
|
||||
// Lazy means "check a proxy when you hand it out"; sweeping the list behind the user's back
|
||||
// is exactly what they switched off.
|
||||
var pool = new ProxyPool(
|
||||
[new CountingSource()],
|
||||
new AliveProbe(),
|
||||
new ProxyOptions { HealthCheck = ProxyHealthCheck.Lazy }
|
||||
);
|
||||
using var loader = new ProxyPoolLoader(pool, new MemoryStateStore(), NullLogger<ProxyPoolLoader>.Instance);
|
||||
|
||||
await loader.EnsureLoadedAsync();
|
||||
|
||||
loader.TopUp.ShouldBeNull();
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Disposing_twice_is_safe()
|
||||
{
|
||||
var loader = Build(out _, out _);
|
||||
await loader.EnsureLoadedAsync();
|
||||
|
||||
loader.Dispose();
|
||||
|
||||
Should.NotThrow(loader.Dispose);
|
||||
}
|
||||
|
||||
private sealed class MemoryStateStore : IProxyStateStore
|
||||
{
|
||||
public Dictionary<string, ProxyStateRecord> State { get; } = new(StringComparer.Ordinal);
|
||||
@@ -150,4 +202,13 @@ public class ProxyPoolLoaderTests
|
||||
CancellationToken cancellationToken = default
|
||||
) => Task.FromResult(ProxyProbeResult.Failure("not used"));
|
||||
}
|
||||
|
||||
private sealed class AliveProbe : IProxyProbe
|
||||
{
|
||||
public Task<ProxyProbeResult> ProbeAsync(
|
||||
ProxyEndpoint endpoint,
|
||||
ProxyOptions options,
|
||||
CancellationToken cancellationToken = default
|
||||
) => Task.FromResult(ProxyProbeResult.Success(TimeSpan.FromMilliseconds(15)));
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user