mrleo1nid/av-parser
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Refactor media source handling and update collection options

Browse Source 
- Updated `IMediaSourceCatalog` to support user-added media sources, allowing dynamic editing and management of sources.
- Removed the `UrlListSource` class as its functionality is now integrated into the new catalog structure.
- Enhanced `CollectOptions` to default `RequireProxy` to true, ensuring stricter handling of proxy requirements.
- Improved error handling in `ParseError` to include a `Subject` field for better context on failures.
- Adjusted dependency injection to reflect changes in media source management, removing old source registrations.
- Introduced background proxy checks to ensure a more robust proxy pool management during collection processes.

These changes streamline the media collection process and improve the overall user experience by providing clearer error reporting and more flexible source management.
This commit is contained in:
Leonid Pershin
2026-08-15 14:20:06 +03:00
parent a4a0ea9a6b
commit eb5061ee23
63 changed files with 5165 additions and 1557 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tests/AvParser.Infrastructure.Tests/Collecting/CollectRunnerTests.cs
+1 -65
View File
@@ -138,8 +138,7 @@ public sealed class CollectRunnerTests : IAsyncLifetime
NullLogger<MediaStore>.Instance
);
// Direct is allowed here: these tests are about the runner, not the proxy gate.
_settings = new FixedSettings(new AppSettings { AllowDirectConnection = true });
_settings = new FixedSettings(new AppSettings());
_fetcher = new ScriptedFetcher(_blobs);
_throttle = new HostThrottle(4, TimeSpan.Zero, NullLogger<CollectRunnerTests>.Instance);
_runner = new CollectRunner(_fetcher, _store, _throttle, NullLogger<CollectRunner>.Instance);
@@ -354,66 +353,3 @@ public sealed class CollectRunnerTests : IAsyncLifetime
}
}
}
public class OwnServiceListingTests
{
private static readonly Uri Endpoint = new("https://own.test/api/list");
[Fact]
public void An_object_with_items_is_read()
{
var page = OwnServiceSource.ReadPage(
"""
{ "items": [ { "url": "https://own.test/a.png", "id": "42", "name": "kitten",
"published": "2026-08-13T10:00:00Z", "size": 4096, "tags": ["cats"] } ],
"next": "page2" }
""",
Endpoint
);
var item = page.Items.ShouldHaveSingleItem();
item.Url.AbsoluteUri.ShouldBe("https://own.test/a.png");
item.ExternalId.ShouldBe("42");
item.SuggestedName.ShouldBe("kitten");
item.ExpectedLength.ShouldBe(4096);
item.Tags.ShouldBe(["cats"]);
page.Next.ShouldBe("page2");
}
[Fact]
public void A_bare_array_of_addresses_is_read()
{
// The service on the other end is the user's own; it should not have to be rewritten to
// match a schema we invented.
var page = OwnServiceSource.ReadPage("""["https://own.test/a.png", "https://own.test/b.gif"]""", Endpoint);
page.Items.Count.ShouldBe(2);
page.Next.ShouldBeNull();
}
[Fact]
public void Relative_addresses_resolve_against_the_endpoint()
{
var page = OwnServiceSource.ReadPage("""{"items":[{"url":"/files/a.png"}]}""", Endpoint);
page.Items.ShouldHaveSingleItem().Url.AbsoluteUri.ShouldBe("https://own.test/files/a.png");
}
[Fact]
public void Entries_without_a_usable_address_are_dropped()
{
var page = OwnServiceSource.ReadPage(
"""{"items":[{"name":"no url"}, {"url":"data:image/png;base64,AA"}, {"url":"https://own.test/ok.png"}]}""",
Endpoint
);
page.Items.ShouldHaveSingleItem().Url.AbsoluteUri.ShouldBe("https://own.test/ok.png");
}
[Fact]
public void An_empty_listing_is_not_an_error()
{
OwnServiceSource.ReadPage("""{"items":[]}""", Endpoint).Items.ShouldBeEmpty();
OwnServiceSource.ReadPage("[]", Endpoint).Items.ShouldBeEmpty();
}
}
tests/AvParser.Infrastructure.Tests/Collecting/JsonUserSourceStoreTests.cs
+124
View File
@@ -0,0 +1,124 @@
using AvParser.Core.Collecting.Sources;
using AvParser.Infrastructure.Collecting;
using AvParser.Infrastructure.Storage;
using Microsoft.Extensions.Logging.Abstractions;
namespace AvParser.Infrastructure.Tests.Collecting;
public sealed class JsonUserSourceStoreTests : IDisposable
{
private readonly string _directory = Path.Combine(
Path.GetTempPath(),
"AvParserTests",
Guid.NewGuid().ToString("N")
);
private JsonUserSourceStore Create() => new(new AppPaths(_directory), NullLogger<JsonUserSourceStore>.Instance);
private static PatternSourceConfig Config(string id, string name = "Test", bool allowDirect = false)
{
PatternSourceConfig.TryCreate(
name,
"https://imgtest.example/test1/",
6,
8,
IdAlphabet.Alphanumeric,
null,
".jpg",
allowDirect,
out var config,
id
);
return config!;
}
public void Dispose()
{
if (Directory.Exists(_directory))
{
Directory.Delete(_directory, recursive: true);
}
}
[Fact]
public void An_absent_file_reads_as_an_empty_list()
{
using var store = Create();
store.List().ShouldBeEmpty();
}
[Fact]
public async Task Added_sources_survive_a_reload()
{
using (var store = Create())
{
await store.AddAsync(Config("s1", "Kept"), TestContext.Current.CancellationToken);
}
// A brand-new instance reads from disk rather than from the in-memory cache.
using var reopened = Create();
var config = reopened.List().ShouldHaveSingleItem();
config.Id.ShouldBe("s1");
config.Name.ShouldBe("Kept");
config.BaseUrl.AbsoluteUri.ShouldBe("https://imgtest.example/test1/");
config.Extension.ShouldBe(".jpg");
}
[Fact]
public async Task Updating_replaces_the_matching_config()
{
using var store = Create();
await store.AddAsync(Config("s1", "Before"), TestContext.Current.CancellationToken);
var updated = Config("s1", "After");
(await store.UpdateAsync(updated, TestContext.Current.CancellationToken)).ShouldBeTrue();
store.List().ShouldHaveSingleItem().Name.ShouldBe("After");
}
[Fact]
public async Task Updating_an_unknown_id_changes_nothing()
{
using var store = Create();
(await store.UpdateAsync(Config("ghost"), TestContext.Current.CancellationToken)).ShouldBeFalse();
store.List().ShouldBeEmpty();
}
[Fact]
public async Task Removing_takes_the_config_out()
{
using var store = Create();
await store.AddAsync(Config("s1"), TestContext.Current.CancellationToken);
(await store.RemoveAsync("s1", TestContext.Current.CancellationToken)).ShouldBeTrue();
store.List().ShouldBeEmpty();
}
[Fact]
public async Task Mutations_raise_the_changed_event()
{
using var store = Create();
var changed = 0;
store.Changed += (_, _) => changed++;
await store.AddAsync(Config("s1"), TestContext.Current.CancellationToken);
await store.RemoveAsync("s1", TestContext.Current.CancellationToken);
changed.ShouldBe(2);
}
[Fact]
public async Task A_corrupt_file_reads_as_an_empty_list()
{
Directory.CreateDirectory(_directory);
IAppPaths paths = new AppPaths(_directory);
await File.WriteAllTextAsync(paths.UserSourcesFile, "{ not json ]", TestContext.Current.CancellationToken);
using var store = Create();
store.List().ShouldBeEmpty();
}
}
tests/AvParser.Infrastructure.Tests/JsonSettingsServiceTests.cs
+6 -3
View File
@@ -156,10 +156,13 @@ public sealed class JsonSettingsServiceTests : IDisposable
}
[Fact]
public void The_proxy_gate_setting_reaches_the_collector()
public void The_collector_defaults_to_proxy_only()
{
new AppSettings(AllowDirectConnection: false).ToCollectOptions().RequireProxy.ShouldBeTrue();
new AppSettings(AllowDirectConnection: true).ToCollectOptions().RequireProxy.ShouldBeFalse();
// Whether a source may go direct is that source's own setting now; what the app-wide
// options must never do is default to the permissive answer.
new AppSettings()
.ToCollectOptions()
.RequireProxy.ShouldBeTrue();
}
[Fact]
tests/AvParser.Infrastructure.Tests/ProxyPoolLoaderTests.cs
+61
View File
@@ -92,6 +92,58 @@ public class ProxyPoolLoaderTests
(await loader.EnsureLoadedAsync()).Total.ShouldBe(0);
}
[Fact]
public async Task What_the_warm_up_skipped_is_checked_in_the_background()
{
// The warm-up stops at the target, which on a real feed leaves thousands unknown. Without
// this pass the pool reports ten live out of a few thousand and the rest were never asked.
var source = new CountingSource();
var pool = new ProxyPool(
[source],
new AliveProbe(),
new ProxyOptions { MinimumLiveProxies = 1, ProbeConcurrency = 1 }
);
using var loader = new ProxyPoolLoader(pool, new MemoryStateStore(), NullLogger<ProxyPoolLoader>.Instance);
var result = await loader.EnsureLoadedAsync();
// One live was enough to finish starting up; the other is still unknown at this point.
result.Live.ShouldBe(1);
await loader.TopUp.ShouldNotBeNull();
pool.LiveCount.ShouldBe(2);
loader.IsToppingUp.ShouldBeFalse();
}
[Fact]
public async Task The_background_check_is_skipped_when_the_user_asked_for_lazy_probing()
{
// Lazy means "check a proxy when you hand it out"; sweeping the list behind the user's back
// is exactly what they switched off.
var pool = new ProxyPool(
[new CountingSource()],
new AliveProbe(),
new ProxyOptions { HealthCheck = ProxyHealthCheck.Lazy }
);
using var loader = new ProxyPoolLoader(pool, new MemoryStateStore(), NullLogger<ProxyPoolLoader>.Instance);
await loader.EnsureLoadedAsync();
loader.TopUp.ShouldBeNull();
}
[Fact]
public async Task Disposing_twice_is_safe()
{
var loader = Build(out _, out _);
await loader.EnsureLoadedAsync();
loader.Dispose();
Should.NotThrow(loader.Dispose);
}
private sealed class MemoryStateStore : IProxyStateStore
{
public Dictionary<string, ProxyStateRecord> State { get; } = new(StringComparer.Ordinal);
@@ -150,4 +202,13 @@ public class ProxyPoolLoaderTests
CancellationToken cancellationToken = default
) => Task.FromResult(ProxyProbeResult.Failure("not used"));
}
private sealed class AliveProbe : IProxyProbe
{
public Task<ProxyProbeResult> ProbeAsync(
ProxyEndpoint endpoint,
ProxyOptions options,
CancellationToken cancellationToken = default
) => Task.FromResult(ProxyProbeResult.Success(TimeSpan.FromMilliseconds(15)));
}
}