mrleo1nid/av-parser
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Refactor media source handling and update collection options

Browse Source 
- Updated `IMediaSourceCatalog` to support user-added media sources, allowing dynamic editing and management of sources.
- Removed the `UrlListSource` class as its functionality is now integrated into the new catalog structure.
- Enhanced `CollectOptions` to default `RequireProxy` to true, ensuring stricter handling of proxy requirements.
- Improved error handling in `ParseError` to include a `Subject` field for better context on failures.
- Adjusted dependency injection to reflect changes in media source management, removing old source registrations.
- Introduced background proxy checks to ensure a more robust proxy pool management during collection processes.

These changes streamline the media collection process and improve the overall user experience by providing clearer error reporting and more flexible source management.
This commit is contained in:
Leonid Pershin
2026-08-15 14:20:06 +03:00
parent a4a0ea9a6b
commit eb5061ee23
63 changed files with 5165 additions and 1557 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tests/AvParser.Infrastructure.Tests/Collecting/CollectRunnerTests.cs
+1 -65
View File
@@ -138,8 +138,7 @@ public sealed class CollectRunnerTests : IAsyncLifetime
NullLogger<MediaStore>.Instance
);
// Direct is allowed here: these tests are about the runner, not the proxy gate.
_settings = new FixedSettings(new AppSettings { AllowDirectConnection = true });
_settings = new FixedSettings(new AppSettings());
_fetcher = new ScriptedFetcher(_blobs);
_throttle = new HostThrottle(4, TimeSpan.Zero, NullLogger<CollectRunnerTests>.Instance);
_runner = new CollectRunner(_fetcher, _store, _throttle, NullLogger<CollectRunner>.Instance);
@@ -354,66 +353,3 @@ public sealed class CollectRunnerTests : IAsyncLifetime
}
}
}
public class OwnServiceListingTests
{
private static readonly Uri Endpoint = new("https://own.test/api/list");
[Fact]
public void An_object_with_items_is_read()
{
var page = OwnServiceSource.ReadPage(
"""
{ "items": [ { "url": "https://own.test/a.png", "id": "42", "name": "kitten",
"published": "2026-08-13T10:00:00Z", "size": 4096, "tags": ["cats"] } ],
"next": "page2" }
""",
Endpoint
);
var item = page.Items.ShouldHaveSingleItem();
item.Url.AbsoluteUri.ShouldBe("https://own.test/a.png");
item.ExternalId.ShouldBe("42");
item.SuggestedName.ShouldBe("kitten");
item.ExpectedLength.ShouldBe(4096);
item.Tags.ShouldBe(["cats"]);
page.Next.ShouldBe("page2");
}
[Fact]
public void A_bare_array_of_addresses_is_read()
{
// The service on the other end is the user's own; it should not have to be rewritten to
// match a schema we invented.
var page = OwnServiceSource.ReadPage("""["https://own.test/a.png", "https://own.test/b.gif"]""", Endpoint);
page.Items.Count.ShouldBe(2);
page.Next.ShouldBeNull();
}
[Fact]
public void Relative_addresses_resolve_against_the_endpoint()
{
var page = OwnServiceSource.ReadPage("""{"items":[{"url":"/files/a.png"}]}""", Endpoint);
page.Items.ShouldHaveSingleItem().Url.AbsoluteUri.ShouldBe("https://own.test/files/a.png");
}
[Fact]
public void Entries_without_a_usable_address_are_dropped()
{
var page = OwnServiceSource.ReadPage(
"""{"items":[{"name":"no url"}, {"url":"data:image/png;base64,AA"}, {"url":"https://own.test/ok.png"}]}""",
Endpoint
);
page.Items.ShouldHaveSingleItem().Url.AbsoluteUri.ShouldBe("https://own.test/ok.png");
}
[Fact]
public void An_empty_listing_is_not_an_error()
{
OwnServiceSource.ReadPage("""{"items":[]}""", Endpoint).Items.ShouldBeEmpty();
OwnServiceSource.ReadPage("[]", Endpoint).Items.ShouldBeEmpty();
}
}