Implement media rule management and enhance proxy handling

- Added methods to `IMediaStore` for loading, saving, and removing media rules, allowing users to manage rules for media items effectively.
- Updated `MediaFetcher` to utilize the new rule management system, integrating rule checks into the fetching process to handle geo-blocks and previously ruled items.
- Enhanced `ProxyPool` to support exclusion of proxies from specific countries during acquisition, improving the handling of geo-blocked content.
- Adjusted `FetchOptions` to include rules instead of tombstones, streamlining the decision-making process during media fetching.
- Updated UI components to support rule editing, providing users with a more interactive experience when managing media rules.

These changes improve the overall media collection process by allowing users to define rules for handling media items and enhancing the proxy management system for better content accessibility.
This commit is contained in:
Leonid Pershin
2026-08-15 15:10:55 +03:00
parent f72f08a7bc
commit 0fa8fb89f6
26 changed files with 947 additions and 46 deletions
@@ -76,6 +76,38 @@ public class PatternMediaSourceTests
candidate.Url.AbsoluteUri.ShouldBe($"https://imgtest.example/test1/{candidate.ExternalId}.jpg");
}
[Fact]
public async Task Several_extensions_become_one_candidate_with_fallbacks_in_order()
{
// One id is one picture: three candidates for three suffixes would fetch the hit and then
// hunt for its imaginary twins, and count one find as three attempts.
var source = new PatternMediaSource(
Config(min: 6, max: 6, alphabet: IdAlphabet.HexLower, extension: ".jpg, png;.gif")
);
var candidate = (await Collect(source, 1)).ShouldHaveSingleItem();
var id = candidate.ExternalId;
candidate.Url.AbsoluteUri.ShouldBe($"https://imgtest.example/test1/{id}.jpg");
candidate
.Alternatives.Select(url => url.AbsoluteUri)
.ShouldBe([$"https://imgtest.example/test1/{id}.png", $"https://imgtest.example/test1/{id}.gif"]);
candidate.Addresses.First().ShouldBe(candidate.Url);
candidate.Addresses.Count().ShouldBe(3);
}
[Fact]
public async Task A_single_extension_leaves_the_candidate_without_fallbacks()
{
var source = new PatternMediaSource(Config(min: 6, max: 6, extension: ".png"));
var candidate = (await Collect(source, 1)).ShouldHaveSingleItem();
candidate.Alternatives.ShouldBeEmpty();
candidate.Addresses.ShouldHaveSingleItem().ShouldBe(candidate.Url);
}
[Fact]
public async Task The_attempt_budget_bounds_how_many_are_generated()
{
@@ -208,6 +240,76 @@ public class PatternSourceConfigTests
config.Id.ShouldNotBeNullOrWhiteSpace();
}
[Theory]
[InlineData(".jpg,.png", ".jpg,.png")]
[InlineData("jpg png gif", ".jpg,.png,.gif")]
[InlineData(".jpg; .PNG ;jpg", ".jpg,.PNG")]
[InlineData(" ", ".jpg")]
public void Extensions_are_normalised_into_an_ordered_list(string typed, string expected)
{
// Order is an instruction, not a detail: it is the order the fetcher will try. Repeats are
// dropped because a repeat means fetching the same address twice before giving up on the id.
PatternSourceConfig.TryCreate(
"N",
"https://h/x/",
6,
8,
IdAlphabet.Digits,
null,
typed,
allowDirectConnection: false,
out var config
);
config.ShouldNotBeNull().Extension.ShouldBe(expected);
config.Extensions.ShouldBe(expected.Split(','));
}
[Fact]
public void A_doubled_trailing_slash_is_collapsed()
{
// "host//" is not "host/": ids resolve against it as //{id}, which most servers answer with
// a 404 for every attempt in the run. Easy to paste, impossible to spot in the field.
var error = PatternSourceConfig.TryCreate(
"N",
"https://imgtest.example/test1//",
6,
8,
IdAlphabet.Digits,
null,
".jpg",
allowDirectConnection: false,
out var config
);
error.ShouldBe(PatternConfigError.None);
config.ShouldNotBeNull().BaseUrl.AbsoluteUri.ShouldBe("https://imgtest.example/test1/");
}
[Fact]
public async Task Ids_hang_off_a_collapsed_base_url_without_a_doubled_slash()
{
PatternSourceConfig.TryCreate(
"N",
"https://imgtest.example/test1//",
6,
6,
IdAlphabet.Digits,
null,
".jpg",
allowDirectConnection: false,
out var config
);
var source = new PatternMediaSource(config!);
await foreach (var outcome in source.ParseAsync(new MediaQuery(Limit: 1), null, CancellationToken.None))
{
var candidate = outcome.Value.ShouldNotBeNull();
candidate.Url.AbsoluteUri.ShouldBe($"https://imgtest.example/test1/{candidate.ExternalId}.jpg");
}
}
[Fact]
public void A_blank_extension_becomes_the_default_one()
{