Files
av-parser/tests/AvParser.Core.Tests/Collecting/PatternMediaSourceTests.cs
T
Leonid Pershin eb5061ee23 Refactor media source handling and update collection options
- Updated `IMediaSourceCatalog` to support user-added media sources, allowing dynamic editing and management of sources.
- Removed the `UrlListSource` class as its functionality is now integrated into the new catalog structure.
- Enhanced `CollectOptions` to default `RequireProxy` to true, ensuring stricter handling of proxy requirements.
- Improved error handling in `ParseError` to include a `Subject` field for better context on failures.
- Adjusted dependency injection to reflect changes in media source management, removing old source registrations.
- Introduced background proxy checks to ensure a more robust proxy pool management during collection processes.

These changes streamline the media collection process and improve the overall user experience by providing clearer error reporting and more flexible source management.
2026-08-15 14:20:06 +03:00

284 lines
9.2 KiB
C#

using AvParser.Core.Collecting;
using AvParser.Core.Collecting.Sources;
namespace AvParser.Core.Tests.Collecting;
public class PatternMediaSourceTests
{
private static PatternSourceConfig Config(
int min = 6,
int max = 8,
IdAlphabet alphabet = IdAlphabet.Alphanumeric,
string? custom = null,
string? extension = ".jpg",
string url = "https://imgtest.example/test1/",
bool allowDirect = false
)
{
PatternSourceConfig.TryCreate("Test", url, min, max, alphabet, custom, extension, allowDirect, out var config);
return config!;
}
private static async Task<List<MediaCandidate>> Collect(IMediaSource source, int limit)
{
var list = new List<MediaCandidate>();
await foreach (var outcome in source.ParseAsync(new MediaQuery(Limit: limit), null, CancellationToken.None))
{
if (outcome.IsSuccess)
{
list.Add(outcome.Value!);
}
}
return list;
}
[Fact]
public void A_pattern_source_needs_the_network()
{
new PatternMediaSource(Config()).RequiresNetwork.ShouldBeTrue();
}
[Fact]
public async Task Generated_ids_stay_within_the_length_range_and_alphabet()
{
var source = new PatternMediaSource(Config(min: 4, max: 6, alphabet: IdAlphabet.Digits, extension: null));
var candidates = await Collect(source, 300);
candidates.Count.ShouldBe(300);
foreach (var candidate in candidates)
{
var id = candidate.ExternalId.ShouldNotBeNull();
id.Length.ShouldBeInRange(4, 6);
id.ShouldAllBe(c => char.IsAsciiDigit(c));
}
}
[Fact]
public async Task Candidates_hang_the_id_off_the_base_url_with_the_extension()
{
var source = new PatternMediaSource(Config(min: 6, max: 6, alphabet: IdAlphabet.HexLower, extension: ".png"));
var candidate = (await Collect(source, 1)).ShouldHaveSingleItem();
candidate.SourceId.ShouldBe(source.Id);
candidate.Url.AbsoluteUri.ShouldBe($"https://imgtest.example/test1/{candidate.ExternalId}.png");
}
[Fact]
public async Task An_unset_extension_still_lands_on_a_jpg_address()
{
var source = new PatternMediaSource(Config(min: 6, max: 6, alphabet: IdAlphabet.HexLower, extension: null));
var candidate = (await Collect(source, 1)).ShouldHaveSingleItem();
candidate.Url.AbsoluteUri.ShouldBe($"https://imgtest.example/test1/{candidate.ExternalId}.jpg");
}
[Fact]
public async Task The_attempt_budget_bounds_how_many_are_generated()
{
var source = new PatternMediaSource(Config(min: 6, max: 8, alphabet: IdAlphabet.Alphanumeric));
(await Collect(source, 40)).Count.ShouldBe(40);
}
[Fact]
public async Task A_small_space_is_never_exceeded_however_large_the_budget()
{
// Digits of length 2 is a hundred ids; asking for a thousand must not loop forever.
var source = new PatternMediaSource(Config(min: 2, max: 2, alphabet: IdAlphabet.Digits, extension: null));
var candidates = await Collect(source, 1000);
candidates.Count.ShouldBeLessThanOrEqualTo(100);
candidates.Select(c => c.ExternalId).Distinct().Count().ShouldBe(candidates.Count);
}
[Fact]
public async Task No_budget_means_the_source_keeps_going_until_it_is_stopped()
{
// A limit of zero used to mean "one candidate", which made an unlimited run collect nothing.
var source = new PatternMediaSource(Config(min: 8, max: 8, alphabet: IdAlphabet.Alphanumeric));
using var stop = new CancellationTokenSource();
var seen = 0;
try
{
await foreach (var outcome in source.ParseAsync(new MediaQuery(), null, stop.Token))
{
outcome.IsSuccess.ShouldBeTrue();
if (++seen >= 5_000)
{
await stop.CancelAsync();
}
}
}
catch (OperationCanceledException)
{
// The only way an unbounded run ends.
}
seen.ShouldBe(5_000);
}
[Fact]
public async Task A_small_space_still_bounds_an_unlimited_run()
{
// Two digits is a hundred ids: with no budget the run must end at the space, not spin on
// collisions for ever.
var source = new PatternMediaSource(Config(min: 2, max: 2, alphabet: IdAlphabet.Digits, extension: null));
var candidates = await Collect(source, 0);
// Not exactly a hundred: the draw is random, so the last few ids may be given up on rather
// than waited for. What matters is that the run ends and never repeats itself.
candidates.Count.ShouldBeInRange(90, 100);
candidates.Select(c => c.ExternalId).Distinct().Count().ShouldBe(candidates.Count);
}
[Fact]
public async Task Generated_ids_are_unique_within_a_run()
{
var source = new PatternMediaSource(Config(min: 8, max: 8, alphabet: IdAlphabet.Alphanumeric, extension: null));
var candidates = await Collect(source, 500);
candidates.Select(c => c.ExternalId).Distinct().Count().ShouldBe(500);
}
}
public class PatternAlphabetTests
{
[Theory]
[InlineData(IdAlphabet.LettersLower, "abcdefghijklmnopqrstuvwxyz")]
[InlineData(IdAlphabet.Digits, "0123456789")]
[InlineData(IdAlphabet.HexLower, "0123456789abcdef")]
public void Named_sets_resolve_to_their_characters(IdAlphabet alphabet, string expected)
{
PatternAlphabet.Resolve(alphabet).ShouldBe(expected);
}
[Fact]
public void Letters_and_alphanumeric_combine_the_sets()
{
PatternAlphabet.Resolve(IdAlphabet.Letters).Length.ShouldBe(52);
PatternAlphabet.Resolve(IdAlphabet.Alphanumeric).Length.ShouldBe(62);
}
[Fact]
public void A_custom_set_is_de_duplicated_and_stripped_of_separators()
{
// Repeats would skew the draw; a slash would break the "one path segment" assumption.
PatternAlphabet.Resolve(IdAlphabet.Custom, "aab/c c").ShouldBe("abc");
}
[Fact]
public void An_empty_custom_set_resolves_to_nothing()
{
PatternAlphabet.Resolve(IdAlphabet.Custom, " ").ShouldBeEmpty();
}
}
public class PatternSourceConfigTests
{
[Fact]
public void A_valid_config_normalises_the_base_url_and_extension()
{
var error = PatternSourceConfig.TryCreate(
" My source ",
"https://imgtest.example/test1",
6,
8,
IdAlphabet.Alphanumeric,
null,
"jpg",
allowDirectConnection: false,
out var config
);
error.ShouldBe(PatternConfigError.None);
config.ShouldNotBeNull();
config.Name.ShouldBe("My source");
config.BaseUrl.AbsoluteUri.ShouldBe("https://imgtest.example/test1/");
config.Extension.ShouldBe(".jpg");
config.Id.ShouldNotBeNullOrWhiteSpace();
}
[Fact]
public void A_blank_extension_becomes_the_default_one()
{
// The editor shows ".jpg" as the placeholder, so an untouched field means "the usual one",
// not "no suffix" — the latter costs a whole run of 404s to discover.
var error = PatternSourceConfig.TryCreate(
"N",
"https://h/x/",
6,
8,
IdAlphabet.Digits,
null,
" ",
allowDirectConnection: false,
out var config
);
error.ShouldBe(PatternConfigError.None);
config.ShouldNotBeNull().Extension.ShouldBe(PatternSourceConfig.DefaultExtension);
}
[Fact]
public void A_supplied_id_is_kept()
{
PatternSourceConfig.TryCreate(
"N",
"https://h/x/",
6,
8,
IdAlphabet.Digits,
null,
null,
allowDirectConnection: false,
out var config,
"fixed-id"
);
config.ShouldNotBeNull().Id.ShouldBe("fixed-id");
}
[Theory]
[InlineData("", "https://h/x/", 6, 8, IdAlphabet.Digits, PatternConfigError.NameRequired)]
[InlineData("N", "not-a-url", 6, 8, IdAlphabet.Digits, PatternConfigError.BaseUrlInvalid)]
[InlineData("N", "ftp://h/x/", 6, 8, IdAlphabet.Digits, PatternConfigError.BaseUrlInvalid)]
[InlineData("N", "https://h/x/", 0, 8, IdAlphabet.Digits, PatternConfigError.LengthRangeInvalid)]
[InlineData("N", "https://h/x/", 8, 6, IdAlphabet.Digits, PatternConfigError.LengthRangeInvalid)]
[InlineData("N", "https://h/x/", 6, 65, IdAlphabet.Digits, PatternConfigError.LengthRangeInvalid)]
[InlineData("N", "https://h/x/", 6, 8, IdAlphabet.Custom, PatternConfigError.AlphabetEmpty)]
public void Invalid_inputs_are_rejected_with_a_reason(
string name,
string url,
int min,
int max,
IdAlphabet alphabet,
PatternConfigError expected
)
{
var error = PatternSourceConfig.TryCreate(
name,
url,
min,
max,
alphabet,
null,
null,
allowDirectConnection: false,
out var config
);
error.ShouldBe(expected);
config.ShouldBeNull();
}
}