diff --git a/docs/collecting.md b/docs/collecting.md index d18b512..4810287 100644 --- a/docs/collecting.md +++ b/docs/collecting.md @@ -42,6 +42,13 @@ ## Источники +- **Несколько расширений — это один кандидат с запасными адресами**, а не несколько кандидатов. У id + одна картинка: три кандидата на три суффикса скачали бы попадание и потом пошли искать его + несуществующих близнецов, а счётчики посчитали бы одну находку за три. Порядок — инструкция: + `MediaFetcher.FetchAddressesAsync` идёт по `MediaCandidate.Addresses` до первого ответа. +- **Дальше по списку двигает только «здесь ничего нет»**: `Gone`, `NotMedia`, `TooSmall`, + `Placeholder`. Тайм-аут или отказ прокси — это отсутствие вердикта, а не вердикт: сжечь на нём + остальные суффиксы значит объявить id отсутствующим везде по одному сломанному соединению. - **Пустое расширение — это `.jpg`, а не «без суффикса».** В редакторе это плейсхолдер, и `PatternSourceConfig.NormaliseExtension` подставляет его в домене, потому что через `TryCreate` проходят и форма, и загрузка `sources.user.json`. Голый `/{id}` почти всегда опечатка, которая diff --git a/docs/proxies.md b/docs/proxies.md index a40fc91..33af5eb 100644 --- a/docs/proxies.md +++ b/docs/proxies.md @@ -12,6 +12,13 @@ - **`ProxyPool` переиспользует существующие `ProxyEntry` по `Endpoint.Key`** — иначе перезагрузка списка стирала бы статистику, а публичные фиды переиздаются каждые несколько минут. +- **Проба обязана ходить по https.** Обычный HTTP через прокси — это форвард GET с абсолютным URI, а + https — `CONNECT`-туннель, и свободные прокси сплошь и рядом умеют первое и отказывают во втором + («The proxy tunnel request to proxy … failed with status code 400»). Все собираемые адреса идут по + https, поэтому прокси без CONNECT мертва *для нас*, и проверять надо ровно то, чем пользуемся. + Раньше умолчание было http с обоснованием «не отбраковывать тех, кто просто не умеет CONNECT» — это + и наполняло пул живыми на бумаге. Старое значение в `settings.json` подменяется на https в + `AppSettings.ToProxyOptions()`; введённое пользователем не трогается. - **Выдаются только подтверждённо живые, пока такие есть.** Сортировки «живые вперёд» не хватало: её видит только липкая стратегия, берущая голову списка, а round-robin и взвешенный выбор тянут из всего набора — то есть из пары тысяч непроверенных адресов, и почти каждый запрос платил полный diff --git a/src/AvParser.Core/Collecting/IMediaStore.cs b/src/AvParser.Core/Collecting/IMediaStore.cs index de59898..1cef18d 100644 --- a/src/AvParser.Core/Collecting/IMediaStore.cs +++ b/src/AvParser.Core/Collecting/IMediaStore.cs @@ -195,6 +195,24 @@ public interface IMediaStore /// How many stored items were removed. Task TombstoneAsync(string sha256, string? reason, CancellationToken cancellationToken = default); + /// Reads every standing rule the user has set on a picture. + /// Loaded once per run and filtered per source; the set is small and the check is hot. + Task> LoadRulesAsync(CancellationToken cancellationToken = default); + + /// + /// Stores a rule, replacing any earlier one for the same hash. + /// + /// How many stored items were removed, which only does. + /// + /// A skip rule also tombstones: the user is looking at the thing in the gallery and saying "never + /// again", and leaving the copy that prompted it on disk would answer half the request. + /// + Task SaveRuleAsync(MediaRule rule, CancellationToken cancellationToken = default); + + /// Drops the rule for a hash. Returns whether one was there. + /// Does not bring back what a skip rule deleted; those bytes are gone. + Task RemoveRuleAsync(string sha256, CancellationToken cancellationToken = default); + /// Takes ownership of a completed download and records it. Task StoreAsync(MediaStoreRequest request, CancellationToken cancellationToken = default); diff --git a/src/AvParser.Core/Collecting/MediaCandidate.cs b/src/AvParser.Core/Collecting/MediaCandidate.cs index 2b0b4ae..626f805 100644 --- a/src/AvParser.Core/Collecting/MediaCandidate.cs +++ b/src/AvParser.Core/Collecting/MediaCandidate.cs @@ -35,4 +35,19 @@ public sealed record MediaCandidate(Uri Url) /// 1-based position within the listing, used to report failures against something. public int Ordinal { get; init; } + + /// + /// Further addresses for the same item, tried in order only if the ones before them are not there. + /// + /// + /// This is what "one id, several extensions" means: a service that serves /{id}.jpg and + /// /{id}.png holds one picture per id, not two, so emitting a candidate per extension + /// would download the hit and then go looking for its twin. One candidate with fallbacks keeps + /// the counters honest — a hit is one item, not one item and two misses — and lets the fetcher + /// stop the moment something answers. + /// + public IReadOnlyList Alternatives { get; init; } = []; + + /// Every address to try, in order, starting with . + public IEnumerable Addresses => Alternatives.Count == 0 ? [Url] : [Url, .. Alternatives]; } diff --git a/src/AvParser.Core/Collecting/MediaRule.cs b/src/AvParser.Core/Collecting/MediaRule.cs new file mode 100644 index 0000000..a600399 --- /dev/null +++ b/src/AvParser.Core/Collecting/MediaRule.cs @@ -0,0 +1,92 @@ +namespace AvParser.Core.Collecting; + +/// What to do when a collected item turns out to be a particular known picture. +public enum MediaRuleAction +{ + /// Never keep it: drop the bytes and settle the address without storing anything. + Skip = 0, + + /// Fetch the same address again through a proxy in another country. + RetryElsewhere = 1, +} + +/// +/// A standing instruction about one exact picture, recognised by its content hash. +/// +/// +/// +/// Born from what a scan actually produces: services answer a missing id with a placeholder and a +/// geo-block with a banner, and both arrive as a perfectly valid image with a 200. Nothing about the +/// address distinguishes them — only the bytes do. So the user points at one in the gallery and says +/// what it means, and every future copy is treated the same way. +/// +/// +/// Matching is by SHA-256, which is exact: a re-encoded or differently sized variant of the same +/// banner is a different rule. That is the same deliberate line the store draws for de-duplication — +/// there is no perceptual hashing here, and pretending otherwise would silently drop real finds. +/// +/// +/// Content hash the rule recognises. +/// What to do with a match. +public sealed record MediaRule(string Sha256, MediaRuleAction Action) +{ + /// Separates ids inside . + public const char SourceSeparator = ';'; + + /// + /// Sources the rule covers, separated by ;; or empty means all of them. + /// + /// + /// A joined string rather than a list so the record keeps value equality and the row stays one + /// column. Scoped because the same bytes mean different things to different services: one host's + /// "removed" placeholder is another host's perfectly ordinary picture. + /// + public string? SourceIds { get; init; } + + /// Why the user made this rule, for the list they will read months later. + public string? Reason { get; init; } + + /// When it was created. + public DateTimeOffset CreatedUtc { get; init; } = DateTimeOffset.UtcNow; + + /// Whether the rule covers . + public bool AppliesTo(string? sourceId) + { + if (string.IsNullOrWhiteSpace(SourceIds)) + { + return true; + } + + if (string.IsNullOrEmpty(sourceId)) + { + return false; + } + + foreach (var id in Split(SourceIds)) + { + if (string.Equals(id, sourceId, StringComparison.OrdinalIgnoreCase)) + { + return true; + } + } + + return false; + } + + /// The ids this rule is scoped to; empty means every source. + public IReadOnlyList Sources => Split(SourceIds); + + /// Joins ids for storage in . + public static string? JoinSources(IEnumerable sourceIds) + { + ArgumentNullException.ThrowIfNull(sourceIds); + + var joined = string.Join(SourceSeparator, sourceIds); + return joined.Length == 0 ? null : joined; + } + + private static string[] Split(string? ids) => + string.IsNullOrWhiteSpace(ids) + ? [] + : ids.Split(SourceSeparator, StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries); +} diff --git a/src/AvParser.Core/Collecting/Sources/PatternMediaSource.cs b/src/AvParser.Core/Collecting/Sources/PatternMediaSource.cs index 761b506..bb621ab 100644 --- a/src/AvParser.Core/Collecting/Sources/PatternMediaSource.cs +++ b/src/AvParser.Core/Collecting/Sources/PatternMediaSource.cs @@ -112,14 +112,7 @@ public sealed class PatternMediaSource : IMediaSource stale = 0; - yield return ParseOutcome.Success( - new MediaCandidate(new Uri(_config.BaseUrl, id + _config.Extension)) - { - SourceId = Id, - ExternalId = id, - Ordinal = emitted + 1, - } - ); + yield return ParseOutcome.Success(BuildCandidate(id, emitted + 1)); emitted++; @@ -133,6 +126,37 @@ public sealed class PatternMediaSource : IMediaSource progress?.Report(new ParseProgress(emitted, budget)); } + /// + /// Builds one candidate for an id: the first suffix as the address, the rest as fallbacks. + /// + /// + /// One candidate rather than one per suffix. A service holds one picture per id, so emitting + /// three candidates for three suffixes would fetch the hit and then go hunting for its + /// non-existent twins, count one item as three, and spend three journal rows on it. + /// + private MediaCandidate BuildCandidate(string id, int ordinal) + { + var suffixes = _config.Extensions; + + if (suffixes.Count <= 1) + { + return new MediaCandidate(new Uri(_config.BaseUrl, id + _config.Extension)) + { + SourceId = Id, + ExternalId = id, + Ordinal = ordinal, + }; + } + + return new MediaCandidate(new Uri(_config.BaseUrl, id + suffixes[0])) + { + SourceId = Id, + ExternalId = id, + Ordinal = ordinal, + Alternatives = [.. suffixes.Skip(1).Select(suffix => new Uri(_config.BaseUrl, id + suffix))], + }; + } + /// /// The number of candidates to generate: the caller's budget, capped at the id space when small. /// diff --git a/src/AvParser.Core/Collecting/Sources/PatternSourceConfig.cs b/src/AvParser.Core/Collecting/Sources/PatternSourceConfig.cs index 889e6ce..924865a 100644 --- a/src/AvParser.Core/Collecting/Sources/PatternSourceConfig.cs +++ b/src/AvParser.Core/Collecting/Sources/PatternSourceConfig.cs @@ -116,8 +116,10 @@ public enum PatternConfigError /// Which characters ids are drawn from. /// Characters used when is . /// -/// Suffix appended after the id. A blank one from the editor becomes ; -/// a config built by hand may still carry for none. +/// Suffixes appended after the id, comma-separated and tried in order until one answers. A blank one +/// from the editor becomes ; a config built by hand may still carry +/// for none. Stored as one string rather than a list so the record keeps value +/// equality and an older sources.user.json holding a single suffix still reads. /// /// /// Whether this source may run from the user's own address when no proxy is live. Per source rather @@ -141,6 +143,9 @@ public sealed record PatternSourceConfig( /// Hard ceiling on id length, guarding both the UI spinner and a hand-edited file. public const int MaxIdLength = 64; + /// Separates suffixes inside . + public const char ExtensionSeparator = ','; + /// Suffix used when the editor's extension field is left empty. /// /// A blank field means "the usual one", not "no suffix": every host worth pointing this at @@ -153,6 +158,15 @@ public sealed record PatternSourceConfig( /// The characters this config draws ids from. public string AlphabetCharacters => PatternAlphabet.Resolve(Alphabet, CustomAlphabet); + /// The suffixes to try, in order. Empty only for a hand-built config that wants none. + public IReadOnlyList Extensions => + string.IsNullOrEmpty(Extension) + ? [] + : Extension.Split( + ExtensionSeparator, + StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries + ); + /// /// Validates and normalises the inputs, generating an id when one is not supplied. /// @@ -219,7 +233,18 @@ public sealed record PatternSourceConfig( private static Uri EnsureTrailingSlash(Uri url) { - if (url.AbsoluteUri.EndsWith('/')) + var text = url.AbsoluteUri; + + // A pasted address that already ended in a slash can easily pick up a second one, and + // "host//" is not "host/": ids resolve against it as //{id}, which most servers answer with + // a 404 for every single attempt. Collapsed rather than rejected — it is a typo, not a + // different intention. + if (text.EndsWith("//", StringComparison.Ordinal)) + { + return new Uri(text.TrimEnd('/') + "/", UriKind.Absolute); + } + + if (text.EndsWith('/')) { return url; } @@ -229,15 +254,41 @@ public sealed record PatternSourceConfig( return new Uri(url.AbsoluteUri + "/", UriKind.Absolute); } + /// + /// Turns whatever the user typed into a canonical, ordered, duplicate-free suffix list. + /// + /// + /// Separators are generous — comma, semicolon or plain spaces — because the field is a text box + /// and every one of those is a reasonable thing to type. Order is the user's and is preserved: + /// it is the order the fetcher will try, so "jpg png" and "png jpg" are different instructions. + /// private static string NormaliseExtension(string? extension) { - var trimmed = extension?.Trim(); - if (string.IsNullOrEmpty(trimmed)) + if (string.IsNullOrWhiteSpace(extension)) { return DefaultExtension; } - return trimmed.StartsWith('.') ? trimmed : "." + trimmed; + var parts = extension.Split( + [ExtensionSeparator, ';', ' ', ' '], + StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries + ); + + var seen = new HashSet(StringComparer.OrdinalIgnoreCase); + var kept = new List(parts.Length); + + foreach (var part in parts) + { + var suffix = part.StartsWith('.') ? part : "." + part; + + // A repeat would mean fetching the same address twice before giving up on the id. + if (suffix.Length > 1 && seen.Add(suffix)) + { + kept.Add(suffix); + } + } + + return kept.Count == 0 ? DefaultExtension : string.Join(ExtensionSeparator, kept); } private static string NewId(string name) diff --git a/src/AvParser.Core/Proxies/IProxyPool.cs b/src/AvParser.Core/Proxies/IProxyPool.cs index 4d91d6a..1db4783 100644 --- a/src/AvParser.Core/Proxies/IProxyPool.cs +++ b/src/AvParser.Core/Proxies/IProxyPool.cs @@ -33,6 +33,19 @@ public interface IProxyPool /// Report the outcome on the lease, otherwise the pool never learns anything. Task AcquireAsync(CancellationToken cancellationToken = default); + /// + /// Takes a proxy that is not in any of , when one exists. + /// + /// ISO codes already tried; matched case-insensitively. + /// Cancellation. + /// A lease, or when nothing outside those countries is usable. + /// + /// For "this picture is a geo-block, try somewhere else". Entries whose country the feed never + /// stated stay eligible: an unknown country is not a known match, and refusing them would turn a + /// missing field into a smaller pool. + /// + Task AcquireAsync(IReadOnlySet? excludedCountries, CancellationToken cancellationToken = default); + /// Probes every entry in parallel and updates their health. /// How many answered. Task SweepAsync(IProgress? progress = null, CancellationToken cancellationToken = default); diff --git a/src/AvParser.Core/Proxies/ProxyOptions.cs b/src/AvParser.Core/Proxies/ProxyOptions.cs index 915baa7..ad3aed7 100644 --- a/src/AvParser.Core/Proxies/ProxyOptions.cs +++ b/src/AvParser.Core/Proxies/ProxyOptions.cs @@ -74,10 +74,19 @@ public sealed record ProxyOptions /// URL fetched to decide whether a proxy works. /// - /// Defaults to a plain-HTTP 204 endpoint: it is tiny, and requiring TLS would fail every - /// proxy that cannot do CONNECT rather than every proxy that is actually dead. + /// + /// Must be https. A plain-HTTP probe is an absolute-URI GET, which a proxy can serve by + /// forwarding; an https request is a CONNECT tunnel, which many free proxies refuse. This + /// used to default to http on the reasoning that requiring TLS would fail proxies that were not + /// actually dead — but every address this app collects from is https, so a proxy that cannot + /// tunnel is dead for our purposes. Probing http marked those live and then failed every + /// real request with "the proxy tunnel request failed with status code 400". + /// + /// + /// A 204 endpoint keeps the check tiny; what earns the "live" verdict is the tunnel, not the body. + /// /// - public Uri ProbeUrl { get; init; } = new("http://www.gstatic.com/generate_204"); + public Uri ProbeUrl { get; init; } = new("https://www.gstatic.com/generate_204"); /// Per-proxy probe timeout. public TimeSpan ProbeTimeout { get; init; } = TimeSpan.FromSeconds(8); @@ -95,7 +104,7 @@ public sealed record ProxyOptions public TimeSpan MaxQuarantine { get; init; } = TimeSpan.FromMinutes(15); /// - /// Proxies tried per call under + /// Proxies tried per call under /// before giving up. /// public int LazyProbeAttempts { get; init; } = 5; diff --git a/src/AvParser.Core/Proxies/ProxyPool.cs b/src/AvParser.Core/Proxies/ProxyPool.cs index 85d05f8..2b373f3 100644 --- a/src/AvParser.Core/Proxies/ProxyPool.cs +++ b/src/AvParser.Core/Proxies/ProxyPool.cs @@ -167,13 +167,20 @@ public sealed class ProxyPool : IProxyPool } /// - public async Task AcquireAsync(CancellationToken cancellationToken = default) + public Task AcquireAsync(CancellationToken cancellationToken = default) => + AcquireAsync(null, cancellationToken); + + /// + public async Task AcquireAsync( + IReadOnlySet? excludedCountries, + CancellationToken cancellationToken = default + ) { var options = Options; if (options.HealthCheck == ProxyHealthCheck.Pool) { - var entry = SelectAvailable(); + var entry = SelectAvailable(excludedCountries); return entry is null ? null : new ProxyLease(this, entry); } @@ -182,7 +189,7 @@ public sealed class ProxyPool : IProxyPool { cancellationToken.ThrowIfCancellationRequested(); - var entry = SelectAvailable(); + var entry = SelectAvailable(excludedCountries); if (entry is null) { return null; @@ -533,13 +540,15 @@ public sealed class ProxyPool : IProxyPool /// on that fallback is marked live by its own success. /// /// - private ProxyEntry? SelectAvailable() + private ProxyEntry? SelectAvailable(IReadOnlySet? excludedCountries = null) { lock (_gate) { var now = _time.GetUtcNow(); - var available = _entries.Where(entry => entry.IsAvailable(now)).ToArray(); + var available = _entries + .Where(entry => entry.IsAvailable(now) && !IsExcluded(entry, excludedCountries)) + .ToArray(); var live = available.Where(entry => entry.Health == ProxyHealthState.Alive).ToArray(); var candidates = (live.Length > 0 ? live : available) @@ -551,5 +560,15 @@ public sealed class ProxyPool : IProxyPool } } + /// Whether the entry sits in a country the caller has already ruled out. + /// + /// An entry the feed gave no country for is never excluded: "unknown" is not a match, and + /// dropping those would shrink the pool over a missing field rather than over a real answer. + /// + private static bool IsExcluded(ProxyEntry entry, IReadOnlySet? excludedCountries) => + excludedCountries is { Count: > 0 } + && entry.Endpoint.Country is { Length: > 0 } country + && excludedCountries.Contains(country); + private void RaiseChanged() => Changed?.Invoke(this, EventArgs.Empty); } diff --git a/src/AvParser.Core/Settings/AppSettings.cs b/src/AvParser.Core/Settings/AppSettings.cs index 84af4bf..f85b2e1 100644 --- a/src/AvParser.Core/Settings/AppSettings.cs +++ b/src/AvParser.Core/Settings/AppSettings.cs @@ -81,7 +81,7 @@ public sealed record AppSettings( ProxyHealthCheck ProxyHealthCheck = ProxyHealthCheck.Pool, bool ProxyUseFeed = true, ProxyProtocolFilter ProxyProtocols = ProxyProtocolFilter.All, - string ProxyProbeUrl = "http://www.gstatic.com/generate_204", + string ProxyProbeUrl = "https://www.gstatic.com/generate_204", int ProxyProbeTimeoutSeconds = 8, int ProxyProbeConcurrency = 64, int ProxyMinimumLive = 10, @@ -104,6 +104,9 @@ public sealed record AppSettings( string? CollectSourceIds = null ) { + /// The probe endpoint shipped before the check moved to https. + private const string LegacyProbeUrl = "http://www.gstatic.com/generate_204"; + /// Separates ids inside . public const char SourceIdSeparator = ';'; @@ -135,7 +138,7 @@ public sealed record AppSettings( public ProxyOptions ToProxyOptions() { var probeUrl = Uri.TryCreate(ProxyProbeUrl, UriKind.Absolute, out var parsed) - ? parsed + ? Upgraded(parsed) : new ProxyOptions().ProbeUrl; return new ProxyOptions @@ -151,6 +154,17 @@ public sealed record AppSettings( }.Validated(); } + /// Replaces the old plain-HTTP probe endpoint with its https twin. + /// + /// Only the value this app used to ship: a file written before the probe moved to https would + /// otherwise keep marking proxies live that cannot open a tunnel, which is the one thing every + /// collected address needs. Anything the user typed themselves is left exactly as they typed it. + /// + private static Uri Upgraded(Uri probeUrl) => + string.Equals(probeUrl.AbsoluteUri, LegacyProbeUrl, StringComparison.OrdinalIgnoreCase) + ? new ProxyOptions().ProbeUrl + : probeUrl; + /// Projects the collector settings onto . /// /// The one place primitives become policy, and it clamps rather than throws for the same reason diff --git a/src/AvParser.Infrastructure/Collecting/CollectRunner.cs b/src/AvParser.Infrastructure/Collecting/CollectRunner.cs index 13d6f4e..48c7dcf 100644 --- a/src/AvParser.Infrastructure/Collecting/CollectRunner.cs +++ b/src/AvParser.Infrastructure/Collecting/CollectRunner.cs @@ -60,8 +60,8 @@ public sealed class CollectRunner( _store.Configure(options.ShowcaseMode); var runId = await _store.BeginRunAsync(source.Id, cancellationToken).ConfigureAwait(false); - var tombstones = await _store.LoadTombstonesAsync(cancellationToken).ConfigureAwait(false); - var fetchOptions = BuildFetchOptions(options, tombstones); + var rules = await LoadRulesAsync(source.Id, cancellationToken).ConfigureAwait(false); + var fetchOptions = BuildFetchOptions(options, rules); var work = Channel.CreateBounded( new BoundedChannelOptions(workers * 4) { FullMode = BoundedChannelFullMode.Wait } @@ -346,8 +346,42 @@ public sealed class CollectRunner( return ParseOutcome.Success(stored with { Elapsed = result.Elapsed }); } + /// + /// Collects everything the user has already ruled on that applies to this source. + /// + /// + /// Merged into one map so the fetcher does a single lookup per download. Dead-link tombstones + /// predate rules and are global, so they come in as unscoped skips; a rule for the same hash + /// wins, because it is the more recent and more specific statement. + /// + private async Task> LoadRulesAsync( + string sourceId, + CancellationToken cancellationToken + ) + { + var merged = new Dictionary(StringComparer.Ordinal); + + foreach (var hash in await _store.LoadTombstonesAsync(cancellationToken).ConfigureAwait(false)) + { + merged[hash] = MediaRuleAction.Skip; + } + + foreach (var rule in await _store.LoadRulesAsync(cancellationToken).ConfigureAwait(false)) + { + if (rule.AppliesTo(sourceId)) + { + merged[rule.Sha256] = rule.Action; + } + } + + return merged; + } + /// Projects the run's policy onto the HTTP layer's parameters. - private static FetchOptions BuildFetchOptions(CollectOptions options, IReadOnlySet tombstones) => + private static FetchOptions BuildFetchOptions( + CollectOptions options, + IReadOnlyDictionary rules + ) => new() { MaxItemBytes = options.MaxItemBytes, @@ -358,7 +392,7 @@ public sealed class CollectRunner( RequireProxy = options.RequireProxy, UserAgent = options.UserAgent, AllowedKinds = MediaKindFilters.ToSet(options.AllowedKinds), - Tombstones = tombstones, + Rules = rules, }; /// Stands in for content on a skipped item, which by definition has none. diff --git a/src/AvParser.Infrastructure/Collecting/FetchOptions.cs b/src/AvParser.Infrastructure/Collecting/FetchOptions.cs index 8d97b16..6ad8f8a 100644 --- a/src/AvParser.Infrastructure/Collecting/FetchOptions.cs +++ b/src/AvParser.Infrastructure/Collecting/FetchOptions.cs @@ -44,8 +44,20 @@ public sealed record FetchOptions MediaKind.WebM, }; - /// Hashes already known to be dead-link placeholders. - public IReadOnlySet Tombstones { get; init; } = new HashSet(StringComparer.Ordinal); + /// + /// What to do about pictures the user has already ruled on, by content hash. + /// + /// + /// Filtered to the running source before it gets here, so the fetcher never has to know which + /// source it is working for. Old dead-link tombstones arrive in the same map as + /// : one lookup on the hot path rather than two. + /// + public IReadOnlyDictionary Rules { get; init; } = + new Dictionary(StringComparer.Ordinal); + + /// Countries the pool must not hand out for this attempt. + /// Set only by the retry that a rule triggers. + public IReadOnlySet? ExcludedCountries { get; init; } } /// What one download attempt produced. @@ -74,6 +86,9 @@ public sealed record FetchResult(SeenOutcome Outcome, Uri FinalUrl) /// Proxy the content came through, or null when direct. public string? ProxyKey { get; init; } + /// Country of that proxy, when the feed stated one. Drives the "try elsewhere" retry. + public string? ProxyCountry { get; init; } + /// Localisation code for the failure, matching a Parse.Error.{Code} key. public string? ErrorCode { get; init; } diff --git a/src/AvParser.Infrastructure/Collecting/MediaFetcher.cs b/src/AvParser.Infrastructure/Collecting/MediaFetcher.cs index 2556cf9..5fee0a6 100644 --- a/src/AvParser.Infrastructure/Collecting/MediaFetcher.cs +++ b/src/AvParser.Infrastructure/Collecting/MediaFetcher.cs @@ -51,6 +51,18 @@ public sealed class MediaFetcher( /// Big enough that a 32 MB file is a few hundred reads, small enough to pool cheaply. private const int BufferSize = 64 * 1024; + /// Internal marker for "the user says this picture means: come back from elsewhere". + /// Never reaches the journal: the retry loop turns it into a real outcome either way. + private const string RetryElsewhereCode = "RetryElsewhere"; + + /// How many countries to try before accepting that the picture is not available here. + /// + /// Small on purpose: each attempt is a full download of a banner, and a service that blocks one + /// country usually blocks a region. Three says "this is not a fluke" without turning one id into + /// a tour of the pool. + /// + private const int MaxCountryAttempts = 3; + private readonly IProxiedHttpClientFactory _clients = clients ?? throw new ArgumentNullException(nameof(clients)); private readonly BlobStore _blobs = blobs ?? throw new ArgumentNullException(nameof(blobs)); private readonly HostThrottle _throttle = throttle ?? throw new ArgumentNullException(nameof(throttle)); @@ -73,7 +85,7 @@ public sealed class MediaFetcher( try { - var result = await FetchCoreAsync(candidate, options, stopwatch, budget.Token, cancellationToken) + var result = await FetchAddressesAsync(candidate, options, stopwatch, budget.Token, cancellationToken) .ConfigureAwait(false); // Cancelling a socket read surfaces as an IOException as often as an @@ -99,6 +111,131 @@ public sealed class MediaFetcher( } } + /// + /// Tries the candidate's addresses in order and stops at the first one that is actually there. + /// + /// + /// + /// This is "several extensions per id": /{id}.jpg, then /{id}.png, until something + /// answers or nothing does. Only a verdict of "not here" moves on — a 404, a placeholder under + /// the size floor, a page that is not media. A timeout or a proxy failure means we do not know + /// what is at that address, and burning the remaining suffixes on a broken connection would + /// report "nothing here" about an id nobody actually asked about. + /// + /// + /// The miss that gets returned is the last one, so the journal records the id as settled by the + /// suffix that was tried last rather than by an early guess. + /// + /// + private async Task FetchAddressesAsync( + MediaCandidate candidate, + FetchOptions options, + Stopwatch stopwatch, + CancellationToken token, + CancellationToken userToken + ) + { + FetchResult? lastMiss = null; + + foreach (var address in candidate.Addresses) + { + var attempt = address == candidate.Url ? candidate : candidate with { Url = address, Alternatives = [] }; + var result = await FetchWithRulesAsync(attempt, options, stopwatch, token, userToken) + .ConfigureAwait(false); + + if (!IsMissing(result.Outcome)) + { + return result; + } + + lastMiss = result; + } + + return lastMiss!; + } + + /// + /// Fetches one address, and when the picture that comes back is one the user has ruled a + /// geo-block, fetches it again from another country. + /// + /// + /// + /// The retry forces a proxy even when the source is allowed to run direct: the whole point is to + /// arrive from somewhere else, and falling back to the user's own address would be the one place + /// we already know shows the banner. + /// + /// + /// Giving up stores nothing and leaves the address retryable — a banner is not the picture, + /// and recording it as settled would burn the id for good on the strength of today's pool. A + /// proxy whose country the feed never stated cannot be excluded from, so that ends the retry too: + /// "somewhere else" is not a thing we can ask for. + /// + /// + private async Task FetchWithRulesAsync( + MediaCandidate candidate, + FetchOptions options, + Stopwatch stopwatch, + CancellationToken token, + CancellationToken userToken + ) + { + var tried = new HashSet(StringComparer.OrdinalIgnoreCase); + var attemptOptions = options; + + for (var attempt = 0; ; attempt++) + { + FetchResult result; + + try + { + result = await FetchCoreAsync(candidate, attemptOptions, stopwatch, token, userToken) + .ConfigureAwait(false); + } + catch (ProxyUnavailableException) when (attempt > 0) + { + // Nothing left outside the countries already tried. + return GeoBlocked(candidate.Url, stopwatch, tried); + } + + if (result.ErrorCode != RetryElsewhereCode) + { + return result; + } + + if (result.ProxyCountry is { Length: > 0 } country) + { + tried.Add(country); + } + + if (tried.Count == 0 || attempt + 1 >= MaxCountryAttempts) + { + return GeoBlocked(candidate.Url, stopwatch, tried); + } + + _logger.LogDebug( + "{Url} matched a try-elsewhere rule; retrying outside {Countries}", + candidate.Url, + string.Join(", ", tried) + ); + + attemptOptions = options with { ExcludedCountries = tried, RequireProxy = true }; + } + } + + /// The verdict when the picture is a known geo-block and there is nowhere else to be. + private static FetchResult GeoBlocked(Uri url, Stopwatch stopwatch, HashSet tried) => + Failure( + url, + SeenOutcome.Failed, + "GeoBlocked", + tried.Count == 0 ? null : string.Join(", ", tried), + stopwatch + ); + + /// Whether the address simply held nothing, which is the only reason to try the next one. + private static bool IsMissing(SeenOutcome outcome) => + outcome is SeenOutcome.Gone or SeenOutcome.NotMedia or SeenOutcome.TooSmall or SeenOutcome.Placeholder; + private async Task FetchCoreAsync( MediaCandidate candidate, FetchOptions options, @@ -108,7 +245,7 @@ public sealed class MediaFetcher( ) { using var leased = await _clients - .LeaseAsync(options.Timeouts, options.RequireProxy, token) + .LeaseAsync(options.Timeouts, options.RequireProxy, options.ExcludedCountries, token) .ConfigureAwait(false); HttpStatusCode? status = null; @@ -457,13 +594,16 @@ public sealed class MediaFetcher( ); } - if (options.Tombstones.Contains(hash!)) + if (options.Rules.TryGetValue(hash!, out var rule)) { - return ( - Failure(url, SeenOutcome.Placeholder, "Placeholder", null, stopwatch, leased.ProxyKey, status), - true, - null - ); + // The bytes are the only thing that identifies these: a service answers a missing id + // and a geo-block with an ordinary 200 and a perfectly valid picture. + var outcome = + rule == MediaRuleAction.Skip + ? Failure(url, SeenOutcome.Placeholder, "Placeholder", null, stopwatch, leased.ProxyKey, status) + : Failure(url, SeenOutcome.Failed, RetryElsewhereCode, null, stopwatch, leased.ProxyKey, status); + + return (outcome with { ProxyCountry = leased.ProxyCountry }, true, null); } var blob = await DescribeAsync(temp, hash!, kind, total, prefix, token).ConfigureAwait(false); diff --git a/src/AvParser.Infrastructure/Media/MediaStore.cs b/src/AvParser.Infrastructure/Media/MediaStore.cs index 46f9f36..665f7ac 100644 --- a/src/AvParser.Infrastructure/Media/MediaStore.cs +++ b/src/AvParser.Infrastructure/Media/MediaStore.cs @@ -83,6 +83,33 @@ public sealed class MediaStore( return plan.ItemsRemoved; } + /// + public Task> LoadRulesAsync(CancellationToken cancellationToken = default) => + _index.LoadRulesAsync(cancellationToken); + + /// + public async Task SaveRuleAsync(MediaRule rule, CancellationToken cancellationToken = default) + { + ArgumentNullException.ThrowIfNull(rule); + + await _index.SaveRuleAsync(rule, cancellationToken).ConfigureAwait(false); + + if (rule.Action != MediaRuleAction.Skip) + { + _logger.LogInformation("Rule for {Hash}: {Action}", rule.Sha256, rule.Action); + return 0; + } + + // A skip rule is made while looking at the thing in the gallery: leaving that copy on disk + // would answer half of "never again". The tombstone path already removes every copy and the + // showcase links pointing at it, so it stays the one place that knows how. + return await TombstoneAsync(rule.Sha256, rule.Reason, cancellationToken).ConfigureAwait(false); + } + + /// + public Task RemoveRuleAsync(string sha256, CancellationToken cancellationToken = default) => + _index.RemoveRuleAsync(sha256, cancellationToken); + /// public async Task StoreAsync( MediaStoreRequest request, diff --git a/src/AvParser.Infrastructure/Media/SqliteMediaIndex.cs b/src/AvParser.Infrastructure/Media/SqliteMediaIndex.cs index 8d53d04..e453a57 100644 --- a/src/AvParser.Infrastructure/Media/SqliteMediaIndex.cs +++ b/src/AvParser.Infrastructure/Media/SqliteMediaIndex.cs @@ -265,6 +265,87 @@ public sealed class SqliteMediaIndex : IDisposable return hashes; } + /// Reads every standing rule the user has set on a picture. + public async Task> LoadRulesAsync(CancellationToken cancellationToken = default) + { + var rules = new List(); + + await using var connection = await OpenAsync(cancellationToken).ConfigureAwait(false); + await using var command = connection.CreateCommand(); + command.CommandText = "SELECT sha256, action, source_ids, reason, created_utc FROM media_rule;"; + + await using var reader = await command.ExecuteReaderAsync(cancellationToken).ConfigureAwait(false); + while (await reader.ReadAsync(cancellationToken).ConfigureAwait(false)) + { + rules.Add( + new MediaRule(reader.GetString(0), (MediaRuleAction)reader.GetInt32(1)) + { + SourceIds = reader.IsDBNull(2) ? null : reader.GetString(2), + Reason = reader.IsDBNull(3) ? null : reader.GetString(3), + CreatedUtc = DateTimeOffset.Parse( + reader.GetString(4), + CultureInfo.InvariantCulture, + DateTimeStyles.RoundtripKind + ), + } + ); + } + + return rules; + } + + /// Writes a rule, replacing any earlier one for the same hash. + public Task SaveRuleAsync(MediaRule rule, CancellationToken cancellationToken = default) + { + ArgumentNullException.ThrowIfNull(rule); + + return WriteAsync( + async connection => + { + await using var command = connection.CreateCommand(); + command.CommandText = """ + INSERT INTO media_rule(sha256, action, source_ids, reason, created_utc) + VALUES($sha, $action, $sources, $reason, $created) + ON CONFLICT(sha256) DO UPDATE SET + action = excluded.action, + source_ids = excluded.source_ids, + reason = excluded.reason; + """; + command.Parameters.AddWithValue("$sha", rule.Sha256); + command.Parameters.AddWithValue("$action", (int)rule.Action); + command.Parameters.AddWithValue("$sources", (object?)rule.SourceIds ?? DBNull.Value); + command.Parameters.AddWithValue("$reason", (object?)rule.Reason ?? DBNull.Value); + command.Parameters.AddWithValue("$created", Now()); + + await command.ExecuteNonQueryAsync(cancellationToken).ConfigureAwait(false); + }, + cancellationToken + ); + } + + /// Drops the rule for a hash. Returns whether one was there. + public async Task RemoveRuleAsync(string sha256, CancellationToken cancellationToken = default) + { + ArgumentException.ThrowIfNullOrWhiteSpace(sha256); + + var removed = 0; + + await WriteAsync( + async connection => + { + await using var command = connection.CreateCommand(); + command.CommandText = "DELETE FROM media_rule WHERE sha256 = $sha;"; + command.Parameters.AddWithValue("$sha", sha256); + + removed = await command.ExecuteNonQueryAsync(cancellationToken).ConfigureAwait(false); + }, + cancellationToken + ) + .ConfigureAwait(false); + + return removed > 0; + } + /// Marks a hash as a placeholder and plans the removal of every copy held. public async Task TombstoneAsync( string sha256, @@ -974,6 +1055,14 @@ public sealed class SqliteMediaIndex : IDisposable CREATE INDEX IF NOT EXISTS ix_seen_outcome ON seen_url(source_id, outcome); + CREATE TABLE IF NOT EXISTS media_rule ( + sha256 TEXT PRIMARY KEY NOT NULL, + action INTEGER NOT NULL, + source_ids TEXT NULL, + reason TEXT NULL, + created_utc TEXT NOT NULL + ) STRICT; + CREATE TABLE IF NOT EXISTS tombstone ( sha256 TEXT PRIMARY KEY NOT NULL, reason TEXT NULL, diff --git a/src/AvParser.Infrastructure/Proxies/ProxiedHttpClientFactory.cs b/src/AvParser.Infrastructure/Proxies/ProxiedHttpClientFactory.cs index ea56ed3..6a20065 100644 --- a/src/AvParser.Infrastructure/Proxies/ProxiedHttpClientFactory.cs +++ b/src/AvParser.Infrastructure/Proxies/ProxiedHttpClientFactory.cs @@ -58,6 +58,9 @@ public sealed class LeasedHttpClient(HttpClient client, ProxyLease? lease) : IDi /// Address of the proxy in use, for recording provenance. public string? ProxyKey => Lease?.Endpoint.Key; + /// Country of the proxy in use, when the feed stated one. + public string? ProxyCountry => Lease?.Endpoint.Country; + /// public void Dispose() { @@ -106,6 +109,9 @@ public interface IProxiedHttpClientFactory /// /// When set, having no live proxy throws instead of quietly connecting directly. /// + /// + /// Countries the pool must not hand out, for a "come back from somewhere else" retry. + /// /// Cancels acquisition. /// /// was set and the pool had nothing live. @@ -113,6 +119,7 @@ public interface IProxiedHttpClientFactory Task LeaseAsync( HttpClientTimeouts timeouts, bool requireProxy, + IReadOnlySet? excludedCountries = null, CancellationToken cancellationToken = default ); } @@ -158,10 +165,11 @@ public sealed class ProxiedHttpClientFactory(IProxyPool pool) : IProxiedHttpClie public async Task LeaseAsync( HttpClientTimeouts timeouts, bool requireProxy, + IReadOnlySet? excludedCountries = null, CancellationToken cancellationToken = default ) { - var lease = await _pool.AcquireAsync(cancellationToken).ConfigureAwait(false); + var lease = await _pool.AcquireAsync(excludedCountries, cancellationToken).ConfigureAwait(false); if (lease is null && requireProxy) { diff --git a/src/AvParser.UI/Localization/Strings.resx b/src/AvParser.UI/Localization/Strings.resx index eaf0ea8..ba9b367 100644 --- a/src/AvParser.UI/Localization/Strings.resx +++ b/src/AvParser.UI/Localization/Strings.resx @@ -269,7 +269,7 @@ PROBE URL - Plain HTTP by default: requiring TLS would fail every proxy that cannot do CONNECT, not just the dead ones. + Keep it https: collected addresses are https, so the check has to open a tunnel the way the collector does. A plain-HTTP probe passes proxies that then refuse every real request. PROBE TIMEOUT (SEC) @@ -491,7 +491,10 @@ Off by default: with no live proxy this source refuses to start rather than fetching from your own address. Turn it on for a host you own, or one you do not mind being seen from here. - EXTENSION + EXTENSIONS + + + Several, comma-separated, are tried in order until one answers: .jpg,.png,.gif. Empty means .jpg. ID CHARACTERS diff --git a/src/AvParser.UI/Localization/Strings.ru.resx b/src/AvParser.UI/Localization/Strings.ru.resx index 59739c2..1b34c49 100644 --- a/src/AvParser.UI/Localization/Strings.ru.resx +++ b/src/AvParser.UI/Localization/Strings.ru.resx @@ -269,7 +269,7 @@ URL ДЛЯ ПРОВЕРКИ - По умолчанию обычный HTTP: требование TLS отбраковало бы все прокси без CONNECT, а не только нерабочие. + Оставьте https: собираемые адреса идут по https, значит и проверка должна открывать туннель так же, как сборщик. Проба по обычному HTTP пропускает прокси, которые потом отказывают на каждом реальном запросе. ТАЙМАУТ ПРОВЕРКИ (СЕК) @@ -491,7 +491,10 @@ По умолчанию выключено: без живой прокси источник не запустится, а не пойдёт с вашего адреса. Включайте для своего хоста или там, где вас не смущает быть видимым отсюда. - РАСШИРЕНИЕ + РАСШИРЕНИЯ + + + Несколько через запятую проверяются по порядку, пока одно не ответит: .jpg,.png,.gif. Пусто — значит .jpg. СИМВОЛЫ ID diff --git a/src/AvParser.UI/ViewModels/GalleryViewModel.cs b/src/AvParser.UI/ViewModels/GalleryViewModel.cs index 4716965..70f4141 100644 --- a/src/AvParser.UI/ViewModels/GalleryViewModel.cs +++ b/src/AvParser.UI/ViewModels/GalleryViewModel.cs @@ -26,6 +26,7 @@ public partial class GalleryViewModel : PageViewModel, IDisposable private const int PreviewWidth = 1400; private readonly IMediaStore _store; + private readonly IMediaSourceCatalog _catalog; private readonly IThumbnailCache _thumbnails; private readonly MediaStore? _paths; private readonly ILogger _logger; @@ -77,6 +78,20 @@ public partial class GalleryViewModel : PageViewModel, IDisposable [Reactive] public partial string? StatusMessage { get; set; } + // ----- Rule editor ----- + + /// Whether the "what to do when this turns up again" panel is showing. + [Reactive] + public partial bool IsRuleEditorOpen { get; set; } + + /// Action chosen in the editor. + [Reactive] + public partial LocalizedOption RuleAction { get; set; } + + /// Whether the open picture already has a rule, which enables removing it. + [Reactive] + public partial bool HasRule { get; private set; } + /// Creates the page. /// Where the content is. /// Decodes and caches tile bitmaps. @@ -84,12 +99,14 @@ public partial class GalleryViewModel : PageViewModel, IDisposable /// Scheduler for UI-affine updates; tests pass an immediate one. public GalleryViewModel( IMediaStore store, + IMediaSourceCatalog catalog, IThumbnailCache thumbnails, ILogger logger, ISequencer? mainThread = null ) { _store = store ?? throw new ArgumentNullException(nameof(store)); + _catalog = catalog ?? throw new ArgumentNullException(nameof(catalog)); _thumbnails = thumbnails ?? throw new ArgumentNullException(nameof(thumbnails)); _logger = logger ?? throw new ArgumentNullException(nameof(logger)); _mainThread = mainThread ?? RxSchedulers.MainThreadScheduler; @@ -99,6 +116,9 @@ public partial class GalleryViewModel : PageViewModel, IDisposable SearchText = string.Empty; SelectedKinds = KindFilters[0]; + RuleActions = LocalizedOption.ForAll(); + RuleAction = RuleActions[0]; + RuleSources = []; var idle = this.WhenAnyValue(x => x.IsLoading).Select(static loading => !loading); @@ -118,6 +138,22 @@ public partial class GalleryViewModel : PageViewModel, IDisposable CloseViewerCommand = ReactiveCommand.Create(() => Selected = null, outputScheduler: _mainThread); + var hasSelection = this.WhenAnyValue(x => x.Selected).Select(static item => item is not null); + var editing = this.WhenAnyValue(x => x.IsRuleEditorOpen); + + EditRuleCommand = ReactiveCommand.CreateFromTask(OpenRuleEditorAsync, hasSelection, _mainThread); + SaveRuleCommand = ReactiveCommand.CreateFromTask(SaveRuleAsync, editing, _mainThread); + RemoveRuleCommand = ReactiveCommand.CreateFromTask( + RemoveRuleAsync, + this.WhenAnyValue(x => x.HasRule), + _mainThread + ); + CancelRuleCommand = ReactiveCommand.Create(() => IsRuleEditorOpen = false, editing, _mainThread); + + EditRuleCommand.ThrownExceptions.Subscribe(OnRuleFailed); + SaveRuleCommand.ThrownExceptions.Subscribe(OnRuleFailed); + RemoveRuleCommand.ThrownExceptions.Subscribe(OnRuleFailed); + // Changing a filter starts again from the first page: staying on page seven of a different // result set shows nothing and looks broken. this.WhenAnyValue( diff --git a/src/AvParser.UI/Views/CollectView.axaml b/src/AvParser.UI/Views/CollectView.axaml index 8115b01..12ec438 100644 --- a/src/AvParser.UI/Views/CollectView.axaml +++ b/src/AvParser.UI/Views/CollectView.axaml @@ -173,10 +173,12 @@ - + + + url.AbsoluteUri) + .ShouldBe([$"https://imgtest.example/test1/{id}.png", $"https://imgtest.example/test1/{id}.gif"]); + + candidate.Addresses.First().ShouldBe(candidate.Url); + candidate.Addresses.Count().ShouldBe(3); + } + + [Fact] + public async Task A_single_extension_leaves_the_candidate_without_fallbacks() + { + var source = new PatternMediaSource(Config(min: 6, max: 6, extension: ".png")); + + var candidate = (await Collect(source, 1)).ShouldHaveSingleItem(); + + candidate.Alternatives.ShouldBeEmpty(); + candidate.Addresses.ShouldHaveSingleItem().ShouldBe(candidate.Url); + } + [Fact] public async Task The_attempt_budget_bounds_how_many_are_generated() { @@ -208,6 +240,76 @@ public class PatternSourceConfigTests config.Id.ShouldNotBeNullOrWhiteSpace(); } + [Theory] + [InlineData(".jpg,.png", ".jpg,.png")] + [InlineData("jpg png gif", ".jpg,.png,.gif")] + [InlineData(".jpg; .PNG ;jpg", ".jpg,.PNG")] + [InlineData(" ", ".jpg")] + public void Extensions_are_normalised_into_an_ordered_list(string typed, string expected) + { + // Order is an instruction, not a detail: it is the order the fetcher will try. Repeats are + // dropped because a repeat means fetching the same address twice before giving up on the id. + PatternSourceConfig.TryCreate( + "N", + "https://h/x/", + 6, + 8, + IdAlphabet.Digits, + null, + typed, + allowDirectConnection: false, + out var config + ); + + config.ShouldNotBeNull().Extension.ShouldBe(expected); + config.Extensions.ShouldBe(expected.Split(',')); + } + + [Fact] + public void A_doubled_trailing_slash_is_collapsed() + { + // "host//" is not "host/": ids resolve against it as //{id}, which most servers answer with + // a 404 for every attempt in the run. Easy to paste, impossible to spot in the field. + var error = PatternSourceConfig.TryCreate( + "N", + "https://imgtest.example/test1//", + 6, + 8, + IdAlphabet.Digits, + null, + ".jpg", + allowDirectConnection: false, + out var config + ); + + error.ShouldBe(PatternConfigError.None); + config.ShouldNotBeNull().BaseUrl.AbsoluteUri.ShouldBe("https://imgtest.example/test1/"); + } + + [Fact] + public async Task Ids_hang_off_a_collapsed_base_url_without_a_doubled_slash() + { + PatternSourceConfig.TryCreate( + "N", + "https://imgtest.example/test1//", + 6, + 6, + IdAlphabet.Digits, + null, + ".jpg", + allowDirectConnection: false, + out var config + ); + + var source = new PatternMediaSource(config!); + + await foreach (var outcome in source.ParseAsync(new MediaQuery(Limit: 1), null, CancellationToken.None)) + { + var candidate = outcome.Value.ShouldNotBeNull(); + candidate.Url.AbsoluteUri.ShouldBe($"https://imgtest.example/test1/{candidate.ExternalId}.jpg"); + } + } + [Fact] public void A_blank_extension_becomes_the_default_one() { diff --git a/tests/AvParser.Infrastructure.Tests/Collecting/MediaFetcherTests.cs b/tests/AvParser.Infrastructure.Tests/Collecting/MediaFetcherTests.cs index 94da8e4..4782ebf 100644 --- a/tests/AvParser.Infrastructure.Tests/Collecting/MediaFetcherTests.cs +++ b/tests/AvParser.Infrastructure.Tests/Collecting/MediaFetcherTests.cs @@ -44,6 +44,7 @@ internal sealed class DirectHttpClientFactory : IProxiedHttpClientFactory public Task LeaseAsync( HttpClientTimeouts timeouts, bool requireProxy, + IReadOnlySet? excludedCountries = null, CancellationToken cancellationToken = default ) => requireProxy @@ -230,6 +231,88 @@ public sealed class MediaFetcherTests : IAsyncLifetime _throttle.CooldownRemaining(url).ShouldBeGreaterThan(TimeSpan.Zero); } + [Fact] + public async Task Several_extensions_are_tried_in_order_until_one_answers() + { + var jpg = _server.Map("/id.jpg", new Reply { Status = 404 }); + var png = _server.MapBody("/id.png", Samples.Png(), "image/png"); + var gif = _server.MapBody("/id.gif", Samples.Gif(), "image/gif"); + + var result = await _fetcher.FetchAsync( + new MediaCandidate(jpg) { SourceId = "test", Alternatives = [png, gif] }, + Options(), + TestContext.Current.CancellationToken + ); + + result.Outcome.ShouldBe(SeenOutcome.Stored); + result.FinalUrl.ShouldBe(png); + result.Blob!.Kind.ShouldBe(MediaKind.Png); + + // The gif exists too, and must not have been fetched: an id holds one picture, and going on + // after a hit would download its imaginary twin and count the id twice. + _server.Requests.ShouldNotContain(request => request.Path == "/id.gif"); + } + + [Fact] + public async Task An_id_that_is_at_none_of_the_extensions_reports_the_last_miss() + { + var jpg = _server.Map("/none.jpg", new Reply { Status = 404 }); + var png = _server.Map("/none.png", new Reply { Status = 404 }); + + var result = await _fetcher.FetchAsync( + new MediaCandidate(jpg) { SourceId = "test", Alternatives = [png] }, + Options(), + TestContext.Current.CancellationToken + ); + + result.Outcome.ShouldBe(SeenOutcome.Gone); + result.FinalUrl.ShouldBe(png); + } + + [Fact] + public async Task A_placeholder_under_the_size_floor_counts_as_not_there_and_moves_on() + { + // What a service actually does with a missing id: 200 with a tiny "removed" image. That is + // "not there" wearing a success, so the next extension still has to be tried. + var jpg = _server.MapBody("/tiny.jpg", Samples.Png(), "image/png"); + var png = _server.Map("/tiny.png", new Reply { Status = 404 }); + + var result = await _fetcher.FetchAsync( + new MediaCandidate(jpg) { SourceId = "test", Alternatives = [png] }, + Options() with + { + MinItemBytes = Samples.Png().Length + 1, + }, + TestContext.Current.CancellationToken + ); + + result.FinalUrl.ShouldBe(png); + result.Outcome.ShouldBe(SeenOutcome.Gone); + } + + [Fact] + public async Task A_timeout_does_not_burn_the_remaining_extensions() + { + // "Not there" is a verdict about the address; a timeout is the absence of one. Moving on + // would report an id as missing everywhere on the strength of one broken connection. + var jpg = _server.Map("/slow.jpg", new Reply { Body = Samples.Png(), HeaderDelay = TimeSpan.FromSeconds(3) }); + var png = _server.MapBody("/slow.png", Samples.Png(), "image/png"); + + var options = Options() with + { + Timeouts = HttpClientTimeouts.Default with { Headers = TimeSpan.FromMilliseconds(300) }, + }; + + var result = await _fetcher.FetchAsync( + new MediaCandidate(jpg) { SourceId = "test", Alternatives = [png] }, + options, + TestContext.Current.CancellationToken + ); + + result.Outcome.ShouldBe(SeenOutcome.Timeout); + _server.Requests.ShouldNotContain(request => request.Path == "/slow.png"); + } + [Fact] public async Task Headers_that_never_arrive_time_out() { @@ -434,7 +517,10 @@ public sealed class MediaFetcherTests : IAsyncLifetime var options = Options() with { - Tombstones = new HashSet(StringComparer.Ordinal) { probe.Blob!.Sha256 }, + Rules = new Dictionary(StringComparer.Ordinal) + { + [probe.Blob!.Sha256] = MediaRuleAction.Skip, + }, }; var result = await FetchAsync(url, options); @@ -443,6 +529,31 @@ public sealed class MediaFetcherTests : IAsyncLifetime StagedFileCount().ShouldBe(0); } + [Fact] + public async Task A_geo_block_rule_with_nowhere_else_to_go_stores_nothing_and_stays_retryable() + { + // Direct connections here, so there is no country to move away from. The picture must not be + // stored — it is a banner, not the thing — and the address must stay worth trying again with + // a different pool tomorrow. + var url = _server.MapBody("/blocked.png", Samples.Png()); + var probe = await FetchAsync(url); + File.Delete(probe.TempPath!); + + var options = Options() with + { + Rules = new Dictionary(StringComparer.Ordinal) + { + [probe.Blob!.Sha256] = MediaRuleAction.RetryElsewhere, + }, + }; + + var result = await FetchAsync(url, options); + + result.ErrorCode.ShouldBe("GeoBlocked"); + SeenOutcomes.IsTerminal(result.Outcome).ShouldBeFalse(); + StagedFileCount().ShouldBe(0); + } + [Fact] public async Task Animation_is_detected_end_to_end() { diff --git a/tests/AvParser.Infrastructure.Tests/JsonSettingsServiceTests.cs b/tests/AvParser.Infrastructure.Tests/JsonSettingsServiceTests.cs index 683fbbf..220f4f3 100644 --- a/tests/AvParser.Infrastructure.Tests/JsonSettingsServiceTests.cs +++ b/tests/AvParser.Infrastructure.Tests/JsonSettingsServiceTests.cs @@ -201,6 +201,34 @@ public sealed class JsonSettingsServiceTests : IDisposable options.ProbeUrl.ShouldBe(new ProxyOptions().ProbeUrl); } + [Fact] + public void The_probe_opens_a_tunnel_the_way_the_collector_does() + { + // A plain-HTTP probe is a forwarded GET; an https one is a CONNECT. Free proxies routinely + // do the first and refuse the second, and every address this app collects is https — so an + // http probe marked them live and they then failed every real request. + new ProxyOptions().ProbeUrl.Scheme.ShouldBe("https"); + new AppSettings().ToProxyOptions().ProbeUrl.Scheme.ShouldBe("https"); + } + + [Fact] + public void A_settings_file_still_holding_the_old_http_probe_is_upgraded() + { + // Nobody would think to edit this by hand, and leaving it would keep confirming proxies that + // cannot do the one thing the collector needs. + var options = new AppSettings(ProxyProbeUrl: "http://www.gstatic.com/generate_204").ToProxyOptions(); + + options.ProbeUrl.ShouldBe(new ProxyOptions().ProbeUrl); + } + + [Fact] + public void A_probe_url_the_user_chose_is_left_alone() + { + var options = new AppSettings(ProxyProbeUrl: "http://example.test/ping").ToProxyOptions(); + + options.ProbeUrl.AbsoluteUri.ShouldBe("http://example.test/ping"); + } + [Fact] public void A_corrupt_file_falls_back_to_the_defaults_instead_of_failing_to_start() { diff --git a/tests/AvParser.UI.HeadlessTests/FakeMediaStore.cs b/tests/AvParser.UI.HeadlessTests/FakeMediaStore.cs index d5d2508..b6c6c7b 100644 --- a/tests/AvParser.UI.HeadlessTests/FakeMediaStore.cs +++ b/tests/AvParser.UI.HeadlessTests/FakeMediaStore.cs @@ -57,6 +57,23 @@ internal sealed class FakeMediaStore : IMediaStore public Task TombstoneAsync(string sha256, string? reason, CancellationToken cancellationToken = default) => Task.FromResult(0); + /// Rules the page under test has saved. + public List Rules { get; } = []; + + public Task> LoadRulesAsync(CancellationToken cancellationToken = default) => + Task.FromResult>([.. Rules]); + + public Task SaveRuleAsync(MediaRule rule, CancellationToken cancellationToken = default) + { + Rules.RemoveAll(existing => existing.Sha256 == rule.Sha256); + Rules.Add(rule); + + return Task.FromResult(rule.Action == MediaRuleAction.Skip ? 1 : 0); + } + + public Task RemoveRuleAsync(string sha256, CancellationToken cancellationToken = default) => + Task.FromResult(Rules.RemoveAll(existing => existing.Sha256 == sha256) > 0); + public Task StoreAsync(MediaStoreRequest request, CancellationToken cancellationToken = default) => Task.FromResult(new CollectedItem(request.Candidate, request.Blob, CollectStatus.Stored)); diff --git a/tests/AvParser.UI.Tests/Fakes/FakeMediaStore.cs b/tests/AvParser.UI.Tests/Fakes/FakeMediaStore.cs index 5ead222..10915ce 100644 --- a/tests/AvParser.UI.Tests/Fakes/FakeMediaStore.cs +++ b/tests/AvParser.UI.Tests/Fakes/FakeMediaStore.cs @@ -57,6 +57,23 @@ internal sealed class FakeMediaStore : IMediaStore public Task TombstoneAsync(string sha256, string? reason, CancellationToken cancellationToken = default) => Task.FromResult(0); + /// Rules the page under test has saved. + public List Rules { get; } = []; + + public Task> LoadRulesAsync(CancellationToken cancellationToken = default) => + Task.FromResult>([.. Rules]); + + public Task SaveRuleAsync(MediaRule rule, CancellationToken cancellationToken = default) + { + Rules.RemoveAll(existing => existing.Sha256 == rule.Sha256); + Rules.Add(rule); + + return Task.FromResult(rule.Action == MediaRuleAction.Skip ? 1 : 0); + } + + public Task RemoveRuleAsync(string sha256, CancellationToken cancellationToken = default) => + Task.FromResult(Rules.RemoveAll(existing => existing.Sha256 == sha256) > 0); + public Task StoreAsync(MediaStoreRequest request, CancellationToken cancellationToken = default) => Task.FromResult(new CollectedItem(request.Candidate, request.Blob, CollectStatus.Stored));