Implement media rule management and enhance proxy handling
- Added methods to `IMediaStore` for loading, saving, and removing media rules, allowing users to manage rules for media items effectively. - Updated `MediaFetcher` to utilize the new rule management system, integrating rule checks into the fetching process to handle geo-blocks and previously ruled items. - Enhanced `ProxyPool` to support exclusion of proxies from specific countries during acquisition, improving the handling of geo-blocked content. - Adjusted `FetchOptions` to include rules instead of tombstones, streamlining the decision-making process during media fetching. - Updated UI components to support rule editing, providing users with a more interactive experience when managing media rules. These changes improve the overall media collection process by allowing users to define rules for handling media items and enhancing the proxy management system for better content accessibility.
This commit is contained in:
@@ -42,6 +42,13 @@
|
||||
|
||||
## Источники
|
||||
|
||||
- **Несколько расширений — это один кандидат с запасными адресами**, а не несколько кандидатов. У id
|
||||
одна картинка: три кандидата на три суффикса скачали бы попадание и потом пошли искать его
|
||||
несуществующих близнецов, а счётчики посчитали бы одну находку за три. Порядок — инструкция:
|
||||
`MediaFetcher.FetchAddressesAsync` идёт по `MediaCandidate.Addresses` до первого ответа.
|
||||
- **Дальше по списку двигает только «здесь ничего нет»**: `Gone`, `NotMedia`, `TooSmall`,
|
||||
`Placeholder`. Тайм-аут или отказ прокси — это отсутствие вердикта, а не вердикт: сжечь на нём
|
||||
остальные суффиксы значит объявить id отсутствующим везде по одному сломанному соединению.
|
||||
- **Пустое расширение — это `.jpg`, а не «без суффикса».** В редакторе это плейсхолдер, и
|
||||
`PatternSourceConfig.NormaliseExtension` подставляет его в домене, потому что через `TryCreate`
|
||||
проходят и форма, и загрузка `sources.user.json`. Голый `/{id}` почти всегда опечатка, которая
|
||||
|
||||
@@ -12,6 +12,13 @@
|
||||
|
||||
- **`ProxyPool` переиспользует существующие `ProxyEntry` по `Endpoint.Key`** — иначе перезагрузка
|
||||
списка стирала бы статистику, а публичные фиды переиздаются каждые несколько минут.
|
||||
- **Проба обязана ходить по https.** Обычный HTTP через прокси — это форвард GET с абсолютным URI, а
|
||||
https — `CONNECT`-туннель, и свободные прокси сплошь и рядом умеют первое и отказывают во втором
|
||||
(«The proxy tunnel request to proxy … failed with status code 400»). Все собираемые адреса идут по
|
||||
https, поэтому прокси без CONNECT мертва *для нас*, и проверять надо ровно то, чем пользуемся.
|
||||
Раньше умолчание было http с обоснованием «не отбраковывать тех, кто просто не умеет CONNECT» — это
|
||||
и наполняло пул живыми на бумаге. Старое значение в `settings.json` подменяется на https в
|
||||
`AppSettings.ToProxyOptions()`; введённое пользователем не трогается.
|
||||
- **Выдаются только подтверждённо живые, пока такие есть.** Сортировки «живые вперёд» не хватало:
|
||||
её видит только липкая стратегия, берущая голову списка, а round-robin и взвешенный выбор тянут из
|
||||
всего набора — то есть из пары тысяч непроверенных адресов, и почти каждый запрос платил полный
|
||||
|
||||
@@ -195,6 +195,24 @@ public interface IMediaStore
|
||||
/// <returns>How many stored items were removed.</returns>
|
||||
Task<int> TombstoneAsync(string sha256, string? reason, CancellationToken cancellationToken = default);
|
||||
|
||||
/// <summary>Reads every standing rule the user has set on a picture.</summary>
|
||||
/// <remarks>Loaded once per run and filtered per source; the set is small and the check is hot.</remarks>
|
||||
Task<IReadOnlyList<MediaRule>> LoadRulesAsync(CancellationToken cancellationToken = default);
|
||||
|
||||
/// <summary>
|
||||
/// Stores a rule, replacing any earlier one for the same hash.
|
||||
/// </summary>
|
||||
/// <returns>How many stored items were removed, which only <see cref="MediaRuleAction.Skip"/> does.</returns>
|
||||
/// <remarks>
|
||||
/// A skip rule also tombstones: the user is looking at the thing in the gallery and saying "never
|
||||
/// again", and leaving the copy that prompted it on disk would answer half the request.
|
||||
/// </remarks>
|
||||
Task<int> SaveRuleAsync(MediaRule rule, CancellationToken cancellationToken = default);
|
||||
|
||||
/// <summary>Drops the rule for a hash. Returns whether one was there.</summary>
|
||||
/// <remarks>Does not bring back what a skip rule deleted; those bytes are gone.</remarks>
|
||||
Task<bool> RemoveRuleAsync(string sha256, CancellationToken cancellationToken = default);
|
||||
|
||||
/// <summary>Takes ownership of a completed download and records it.</summary>
|
||||
Task<CollectedItem> StoreAsync(MediaStoreRequest request, CancellationToken cancellationToken = default);
|
||||
|
||||
|
||||
@@ -35,4 +35,19 @@ public sealed record MediaCandidate(Uri Url)
|
||||
|
||||
/// <summary>1-based position within the listing, used to report failures against something.</summary>
|
||||
public int Ordinal { get; init; }
|
||||
|
||||
/// <summary>
|
||||
/// Further addresses for the same item, tried in order only if the ones before them are not there.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// This is what "one id, several extensions" means: a service that serves <c>/{id}.jpg</c> and
|
||||
/// <c>/{id}.png</c> holds one picture per id, not two, so emitting a candidate per extension
|
||||
/// would download the hit and then go looking for its twin. One candidate with fallbacks keeps
|
||||
/// the counters honest — a hit is one item, not one item and two misses — and lets the fetcher
|
||||
/// stop the moment something answers.
|
||||
/// </remarks>
|
||||
public IReadOnlyList<Uri> Alternatives { get; init; } = [];
|
||||
|
||||
/// <summary>Every address to try, in order, starting with <see cref="Url"/>.</summary>
|
||||
public IEnumerable<Uri> Addresses => Alternatives.Count == 0 ? [Url] : [Url, .. Alternatives];
|
||||
}
|
||||
|
||||
@@ -0,0 +1,92 @@
|
||||
namespace AvParser.Core.Collecting;
|
||||
|
||||
/// <summary>What to do when a collected item turns out to be a particular known picture.</summary>
|
||||
public enum MediaRuleAction
|
||||
{
|
||||
/// <summary>Never keep it: drop the bytes and settle the address without storing anything.</summary>
|
||||
Skip = 0,
|
||||
|
||||
/// <summary>Fetch the same address again through a proxy in another country.</summary>
|
||||
RetryElsewhere = 1,
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// A standing instruction about one exact picture, recognised by its content hash.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// <para>
|
||||
/// Born from what a scan actually produces: services answer a missing id with a placeholder and a
|
||||
/// geo-block with a banner, and both arrive as a perfectly valid image with a 200. Nothing about the
|
||||
/// address distinguishes them — only the bytes do. So the user points at one in the gallery and says
|
||||
/// what it means, and every future copy is treated the same way.
|
||||
/// </para>
|
||||
/// <para>
|
||||
/// Matching is by SHA-256, which is exact: a re-encoded or differently sized variant of the same
|
||||
/// banner is a different rule. That is the same deliberate line the store draws for de-duplication —
|
||||
/// there is no perceptual hashing here, and pretending otherwise would silently drop real finds.
|
||||
/// </para>
|
||||
/// </remarks>
|
||||
/// <param name="Sha256">Content hash the rule recognises.</param>
|
||||
/// <param name="Action">What to do with a match.</param>
|
||||
public sealed record MediaRule(string Sha256, MediaRuleAction Action)
|
||||
{
|
||||
/// <summary>Separates ids inside <see cref="SourceIds"/>.</summary>
|
||||
public const char SourceSeparator = ';';
|
||||
|
||||
/// <summary>
|
||||
/// Sources the rule covers, separated by <c>;</c>; <see langword="null"/> or empty means all of them.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// A joined string rather than a list so the record keeps value equality and the row stays one
|
||||
/// column. Scoped because the same bytes mean different things to different services: one host's
|
||||
/// "removed" placeholder is another host's perfectly ordinary picture.
|
||||
/// </remarks>
|
||||
public string? SourceIds { get; init; }
|
||||
|
||||
/// <summary>Why the user made this rule, for the list they will read months later.</summary>
|
||||
public string? Reason { get; init; }
|
||||
|
||||
/// <summary>When it was created.</summary>
|
||||
public DateTimeOffset CreatedUtc { get; init; } = DateTimeOffset.UtcNow;
|
||||
|
||||
/// <summary>Whether the rule covers <paramref name="sourceId"/>.</summary>
|
||||
public bool AppliesTo(string? sourceId)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(SourceIds))
|
||||
{
|
||||
return true;
|
||||
}
|
||||
|
||||
if (string.IsNullOrEmpty(sourceId))
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
foreach (var id in Split(SourceIds))
|
||||
{
|
||||
if (string.Equals(id, sourceId, StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
return true;
|
||||
}
|
||||
}
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
/// <summary>The ids this rule is scoped to; empty means every source.</summary>
|
||||
public IReadOnlyList<string> Sources => Split(SourceIds);
|
||||
|
||||
/// <summary>Joins ids for storage in <see cref="SourceIds"/>.</summary>
|
||||
public static string? JoinSources(IEnumerable<string> sourceIds)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(sourceIds);
|
||||
|
||||
var joined = string.Join(SourceSeparator, sourceIds);
|
||||
return joined.Length == 0 ? null : joined;
|
||||
}
|
||||
|
||||
private static string[] Split(string? ids) =>
|
||||
string.IsNullOrWhiteSpace(ids)
|
||||
? []
|
||||
: ids.Split(SourceSeparator, StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries);
|
||||
}
|
||||
@@ -112,14 +112,7 @@ public sealed class PatternMediaSource : IMediaSource
|
||||
|
||||
stale = 0;
|
||||
|
||||
yield return ParseOutcome<MediaCandidate>.Success(
|
||||
new MediaCandidate(new Uri(_config.BaseUrl, id + _config.Extension))
|
||||
{
|
||||
SourceId = Id,
|
||||
ExternalId = id,
|
||||
Ordinal = emitted + 1,
|
||||
}
|
||||
);
|
||||
yield return ParseOutcome<MediaCandidate>.Success(BuildCandidate(id, emitted + 1));
|
||||
|
||||
emitted++;
|
||||
|
||||
@@ -133,6 +126,37 @@ public sealed class PatternMediaSource : IMediaSource
|
||||
progress?.Report(new ParseProgress(emitted, budget));
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Builds one candidate for an id: the first suffix as the address, the rest as fallbacks.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// One candidate rather than one per suffix. A service holds one picture per id, so emitting
|
||||
/// three candidates for three suffixes would fetch the hit and then go hunting for its
|
||||
/// non-existent twins, count one item as three, and spend three journal rows on it.
|
||||
/// </remarks>
|
||||
private MediaCandidate BuildCandidate(string id, int ordinal)
|
||||
{
|
||||
var suffixes = _config.Extensions;
|
||||
|
||||
if (suffixes.Count <= 1)
|
||||
{
|
||||
return new MediaCandidate(new Uri(_config.BaseUrl, id + _config.Extension))
|
||||
{
|
||||
SourceId = Id,
|
||||
ExternalId = id,
|
||||
Ordinal = ordinal,
|
||||
};
|
||||
}
|
||||
|
||||
return new MediaCandidate(new Uri(_config.BaseUrl, id + suffixes[0]))
|
||||
{
|
||||
SourceId = Id,
|
||||
ExternalId = id,
|
||||
Ordinal = ordinal,
|
||||
Alternatives = [.. suffixes.Skip(1).Select(suffix => new Uri(_config.BaseUrl, id + suffix))],
|
||||
};
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// The number of candidates to generate: the caller's budget, capped at the id space when small.
|
||||
/// </summary>
|
||||
|
||||
@@ -116,8 +116,10 @@ public enum PatternConfigError
|
||||
/// <param name="Alphabet">Which characters ids are drawn from.</param>
|
||||
/// <param name="CustomAlphabet">Characters used when <paramref name="Alphabet"/> is <see cref="IdAlphabet.Custom"/>.</param>
|
||||
/// <param name="Extension">
|
||||
/// Suffix appended after the id. A blank one from the editor becomes <see cref="DefaultExtension"/>;
|
||||
/// a config built by hand may still carry <see langword="null"/> for none.
|
||||
/// Suffixes appended after the id, comma-separated and tried in order until one answers. A blank one
|
||||
/// from the editor becomes <see cref="DefaultExtension"/>; a config built by hand may still carry
|
||||
/// <see langword="null"/> for none. Stored as one string rather than a list so the record keeps value
|
||||
/// equality and an older <c>sources.user.json</c> holding a single suffix still reads.
|
||||
/// </param>
|
||||
/// <param name="AllowDirectConnection">
|
||||
/// Whether this source may run from the user's own address when no proxy is live. Per source rather
|
||||
@@ -141,6 +143,9 @@ public sealed record PatternSourceConfig(
|
||||
/// <summary>Hard ceiling on id length, guarding both the UI spinner and a hand-edited file.</summary>
|
||||
public const int MaxIdLength = 64;
|
||||
|
||||
/// <summary>Separates suffixes inside <see cref="Extension"/>.</summary>
|
||||
public const char ExtensionSeparator = ',';
|
||||
|
||||
/// <summary>Suffix used when the editor's extension field is left empty.</summary>
|
||||
/// <remarks>
|
||||
/// A blank field means "the usual one", not "no suffix": every host worth pointing this at
|
||||
@@ -153,6 +158,15 @@ public sealed record PatternSourceConfig(
|
||||
/// <summary>The characters this config draws ids from.</summary>
|
||||
public string AlphabetCharacters => PatternAlphabet.Resolve(Alphabet, CustomAlphabet);
|
||||
|
||||
/// <summary>The suffixes to try, in order. Empty only for a hand-built config that wants none.</summary>
|
||||
public IReadOnlyList<string> Extensions =>
|
||||
string.IsNullOrEmpty(Extension)
|
||||
? []
|
||||
: Extension.Split(
|
||||
ExtensionSeparator,
|
||||
StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries
|
||||
);
|
||||
|
||||
/// <summary>
|
||||
/// Validates and normalises the inputs, generating an id when one is not supplied.
|
||||
/// </summary>
|
||||
@@ -219,7 +233,18 @@ public sealed record PatternSourceConfig(
|
||||
|
||||
private static Uri EnsureTrailingSlash(Uri url)
|
||||
{
|
||||
if (url.AbsoluteUri.EndsWith('/'))
|
||||
var text = url.AbsoluteUri;
|
||||
|
||||
// A pasted address that already ended in a slash can easily pick up a second one, and
|
||||
// "host//" is not "host/": ids resolve against it as //{id}, which most servers answer with
|
||||
// a 404 for every single attempt. Collapsed rather than rejected — it is a typo, not a
|
||||
// different intention.
|
||||
if (text.EndsWith("//", StringComparison.Ordinal))
|
||||
{
|
||||
return new Uri(text.TrimEnd('/') + "/", UriKind.Absolute);
|
||||
}
|
||||
|
||||
if (text.EndsWith('/'))
|
||||
{
|
||||
return url;
|
||||
}
|
||||
@@ -229,15 +254,41 @@ public sealed record PatternSourceConfig(
|
||||
return new Uri(url.AbsoluteUri + "/", UriKind.Absolute);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Turns whatever the user typed into a canonical, ordered, duplicate-free suffix list.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// Separators are generous — comma, semicolon or plain spaces — because the field is a text box
|
||||
/// and every one of those is a reasonable thing to type. Order is the user's and is preserved:
|
||||
/// it is the order the fetcher will try, so "jpg png" and "png jpg" are different instructions.
|
||||
/// </remarks>
|
||||
private static string NormaliseExtension(string? extension)
|
||||
{
|
||||
var trimmed = extension?.Trim();
|
||||
if (string.IsNullOrEmpty(trimmed))
|
||||
if (string.IsNullOrWhiteSpace(extension))
|
||||
{
|
||||
return DefaultExtension;
|
||||
}
|
||||
|
||||
return trimmed.StartsWith('.') ? trimmed : "." + trimmed;
|
||||
var parts = extension.Split(
|
||||
[ExtensionSeparator, ';', ' ', ' '],
|
||||
StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries
|
||||
);
|
||||
|
||||
var seen = new HashSet<string>(StringComparer.OrdinalIgnoreCase);
|
||||
var kept = new List<string>(parts.Length);
|
||||
|
||||
foreach (var part in parts)
|
||||
{
|
||||
var suffix = part.StartsWith('.') ? part : "." + part;
|
||||
|
||||
// A repeat would mean fetching the same address twice before giving up on the id.
|
||||
if (suffix.Length > 1 && seen.Add(suffix))
|
||||
{
|
||||
kept.Add(suffix);
|
||||
}
|
||||
}
|
||||
|
||||
return kept.Count == 0 ? DefaultExtension : string.Join(ExtensionSeparator, kept);
|
||||
}
|
||||
|
||||
private static string NewId(string name)
|
||||
|
||||
@@ -33,6 +33,19 @@ public interface IProxyPool
|
||||
/// <remarks>Report the outcome on the lease, otherwise the pool never learns anything.</remarks>
|
||||
Task<ProxyLease?> AcquireAsync(CancellationToken cancellationToken = default);
|
||||
|
||||
/// <summary>
|
||||
/// Takes a proxy that is not in any of <paramref name="excludedCountries"/>, when one exists.
|
||||
/// </summary>
|
||||
/// <param name="excludedCountries">ISO codes already tried; matched case-insensitively.</param>
|
||||
/// <param name="cancellationToken">Cancellation.</param>
|
||||
/// <returns>A lease, or <see langword="null"/> when nothing outside those countries is usable.</returns>
|
||||
/// <remarks>
|
||||
/// For "this picture is a geo-block, try somewhere else". Entries whose country the feed never
|
||||
/// stated stay eligible: an unknown country is not a known match, and refusing them would turn a
|
||||
/// missing field into a smaller pool.
|
||||
/// </remarks>
|
||||
Task<ProxyLease?> AcquireAsync(IReadOnlySet<string>? excludedCountries, CancellationToken cancellationToken = default);
|
||||
|
||||
/// <summary>Probes every entry in parallel and updates their health.</summary>
|
||||
/// <returns>How many answered.</returns>
|
||||
Task<int> SweepAsync(IProgress<ProxySweepProgress>? progress = null, CancellationToken cancellationToken = default);
|
||||
|
||||
@@ -74,10 +74,19 @@ public sealed record ProxyOptions
|
||||
|
||||
/// <summary>URL fetched to decide whether a proxy works.</summary>
|
||||
/// <remarks>
|
||||
/// Defaults to a plain-HTTP 204 endpoint: it is tiny, and requiring TLS would fail every
|
||||
/// proxy that cannot do CONNECT rather than every proxy that is actually dead.
|
||||
/// <para>
|
||||
/// <b>Must be https.</b> A plain-HTTP probe is an absolute-URI GET, which a proxy can serve by
|
||||
/// forwarding; an https request is a <c>CONNECT</c> tunnel, which many free proxies refuse. This
|
||||
/// used to default to http on the reasoning that requiring TLS would fail proxies that were not
|
||||
/// actually dead — but every address this app collects from is https, so a proxy that cannot
|
||||
/// tunnel is dead <i>for our purposes</i>. Probing http marked those live and then failed every
|
||||
/// real request with "the proxy tunnel request failed with status code 400".
|
||||
/// </para>
|
||||
/// <para>
|
||||
/// A 204 endpoint keeps the check tiny; what earns the "live" verdict is the tunnel, not the body.
|
||||
/// </para>
|
||||
/// </remarks>
|
||||
public Uri ProbeUrl { get; init; } = new("http://www.gstatic.com/generate_204");
|
||||
public Uri ProbeUrl { get; init; } = new("https://www.gstatic.com/generate_204");
|
||||
|
||||
/// <summary>Per-proxy probe timeout.</summary>
|
||||
public TimeSpan ProbeTimeout { get; init; } = TimeSpan.FromSeconds(8);
|
||||
@@ -95,7 +104,7 @@ public sealed record ProxyOptions
|
||||
public TimeSpan MaxQuarantine { get; init; } = TimeSpan.FromMinutes(15);
|
||||
|
||||
/// <summary>
|
||||
/// Proxies tried per <see cref="IProxyPool.AcquireAsync"/> call under
|
||||
/// Proxies tried per <see cref="IProxyPool.AcquireAsync(CancellationToken)"/> call under
|
||||
/// <see cref="ProxyHealthCheck.Lazy"/> before giving up.
|
||||
/// </summary>
|
||||
public int LazyProbeAttempts { get; init; } = 5;
|
||||
|
||||
@@ -167,13 +167,20 @@ public sealed class ProxyPool : IProxyPool
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
public async Task<ProxyLease?> AcquireAsync(CancellationToken cancellationToken = default)
|
||||
public Task<ProxyLease?> AcquireAsync(CancellationToken cancellationToken = default) =>
|
||||
AcquireAsync(null, cancellationToken);
|
||||
|
||||
/// <inheritdoc />
|
||||
public async Task<ProxyLease?> AcquireAsync(
|
||||
IReadOnlySet<string>? excludedCountries,
|
||||
CancellationToken cancellationToken = default
|
||||
)
|
||||
{
|
||||
var options = Options;
|
||||
|
||||
if (options.HealthCheck == ProxyHealthCheck.Pool)
|
||||
{
|
||||
var entry = SelectAvailable();
|
||||
var entry = SelectAvailable(excludedCountries);
|
||||
return entry is null ? null : new ProxyLease(this, entry);
|
||||
}
|
||||
|
||||
@@ -182,7 +189,7 @@ public sealed class ProxyPool : IProxyPool
|
||||
{
|
||||
cancellationToken.ThrowIfCancellationRequested();
|
||||
|
||||
var entry = SelectAvailable();
|
||||
var entry = SelectAvailable(excludedCountries);
|
||||
if (entry is null)
|
||||
{
|
||||
return null;
|
||||
@@ -533,13 +540,15 @@ public sealed class ProxyPool : IProxyPool
|
||||
/// on that fallback is marked live by its own success.
|
||||
/// </para>
|
||||
/// </remarks>
|
||||
private ProxyEntry? SelectAvailable()
|
||||
private ProxyEntry? SelectAvailable(IReadOnlySet<string>? excludedCountries = null)
|
||||
{
|
||||
lock (_gate)
|
||||
{
|
||||
var now = _time.GetUtcNow();
|
||||
|
||||
var available = _entries.Where(entry => entry.IsAvailable(now)).ToArray();
|
||||
var available = _entries
|
||||
.Where(entry => entry.IsAvailable(now) && !IsExcluded(entry, excludedCountries))
|
||||
.ToArray();
|
||||
var live = available.Where(entry => entry.Health == ProxyHealthState.Alive).ToArray();
|
||||
|
||||
var candidates = (live.Length > 0 ? live : available)
|
||||
@@ -551,5 +560,15 @@ public sealed class ProxyPool : IProxyPool
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>Whether the entry sits in a country the caller has already ruled out.</summary>
|
||||
/// <remarks>
|
||||
/// An entry the feed gave no country for is never excluded: "unknown" is not a match, and
|
||||
/// dropping those would shrink the pool over a missing field rather than over a real answer.
|
||||
/// </remarks>
|
||||
private static bool IsExcluded(ProxyEntry entry, IReadOnlySet<string>? excludedCountries) =>
|
||||
excludedCountries is { Count: > 0 }
|
||||
&& entry.Endpoint.Country is { Length: > 0 } country
|
||||
&& excludedCountries.Contains(country);
|
||||
|
||||
private void RaiseChanged() => Changed?.Invoke(this, EventArgs.Empty);
|
||||
}
|
||||
|
||||
@@ -81,7 +81,7 @@ public sealed record AppSettings(
|
||||
ProxyHealthCheck ProxyHealthCheck = ProxyHealthCheck.Pool,
|
||||
bool ProxyUseFeed = true,
|
||||
ProxyProtocolFilter ProxyProtocols = ProxyProtocolFilter.All,
|
||||
string ProxyProbeUrl = "http://www.gstatic.com/generate_204",
|
||||
string ProxyProbeUrl = "https://www.gstatic.com/generate_204",
|
||||
int ProxyProbeTimeoutSeconds = 8,
|
||||
int ProxyProbeConcurrency = 64,
|
||||
int ProxyMinimumLive = 10,
|
||||
@@ -104,6 +104,9 @@ public sealed record AppSettings(
|
||||
string? CollectSourceIds = null
|
||||
)
|
||||
{
|
||||
/// <summary>The probe endpoint shipped before the check moved to https.</summary>
|
||||
private const string LegacyProbeUrl = "http://www.gstatic.com/generate_204";
|
||||
|
||||
/// <summary>Separates ids inside <see cref="CollectSourceIds"/>.</summary>
|
||||
public const char SourceIdSeparator = ';';
|
||||
|
||||
@@ -135,7 +138,7 @@ public sealed record AppSettings(
|
||||
public ProxyOptions ToProxyOptions()
|
||||
{
|
||||
var probeUrl = Uri.TryCreate(ProxyProbeUrl, UriKind.Absolute, out var parsed)
|
||||
? parsed
|
||||
? Upgraded(parsed)
|
||||
: new ProxyOptions().ProbeUrl;
|
||||
|
||||
return new ProxyOptions
|
||||
@@ -151,6 +154,17 @@ public sealed record AppSettings(
|
||||
}.Validated();
|
||||
}
|
||||
|
||||
/// <summary>Replaces the old plain-HTTP probe endpoint with its https twin.</summary>
|
||||
/// <remarks>
|
||||
/// Only the value this app used to ship: a file written before the probe moved to https would
|
||||
/// otherwise keep marking proxies live that cannot open a tunnel, which is the one thing every
|
||||
/// collected address needs. Anything the user typed themselves is left exactly as they typed it.
|
||||
/// </remarks>
|
||||
private static Uri Upgraded(Uri probeUrl) =>
|
||||
string.Equals(probeUrl.AbsoluteUri, LegacyProbeUrl, StringComparison.OrdinalIgnoreCase)
|
||||
? new ProxyOptions().ProbeUrl
|
||||
: probeUrl;
|
||||
|
||||
/// <summary>Projects the collector settings onto <see cref="CollectOptions"/>.</summary>
|
||||
/// <remarks>
|
||||
/// The one place primitives become policy, and it clamps rather than throws for the same reason
|
||||
|
||||
@@ -60,8 +60,8 @@ public sealed class CollectRunner(
|
||||
_store.Configure(options.ShowcaseMode);
|
||||
|
||||
var runId = await _store.BeginRunAsync(source.Id, cancellationToken).ConfigureAwait(false);
|
||||
var tombstones = await _store.LoadTombstonesAsync(cancellationToken).ConfigureAwait(false);
|
||||
var fetchOptions = BuildFetchOptions(options, tombstones);
|
||||
var rules = await LoadRulesAsync(source.Id, cancellationToken).ConfigureAwait(false);
|
||||
var fetchOptions = BuildFetchOptions(options, rules);
|
||||
|
||||
var work = Channel.CreateBounded<MediaCandidate>(
|
||||
new BoundedChannelOptions(workers * 4) { FullMode = BoundedChannelFullMode.Wait }
|
||||
@@ -346,8 +346,42 @@ public sealed class CollectRunner(
|
||||
return ParseOutcome<CollectedItem>.Success(stored with { Elapsed = result.Elapsed });
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Collects everything the user has already ruled on that applies to this source.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// Merged into one map so the fetcher does a single lookup per download. Dead-link tombstones
|
||||
/// predate rules and are global, so they come in as unscoped skips; a rule for the same hash
|
||||
/// wins, because it is the more recent and more specific statement.
|
||||
/// </remarks>
|
||||
private async Task<IReadOnlyDictionary<string, MediaRuleAction>> LoadRulesAsync(
|
||||
string sourceId,
|
||||
CancellationToken cancellationToken
|
||||
)
|
||||
{
|
||||
var merged = new Dictionary<string, MediaRuleAction>(StringComparer.Ordinal);
|
||||
|
||||
foreach (var hash in await _store.LoadTombstonesAsync(cancellationToken).ConfigureAwait(false))
|
||||
{
|
||||
merged[hash] = MediaRuleAction.Skip;
|
||||
}
|
||||
|
||||
foreach (var rule in await _store.LoadRulesAsync(cancellationToken).ConfigureAwait(false))
|
||||
{
|
||||
if (rule.AppliesTo(sourceId))
|
||||
{
|
||||
merged[rule.Sha256] = rule.Action;
|
||||
}
|
||||
}
|
||||
|
||||
return merged;
|
||||
}
|
||||
|
||||
/// <summary>Projects the run's policy onto the HTTP layer's parameters.</summary>
|
||||
private static FetchOptions BuildFetchOptions(CollectOptions options, IReadOnlySet<string> tombstones) =>
|
||||
private static FetchOptions BuildFetchOptions(
|
||||
CollectOptions options,
|
||||
IReadOnlyDictionary<string, MediaRuleAction> rules
|
||||
) =>
|
||||
new()
|
||||
{
|
||||
MaxItemBytes = options.MaxItemBytes,
|
||||
@@ -358,7 +392,7 @@ public sealed class CollectRunner(
|
||||
RequireProxy = options.RequireProxy,
|
||||
UserAgent = options.UserAgent,
|
||||
AllowedKinds = MediaKindFilters.ToSet(options.AllowedKinds),
|
||||
Tombstones = tombstones,
|
||||
Rules = rules,
|
||||
};
|
||||
|
||||
/// <summary>Stands in for content on a skipped item, which by definition has none.</summary>
|
||||
|
||||
@@ -44,8 +44,20 @@ public sealed record FetchOptions
|
||||
MediaKind.WebM,
|
||||
};
|
||||
|
||||
/// <summary>Hashes already known to be dead-link placeholders.</summary>
|
||||
public IReadOnlySet<string> Tombstones { get; init; } = new HashSet<string>(StringComparer.Ordinal);
|
||||
/// <summary>
|
||||
/// What to do about pictures the user has already ruled on, by content hash.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// Filtered to the running source before it gets here, so the fetcher never has to know which
|
||||
/// source it is working for. Old dead-link tombstones arrive in the same map as
|
||||
/// <see cref="MediaRuleAction.Skip"/>: one lookup on the hot path rather than two.
|
||||
/// </remarks>
|
||||
public IReadOnlyDictionary<string, MediaRuleAction> Rules { get; init; } =
|
||||
new Dictionary<string, MediaRuleAction>(StringComparer.Ordinal);
|
||||
|
||||
/// <summary>Countries the pool must not hand out for this attempt.</summary>
|
||||
/// <remarks>Set only by the retry that a <see cref="MediaRuleAction.RetryElsewhere"/> rule triggers.</remarks>
|
||||
public IReadOnlySet<string>? ExcludedCountries { get; init; }
|
||||
}
|
||||
|
||||
/// <summary>What one download attempt produced.</summary>
|
||||
@@ -74,6 +86,9 @@ public sealed record FetchResult(SeenOutcome Outcome, Uri FinalUrl)
|
||||
/// <summary>Proxy the content came through, or null when direct.</summary>
|
||||
public string? ProxyKey { get; init; }
|
||||
|
||||
/// <summary>Country of that proxy, when the feed stated one. Drives the "try elsewhere" retry.</summary>
|
||||
public string? ProxyCountry { get; init; }
|
||||
|
||||
/// <summary>Localisation code for the failure, matching a <c>Parse.Error.{Code}</c> key.</summary>
|
||||
public string? ErrorCode { get; init; }
|
||||
|
||||
|
||||
@@ -51,6 +51,18 @@ public sealed class MediaFetcher(
|
||||
/// <summary>Big enough that a 32 MB file is a few hundred reads, small enough to pool cheaply.</summary>
|
||||
private const int BufferSize = 64 * 1024;
|
||||
|
||||
/// <summary>Internal marker for "the user says this picture means: come back from elsewhere".</summary>
|
||||
/// <remarks>Never reaches the journal: the retry loop turns it into a real outcome either way.</remarks>
|
||||
private const string RetryElsewhereCode = "RetryElsewhere";
|
||||
|
||||
/// <summary>How many countries to try before accepting that the picture is not available here.</summary>
|
||||
/// <remarks>
|
||||
/// Small on purpose: each attempt is a full download of a banner, and a service that blocks one
|
||||
/// country usually blocks a region. Three says "this is not a fluke" without turning one id into
|
||||
/// a tour of the pool.
|
||||
/// </remarks>
|
||||
private const int MaxCountryAttempts = 3;
|
||||
|
||||
private readonly IProxiedHttpClientFactory _clients = clients ?? throw new ArgumentNullException(nameof(clients));
|
||||
private readonly BlobStore _blobs = blobs ?? throw new ArgumentNullException(nameof(blobs));
|
||||
private readonly HostThrottle _throttle = throttle ?? throw new ArgumentNullException(nameof(throttle));
|
||||
@@ -73,7 +85,7 @@ public sealed class MediaFetcher(
|
||||
|
||||
try
|
||||
{
|
||||
var result = await FetchCoreAsync(candidate, options, stopwatch, budget.Token, cancellationToken)
|
||||
var result = await FetchAddressesAsync(candidate, options, stopwatch, budget.Token, cancellationToken)
|
||||
.ConfigureAwait(false);
|
||||
|
||||
// Cancelling a socket read surfaces as an IOException as often as an
|
||||
@@ -99,6 +111,131 @@ public sealed class MediaFetcher(
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Tries the candidate's addresses in order and stops at the first one that is actually there.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// <para>
|
||||
/// This is "several extensions per id": <c>/{id}.jpg</c>, then <c>/{id}.png</c>, until something
|
||||
/// answers or nothing does. Only a verdict of "not here" moves on — a 404, a placeholder under
|
||||
/// the size floor, a page that is not media. A timeout or a proxy failure means we do not know
|
||||
/// what is at that address, and burning the remaining suffixes on a broken connection would
|
||||
/// report "nothing here" about an id nobody actually asked about.
|
||||
/// </para>
|
||||
/// <para>
|
||||
/// The miss that gets returned is the last one, so the journal records the id as settled by the
|
||||
/// suffix that was tried last rather than by an early guess.
|
||||
/// </para>
|
||||
/// </remarks>
|
||||
private async Task<FetchResult> FetchAddressesAsync(
|
||||
MediaCandidate candidate,
|
||||
FetchOptions options,
|
||||
Stopwatch stopwatch,
|
||||
CancellationToken token,
|
||||
CancellationToken userToken
|
||||
)
|
||||
{
|
||||
FetchResult? lastMiss = null;
|
||||
|
||||
foreach (var address in candidate.Addresses)
|
||||
{
|
||||
var attempt = address == candidate.Url ? candidate : candidate with { Url = address, Alternatives = [] };
|
||||
var result = await FetchWithRulesAsync(attempt, options, stopwatch, token, userToken)
|
||||
.ConfigureAwait(false);
|
||||
|
||||
if (!IsMissing(result.Outcome))
|
||||
{
|
||||
return result;
|
||||
}
|
||||
|
||||
lastMiss = result;
|
||||
}
|
||||
|
||||
return lastMiss!;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Fetches one address, and when the picture that comes back is one the user has ruled a
|
||||
/// geo-block, fetches it again from another country.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// <para>
|
||||
/// The retry forces a proxy even when the source is allowed to run direct: the whole point is to
|
||||
/// arrive from somewhere else, and falling back to the user's own address would be the one place
|
||||
/// we already know shows the banner.
|
||||
/// </para>
|
||||
/// <para>
|
||||
/// Giving up stores nothing and leaves the address <b>retryable</b> — a banner is not the picture,
|
||||
/// and recording it as settled would burn the id for good on the strength of today's pool. A
|
||||
/// proxy whose country the feed never stated cannot be excluded from, so that ends the retry too:
|
||||
/// "somewhere else" is not a thing we can ask for.
|
||||
/// </para>
|
||||
/// </remarks>
|
||||
private async Task<FetchResult> FetchWithRulesAsync(
|
||||
MediaCandidate candidate,
|
||||
FetchOptions options,
|
||||
Stopwatch stopwatch,
|
||||
CancellationToken token,
|
||||
CancellationToken userToken
|
||||
)
|
||||
{
|
||||
var tried = new HashSet<string>(StringComparer.OrdinalIgnoreCase);
|
||||
var attemptOptions = options;
|
||||
|
||||
for (var attempt = 0; ; attempt++)
|
||||
{
|
||||
FetchResult result;
|
||||
|
||||
try
|
||||
{
|
||||
result = await FetchCoreAsync(candidate, attemptOptions, stopwatch, token, userToken)
|
||||
.ConfigureAwait(false);
|
||||
}
|
||||
catch (ProxyUnavailableException) when (attempt > 0)
|
||||
{
|
||||
// Nothing left outside the countries already tried.
|
||||
return GeoBlocked(candidate.Url, stopwatch, tried);
|
||||
}
|
||||
|
||||
if (result.ErrorCode != RetryElsewhereCode)
|
||||
{
|
||||
return result;
|
||||
}
|
||||
|
||||
if (result.ProxyCountry is { Length: > 0 } country)
|
||||
{
|
||||
tried.Add(country);
|
||||
}
|
||||
|
||||
if (tried.Count == 0 || attempt + 1 >= MaxCountryAttempts)
|
||||
{
|
||||
return GeoBlocked(candidate.Url, stopwatch, tried);
|
||||
}
|
||||
|
||||
_logger.LogDebug(
|
||||
"{Url} matched a try-elsewhere rule; retrying outside {Countries}",
|
||||
candidate.Url,
|
||||
string.Join(", ", tried)
|
||||
);
|
||||
|
||||
attemptOptions = options with { ExcludedCountries = tried, RequireProxy = true };
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>The verdict when the picture is a known geo-block and there is nowhere else to be.</summary>
|
||||
private static FetchResult GeoBlocked(Uri url, Stopwatch stopwatch, HashSet<string> tried) =>
|
||||
Failure(
|
||||
url,
|
||||
SeenOutcome.Failed,
|
||||
"GeoBlocked",
|
||||
tried.Count == 0 ? null : string.Join(", ", tried),
|
||||
stopwatch
|
||||
);
|
||||
|
||||
/// <summary>Whether the address simply held nothing, which is the only reason to try the next one.</summary>
|
||||
private static bool IsMissing(SeenOutcome outcome) =>
|
||||
outcome is SeenOutcome.Gone or SeenOutcome.NotMedia or SeenOutcome.TooSmall or SeenOutcome.Placeholder;
|
||||
|
||||
private async Task<FetchResult> FetchCoreAsync(
|
||||
MediaCandidate candidate,
|
||||
FetchOptions options,
|
||||
@@ -108,7 +245,7 @@ public sealed class MediaFetcher(
|
||||
)
|
||||
{
|
||||
using var leased = await _clients
|
||||
.LeaseAsync(options.Timeouts, options.RequireProxy, token)
|
||||
.LeaseAsync(options.Timeouts, options.RequireProxy, options.ExcludedCountries, token)
|
||||
.ConfigureAwait(false);
|
||||
|
||||
HttpStatusCode? status = null;
|
||||
@@ -457,13 +594,16 @@ public sealed class MediaFetcher(
|
||||
);
|
||||
}
|
||||
|
||||
if (options.Tombstones.Contains(hash!))
|
||||
if (options.Rules.TryGetValue(hash!, out var rule))
|
||||
{
|
||||
return (
|
||||
Failure(url, SeenOutcome.Placeholder, "Placeholder", null, stopwatch, leased.ProxyKey, status),
|
||||
true,
|
||||
null
|
||||
);
|
||||
// The bytes are the only thing that identifies these: a service answers a missing id
|
||||
// and a geo-block with an ordinary 200 and a perfectly valid picture.
|
||||
var outcome =
|
||||
rule == MediaRuleAction.Skip
|
||||
? Failure(url, SeenOutcome.Placeholder, "Placeholder", null, stopwatch, leased.ProxyKey, status)
|
||||
: Failure(url, SeenOutcome.Failed, RetryElsewhereCode, null, stopwatch, leased.ProxyKey, status);
|
||||
|
||||
return (outcome with { ProxyCountry = leased.ProxyCountry }, true, null);
|
||||
}
|
||||
|
||||
var blob = await DescribeAsync(temp, hash!, kind, total, prefix, token).ConfigureAwait(false);
|
||||
|
||||
@@ -83,6 +83,33 @@ public sealed class MediaStore(
|
||||
return plan.ItemsRemoved;
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
public Task<IReadOnlyList<MediaRule>> LoadRulesAsync(CancellationToken cancellationToken = default) =>
|
||||
_index.LoadRulesAsync(cancellationToken);
|
||||
|
||||
/// <inheritdoc />
|
||||
public async Task<int> SaveRuleAsync(MediaRule rule, CancellationToken cancellationToken = default)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(rule);
|
||||
|
||||
await _index.SaveRuleAsync(rule, cancellationToken).ConfigureAwait(false);
|
||||
|
||||
if (rule.Action != MediaRuleAction.Skip)
|
||||
{
|
||||
_logger.LogInformation("Rule for {Hash}: {Action}", rule.Sha256, rule.Action);
|
||||
return 0;
|
||||
}
|
||||
|
||||
// A skip rule is made while looking at the thing in the gallery: leaving that copy on disk
|
||||
// would answer half of "never again". The tombstone path already removes every copy and the
|
||||
// showcase links pointing at it, so it stays the one place that knows how.
|
||||
return await TombstoneAsync(rule.Sha256, rule.Reason, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
public Task<bool> RemoveRuleAsync(string sha256, CancellationToken cancellationToken = default) =>
|
||||
_index.RemoveRuleAsync(sha256, cancellationToken);
|
||||
|
||||
/// <inheritdoc />
|
||||
public async Task<CollectedItem> StoreAsync(
|
||||
MediaStoreRequest request,
|
||||
|
||||
@@ -265,6 +265,87 @@ public sealed class SqliteMediaIndex : IDisposable
|
||||
return hashes;
|
||||
}
|
||||
|
||||
/// <summary>Reads every standing rule the user has set on a picture.</summary>
|
||||
public async Task<IReadOnlyList<MediaRule>> LoadRulesAsync(CancellationToken cancellationToken = default)
|
||||
{
|
||||
var rules = new List<MediaRule>();
|
||||
|
||||
await using var connection = await OpenAsync(cancellationToken).ConfigureAwait(false);
|
||||
await using var command = connection.CreateCommand();
|
||||
command.CommandText = "SELECT sha256, action, source_ids, reason, created_utc FROM media_rule;";
|
||||
|
||||
await using var reader = await command.ExecuteReaderAsync(cancellationToken).ConfigureAwait(false);
|
||||
while (await reader.ReadAsync(cancellationToken).ConfigureAwait(false))
|
||||
{
|
||||
rules.Add(
|
||||
new MediaRule(reader.GetString(0), (MediaRuleAction)reader.GetInt32(1))
|
||||
{
|
||||
SourceIds = reader.IsDBNull(2) ? null : reader.GetString(2),
|
||||
Reason = reader.IsDBNull(3) ? null : reader.GetString(3),
|
||||
CreatedUtc = DateTimeOffset.Parse(
|
||||
reader.GetString(4),
|
||||
CultureInfo.InvariantCulture,
|
||||
DateTimeStyles.RoundtripKind
|
||||
),
|
||||
}
|
||||
);
|
||||
}
|
||||
|
||||
return rules;
|
||||
}
|
||||
|
||||
/// <summary>Writes a rule, replacing any earlier one for the same hash.</summary>
|
||||
public Task SaveRuleAsync(MediaRule rule, CancellationToken cancellationToken = default)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(rule);
|
||||
|
||||
return WriteAsync(
|
||||
async connection =>
|
||||
{
|
||||
await using var command = connection.CreateCommand();
|
||||
command.CommandText = """
|
||||
INSERT INTO media_rule(sha256, action, source_ids, reason, created_utc)
|
||||
VALUES($sha, $action, $sources, $reason, $created)
|
||||
ON CONFLICT(sha256) DO UPDATE SET
|
||||
action = excluded.action,
|
||||
source_ids = excluded.source_ids,
|
||||
reason = excluded.reason;
|
||||
""";
|
||||
command.Parameters.AddWithValue("$sha", rule.Sha256);
|
||||
command.Parameters.AddWithValue("$action", (int)rule.Action);
|
||||
command.Parameters.AddWithValue("$sources", (object?)rule.SourceIds ?? DBNull.Value);
|
||||
command.Parameters.AddWithValue("$reason", (object?)rule.Reason ?? DBNull.Value);
|
||||
command.Parameters.AddWithValue("$created", Now());
|
||||
|
||||
await command.ExecuteNonQueryAsync(cancellationToken).ConfigureAwait(false);
|
||||
},
|
||||
cancellationToken
|
||||
);
|
||||
}
|
||||
|
||||
/// <summary>Drops the rule for a hash. Returns whether one was there.</summary>
|
||||
public async Task<bool> RemoveRuleAsync(string sha256, CancellationToken cancellationToken = default)
|
||||
{
|
||||
ArgumentException.ThrowIfNullOrWhiteSpace(sha256);
|
||||
|
||||
var removed = 0;
|
||||
|
||||
await WriteAsync(
|
||||
async connection =>
|
||||
{
|
||||
await using var command = connection.CreateCommand();
|
||||
command.CommandText = "DELETE FROM media_rule WHERE sha256 = $sha;";
|
||||
command.Parameters.AddWithValue("$sha", sha256);
|
||||
|
||||
removed = await command.ExecuteNonQueryAsync(cancellationToken).ConfigureAwait(false);
|
||||
},
|
||||
cancellationToken
|
||||
)
|
||||
.ConfigureAwait(false);
|
||||
|
||||
return removed > 0;
|
||||
}
|
||||
|
||||
/// <summary>Marks a hash as a placeholder and plans the removal of every copy held.</summary>
|
||||
public async Task<RemovalPlan> TombstoneAsync(
|
||||
string sha256,
|
||||
@@ -974,6 +1055,14 @@ public sealed class SqliteMediaIndex : IDisposable
|
||||
|
||||
CREATE INDEX IF NOT EXISTS ix_seen_outcome ON seen_url(source_id, outcome);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS media_rule (
|
||||
sha256 TEXT PRIMARY KEY NOT NULL,
|
||||
action INTEGER NOT NULL,
|
||||
source_ids TEXT NULL,
|
||||
reason TEXT NULL,
|
||||
created_utc TEXT NOT NULL
|
||||
) STRICT;
|
||||
|
||||
CREATE TABLE IF NOT EXISTS tombstone (
|
||||
sha256 TEXT PRIMARY KEY NOT NULL,
|
||||
reason TEXT NULL,
|
||||
|
||||
@@ -58,6 +58,9 @@ public sealed class LeasedHttpClient(HttpClient client, ProxyLease? lease) : IDi
|
||||
/// <summary>Address of the proxy in use, for recording provenance.</summary>
|
||||
public string? ProxyKey => Lease?.Endpoint.Key;
|
||||
|
||||
/// <summary>Country of the proxy in use, when the feed stated one.</summary>
|
||||
public string? ProxyCountry => Lease?.Endpoint.Country;
|
||||
|
||||
/// <inheritdoc />
|
||||
public void Dispose()
|
||||
{
|
||||
@@ -106,6 +109,9 @@ public interface IProxiedHttpClientFactory
|
||||
/// <param name="requireProxy">
|
||||
/// When set, having no live proxy throws instead of quietly connecting directly.
|
||||
/// </param>
|
||||
/// <param name="excludedCountries">
|
||||
/// Countries the pool must not hand out, for a "come back from somewhere else" retry.
|
||||
/// </param>
|
||||
/// <param name="cancellationToken">Cancels acquisition.</param>
|
||||
/// <exception cref="ProxyUnavailableException">
|
||||
/// <paramref name="requireProxy"/> was set and the pool had nothing live.
|
||||
@@ -113,6 +119,7 @@ public interface IProxiedHttpClientFactory
|
||||
Task<LeasedHttpClient> LeaseAsync(
|
||||
HttpClientTimeouts timeouts,
|
||||
bool requireProxy,
|
||||
IReadOnlySet<string>? excludedCountries = null,
|
||||
CancellationToken cancellationToken = default
|
||||
);
|
||||
}
|
||||
@@ -158,10 +165,11 @@ public sealed class ProxiedHttpClientFactory(IProxyPool pool) : IProxiedHttpClie
|
||||
public async Task<LeasedHttpClient> LeaseAsync(
|
||||
HttpClientTimeouts timeouts,
|
||||
bool requireProxy,
|
||||
IReadOnlySet<string>? excludedCountries = null,
|
||||
CancellationToken cancellationToken = default
|
||||
)
|
||||
{
|
||||
var lease = await _pool.AcquireAsync(cancellationToken).ConfigureAwait(false);
|
||||
var lease = await _pool.AcquireAsync(excludedCountries, cancellationToken).ConfigureAwait(false);
|
||||
|
||||
if (lease is null && requireProxy)
|
||||
{
|
||||
|
||||
@@ -269,7 +269,7 @@
|
||||
<value>PROBE URL</value>
|
||||
</data>
|
||||
<data name="Settings.ProbeUrlHint" xml:space="preserve">
|
||||
<value>Plain HTTP by default: requiring TLS would fail every proxy that cannot do CONNECT, not just the dead ones.</value>
|
||||
<value>Keep it https: collected addresses are https, so the check has to open a tunnel the way the collector does. A plain-HTTP probe passes proxies that then refuse every real request.</value>
|
||||
</data>
|
||||
<data name="Settings.ProbeTimeout" xml:space="preserve">
|
||||
<value>PROBE TIMEOUT (SEC)</value>
|
||||
@@ -491,7 +491,10 @@
|
||||
<value>Off by default: with no live proxy this source refuses to start rather than fetching from your own address. Turn it on for a host you own, or one you do not mind being seen from here.</value>
|
||||
</data>
|
||||
<data name="Collect.SourceExtension" xml:space="preserve">
|
||||
<value>EXTENSION</value>
|
||||
<value>EXTENSIONS</value>
|
||||
</data>
|
||||
<data name="Collect.SourceExtensionHint" xml:space="preserve">
|
||||
<value>Several, comma-separated, are tried in order until one answers: .jpg,.png,.gif. Empty means .jpg.</value>
|
||||
</data>
|
||||
<data name="Collect.SourceAlphabet" xml:space="preserve">
|
||||
<value>ID CHARACTERS</value>
|
||||
|
||||
@@ -269,7 +269,7 @@
|
||||
<value>URL ДЛЯ ПРОВЕРКИ</value>
|
||||
</data>
|
||||
<data name="Settings.ProbeUrlHint" xml:space="preserve">
|
||||
<value>По умолчанию обычный HTTP: требование TLS отбраковало бы все прокси без CONNECT, а не только нерабочие.</value>
|
||||
<value>Оставьте https: собираемые адреса идут по https, значит и проверка должна открывать туннель так же, как сборщик. Проба по обычному HTTP пропускает прокси, которые потом отказывают на каждом реальном запросе.</value>
|
||||
</data>
|
||||
<data name="Settings.ProbeTimeout" xml:space="preserve">
|
||||
<value>ТАЙМАУТ ПРОВЕРКИ (СЕК)</value>
|
||||
@@ -491,7 +491,10 @@
|
||||
<value>По умолчанию выключено: без живой прокси источник не запустится, а не пойдёт с вашего адреса. Включайте для своего хоста или там, где вас не смущает быть видимым отсюда.</value>
|
||||
</data>
|
||||
<data name="Collect.SourceExtension" xml:space="preserve">
|
||||
<value>РАСШИРЕНИЕ</value>
|
||||
<value>РАСШИРЕНИЯ</value>
|
||||
</data>
|
||||
<data name="Collect.SourceExtensionHint" xml:space="preserve">
|
||||
<value>Несколько через запятую проверяются по порядку, пока одно не ответит: .jpg,.png,.gif. Пусто — значит .jpg.</value>
|
||||
</data>
|
||||
<data name="Collect.SourceAlphabet" xml:space="preserve">
|
||||
<value>СИМВОЛЫ ID</value>
|
||||
|
||||
@@ -26,6 +26,7 @@ public partial class GalleryViewModel : PageViewModel, IDisposable
|
||||
private const int PreviewWidth = 1400;
|
||||
|
||||
private readonly IMediaStore _store;
|
||||
private readonly IMediaSourceCatalog _catalog;
|
||||
private readonly IThumbnailCache _thumbnails;
|
||||
private readonly MediaStore? _paths;
|
||||
private readonly ILogger<GalleryViewModel> _logger;
|
||||
@@ -77,6 +78,20 @@ public partial class GalleryViewModel : PageViewModel, IDisposable
|
||||
[Reactive]
|
||||
public partial string? StatusMessage { get; set; }
|
||||
|
||||
// ----- Rule editor -----
|
||||
|
||||
/// <summary>Whether the "what to do when this turns up again" panel is showing.</summary>
|
||||
[Reactive]
|
||||
public partial bool IsRuleEditorOpen { get; set; }
|
||||
|
||||
/// <summary>Action chosen in the editor.</summary>
|
||||
[Reactive]
|
||||
public partial LocalizedOption<MediaRuleAction> RuleAction { get; set; }
|
||||
|
||||
/// <summary>Whether the open picture already has a rule, which enables removing it.</summary>
|
||||
[Reactive]
|
||||
public partial bool HasRule { get; private set; }
|
||||
|
||||
/// <summary>Creates the page.</summary>
|
||||
/// <param name="store">Where the content is.</param>
|
||||
/// <param name="thumbnails">Decodes and caches tile bitmaps.</param>
|
||||
@@ -84,12 +99,14 @@ public partial class GalleryViewModel : PageViewModel, IDisposable
|
||||
/// <param name="mainThread">Scheduler for UI-affine updates; tests pass an immediate one.</param>
|
||||
public GalleryViewModel(
|
||||
IMediaStore store,
|
||||
IMediaSourceCatalog catalog,
|
||||
IThumbnailCache thumbnails,
|
||||
ILogger<GalleryViewModel> logger,
|
||||
ISequencer? mainThread = null
|
||||
)
|
||||
{
|
||||
_store = store ?? throw new ArgumentNullException(nameof(store));
|
||||
_catalog = catalog ?? throw new ArgumentNullException(nameof(catalog));
|
||||
_thumbnails = thumbnails ?? throw new ArgumentNullException(nameof(thumbnails));
|
||||
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
|
||||
_mainThread = mainThread ?? RxSchedulers.MainThreadScheduler;
|
||||
@@ -99,6 +116,9 @@ public partial class GalleryViewModel : PageViewModel, IDisposable
|
||||
|
||||
SearchText = string.Empty;
|
||||
SelectedKinds = KindFilters[0];
|
||||
RuleActions = LocalizedOption<MediaRuleAction>.ForAll();
|
||||
RuleAction = RuleActions[0];
|
||||
RuleSources = [];
|
||||
|
||||
var idle = this.WhenAnyValue(x => x.IsLoading).Select(static loading => !loading);
|
||||
|
||||
@@ -118,6 +138,22 @@ public partial class GalleryViewModel : PageViewModel, IDisposable
|
||||
|
||||
CloseViewerCommand = ReactiveCommand.Create(() => Selected = null, outputScheduler: _mainThread);
|
||||
|
||||
var hasSelection = this.WhenAnyValue(x => x.Selected).Select(static item => item is not null);
|
||||
var editing = this.WhenAnyValue(x => x.IsRuleEditorOpen);
|
||||
|
||||
EditRuleCommand = ReactiveCommand.CreateFromTask(OpenRuleEditorAsync, hasSelection, _mainThread);
|
||||
SaveRuleCommand = ReactiveCommand.CreateFromTask(SaveRuleAsync, editing, _mainThread);
|
||||
RemoveRuleCommand = ReactiveCommand.CreateFromTask(
|
||||
RemoveRuleAsync,
|
||||
this.WhenAnyValue(x => x.HasRule),
|
||||
_mainThread
|
||||
);
|
||||
CancelRuleCommand = ReactiveCommand.Create(() => IsRuleEditorOpen = false, editing, _mainThread);
|
||||
|
||||
EditRuleCommand.ThrownExceptions.Subscribe(OnRuleFailed);
|
||||
SaveRuleCommand.ThrownExceptions.Subscribe(OnRuleFailed);
|
||||
RemoveRuleCommand.ThrownExceptions.Subscribe(OnRuleFailed);
|
||||
|
||||
// Changing a filter starts again from the first page: staying on page seven of a different
|
||||
// result set shows nothing and looks broken.
|
||||
this.WhenAnyValue(
|
||||
|
||||
@@ -173,10 +173,12 @@
|
||||
</StackPanel>
|
||||
<StackPanel Grid.Column="4" Spacing="4">
|
||||
<TextBlock Classes="caption" Text="{l:Loc Collect.SourceExtension}" />
|
||||
<TextBox Text="{Binding EditorExtension}" PlaceholderText=".jpg" />
|
||||
<TextBox Text="{Binding EditorExtension}" PlaceholderText=".jpg,.png" />
|
||||
</StackPanel>
|
||||
</Grid>
|
||||
|
||||
<TextBlock Classes="muted caption" Text="{l:Loc Collect.SourceExtensionHint}" TextWrapping="Wrap" />
|
||||
|
||||
<StackPanel Spacing="4">
|
||||
<TextBlock Classes="caption" Text="{l:Loc Collect.SourceAlphabet}" />
|
||||
<ComboBox
|
||||
|
||||
@@ -76,6 +76,38 @@ public class PatternMediaSourceTests
|
||||
candidate.Url.AbsoluteUri.ShouldBe($"https://imgtest.example/test1/{candidate.ExternalId}.jpg");
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Several_extensions_become_one_candidate_with_fallbacks_in_order()
|
||||
{
|
||||
// One id is one picture: three candidates for three suffixes would fetch the hit and then
|
||||
// hunt for its imaginary twins, and count one find as three attempts.
|
||||
var source = new PatternMediaSource(
|
||||
Config(min: 6, max: 6, alphabet: IdAlphabet.HexLower, extension: ".jpg, png;.gif")
|
||||
);
|
||||
|
||||
var candidate = (await Collect(source, 1)).ShouldHaveSingleItem();
|
||||
var id = candidate.ExternalId;
|
||||
|
||||
candidate.Url.AbsoluteUri.ShouldBe($"https://imgtest.example/test1/{id}.jpg");
|
||||
candidate
|
||||
.Alternatives.Select(url => url.AbsoluteUri)
|
||||
.ShouldBe([$"https://imgtest.example/test1/{id}.png", $"https://imgtest.example/test1/{id}.gif"]);
|
||||
|
||||
candidate.Addresses.First().ShouldBe(candidate.Url);
|
||||
candidate.Addresses.Count().ShouldBe(3);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task A_single_extension_leaves_the_candidate_without_fallbacks()
|
||||
{
|
||||
var source = new PatternMediaSource(Config(min: 6, max: 6, extension: ".png"));
|
||||
|
||||
var candidate = (await Collect(source, 1)).ShouldHaveSingleItem();
|
||||
|
||||
candidate.Alternatives.ShouldBeEmpty();
|
||||
candidate.Addresses.ShouldHaveSingleItem().ShouldBe(candidate.Url);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task The_attempt_budget_bounds_how_many_are_generated()
|
||||
{
|
||||
@@ -208,6 +240,76 @@ public class PatternSourceConfigTests
|
||||
config.Id.ShouldNotBeNullOrWhiteSpace();
|
||||
}
|
||||
|
||||
[Theory]
|
||||
[InlineData(".jpg,.png", ".jpg,.png")]
|
||||
[InlineData("jpg png gif", ".jpg,.png,.gif")]
|
||||
[InlineData(".jpg; .PNG ;jpg", ".jpg,.PNG")]
|
||||
[InlineData(" ", ".jpg")]
|
||||
public void Extensions_are_normalised_into_an_ordered_list(string typed, string expected)
|
||||
{
|
||||
// Order is an instruction, not a detail: it is the order the fetcher will try. Repeats are
|
||||
// dropped because a repeat means fetching the same address twice before giving up on the id.
|
||||
PatternSourceConfig.TryCreate(
|
||||
"N",
|
||||
"https://h/x/",
|
||||
6,
|
||||
8,
|
||||
IdAlphabet.Digits,
|
||||
null,
|
||||
typed,
|
||||
allowDirectConnection: false,
|
||||
out var config
|
||||
);
|
||||
|
||||
config.ShouldNotBeNull().Extension.ShouldBe(expected);
|
||||
config.Extensions.ShouldBe(expected.Split(','));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void A_doubled_trailing_slash_is_collapsed()
|
||||
{
|
||||
// "host//" is not "host/": ids resolve against it as //{id}, which most servers answer with
|
||||
// a 404 for every attempt in the run. Easy to paste, impossible to spot in the field.
|
||||
var error = PatternSourceConfig.TryCreate(
|
||||
"N",
|
||||
"https://imgtest.example/test1//",
|
||||
6,
|
||||
8,
|
||||
IdAlphabet.Digits,
|
||||
null,
|
||||
".jpg",
|
||||
allowDirectConnection: false,
|
||||
out var config
|
||||
);
|
||||
|
||||
error.ShouldBe(PatternConfigError.None);
|
||||
config.ShouldNotBeNull().BaseUrl.AbsoluteUri.ShouldBe("https://imgtest.example/test1/");
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Ids_hang_off_a_collapsed_base_url_without_a_doubled_slash()
|
||||
{
|
||||
PatternSourceConfig.TryCreate(
|
||||
"N",
|
||||
"https://imgtest.example/test1//",
|
||||
6,
|
||||
6,
|
||||
IdAlphabet.Digits,
|
||||
null,
|
||||
".jpg",
|
||||
allowDirectConnection: false,
|
||||
out var config
|
||||
);
|
||||
|
||||
var source = new PatternMediaSource(config!);
|
||||
|
||||
await foreach (var outcome in source.ParseAsync(new MediaQuery(Limit: 1), null, CancellationToken.None))
|
||||
{
|
||||
var candidate = outcome.Value.ShouldNotBeNull();
|
||||
candidate.Url.AbsoluteUri.ShouldBe($"https://imgtest.example/test1/{candidate.ExternalId}.jpg");
|
||||
}
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void A_blank_extension_becomes_the_default_one()
|
||||
{
|
||||
|
||||
@@ -44,6 +44,7 @@ internal sealed class DirectHttpClientFactory : IProxiedHttpClientFactory
|
||||
public Task<LeasedHttpClient> LeaseAsync(
|
||||
HttpClientTimeouts timeouts,
|
||||
bool requireProxy,
|
||||
IReadOnlySet<string>? excludedCountries = null,
|
||||
CancellationToken cancellationToken = default
|
||||
) =>
|
||||
requireProxy
|
||||
@@ -230,6 +231,88 @@ public sealed class MediaFetcherTests : IAsyncLifetime
|
||||
_throttle.CooldownRemaining(url).ShouldBeGreaterThan(TimeSpan.Zero);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Several_extensions_are_tried_in_order_until_one_answers()
|
||||
{
|
||||
var jpg = _server.Map("/id.jpg", new Reply { Status = 404 });
|
||||
var png = _server.MapBody("/id.png", Samples.Png(), "image/png");
|
||||
var gif = _server.MapBody("/id.gif", Samples.Gif(), "image/gif");
|
||||
|
||||
var result = await _fetcher.FetchAsync(
|
||||
new MediaCandidate(jpg) { SourceId = "test", Alternatives = [png, gif] },
|
||||
Options(),
|
||||
TestContext.Current.CancellationToken
|
||||
);
|
||||
|
||||
result.Outcome.ShouldBe(SeenOutcome.Stored);
|
||||
result.FinalUrl.ShouldBe(png);
|
||||
result.Blob!.Kind.ShouldBe(MediaKind.Png);
|
||||
|
||||
// The gif exists too, and must not have been fetched: an id holds one picture, and going on
|
||||
// after a hit would download its imaginary twin and count the id twice.
|
||||
_server.Requests.ShouldNotContain(request => request.Path == "/id.gif");
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task An_id_that_is_at_none_of_the_extensions_reports_the_last_miss()
|
||||
{
|
||||
var jpg = _server.Map("/none.jpg", new Reply { Status = 404 });
|
||||
var png = _server.Map("/none.png", new Reply { Status = 404 });
|
||||
|
||||
var result = await _fetcher.FetchAsync(
|
||||
new MediaCandidate(jpg) { SourceId = "test", Alternatives = [png] },
|
||||
Options(),
|
||||
TestContext.Current.CancellationToken
|
||||
);
|
||||
|
||||
result.Outcome.ShouldBe(SeenOutcome.Gone);
|
||||
result.FinalUrl.ShouldBe(png);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task A_placeholder_under_the_size_floor_counts_as_not_there_and_moves_on()
|
||||
{
|
||||
// What a service actually does with a missing id: 200 with a tiny "removed" image. That is
|
||||
// "not there" wearing a success, so the next extension still has to be tried.
|
||||
var jpg = _server.MapBody("/tiny.jpg", Samples.Png(), "image/png");
|
||||
var png = _server.Map("/tiny.png", new Reply { Status = 404 });
|
||||
|
||||
var result = await _fetcher.FetchAsync(
|
||||
new MediaCandidate(jpg) { SourceId = "test", Alternatives = [png] },
|
||||
Options() with
|
||||
{
|
||||
MinItemBytes = Samples.Png().Length + 1,
|
||||
},
|
||||
TestContext.Current.CancellationToken
|
||||
);
|
||||
|
||||
result.FinalUrl.ShouldBe(png);
|
||||
result.Outcome.ShouldBe(SeenOutcome.Gone);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task A_timeout_does_not_burn_the_remaining_extensions()
|
||||
{
|
||||
// "Not there" is a verdict about the address; a timeout is the absence of one. Moving on
|
||||
// would report an id as missing everywhere on the strength of one broken connection.
|
||||
var jpg = _server.Map("/slow.jpg", new Reply { Body = Samples.Png(), HeaderDelay = TimeSpan.FromSeconds(3) });
|
||||
var png = _server.MapBody("/slow.png", Samples.Png(), "image/png");
|
||||
|
||||
var options = Options() with
|
||||
{
|
||||
Timeouts = HttpClientTimeouts.Default with { Headers = TimeSpan.FromMilliseconds(300) },
|
||||
};
|
||||
|
||||
var result = await _fetcher.FetchAsync(
|
||||
new MediaCandidate(jpg) { SourceId = "test", Alternatives = [png] },
|
||||
options,
|
||||
TestContext.Current.CancellationToken
|
||||
);
|
||||
|
||||
result.Outcome.ShouldBe(SeenOutcome.Timeout);
|
||||
_server.Requests.ShouldNotContain(request => request.Path == "/slow.png");
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Headers_that_never_arrive_time_out()
|
||||
{
|
||||
@@ -434,7 +517,10 @@ public sealed class MediaFetcherTests : IAsyncLifetime
|
||||
|
||||
var options = Options() with
|
||||
{
|
||||
Tombstones = new HashSet<string>(StringComparer.Ordinal) { probe.Blob!.Sha256 },
|
||||
Rules = new Dictionary<string, MediaRuleAction>(StringComparer.Ordinal)
|
||||
{
|
||||
[probe.Blob!.Sha256] = MediaRuleAction.Skip,
|
||||
},
|
||||
};
|
||||
|
||||
var result = await FetchAsync(url, options);
|
||||
@@ -443,6 +529,31 @@ public sealed class MediaFetcherTests : IAsyncLifetime
|
||||
StagedFileCount().ShouldBe(0);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task A_geo_block_rule_with_nowhere_else_to_go_stores_nothing_and_stays_retryable()
|
||||
{
|
||||
// Direct connections here, so there is no country to move away from. The picture must not be
|
||||
// stored — it is a banner, not the thing — and the address must stay worth trying again with
|
||||
// a different pool tomorrow.
|
||||
var url = _server.MapBody("/blocked.png", Samples.Png());
|
||||
var probe = await FetchAsync(url);
|
||||
File.Delete(probe.TempPath!);
|
||||
|
||||
var options = Options() with
|
||||
{
|
||||
Rules = new Dictionary<string, MediaRuleAction>(StringComparer.Ordinal)
|
||||
{
|
||||
[probe.Blob!.Sha256] = MediaRuleAction.RetryElsewhere,
|
||||
},
|
||||
};
|
||||
|
||||
var result = await FetchAsync(url, options);
|
||||
|
||||
result.ErrorCode.ShouldBe("GeoBlocked");
|
||||
SeenOutcomes.IsTerminal(result.Outcome).ShouldBeFalse();
|
||||
StagedFileCount().ShouldBe(0);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Animation_is_detected_end_to_end()
|
||||
{
|
||||
|
||||
@@ -201,6 +201,34 @@ public sealed class JsonSettingsServiceTests : IDisposable
|
||||
options.ProbeUrl.ShouldBe(new ProxyOptions().ProbeUrl);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void The_probe_opens_a_tunnel_the_way_the_collector_does()
|
||||
{
|
||||
// A plain-HTTP probe is a forwarded GET; an https one is a CONNECT. Free proxies routinely
|
||||
// do the first and refuse the second, and every address this app collects is https — so an
|
||||
// http probe marked them live and they then failed every real request.
|
||||
new ProxyOptions().ProbeUrl.Scheme.ShouldBe("https");
|
||||
new AppSettings().ToProxyOptions().ProbeUrl.Scheme.ShouldBe("https");
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void A_settings_file_still_holding_the_old_http_probe_is_upgraded()
|
||||
{
|
||||
// Nobody would think to edit this by hand, and leaving it would keep confirming proxies that
|
||||
// cannot do the one thing the collector needs.
|
||||
var options = new AppSettings(ProxyProbeUrl: "http://www.gstatic.com/generate_204").ToProxyOptions();
|
||||
|
||||
options.ProbeUrl.ShouldBe(new ProxyOptions().ProbeUrl);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void A_probe_url_the_user_chose_is_left_alone()
|
||||
{
|
||||
var options = new AppSettings(ProxyProbeUrl: "http://example.test/ping").ToProxyOptions();
|
||||
|
||||
options.ProbeUrl.AbsoluteUri.ShouldBe("http://example.test/ping");
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void A_corrupt_file_falls_back_to_the_defaults_instead_of_failing_to_start()
|
||||
{
|
||||
|
||||
@@ -57,6 +57,23 @@ internal sealed class FakeMediaStore : IMediaStore
|
||||
public Task<int> TombstoneAsync(string sha256, string? reason, CancellationToken cancellationToken = default) =>
|
||||
Task.FromResult(0);
|
||||
|
||||
/// <summary>Rules the page under test has saved.</summary>
|
||||
public List<MediaRule> Rules { get; } = [];
|
||||
|
||||
public Task<IReadOnlyList<MediaRule>> LoadRulesAsync(CancellationToken cancellationToken = default) =>
|
||||
Task.FromResult<IReadOnlyList<MediaRule>>([.. Rules]);
|
||||
|
||||
public Task<int> SaveRuleAsync(MediaRule rule, CancellationToken cancellationToken = default)
|
||||
{
|
||||
Rules.RemoveAll(existing => existing.Sha256 == rule.Sha256);
|
||||
Rules.Add(rule);
|
||||
|
||||
return Task.FromResult(rule.Action == MediaRuleAction.Skip ? 1 : 0);
|
||||
}
|
||||
|
||||
public Task<bool> RemoveRuleAsync(string sha256, CancellationToken cancellationToken = default) =>
|
||||
Task.FromResult(Rules.RemoveAll(existing => existing.Sha256 == sha256) > 0);
|
||||
|
||||
public Task<CollectedItem> StoreAsync(MediaStoreRequest request, CancellationToken cancellationToken = default) =>
|
||||
Task.FromResult(new CollectedItem(request.Candidate, request.Blob, CollectStatus.Stored));
|
||||
|
||||
|
||||
@@ -57,6 +57,23 @@ internal sealed class FakeMediaStore : IMediaStore
|
||||
public Task<int> TombstoneAsync(string sha256, string? reason, CancellationToken cancellationToken = default) =>
|
||||
Task.FromResult(0);
|
||||
|
||||
/// <summary>Rules the page under test has saved.</summary>
|
||||
public List<MediaRule> Rules { get; } = [];
|
||||
|
||||
public Task<IReadOnlyList<MediaRule>> LoadRulesAsync(CancellationToken cancellationToken = default) =>
|
||||
Task.FromResult<IReadOnlyList<MediaRule>>([.. Rules]);
|
||||
|
||||
public Task<int> SaveRuleAsync(MediaRule rule, CancellationToken cancellationToken = default)
|
||||
{
|
||||
Rules.RemoveAll(existing => existing.Sha256 == rule.Sha256);
|
||||
Rules.Add(rule);
|
||||
|
||||
return Task.FromResult(rule.Action == MediaRuleAction.Skip ? 1 : 0);
|
||||
}
|
||||
|
||||
public Task<bool> RemoveRuleAsync(string sha256, CancellationToken cancellationToken = default) =>
|
||||
Task.FromResult(Rules.RemoveAll(existing => existing.Sha256 == sha256) > 0);
|
||||
|
||||
public Task<CollectedItem> StoreAsync(MediaStoreRequest request, CancellationToken cancellationToken = default) =>
|
||||
Task.FromResult(new CollectedItem(request.Candidate, request.Blob, CollectStatus.Stored));
|
||||
|
||||
|
||||
Reference in New Issue
Block a user