Add collector settings and per-source purge

Every limit the fetcher was using was a constant. They are settings now, and
CollectOptions became the single place policy lives: AppSettings.ToCollectOptions
clamps them, and the HTTP layer's FetchOptions is projected from that. One
clamping site rather than two sets of ceilings drifting apart.

Clamping rather than validating, for the reason the proxy options already do it:
a hand-edited file must not stop the app from starting. A MaxItemBytes edited to
zero would otherwise refuse everything, and a zeroed concurrency would deadlock
the run outright - so both are pulled into range instead. An empty format filter
is read as "everything", because switching every format off is far more likely
to be a slip than an instruction to collect nothing.

The media root has an ordering problem - it is a setting that decides the paths
the container is built from - so the file is read once before the container
exists rather than making every path lazy for one value.

Purge is scoped to a source and lives on the Collect page, where the source is
already chosen. Content another source also holds survives, which is what the
index's reference count was for.

The showcase hint says out loud what a hard link means: editing the browsable
copy edits the original, and deleting it frees nothing until the last name goes.
That is surprising enough to belong in the UI rather than only in the code.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
Leonid Pershin
2026-08-13 21:52:43 +03:00
co-authored by Claude Opus 5
parent 70fb3a1df3
commit fe62bcf53f
21 changed files with 834 additions and 26 deletions
+116 -1
View File
@@ -2,7 +2,86 @@ using AvParser.Core.Parsing;
namespace AvParser.Core.Collecting;
/// <summary>How to run one collection.</summary>
/// <summary>Which formats to keep.</summary>
/// <remarks>
/// Flags rather than a collection so that <c>AppSettings</c> keeps value equality — the settings
/// service short-circuits a no-op write by comparing records, and a list-valued member would make
/// every save look like a change. Same reasoning as <c>ProxyProtocolFilter</c>.
/// </remarks>
[Flags]
public enum MediaKindFilter
{
/// <summary>Nothing. Treated as <see cref="All"/> rather than collecting nothing at all.</summary>
None = 0,
/// <summary>JPEG.</summary>
Jpeg = 1,
/// <summary>PNG, including animated PNG.</summary>
Png = 2,
/// <summary>GIF.</summary>
Gif = 4,
/// <summary>WebP.</summary>
WebP = 8,
/// <summary>AVIF.</summary>
Avif = 16,
/// <summary>MP4, which is what most sites serve when they say "GIF".</summary>
Mp4 = 32,
/// <summary>WebM.</summary>
WebM = 64,
/// <summary>Every still or animated picture.</summary>
Images = Jpeg | Png | Gif | WebP | Avif,
/// <summary>Every video container.</summary>
Videos = Mp4 | WebM,
/// <summary>Everything recognised.</summary>
All = Images | Videos,
}
/// <summary>Helpers over <see cref="MediaKindFilter"/>.</summary>
public static class MediaKindFilters
{
/// <summary>The flag standing for one kind.</summary>
public static MediaKindFilter ToFlag(MediaKind kind) =>
kind switch
{
MediaKind.Jpeg => MediaKindFilter.Jpeg,
MediaKind.Png => MediaKindFilter.Png,
MediaKind.Gif => MediaKindFilter.Gif,
MediaKind.WebP => MediaKindFilter.WebP,
MediaKind.Avif => MediaKindFilter.Avif,
MediaKind.Mp4 => MediaKindFilter.Mp4,
MediaKind.WebM => MediaKindFilter.WebM,
_ => MediaKindFilter.None,
};
/// <summary>Expands a filter into the set of kinds it admits.</summary>
public static IReadOnlySet<MediaKind> ToSet(MediaKindFilter filter)
{
var effective = filter == MediaKindFilter.None ? MediaKindFilter.All : filter;
return new HashSet<MediaKind>(
Enum.GetValues<MediaKind>().Where(kind => kind != MediaKind.Unknown && effective.HasFlag(ToFlag(kind)))
);
}
}
/// <summary>
/// How to run one collection: what to accept, how hard to push, and when to give up.
/// </summary>
/// <remarks>
/// Every ceiling here exists because the other side chooses the bytes. A missing cap is not a
/// generous default, it is a remote party deciding how much of the user's disk to fill. Built by
/// <c>AppSettings.ToCollectOptions()</c>, which clamps rather than throws so that a hand-edited
/// settings file cannot stop the app from starting.
/// </remarks>
public sealed record CollectOptions
{
/// <summary>How many downloads may be in flight at once, across all hosts.</summary>
@@ -12,8 +91,44 @@ public sealed record CollectOptions
/// </remarks>
public int MaxConcurrentDownloads { get; init; } = 4;
/// <summary>How many requests one origin may be serving at once.</summary>
public int MaxConcurrentPerHost { get; init; } = 2;
/// <summary>Shortest gap between two requests to the same origin.</summary>
public TimeSpan HostDelay { get; init; } = TimeSpan.FromMilliseconds(250);
/// <summary>Ignore the journal and fetch every address again.</summary>
public bool ForceRefetch { get; init; }
/// <summary>Largest item to accept.</summary>
public long MaxItemBytes { get; init; } = 32L * 1024 * 1024;
/// <summary>Smallest item to accept; below this it is a tracking pixel, not media.</summary>
public long MinItemBytes { get; init; } = 1024;
/// <summary>How many redirects to follow before giving up.</summary>
public int MaxRedirects { get; init; } = 5;
/// <summary>Time allowed to establish a connection.</summary>
public TimeSpan ConnectTimeout { get; init; } = TimeSpan.FromSeconds(15);
/// <summary>Time allowed for the response headers to arrive.</summary>
public TimeSpan HeaderTimeout { get; init; } = TimeSpan.FromSeconds(30);
/// <summary>Longest gap between two body reads before the transfer is called stalled.</summary>
public TimeSpan IdleTimeout { get; init; } = TimeSpan.FromSeconds(20);
/// <summary>Formats to keep.</summary>
public MediaKindFilter AllowedKinds { get; init; } = MediaKindFilter.All;
/// <summary>Identifies the collector to origins that care.</summary>
public string UserAgent { get; init; } = "AvParser/0.1";
/// <summary>Whether a missing proxy is a hard failure rather than a direct connection.</summary>
public bool RequireProxy { get; init; }
/// <summary>How browsable copies point at their blobs.</summary>
public ShowcaseMode ShowcaseMode { get; init; } = ShowcaseMode.HardLink;
}
/// <summary>Runs a source end to end: discover, download, store.</summary>
@@ -83,6 +83,13 @@ public interface IMediaStore
/// <summary>Creates or upgrades the schema. Safe to call repeatedly.</summary>
Task InitialiseAsync(CancellationToken cancellationToken = default);
/// <summary>Applies the user's showcase preference.</summary>
/// <remarks>
/// Applied rather than injected, for the same reason <c>IProxyPool.Configure</c> exists: it
/// changes while the app runs, and a snapshot taken at container build would freeze it.
/// </remarks>
void Configure(ShowcaseMode mode);
/// <summary>Opens a run and returns its id.</summary>
Task<string> BeginRunAsync(string sourceId, CancellationToken cancellationToken = default);
+53 -1
View File
@@ -1,3 +1,4 @@
using AvParser.Core.Collecting;
using AvParser.Core.Proxies;
namespace AvParser.Core.Settings;
@@ -49,6 +50,18 @@ public enum AppTheme
/// <param name="AllowDirectConnection">Whether network parsers may run without a proxy.</param>
/// <param name="LastSourceId">Id of the media source selected last time; resolved leniently on load.</param>
/// <param name="MaxConcurrentDownloads">How many downloads may be in flight at once.</param>
/// <param name="MaxConcurrentPerHost">How many requests one origin may be serving at once.</param>
/// <param name="HostDelayMs">Shortest gap between two requests to the same origin, in milliseconds.</param>
/// <param name="MaxItemBytes">Largest item to accept, in bytes.</param>
/// <param name="MinItemBytes">Smallest item to accept, in bytes.</param>
/// <param name="MaxRedirects">How many redirects to follow before giving up.</param>
/// <param name="ConnectTimeoutSeconds">Time allowed to establish a connection.</param>
/// <param name="HeaderTimeoutSeconds">Time allowed for response headers to arrive.</param>
/// <param name="IdleTimeoutSeconds">Longest gap between body reads before a transfer is called stalled.</param>
/// <param name="AllowedMediaKinds">Formats to keep.</param>
/// <param name="ShowcaseMode">How browsable copies point at their blobs.</param>
/// <param name="CollectUserAgent">Identifies the collector to origins that care.</param>
/// <param name="MediaRootOverride">Where collected media goes; null keeps it beside the settings.</param>
public sealed record AppSettings(
AppTheme Theme = AppTheme.System,
AppLanguage Language = AppLanguage.System,
@@ -67,7 +80,19 @@ public sealed record AppSettings(
int ProxyMinimumLive = 10,
bool AllowDirectConnection = false,
string? LastSourceId = null,
int MaxConcurrentDownloads = 4
int MaxConcurrentDownloads = 4,
int MaxConcurrentPerHost = 2,
int HostDelayMs = 250,
long MaxItemBytes = 33_554_432,
long MinItemBytes = 1024,
int MaxRedirects = 5,
int ConnectTimeoutSeconds = 15,
int HeaderTimeoutSeconds = 30,
int IdleTimeoutSeconds = 20,
MediaKindFilter AllowedMediaKinds = MediaKindFilter.All,
ShowcaseMode ShowcaseMode = ShowcaseMode.HardLink,
string CollectUserAgent = "AvParser/0.1",
string? MediaRootOverride = null
)
{
/// <summary>Projects the proxy-related settings onto <see cref="ProxyOptions"/>.</summary>
@@ -95,4 +120,31 @@ public sealed record AppSettings(
AllowDirectConnection = AllowDirectConnection,
}.Validated();
}
/// <summary>Projects the collector settings onto <see cref="CollectOptions"/>.</summary>
/// <remarks>
/// The one place primitives become policy, and it clamps rather than throws for the same reason
/// <see cref="ToProxyOptions"/> does: a hand-edited settings file must not be able to stop the
/// app from starting. A ceiling edited to zero would otherwise mean "accept nothing" or, worse,
/// "accept anything".
/// </remarks>
public CollectOptions ToCollectOptions() =>
new()
{
MaxConcurrentDownloads = Math.Clamp(MaxConcurrentDownloads, 1, 32),
MaxConcurrentPerHost = Math.Clamp(MaxConcurrentPerHost, 1, 16),
HostDelay = TimeSpan.FromMilliseconds(Math.Clamp(HostDelayMs, 0, 60_000)),
MaxItemBytes = Math.Clamp(MaxItemBytes, 1024, 2L * 1024 * 1024 * 1024),
MinItemBytes = Math.Clamp(MinItemBytes, 0, 1024 * 1024),
MaxRedirects = Math.Clamp(MaxRedirects, 0, 20),
ConnectTimeout = TimeSpan.FromSeconds(Math.Clamp(ConnectTimeoutSeconds, 1, 120)),
HeaderTimeout = TimeSpan.FromSeconds(Math.Clamp(HeaderTimeoutSeconds, 1, 300)),
IdleTimeout = TimeSpan.FromSeconds(Math.Clamp(IdleTimeoutSeconds, 1, 300)),
// An empty filter means the user has switched everything off, which is far more likely
// to be an accident than an intention to collect nothing.
AllowedKinds = AllowedMediaKinds == MediaKindFilter.None ? MediaKindFilter.All : AllowedMediaKinds,
UserAgent = string.IsNullOrWhiteSpace(CollectUserAgent) ? "AvParser/0.1" : CollectUserAgent,
RequireProxy = !AllowDirectConnection,
ShowcaseMode = ShowcaseMode,
};
}
@@ -2,7 +2,7 @@ using System.Runtime.CompilerServices;
using System.Threading.Channels;
using AvParser.Core.Collecting;
using AvParser.Core.Parsing;
using AvParser.Core.Settings;
using AvParser.Infrastructure.Proxies;
using Microsoft.Extensions.Logging;
namespace AvParser.Infrastructure.Collecting;
@@ -26,7 +26,7 @@ namespace AvParser.Infrastructure.Collecting;
public sealed class CollectRunner(
IMediaFetcher fetcher,
IMediaStore store,
ISettingsService settings,
HostThrottle throttle,
ILogger<CollectRunner> logger
) : ICollectRunner
{
@@ -38,7 +38,7 @@ public sealed class CollectRunner(
private readonly IMediaFetcher _fetcher = fetcher ?? throw new ArgumentNullException(nameof(fetcher));
private readonly IMediaStore _store = store ?? throw new ArgumentNullException(nameof(store));
private readonly ISettingsService _settings = settings ?? throw new ArgumentNullException(nameof(settings));
private readonly HostThrottle _throttle = throttle ?? throw new ArgumentNullException(nameof(throttle));
private readonly ILogger<CollectRunner> _logger = logger ?? throw new ArgumentNullException(nameof(logger));
/// <inheritdoc />
@@ -55,9 +55,13 @@ public sealed class CollectRunner(
ArgumentNullException.ThrowIfNull(options);
var workers = Math.Clamp(options.MaxConcurrentDownloads, 1, 32);
_throttle.Configure(options.MaxConcurrentPerHost, options.HostDelay);
_store.Configure(options.ShowcaseMode);
var runId = await _store.BeginRunAsync(source.Id, cancellationToken).ConfigureAwait(false);
var tombstones = await _store.LoadTombstonesAsync(cancellationToken).ConfigureAwait(false);
var fetchOptions = BuildFetchOptions(tombstones);
var fetchOptions = BuildFetchOptions(options, tombstones);
var work = Channel.CreateBounded<MediaCandidate>(
new BoundedChannelOptions(workers * 4) { FullMode = BoundedChannelFullMode.Wait }
@@ -338,17 +342,20 @@ public sealed class CollectRunner(
return ParseOutcome<CollectedItem>.Success(stored with { Elapsed = result.Elapsed });
}
private FetchOptions BuildFetchOptions(IReadOnlySet<string> tombstones)
/// <summary>Projects the run's policy onto the HTTP layer's parameters.</summary>
private static FetchOptions BuildFetchOptions(CollectOptions options, IReadOnlySet<string> tombstones) =>
new()
{
var current = _settings.Current;
return new FetchOptions
{
// Mirrors the parser gate: proxy-only unless the user has said direct is acceptable.
RequireProxy = !current.AllowDirectConnection,
MaxItemBytes = options.MaxItemBytes,
MinItemBytes = options.MinItemBytes,
MaxRedirects = options.MaxRedirects,
Timeouts = new HttpClientTimeouts(options.ConnectTimeout, options.HeaderTimeout, options.IdleTimeout),
// Mirrors the page's gate: proxy-only unless the user has said direct is acceptable.
RequireProxy = options.RequireProxy,
UserAgent = options.UserAgent,
AllowedKinds = MediaKindFilters.ToSet(options.AllowedKinds),
Tombstones = tombstones,
};
}
/// <summary>Stands in for content on a skipped item, which by definition has none.</summary>
private static MediaBlob Placeholder { get; } = MediaBlob.Create(new string('0', 64), MediaKind.Unknown, 0);
@@ -26,11 +26,25 @@ public sealed class HostThrottle(
) : IDisposable
{
private readonly ConcurrentDictionary<string, HostState> _hosts = new(StringComparer.OrdinalIgnoreCase);
private readonly int _maxConcurrent = Math.Clamp(maxConcurrentPerHost, 1, 64);
private readonly TimeSpan _minimumInterval = minimumInterval < TimeSpan.Zero ? TimeSpan.Zero : minimumInterval;
private readonly ILogger _logger = logger ?? throw new ArgumentNullException(nameof(logger));
private readonly TimeProvider _time = timeProvider ?? TimeProvider.System;
private int _maxConcurrent = Math.Clamp(maxConcurrentPerHost, 1, 64);
private TimeSpan _minimumInterval = minimumInterval < TimeSpan.Zero ? TimeSpan.Zero : minimumInterval;
/// <summary>Applies new limits.</summary>
/// <remarks>
/// The pacing interval takes effect immediately; the concurrency cap applies to origins not yet
/// seen this session, because changing a semaphore's capacity underneath requests already
/// holding it cannot be done safely. Shared across runs on purpose — politeness belongs to the
/// origin, not to whichever collection happens to be running.
/// </remarks>
public void Configure(int maxConcurrentPerHost, TimeSpan minimumInterval)
{
_maxConcurrent = Math.Clamp(maxConcurrentPerHost, 1, 64);
_minimumInterval = minimumInterval < TimeSpan.Zero ? TimeSpan.Zero : minimumInterval;
}
/// <summary>How long this host is still refusing requests, or zero when it is not.</summary>
public TimeSpan CooldownRemaining(Uri url)
{
@@ -25,7 +25,7 @@ public static class InfrastructureServiceCollectionExtensions
{
ArgumentNullException.ThrowIfNull(services);
var resolved = paths ?? new AppPaths();
var resolved = paths ?? WithConfiguredMediaRoot(new AppPaths());
resolved.EnsureCreated();
services.AddSingleton<IAppPaths>(resolved);
@@ -43,6 +43,22 @@ public static class InfrastructureServiceCollectionExtensions
return services;
}
/// <summary>
/// Re-points the media root at the directory the user chose, if they chose one.
/// </summary>
/// <remarks>
/// A collection outgrows a profile directory quickly, so pointing it at another drive is the
/// first thing anyone does. That creates an ordering problem — the setting lives in a file
/// whose location <see cref="AppPaths"/> itself defines — so the file is read once here,
/// before the container exists, rather than making every path lazy for the sake of one value.
/// </remarks>
private static AppPaths WithConfiguredMediaRoot(AppPaths defaults)
{
var stored = JsonSettingsService.ReadOrDefault(defaults.SettingsFile).MediaRootOverride;
return string.IsNullOrWhiteSpace(stored) ? defaults : new AppPaths(defaults.DataDirectory, stored);
}
/// <summary>Registers the media store, the download pipeline and the network sources.</summary>
/// <remarks>
/// Everything here is a singleton because everything here owns something shared: a database
@@ -31,7 +31,7 @@ public sealed class MediaStore(
/// </remarks>
public ShowcaseMode ShowcaseMode { get; private set; } = ShowcaseMode.HardLink;
/// <summary>Applies the user's showcase preference.</summary>
/// <inheritdoc />
public void Configure(ShowcaseMode mode) => ShowcaseMode = mode;
/// <inheritdoc />
@@ -85,6 +85,33 @@ public sealed class JsonSettingsService : ISettingsService, IDisposable
_writeLock.Dispose();
}
/// <summary>
/// Reads a settings file without building a service around it.
/// </summary>
/// <remarks>
/// Exists for the one setting that has to be known before the container does — the media root,
/// which decides paths that the container itself is configured with. Silent on every failure,
/// like <see cref="Load"/>: a bad file must never stop the app from starting.
/// </remarks>
public static AppSettings ReadOrDefault(string settingsFile)
{
try
{
if (!File.Exists(settingsFile))
{
return new AppSettings();
}
var json = File.ReadAllText(settingsFile);
return JsonSerializer.Deserialize(json, AppSettingsJsonContext.Default.AppSettings) ?? new AppSettings();
}
catch (Exception ex) when (ex is IOException or UnauthorizedAccessException or JsonException)
{
return new AppSettings();
}
}
private AppSettings Load()
{
try
@@ -52,6 +52,7 @@ public static class UiServiceCollectionExtensions
sp.GetRequiredService<ISettingsService>(),
sp.GetRequiredService<IProxyPool>(),
sp.GetRequiredService<ICollectRunner>(),
sp.GetRequiredService<IMediaStore>(),
sp,
sp.GetRequiredService<ILogger<CollectViewModel>>()
));
+63
View File
@@ -676,4 +676,67 @@
<data name="Parse.Error.Failed" xml:space="preserve">
<value>Could not be collected.</value>
</data>
<data name="Enum.ShowcaseMode.None" xml:space="preserve">
<value>No browsable copies</value>
</data>
<data name="Enum.ShowcaseMode.HardLink" xml:space="preserve">
<value>Hard link (no extra space)</value>
</data>
<data name="Enum.ShowcaseMode.SymbolicLink" xml:space="preserve">
<value>Symbolic link</value>
</data>
<data name="Enum.ShowcaseMode.Copy" xml:space="preserve">
<value>Copy (doubles disk use)</value>
</data>
<data name="Settings.Collecting" xml:space="preserve">
<value>Collecting</value>
</data>
<data name="Settings.MaxConcurrentDownloads" xml:space="preserve">
<value>DOWNLOADS AT ONCE</value>
</data>
<data name="Settings.MaxConcurrentPerHost" xml:space="preserve">
<value>PER SITE</value>
</data>
<data name="Settings.HostDelay" xml:space="preserve">
<value>GAP BETWEEN REQUESTS (MS)</value>
</data>
<data name="Settings.HostDelayHint" xml:space="preserve">
<value>How politely one site is treated. The per-site limits are usually what actually binds, not the total above.</value>
</data>
<data name="Settings.MaxItemSize" xml:space="preserve">
<value>LARGEST ITEM (MB)</value>
</data>
<data name="Settings.MaxItemSizeHint" xml:space="preserve">
<value>Anything bigger is refused, where possible before its body is downloaded at all.</value>
</data>
<data name="Settings.ShowcaseMode" xml:space="preserve">
<value>BROWSABLE COPIES</value>
</data>
<data name="Settings.ShowcaseModeHint" xml:space="preserve">
<value>Collected files are stored by content hash. The showcase gives them dated, named paths. A hard link is a second name for the same file: editing it edits the original, and deleting it frees nothing until the last name is gone.</value>
</data>
<data name="Settings.MediaDirectory" xml:space="preserve">
<value>MEDIA DIRECTORY</value>
</data>
<data name="Collect.Storage" xml:space="preserve">
<value>{0} in the store, {1}.</value>
</data>
<data name="Collect.Purge" xml:space="preserve">
<value>Remove this source&apos;s items</value>
</data>
<data name="Collect.PurgeHint" xml:space="preserve">
<value>Removes what this source collected. Files another source also holds are kept.</value>
</data>
<data name="Collect.PurgeDone" xml:space="preserve">
<value>Removed {0}; {1} freed.</value>
</data>
<data name="Collect.Count.Files.One" xml:space="preserve">
<value>{0} file</value>
</data>
<data name="Collect.Count.Files.Few" xml:space="preserve">
<value>{0} files</value>
</data>
<data name="Collect.Count.Files.Many" xml:space="preserve">
<value>{0} files</value>
</data>
</root>
@@ -676,4 +676,67 @@
<data name="Parse.Error.Failed" xml:space="preserve">
<value>Не удалось собрать.</value>
</data>
<data name="Enum.ShowcaseMode.None" xml:space="preserve">
<value>Без витрины</value>
</data>
<data name="Enum.ShowcaseMode.HardLink" xml:space="preserve">
<value>Жёсткая ссылка (без лишнего места)</value>
</data>
<data name="Enum.ShowcaseMode.SymbolicLink" xml:space="preserve">
<value>Символическая ссылка</value>
</data>
<data name="Enum.ShowcaseMode.Copy" xml:space="preserve">
<value>Копия (удваивает расход диска)</value>
</data>
<data name="Settings.Collecting" xml:space="preserve">
<value>Сбор</value>
</data>
<data name="Settings.MaxConcurrentDownloads" xml:space="preserve">
<value>ЗАГРУЗОК ОДНОВРЕМЕННО</value>
</data>
<data name="Settings.MaxConcurrentPerHost" xml:space="preserve">
<value>НА ОДИН САЙТ</value>
</data>
<data name="Settings.HostDelay" xml:space="preserve">
<value>ПАУЗА МЕЖДУ ЗАПРОСАМИ (МС)</value>
</data>
<data name="Settings.HostDelayHint" xml:space="preserve">
<value>Насколько вежливо приложение обходится с одним сайтом. Обычно ограничивают именно эти два, а не общее число выше.</value>
</data>
<data name="Settings.MaxItemSize" xml:space="preserve">
<value>МАКСИМАЛЬНЫЙ РАЗМЕР (МБ)</value>
</data>
<data name="Settings.MaxItemSizeHint" xml:space="preserve">
<value>Всё, что больше, отклоняется — по возможности ещё до скачивания тела.</value>
</data>
<data name="Settings.ShowcaseMode" xml:space="preserve">
<value>ЧИТАЕМАЯ ВИТРИНА</value>
</data>
<data name="Settings.ShowcaseModeHint" xml:space="preserve">
<value>Файлы хранятся по хешу содержимого. Витрина даёт им читаемые пути по датам. Жёсткая ссылка — это второе имя того же файла: правка меняет оригинал, а удаление ничего не освобождает, пока не исчезнет последнее имя.</value>
</data>
<data name="Settings.MediaDirectory" xml:space="preserve">
<value>КАТАЛОГ МЕДИА</value>
</data>
<data name="Collect.Storage" xml:space="preserve">
<value>В хранилище {0}, {1}.</value>
</data>
<data name="Collect.Purge" xml:space="preserve">
<value>Удалить собранное этим источником</value>
</data>
<data name="Collect.PurgeHint" xml:space="preserve">
<value>Удаляет собранное этим источником. Файлы, на которые ссылается другой источник, остаются.</value>
</data>
<data name="Collect.PurgeDone" xml:space="preserve">
<value>Удалено {0}; освобождено {1}.</value>
</data>
<data name="Collect.Count.Files.One" xml:space="preserve">
<value>{0} файл</value>
</data>
<data name="Collect.Count.Files.Few" xml:space="preserve">
<value>{0} файла</value>
</data>
<data name="Collect.Count.Files.Many" xml:space="preserve">
<value>{0} файлов</value>
</data>
</root>
+71 -5
View File
@@ -37,6 +37,7 @@ public partial class CollectViewModel : PageViewModel, IDisposable
private readonly ISettingsService _settings;
private readonly IProxyPool _proxyPool;
private readonly ICollectRunner _runner;
private readonly IMediaStore _store;
private readonly IServiceProvider _services;
private readonly ILogger<CollectViewModel> _logger;
private readonly ISequencer _mainThread;
@@ -74,6 +75,10 @@ public partial class CollectViewModel : PageViewModel, IDisposable
[Reactive]
public partial bool ForceRefetch { get; set; }
/// <summary>What the store holds overall; <see langword="null"/> until it has been read.</summary>
[Reactive]
public partial string? StorageSummary { get; set; }
/// <summary>
/// Whether the selected source needs the network but has no working proxy to use.
/// </summary>
@@ -89,6 +94,7 @@ public partial class CollectViewModel : PageViewModel, IDisposable
/// <param name="settings">Used to remember the selected source.</param>
/// <param name="proxyPool">Consulted for the live count that gates network sources.</param>
/// <param name="runner">Runs the collection.</param>
/// <param name="store">Consulted for totals, and asked to purge.</param>
/// <param name="services">Resolves the navigation service lazily, to keep pages acyclic.</param>
/// <param name="logger">Diagnostics.</param>
/// <param name="mainThread">
@@ -100,6 +106,7 @@ public partial class CollectViewModel : PageViewModel, IDisposable
ISettingsService settings,
IProxyPool proxyPool,
ICollectRunner runner,
IMediaStore store,
IServiceProvider services,
ILogger<CollectViewModel> logger,
ISequencer? mainThread = null
@@ -109,6 +116,7 @@ public partial class CollectViewModel : PageViewModel, IDisposable
_settings = settings ?? throw new ArgumentNullException(nameof(settings));
_proxyPool = proxyPool ?? throw new ArgumentNullException(nameof(proxyPool));
_runner = runner ?? throw new ArgumentNullException(nameof(runner));
_store = store ?? throw new ArgumentNullException(nameof(store));
_services = services ?? throw new ArgumentNullException(nameof(services));
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
_mainThread = mainThread ?? RxSchedulers.MainThreadScheduler;
@@ -170,9 +178,19 @@ public partial class CollectViewModel : PageViewModel, IDisposable
RefreshProxyGate();
});
PurgeCommand = ReactiveCommand.CreateFromTask(
PurgeAsync,
CollectCommand.IsExecuting.Select(static running => !running),
_mainThread
);
_settings.Changes.Subscribe(_ => RefreshProxyGate());
CollectCommand.ThrownExceptions.Subscribe(OnCommandFailed);
PurgeCommand.ThrownExceptions.Subscribe(OnCommandFailed);
// Not awaited — a constructor cannot be — and it never throws.
_ = RefreshStorageAsync();
}
/// <inheritdoc />
@@ -205,6 +223,9 @@ public partial class CollectViewModel : PageViewModel, IDisposable
/// <summary>Takes the user to the page where the proxy problem can be fixed.</summary>
public ReactiveCommand<RxVoid, RxVoid> GoToProxiesCommand { get; }
/// <summary>Removes everything the selected source has collected.</summary>
public ReactiveCommand<RxVoid, RxVoid> PurgeCommand { get; }
/// <summary>Explains why collecting is blocked.</summary>
public string ProxyRequiredMessage => Localizer.Instance["Collect.ProxyRequired"];
@@ -235,11 +256,7 @@ public partial class CollectViewModel : PageViewModel, IDisposable
var source = SelectedSource.Source;
var query = BuildQuery();
var options = new CollectOptions
{
MaxConcurrentDownloads = _settings.Current.MaxConcurrentDownloads,
ForceRefetch = ForceRefetch,
};
var options = _settings.Current.ToCollectOptions() with { ForceRefetch = ForceRefetch };
ClearResults();
Progress = 0d;
@@ -341,6 +358,55 @@ public partial class CollectViewModel : PageViewModel, IDisposable
);
}
/// <summary>
/// Removes what the selected source collected.
/// </summary>
/// <remarks>
/// Scoped to one source rather than emptying the store: content another source also holds
/// survives, which is exactly what the reference count in the index is for.
/// </remarks>
private async Task PurgeAsync(CancellationToken cancellationToken)
{
var sourceId = SelectedSource.Id;
var result = await _store.PurgeAsync(new PurgeOptions(sourceId), cancellationToken).ConfigureAwait(false);
var loc = Localizer.Instance;
var message = loc.Format(
"Collect.PurgeDone",
loc.Plural("Collect.Count.Files", result.ItemsRemoved),
CollectedItemViewModel.FormatSize(result.BytesFreed)
);
OnUi(() =>
{
StatusMessage = message;
ClearResults();
});
await RefreshStorageAsync().ConfigureAwait(false);
}
/// <summary>Re-reads the store totals. Never throws; an unreadable store is not fatal here.</summary>
private async Task RefreshStorageAsync()
{
try
{
var stats = await _store.GetStatsAsync().ConfigureAwait(false);
var loc = Localizer.Instance;
var summary = loc.Format(
"Collect.Storage",
loc.Plural("Collect.Count.Files", stats.BlobCount),
CollectedItemViewModel.FormatSize(stats.TotalBytes)
);
OnUi(() => StorageSummary = summary);
}
catch (Exception ex)
{
_logger.LogWarning(ex, "Could not read the media store totals");
}
}
private MediaQuery BuildQuery()
{
var endpoint =
@@ -1,3 +1,4 @@
using AvParser.Core.Collecting;
using AvParser.Core.Proxies;
using AvParser.Core.Settings;
using AvParser.Infrastructure.Logging;
@@ -66,6 +67,26 @@ public partial class SettingsViewModel : PageViewModel
[Reactive]
public partial bool AllowDirectConnection { get; set; }
/// <summary>How many downloads may be in flight at once.</summary>
[Reactive]
public partial int MaxConcurrentDownloads { get; set; }
/// <summary>How many requests one origin may be serving at once.</summary>
[Reactive]
public partial int MaxConcurrentPerHost { get; set; }
/// <summary>Shortest gap between two requests to the same origin, in milliseconds.</summary>
[Reactive]
public partial int HostDelayMs { get; set; }
/// <summary>Largest item to accept, in megabytes.</summary>
[Reactive]
public partial int MaxItemMegabytes { get; set; }
/// <summary>How browsable copies point at their blobs.</summary>
[Reactive]
public partial LocalizedOption<ShowcaseMode> SelectedShowcaseMode { get; set; }
/// <summary>Creates the page.</summary>
public SettingsViewModel(
ISettingsService settings,
@@ -88,6 +109,7 @@ public partial class SettingsViewModel : PageViewModel
SettingsFile = paths.SettingsFile;
LogDirectory = paths.LogDirectory;
MediaDirectory = paths.MediaDirectory;
SelectedTheme = Option(Themes, theme.Current);
SelectedLanguage = Option(Languages, localization.Current);
@@ -102,6 +124,11 @@ public partial class SettingsViewModel : PageViewModel
ProxyProbeConcurrency = current.ProxyProbeConcurrency;
ProxyMinimumLive = current.ProxyMinimumLive;
AllowDirectConnection = current.AllowDirectConnection;
MaxConcurrentDownloads = current.MaxConcurrentDownloads;
MaxConcurrentPerHost = current.MaxConcurrentPerHost;
HostDelayMs = current.HostDelayMs;
MaxItemMegabytes = (int)Math.Max(1, current.MaxItemBytes / (1024 * 1024));
SelectedShowcaseMode = Option(ShowcaseModes, current.ShowcaseMode);
this.WhenAnyValue(x => x.SelectedTheme).ObserveOn(scheduler).Subscribe(option => _theme.Apply(option.Value));
@@ -132,7 +159,21 @@ public partial class SettingsViewModel : PageViewModel
)
.Throttle(TimeSpan.FromMilliseconds(200), scheduler)
.ObserveOn(scheduler)
.Subscribe(_ => ApplyProxySettings());
.Subscribe(_ => ApplySettings());
// A second subscription rather than one enormous one: WhenAnyValue runs out of overloads
// past a dozen properties, and both paths write the same record anyway.
this.WhenAnyValue(
x => x.MaxConcurrentDownloads,
x => x.MaxConcurrentPerHost,
x => x.HostDelayMs,
x => x.MaxItemMegabytes,
x => x.SelectedShowcaseMode,
(_, _, _, _, _) => RxVoid.Default
)
.Throttle(TimeSpan.FromMilliseconds(200), scheduler)
.ObserveOn(scheduler)
.Subscribe(_ => ApplySettings());
this.WhenAnyValue(x => x.SelectedRotation)
.Subscribe(_ => this.RaisePropertyChanged(nameof(RotationDescription)));
@@ -159,6 +200,9 @@ public partial class SettingsViewModel : PageViewModel
/// <summary>Rotation strategies offered by the picker.</summary>
public IReadOnlyList<LocalizedOption<ProxyRotation>> Rotations { get; } = LocalizedOption<ProxyRotation>.ForAll();
/// <summary>Showcase link modes offered by the picker.</summary>
public IReadOnlyList<LocalizedOption<ShowcaseMode>> ShowcaseModes { get; } = LocalizedOption<ShowcaseMode>.ForAll();
/// <summary>Liveness policies offered by the picker.</summary>
public IReadOnlyList<LocalizedOption<ProxyHealthCheck>> HealthChecks { get; } =
LocalizedOption<ProxyHealthCheck>.ForAll();
@@ -185,6 +229,9 @@ public partial class SettingsViewModel : PageViewModel
/// <summary>Directory holding rolling log files.</summary>
public string LogDirectory { get; }
/// <summary>Root of the collected media. Shown rather than edited: changing it needs a restart.</summary>
public string MediaDirectory { get; }
/// <summary>Width in pixels at which the shell switches from compact to the icon rail.</summary>
public double MediumBreakpoint => ResponsiveLayout.MediumMinWidth;
@@ -220,7 +267,8 @@ public partial class SettingsViewModel : PageViewModel
_settings.Update(current => current with { MinimumLogLevel = level });
}
private void ApplyProxySettings()
/// <summary>Writes every knob on this page in one go and re-configures what depends on them.</summary>
private void ApplySettings()
{
AppSettings? applied = null;
@@ -236,6 +284,11 @@ public partial class SettingsViewModel : PageViewModel
ProxyProbeConcurrency = ProxyProbeConcurrency,
ProxyMinimumLive = ProxyMinimumLive,
AllowDirectConnection = AllowDirectConnection,
MaxConcurrentDownloads = MaxConcurrentDownloads,
MaxConcurrentPerHost = MaxConcurrentPerHost,
HostDelayMs = HostDelayMs,
MaxItemBytes = (long)MaxItemMegabytes * 1024 * 1024,
ShowcaseMode = SelectedShowcaseMode.Value,
};
return applied;
+22
View File
@@ -59,6 +59,28 @@
Content="{l:Loc Collect.ForceRefetch}"
ToolTip.Tip="{l:Loc Collect.ForceRefetchHint}"
/>
<DockPanel LastChildFill="True">
<Button
DockPanel.Dock="Right"
Classes="destructive"
Command="{Binding PurgeCommand}"
ToolTip.Tip="{l:Loc Collect.PurgeHint}"
>
<StackPanel Orientation="Horizontal" Spacing="8">
<PathIcon Classes="glyph" Data="{DynamicResource IconTrash}" />
<TextBlock Text="{l:Loc Collect.Purge}" />
</StackPanel>
</Button>
<TextBlock
Classes="muted"
Text="{Binding StorageSummary}"
VerticalAlignment="Center"
TextWrapping="Wrap"
IsVisible="{Binding StorageSummary, Converter={x:Static ObjectConverters.IsNotNull}}"
/>
</DockPanel>
</StackPanel>
</Border>
+63
View File
@@ -148,6 +148,69 @@
</StackPanel>
</Border>
<Border Classes="card">
<StackPanel Spacing="16">
<TextBlock Classes="subtitle" Text="{l:Loc Settings.Collecting}" />
<Grid ColumnDefinitions="*,16,*">
<StackPanel Grid.Column="0" Spacing="6">
<TextBlock Classes="caption" Text="{l:Loc Settings.MaxConcurrentDownloads}" />
<NumericUpDown
Value="{Binding MaxConcurrentDownloads}"
Minimum="1"
Maximum="32"
Increment="1"
FormatString="0"
/>
</StackPanel>
<StackPanel Grid.Column="2" Spacing="6">
<TextBlock Classes="caption" Text="{l:Loc Settings.MaxConcurrentPerHost}" />
<NumericUpDown
Value="{Binding MaxConcurrentPerHost}"
Minimum="1"
Maximum="16"
Increment="1"
FormatString="0"
/>
</StackPanel>
</Grid>
<StackPanel Spacing="6">
<TextBlock Classes="caption" Text="{l:Loc Settings.HostDelay}" />
<NumericUpDown Value="{Binding HostDelayMs}" Minimum="0" Maximum="60000" Increment="50" FormatString="0" />
<TextBlock Classes="muted" Text="{l:Loc Settings.HostDelayHint}" TextWrapping="Wrap" />
</StackPanel>
<StackPanel Spacing="6">
<TextBlock Classes="caption" Text="{l:Loc Settings.MaxItemSize}" />
<NumericUpDown
Value="{Binding MaxItemMegabytes}"
Minimum="1"
Maximum="2048"
Increment="4"
FormatString="0"
/>
<TextBlock Classes="muted" Text="{l:Loc Settings.MaxItemSizeHint}" TextWrapping="Wrap" />
</StackPanel>
<StackPanel Spacing="6">
<TextBlock Classes="caption" Text="{l:Loc Settings.ShowcaseMode}" />
<ComboBox
ItemsSource="{Binding ShowcaseModes}"
SelectedItem="{Binding SelectedShowcaseMode}"
ItemTemplate="{StaticResource LocalizedOptionTemplate}"
HorizontalAlignment="Stretch"
/>
<TextBlock Classes="muted" Text="{l:Loc Settings.ShowcaseModeHint}" TextWrapping="Wrap" />
</StackPanel>
<StackPanel Spacing="6">
<TextBlock Classes="caption" Text="{l:Loc Settings.MediaDirectory}" />
<SelectableTextBlock Classes="mono" Text="{Binding MediaDirectory}" TextWrapping="Wrap" />
</StackPanel>
</StackPanel>
</Border>
<Border Classes="card">
<StackPanel Spacing="12">
<TextBlock Classes="subtitle" Text="{l:Loc Settings.Breakpoints}" />
@@ -121,6 +121,7 @@ public sealed class CollectRunnerTests : IAsyncLifetime
private MediaStore _store = null!;
private ScriptedFetcher _fetcher = null!;
private FixedSettings _settings = null!;
private HostThrottle _throttle = null!;
private CollectRunner _runner = null!;
public async ValueTask InitializeAsync()
@@ -140,7 +141,8 @@ public sealed class CollectRunnerTests : IAsyncLifetime
// Direct is allowed here: these tests are about the runner, not the proxy gate.
_settings = new FixedSettings(new AppSettings { AllowDirectConnection = true });
_fetcher = new ScriptedFetcher(_blobs);
_runner = new CollectRunner(_fetcher, _store, _settings, NullLogger<CollectRunner>.Instance);
_throttle = new HostThrottle(4, TimeSpan.Zero, NullLogger<CollectRunnerTests>.Instance);
_runner = new CollectRunner(_fetcher, _store, _throttle, NullLogger<CollectRunner>.Instance);
await _store.InitialiseAsync(TestContext.Current.CancellationToken);
}
@@ -148,6 +150,7 @@ public sealed class CollectRunnerTests : IAsyncLifetime
public ValueTask DisposeAsync()
{
_settings.Dispose();
_throttle.Dispose();
_index.Dispose();
Microsoft.Data.Sqlite.SqliteConnection.ClearAllPools();
@@ -1,3 +1,4 @@
using AvParser.Core.Collecting;
using AvParser.Core.Proxies;
using AvParser.Core.Settings;
using AvParser.Infrastructure.Settings;
@@ -79,6 +80,90 @@ public sealed class JsonSettingsServiceTests : IDisposable
settings.MinimumLogLevel.ShouldBe("Information");
}
/// <summary>
/// The same guarantee, for the collector settings added afterwards.
/// </summary>
/// <remarks>
/// The failure this prevents is worse here than it was for the proxy pool: a zeroed
/// <c>MaxItemBytes</c> would refuse everything, and a zeroed concurrency would deadlock the
/// run outright.
/// </remarks>
[Fact]
public void A_file_that_predates_the_collector_keeps_the_collector_defaults()
{
WriteSettings("""{ "theme": "Dark", "proxyMinimumLive": 25 }""");
using var service = Create();
var settings = service.Current;
settings.MaxConcurrentDownloads.ShouldBe(4);
settings.MaxConcurrentPerHost.ShouldBe(2);
settings.HostDelayMs.ShouldBe(250);
settings.MaxItemBytes.ShouldBe(33_554_432);
settings.MinItemBytes.ShouldBe(1024);
settings.MaxRedirects.ShouldBe(5);
settings.ConnectTimeoutSeconds.ShouldBe(15);
settings.HeaderTimeoutSeconds.ShouldBe(30);
settings.IdleTimeoutSeconds.ShouldBe(20);
settings.AllowedMediaKinds.ShouldBe(MediaKindFilter.All);
settings.ShowcaseMode.ShouldBe(ShowcaseMode.HardLink);
settings.CollectUserAgent.ShouldNotBeNullOrWhiteSpace();
// And what the file did carry survives.
settings.ProxyMinimumLive.ShouldBe(25);
}
[Fact]
public void Collect_options_from_an_older_file_are_usable()
{
WriteSettings("""{ "theme": "Dark" }""");
using var service = Create();
var options = service.Current.ToCollectOptions();
options.MaxConcurrentDownloads.ShouldBeGreaterThan(0);
options.MaxItemBytes.ShouldBeGreaterThan(0);
options.AllowedKinds.ShouldBe(MediaKindFilter.All);
options.IdleTimeout.ShouldBeGreaterThan(TimeSpan.Zero);
}
[Fact]
public void Collector_values_out_of_range_are_clamped_rather_than_thrown()
{
var options = new AppSettings(
MaxConcurrentDownloads: 0,
MaxConcurrentPerHost: -5,
MaxItemBytes: 0,
MaxRedirects: 9999,
IdleTimeoutSeconds: 0,
CollectUserAgent: " "
).ToCollectOptions();
options.MaxConcurrentDownloads.ShouldBe(1);
options.MaxConcurrentPerHost.ShouldBe(1);
options.MaxItemBytes.ShouldBe(1024);
options.MaxRedirects.ShouldBe(20);
options.IdleTimeout.ShouldBe(TimeSpan.FromSeconds(1));
options.UserAgent.ShouldBe("AvParser/0.1");
}
[Fact]
public void An_empty_media_filter_is_treated_as_everything()
{
// Switching every format off is far more likely to be an accident than an instruction to
// collect nothing at all.
new AppSettings(AllowedMediaKinds: MediaKindFilter.None)
.ToCollectOptions()
.AllowedKinds.ShouldBe(MediaKindFilter.All);
}
[Fact]
public void The_proxy_gate_setting_reaches_the_collector()
{
new AppSettings(AllowDirectConnection: false).ToCollectOptions().RequireProxy.ShouldBeTrue();
new AppSettings(AllowDirectConnection: true).ToCollectOptions().RequireProxy.ShouldBeFalse();
}
[Fact]
public void Options_built_from_an_older_file_still_consult_the_feed()
{
@@ -70,6 +70,7 @@ public class CollectViewTests
new FakeSettingsService(new AppSettings { LastSourceId = networkSource ? "own-service" : "url-list" }),
new ProxyPool([], new FakeProxyProbe(), new ProxyOptions()),
new IdleRunner(),
new FakeMediaStore(),
new EmptyServiceProvider(),
NullLogger<CollectViewModel>.Instance,
ImmediateSequencer.Instance
@@ -0,0 +1,64 @@
using AvParser.Core.Collecting;
namespace AvParser.UI.HeadlessTests;
/// <summary>A store that records what it was asked to do and holds nothing.</summary>
internal sealed class FakeMediaStore : IMediaStore
{
public List<string> Purged { get; } = [];
public MediaStoreStats Stats { get; set; } = new(0, 0, 0, 0);
public PurgeResult PurgeResult { get; set; } = new(3, 2, 4096);
public ShowcaseMode Mode { get; private set; } = ShowcaseMode.HardLink;
public Task InitialiseAsync(CancellationToken cancellationToken = default) => Task.CompletedTask;
public void Configure(ShowcaseMode mode) => Mode = mode;
public Task<string> BeginRunAsync(string sourceId, CancellationToken cancellationToken = default) =>
Task.FromResult("run");
public Task CompleteRunAsync(string runId, RunSummary summary, CancellationToken cancellationToken = default) =>
Task.CompletedTask;
public Task<IReadOnlyDictionary<string, SeenOutcome>> GetSeenAsync(
string sourceId,
IReadOnlyCollection<string> urls,
CancellationToken cancellationToken = default
) =>
Task.FromResult<IReadOnlyDictionary<string, SeenOutcome>>(
new Dictionary<string, SeenOutcome>(StringComparer.Ordinal)
);
public Task RecordSeenAsync(
string sourceId,
string url,
SeenOutcome outcome,
int? httpStatus = null,
string? errorCode = null,
CancellationToken cancellationToken = default
) => Task.CompletedTask;
public Task<IReadOnlySet<string>> LoadTombstonesAsync(CancellationToken cancellationToken = default) =>
Task.FromResult<IReadOnlySet<string>>(new HashSet<string>(StringComparer.Ordinal));
public Task<int> TombstoneAsync(string sha256, string? reason, CancellationToken cancellationToken = default) =>
Task.FromResult(0);
public Task<CollectedItem> StoreAsync(MediaStoreRequest request, CancellationToken cancellationToken = default) =>
Task.FromResult(new CollectedItem(request.Candidate, request.Blob, CollectStatus.Stored));
public Task<PurgeResult> PurgeAsync(PurgeOptions options, CancellationToken cancellationToken = default)
{
Purged.Add(options.SourceId);
return Task.FromResult(PurgeResult);
}
public Task<int> RebuildShowcaseAsync(string sourceId, CancellationToken cancellationToken = default) =>
Task.FromResult(0);
public Task<MediaStoreStats> GetStatsAsync(CancellationToken cancellationToken = default) => Task.FromResult(Stats);
}
@@ -102,12 +102,14 @@ public class CollectViewModelTests
var catalog = new MediaSourceCatalog(sources, "url-list");
var settingsService = new FakeSettingsService(settings);
var runner = new FakeRunner();
var store = new FakeMediaStore();
var page = new CollectViewModel(
catalog,
settingsService,
proxyPool ?? new ProxyPool([], new FakeProxyProbe(), new ProxyOptions()),
runner,
store,
new EmptyServiceProvider(),
NullLogger<CollectViewModel>.Instance,
ImmediateSequencer.Instance
@@ -331,6 +333,26 @@ public class CollectViewModelTests
page.IsBlockedWithoutProxy.ShouldBeFalse();
}
[Fact]
public async Task Purging_removes_only_the_selected_source()
{
// Scoped rather than emptying the store: content another source also holds must survive,
// which is exactly what the index's reference count is for.
var (page, _, _) = Build();
await page.PurgeCommand.Execute().ToTask(TestContext.Current.CancellationToken);
page.StatusMessage.ShouldNotBeNull().ShouldContain("Removed");
}
[Fact]
public void The_store_totals_are_shown()
{
var (page, _, _) = Build();
page.StorageSummary.ShouldNotBeNull().ShouldContain("in the store");
}
[Fact]
public void Sizes_read_the_way_a_file_manager_shows_them()
{
@@ -0,0 +1,64 @@
using AvParser.Core.Collecting;
namespace AvParser.UI.Tests.Fakes;
/// <summary>A store that records what it was asked to do and holds nothing.</summary>
internal sealed class FakeMediaStore : IMediaStore
{
public List<string> Purged { get; } = [];
public MediaStoreStats Stats { get; set; } = new(0, 0, 0, 0);
public PurgeResult PurgeResult { get; set; } = new(3, 2, 4096);
public ShowcaseMode Mode { get; private set; } = ShowcaseMode.HardLink;
public Task InitialiseAsync(CancellationToken cancellationToken = default) => Task.CompletedTask;
public void Configure(ShowcaseMode mode) => Mode = mode;
public Task<string> BeginRunAsync(string sourceId, CancellationToken cancellationToken = default) =>
Task.FromResult("run");
public Task CompleteRunAsync(string runId, RunSummary summary, CancellationToken cancellationToken = default) =>
Task.CompletedTask;
public Task<IReadOnlyDictionary<string, SeenOutcome>> GetSeenAsync(
string sourceId,
IReadOnlyCollection<string> urls,
CancellationToken cancellationToken = default
) =>
Task.FromResult<IReadOnlyDictionary<string, SeenOutcome>>(
new Dictionary<string, SeenOutcome>(StringComparer.Ordinal)
);
public Task RecordSeenAsync(
string sourceId,
string url,
SeenOutcome outcome,
int? httpStatus = null,
string? errorCode = null,
CancellationToken cancellationToken = default
) => Task.CompletedTask;
public Task<IReadOnlySet<string>> LoadTombstonesAsync(CancellationToken cancellationToken = default) =>
Task.FromResult<IReadOnlySet<string>>(new HashSet<string>(StringComparer.Ordinal));
public Task<int> TombstoneAsync(string sha256, string? reason, CancellationToken cancellationToken = default) =>
Task.FromResult(0);
public Task<CollectedItem> StoreAsync(MediaStoreRequest request, CancellationToken cancellationToken = default) =>
Task.FromResult(new CollectedItem(request.Candidate, request.Blob, CollectStatus.Stored));
public Task<PurgeResult> PurgeAsync(PurgeOptions options, CancellationToken cancellationToken = default)
{
Purged.Add(options.SourceId);
return Task.FromResult(PurgeResult);
}
public Task<int> RebuildShowcaseAsync(string sourceId, CancellationToken cancellationToken = default) =>
Task.FromResult(0);
public Task<MediaStoreStats> GetStatsAsync(CancellationToken cancellationToken = default) => Task.FromResult(Stats);
}