diff --git a/src/AvParser.Core/Collecting/ICollectRunner.cs b/src/AvParser.Core/Collecting/ICollectRunner.cs index a8a04d8..d3b7533 100644 --- a/src/AvParser.Core/Collecting/ICollectRunner.cs +++ b/src/AvParser.Core/Collecting/ICollectRunner.cs @@ -2,7 +2,86 @@ using AvParser.Core.Parsing; namespace AvParser.Core.Collecting; -/// How to run one collection. +/// Which formats to keep. +/// +/// Flags rather than a collection so that AppSettings keeps value equality — the settings +/// service short-circuits a no-op write by comparing records, and a list-valued member would make +/// every save look like a change. Same reasoning as ProxyProtocolFilter. +/// +[Flags] +public enum MediaKindFilter +{ + /// Nothing. Treated as rather than collecting nothing at all. + None = 0, + + /// JPEG. + Jpeg = 1, + + /// PNG, including animated PNG. + Png = 2, + + /// GIF. + Gif = 4, + + /// WebP. + WebP = 8, + + /// AVIF. + Avif = 16, + + /// MP4, which is what most sites serve when they say "GIF". + Mp4 = 32, + + /// WebM. + WebM = 64, + + /// Every still or animated picture. + Images = Jpeg | Png | Gif | WebP | Avif, + + /// Every video container. + Videos = Mp4 | WebM, + + /// Everything recognised. + All = Images | Videos, +} + +/// Helpers over . +public static class MediaKindFilters +{ + /// The flag standing for one kind. + public static MediaKindFilter ToFlag(MediaKind kind) => + kind switch + { + MediaKind.Jpeg => MediaKindFilter.Jpeg, + MediaKind.Png => MediaKindFilter.Png, + MediaKind.Gif => MediaKindFilter.Gif, + MediaKind.WebP => MediaKindFilter.WebP, + MediaKind.Avif => MediaKindFilter.Avif, + MediaKind.Mp4 => MediaKindFilter.Mp4, + MediaKind.WebM => MediaKindFilter.WebM, + _ => MediaKindFilter.None, + }; + + /// Expands a filter into the set of kinds it admits. + public static IReadOnlySet ToSet(MediaKindFilter filter) + { + var effective = filter == MediaKindFilter.None ? MediaKindFilter.All : filter; + + return new HashSet( + Enum.GetValues().Where(kind => kind != MediaKind.Unknown && effective.HasFlag(ToFlag(kind))) + ); + } +} + +/// +/// How to run one collection: what to accept, how hard to push, and when to give up. +/// +/// +/// Every ceiling here exists because the other side chooses the bytes. A missing cap is not a +/// generous default, it is a remote party deciding how much of the user's disk to fill. Built by +/// AppSettings.ToCollectOptions(), which clamps rather than throws so that a hand-edited +/// settings file cannot stop the app from starting. +/// public sealed record CollectOptions { /// How many downloads may be in flight at once, across all hosts. @@ -12,8 +91,44 @@ public sealed record CollectOptions /// public int MaxConcurrentDownloads { get; init; } = 4; + /// How many requests one origin may be serving at once. + public int MaxConcurrentPerHost { get; init; } = 2; + + /// Shortest gap between two requests to the same origin. + public TimeSpan HostDelay { get; init; } = TimeSpan.FromMilliseconds(250); + /// Ignore the journal and fetch every address again. public bool ForceRefetch { get; init; } + + /// Largest item to accept. + public long MaxItemBytes { get; init; } = 32L * 1024 * 1024; + + /// Smallest item to accept; below this it is a tracking pixel, not media. + public long MinItemBytes { get; init; } = 1024; + + /// How many redirects to follow before giving up. + public int MaxRedirects { get; init; } = 5; + + /// Time allowed to establish a connection. + public TimeSpan ConnectTimeout { get; init; } = TimeSpan.FromSeconds(15); + + /// Time allowed for the response headers to arrive. + public TimeSpan HeaderTimeout { get; init; } = TimeSpan.FromSeconds(30); + + /// Longest gap between two body reads before the transfer is called stalled. + public TimeSpan IdleTimeout { get; init; } = TimeSpan.FromSeconds(20); + + /// Formats to keep. + public MediaKindFilter AllowedKinds { get; init; } = MediaKindFilter.All; + + /// Identifies the collector to origins that care. + public string UserAgent { get; init; } = "AvParser/0.1"; + + /// Whether a missing proxy is a hard failure rather than a direct connection. + public bool RequireProxy { get; init; } + + /// How browsable copies point at their blobs. + public ShowcaseMode ShowcaseMode { get; init; } = ShowcaseMode.HardLink; } /// Runs a source end to end: discover, download, store. diff --git a/src/AvParser.Core/Collecting/IMediaStore.cs b/src/AvParser.Core/Collecting/IMediaStore.cs index fa618db..7717fdf 100644 --- a/src/AvParser.Core/Collecting/IMediaStore.cs +++ b/src/AvParser.Core/Collecting/IMediaStore.cs @@ -83,6 +83,13 @@ public interface IMediaStore /// Creates or upgrades the schema. Safe to call repeatedly. Task InitialiseAsync(CancellationToken cancellationToken = default); + /// Applies the user's showcase preference. + /// + /// Applied rather than injected, for the same reason IProxyPool.Configure exists: it + /// changes while the app runs, and a snapshot taken at container build would freeze it. + /// + void Configure(ShowcaseMode mode); + /// Opens a run and returns its id. Task BeginRunAsync(string sourceId, CancellationToken cancellationToken = default); diff --git a/src/AvParser.Core/Settings/AppSettings.cs b/src/AvParser.Core/Settings/AppSettings.cs index ab6f69b..920db93 100644 --- a/src/AvParser.Core/Settings/AppSettings.cs +++ b/src/AvParser.Core/Settings/AppSettings.cs @@ -1,3 +1,4 @@ +using AvParser.Core.Collecting; using AvParser.Core.Proxies; namespace AvParser.Core.Settings; @@ -49,6 +50,18 @@ public enum AppTheme /// Whether network parsers may run without a proxy. /// Id of the media source selected last time; resolved leniently on load. /// How many downloads may be in flight at once. +/// How many requests one origin may be serving at once. +/// Shortest gap between two requests to the same origin, in milliseconds. +/// Largest item to accept, in bytes. +/// Smallest item to accept, in bytes. +/// How many redirects to follow before giving up. +/// Time allowed to establish a connection. +/// Time allowed for response headers to arrive. +/// Longest gap between body reads before a transfer is called stalled. +/// Formats to keep. +/// How browsable copies point at their blobs. +/// Identifies the collector to origins that care. +/// Where collected media goes; null keeps it beside the settings. public sealed record AppSettings( AppTheme Theme = AppTheme.System, AppLanguage Language = AppLanguage.System, @@ -67,7 +80,19 @@ public sealed record AppSettings( int ProxyMinimumLive = 10, bool AllowDirectConnection = false, string? LastSourceId = null, - int MaxConcurrentDownloads = 4 + int MaxConcurrentDownloads = 4, + int MaxConcurrentPerHost = 2, + int HostDelayMs = 250, + long MaxItemBytes = 33_554_432, + long MinItemBytes = 1024, + int MaxRedirects = 5, + int ConnectTimeoutSeconds = 15, + int HeaderTimeoutSeconds = 30, + int IdleTimeoutSeconds = 20, + MediaKindFilter AllowedMediaKinds = MediaKindFilter.All, + ShowcaseMode ShowcaseMode = ShowcaseMode.HardLink, + string CollectUserAgent = "AvParser/0.1", + string? MediaRootOverride = null ) { /// Projects the proxy-related settings onto . @@ -95,4 +120,31 @@ public sealed record AppSettings( AllowDirectConnection = AllowDirectConnection, }.Validated(); } + + /// Projects the collector settings onto . + /// + /// The one place primitives become policy, and it clamps rather than throws for the same reason + /// does: a hand-edited settings file must not be able to stop the + /// app from starting. A ceiling edited to zero would otherwise mean "accept nothing" or, worse, + /// "accept anything". + /// + public CollectOptions ToCollectOptions() => + new() + { + MaxConcurrentDownloads = Math.Clamp(MaxConcurrentDownloads, 1, 32), + MaxConcurrentPerHost = Math.Clamp(MaxConcurrentPerHost, 1, 16), + HostDelay = TimeSpan.FromMilliseconds(Math.Clamp(HostDelayMs, 0, 60_000)), + MaxItemBytes = Math.Clamp(MaxItemBytes, 1024, 2L * 1024 * 1024 * 1024), + MinItemBytes = Math.Clamp(MinItemBytes, 0, 1024 * 1024), + MaxRedirects = Math.Clamp(MaxRedirects, 0, 20), + ConnectTimeout = TimeSpan.FromSeconds(Math.Clamp(ConnectTimeoutSeconds, 1, 120)), + HeaderTimeout = TimeSpan.FromSeconds(Math.Clamp(HeaderTimeoutSeconds, 1, 300)), + IdleTimeout = TimeSpan.FromSeconds(Math.Clamp(IdleTimeoutSeconds, 1, 300)), + // An empty filter means the user has switched everything off, which is far more likely + // to be an accident than an intention to collect nothing. + AllowedKinds = AllowedMediaKinds == MediaKindFilter.None ? MediaKindFilter.All : AllowedMediaKinds, + UserAgent = string.IsNullOrWhiteSpace(CollectUserAgent) ? "AvParser/0.1" : CollectUserAgent, + RequireProxy = !AllowDirectConnection, + ShowcaseMode = ShowcaseMode, + }; } diff --git a/src/AvParser.Infrastructure/Collecting/CollectRunner.cs b/src/AvParser.Infrastructure/Collecting/CollectRunner.cs index 48b5fba..11bf5f1 100644 --- a/src/AvParser.Infrastructure/Collecting/CollectRunner.cs +++ b/src/AvParser.Infrastructure/Collecting/CollectRunner.cs @@ -2,7 +2,7 @@ using System.Runtime.CompilerServices; using System.Threading.Channels; using AvParser.Core.Collecting; using AvParser.Core.Parsing; -using AvParser.Core.Settings; +using AvParser.Infrastructure.Proxies; using Microsoft.Extensions.Logging; namespace AvParser.Infrastructure.Collecting; @@ -26,7 +26,7 @@ namespace AvParser.Infrastructure.Collecting; public sealed class CollectRunner( IMediaFetcher fetcher, IMediaStore store, - ISettingsService settings, + HostThrottle throttle, ILogger logger ) : ICollectRunner { @@ -38,7 +38,7 @@ public sealed class CollectRunner( private readonly IMediaFetcher _fetcher = fetcher ?? throw new ArgumentNullException(nameof(fetcher)); private readonly IMediaStore _store = store ?? throw new ArgumentNullException(nameof(store)); - private readonly ISettingsService _settings = settings ?? throw new ArgumentNullException(nameof(settings)); + private readonly HostThrottle _throttle = throttle ?? throw new ArgumentNullException(nameof(throttle)); private readonly ILogger _logger = logger ?? throw new ArgumentNullException(nameof(logger)); /// @@ -55,9 +55,13 @@ public sealed class CollectRunner( ArgumentNullException.ThrowIfNull(options); var workers = Math.Clamp(options.MaxConcurrentDownloads, 1, 32); + + _throttle.Configure(options.MaxConcurrentPerHost, options.HostDelay); + _store.Configure(options.ShowcaseMode); + var runId = await _store.BeginRunAsync(source.Id, cancellationToken).ConfigureAwait(false); var tombstones = await _store.LoadTombstonesAsync(cancellationToken).ConfigureAwait(false); - var fetchOptions = BuildFetchOptions(tombstones); + var fetchOptions = BuildFetchOptions(options, tombstones); var work = Channel.CreateBounded( new BoundedChannelOptions(workers * 4) { FullMode = BoundedChannelFullMode.Wait } @@ -338,17 +342,20 @@ public sealed class CollectRunner( return ParseOutcome.Success(stored with { Elapsed = result.Elapsed }); } - private FetchOptions BuildFetchOptions(IReadOnlySet tombstones) - { - var current = _settings.Current; - - return new FetchOptions + /// Projects the run's policy onto the HTTP layer's parameters. + private static FetchOptions BuildFetchOptions(CollectOptions options, IReadOnlySet tombstones) => + new() { - // Mirrors the parser gate: proxy-only unless the user has said direct is acceptable. - RequireProxy = !current.AllowDirectConnection, + MaxItemBytes = options.MaxItemBytes, + MinItemBytes = options.MinItemBytes, + MaxRedirects = options.MaxRedirects, + Timeouts = new HttpClientTimeouts(options.ConnectTimeout, options.HeaderTimeout, options.IdleTimeout), + // Mirrors the page's gate: proxy-only unless the user has said direct is acceptable. + RequireProxy = options.RequireProxy, + UserAgent = options.UserAgent, + AllowedKinds = MediaKindFilters.ToSet(options.AllowedKinds), Tombstones = tombstones, }; - } /// Stands in for content on a skipped item, which by definition has none. private static MediaBlob Placeholder { get; } = MediaBlob.Create(new string('0', 64), MediaKind.Unknown, 0); diff --git a/src/AvParser.Infrastructure/Collecting/HostThrottle.cs b/src/AvParser.Infrastructure/Collecting/HostThrottle.cs index 03812bd..9f212dd 100644 --- a/src/AvParser.Infrastructure/Collecting/HostThrottle.cs +++ b/src/AvParser.Infrastructure/Collecting/HostThrottle.cs @@ -26,11 +26,25 @@ public sealed class HostThrottle( ) : IDisposable { private readonly ConcurrentDictionary _hosts = new(StringComparer.OrdinalIgnoreCase); - private readonly int _maxConcurrent = Math.Clamp(maxConcurrentPerHost, 1, 64); - private readonly TimeSpan _minimumInterval = minimumInterval < TimeSpan.Zero ? TimeSpan.Zero : minimumInterval; private readonly ILogger _logger = logger ?? throw new ArgumentNullException(nameof(logger)); private readonly TimeProvider _time = timeProvider ?? TimeProvider.System; + private int _maxConcurrent = Math.Clamp(maxConcurrentPerHost, 1, 64); + private TimeSpan _minimumInterval = minimumInterval < TimeSpan.Zero ? TimeSpan.Zero : minimumInterval; + + /// Applies new limits. + /// + /// The pacing interval takes effect immediately; the concurrency cap applies to origins not yet + /// seen this session, because changing a semaphore's capacity underneath requests already + /// holding it cannot be done safely. Shared across runs on purpose — politeness belongs to the + /// origin, not to whichever collection happens to be running. + /// + public void Configure(int maxConcurrentPerHost, TimeSpan minimumInterval) + { + _maxConcurrent = Math.Clamp(maxConcurrentPerHost, 1, 64); + _minimumInterval = minimumInterval < TimeSpan.Zero ? TimeSpan.Zero : minimumInterval; + } + /// How long this host is still refusing requests, or zero when it is not. public TimeSpan CooldownRemaining(Uri url) { diff --git a/src/AvParser.Infrastructure/DependencyInjection/InfrastructureServiceCollectionExtensions.cs b/src/AvParser.Infrastructure/DependencyInjection/InfrastructureServiceCollectionExtensions.cs index bf2c82b..eae9626 100644 --- a/src/AvParser.Infrastructure/DependencyInjection/InfrastructureServiceCollectionExtensions.cs +++ b/src/AvParser.Infrastructure/DependencyInjection/InfrastructureServiceCollectionExtensions.cs @@ -25,7 +25,7 @@ public static class InfrastructureServiceCollectionExtensions { ArgumentNullException.ThrowIfNull(services); - var resolved = paths ?? new AppPaths(); + var resolved = paths ?? WithConfiguredMediaRoot(new AppPaths()); resolved.EnsureCreated(); services.AddSingleton(resolved); @@ -43,6 +43,22 @@ public static class InfrastructureServiceCollectionExtensions return services; } + /// + /// Re-points the media root at the directory the user chose, if they chose one. + /// + /// + /// A collection outgrows a profile directory quickly, so pointing it at another drive is the + /// first thing anyone does. That creates an ordering problem — the setting lives in a file + /// whose location itself defines — so the file is read once here, + /// before the container exists, rather than making every path lazy for the sake of one value. + /// + private static AppPaths WithConfiguredMediaRoot(AppPaths defaults) + { + var stored = JsonSettingsService.ReadOrDefault(defaults.SettingsFile).MediaRootOverride; + + return string.IsNullOrWhiteSpace(stored) ? defaults : new AppPaths(defaults.DataDirectory, stored); + } + /// Registers the media store, the download pipeline and the network sources. /// /// Everything here is a singleton because everything here owns something shared: a database diff --git a/src/AvParser.Infrastructure/Media/MediaStore.cs b/src/AvParser.Infrastructure/Media/MediaStore.cs index 3309da2..bf9c260 100644 --- a/src/AvParser.Infrastructure/Media/MediaStore.cs +++ b/src/AvParser.Infrastructure/Media/MediaStore.cs @@ -31,7 +31,7 @@ public sealed class MediaStore( /// public ShowcaseMode ShowcaseMode { get; private set; } = ShowcaseMode.HardLink; - /// Applies the user's showcase preference. + /// public void Configure(ShowcaseMode mode) => ShowcaseMode = mode; /// diff --git a/src/AvParser.Infrastructure/Settings/JsonSettingsService.cs b/src/AvParser.Infrastructure/Settings/JsonSettingsService.cs index 643146a..2c56fb0 100644 --- a/src/AvParser.Infrastructure/Settings/JsonSettingsService.cs +++ b/src/AvParser.Infrastructure/Settings/JsonSettingsService.cs @@ -85,6 +85,33 @@ public sealed class JsonSettingsService : ISettingsService, IDisposable _writeLock.Dispose(); } + /// + /// Reads a settings file without building a service around it. + /// + /// + /// Exists for the one setting that has to be known before the container does — the media root, + /// which decides paths that the container itself is configured with. Silent on every failure, + /// like : a bad file must never stop the app from starting. + /// + public static AppSettings ReadOrDefault(string settingsFile) + { + try + { + if (!File.Exists(settingsFile)) + { + return new AppSettings(); + } + + var json = File.ReadAllText(settingsFile); + + return JsonSerializer.Deserialize(json, AppSettingsJsonContext.Default.AppSettings) ?? new AppSettings(); + } + catch (Exception ex) when (ex is IOException or UnauthorizedAccessException or JsonException) + { + return new AppSettings(); + } + } + private AppSettings Load() { try diff --git a/src/AvParser.UI/DependencyInjection/UiServiceCollectionExtensions.cs b/src/AvParser.UI/DependencyInjection/UiServiceCollectionExtensions.cs index 434161b..3dfe9f8 100644 --- a/src/AvParser.UI/DependencyInjection/UiServiceCollectionExtensions.cs +++ b/src/AvParser.UI/DependencyInjection/UiServiceCollectionExtensions.cs @@ -52,6 +52,7 @@ public static class UiServiceCollectionExtensions sp.GetRequiredService(), sp.GetRequiredService(), sp.GetRequiredService(), + sp.GetRequiredService(), sp, sp.GetRequiredService>() )); diff --git a/src/AvParser.UI/Localization/Strings.resx b/src/AvParser.UI/Localization/Strings.resx index f30f4e6..f06f18a 100644 --- a/src/AvParser.UI/Localization/Strings.resx +++ b/src/AvParser.UI/Localization/Strings.resx @@ -676,4 +676,67 @@ Could not be collected. + + No browsable copies + + + Hard link (no extra space) + + + Symbolic link + + + Copy (doubles disk use) + + + Collecting + + + DOWNLOADS AT ONCE + + + PER SITE + + + GAP BETWEEN REQUESTS (MS) + + + How politely one site is treated. The per-site limits are usually what actually binds, not the total above. + + + LARGEST ITEM (MB) + + + Anything bigger is refused, where possible before its body is downloaded at all. + + + BROWSABLE COPIES + + + Collected files are stored by content hash. The showcase gives them dated, named paths. A hard link is a second name for the same file: editing it edits the original, and deleting it frees nothing until the last name is gone. + + + MEDIA DIRECTORY + + + {0} in the store, {1}. + + + Remove this source's items + + + Removes what this source collected. Files another source also holds are kept. + + + Removed {0}; {1} freed. + + + {0} file + + + {0} files + + + {0} files + diff --git a/src/AvParser.UI/Localization/Strings.ru.resx b/src/AvParser.UI/Localization/Strings.ru.resx index 15a1e32..2b87e3b 100644 --- a/src/AvParser.UI/Localization/Strings.ru.resx +++ b/src/AvParser.UI/Localization/Strings.ru.resx @@ -676,4 +676,67 @@ Не удалось собрать. + + Без витрины + + + Жёсткая ссылка (без лишнего места) + + + Символическая ссылка + + + Копия (удваивает расход диска) + + + Сбор + + + ЗАГРУЗОК ОДНОВРЕМЕННО + + + НА ОДИН САЙТ + + + ПАУЗА МЕЖДУ ЗАПРОСАМИ (МС) + + + Насколько вежливо приложение обходится с одним сайтом. Обычно ограничивают именно эти два, а не общее число выше. + + + МАКСИМАЛЬНЫЙ РАЗМЕР (МБ) + + + Всё, что больше, отклоняется — по возможности ещё до скачивания тела. + + + ЧИТАЕМАЯ ВИТРИНА + + + Файлы хранятся по хешу содержимого. Витрина даёт им читаемые пути по датам. Жёсткая ссылка — это второе имя того же файла: правка меняет оригинал, а удаление ничего не освобождает, пока не исчезнет последнее имя. + + + КАТАЛОГ МЕДИА + + + В хранилище {0}, {1}. + + + Удалить собранное этим источником + + + Удаляет собранное этим источником. Файлы, на которые ссылается другой источник, остаются. + + + Удалено {0}; освобождено {1}. + + + {0} файл + + + {0} файла + + + {0} файлов + diff --git a/src/AvParser.UI/ViewModels/CollectViewModel.cs b/src/AvParser.UI/ViewModels/CollectViewModel.cs index 695d722..fcd741c 100644 --- a/src/AvParser.UI/ViewModels/CollectViewModel.cs +++ b/src/AvParser.UI/ViewModels/CollectViewModel.cs @@ -37,6 +37,7 @@ public partial class CollectViewModel : PageViewModel, IDisposable private readonly ISettingsService _settings; private readonly IProxyPool _proxyPool; private readonly ICollectRunner _runner; + private readonly IMediaStore _store; private readonly IServiceProvider _services; private readonly ILogger _logger; private readonly ISequencer _mainThread; @@ -74,6 +75,10 @@ public partial class CollectViewModel : PageViewModel, IDisposable [Reactive] public partial bool ForceRefetch { get; set; } + /// What the store holds overall; until it has been read. + [Reactive] + public partial string? StorageSummary { get; set; } + /// /// Whether the selected source needs the network but has no working proxy to use. /// @@ -89,6 +94,7 @@ public partial class CollectViewModel : PageViewModel, IDisposable /// Used to remember the selected source. /// Consulted for the live count that gates network sources. /// Runs the collection. + /// Consulted for totals, and asked to purge. /// Resolves the navigation service lazily, to keep pages acyclic. /// Diagnostics. /// @@ -100,6 +106,7 @@ public partial class CollectViewModel : PageViewModel, IDisposable ISettingsService settings, IProxyPool proxyPool, ICollectRunner runner, + IMediaStore store, IServiceProvider services, ILogger logger, ISequencer? mainThread = null @@ -109,6 +116,7 @@ public partial class CollectViewModel : PageViewModel, IDisposable _settings = settings ?? throw new ArgumentNullException(nameof(settings)); _proxyPool = proxyPool ?? throw new ArgumentNullException(nameof(proxyPool)); _runner = runner ?? throw new ArgumentNullException(nameof(runner)); + _store = store ?? throw new ArgumentNullException(nameof(store)); _services = services ?? throw new ArgumentNullException(nameof(services)); _logger = logger ?? throw new ArgumentNullException(nameof(logger)); _mainThread = mainThread ?? RxSchedulers.MainThreadScheduler; @@ -170,9 +178,19 @@ public partial class CollectViewModel : PageViewModel, IDisposable RefreshProxyGate(); }); + PurgeCommand = ReactiveCommand.CreateFromTask( + PurgeAsync, + CollectCommand.IsExecuting.Select(static running => !running), + _mainThread + ); + _settings.Changes.Subscribe(_ => RefreshProxyGate()); CollectCommand.ThrownExceptions.Subscribe(OnCommandFailed); + PurgeCommand.ThrownExceptions.Subscribe(OnCommandFailed); + + // Not awaited — a constructor cannot be — and it never throws. + _ = RefreshStorageAsync(); } /// @@ -205,6 +223,9 @@ public partial class CollectViewModel : PageViewModel, IDisposable /// Takes the user to the page where the proxy problem can be fixed. public ReactiveCommand GoToProxiesCommand { get; } + /// Removes everything the selected source has collected. + public ReactiveCommand PurgeCommand { get; } + /// Explains why collecting is blocked. public string ProxyRequiredMessage => Localizer.Instance["Collect.ProxyRequired"]; @@ -235,11 +256,7 @@ public partial class CollectViewModel : PageViewModel, IDisposable var source = SelectedSource.Source; var query = BuildQuery(); - var options = new CollectOptions - { - MaxConcurrentDownloads = _settings.Current.MaxConcurrentDownloads, - ForceRefetch = ForceRefetch, - }; + var options = _settings.Current.ToCollectOptions() with { ForceRefetch = ForceRefetch }; ClearResults(); Progress = 0d; @@ -341,6 +358,55 @@ public partial class CollectViewModel : PageViewModel, IDisposable ); } + /// + /// Removes what the selected source collected. + /// + /// + /// Scoped to one source rather than emptying the store: content another source also holds + /// survives, which is exactly what the reference count in the index is for. + /// + private async Task PurgeAsync(CancellationToken cancellationToken) + { + var sourceId = SelectedSource.Id; + var result = await _store.PurgeAsync(new PurgeOptions(sourceId), cancellationToken).ConfigureAwait(false); + + var loc = Localizer.Instance; + var message = loc.Format( + "Collect.PurgeDone", + loc.Plural("Collect.Count.Files", result.ItemsRemoved), + CollectedItemViewModel.FormatSize(result.BytesFreed) + ); + + OnUi(() => + { + StatusMessage = message; + ClearResults(); + }); + + await RefreshStorageAsync().ConfigureAwait(false); + } + + /// Re-reads the store totals. Never throws; an unreadable store is not fatal here. + private async Task RefreshStorageAsync() + { + try + { + var stats = await _store.GetStatsAsync().ConfigureAwait(false); + var loc = Localizer.Instance; + var summary = loc.Format( + "Collect.Storage", + loc.Plural("Collect.Count.Files", stats.BlobCount), + CollectedItemViewModel.FormatSize(stats.TotalBytes) + ); + + OnUi(() => StorageSummary = summary); + } + catch (Exception ex) + { + _logger.LogWarning(ex, "Could not read the media store totals"); + } + } + private MediaQuery BuildQuery() { var endpoint = diff --git a/src/AvParser.UI/ViewModels/SettingsViewModel.cs b/src/AvParser.UI/ViewModels/SettingsViewModel.cs index 03a460d..a8f7cfc 100644 --- a/src/AvParser.UI/ViewModels/SettingsViewModel.cs +++ b/src/AvParser.UI/ViewModels/SettingsViewModel.cs @@ -1,3 +1,4 @@ +using AvParser.Core.Collecting; using AvParser.Core.Proxies; using AvParser.Core.Settings; using AvParser.Infrastructure.Logging; @@ -66,6 +67,26 @@ public partial class SettingsViewModel : PageViewModel [Reactive] public partial bool AllowDirectConnection { get; set; } + /// How many downloads may be in flight at once. + [Reactive] + public partial int MaxConcurrentDownloads { get; set; } + + /// How many requests one origin may be serving at once. + [Reactive] + public partial int MaxConcurrentPerHost { get; set; } + + /// Shortest gap between two requests to the same origin, in milliseconds. + [Reactive] + public partial int HostDelayMs { get; set; } + + /// Largest item to accept, in megabytes. + [Reactive] + public partial int MaxItemMegabytes { get; set; } + + /// How browsable copies point at their blobs. + [Reactive] + public partial LocalizedOption SelectedShowcaseMode { get; set; } + /// Creates the page. public SettingsViewModel( ISettingsService settings, @@ -88,6 +109,7 @@ public partial class SettingsViewModel : PageViewModel SettingsFile = paths.SettingsFile; LogDirectory = paths.LogDirectory; + MediaDirectory = paths.MediaDirectory; SelectedTheme = Option(Themes, theme.Current); SelectedLanguage = Option(Languages, localization.Current); @@ -102,6 +124,11 @@ public partial class SettingsViewModel : PageViewModel ProxyProbeConcurrency = current.ProxyProbeConcurrency; ProxyMinimumLive = current.ProxyMinimumLive; AllowDirectConnection = current.AllowDirectConnection; + MaxConcurrentDownloads = current.MaxConcurrentDownloads; + MaxConcurrentPerHost = current.MaxConcurrentPerHost; + HostDelayMs = current.HostDelayMs; + MaxItemMegabytes = (int)Math.Max(1, current.MaxItemBytes / (1024 * 1024)); + SelectedShowcaseMode = Option(ShowcaseModes, current.ShowcaseMode); this.WhenAnyValue(x => x.SelectedTheme).ObserveOn(scheduler).Subscribe(option => _theme.Apply(option.Value)); @@ -132,7 +159,21 @@ public partial class SettingsViewModel : PageViewModel ) .Throttle(TimeSpan.FromMilliseconds(200), scheduler) .ObserveOn(scheduler) - .Subscribe(_ => ApplyProxySettings()); + .Subscribe(_ => ApplySettings()); + + // A second subscription rather than one enormous one: WhenAnyValue runs out of overloads + // past a dozen properties, and both paths write the same record anyway. + this.WhenAnyValue( + x => x.MaxConcurrentDownloads, + x => x.MaxConcurrentPerHost, + x => x.HostDelayMs, + x => x.MaxItemMegabytes, + x => x.SelectedShowcaseMode, + (_, _, _, _, _) => RxVoid.Default + ) + .Throttle(TimeSpan.FromMilliseconds(200), scheduler) + .ObserveOn(scheduler) + .Subscribe(_ => ApplySettings()); this.WhenAnyValue(x => x.SelectedRotation) .Subscribe(_ => this.RaisePropertyChanged(nameof(RotationDescription))); @@ -159,6 +200,9 @@ public partial class SettingsViewModel : PageViewModel /// Rotation strategies offered by the picker. public IReadOnlyList> Rotations { get; } = LocalizedOption.ForAll(); + /// Showcase link modes offered by the picker. + public IReadOnlyList> ShowcaseModes { get; } = LocalizedOption.ForAll(); + /// Liveness policies offered by the picker. public IReadOnlyList> HealthChecks { get; } = LocalizedOption.ForAll(); @@ -185,6 +229,9 @@ public partial class SettingsViewModel : PageViewModel /// Directory holding rolling log files. public string LogDirectory { get; } + /// Root of the collected media. Shown rather than edited: changing it needs a restart. + public string MediaDirectory { get; } + /// Width in pixels at which the shell switches from compact to the icon rail. public double MediumBreakpoint => ResponsiveLayout.MediumMinWidth; @@ -220,7 +267,8 @@ public partial class SettingsViewModel : PageViewModel _settings.Update(current => current with { MinimumLogLevel = level }); } - private void ApplyProxySettings() + /// Writes every knob on this page in one go and re-configures what depends on them. + private void ApplySettings() { AppSettings? applied = null; @@ -236,6 +284,11 @@ public partial class SettingsViewModel : PageViewModel ProxyProbeConcurrency = ProxyProbeConcurrency, ProxyMinimumLive = ProxyMinimumLive, AllowDirectConnection = AllowDirectConnection, + MaxConcurrentDownloads = MaxConcurrentDownloads, + MaxConcurrentPerHost = MaxConcurrentPerHost, + HostDelayMs = HostDelayMs, + MaxItemBytes = (long)MaxItemMegabytes * 1024 * 1024, + ShowcaseMode = SelectedShowcaseMode.Value, }; return applied; diff --git a/src/AvParser.UI/Views/CollectView.axaml b/src/AvParser.UI/Views/CollectView.axaml index efbdc40..6f6bd7f 100644 --- a/src/AvParser.UI/Views/CollectView.axaml +++ b/src/AvParser.UI/Views/CollectView.axaml @@ -59,6 +59,28 @@ Content="{l:Loc Collect.ForceRefetch}" ToolTip.Tip="{l:Loc Collect.ForceRefetchHint}" /> + + + + + + diff --git a/src/AvParser.UI/Views/SettingsView.axaml b/src/AvParser.UI/Views/SettingsView.axaml index 95767d5..c1775a0 100644 --- a/src/AvParser.UI/Views/SettingsView.axaml +++ b/src/AvParser.UI/Views/SettingsView.axaml @@ -148,6 +148,69 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/tests/AvParser.Infrastructure.Tests/Collecting/CollectRunnerTests.cs b/tests/AvParser.Infrastructure.Tests/Collecting/CollectRunnerTests.cs index bc77bae..9476e99 100644 --- a/tests/AvParser.Infrastructure.Tests/Collecting/CollectRunnerTests.cs +++ b/tests/AvParser.Infrastructure.Tests/Collecting/CollectRunnerTests.cs @@ -121,6 +121,7 @@ public sealed class CollectRunnerTests : IAsyncLifetime private MediaStore _store = null!; private ScriptedFetcher _fetcher = null!; private FixedSettings _settings = null!; + private HostThrottle _throttle = null!; private CollectRunner _runner = null!; public async ValueTask InitializeAsync() @@ -140,7 +141,8 @@ public sealed class CollectRunnerTests : IAsyncLifetime // Direct is allowed here: these tests are about the runner, not the proxy gate. _settings = new FixedSettings(new AppSettings { AllowDirectConnection = true }); _fetcher = new ScriptedFetcher(_blobs); - _runner = new CollectRunner(_fetcher, _store, _settings, NullLogger.Instance); + _throttle = new HostThrottle(4, TimeSpan.Zero, NullLogger.Instance); + _runner = new CollectRunner(_fetcher, _store, _throttle, NullLogger.Instance); await _store.InitialiseAsync(TestContext.Current.CancellationToken); } @@ -148,6 +150,7 @@ public sealed class CollectRunnerTests : IAsyncLifetime public ValueTask DisposeAsync() { _settings.Dispose(); + _throttle.Dispose(); _index.Dispose(); Microsoft.Data.Sqlite.SqliteConnection.ClearAllPools(); diff --git a/tests/AvParser.Infrastructure.Tests/JsonSettingsServiceTests.cs b/tests/AvParser.Infrastructure.Tests/JsonSettingsServiceTests.cs index 0919135..a29468b 100644 --- a/tests/AvParser.Infrastructure.Tests/JsonSettingsServiceTests.cs +++ b/tests/AvParser.Infrastructure.Tests/JsonSettingsServiceTests.cs @@ -1,3 +1,4 @@ +using AvParser.Core.Collecting; using AvParser.Core.Proxies; using AvParser.Core.Settings; using AvParser.Infrastructure.Settings; @@ -79,6 +80,90 @@ public sealed class JsonSettingsServiceTests : IDisposable settings.MinimumLogLevel.ShouldBe("Information"); } + /// + /// The same guarantee, for the collector settings added afterwards. + /// + /// + /// The failure this prevents is worse here than it was for the proxy pool: a zeroed + /// MaxItemBytes would refuse everything, and a zeroed concurrency would deadlock the + /// run outright. + /// + [Fact] + public void A_file_that_predates_the_collector_keeps_the_collector_defaults() + { + WriteSettings("""{ "theme": "Dark", "proxyMinimumLive": 25 }"""); + + using var service = Create(); + var settings = service.Current; + + settings.MaxConcurrentDownloads.ShouldBe(4); + settings.MaxConcurrentPerHost.ShouldBe(2); + settings.HostDelayMs.ShouldBe(250); + settings.MaxItemBytes.ShouldBe(33_554_432); + settings.MinItemBytes.ShouldBe(1024); + settings.MaxRedirects.ShouldBe(5); + settings.ConnectTimeoutSeconds.ShouldBe(15); + settings.HeaderTimeoutSeconds.ShouldBe(30); + settings.IdleTimeoutSeconds.ShouldBe(20); + settings.AllowedMediaKinds.ShouldBe(MediaKindFilter.All); + settings.ShowcaseMode.ShouldBe(ShowcaseMode.HardLink); + settings.CollectUserAgent.ShouldNotBeNullOrWhiteSpace(); + + // And what the file did carry survives. + settings.ProxyMinimumLive.ShouldBe(25); + } + + [Fact] + public void Collect_options_from_an_older_file_are_usable() + { + WriteSettings("""{ "theme": "Dark" }"""); + + using var service = Create(); + var options = service.Current.ToCollectOptions(); + + options.MaxConcurrentDownloads.ShouldBeGreaterThan(0); + options.MaxItemBytes.ShouldBeGreaterThan(0); + options.AllowedKinds.ShouldBe(MediaKindFilter.All); + options.IdleTimeout.ShouldBeGreaterThan(TimeSpan.Zero); + } + + [Fact] + public void Collector_values_out_of_range_are_clamped_rather_than_thrown() + { + var options = new AppSettings( + MaxConcurrentDownloads: 0, + MaxConcurrentPerHost: -5, + MaxItemBytes: 0, + MaxRedirects: 9999, + IdleTimeoutSeconds: 0, + CollectUserAgent: " " + ).ToCollectOptions(); + + options.MaxConcurrentDownloads.ShouldBe(1); + options.MaxConcurrentPerHost.ShouldBe(1); + options.MaxItemBytes.ShouldBe(1024); + options.MaxRedirects.ShouldBe(20); + options.IdleTimeout.ShouldBe(TimeSpan.FromSeconds(1)); + options.UserAgent.ShouldBe("AvParser/0.1"); + } + + [Fact] + public void An_empty_media_filter_is_treated_as_everything() + { + // Switching every format off is far more likely to be an accident than an instruction to + // collect nothing at all. + new AppSettings(AllowedMediaKinds: MediaKindFilter.None) + .ToCollectOptions() + .AllowedKinds.ShouldBe(MediaKindFilter.All); + } + + [Fact] + public void The_proxy_gate_setting_reaches_the_collector() + { + new AppSettings(AllowDirectConnection: false).ToCollectOptions().RequireProxy.ShouldBeTrue(); + new AppSettings(AllowDirectConnection: true).ToCollectOptions().RequireProxy.ShouldBeFalse(); + } + [Fact] public void Options_built_from_an_older_file_still_consult_the_feed() { diff --git a/tests/AvParser.UI.HeadlessTests/CollectViewTests.cs b/tests/AvParser.UI.HeadlessTests/CollectViewTests.cs index d759624..e26f05a 100644 --- a/tests/AvParser.UI.HeadlessTests/CollectViewTests.cs +++ b/tests/AvParser.UI.HeadlessTests/CollectViewTests.cs @@ -70,6 +70,7 @@ public class CollectViewTests new FakeSettingsService(new AppSettings { LastSourceId = networkSource ? "own-service" : "url-list" }), new ProxyPool([], new FakeProxyProbe(), new ProxyOptions()), new IdleRunner(), + new FakeMediaStore(), new EmptyServiceProvider(), NullLogger.Instance, ImmediateSequencer.Instance diff --git a/tests/AvParser.UI.HeadlessTests/FakeMediaStore.cs b/tests/AvParser.UI.HeadlessTests/FakeMediaStore.cs new file mode 100644 index 0000000..2d8a309 --- /dev/null +++ b/tests/AvParser.UI.HeadlessTests/FakeMediaStore.cs @@ -0,0 +1,64 @@ +using AvParser.Core.Collecting; + +namespace AvParser.UI.HeadlessTests; + +/// A store that records what it was asked to do and holds nothing. +internal sealed class FakeMediaStore : IMediaStore +{ + public List Purged { get; } = []; + + public MediaStoreStats Stats { get; set; } = new(0, 0, 0, 0); + + public PurgeResult PurgeResult { get; set; } = new(3, 2, 4096); + + public ShowcaseMode Mode { get; private set; } = ShowcaseMode.HardLink; + + public Task InitialiseAsync(CancellationToken cancellationToken = default) => Task.CompletedTask; + + public void Configure(ShowcaseMode mode) => Mode = mode; + + public Task BeginRunAsync(string sourceId, CancellationToken cancellationToken = default) => + Task.FromResult("run"); + + public Task CompleteRunAsync(string runId, RunSummary summary, CancellationToken cancellationToken = default) => + Task.CompletedTask; + + public Task> GetSeenAsync( + string sourceId, + IReadOnlyCollection urls, + CancellationToken cancellationToken = default + ) => + Task.FromResult>( + new Dictionary(StringComparer.Ordinal) + ); + + public Task RecordSeenAsync( + string sourceId, + string url, + SeenOutcome outcome, + int? httpStatus = null, + string? errorCode = null, + CancellationToken cancellationToken = default + ) => Task.CompletedTask; + + public Task> LoadTombstonesAsync(CancellationToken cancellationToken = default) => + Task.FromResult>(new HashSet(StringComparer.Ordinal)); + + public Task TombstoneAsync(string sha256, string? reason, CancellationToken cancellationToken = default) => + Task.FromResult(0); + + public Task StoreAsync(MediaStoreRequest request, CancellationToken cancellationToken = default) => + Task.FromResult(new CollectedItem(request.Candidate, request.Blob, CollectStatus.Stored)); + + public Task PurgeAsync(PurgeOptions options, CancellationToken cancellationToken = default) + { + Purged.Add(options.SourceId); + + return Task.FromResult(PurgeResult); + } + + public Task RebuildShowcaseAsync(string sourceId, CancellationToken cancellationToken = default) => + Task.FromResult(0); + + public Task GetStatsAsync(CancellationToken cancellationToken = default) => Task.FromResult(Stats); +} diff --git a/tests/AvParser.UI.Tests/CollectViewModelTests.cs b/tests/AvParser.UI.Tests/CollectViewModelTests.cs index ea41c8a..fcdbe10 100644 --- a/tests/AvParser.UI.Tests/CollectViewModelTests.cs +++ b/tests/AvParser.UI.Tests/CollectViewModelTests.cs @@ -102,12 +102,14 @@ public class CollectViewModelTests var catalog = new MediaSourceCatalog(sources, "url-list"); var settingsService = new FakeSettingsService(settings); var runner = new FakeRunner(); + var store = new FakeMediaStore(); var page = new CollectViewModel( catalog, settingsService, proxyPool ?? new ProxyPool([], new FakeProxyProbe(), new ProxyOptions()), runner, + store, new EmptyServiceProvider(), NullLogger.Instance, ImmediateSequencer.Instance @@ -331,6 +333,26 @@ public class CollectViewModelTests page.IsBlockedWithoutProxy.ShouldBeFalse(); } + [Fact] + public async Task Purging_removes_only_the_selected_source() + { + // Scoped rather than emptying the store: content another source also holds must survive, + // which is exactly what the index's reference count is for. + var (page, _, _) = Build(); + + await page.PurgeCommand.Execute().ToTask(TestContext.Current.CancellationToken); + + page.StatusMessage.ShouldNotBeNull().ShouldContain("Removed"); + } + + [Fact] + public void The_store_totals_are_shown() + { + var (page, _, _) = Build(); + + page.StorageSummary.ShouldNotBeNull().ShouldContain("in the store"); + } + [Fact] public void Sizes_read_the_way_a_file_manager_shows_them() { diff --git a/tests/AvParser.UI.Tests/Fakes/FakeMediaStore.cs b/tests/AvParser.UI.Tests/Fakes/FakeMediaStore.cs new file mode 100644 index 0000000..bfd6b68 --- /dev/null +++ b/tests/AvParser.UI.Tests/Fakes/FakeMediaStore.cs @@ -0,0 +1,64 @@ +using AvParser.Core.Collecting; + +namespace AvParser.UI.Tests.Fakes; + +/// A store that records what it was asked to do and holds nothing. +internal sealed class FakeMediaStore : IMediaStore +{ + public List Purged { get; } = []; + + public MediaStoreStats Stats { get; set; } = new(0, 0, 0, 0); + + public PurgeResult PurgeResult { get; set; } = new(3, 2, 4096); + + public ShowcaseMode Mode { get; private set; } = ShowcaseMode.HardLink; + + public Task InitialiseAsync(CancellationToken cancellationToken = default) => Task.CompletedTask; + + public void Configure(ShowcaseMode mode) => Mode = mode; + + public Task BeginRunAsync(string sourceId, CancellationToken cancellationToken = default) => + Task.FromResult("run"); + + public Task CompleteRunAsync(string runId, RunSummary summary, CancellationToken cancellationToken = default) => + Task.CompletedTask; + + public Task> GetSeenAsync( + string sourceId, + IReadOnlyCollection urls, + CancellationToken cancellationToken = default + ) => + Task.FromResult>( + new Dictionary(StringComparer.Ordinal) + ); + + public Task RecordSeenAsync( + string sourceId, + string url, + SeenOutcome outcome, + int? httpStatus = null, + string? errorCode = null, + CancellationToken cancellationToken = default + ) => Task.CompletedTask; + + public Task> LoadTombstonesAsync(CancellationToken cancellationToken = default) => + Task.FromResult>(new HashSet(StringComparer.Ordinal)); + + public Task TombstoneAsync(string sha256, string? reason, CancellationToken cancellationToken = default) => + Task.FromResult(0); + + public Task StoreAsync(MediaStoreRequest request, CancellationToken cancellationToken = default) => + Task.FromResult(new CollectedItem(request.Candidate, request.Blob, CollectStatus.Stored)); + + public Task PurgeAsync(PurgeOptions options, CancellationToken cancellationToken = default) + { + Purged.Add(options.SourceId); + + return Task.FromResult(PurgeResult); + } + + public Task RebuildShowcaseAsync(string sourceId, CancellationToken cancellationToken = default) => + Task.FromResult(0); + + public Task GetStatsAsync(CancellationToken cancellationToken = default) => Task.FromResult(Stats); +}