Add collector settings and per-source purge

Every limit the fetcher was using was a constant. They are settings now, and
CollectOptions became the single place policy lives: AppSettings.ToCollectOptions
clamps them, and the HTTP layer's FetchOptions is projected from that. One
clamping site rather than two sets of ceilings drifting apart.

Clamping rather than validating, for the reason the proxy options already do it:
a hand-edited file must not stop the app from starting. A MaxItemBytes edited to
zero would otherwise refuse everything, and a zeroed concurrency would deadlock
the run outright - so both are pulled into range instead. An empty format filter
is read as "everything", because switching every format off is far more likely
to be a slip than an instruction to collect nothing.

The media root has an ordering problem - it is a setting that decides the paths
the container is built from - so the file is read once before the container
exists rather than making every path lazy for one value.

Purge is scoped to a source and lives on the Collect page, where the source is
already chosen. Content another source also holds survives, which is what the
index's reference count was for.

The showcase hint says out loud what a hard link means: editing the browsable
copy edits the original, and deleting it frees nothing until the last name goes.
That is surprising enough to belong in the UI rather than only in the code.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
Leonid Pershin
2026-08-13 21:52:43 +03:00
co-authored by Claude Opus 5
parent 70fb3a1df3
commit fe62bcf53f
21 changed files with 834 additions and 26 deletions
@@ -70,6 +70,7 @@ public class CollectViewTests
new FakeSettingsService(new AppSettings { LastSourceId = networkSource ? "own-service" : "url-list" }),
new ProxyPool([], new FakeProxyProbe(), new ProxyOptions()),
new IdleRunner(),
new FakeMediaStore(),
new EmptyServiceProvider(),
NullLogger<CollectViewModel>.Instance,
ImmediateSequencer.Instance
@@ -0,0 +1,64 @@
using AvParser.Core.Collecting;
namespace AvParser.UI.HeadlessTests;
/// <summary>A store that records what it was asked to do and holds nothing.</summary>
internal sealed class FakeMediaStore : IMediaStore
{
public List<string> Purged { get; } = [];
public MediaStoreStats Stats { get; set; } = new(0, 0, 0, 0);
public PurgeResult PurgeResult { get; set; } = new(3, 2, 4096);
public ShowcaseMode Mode { get; private set; } = ShowcaseMode.HardLink;
public Task InitialiseAsync(CancellationToken cancellationToken = default) => Task.CompletedTask;
public void Configure(ShowcaseMode mode) => Mode = mode;
public Task<string> BeginRunAsync(string sourceId, CancellationToken cancellationToken = default) =>
Task.FromResult("run");
public Task CompleteRunAsync(string runId, RunSummary summary, CancellationToken cancellationToken = default) =>
Task.CompletedTask;
public Task<IReadOnlyDictionary<string, SeenOutcome>> GetSeenAsync(
string sourceId,
IReadOnlyCollection<string> urls,
CancellationToken cancellationToken = default
) =>
Task.FromResult<IReadOnlyDictionary<string, SeenOutcome>>(
new Dictionary<string, SeenOutcome>(StringComparer.Ordinal)
);
public Task RecordSeenAsync(
string sourceId,
string url,
SeenOutcome outcome,
int? httpStatus = null,
string? errorCode = null,
CancellationToken cancellationToken = default
) => Task.CompletedTask;
public Task<IReadOnlySet<string>> LoadTombstonesAsync(CancellationToken cancellationToken = default) =>
Task.FromResult<IReadOnlySet<string>>(new HashSet<string>(StringComparer.Ordinal));
public Task<int> TombstoneAsync(string sha256, string? reason, CancellationToken cancellationToken = default) =>
Task.FromResult(0);
public Task<CollectedItem> StoreAsync(MediaStoreRequest request, CancellationToken cancellationToken = default) =>
Task.FromResult(new CollectedItem(request.Candidate, request.Blob, CollectStatus.Stored));
public Task<PurgeResult> PurgeAsync(PurgeOptions options, CancellationToken cancellationToken = default)
{
Purged.Add(options.SourceId);
return Task.FromResult(PurgeResult);
}
public Task<int> RebuildShowcaseAsync(string sourceId, CancellationToken cancellationToken = default) =>
Task.FromResult(0);
public Task<MediaStoreStats> GetStatsAsync(CancellationToken cancellationToken = default) => Task.FromResult(Stats);
}