Add the media store: content-addressed blobs, SQLite index, showcase
First half of replacing the stub text domain with a media collector. Nothing references this yet - the store is standalone and fully tested before anything depends on it. Blobs are addressed by SHA-256 and sharded two levels deep, so the same picture re-uploaded at a dozen addresses costs one file. Downloads stage in a sibling temp directory on the same volume and are promoted by rename, which is what keeps blobs/ free of truncated files: a crash leaves a stray .part that the next startup sweeps, never a half-image indistinguishable from a real one. The SQLite index holds provenance separately from content, so purging one source leaves blobs another source still references - that is what ref_count buys, and it is recomputed rather than incremented because the item upsert can replace a row pointing at a different blob. The seen_url journal deliberately outlives a purge: without that, the next run downloads again exactly what the user just deleted. Terminal outcomes are split from retryable ones so a flaky network does not permanently lose content. The showcase gives every item a dated, named path via hard links - a second name for one file, not a second file. Hard links are a filesystem privilege rather than a guarantee, so it degrades to copying and records which it achieved; the UI has to be able to admit that. Names suggested by the origin are treated as hostile: only the last path segment survives, Windows device names are pushed aside, and the extension comes from the sniffed kind, never from the remote. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
Leonid Pershin
co-authored by
Claude Opus 5
Claude Opus 5
parent
44fb0d3a5f
commit
181f974a37
@@ -0,0 +1,351 @@
|
||||
using System.Security.Cryptography;
|
||||
using System.Text;
|
||||
using AvParser.Core.Collecting;
|
||||
using AvParser.Infrastructure.Media;
|
||||
using AvParser.Infrastructure.Storage;
|
||||
using Microsoft.Extensions.Logging.Abstractions;
|
||||
|
||||
namespace AvParser.Infrastructure.Tests.Media;
|
||||
|
||||
public sealed class MediaStoreTests : IAsyncLifetime
|
||||
{
|
||||
private readonly string _root = Path.Combine(Path.GetTempPath(), "AvParserTests", Guid.NewGuid().ToString("N"));
|
||||
|
||||
private AppPaths _paths = null!;
|
||||
private SqliteMediaIndex _index = null!;
|
||||
private BlobStore _blobs = null!;
|
||||
private MediaStore _store = null!;
|
||||
|
||||
public async ValueTask InitializeAsync()
|
||||
{
|
||||
_paths = new AppPaths(_root);
|
||||
_paths.EnsureCreated();
|
||||
|
||||
_index = new SqliteMediaIndex(_paths, NullLogger<SqliteMediaIndex>.Instance);
|
||||
_blobs = new BlobStore(_paths, NullLogger<BlobStore>.Instance);
|
||||
_store = Build();
|
||||
|
||||
await _store.InitialiseAsync(TestContext.Current.CancellationToken);
|
||||
}
|
||||
|
||||
public ValueTask DisposeAsync()
|
||||
{
|
||||
_index.Dispose();
|
||||
|
||||
// Pooled SQLite connections keep the file open; without this the delete races the pool.
|
||||
Microsoft.Data.Sqlite.SqliteConnection.ClearAllPools();
|
||||
|
||||
if (Directory.Exists(_root))
|
||||
{
|
||||
try
|
||||
{
|
||||
Directory.Delete(_root, recursive: true);
|
||||
}
|
||||
catch (IOException)
|
||||
{
|
||||
// A leaked handle is not worth failing a green test over.
|
||||
}
|
||||
}
|
||||
|
||||
return ValueTask.CompletedTask;
|
||||
}
|
||||
|
||||
private MediaStore Build(LinkStrategy? hardLink = null) =>
|
||||
new(
|
||||
_index,
|
||||
_blobs,
|
||||
new ShowcaseLinker(_paths, _blobs, NullLogger<ShowcaseLinker>.Instance, hardLink),
|
||||
NullLogger<MediaStore>.Instance
|
||||
);
|
||||
|
||||
/// <summary>Writes bytes to a staged temp file and returns the blob describing them.</summary>
|
||||
private (MediaBlob Blob, string TempPath) Stage(string content, MediaKind kind = MediaKind.Png)
|
||||
{
|
||||
var bytes = Encoding.UTF8.GetBytes(content);
|
||||
var hash = Convert.ToHexStringLower(SHA256.HashData(bytes));
|
||||
var temp = _blobs.CreateTempPath();
|
||||
|
||||
File.WriteAllBytes(temp, bytes);
|
||||
|
||||
return (MediaBlob.Create(hash, kind, bytes.Length), temp);
|
||||
}
|
||||
|
||||
private async Task<CollectedItem> StoreAsync(
|
||||
string content,
|
||||
string sourceId = "url-list",
|
||||
string? url = null,
|
||||
string? suggestedName = null,
|
||||
MediaStore? store = null
|
||||
)
|
||||
{
|
||||
var (blob, temp) = Stage(content);
|
||||
var candidate = new MediaCandidate(new Uri(url ?? $"https://example.test/{content}.png"))
|
||||
{
|
||||
SourceId = sourceId,
|
||||
SuggestedName = suggestedName,
|
||||
};
|
||||
|
||||
var request = new MediaStoreRequest(candidate, blob, temp, "run-1", candidate.Url, 200);
|
||||
|
||||
return await (store ?? _store).StoreAsync(request, TestContext.Current.CancellationToken);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Storing_writes_one_blob_and_reports_it_as_new()
|
||||
{
|
||||
var item = await StoreAsync("alpha");
|
||||
|
||||
item.Status.ShouldBe(CollectStatus.Stored);
|
||||
File.Exists(_blobs.PathFor(item.Blob)).ShouldBeTrue();
|
||||
|
||||
var stats = await _store.GetStatsAsync(TestContext.Current.CancellationToken);
|
||||
stats.BlobCount.ShouldBe(1);
|
||||
stats.ItemCount.ShouldBe(1);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task The_same_bytes_at_two_addresses_produce_one_blob_and_two_provenance_rows()
|
||||
{
|
||||
// The point of hashing the content instead of the address: the web re-uploads constantly.
|
||||
var first = await StoreAsync("same", url: "https://example.test/a.png");
|
||||
var second = await StoreAsync("same", url: "https://example.test/b.png");
|
||||
|
||||
first.Status.ShouldBe(CollectStatus.Stored);
|
||||
second.Status.ShouldBe(CollectStatus.Duplicate);
|
||||
first.Blob.Sha256.ShouldBe(second.Blob.Sha256);
|
||||
|
||||
var stats = await _store.GetStatsAsync(TestContext.Current.CancellationToken);
|
||||
stats.BlobCount.ShouldBe(1);
|
||||
stats.ItemCount.ShouldBe(2);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Re_collecting_the_same_address_does_not_add_a_second_row()
|
||||
{
|
||||
await StoreAsync("alpha", url: "https://example.test/a.png");
|
||||
await StoreAsync("alpha", url: "https://example.test/a.png");
|
||||
|
||||
(await _store.GetStatsAsync(TestContext.Current.CancellationToken)).ItemCount.ShouldBe(1);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task A_stored_address_is_journalled_so_a_re_run_can_skip_it()
|
||||
{
|
||||
await StoreAsync("alpha", url: "https://example.test/a.png");
|
||||
|
||||
var seen = await _store.GetSeenAsync(
|
||||
"url-list",
|
||||
["https://example.test/a.png"],
|
||||
TestContext.Current.CancellationToken
|
||||
);
|
||||
|
||||
seen["https://example.test/a.png"].ShouldBe(SeenOutcome.Stored);
|
||||
SeenOutcomes.IsTerminal(seen["https://example.test/a.png"]).ShouldBeTrue();
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Repeated_failures_at_one_address_count_up_without_becoming_terminal()
|
||||
{
|
||||
// A failure describes the moment, not the resource; treating it as final would make a
|
||||
// flaky network permanently lose content.
|
||||
for (var attempt = 0; attempt < 3; attempt++)
|
||||
{
|
||||
await _store.RecordSeenAsync(
|
||||
"url-list",
|
||||
"https://example.test/flaky.png",
|
||||
SeenOutcome.Failed,
|
||||
cancellationToken: TestContext.Current.CancellationToken
|
||||
);
|
||||
}
|
||||
|
||||
var seen = await _store.GetSeenAsync(
|
||||
"url-list",
|
||||
["https://example.test/flaky.png"],
|
||||
TestContext.Current.CancellationToken
|
||||
);
|
||||
|
||||
seen["https://example.test/flaky.png"].ShouldBe(SeenOutcome.Failed);
|
||||
SeenOutcomes.IsTerminal(SeenOutcome.Failed).ShouldBeFalse();
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Purging_a_source_leaves_a_blob_that_another_source_still_references()
|
||||
{
|
||||
// This is exactly what the reference count buys; without it the second source would find
|
||||
// its own index rows pointing at a file that the first source's purge deleted.
|
||||
await StoreAsync("shared", sourceId: "url-list", url: "https://example.test/a.png");
|
||||
await StoreAsync("shared", sourceId: "own-service", url: "https://own.test/a.png");
|
||||
|
||||
var result = await _store.PurgeAsync(new PurgeOptions("url-list"), TestContext.Current.CancellationToken);
|
||||
|
||||
result.ItemsRemoved.ShouldBe(1);
|
||||
result.BlobsRemoved.ShouldBe(0);
|
||||
|
||||
var stats = await _store.GetStatsAsync(TestContext.Current.CancellationToken);
|
||||
stats.BlobCount.ShouldBe(1);
|
||||
stats.ItemCount.ShouldBe(1);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Purging_the_last_reference_deletes_the_file()
|
||||
{
|
||||
var item = await StoreAsync("lonely", sourceId: "url-list");
|
||||
var path = _blobs.PathFor(item.Blob);
|
||||
|
||||
var result = await _store.PurgeAsync(new PurgeOptions("url-list"), TestContext.Current.CancellationToken);
|
||||
|
||||
result.BlobsRemoved.ShouldBe(1);
|
||||
result.BytesFreed.ShouldBe(item.Blob.Length);
|
||||
File.Exists(path).ShouldBeFalse();
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Purging_keeps_the_journal_unless_asked_to_forget_it()
|
||||
{
|
||||
await StoreAsync("alpha", url: "https://example.test/a.png");
|
||||
await _store.PurgeAsync(new PurgeOptions("url-list"), TestContext.Current.CancellationToken);
|
||||
|
||||
// Deliberate: otherwise the next run downloads again exactly what was just deleted.
|
||||
var kept = await _store.GetSeenAsync(
|
||||
"url-list",
|
||||
["https://example.test/a.png"],
|
||||
TestContext.Current.CancellationToken
|
||||
);
|
||||
kept.ShouldContainKey("https://example.test/a.png");
|
||||
|
||||
await _store.PurgeAsync(
|
||||
new PurgeOptions("url-list") { ForgetSeenUrls = true },
|
||||
TestContext.Current.CancellationToken
|
||||
);
|
||||
|
||||
var forgotten = await _store.GetSeenAsync(
|
||||
"url-list",
|
||||
["https://example.test/a.png"],
|
||||
TestContext.Current.CancellationToken
|
||||
);
|
||||
forgotten.ShouldBeEmpty();
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Marking_a_placeholder_removes_every_copy_already_held()
|
||||
{
|
||||
var first = await StoreAsync("placeholder", url: "https://example.test/a.png");
|
||||
await StoreAsync("placeholder", url: "https://example.test/b.png");
|
||||
var path = _blobs.PathFor(first.Blob);
|
||||
|
||||
var removed = await _store.TombstoneAsync(
|
||||
first.Blob.Sha256,
|
||||
"dead link image",
|
||||
TestContext.Current.CancellationToken
|
||||
);
|
||||
|
||||
removed.ShouldBe(2);
|
||||
File.Exists(path).ShouldBeFalse();
|
||||
(await _store.LoadTombstonesAsync(TestContext.Current.CancellationToken)).ShouldContain(first.Blob.Sha256);
|
||||
|
||||
// And the addresses are remembered as placeholders, so a re-run does not fetch them again.
|
||||
var seen = await _store.GetSeenAsync(
|
||||
"url-list",
|
||||
["https://example.test/a.png"],
|
||||
TestContext.Current.CancellationToken
|
||||
);
|
||||
seen["https://example.test/a.png"].ShouldBe(SeenOutcome.Placeholder);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task A_showcase_entry_is_the_same_content_and_deleting_it_leaves_the_blob()
|
||||
{
|
||||
var item = await StoreAsync("linked", suggestedName: "kitten");
|
||||
|
||||
item.ShowcasePath.ShouldNotBeNull();
|
||||
var showcasePath = Path.Combine(
|
||||
_paths.ShowcaseDirectory,
|
||||
item.ShowcasePath!.Replace('/', Path.DirectorySeparatorChar)
|
||||
);
|
||||
|
||||
File.ReadAllBytes(showcasePath).ShouldBe(File.ReadAllBytes(_blobs.PathFor(item.Blob)));
|
||||
item.ShowcasePath.ShouldContain("kitten");
|
||||
|
||||
File.Delete(showcasePath);
|
||||
File.Exists(_blobs.PathFor(item.Blob)).ShouldBeTrue();
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task A_filesystem_without_hard_links_falls_back_to_copying()
|
||||
{
|
||||
// Forced, because the developer's own disk supports hard links and this path would
|
||||
// otherwise only ever run on a user's exFAT drive or network share.
|
||||
var store = Build(
|
||||
(string _, string _, out string? error) =>
|
||||
{
|
||||
error = "cross-volume";
|
||||
return false;
|
||||
}
|
||||
);
|
||||
|
||||
var item = await StoreAsync("copied", store: store);
|
||||
|
||||
item.ShowcasePath.ShouldNotBeNull();
|
||||
var showcasePath = Path.Combine(
|
||||
_paths.ShowcaseDirectory,
|
||||
item.ShowcasePath!.Replace('/', Path.DirectorySeparatorChar)
|
||||
);
|
||||
File.Exists(showcasePath).ShouldBeTrue();
|
||||
File.ReadAllBytes(showcasePath).ShouldBe(File.ReadAllBytes(_blobs.PathFor(item.Blob)));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task The_showcase_can_be_rebuilt_after_the_user_deletes_it()
|
||||
{
|
||||
await StoreAsync("one", url: "https://example.test/1.png", suggestedName: "one");
|
||||
await StoreAsync("two", url: "https://example.test/2.png", suggestedName: "two");
|
||||
|
||||
Directory.Delete(_paths.ShowcaseDirectory, recursive: true);
|
||||
|
||||
var linked = await _store.RebuildShowcaseAsync("url-list", TestContext.Current.CancellationToken);
|
||||
|
||||
linked.ShouldBe(2);
|
||||
Directory.EnumerateFiles(_paths.ShowcaseDirectory, "*", SearchOption.AllDirectories).Count().ShouldBe(2);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Reference_counts_do_not_drift()
|
||||
{
|
||||
await StoreAsync("a", url: "https://example.test/1.png");
|
||||
await StoreAsync("a", url: "https://example.test/2.png");
|
||||
await StoreAsync("b", url: "https://example.test/3.png");
|
||||
await _store.PurgeAsync(new PurgeOptions("nobody"), TestContext.Current.CancellationToken);
|
||||
|
||||
(await _index.VerifyReferenceCountsAsync(TestContext.Current.CancellationToken)).ShouldBe(0);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task A_run_records_what_it_did()
|
||||
{
|
||||
var runId = await _store.BeginRunAsync("url-list", TestContext.Current.CancellationToken);
|
||||
|
||||
runId.ShouldNotBeNullOrWhiteSpace();
|
||||
|
||||
await _store.CompleteRunAsync(
|
||||
runId,
|
||||
new RunSummary(10, 7, 2, 1, 4096, Cancelled: false),
|
||||
TestContext.Current.CancellationToken
|
||||
);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task An_unfinished_download_from_a_previous_process_is_swept()
|
||||
{
|
||||
var stale = _blobs.CreateTempPath();
|
||||
await File.WriteAllTextAsync(stale, "half a picture", TestContext.Current.CancellationToken);
|
||||
File.SetLastWriteTimeUtc(stale, DateTime.UtcNow.AddDays(-1));
|
||||
|
||||
var fresh = _blobs.CreateTempPath();
|
||||
await File.WriteAllTextAsync(fresh, "in progress right now", TestContext.Current.CancellationToken);
|
||||
|
||||
_blobs.SweepTemp(TimeSpan.FromHours(6)).ShouldBe(1);
|
||||
|
||||
File.Exists(stale).ShouldBeFalse();
|
||||
File.Exists(fresh).ShouldBeTrue();
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,76 @@
|
||||
using AvParser.Infrastructure.Media;
|
||||
|
||||
namespace AvParser.Infrastructure.Tests.Media;
|
||||
|
||||
public class ShowcaseNameTests
|
||||
{
|
||||
private const string Hash = "a3f19b7c5d2e8f0142536475869708192a3b4c5d6e7f8091a2b3c4d5e6f70819";
|
||||
|
||||
[Fact]
|
||||
public void A_plain_name_is_kept()
|
||||
{
|
||||
ShowcaseLinker.SanitiseName("kitten", Hash).ShouldBe("kitten");
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void An_absent_name_falls_back_to_the_hash()
|
||||
{
|
||||
ShowcaseLinker.SanitiseName(null, Hash).ShouldBe(Hash[..12]);
|
||||
ShowcaseLinker.SanitiseName(" ", Hash).ShouldBe(Hash[..12]);
|
||||
}
|
||||
|
||||
[Theory]
|
||||
[InlineData("../../etc/passwd")]
|
||||
[InlineData("..\\..\\windows\\system32\\config")]
|
||||
[InlineData("/absolute/path/name")]
|
||||
public void A_name_cannot_walk_out_of_its_directory(string suggested)
|
||||
{
|
||||
// The suggestion comes from a remote server, so it is treated as hostile rather than
|
||||
// merely untidy: only the last segment survives, and separators never do.
|
||||
var name = ShowcaseLinker.SanitiseName(suggested, Hash);
|
||||
|
||||
name.ShouldNotContain("/");
|
||||
name.ShouldNotContain("\\");
|
||||
name.ShouldNotContain("..");
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void The_extension_the_origin_suggested_is_dropped()
|
||||
{
|
||||
// The extension is derived from the sniffed kind, never from what the origin claimed.
|
||||
ShowcaseLinker.SanitiseName("photo.exe", Hash).ShouldBe("photo");
|
||||
}
|
||||
|
||||
[Theory]
|
||||
[InlineData("CON")]
|
||||
[InlineData("nul")]
|
||||
[InlineData("COM1")]
|
||||
[InlineData("lpt9")]
|
||||
public void Windows_device_names_are_pushed_out_of_the_way(string reserved)
|
||||
{
|
||||
ShowcaseLinker.SanitiseName(reserved, Hash).ShouldBe($"_{reserved}");
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void An_over_long_name_is_trimmed()
|
||||
{
|
||||
var name = ShowcaseLinker.SanitiseName(new string('x', 400), Hash);
|
||||
|
||||
name.Length.ShouldBeLessThanOrEqualTo(80);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void A_name_of_nothing_but_junk_falls_back_to_the_hash()
|
||||
{
|
||||
// Dots rather than, say, '?': the set of invalid filename characters differs by platform,
|
||||
// but a name that trims away to nothing does so everywhere.
|
||||
ShowcaseLinker.SanitiseName("...", Hash).ShouldBe(Hash[..12]);
|
||||
ShowcaseLinker.SanitiseName("\t\t", Hash).ShouldBe(Hash[..12]);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Control_characters_are_replaced_but_spaces_survive()
|
||||
{
|
||||
ShowcaseLinker.SanitiseName("a b\tc", Hash).ShouldBe("a b_c");
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user