Add the media store: content-addressed blobs, SQLite index, showcase
First half of replacing the stub text domain with a media collector. Nothing references this yet - the store is standalone and fully tested before anything depends on it. Blobs are addressed by SHA-256 and sharded two levels deep, so the same picture re-uploaded at a dozen addresses costs one file. Downloads stage in a sibling temp directory on the same volume and are promoted by rename, which is what keeps blobs/ free of truncated files: a crash leaves a stray .part that the next startup sweeps, never a half-image indistinguishable from a real one. The SQLite index holds provenance separately from content, so purging one source leaves blobs another source still references - that is what ref_count buys, and it is recomputed rather than incremented because the item upsert can replace a row pointing at a different blob. The seen_url journal deliberately outlives a purge: without that, the next run downloads again exactly what the user just deleted. Terminal outcomes are split from retryable ones so a flaky network does not permanently lose content. The showcase gives every item a dated, named path via hard links - a second name for one file, not a second file. Hard links are a filesystem privilege rather than a guarantee, so it degrades to copying and records which it achieved; the UI has to be able to admit that. Names suggested by the origin are treated as hostile: only the last path segment survives, Windows device names are pushed aside, and the extension comes from the sniffed kind, never from the remote. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 5
parent
44fb0d3a5f
commit
181f974a37
@@ -0,0 +1,209 @@
|
||||
using AvParser.Core.Collecting;
|
||||
using AvParser.Infrastructure.Storage;
|
||||
using Microsoft.Extensions.Logging;
|
||||
|
||||
namespace AvParser.Infrastructure.Media;
|
||||
|
||||
/// <summary>Whether a promotion actually wrote anything.</summary>
|
||||
public enum BlobPromotion
|
||||
{
|
||||
/// <summary>The file was moved into the store; these bytes were new.</summary>
|
||||
Written = 0,
|
||||
|
||||
/// <summary>The content was already held, so the temporary file was discarded.</summary>
|
||||
AlreadyPresent = 1,
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// The files on disk: content-addressed, sharded, and written only once complete.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// The invariant this type exists to hold is that <c>blobs/</c> contains nothing but complete,
|
||||
/// hashed content. Downloads are staged in a sibling temporary directory on the same volume and
|
||||
/// promoted by rename, so a crash mid-download leaves a stray temporary file rather than a
|
||||
/// truncated image that would then be indistinguishable from a real one for ever.
|
||||
/// </remarks>
|
||||
public sealed class BlobStore(IAppPaths paths, ILogger<BlobStore> logger)
|
||||
{
|
||||
private readonly IAppPaths _paths = paths ?? throw new ArgumentNullException(nameof(paths));
|
||||
private readonly ILogger<BlobStore> _logger = logger ?? throw new ArgumentNullException(nameof(logger));
|
||||
|
||||
/// <summary>Absolute path this content would live at.</summary>
|
||||
public string PathFor(MediaBlob blob)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(blob);
|
||||
|
||||
return Path.Combine(_paths.BlobDirectory, blob.RelativePath);
|
||||
}
|
||||
|
||||
/// <summary>Absolute path for a hash and extension already known.</summary>
|
||||
public string PathFor(string sha256, string extension)
|
||||
{
|
||||
ArgumentException.ThrowIfNullOrWhiteSpace(sha256);
|
||||
|
||||
return Path.Combine(
|
||||
_paths.BlobDirectory,
|
||||
sha256[..2],
|
||||
sha256.Substring(2, 2),
|
||||
string.Concat(sha256, extension)
|
||||
);
|
||||
}
|
||||
|
||||
/// <summary>Whether this content is already held.</summary>
|
||||
public bool Exists(MediaBlob blob) => File.Exists(PathFor(blob));
|
||||
|
||||
/// <summary>Reserves a path for an in-progress download.</summary>
|
||||
/// <remarks>
|
||||
/// The <c>.part</c> suffix is load-bearing for the sweep below: it is how a leftover from a
|
||||
/// killed process is told apart from a file some other part of the app is using.
|
||||
/// </remarks>
|
||||
public string CreateTempPath()
|
||||
{
|
||||
Directory.CreateDirectory(_paths.MediaTempDirectory);
|
||||
|
||||
return Path.Combine(_paths.MediaTempDirectory, $"{Guid.NewGuid():N}.part");
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Takes ownership of a completed temporary file and moves it into the store.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// The temporary file is consumed either way: moved when the content is new, deleted when it
|
||||
/// was already held. Callers must not touch it afterwards.
|
||||
/// </remarks>
|
||||
public BlobPromotion Promote(MediaBlob blob, string tempFilePath)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(blob);
|
||||
ArgumentException.ThrowIfNullOrWhiteSpace(tempFilePath);
|
||||
|
||||
var target = PathFor(blob);
|
||||
|
||||
if (File.Exists(target))
|
||||
{
|
||||
// Identical hash means identical bytes; there is nothing to compare and nothing to write.
|
||||
TryDelete(tempFilePath);
|
||||
return BlobPromotion.AlreadyPresent;
|
||||
}
|
||||
|
||||
Directory.CreateDirectory(Path.GetDirectoryName(target)!);
|
||||
|
||||
try
|
||||
{
|
||||
// Deliberately without overwrite: if a concurrent worker just wrote the same content,
|
||||
// its file is as good as ours and clobbering it would break any reader mid-stream.
|
||||
File.Move(tempFilePath, target);
|
||||
return BlobPromotion.Written;
|
||||
}
|
||||
catch (IOException) when (File.Exists(target))
|
||||
{
|
||||
TryDelete(tempFilePath);
|
||||
return BlobPromotion.AlreadyPresent;
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>Removes content from disk. Missing is success — the caller wants it gone.</summary>
|
||||
public bool Delete(string sha256, string extension)
|
||||
{
|
||||
var path = PathFor(sha256, extension);
|
||||
|
||||
if (!File.Exists(path))
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
if (!TryDelete(path))
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
TrimEmptyShards(Path.GetDirectoryName(path));
|
||||
return true;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Deletes temporary files left behind by a previous process.
|
||||
/// </summary>
|
||||
/// <returns>How many were removed.</returns>
|
||||
/// <remarks>
|
||||
/// Only files older than <paramref name="olderThan"/> are touched, so a sweep at startup
|
||||
/// cannot delete a download that another instance of the app is running right now.
|
||||
/// </remarks>
|
||||
public int SweepTemp(TimeSpan olderThan)
|
||||
{
|
||||
if (!Directory.Exists(_paths.MediaTempDirectory))
|
||||
{
|
||||
return 0;
|
||||
}
|
||||
|
||||
var cutoff = DateTime.UtcNow - olderThan;
|
||||
var removed = 0;
|
||||
|
||||
foreach (var file in Directory.EnumerateFiles(_paths.MediaTempDirectory, "*.part"))
|
||||
{
|
||||
try
|
||||
{
|
||||
if (File.GetLastWriteTimeUtc(file) < cutoff && TryDelete(file))
|
||||
{
|
||||
removed++;
|
||||
}
|
||||
}
|
||||
catch (Exception ex) when (ex is IOException or UnauthorizedAccessException)
|
||||
{
|
||||
// Being unable to tidy up is not a reason to fail whatever asked for the sweep.
|
||||
}
|
||||
}
|
||||
|
||||
if (removed > 0)
|
||||
{
|
||||
_logger.LogInformation("Removed {Count} unfinished download(s) from a previous run", removed);
|
||||
}
|
||||
|
||||
return removed;
|
||||
}
|
||||
|
||||
private bool TryDelete(string path)
|
||||
{
|
||||
try
|
||||
{
|
||||
File.Delete(path);
|
||||
return true;
|
||||
}
|
||||
catch (Exception ex) when (ex is IOException or UnauthorizedAccessException)
|
||||
{
|
||||
_logger.LogWarning(ex, "Could not delete {Path}", path);
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>Removes the two shard directories once the last file in them is gone.</summary>
|
||||
private void TrimEmptyShards(string? leaf)
|
||||
{
|
||||
for (var directory = leaf; directory is not null; directory = Path.GetDirectoryName(directory))
|
||||
{
|
||||
if (
|
||||
string.Equals(
|
||||
Path.GetFullPath(directory).TrimEnd(Path.DirectorySeparatorChar),
|
||||
Path.GetFullPath(_paths.BlobDirectory).TrimEnd(Path.DirectorySeparatorChar),
|
||||
StringComparison.OrdinalIgnoreCase
|
||||
)
|
||||
)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
try
|
||||
{
|
||||
if (Directory.EnumerateFileSystemEntries(directory).Any())
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
Directory.Delete(directory);
|
||||
}
|
||||
catch (Exception ex) when (ex is IOException or UnauthorizedAccessException)
|
||||
{
|
||||
return;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user