Add the download pipeline: sniffing, redirects, throttling, verdicts
Second half of the collector foundation. Still nothing in the app references it; the pipeline is tested end to end against a deliberately badly behaved loopback server before anything depends on it. Types come from the bytes, never from the URL, the extension or Content-Type - two of those three are chosen by whoever serves the file, and a host must not get to pick the extension of a file written to the user's disk. Animation is a separate question from kind: GIF89a proves nothing without a second image descriptor, and a PNG is an APNG only if acTL precedes the first IDAT, so both are walked properly rather than guessed. Timeouts are split three ways because HttpClient.Timeout covers the whole response: any value large enough for a 30 MB file is also large enough for a dead connection to hang on. Connect, headers and a per-read idle deadline let both be strict. Redirects are followed by hand since the shared proxy handler disables them, which is what allows a hop cap, loop detection and refusing a jump to a data: URL. The lease verdict is a pure function, because ProxyLease's constructor is internal to the domain and no test can fabricate one. Its rule is that the verdict describes the transport, not the resource: a 404 is a working proxy, and so is a 429 - blaming the proxy for an origin's rate limit would make the pool rotate away from a good address in response to being asked to slow down. Cancellation reports nothing at all. Throttling exists to be obeyed. It is raised only by the host's own 429 and 503, and never by rotating to another proxy. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
Leonid Pershin
co-authored by
Claude Opus 5
Claude Opus 5
parent
181f974a37
commit
1742c094e9
@@ -0,0 +1,159 @@
|
||||
using System.Net;
|
||||
using System.Net.Sockets;
|
||||
using System.Security.Authentication;
|
||||
using AvParser.Infrastructure.Collecting;
|
||||
|
||||
namespace AvParser.Infrastructure.Tests.Collecting;
|
||||
|
||||
/// <summary>
|
||||
/// The verdict table, tested as a pure function.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// It has to be a pure function to be testable at all: <c>ProxyLease</c>'s constructor is internal
|
||||
/// to <c>AvParser.Core</c>, which declares no <c>InternalsVisibleTo</c>, so nothing outside the
|
||||
/// domain assembly can fabricate one to assert against.
|
||||
/// </remarks>
|
||||
public class LeaseVerdictTests
|
||||
{
|
||||
[Theory]
|
||||
[InlineData(200)]
|
||||
[InlineData(204)]
|
||||
public void A_delivered_response_is_a_working_proxy(int status) =>
|
||||
Decide(status).Verdict.ShouldBe(LeaseVerdict.Success);
|
||||
|
||||
[Theory]
|
||||
[InlineData(400)]
|
||||
[InlineData(403)]
|
||||
[InlineData(404)]
|
||||
[InlineData(410)]
|
||||
[InlineData(451)]
|
||||
[InlineData(500)]
|
||||
[InlineData(503)]
|
||||
public void The_resource_being_broken_is_not_the_proxy_being_broken(int status)
|
||||
{
|
||||
// A dead link is the normal case on a free list. Recording it against the proxy would
|
||||
// quarantine working proxies at exactly the rate that dead links appear.
|
||||
Decide(status).Verdict.ShouldBe(LeaseVerdict.Success);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void A_rate_limit_is_never_the_proxy_s_fault()
|
||||
{
|
||||
// And treating it as one would make the pool rotate away from a working address in
|
||||
// response to being asked politely to slow down.
|
||||
Decide(429).Verdict.ShouldBe(LeaseVerdict.Success);
|
||||
}
|
||||
|
||||
[Theory]
|
||||
[InlineData(407, "proxy auth")]
|
||||
[InlineData(502, "bad gateway")]
|
||||
[InlineData(504, "gateway timeout")]
|
||||
public void Statuses_the_proxy_itself_produces_count_against_it(int status, string reason)
|
||||
{
|
||||
var outcome = Decide(status);
|
||||
|
||||
outcome.Verdict.ShouldBe(LeaseVerdict.Failure);
|
||||
outcome.Reason.ShouldBe(reason);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Cancellation_teaches_the_pool_nothing()
|
||||
{
|
||||
// The deliberate neutral case: pressing Stop must not quarantine healthy proxies.
|
||||
LeaseVerdicts.Decide(null, null, bytesReceived: true, cancelled: true).Verdict.ShouldBe(LeaseVerdict.Neutral);
|
||||
LeaseVerdicts
|
||||
.Decide(HttpStatusCode.OK, new OperationCanceledException(), true, false)
|
||||
.Verdict.ShouldBe(LeaseVerdict.Neutral);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void No_response_at_all_is_a_failure() =>
|
||||
LeaseVerdicts.Decide(null, null, false, false).Verdict.ShouldBe(LeaseVerdict.Failure);
|
||||
|
||||
[Fact]
|
||||
public void A_refused_connection_is_a_failure()
|
||||
{
|
||||
var failure = new HttpRequestException("boom", new SocketException((int)SocketError.ConnectionRefused));
|
||||
var outcome = LeaseVerdicts.Decide(null, failure, false, false);
|
||||
|
||||
outcome.Verdict.ShouldBe(LeaseVerdict.Failure);
|
||||
outcome.Reason.ShouldBe("refused");
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void A_tls_handshake_failure_names_itself()
|
||||
{
|
||||
var outcome = LeaseVerdicts.Decide(null, new AuthenticationException("no"), false, false);
|
||||
|
||||
outcome.Verdict.ShouldBe(LeaseVerdict.Failure);
|
||||
outcome.Reason.ShouldBe("tls");
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void A_timeout_before_any_bytes_reads_differently_from_one_after()
|
||||
{
|
||||
LeaseVerdicts.Decide(null, new TimeoutException(), false, false).Reason.ShouldBe("timeout");
|
||||
LeaseVerdicts.Decide(null, new TimeoutException(), true, false).Reason.ShouldBe("stalled");
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void A_body_cut_short_counts_against_the_proxy()
|
||||
{
|
||||
var outcome = LeaseVerdicts.Decide(HttpStatusCode.OK, new IOException("short"), true, false);
|
||||
|
||||
outcome.Verdict.ShouldBe(LeaseVerdict.Failure);
|
||||
outcome.Reason.ShouldBe("truncated");
|
||||
}
|
||||
|
||||
private static LeaseOutcome Decide(int status) =>
|
||||
LeaseVerdicts.Decide((HttpStatusCode)status, null, bytesReceived: true, cancelled: false);
|
||||
}
|
||||
|
||||
public class RetryAfterTests
|
||||
{
|
||||
private static readonly DateTimeOffset Now = new(2026, 8, 13, 12, 0, 0, TimeSpan.Zero);
|
||||
|
||||
[Fact]
|
||||
public void Delta_seconds_are_read()
|
||||
{
|
||||
RetryAfter.Parse("30", Now, out var clamped).ShouldBe(TimeSpan.FromSeconds(30));
|
||||
clamped.ShouldBeFalse();
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void An_http_date_is_read_relative_to_now() =>
|
||||
RetryAfter.Parse(Now.AddSeconds(45).ToString("r"), Now, out _).ShouldBe(TimeSpan.FromSeconds(45));
|
||||
|
||||
[Fact]
|
||||
public void An_absurd_wait_is_pulled_back_into_range()
|
||||
{
|
||||
// A date measured against a skewed server clock otherwise turns a short pause into
|
||||
// something indistinguishable from a hang.
|
||||
RetryAfter.Parse("86400", Now, out var clamped).ShouldBe(RetryAfter.Maximum);
|
||||
clamped.ShouldBeTrue();
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void A_date_already_in_the_past_still_pauses_briefly()
|
||||
{
|
||||
RetryAfter.Parse(Now.AddMinutes(-5).ToString("r"), Now, out _).ShouldBe(RetryAfter.Minimum);
|
||||
RetryAfter.Parse("0", Now, out _).ShouldBe(RetryAfter.Minimum);
|
||||
RetryAfter.Parse("-30", Now, out _).ShouldBe(RetryAfter.Minimum);
|
||||
}
|
||||
|
||||
[Theory]
|
||||
[InlineData(null)]
|
||||
[InlineData("")]
|
||||
[InlineData(" ")]
|
||||
[InlineData("soon")]
|
||||
public void Nonsense_reads_as_absent(string? value) => RetryAfter.Parse(value, Now, out _).ShouldBeNull();
|
||||
|
||||
[Fact]
|
||||
public void Backoff_grows_and_then_stops_growing()
|
||||
{
|
||||
RetryAfter.Backoff(1).ShouldBe(TimeSpan.FromSeconds(2));
|
||||
RetryAfter.Backoff(2).ShouldBe(TimeSpan.FromSeconds(4));
|
||||
RetryAfter.Backoff(3).ShouldBe(TimeSpan.FromSeconds(8));
|
||||
RetryAfter.Backoff(20).ShouldBe(RetryAfter.Maximum);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,344 @@
|
||||
using System.Collections.Concurrent;
|
||||
using System.Globalization;
|
||||
using System.IO.Compression;
|
||||
using System.Net;
|
||||
using System.Net.Sockets;
|
||||
using System.Text;
|
||||
|
||||
namespace AvParser.Infrastructure.Tests.Collecting;
|
||||
|
||||
/// <summary>One request as the server saw it.</summary>
|
||||
/// <param name="Path">Requested path.</param>
|
||||
/// <param name="Headers">Request headers, lower-cased keys.</param>
|
||||
/// <param name="ArrivedUtc">When the request line was read.</param>
|
||||
/// <param name="CompletedUtc">When the response finished being written.</param>
|
||||
internal sealed record RecordedRequest(
|
||||
string Path,
|
||||
IReadOnlyDictionary<string, string> Headers,
|
||||
DateTimeOffset ArrivedUtc,
|
||||
DateTimeOffset CompletedUtc
|
||||
);
|
||||
|
||||
/// <summary>How the server should answer one path.</summary>
|
||||
internal sealed record Reply
|
||||
{
|
||||
/// <summary>Status code to send.</summary>
|
||||
public int Status { get; init; } = 200;
|
||||
|
||||
/// <summary>Body bytes. Ignored for statuses that carry none.</summary>
|
||||
public byte[] Body { get; init; } = [];
|
||||
|
||||
/// <summary>Value for the <c>Content-Type</c> header, if any.</summary>
|
||||
public string? ContentType { get; init; }
|
||||
|
||||
/// <summary>Extra headers, sent verbatim.</summary>
|
||||
public IReadOnlyDictionary<string, string> Headers { get; init; } =
|
||||
new Dictionary<string, string>(StringComparer.OrdinalIgnoreCase);
|
||||
|
||||
/// <summary>Send the body with chunked transfer encoding instead of a length.</summary>
|
||||
public bool Chunked { get; init; }
|
||||
|
||||
/// <summary>Compress the body and declare <c>Content-Encoding: gzip</c>.</summary>
|
||||
public bool Gzip { get; init; }
|
||||
|
||||
/// <summary>Declare this length instead of the real one — for testing truncation.</summary>
|
||||
public long? DeclaredLength { get; init; }
|
||||
|
||||
/// <summary>Write only this many body bytes, then close the connection mid-transfer.</summary>
|
||||
public int? TruncateAfter { get; init; }
|
||||
|
||||
/// <summary>Wait this long before sending the status line.</summary>
|
||||
public TimeSpan HeaderDelay { get; init; }
|
||||
|
||||
/// <summary>Wait this long between body writes.</summary>
|
||||
public TimeSpan BodyDelay { get; init; }
|
||||
|
||||
/// <summary>Bytes per body write, when dripping.</summary>
|
||||
public int BodyChunkSize { get; init; } = int.MaxValue;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// A deliberately badly behaved HTTP server on loopback.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// <para>
|
||||
/// Raw sockets rather than <c>HttpListener</c> or Kestrel, because most of what needs testing is
|
||||
/// below the level either of those will let you reach: declaring four kilobytes and sending three
|
||||
/// hundred bytes, dripping one byte at a time, or closing mid-chunk. A well-behaved server cannot
|
||||
/// produce the failures a collector has to survive.
|
||||
/// </para>
|
||||
/// <para>
|
||||
/// Port 0 lets the OS assign a free port, so test classes never collide and nothing has to be
|
||||
/// reserved or cleaned up between runs.
|
||||
/// </para>
|
||||
/// </remarks>
|
||||
internal sealed class LoopbackServer : IAsyncDisposable
|
||||
{
|
||||
private readonly TcpListener _listener;
|
||||
private readonly CancellationTokenSource _shutdown = new();
|
||||
private readonly ConcurrentDictionary<string, Func<Reply>> _routes = new(StringComparer.Ordinal);
|
||||
private readonly ConcurrentBag<RecordedRequest> _requests = [];
|
||||
private readonly Task _acceptLoop;
|
||||
|
||||
public LoopbackServer()
|
||||
{
|
||||
_listener = new TcpListener(IPAddress.Loopback, 0);
|
||||
_listener.Start();
|
||||
|
||||
var port = ((IPEndPoint)_listener.LocalEndpoint).Port;
|
||||
BaseAddress = new Uri($"http://127.0.0.1:{port.ToString(CultureInfo.InvariantCulture)}/");
|
||||
|
||||
_acceptLoop = Task.Run(AcceptAsync);
|
||||
}
|
||||
|
||||
/// <summary>Root of this server.</summary>
|
||||
public Uri BaseAddress { get; }
|
||||
|
||||
/// <summary>Every request served, in no particular order.</summary>
|
||||
public IReadOnlyCollection<RecordedRequest> Requests => [.. _requests];
|
||||
|
||||
/// <summary>How many requests were served.</summary>
|
||||
public int RequestCount => _requests.Count;
|
||||
|
||||
/// <summary>Registers a fixed reply for a path.</summary>
|
||||
public Uri Map(string path, Reply reply) => Map(path, () => reply);
|
||||
|
||||
/// <summary>Registers a reply computed per request, for sequences.</summary>
|
||||
public Uri Map(string path, Func<Reply> reply)
|
||||
{
|
||||
_routes[path] = reply;
|
||||
|
||||
return new Uri(BaseAddress, path);
|
||||
}
|
||||
|
||||
/// <summary>Registers a body served as-is with a 200.</summary>
|
||||
public Uri MapBody(string path, byte[] body, string? contentType = null) =>
|
||||
Map(path, new Reply { Body = body, ContentType = contentType });
|
||||
|
||||
/// <inheritdoc />
|
||||
public async ValueTask DisposeAsync()
|
||||
{
|
||||
await _shutdown.CancelAsync();
|
||||
_listener.Stop();
|
||||
|
||||
try
|
||||
{
|
||||
await _acceptLoop;
|
||||
}
|
||||
catch (Exception ex) when (ex is OperationCanceledException or SocketException or ObjectDisposedException)
|
||||
{
|
||||
// Shutting down a listener out from under its accept loop is the normal way to stop it.
|
||||
}
|
||||
|
||||
_shutdown.Dispose();
|
||||
}
|
||||
|
||||
private async Task AcceptAsync()
|
||||
{
|
||||
while (!_shutdown.IsCancellationRequested)
|
||||
{
|
||||
TcpClient client;
|
||||
|
||||
try
|
||||
{
|
||||
client = await _listener.AcceptTcpClientAsync(_shutdown.Token);
|
||||
}
|
||||
catch (Exception ex) when (ex is OperationCanceledException or SocketException or ObjectDisposedException)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
_ = Task.Run(() => ServeAsync(client));
|
||||
}
|
||||
}
|
||||
|
||||
private async Task ServeAsync(TcpClient client)
|
||||
{
|
||||
using (client)
|
||||
{
|
||||
try
|
||||
{
|
||||
await using var stream = client.GetStream();
|
||||
|
||||
var (path, headers) = await ReadRequestAsync(stream);
|
||||
var arrived = DateTimeOffset.UtcNow;
|
||||
|
||||
try
|
||||
{
|
||||
var reply = _routes.TryGetValue(path, out var factory)
|
||||
? factory()
|
||||
: new Reply { Status = 404, Body = "not found"u8.ToArray() };
|
||||
|
||||
if (reply.HeaderDelay > TimeSpan.Zero)
|
||||
{
|
||||
await Task.Delay(reply.HeaderDelay, _shutdown.Token);
|
||||
}
|
||||
|
||||
await WriteReplyAsync(stream, reply);
|
||||
}
|
||||
finally
|
||||
{
|
||||
// Recorded even when the client hung up mid-response: several tests arrange
|
||||
// exactly that, and they still need to know the request arrived.
|
||||
_requests.Add(new RecordedRequest(path, headers, arrived, DateTimeOffset.UtcNow));
|
||||
}
|
||||
}
|
||||
catch (Exception ex) when (ex is IOException or OperationCanceledException or SocketException)
|
||||
{
|
||||
// A client that hangs up mid-response is exactly what several tests arrange.
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private static async Task<(string Path, Dictionary<string, string> Headers)> ReadRequestAsync(NetworkStream stream)
|
||||
{
|
||||
var buffer = new List<byte>(1024);
|
||||
var single = new byte[1];
|
||||
|
||||
while (buffer.Count < 16 * 1024)
|
||||
{
|
||||
var read = await stream.ReadAsync(single);
|
||||
|
||||
if (read == 0)
|
||||
{
|
||||
break;
|
||||
}
|
||||
|
||||
buffer.Add(single[0]);
|
||||
|
||||
if (
|
||||
buffer.Count >= 4
|
||||
&& buffer[^4] == (byte)'\r'
|
||||
&& buffer[^3] == (byte)'\n'
|
||||
&& buffer[^2] == (byte)'\r'
|
||||
&& buffer[^1] == (byte)'\n'
|
||||
)
|
||||
{
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
var text = Encoding.ASCII.GetString([.. buffer]);
|
||||
var lines = text.Split("\r\n", StringSplitOptions.RemoveEmptyEntries);
|
||||
var path =
|
||||
lines.Length > 0
|
||||
? lines[0].Split(' ') is { Length: >= 2 } parts
|
||||
? parts[1]
|
||||
: "/"
|
||||
: "/";
|
||||
|
||||
var headers = new Dictionary<string, string>(StringComparer.OrdinalIgnoreCase);
|
||||
foreach (var line in lines.Skip(1))
|
||||
{
|
||||
var colon = line.IndexOf(':', StringComparison.Ordinal);
|
||||
|
||||
if (colon > 0)
|
||||
{
|
||||
headers[line[..colon].Trim()] = line[(colon + 1)..].Trim();
|
||||
}
|
||||
}
|
||||
|
||||
return (path, headers);
|
||||
}
|
||||
|
||||
private async Task WriteReplyAsync(NetworkStream stream, Reply reply)
|
||||
{
|
||||
var body = reply.Gzip ? Compress(reply.Body) : reply.Body;
|
||||
|
||||
var head = new StringBuilder();
|
||||
head.Append(CultureInfo.InvariantCulture, $"HTTP/1.1 {reply.Status} {Describe(reply.Status)}\r\n");
|
||||
head.Append("Connection: close\r\n");
|
||||
|
||||
if (reply.ContentType is not null)
|
||||
{
|
||||
head.Append(CultureInfo.InvariantCulture, $"Content-Type: {reply.ContentType}\r\n");
|
||||
}
|
||||
|
||||
if (reply.Gzip)
|
||||
{
|
||||
head.Append("Content-Encoding: gzip\r\n");
|
||||
}
|
||||
|
||||
foreach (var (name, value) in reply.Headers)
|
||||
{
|
||||
head.Append(CultureInfo.InvariantCulture, $"{name}: {value}\r\n");
|
||||
}
|
||||
|
||||
if (reply.Chunked)
|
||||
{
|
||||
head.Append("Transfer-Encoding: chunked\r\n");
|
||||
}
|
||||
else
|
||||
{
|
||||
var declared = reply.DeclaredLength ?? body.Length;
|
||||
head.Append(CultureInfo.InvariantCulture, $"Content-Length: {declared}\r\n");
|
||||
}
|
||||
|
||||
head.Append("\r\n");
|
||||
|
||||
await stream.WriteAsync(Encoding.ASCII.GetBytes(head.ToString()), _shutdown.Token);
|
||||
await stream.FlushAsync(_shutdown.Token);
|
||||
|
||||
var limit = Math.Min(reply.TruncateAfter ?? body.Length, body.Length);
|
||||
var chunk = Math.Max(1, Math.Min(reply.BodyChunkSize, limit == 0 ? 1 : limit));
|
||||
|
||||
for (var offset = 0; offset < limit; offset += chunk)
|
||||
{
|
||||
var count = Math.Min(chunk, limit - offset);
|
||||
|
||||
if (reply.Chunked)
|
||||
{
|
||||
var header = count.ToString("x", CultureInfo.InvariantCulture) + "\r\n";
|
||||
await stream.WriteAsync(Encoding.ASCII.GetBytes(header), _shutdown.Token);
|
||||
}
|
||||
|
||||
await stream.WriteAsync(body.AsMemory(offset, count), _shutdown.Token);
|
||||
|
||||
if (reply.Chunked)
|
||||
{
|
||||
await stream.WriteAsync("\r\n"u8.ToArray(), _shutdown.Token);
|
||||
}
|
||||
|
||||
await stream.FlushAsync(_shutdown.Token);
|
||||
|
||||
if (reply.BodyDelay > TimeSpan.Zero)
|
||||
{
|
||||
await Task.Delay(reply.BodyDelay, _shutdown.Token);
|
||||
}
|
||||
}
|
||||
|
||||
// A truncated reply deliberately omits the terminal chunk: the client must notice.
|
||||
if (reply.Chunked && reply.TruncateAfter is null)
|
||||
{
|
||||
await stream.WriteAsync("0\r\n\r\n"u8.ToArray(), _shutdown.Token);
|
||||
await stream.FlushAsync(_shutdown.Token);
|
||||
}
|
||||
}
|
||||
|
||||
private static byte[] Compress(byte[] body)
|
||||
{
|
||||
using var output = new MemoryStream();
|
||||
|
||||
using (var gzip = new GZipStream(output, CompressionLevel.Fastest, leaveOpen: true))
|
||||
{
|
||||
gzip.Write(body);
|
||||
}
|
||||
|
||||
return output.ToArray();
|
||||
}
|
||||
|
||||
private static string Describe(int status) =>
|
||||
status switch
|
||||
{
|
||||
200 => "OK",
|
||||
301 => "Moved Permanently",
|
||||
302 => "Found",
|
||||
403 => "Forbidden",
|
||||
404 => "Not Found",
|
||||
410 => "Gone",
|
||||
429 => "Too Many Requests",
|
||||
500 => "Internal Server Error",
|
||||
503 => "Service Unavailable",
|
||||
_ => "Status",
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,571 @@
|
||||
using System.Net;
|
||||
using System.Text;
|
||||
using AvParser.Core.Collecting;
|
||||
using AvParser.Core.Proxies;
|
||||
using AvParser.Infrastructure.Collecting;
|
||||
using AvParser.Infrastructure.Media;
|
||||
using AvParser.Infrastructure.Proxies;
|
||||
using AvParser.Infrastructure.Storage;
|
||||
using Microsoft.Extensions.Logging.Abstractions;
|
||||
|
||||
namespace AvParser.Infrastructure.Tests.Collecting;
|
||||
|
||||
/// <summary>
|
||||
/// A factory that never uses a proxy but mirrors the real handler's configuration.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// Matching <c>AllowAutoRedirect = false</c> and <c>AutomaticDecompression = All</c> matters: with
|
||||
/// redirects handled by the BCL these tests would exercise its logic rather than the fetcher's.
|
||||
/// </remarks>
|
||||
internal sealed class DirectHttpClientFactory : IProxiedHttpClientFactory
|
||||
{
|
||||
public HttpClient Create(ProxyEndpoint? endpoint, TimeSpan? timeout = null) =>
|
||||
Create(endpoint, HttpClientTimeouts.Default);
|
||||
|
||||
public HttpClient Create(ProxyEndpoint? endpoint, HttpClientTimeouts timeouts) =>
|
||||
new(
|
||||
new SocketsHttpHandler
|
||||
{
|
||||
AllowAutoRedirect = false,
|
||||
ConnectTimeout = timeouts.Connect,
|
||||
AutomaticDecompression = DecompressionMethods.All,
|
||||
},
|
||||
disposeHandler: true
|
||||
)
|
||||
{
|
||||
Timeout = Timeout.InfiniteTimeSpan,
|
||||
};
|
||||
|
||||
public Task<(HttpClient Client, ProxyLease? Lease)> CreateFromPoolAsync(
|
||||
TimeSpan? timeout = null,
|
||||
CancellationToken cancellationToken = default
|
||||
) => Task.FromResult<(HttpClient, ProxyLease?)>((Create(null, timeout), null));
|
||||
|
||||
public Task<LeasedHttpClient> LeaseAsync(
|
||||
HttpClientTimeouts timeouts,
|
||||
bool requireProxy,
|
||||
CancellationToken cancellationToken = default
|
||||
) =>
|
||||
requireProxy
|
||||
? throw new ProxyUnavailableException()
|
||||
: Task.FromResult(new LeasedHttpClient(Create(null, timeouts), null));
|
||||
}
|
||||
|
||||
public sealed class MediaFetcherTests : IAsyncLifetime
|
||||
{
|
||||
private readonly string _root = Path.Combine(Path.GetTempPath(), "AvParserTests", Guid.NewGuid().ToString("N"));
|
||||
|
||||
private LoopbackServer _server = null!;
|
||||
private AppPaths _paths = null!;
|
||||
private BlobStore _blobs = null!;
|
||||
private HostThrottle _throttle = null!;
|
||||
private MediaFetcher _fetcher = null!;
|
||||
|
||||
public ValueTask InitializeAsync()
|
||||
{
|
||||
_paths = new AppPaths(_root);
|
||||
_paths.EnsureCreated();
|
||||
|
||||
_server = new LoopbackServer();
|
||||
_blobs = new BlobStore(_paths, NullLogger<BlobStore>.Instance);
|
||||
_throttle = Throttle();
|
||||
_fetcher = new MediaFetcher(
|
||||
new DirectHttpClientFactory(),
|
||||
_blobs,
|
||||
_throttle,
|
||||
NullLogger<MediaFetcher>.Instance
|
||||
);
|
||||
|
||||
return ValueTask.CompletedTask;
|
||||
}
|
||||
|
||||
public async ValueTask DisposeAsync()
|
||||
{
|
||||
await _server.DisposeAsync();
|
||||
_throttle.Dispose();
|
||||
|
||||
if (Directory.Exists(_root))
|
||||
{
|
||||
try
|
||||
{
|
||||
Directory.Delete(_root, recursive: true);
|
||||
}
|
||||
catch (IOException)
|
||||
{
|
||||
// Not worth failing a green test over.
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private HostThrottle Throttle(int concurrent = 4, int intervalMs = 0) =>
|
||||
new(concurrent, TimeSpan.FromMilliseconds(intervalMs), NullLogger<MediaFetcherTests>.Instance);
|
||||
|
||||
/// <summary>Options with the floor lowered, since the sample files are deliberately tiny.</summary>
|
||||
private static FetchOptions Options(params (string Key, object Value)[] _) =>
|
||||
new() { MinItemBytes = 1, Timeouts = HttpClientTimeouts.Default };
|
||||
|
||||
private Task<FetchResult> FetchAsync(Uri url, FetchOptions? options = null) =>
|
||||
_fetcher.FetchAsync(
|
||||
new MediaCandidate(url) { SourceId = "test" },
|
||||
options ?? Options(),
|
||||
TestContext.Current.CancellationToken
|
||||
);
|
||||
|
||||
/// <summary>Waits for the server's own bookkeeping to catch up with the client.</summary>
|
||||
private async Task<List<RecordedRequest>> WaitForRequestsAsync(int count, Func<RecordedRequest, bool> match)
|
||||
{
|
||||
for (var attempt = 0; attempt < 100; attempt++)
|
||||
{
|
||||
var matched = _server.Requests.Where(match).OrderBy(r => r.ArrivedUtc).ToList();
|
||||
|
||||
if (matched.Count >= count)
|
||||
{
|
||||
return matched;
|
||||
}
|
||||
|
||||
await Task.Delay(20, TestContext.Current.CancellationToken);
|
||||
}
|
||||
|
||||
var found = _server.Requests.Where(match).OrderBy(r => r.ArrivedUtc).ToList();
|
||||
found.Count.ShouldBe(count);
|
||||
|
||||
return found;
|
||||
}
|
||||
|
||||
private int StagedFileCount() =>
|
||||
Directory.Exists(_paths.MediaTempDirectory) ? Directory.EnumerateFiles(_paths.MediaTempDirectory).Count() : 0;
|
||||
|
||||
[Fact]
|
||||
public async Task A_valid_image_is_downloaded_and_hashed()
|
||||
{
|
||||
var url = _server.MapBody("/a.png", Samples.Png(), "image/png");
|
||||
|
||||
var result = await FetchAsync(url);
|
||||
|
||||
result.Outcome.ShouldBe(SeenOutcome.Stored);
|
||||
result.Blob!.Kind.ShouldBe(MediaKind.Png);
|
||||
result.Blob.Sha256.Length.ShouldBe(64);
|
||||
result.TempPath.ShouldNotBeNull();
|
||||
File.Exists(result.TempPath).ShouldBeTrue();
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task The_extension_comes_from_the_signature_not_from_the_content_type()
|
||||
{
|
||||
// The origin says PNG and serves a GIF. Believing the header would write a .png that no
|
||||
// viewer opens, and would let a host dictate the extension of a file on the user's disk.
|
||||
var url = _server.MapBody("/liar.png", Samples.Gif(), "image/png");
|
||||
|
||||
var result = await FetchAsync(url);
|
||||
|
||||
result.Blob!.Kind.ShouldBe(MediaKind.Gif);
|
||||
result.Blob.Extension.ShouldBe(".gif");
|
||||
result.ContentType.ShouldBe("image/png");
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task An_error_page_behind_a_200_is_refused()
|
||||
{
|
||||
var url = _server.MapBody("/gone.jpg", Samples.Html(), "image/jpeg");
|
||||
|
||||
var result = await FetchAsync(url);
|
||||
|
||||
result.Outcome.ShouldBe(SeenOutcome.NotMedia);
|
||||
result.TempPath.ShouldBeNull();
|
||||
StagedFileCount().ShouldBe(0);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task A_kind_the_user_excluded_is_refused()
|
||||
{
|
||||
var url = _server.MapBody("/clip.mp4", Samples.Mp4());
|
||||
var options = Options() with { AllowedKinds = new HashSet<MediaKind> { MediaKind.Png } };
|
||||
|
||||
(await FetchAsync(url, options)).Outcome.ShouldBe(SeenOutcome.UnsupportedType);
|
||||
}
|
||||
|
||||
[Theory]
|
||||
[InlineData(403, SeenOutcome.Failed)]
|
||||
[InlineData(404, SeenOutcome.Gone)]
|
||||
[InlineData(410, SeenOutcome.Gone)]
|
||||
[InlineData(500, SeenOutcome.Failed)]
|
||||
public async Task Error_statuses_are_reported_without_staging_anything(int status, SeenOutcome expected)
|
||||
{
|
||||
var url = _server.Map($"/e{status}", new Reply { Status = status, Body = "no"u8.ToArray() });
|
||||
|
||||
var result = await FetchAsync(url);
|
||||
|
||||
result.Outcome.ShouldBe(expected);
|
||||
result.HttpStatus.ShouldBe(status);
|
||||
StagedFileCount().ShouldBe(0);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task A_rate_limit_is_honoured_rather_than_worked_around()
|
||||
{
|
||||
var url = _server.Map(
|
||||
"/busy",
|
||||
new Reply
|
||||
{
|
||||
Status = 429,
|
||||
Headers = new Dictionary<string, string>(StringComparer.OrdinalIgnoreCase) { ["Retry-After"] = "3" },
|
||||
}
|
||||
);
|
||||
|
||||
var result = await FetchAsync(url);
|
||||
|
||||
result.Outcome.ShouldBe(SeenOutcome.RateLimited);
|
||||
result.RetryAfter.ShouldBe(TimeSpan.FromSeconds(3));
|
||||
|
||||
// The host is now cooling: the collector slows down instead of switching proxy.
|
||||
_throttle.CooldownRemaining(url).ShouldBeGreaterThan(TimeSpan.Zero);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task A_service_unavailable_without_a_hint_still_backs_off()
|
||||
{
|
||||
var url = _server.Map("/down", new Reply { Status = 503 });
|
||||
|
||||
(await FetchAsync(url)).Outcome.ShouldBe(SeenOutcome.RateLimited);
|
||||
_throttle.CooldownRemaining(url).ShouldBeGreaterThan(TimeSpan.Zero);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Headers_that_never_arrive_time_out()
|
||||
{
|
||||
var url = _server.Map("/slowhead", new Reply { Body = Samples.Png(), HeaderDelay = TimeSpan.FromSeconds(3) });
|
||||
|
||||
var options = Options() with
|
||||
{
|
||||
Timeouts = HttpClientTimeouts.Default with { Headers = TimeSpan.FromMilliseconds(300) },
|
||||
};
|
||||
|
||||
(await FetchAsync(url, options)).Outcome.ShouldBe(SeenOutcome.Timeout);
|
||||
StagedFileCount().ShouldBe(0);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task A_body_that_stops_arriving_is_abandoned_and_leaves_nothing_behind()
|
||||
{
|
||||
var url = _server.Map(
|
||||
"/drip",
|
||||
new Reply
|
||||
{
|
||||
Body = Encoding.ASCII.GetBytes(new string('x', 4096)),
|
||||
BodyChunkSize = 8,
|
||||
BodyDelay = TimeSpan.FromSeconds(2),
|
||||
}
|
||||
);
|
||||
|
||||
var options = Options() with
|
||||
{
|
||||
Timeouts = HttpClientTimeouts.Default with { Idle = TimeSpan.FromMilliseconds(300) },
|
||||
};
|
||||
|
||||
var result = await FetchAsync(url, options);
|
||||
|
||||
result.Outcome.ShouldBe(SeenOutcome.Timeout);
|
||||
StagedFileCount().ShouldBe(0);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task A_body_shorter_than_declared_is_a_truncated_transfer_not_a_small_file()
|
||||
{
|
||||
// The single most important rejection: a partial image promoted into the blob store would
|
||||
// be indistinguishable from a real one for ever afterwards.
|
||||
var body = Samples.Png();
|
||||
var url = _server.Map(
|
||||
"/short",
|
||||
new Reply
|
||||
{
|
||||
Body = body,
|
||||
DeclaredLength = body.Length + 5000,
|
||||
TruncateAfter = body.Length,
|
||||
}
|
||||
);
|
||||
|
||||
var result = await FetchAsync(url);
|
||||
|
||||
result.Outcome.ShouldBe(SeenOutcome.Failed);
|
||||
result.ErrorCode.ShouldBe("Truncated");
|
||||
result.TempPath.ShouldBeNull();
|
||||
StagedFileCount().ShouldBe(0);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task A_declared_length_over_the_cap_is_refused_before_the_body_is_read()
|
||||
{
|
||||
var body = Encoding.ASCII.GetBytes(new string('x', 8192));
|
||||
var url = _server.Map("/huge", new Reply { Body = body });
|
||||
var options = Options() with { MaxItemBytes = 1024 };
|
||||
|
||||
var result = await FetchAsync(url, options);
|
||||
|
||||
result.Outcome.ShouldBe(SeenOutcome.TooLarge);
|
||||
StagedFileCount().ShouldBe(0);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task A_chunked_body_that_grows_past_the_cap_is_cut_off()
|
||||
{
|
||||
// No Content-Length to check against, so the cap has to be enforced as bytes arrive.
|
||||
var body = Encoding.ASCII.GetBytes(new string('x', 8192));
|
||||
var url = _server.Map(
|
||||
"/huge-chunked",
|
||||
new Reply
|
||||
{
|
||||
Body = body,
|
||||
Chunked = true,
|
||||
BodyChunkSize = 256,
|
||||
}
|
||||
);
|
||||
var options = Options() with { MaxItemBytes = 1024 };
|
||||
|
||||
(await FetchAsync(url, options)).Outcome.ShouldBe(SeenOutcome.TooLarge);
|
||||
StagedFileCount().ShouldBe(0);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task A_well_formed_chunked_body_is_accepted()
|
||||
{
|
||||
var url = _server.Map(
|
||||
"/chunked.png",
|
||||
new Reply
|
||||
{
|
||||
Body = Samples.Png(),
|
||||
Chunked = true,
|
||||
BodyChunkSize = 7,
|
||||
}
|
||||
);
|
||||
|
||||
var result = await FetchAsync(url);
|
||||
|
||||
result.Outcome.ShouldBe(SeenOutcome.Stored);
|
||||
result.Blob!.Length.ShouldBe(Samples.Png().Length);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task The_same_image_served_gzipped_and_plain_is_one_blob()
|
||||
{
|
||||
// The handler decompresses transparently, so the hash must be over the decoded bytes.
|
||||
var plain = _server.MapBody("/plain.png", Samples.Png());
|
||||
var zipped = _server.Map("/zipped.png", new Reply { Body = Samples.Png(), Gzip = true });
|
||||
|
||||
var first = await FetchAsync(plain);
|
||||
var second = await FetchAsync(zipped);
|
||||
|
||||
first.Blob!.Sha256.ShouldBe(second.Blob!.Sha256);
|
||||
first.Blob.Length.ShouldBe(second.Blob.Length);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task A_tracking_pixel_is_below_the_floor()
|
||||
{
|
||||
var url = _server.MapBody("/pixel.gif", Samples.Gif());
|
||||
|
||||
// The default floor, rather than the lowered one the other tests use.
|
||||
var result = await FetchAsync(url, new FetchOptions());
|
||||
|
||||
result.Outcome.ShouldBe(SeenOutcome.TooSmall);
|
||||
StagedFileCount().ShouldBe(0);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task A_redirect_chain_is_followed_and_the_final_address_recorded()
|
||||
{
|
||||
var final = _server.MapBody("/final.png", Samples.Png());
|
||||
_server.Map("/hop2", Redirect(302, "/final.png"));
|
||||
var start = _server.Map("/hop1", Redirect(301, "/hop2"));
|
||||
|
||||
var result = await FetchAsync(start);
|
||||
|
||||
result.Outcome.ShouldBe(SeenOutcome.Stored);
|
||||
result.FinalUrl.ShouldBe(final);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task A_relative_location_is_resolved_against_the_current_address()
|
||||
{
|
||||
_server.MapBody("/other.png", Samples.Png());
|
||||
var start = _server.Map("/rel", Redirect(302, "/other.png"));
|
||||
|
||||
(await FetchAsync(start)).Outcome.ShouldBe(SeenOutcome.Stored);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task A_redirect_loop_is_broken()
|
||||
{
|
||||
_server.Map("/loopB", Redirect(302, "/loopA"));
|
||||
var start = _server.Map("/loopA", Redirect(302, "/loopB"));
|
||||
|
||||
var result = await FetchAsync(start);
|
||||
|
||||
result.ErrorCode.ShouldBe("RedirectLoop");
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task A_chain_longer_than_the_cap_gives_up()
|
||||
{
|
||||
// Distinct hops, so this is the hop budget rather than the loop detector doing the work.
|
||||
for (var hop = 0; hop < 10; hop++)
|
||||
{
|
||||
_server.Map($"/chain{hop}", Redirect(302, $"/chain{hop + 1}"));
|
||||
}
|
||||
|
||||
var result = await FetchAsync(new Uri(_server.BaseAddress, "/chain0"), Options() with { MaxRedirects = 3 });
|
||||
|
||||
result.ErrorCode.ShouldBe("TooManyRedirects");
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task A_redirect_to_a_non_http_scheme_is_refused()
|
||||
{
|
||||
var start = _server.Map("/evil", Redirect(302, "data:image/png;base64,AAAA"));
|
||||
|
||||
(await FetchAsync(start)).ErrorCode.ShouldBe("BadRedirect");
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task A_known_placeholder_is_recognised_by_its_hash()
|
||||
{
|
||||
var url = _server.MapBody("/dead.png", Samples.Png());
|
||||
var probe = await FetchAsync(url);
|
||||
File.Delete(probe.TempPath!);
|
||||
|
||||
var options = Options() with
|
||||
{
|
||||
Tombstones = new HashSet<string>(StringComparer.Ordinal) { probe.Blob!.Sha256 },
|
||||
};
|
||||
|
||||
var result = await FetchAsync(url, options);
|
||||
|
||||
result.Outcome.ShouldBe(SeenOutcome.Placeholder);
|
||||
StagedFileCount().ShouldBe(0);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Animation_is_detected_end_to_end()
|
||||
{
|
||||
var still = _server.MapBody("/still.gif", Samples.Gif(frames: 1));
|
||||
var moving = _server.MapBody("/moving.gif", Samples.Gif(frames: 3));
|
||||
|
||||
(await FetchAsync(still)).Blob!.IsAnimated.ShouldBeFalse();
|
||||
(await FetchAsync(moving)).Blob!.IsAnimated.ShouldBeTrue();
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Dimensions_are_recorded_when_the_header_carries_them()
|
||||
{
|
||||
var url = _server.MapBody("/big.png", Samples.Png(1024, 768));
|
||||
|
||||
var blob = (await FetchAsync(url)).Blob!;
|
||||
|
||||
blob.Width.ShouldBe(1024);
|
||||
blob.Height.ShouldBe(768);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task A_referer_is_sent_when_the_candidate_carries_one()
|
||||
{
|
||||
var url = _server.MapBody("/ref.png", Samples.Png());
|
||||
var referer = new Uri("https://example.test/page");
|
||||
|
||||
await _fetcher.FetchAsync(
|
||||
new MediaCandidate(url) { SourceId = "test", Referer = referer },
|
||||
Options(),
|
||||
TestContext.Current.CancellationToken
|
||||
);
|
||||
|
||||
var request = _server.Requests.Single(r => r.Path == "/ref.png");
|
||||
request.Headers["Referer"].ShouldBe(referer.AbsoluteUri);
|
||||
request.Headers["User-Agent"].ShouldContain("AvParser");
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task One_request_at_a_time_per_host_when_the_cap_says_so()
|
||||
{
|
||||
_throttle.Dispose();
|
||||
_throttle = Throttle(concurrent: 1);
|
||||
_fetcher = new MediaFetcher(
|
||||
new DirectHttpClientFactory(),
|
||||
_blobs,
|
||||
_throttle,
|
||||
NullLogger<MediaFetcher>.Instance
|
||||
);
|
||||
|
||||
// Real media padded out, so neither request is refused after 32 bytes and cut short —
|
||||
// the point of the test is two full responses that could have overlapped.
|
||||
var body = (byte[])[.. Samples.Png(), .. new byte[2048]];
|
||||
var slow = new Reply
|
||||
{
|
||||
Body = body,
|
||||
BodyChunkSize = 256,
|
||||
BodyDelay = TimeSpan.FromMilliseconds(40),
|
||||
};
|
||||
|
||||
_server.Map("/one", slow);
|
||||
_server.Map("/two", slow);
|
||||
|
||||
await Task.WhenAll(
|
||||
FetchAsync(new Uri(_server.BaseAddress, "/one")),
|
||||
FetchAsync(new Uri(_server.BaseAddress, "/two"))
|
||||
);
|
||||
|
||||
// The server records a request from its own task, which can lag the client by a moment.
|
||||
var requests = await WaitForRequestsAsync(2, r => r.Path is "/one" or "/two");
|
||||
|
||||
// Serialised, the second request cannot arrive until the first body has been written:
|
||||
// nine 256-byte writes at 40 ms apiece. Run concurrently the gap would be near zero.
|
||||
// Compared against arrival rather than completion because the server records completion
|
||||
// from its own task, which can lag the client by a few milliseconds.
|
||||
var gap = requests[1].ArrivedUtc - requests[0].ArrivedUtc;
|
||||
gap.ShouldBeGreaterThan(TimeSpan.FromMilliseconds(200));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Cancelling_mid_download_leaves_no_staged_file()
|
||||
{
|
||||
// Real media, or the fetcher rightly rejects it after 32 bytes and there is nothing left
|
||||
// to cancel.
|
||||
var url = _server.Map(
|
||||
"/cancel",
|
||||
new Reply
|
||||
{
|
||||
Body = [.. Samples.Png(), .. new byte[65536]],
|
||||
BodyChunkSize = 128,
|
||||
BodyDelay = TimeSpan.FromMilliseconds(30),
|
||||
}
|
||||
);
|
||||
|
||||
using var cancellation = new CancellationTokenSource();
|
||||
var fetch = _fetcher.FetchAsync(new MediaCandidate(url) { SourceId = "test" }, Options(), cancellation.Token);
|
||||
|
||||
await Task.Delay(200, TestContext.Current.CancellationToken);
|
||||
await cancellation.CancelAsync();
|
||||
|
||||
await Should.ThrowAsync<OperationCanceledException>(async () => await fetch);
|
||||
|
||||
StagedFileCount().ShouldBe(0);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Demanding_a_proxy_that_is_not_there_fails_the_item_rather_than_going_direct()
|
||||
{
|
||||
// Otherwise the request leaves from the user's own address at precisely the moment they
|
||||
// asked for it not to.
|
||||
var url = _server.MapBody("/direct.png", Samples.Png());
|
||||
var options = Options() with { RequireProxy = true };
|
||||
|
||||
var result = await FetchAsync(url, options);
|
||||
|
||||
result.ErrorCode.ShouldBe("NoProxy");
|
||||
_server.Requests.ShouldNotContain(r => r.Path == "/direct.png");
|
||||
}
|
||||
|
||||
private static Reply Redirect(int status, string location) =>
|
||||
new()
|
||||
{
|
||||
Status = status,
|
||||
Headers = new Dictionary<string, string>(StringComparer.OrdinalIgnoreCase) { ["Location"] = location },
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,383 @@
|
||||
using System.Buffers.Binary;
|
||||
using System.Text;
|
||||
using AvParser.Core.Collecting;
|
||||
using AvParser.Infrastructure.Collecting;
|
||||
|
||||
namespace AvParser.Infrastructure.Tests.Collecting;
|
||||
|
||||
/// <summary>Builds the smallest byte sequences that are still legitimately each format.</summary>
|
||||
internal static class Samples
|
||||
{
|
||||
private static readonly byte[] PngSignature = [0x89, 0x50, 0x4E, 0x47, 0x0D, 0x0A, 0x1A, 0x0A];
|
||||
|
||||
public static byte[] Jpeg() => [0xFF, 0xD8, 0xFF, 0xE0, 0x00, 0x10, .. "JFIF\0"u8, .. new byte[24]];
|
||||
|
||||
public static byte[] Png(int width = 640, int height = 480)
|
||||
{
|
||||
var bytes = new List<byte>(PngSignature);
|
||||
bytes.AddRange(Chunk("IHDR", Ihdr(width, height)));
|
||||
bytes.AddRange(Chunk("IDAT", new byte[8]));
|
||||
|
||||
return [.. bytes];
|
||||
}
|
||||
|
||||
public static byte[] Apng(int width = 64, int height = 64)
|
||||
{
|
||||
var bytes = new List<byte>(PngSignature);
|
||||
bytes.AddRange(Chunk("IHDR", Ihdr(width, height)));
|
||||
// acTL must precede IDAT to count; a decoder ignores it afterwards and so do we.
|
||||
bytes.AddRange(Chunk("acTL", [0, 0, 0, 2, 0, 0, 0, 0]));
|
||||
bytes.AddRange(Chunk("IDAT", new byte[8]));
|
||||
|
||||
return [.. bytes];
|
||||
}
|
||||
|
||||
/// <summary>A PNG carrying acTL only after IDAT — a still image with junk appended.</summary>
|
||||
public static byte[] PngWithLateAnimationChunk()
|
||||
{
|
||||
var bytes = new List<byte>(PngSignature);
|
||||
bytes.AddRange(Chunk("IHDR", Ihdr(8, 8)));
|
||||
bytes.AddRange(Chunk("IDAT", new byte[8]));
|
||||
bytes.AddRange(Chunk("acTL", [0, 0, 0, 2, 0, 0, 0, 0]));
|
||||
|
||||
return [.. bytes];
|
||||
}
|
||||
|
||||
public static byte[] Gif(int frames = 1, int width = 12, int height = 34, bool modern = true)
|
||||
{
|
||||
var bytes = new List<byte>(Encoding.ASCII.GetBytes(modern ? "GIF89a" : "GIF87a"));
|
||||
|
||||
bytes.AddRange(LittleEndian16(width));
|
||||
bytes.AddRange(LittleEndian16(height));
|
||||
bytes.Add(0x00); // no global colour table, which keeps the offsets simple
|
||||
bytes.Add(0x00);
|
||||
bytes.Add(0x00);
|
||||
|
||||
for (var frame = 0; frame < frames; frame++)
|
||||
{
|
||||
bytes.Add(0x2C);
|
||||
bytes.AddRange(LittleEndian16(0));
|
||||
bytes.AddRange(LittleEndian16(0));
|
||||
bytes.AddRange(LittleEndian16(width));
|
||||
bytes.AddRange(LittleEndian16(height));
|
||||
bytes.Add(0x00); // no local colour table
|
||||
bytes.Add(0x02); // LZW minimum code size
|
||||
bytes.Add(0x03); // one sub-block of three bytes
|
||||
bytes.AddRange([0x01, 0x02, 0x03]);
|
||||
bytes.Add(0x00); // sub-block terminator
|
||||
}
|
||||
|
||||
bytes.Add(0x3B);
|
||||
|
||||
return [.. bytes];
|
||||
}
|
||||
|
||||
/// <summary>An animated GIF whose frames are separated by graphic control extensions.</summary>
|
||||
public static byte[] GifWithExtensions()
|
||||
{
|
||||
var bytes = new List<byte>(Encoding.ASCII.GetBytes("GIF89a"));
|
||||
bytes.AddRange(LittleEndian16(4));
|
||||
bytes.AddRange(LittleEndian16(4));
|
||||
bytes.Add(0x80 | 0x01); // global colour table, 2^(1+1) = 4 entries
|
||||
bytes.Add(0x00);
|
||||
bytes.Add(0x00);
|
||||
bytes.AddRange(new byte[3 * 4]);
|
||||
|
||||
for (var frame = 0; frame < 2; frame++)
|
||||
{
|
||||
bytes.Add(0x21);
|
||||
bytes.Add(0xF9);
|
||||
bytes.Add(0x04);
|
||||
bytes.AddRange([0x00, 0x0A, 0x00, 0x00]);
|
||||
bytes.Add(0x00);
|
||||
|
||||
bytes.Add(0x2C);
|
||||
bytes.AddRange(LittleEndian16(0));
|
||||
bytes.AddRange(LittleEndian16(0));
|
||||
bytes.AddRange(LittleEndian16(4));
|
||||
bytes.AddRange(LittleEndian16(4));
|
||||
bytes.Add(0x00);
|
||||
bytes.Add(0x02);
|
||||
bytes.Add(0x02);
|
||||
bytes.AddRange([0x01, 0x02]);
|
||||
bytes.Add(0x00);
|
||||
}
|
||||
|
||||
bytes.Add(0x3B);
|
||||
|
||||
return [.. bytes];
|
||||
}
|
||||
|
||||
public static byte[] WebpStill()
|
||||
{
|
||||
var body = new List<byte>("WEBP"u8.ToArray());
|
||||
body.AddRange("VP8 "u8.ToArray());
|
||||
body.AddRange(LittleEndian32(16));
|
||||
body.AddRange(new byte[16]);
|
||||
|
||||
return Riff(body);
|
||||
}
|
||||
|
||||
public static byte[] WebpLossless()
|
||||
{
|
||||
var body = new List<byte>("WEBP"u8.ToArray());
|
||||
body.AddRange("VP8L"u8.ToArray());
|
||||
body.AddRange(LittleEndian32(16));
|
||||
body.AddRange(new byte[16]);
|
||||
|
||||
return Riff(body);
|
||||
}
|
||||
|
||||
public static byte[] WebpExtended(bool animated, int width = 100, int height = 50)
|
||||
{
|
||||
var body = new List<byte>("WEBP"u8.ToArray());
|
||||
body.AddRange("VP8X"u8.ToArray());
|
||||
body.AddRange(LittleEndian32(10));
|
||||
body.Add((byte)(animated ? 0x02 : 0x10)); // ANIM, versus a plain alpha flag
|
||||
body.AddRange([0x00, 0x00, 0x00]);
|
||||
body.AddRange(LittleEndian24(width - 1));
|
||||
body.AddRange(LittleEndian24(height - 1));
|
||||
body.AddRange(new byte[8]);
|
||||
|
||||
return Riff(body);
|
||||
}
|
||||
|
||||
public static byte[] Mp4(string brand = "isom") =>
|
||||
[0x00, 0x00, 0x00, 0x20, .. "ftyp"u8, .. Encoding.ASCII.GetBytes(brand), .. new byte[20]];
|
||||
|
||||
public static byte[] Avif() => Mp4("avif");
|
||||
|
||||
public static byte[] WebM() => Ebml("webm");
|
||||
|
||||
public static byte[] Matroska() => Ebml("matroska");
|
||||
|
||||
public static byte[] Html(string body = "<!DOCTYPE html><html><body>Image not found</body></html>") =>
|
||||
Encoding.UTF8.GetBytes(body);
|
||||
|
||||
private static byte[] Ebml(string docType)
|
||||
{
|
||||
var bytes = new List<byte> { 0x1A, 0x45, 0xDF, 0xA3 };
|
||||
bytes.AddRange([0x9F, 0x42, 0x86, 0x81, 0x01]); // a plausible-looking EBML header preamble
|
||||
bytes.Add(0x42);
|
||||
bytes.Add(0x82);
|
||||
bytes.Add((byte)docType.Length);
|
||||
bytes.AddRange(Encoding.ASCII.GetBytes(docType));
|
||||
bytes.AddRange(new byte[16]);
|
||||
|
||||
return [.. bytes];
|
||||
}
|
||||
|
||||
private static byte[] Riff(List<byte> body)
|
||||
{
|
||||
var bytes = new List<byte>("RIFF"u8.ToArray());
|
||||
bytes.AddRange(LittleEndian32(body.Count));
|
||||
bytes.AddRange(body);
|
||||
|
||||
return [.. bytes];
|
||||
}
|
||||
|
||||
private static byte[] Ihdr(int width, int height)
|
||||
{
|
||||
var data = new byte[13];
|
||||
BinaryPrimitives.WriteUInt32BigEndian(data.AsSpan(0, 4), (uint)width);
|
||||
BinaryPrimitives.WriteUInt32BigEndian(data.AsSpan(4, 4), (uint)height);
|
||||
data[8] = 8;
|
||||
|
||||
return data;
|
||||
}
|
||||
|
||||
private static byte[] Chunk(string type, byte[] data)
|
||||
{
|
||||
var bytes = new List<byte>();
|
||||
var length = new byte[4];
|
||||
BinaryPrimitives.WriteUInt32BigEndian(length, (uint)data.Length);
|
||||
|
||||
bytes.AddRange(length);
|
||||
bytes.AddRange(Encoding.ASCII.GetBytes(type));
|
||||
bytes.AddRange(data);
|
||||
bytes.AddRange(new byte[4]); // CRC, never checked here
|
||||
|
||||
return [.. bytes];
|
||||
}
|
||||
|
||||
private static byte[] LittleEndian16(int value) => [(byte)(value & 0xFF), (byte)((value >> 8) & 0xFF)];
|
||||
|
||||
private static byte[] LittleEndian24(int value) =>
|
||||
[(byte)(value & 0xFF), (byte)((value >> 8) & 0xFF), (byte)((value >> 16) & 0xFF)];
|
||||
|
||||
private static byte[] LittleEndian32(int value) =>
|
||||
[(byte)(value & 0xFF), (byte)((value >> 8) & 0xFF), (byte)((value >> 16) & 0xFF), (byte)((value >> 24) & 0xFF)];
|
||||
}
|
||||
|
||||
public class MediaSignatureTests
|
||||
{
|
||||
[Fact]
|
||||
public void Jpeg_is_recognised() => MediaSignatures.Detect(Samples.Jpeg()).ShouldBe(MediaKind.Jpeg);
|
||||
|
||||
[Fact]
|
||||
public void Png_is_recognised() => MediaSignatures.Detect(Samples.Png()).ShouldBe(MediaKind.Png);
|
||||
|
||||
[Theory]
|
||||
[InlineData(true)]
|
||||
[InlineData(false)]
|
||||
public void Both_gif_versions_are_recognised(bool modern) =>
|
||||
MediaSignatures.Detect(Samples.Gif(modern: modern)).ShouldBe(MediaKind.Gif);
|
||||
|
||||
[Fact]
|
||||
public void Webp_is_recognised() => MediaSignatures.Detect(Samples.WebpStill()).ShouldBe(MediaKind.WebP);
|
||||
|
||||
[Theory]
|
||||
[InlineData("isom")]
|
||||
[InlineData("mp42")]
|
||||
[InlineData("avc1")]
|
||||
public void Mp4_brands_are_recognised(string brand) =>
|
||||
MediaSignatures.Detect(Samples.Mp4(brand)).ShouldBe(MediaKind.Mp4);
|
||||
|
||||
[Fact]
|
||||
public void Avif_is_told_apart_from_mp4() => MediaSignatures.Detect(Samples.Avif()).ShouldBe(MediaKind.Avif);
|
||||
|
||||
[Fact]
|
||||
public void WebM_is_recognised() => MediaSignatures.Detect(Samples.WebM()).ShouldBe(MediaKind.WebM);
|
||||
|
||||
[Fact]
|
||||
public void Matroska_is_not_mistaken_for_webm()
|
||||
{
|
||||
// Both open with the same EBML magic; only the DocType separates them.
|
||||
MediaSignatures.Detect(Samples.Matroska()).ShouldBe(MediaKind.Unknown);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Anything_unrecognised_is_unknown()
|
||||
{
|
||||
MediaSignatures.Detect(Samples.Html()).ShouldBe(MediaKind.Unknown);
|
||||
MediaSignatures.Detect(new byte[32]).ShouldBe(MediaKind.Unknown);
|
||||
MediaSignatures.Detect([]).ShouldBe(MediaKind.Unknown);
|
||||
MediaSignatures.Detect([0xFF]).ShouldBe(MediaKind.Unknown);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void A_truncated_prefix_never_throws()
|
||||
{
|
||||
var png = Samples.Png();
|
||||
|
||||
for (var length = 0; length < png.Length; length++)
|
||||
{
|
||||
_ = MediaSignatures.Detect(png.AsSpan(0, length));
|
||||
_ = MediaSignatures.DetectAnimation(MediaKind.Png, png.AsSpan(0, length));
|
||||
_ = MediaSignatures.ReadDimensions(MediaKind.Png, png.AsSpan(0, length));
|
||||
}
|
||||
}
|
||||
|
||||
[Theory]
|
||||
[InlineData("<!DOCTYPE html><html>")]
|
||||
[InlineData("<html><body>gone</body></html>")]
|
||||
[InlineData(" \n<!doctype HTML>")]
|
||||
[InlineData("<?xml version=\"1.0\"?>")]
|
||||
public void An_error_page_behind_a_200_is_spotted(string body) =>
|
||||
MediaSignatures.LooksLikeHtml(Samples.Html(body)).ShouldBeTrue();
|
||||
|
||||
[Fact]
|
||||
public void Real_media_is_not_mistaken_for_html()
|
||||
{
|
||||
MediaSignatures.LooksLikeHtml(Samples.Png()).ShouldBeFalse();
|
||||
MediaSignatures.LooksLikeHtml(Samples.Gif()).ShouldBeFalse();
|
||||
MediaSignatures.LooksLikeHtml(Samples.Jpeg()).ShouldBeFalse();
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void A_single_frame_gif_does_not_animate() =>
|
||||
MediaSignatures.DetectAnimation(MediaKind.Gif, Samples.Gif(frames: 1)).ShouldBeFalse();
|
||||
|
||||
[Fact]
|
||||
public void A_two_frame_gif_animates() =>
|
||||
MediaSignatures.DetectAnimation(MediaKind.Gif, Samples.Gif(frames: 2)).ShouldBeTrue();
|
||||
|
||||
[Fact]
|
||||
public void A_gif_with_control_extensions_between_frames_animates()
|
||||
{
|
||||
// Real animated GIFs interleave graphic control extensions and use a global colour table;
|
||||
// walking past both is where a naive scanner goes wrong.
|
||||
MediaSignatures.DetectAnimation(MediaKind.Gif, Samples.GifWithExtensions()).ShouldBeTrue();
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void A_plain_png_does_not_animate() =>
|
||||
MediaSignatures.DetectAnimation(MediaKind.Png, Samples.Png()).ShouldBeFalse();
|
||||
|
||||
[Fact]
|
||||
public void An_apng_animates() => MediaSignatures.DetectAnimation(MediaKind.Png, Samples.Apng()).ShouldBeTrue();
|
||||
|
||||
[Fact]
|
||||
public void An_animation_chunk_after_the_first_idat_does_not_count()
|
||||
{
|
||||
// Decoders ignore acTL once IDAT has started, so claiming animation here would be a lie.
|
||||
MediaSignatures.DetectAnimation(MediaKind.Png, Samples.PngWithLateAnimationChunk()).ShouldBeFalse();
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void A_plain_webp_does_not_animate()
|
||||
{
|
||||
MediaSignatures.DetectAnimation(MediaKind.WebP, Samples.WebpStill()).ShouldBeFalse();
|
||||
MediaSignatures.DetectAnimation(MediaKind.WebP, Samples.WebpLossless()).ShouldBeFalse();
|
||||
MediaSignatures.DetectAnimation(MediaKind.WebP, Samples.WebpExtended(animated: false)).ShouldBeFalse();
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void An_extended_webp_with_the_anim_flag_animates() =>
|
||||
MediaSignatures.DetectAnimation(MediaKind.WebP, Samples.WebpExtended(animated: true)).ShouldBeTrue();
|
||||
|
||||
[Fact]
|
||||
public void Png_dimensions_are_read()
|
||||
{
|
||||
var (width, height) = MediaSignatures.ReadDimensions(MediaKind.Png, Samples.Png(1920, 1080));
|
||||
|
||||
width.ShouldBe(1920);
|
||||
height.ShouldBe(1080);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Gif_dimensions_are_read()
|
||||
{
|
||||
var (width, height) = MediaSignatures.ReadDimensions(MediaKind.Gif, Samples.Gif(width: 320, height: 240));
|
||||
|
||||
width.ShouldBe(320);
|
||||
height.ShouldBe(240);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Extended_webp_dimensions_are_read()
|
||||
{
|
||||
// Stored as canvas-minus-one, which is the kind of off-by-one that ships unnoticed.
|
||||
var (width, height) = MediaSignatures.ReadDimensions(
|
||||
MediaKind.WebP,
|
||||
Samples.WebpExtended(animated: true, width: 800, height: 600)
|
||||
);
|
||||
|
||||
width.ShouldBe(800);
|
||||
height.ShouldBe(600);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Dimensions_are_absent_rather_than_guessed()
|
||||
{
|
||||
MediaSignatures.ReadDimensions(MediaKind.Jpeg, Samples.Jpeg()).ShouldBe((null, null));
|
||||
MediaSignatures.ReadDimensions(MediaKind.WebP, Samples.WebpStill()).ShouldBe((null, null));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Garbage_never_throws()
|
||||
{
|
||||
var random = new Random(20260813);
|
||||
|
||||
for (var attempt = 0; attempt < 500; attempt++)
|
||||
{
|
||||
var bytes = new byte[random.Next(0, 200)];
|
||||
random.NextBytes(bytes);
|
||||
|
||||
var kind = MediaSignatures.Detect(bytes);
|
||||
_ = MediaSignatures.DetectAnimation(kind, bytes);
|
||||
_ = MediaSignatures.ReadDimensions(kind, bytes);
|
||||
_ = MediaSignatures.LooksLikeHtml(bytes);
|
||||
}
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user