Add the download pipeline: sniffing, redirects, throttling, verdicts
Second half of the collector foundation. Still nothing in the app references it; the pipeline is tested end to end against a deliberately badly behaved loopback server before anything depends on it. Types come from the bytes, never from the URL, the extension or Content-Type - two of those three are chosen by whoever serves the file, and a host must not get to pick the extension of a file written to the user's disk. Animation is a separate question from kind: GIF89a proves nothing without a second image descriptor, and a PNG is an APNG only if acTL precedes the first IDAT, so both are walked properly rather than guessed. Timeouts are split three ways because HttpClient.Timeout covers the whole response: any value large enough for a 30 MB file is also large enough for a dead connection to hang on. Connect, headers and a per-read idle deadline let both be strict. Redirects are followed by hand since the shared proxy handler disables them, which is what allows a hop cap, loop detection and refusing a jump to a data: URL. The lease verdict is a pure function, because ProxyLease's constructor is internal to the domain and no test can fabricate one. Its rule is that the verdict describes the transport, not the resource: a 404 is a working proxy, and so is a 429 - blaming the proxy for an origin's rate limit would make the pool rotate away from a good address in response to being asked to slow down. Cancellation reports nothing at all. Throttling exists to be obeyed. It is raised only by the host's own 429 and 503, and never by rotating to another proxy. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
Leonid Pershin
co-authored by
Claude Opus 5
Claude Opus 5
parent
181f974a37
commit
1742c094e9
@@ -0,0 +1,636 @@
|
||||
using System.Buffers;
|
||||
using System.Diagnostics;
|
||||
using System.Net;
|
||||
using System.Net.Http.Headers;
|
||||
using System.Security.Cryptography;
|
||||
using AvParser.Core.Collecting;
|
||||
using AvParser.Infrastructure.Media;
|
||||
using AvParser.Infrastructure.Proxies;
|
||||
using Microsoft.Extensions.Logging;
|
||||
|
||||
namespace AvParser.Infrastructure.Collecting;
|
||||
|
||||
/// <summary>Downloads one candidate, verifies it, and stages it for the store.</summary>
|
||||
public interface IMediaFetcher
|
||||
{
|
||||
/// <summary>Fetches one candidate.</summary>
|
||||
/// <remarks>
|
||||
/// Never throws for an ordinary network or content failure — those come back as a
|
||||
/// <see cref="FetchResult"/> so the run continues. Cancellation still propagates.
|
||||
/// </remarks>
|
||||
Task<FetchResult> FetchAsync(
|
||||
MediaCandidate candidate,
|
||||
FetchOptions options,
|
||||
CancellationToken cancellationToken = default
|
||||
);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// The one place that knows how to get bytes off the network safely.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// <para>
|
||||
/// Redirects are followed here because the shared proxy handler disables automatic redirects, and
|
||||
/// following them by hand is what makes it possible to cap the hops, spot a loop, refuse a jump to
|
||||
/// a non-HTTP scheme and re-enter the throttle when the host changes.
|
||||
/// </para>
|
||||
/// <para>
|
||||
/// The invariant everything else depends on: a staged file is returned only when the transfer
|
||||
/// reached the end and, where the length was knowable, matched it. Every other path deletes it in a
|
||||
/// <c>finally</c>. A partial image that reached the blob store would be indistinguishable from a
|
||||
/// real one for ever after.
|
||||
/// </para>
|
||||
/// </remarks>
|
||||
public sealed class MediaFetcher(
|
||||
IProxiedHttpClientFactory clients,
|
||||
BlobStore blobs,
|
||||
HostThrottle throttle,
|
||||
ILogger<MediaFetcher> logger
|
||||
) : IMediaFetcher
|
||||
{
|
||||
/// <summary>Big enough that a 32 MB file is a few hundred reads, small enough to pool cheaply.</summary>
|
||||
private const int BufferSize = 64 * 1024;
|
||||
|
||||
private readonly IProxiedHttpClientFactory _clients = clients ?? throw new ArgumentNullException(nameof(clients));
|
||||
private readonly BlobStore _blobs = blobs ?? throw new ArgumentNullException(nameof(blobs));
|
||||
private readonly HostThrottle _throttle = throttle ?? throw new ArgumentNullException(nameof(throttle));
|
||||
private readonly ILogger<MediaFetcher> _logger = logger ?? throw new ArgumentNullException(nameof(logger));
|
||||
|
||||
/// <inheritdoc />
|
||||
public async Task<FetchResult> FetchAsync(
|
||||
MediaCandidate candidate,
|
||||
FetchOptions options,
|
||||
CancellationToken cancellationToken = default
|
||||
)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(candidate);
|
||||
ArgumentNullException.ThrowIfNull(options);
|
||||
|
||||
var stopwatch = Stopwatch.StartNew();
|
||||
|
||||
using var budget = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken);
|
||||
budget.CancelAfter(options.MaxItemDuration);
|
||||
|
||||
try
|
||||
{
|
||||
var result = await FetchCoreAsync(candidate, options, stopwatch, budget.Token, cancellationToken)
|
||||
.ConfigureAwait(false);
|
||||
|
||||
// Cancelling a socket read surfaces as an IOException as often as an
|
||||
// OperationCanceledException, and the handlers below would otherwise turn a stop into
|
||||
// an ordinary per-item failure. One check here covers every return path.
|
||||
cancellationToken.ThrowIfCancellationRequested();
|
||||
|
||||
return result;
|
||||
}
|
||||
catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested)
|
||||
{
|
||||
throw;
|
||||
}
|
||||
catch (OperationCanceledException)
|
||||
{
|
||||
// The backstop fired rather than the user: an item that takes minutes is a failure of
|
||||
// this attempt, not of the run.
|
||||
return Failure(candidate.Url, SeenOutcome.Timeout, "Stalled", "item budget exceeded", stopwatch);
|
||||
}
|
||||
catch (ProxyUnavailableException)
|
||||
{
|
||||
return Failure(candidate.Url, SeenOutcome.Failed, "NoProxy", "no live proxy", stopwatch);
|
||||
}
|
||||
}
|
||||
|
||||
private async Task<FetchResult> FetchCoreAsync(
|
||||
MediaCandidate candidate,
|
||||
FetchOptions options,
|
||||
Stopwatch stopwatch,
|
||||
CancellationToken token,
|
||||
CancellationToken userToken
|
||||
)
|
||||
{
|
||||
using var leased = await _clients
|
||||
.LeaseAsync(options.Timeouts, options.RequireProxy, token)
|
||||
.ConfigureAwait(false);
|
||||
|
||||
HttpStatusCode? status = null;
|
||||
Exception? failure = null;
|
||||
var bytesReceived = false;
|
||||
|
||||
try
|
||||
{
|
||||
var visited = new HashSet<string>(StringComparer.OrdinalIgnoreCase);
|
||||
var current = candidate.Url;
|
||||
var referer = candidate.Referer;
|
||||
|
||||
for (var hop = 0; ; hop++)
|
||||
{
|
||||
if (hop > options.MaxRedirects)
|
||||
{
|
||||
return Failure(current, SeenOutcome.Failed, "TooManyRedirects", null, stopwatch, leased.ProxyKey);
|
||||
}
|
||||
|
||||
if (!visited.Add(current.AbsoluteUri))
|
||||
{
|
||||
return Failure(current, SeenOutcome.Failed, "RedirectLoop", null, stopwatch, leased.ProxyKey);
|
||||
}
|
||||
|
||||
using var turn = await _throttle.AcquireAsync(current, token).ConfigureAwait(false);
|
||||
|
||||
using var request = BuildRequest(current, referer, options);
|
||||
using var headerBudget = CancellationTokenSource.CreateLinkedTokenSource(token);
|
||||
headerBudget.CancelAfter(options.Timeouts.Headers);
|
||||
|
||||
HttpResponseMessage response;
|
||||
try
|
||||
{
|
||||
response = await leased
|
||||
.Client.SendAsync(request, HttpCompletionOption.ResponseHeadersRead, headerBudget.Token)
|
||||
.ConfigureAwait(false);
|
||||
}
|
||||
catch (OperationCanceledException) when (!userToken.IsCancellationRequested)
|
||||
{
|
||||
failure = new TimeoutException("headers");
|
||||
return Failure(current, SeenOutcome.Timeout, "Stalled", "headers", stopwatch, leased.ProxyKey);
|
||||
}
|
||||
|
||||
using (response)
|
||||
{
|
||||
status = response.StatusCode;
|
||||
|
||||
if (IsRedirect(response.StatusCode))
|
||||
{
|
||||
var next = ResolveRedirect(current, response.Headers.Location);
|
||||
|
||||
if (next is null)
|
||||
{
|
||||
return Failure(
|
||||
current,
|
||||
SeenOutcome.Failed,
|
||||
"BadRedirect",
|
||||
null,
|
||||
stopwatch,
|
||||
leased.ProxyKey
|
||||
);
|
||||
}
|
||||
|
||||
referer = current;
|
||||
current = next;
|
||||
continue;
|
||||
}
|
||||
|
||||
if (response.StatusCode is HttpStatusCode.TooManyRequests or HttpStatusCode.ServiceUnavailable)
|
||||
{
|
||||
var wait = ReadRetryAfter(response.Headers.RetryAfter);
|
||||
_throttle.ReportRateLimited(current, wait);
|
||||
|
||||
return Failure(
|
||||
current,
|
||||
SeenOutcome.RateLimited,
|
||||
"RateLimited",
|
||||
null,
|
||||
stopwatch,
|
||||
leased.ProxyKey,
|
||||
(int)response.StatusCode
|
||||
) with
|
||||
{
|
||||
RetryAfter = wait,
|
||||
};
|
||||
}
|
||||
|
||||
if (!response.IsSuccessStatusCode)
|
||||
{
|
||||
var outcome = response.StatusCode is HttpStatusCode.Gone or HttpStatusCode.NotFound
|
||||
? SeenOutcome.Gone
|
||||
: SeenOutcome.Failed;
|
||||
|
||||
return Failure(
|
||||
current,
|
||||
outcome,
|
||||
"HttpStatus",
|
||||
((int)response.StatusCode).ToString(System.Globalization.CultureInfo.InvariantCulture),
|
||||
stopwatch,
|
||||
leased.ProxyKey,
|
||||
(int)response.StatusCode
|
||||
);
|
||||
}
|
||||
|
||||
_throttle.ReportSuccess(current);
|
||||
|
||||
var declared = response.Content.Headers.ContentLength;
|
||||
if (declared > options.MaxItemBytes)
|
||||
{
|
||||
// Refused without reading a byte of it: the point of a size cap is not to
|
||||
// download the thing and then decide it was too big.
|
||||
return Failure(
|
||||
current,
|
||||
SeenOutcome.TooLarge,
|
||||
"TooLarge",
|
||||
null,
|
||||
stopwatch,
|
||||
leased.ProxyKey,
|
||||
(int)response.StatusCode
|
||||
);
|
||||
}
|
||||
|
||||
var download = await DownloadAsync(response, current, options, stopwatch, leased, token, userToken)
|
||||
.ConfigureAwait(false);
|
||||
|
||||
bytesReceived = download.BytesReceived;
|
||||
failure = download.Failure;
|
||||
|
||||
return download.Result;
|
||||
}
|
||||
}
|
||||
}
|
||||
catch (HttpRequestException ex)
|
||||
{
|
||||
failure = ex;
|
||||
return Failure(candidate.Url, SeenOutcome.Failed, "RequestFailed", ex.Message, stopwatch, leased.ProxyKey);
|
||||
}
|
||||
catch (IOException ex)
|
||||
{
|
||||
failure = ex;
|
||||
return Failure(candidate.Url, SeenOutcome.Failed, "RequestFailed", ex.Message, stopwatch, leased.ProxyKey);
|
||||
}
|
||||
finally
|
||||
{
|
||||
var verdict = LeaseVerdicts.Decide(status, failure, bytesReceived, userToken.IsCancellationRequested);
|
||||
|
||||
switch (verdict.Verdict)
|
||||
{
|
||||
case LeaseVerdict.Success:
|
||||
leased.Lease?.ReportSuccess(stopwatch.Elapsed);
|
||||
break;
|
||||
case LeaseVerdict.Failure:
|
||||
leased.Lease?.ReportFailure(verdict.Reason);
|
||||
break;
|
||||
default:
|
||||
// Neutral: say nothing, so the pool learns nothing from a cancelled attempt.
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private async Task<(FetchResult Result, bool BytesReceived, Exception? Failure)> DownloadAsync(
|
||||
HttpResponseMessage response,
|
||||
Uri url,
|
||||
FetchOptions options,
|
||||
Stopwatch stopwatch,
|
||||
LeasedHttpClient leased,
|
||||
CancellationToken token,
|
||||
CancellationToken userToken
|
||||
)
|
||||
{
|
||||
var contentType = response.Content.Headers.ContentType?.MediaType;
|
||||
var status = (int)response.StatusCode;
|
||||
var declared = response.Content.Headers.ContentLength;
|
||||
|
||||
// Automatic decompression strips the encoding, so a declared length that came with one
|
||||
// described the compressed body and cannot be compared against what we read.
|
||||
var lengthIsComparable = declared.HasValue && response.Content.Headers.ContentEncoding.Count == 0;
|
||||
|
||||
var temp = _blobs.CreateTempPath();
|
||||
var buffer = ArrayPool<byte>.Shared.Rent(BufferSize);
|
||||
var prefix = new byte[MediaSignatures.PrefixLength];
|
||||
var prefixLength = 0;
|
||||
var total = 0L;
|
||||
var kind = MediaKind.Unknown;
|
||||
var sniffed = false;
|
||||
string? hash = null;
|
||||
Exception? failure = null;
|
||||
|
||||
try
|
||||
{
|
||||
// Scoped so the file is closed before it is read back below. Writing with FileShare.None
|
||||
// is deliberate — nothing else may see a partial file — which makes closing it first a
|
||||
// requirement rather than tidiness.
|
||||
{
|
||||
using var hasher = IncrementalHash.CreateHash(HashAlgorithmName.SHA256);
|
||||
await using var source = await response.Content.ReadAsStreamAsync(token).ConfigureAwait(false);
|
||||
await using var destination = new FileStream(
|
||||
temp,
|
||||
FileMode.Create,
|
||||
FileAccess.Write,
|
||||
FileShare.None,
|
||||
BufferSize,
|
||||
useAsync: true
|
||||
);
|
||||
|
||||
using var idle = CancellationTokenSource.CreateLinkedTokenSource(token);
|
||||
|
||||
while (true)
|
||||
{
|
||||
// Rescheduled before every read: the rule is "no bytes for N seconds", not
|
||||
// "the whole body within N seconds".
|
||||
idle.CancelAfter(options.Timeouts.Idle);
|
||||
|
||||
int read;
|
||||
try
|
||||
{
|
||||
read = await source.ReadAsync(buffer.AsMemory(0, BufferSize), idle.Token).ConfigureAwait(false);
|
||||
}
|
||||
catch (OperationCanceledException) when (!userToken.IsCancellationRequested)
|
||||
{
|
||||
failure = new TimeoutException("idle");
|
||||
return (
|
||||
Failure(url, SeenOutcome.Timeout, "Stalled", null, stopwatch, leased.ProxyKey, status),
|
||||
total > 0,
|
||||
failure
|
||||
);
|
||||
}
|
||||
catch (Exception ex) when (ex is HttpRequestException or IOException)
|
||||
{
|
||||
// The origin hung up mid-body. When it had declared a length, that is
|
||||
// precisely a truncated transfer; the distinction matters because a
|
||||
// truncated file must never reach the store.
|
||||
failure = ex;
|
||||
var truncated = lengthIsComparable && total > 0;
|
||||
|
||||
return (
|
||||
Failure(
|
||||
url,
|
||||
SeenOutcome.Failed,
|
||||
truncated ? "Truncated" : "RequestFailed",
|
||||
truncated ? null : ex.Message,
|
||||
stopwatch,
|
||||
leased.ProxyKey,
|
||||
status
|
||||
),
|
||||
total > 0,
|
||||
failure
|
||||
);
|
||||
}
|
||||
|
||||
if (read == 0)
|
||||
{
|
||||
break;
|
||||
}
|
||||
|
||||
total += read;
|
||||
|
||||
if (total > options.MaxItemBytes)
|
||||
{
|
||||
return (
|
||||
Failure(url, SeenOutcome.TooLarge, "TooLarge", null, stopwatch, leased.ProxyKey, status),
|
||||
true,
|
||||
null
|
||||
);
|
||||
}
|
||||
|
||||
if (prefixLength < prefix.Length)
|
||||
{
|
||||
var take = Math.Min(prefix.Length - prefixLength, read);
|
||||
buffer.AsSpan(0, take).CopyTo(prefix.AsSpan(prefixLength));
|
||||
prefixLength += take;
|
||||
}
|
||||
|
||||
if (!sniffed && prefixLength >= prefix.Length)
|
||||
{
|
||||
sniffed = true;
|
||||
kind = MediaSignatures.Detect(prefix);
|
||||
|
||||
if (Reject(kind, prefix, options) is { } early)
|
||||
{
|
||||
return (
|
||||
Failure(
|
||||
url,
|
||||
early.Outcome,
|
||||
early.Code,
|
||||
contentType,
|
||||
stopwatch,
|
||||
leased.ProxyKey,
|
||||
status
|
||||
),
|
||||
true,
|
||||
null
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
hasher.AppendData(buffer, 0, read);
|
||||
await destination.WriteAsync(buffer.AsMemory(0, read), token).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
await destination.FlushAsync(token).ConfigureAwait(false);
|
||||
hash = Convert.ToHexStringLower(hasher.GetCurrentHash());
|
||||
}
|
||||
|
||||
// A body shorter than the length it announced is a truncated transfer, not a small file.
|
||||
if (lengthIsComparable && total != declared!.Value)
|
||||
{
|
||||
failure = new IOException("truncated");
|
||||
return (
|
||||
Failure(url, SeenOutcome.Failed, "Truncated", null, stopwatch, leased.ProxyKey, status),
|
||||
true,
|
||||
failure
|
||||
);
|
||||
}
|
||||
|
||||
// Files shorter than the sniff window never reached the check inside the loop.
|
||||
if (!sniffed)
|
||||
{
|
||||
kind = MediaSignatures.Detect(prefix.AsSpan(0, prefixLength));
|
||||
|
||||
if (Reject(kind, prefix.AsSpan(0, prefixLength), options) is { } rejection)
|
||||
{
|
||||
return (
|
||||
Failure(
|
||||
url,
|
||||
rejection.Outcome,
|
||||
rejection.Code,
|
||||
contentType,
|
||||
stopwatch,
|
||||
leased.ProxyKey,
|
||||
status
|
||||
),
|
||||
true,
|
||||
null
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
if (total < options.MinItemBytes)
|
||||
{
|
||||
return (
|
||||
Failure(url, SeenOutcome.TooSmall, "TooSmall", null, stopwatch, leased.ProxyKey, status),
|
||||
true,
|
||||
null
|
||||
);
|
||||
}
|
||||
|
||||
if (options.Tombstones.Contains(hash!))
|
||||
{
|
||||
return (
|
||||
Failure(url, SeenOutcome.Placeholder, "Placeholder", null, stopwatch, leased.ProxyKey, status),
|
||||
true,
|
||||
null
|
||||
);
|
||||
}
|
||||
|
||||
var blob = await DescribeAsync(temp, hash!, kind, total, prefix, token).ConfigureAwait(false);
|
||||
|
||||
var result = new FetchResult(SeenOutcome.Stored, url)
|
||||
{
|
||||
Blob = blob,
|
||||
TempPath = temp,
|
||||
HttpStatus = status,
|
||||
ContentType = contentType,
|
||||
ProxyKey = leased.ProxyKey,
|
||||
Elapsed = stopwatch.Elapsed,
|
||||
};
|
||||
|
||||
temp = null; // ownership passes to the caller
|
||||
return (result, true, null);
|
||||
}
|
||||
finally
|
||||
{
|
||||
ArrayPool<byte>.Shared.Return(buffer);
|
||||
|
||||
// Anything that did not run to a verified end leaves nothing behind. This is what keeps
|
||||
// the blob store free of half-written files.
|
||||
if (temp is not null)
|
||||
{
|
||||
TryDelete(temp);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>Re-reads the head of the staged file to settle animation and dimensions.</summary>
|
||||
/// <remarks>
|
||||
/// A second pass rather than a streaming state machine: whether a PNG is animated depends on a
|
||||
/// chunk that can sit anywhere before the first <c>IDAT</c>, and tracking that across arbitrary
|
||||
/// read boundaries is a great deal of fiddly code to avoid re-reading a megabyte that is still
|
||||
/// in the page cache.
|
||||
/// </remarks>
|
||||
private static async Task<MediaBlob> DescribeAsync(
|
||||
string path,
|
||||
string hash,
|
||||
MediaKind kind,
|
||||
long length,
|
||||
byte[] prefix,
|
||||
CancellationToken token
|
||||
)
|
||||
{
|
||||
var blob = MediaBlob.Create(hash, kind, length);
|
||||
var (width, height) = MediaSignatures.ReadDimensions(kind, prefix);
|
||||
|
||||
var headLength = (int)Math.Min(length, MediaSignatures.AnimationScanLimit);
|
||||
var head = new byte[headLength];
|
||||
|
||||
await using (var stream = new FileStream(path, FileMode.Open, FileAccess.Read, FileShare.Read))
|
||||
{
|
||||
await stream.ReadExactlyAsync(head.AsMemory(0, headLength), token).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
return blob with
|
||||
{
|
||||
Width = width,
|
||||
Height = height,
|
||||
IsAnimated = MediaSignatures.DetectAnimation(kind, head),
|
||||
};
|
||||
}
|
||||
|
||||
private static (SeenOutcome Outcome, string Code)? Reject(
|
||||
MediaKind kind,
|
||||
ReadOnlySpan<byte> prefix,
|
||||
FetchOptions options
|
||||
)
|
||||
{
|
||||
if (MediaSignatures.LooksLikeHtml(prefix))
|
||||
{
|
||||
// A dead link answered with an error page and a 200 is the normal behaviour of several
|
||||
// large image hosts; without this the store fills with copies of their apology page.
|
||||
return (SeenOutcome.NotMedia, "NotMedia");
|
||||
}
|
||||
|
||||
if (kind == MediaKind.Unknown)
|
||||
{
|
||||
return (SeenOutcome.NotMedia, "NotMedia");
|
||||
}
|
||||
|
||||
return options.AllowedKinds.Contains(kind) ? null : (SeenOutcome.UnsupportedType, "UnsupportedType");
|
||||
}
|
||||
|
||||
private static HttpRequestMessage BuildRequest(Uri url, Uri? referer, FetchOptions options)
|
||||
{
|
||||
var request = new HttpRequestMessage(HttpMethod.Get, url);
|
||||
|
||||
request.Headers.UserAgent.ParseAdd(options.UserAgent);
|
||||
request.Headers.Accept.ParseAdd("image/avif,image/webp,image/apng,image/*,video/*;q=0.8,*/*;q=0.5");
|
||||
|
||||
if (referer is not null)
|
||||
{
|
||||
request.Headers.Referrer = referer;
|
||||
}
|
||||
|
||||
return request;
|
||||
}
|
||||
|
||||
private static bool IsRedirect(HttpStatusCode status) =>
|
||||
status
|
||||
is HttpStatusCode.MovedPermanently
|
||||
or HttpStatusCode.Found
|
||||
or HttpStatusCode.SeeOther
|
||||
or HttpStatusCode.TemporaryRedirect
|
||||
or HttpStatusCode.PermanentRedirect;
|
||||
|
||||
/// <summary>Resolves a <c>Location</c> header, refusing anything that is not HTTP.</summary>
|
||||
private static Uri? ResolveRedirect(Uri current, Uri? location)
|
||||
{
|
||||
if (location is null)
|
||||
{
|
||||
return null;
|
||||
}
|
||||
|
||||
var resolved = location.IsAbsoluteUri ? location : new Uri(current, location);
|
||||
|
||||
// data:, file: and friends have no business being the target of a media fetch.
|
||||
return resolved.Scheme is "http" or "https" ? resolved : null;
|
||||
}
|
||||
|
||||
private static TimeSpan? ReadRetryAfter(RetryConditionHeaderValue? header)
|
||||
{
|
||||
if (header is null)
|
||||
{
|
||||
return null;
|
||||
}
|
||||
|
||||
var raw = header.Delta is { } delta
|
||||
? ((int)delta.TotalSeconds).ToString(System.Globalization.CultureInfo.InvariantCulture)
|
||||
: header.Date?.ToString("r", System.Globalization.CultureInfo.InvariantCulture);
|
||||
|
||||
return RetryAfter.Parse(raw, DateTimeOffset.UtcNow, out _);
|
||||
}
|
||||
|
||||
private static FetchResult Failure(
|
||||
Uri url,
|
||||
SeenOutcome outcome,
|
||||
string code,
|
||||
string? detail,
|
||||
Stopwatch stopwatch,
|
||||
string? proxyKey = null,
|
||||
int status = 0
|
||||
) =>
|
||||
new(outcome, url)
|
||||
{
|
||||
ErrorCode = code,
|
||||
ErrorDetail = detail,
|
||||
HttpStatus = status,
|
||||
ProxyKey = proxyKey,
|
||||
Elapsed = stopwatch.Elapsed,
|
||||
};
|
||||
|
||||
private void TryDelete(string path)
|
||||
{
|
||||
try
|
||||
{
|
||||
if (File.Exists(path))
|
||||
{
|
||||
File.Delete(path);
|
||||
}
|
||||
}
|
||||
catch (Exception ex) when (ex is IOException or UnauthorizedAccessException)
|
||||
{
|
||||
_logger.LogWarning(ex, "Could not remove the staged file {Path}", path);
|
||||
}
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user