Third step: the collector becomes wireable. Both catalogs coexist for exactly this one step, so ParseViewModel and every existing test stay green while the new domain is proven. IMediaSource reuses the closed-generic trick ITextParser used, and for the same reason - the container cannot resolve an open generic as IEnumerable<T>, so adding a source stays a one-line registration. Its input is a MediaQuery rather than text, because a source that walks a paginated listing needs an endpoint and a cursor, not a string. Sources discover; they do not download. That split is why UrlListSource lives in the domain with no network at all, and why everything hard about fetching lives in one place instead of once per source. The catalog takes an explicit default id. Left to alphabetical order the landing source would be the network one, so the app would open behind the proxy gate before the user had asked for anything. The runner decouples discovery from downloading with a bounded channel - a listing of two hundred thousand items must not materialise because the workers are slower than the source - and owns its workers, waiting for them even when cancelled. Without that a stopped run keeps writing to the store after the page has said it stopped. The own-service listing is read leniently: the service on the other end is the user's own and should not have to be rewritten to match a schema we invented, so both a bare array of addresses and an object with items and a cursor work. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
136 lines
5.9 KiB
C#
136 lines
5.9 KiB
C#
using AvParser.Core.Collecting;
|
|
using AvParser.Core.DependencyInjection;
|
|
using AvParser.Core.Proxies;
|
|
using AvParser.Core.Settings;
|
|
using AvParser.Infrastructure.Collecting;
|
|
using AvParser.Infrastructure.Media;
|
|
using AvParser.Infrastructure.Proxies;
|
|
using AvParser.Infrastructure.Settings;
|
|
using AvParser.Infrastructure.Storage;
|
|
using Microsoft.Extensions.DependencyInjection;
|
|
using Microsoft.Extensions.Logging;
|
|
|
|
namespace AvParser.Infrastructure.DependencyInjection;
|
|
|
|
/// <summary>Composition root for the infrastructure layer.</summary>
|
|
public static class InfrastructureServiceCollectionExtensions
|
|
{
|
|
/// <summary>Registers filesystem paths and the persisted settings service.</summary>
|
|
/// <param name="services">The collection to add to.</param>
|
|
/// <param name="paths">
|
|
/// Explicit paths, or <see langword="null"/> to use the current user's application-data folder.
|
|
/// Tests pass a temp directory here.
|
|
/// </param>
|
|
public static IServiceCollection AddAvParserInfrastructure(this IServiceCollection services, AppPaths? paths = null)
|
|
{
|
|
ArgumentNullException.ThrowIfNull(services);
|
|
|
|
var resolved = paths ?? new AppPaths();
|
|
resolved.EnsureCreated();
|
|
|
|
services.AddSingleton<IAppPaths>(resolved);
|
|
|
|
// Constructed explicitly rather than by type: the optional ISequencer parameter would
|
|
// otherwise make the container's constructor choice depend on registration order.
|
|
services.AddSingleton<ISettingsService>(sp => new JsonSettingsService(
|
|
sp.GetRequiredService<IAppPaths>(),
|
|
sp.GetRequiredService<ILogger<JsonSettingsService>>()
|
|
));
|
|
|
|
services.AddAvParserProxies();
|
|
services.AddAvParserCollecting();
|
|
|
|
return services;
|
|
}
|
|
|
|
/// <summary>Registers the media store, the download pipeline and the network sources.</summary>
|
|
/// <remarks>
|
|
/// Everything here is a singleton because everything here owns something shared: a database
|
|
/// connection pool, a per-host throttle whose whole purpose is being common to all workers, and
|
|
/// a blob directory that must have exactly one owner deciding what is complete.
|
|
/// </remarks>
|
|
public static IServiceCollection AddAvParserCollecting(this IServiceCollection services)
|
|
{
|
|
ArgumentNullException.ThrowIfNull(services);
|
|
|
|
services.AddSingleton<BlobStore>();
|
|
services.AddSingleton<SqliteMediaIndex>();
|
|
|
|
// Explicit factories: both take an optional parameter, and letting the container pick a
|
|
// constructor by registration order is how that goes wrong quietly.
|
|
services.AddSingleton(sp => new ShowcaseLinker(
|
|
sp.GetRequiredService<IAppPaths>(),
|
|
sp.GetRequiredService<BlobStore>(),
|
|
sp.GetRequiredService<ILogger<ShowcaseLinker>>()
|
|
));
|
|
|
|
services.AddSingleton<MediaStore>();
|
|
services.AddSingleton<IMediaStore>(sp => sp.GetRequiredService<MediaStore>());
|
|
|
|
// Two concurrent requests per origin, a quarter of a second apart. These become settings in
|
|
// their own right; until then the defaults are the polite ones.
|
|
services.AddSingleton(sp => new HostThrottle(
|
|
maxConcurrentPerHost: 2,
|
|
minimumInterval: TimeSpan.FromMilliseconds(250),
|
|
sp.GetRequiredService<ILogger<HostThrottle>>()
|
|
));
|
|
|
|
services.AddSingleton<IMediaFetcher, MediaFetcher>();
|
|
services.AddSingleton<ICollectRunner, CollectRunner>();
|
|
|
|
services.AddSingleton<IMediaSource, OwnServiceSource>();
|
|
|
|
// Resolves sources from both assemblies: the container gathers every IMediaSource
|
|
// registration regardless of which project declared it.
|
|
services.AddSingleton<IMediaSourceCatalog>(sp => new MediaSourceCatalog(
|
|
sp.GetServices<IMediaSource>(),
|
|
CoreServiceCollectionExtensions.DefaultMediaSourceId
|
|
));
|
|
|
|
return services;
|
|
}
|
|
|
|
/// <summary>Registers the proxy sources, the probe, the pool and the proxied client factory.</summary>
|
|
/// <remarks>
|
|
/// The feed source gets a pooled <see cref="HttpClient"/> because it always talks to the same
|
|
/// CDN host. The probe deliberately does not: its handler carries the proxy, so it has to
|
|
/// build one per check.
|
|
/// </remarks>
|
|
public static IServiceCollection AddAvParserProxies(this IServiceCollection services)
|
|
{
|
|
ArgumentNullException.ThrowIfNull(services);
|
|
|
|
services
|
|
.AddHttpClient<ProxiflyProxySource>(client =>
|
|
{
|
|
client.Timeout = TimeSpan.FromSeconds(30);
|
|
client.DefaultRequestHeaders.UserAgent.ParseAdd("AvParser/0.1");
|
|
})
|
|
.ConfigurePrimaryHttpMessageHandler(() =>
|
|
new SocketsHttpHandler { AutomaticDecompression = System.Net.DecompressionMethods.All }
|
|
);
|
|
|
|
services.AddSingleton<CustomProxySource>();
|
|
services.AddSingleton<IMutableProxySource>(sp => sp.GetRequiredService<CustomProxySource>());
|
|
|
|
// Registration order here is the order the pool merges sources; the custom list comes
|
|
// last so a user-entered address wins over a feed entry for the same host and port.
|
|
services.AddSingleton<IProxySource>(sp => sp.GetRequiredService<ProxiflyProxySource>());
|
|
services.AddSingleton<IProxySource>(sp => sp.GetRequiredService<CustomProxySource>());
|
|
|
|
services.AddSingleton<IProxyProbe, HttpProxyProbe>();
|
|
|
|
services.AddSingleton<IProxyPool>(sp => new ProxyPool(
|
|
sp.GetServices<IProxySource>(),
|
|
sp.GetRequiredService<IProxyProbe>(),
|
|
sp.GetRequiredService<ISettingsService>().Current.ToProxyOptions()
|
|
));
|
|
|
|
services.AddSingleton<IProxiedHttpClientFactory, ProxiedHttpClientFactory>();
|
|
services.AddSingleton<IProxyStateStore, ProxyStateStore>();
|
|
services.AddSingleton<IProxyPoolLoader, ProxyPoolLoader>();
|
|
|
|
return services;
|
|
}
|
|
}
|