mrleo1nid/av-parser
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Gate network parsers on a working proxy and remember what worked

Browse Source 
The pool now warms up from what the previous run learned instead of starting
cold every launch. Startup probes the remembered proxies first, stops as soon
as ProxyMinimumLive of them answer, and writes the survivors to
proxies.state.json after the warm-up and again on shutdown. Only proxies that
ever answered are stored: the feed republishes a few thousand dead addresses
every five minutes, and "was dead an hour ago" says almost nothing.

Remembered state is a hint, not a verdict. A restored proxy sorts first in the
warm-up queue but is not counted live until it answers in this session -
otherwise a launch a week later would report live proxies it had never spoken
to, the warm-up would skip the very entries it exists to re-check, and the
parser gate would open on week-old evidence.

That gate is the other half: a parser declaring RequiresNetwork will not run
while the pool has nothing live. The Parse page disables the run button and
shows a banner that leads to the Proxies page. Parsers that work on pasted text
are never gated - they have nothing to route, and blocking them would make the
app useless whenever the public lists are down. Two new settings cover the
escape hatch and the target: "allow network parsers without a proxy" and how
many live proxies to find at startup.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
Leonid Pershin
2026-08-13 19:11:51 +03:00
co-authored by Claude Opus 5
parent 85656e70b0
commit 44fb0d3a5f
31 changed files with 1349 additions and 41 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tests/AvParser.UI.Tests/ParseViewModelTests.cs
+113 -2
View File
@@ -1,5 +1,6 @@
using AvParser.Core.Parsing;
using AvParser.Core.Parsing.Samples;
using AvParser.Core.Proxies;
using AvParser.Core.Settings;
using AvParser.UI.Tests.Fakes;
using AvParser.UI.ViewModels;
@@ -11,15 +12,21 @@ namespace AvParser.UI.Tests;
public class ParseViewModelTests
{
private static (ParseViewModel Page, FakeSettingsService Settings) Build(AppSettings? settings = null)
private static (ParseViewModel Page, FakeSettingsService Settings) Build(
AppSettings? settings = null,
IProxyPool? proxyPool = null,
params ITextParser[] extraParsers
)
{
var catalog = new ParserCatalog([new DelimitedTextParser(), new KeyValueTextParser()]);
var catalog = new ParserCatalog([new DelimitedTextParser(), new KeyValueTextParser(), .. extraParsers]);
var settingsService = new FakeSettingsService(settings);
return (
new ParseViewModel(
catalog,
settingsService,
proxyPool ?? new ProxyPool([], new FakeProxyProbe(), new ProxyOptions()),
new EmptyServiceProvider(),
NullLogger<ParseViewModel>.Instance,
ImmediateSequencer.Instance
),
@@ -27,6 +34,36 @@ public class ParseViewModelTests
);
}
/// <summary>A container that resolves nothing; the page only reaches for navigation on click.</summary>
private sealed class EmptyServiceProvider : IServiceProvider
{
public object? GetService(Type serviceType) => null;
}
/// <summary>Stands in for the kind of parser the proxy gate exists for.</summary>
private sealed class NetworkParser : ITextParser
{
public string Id => "network";
public string DisplayName => "Network parser";
public string Description => "Fetches something";
public bool RequiresNetwork => true;
public bool CanParse(string input) => true;
public async IAsyncEnumerable<ParseOutcome<ParsedRecord>> ParseAsync(
string input,
IProgress<ParseProgress>? progress,
[System.Runtime.CompilerServices.EnumeratorCancellation] CancellationToken cancellationToken
)
{
await Task.Yield();
yield break;
}
}
private static Task RunAsync(ParseViewModel page) => page.ParseCommand.Execute().ToTask();
/// <summary>
@@ -67,6 +104,80 @@ public class ParseViewModelTests
settings.Current.LastParserId.ShouldBe("key-value");
}
[Fact]
public void A_local_parser_runs_even_with_no_proxy_at_all()
{
// Only network parsers are gated. Blocking a parser that works on pasted text would make
// the app unusable whenever the public lists are down, for no benefit at all.
var (page, _) = Build();
page.IsBlockedWithoutProxy.ShouldBeFalse();
}
[Fact]
public void A_network_parser_is_blocked_while_nothing_is_live()
{
var (page, _) = Build(
new AppSettings { LastParserId = "network" },
new ProxyPool([], new FakeProxyProbe(), new ProxyOptions()),
new NetworkParser()
);
var canExecute = true;
using var subscription = page.ParseCommand.CanExecute.Subscribe(value => canExecute = value);
page.IsBlockedWithoutProxy.ShouldBeTrue();
page.InputText = "anything";
canExecute.ShouldBeFalse();
}
[Fact]
public async Task A_network_parser_runs_once_a_proxy_answers()
{
var endpoint = new ProxyEndpoint(ProxyProtocol.Http, "1.2.3.4", 8080);
var pool = new ProxyPool(
[new FakeProxySource([endpoint])],
new FakeProxyProbe().Set(endpoint, alive: true),
new ProxyOptions()
);
var (page, _) = Build(new AppSettings { LastParserId = "network" }, pool, new NetworkParser());
page.IsBlockedWithoutProxy.ShouldBeTrue();
await pool.RefreshAsync(TestContext.Current.CancellationToken);
await pool.WarmUpAsync(1, cancellationToken: TestContext.Current.CancellationToken);
page.RefreshProxyGate();
page.IsBlockedWithoutProxy.ShouldBeFalse();
}
[Fact]
public void Allowing_direct_connections_lifts_the_gate()
{
var (page, _) = Build(
new AppSettings { LastParserId = "network", AllowDirectConnection = true },
new ProxyPool([], new FakeProxyProbe(), new ProxyOptions()),
new NetworkParser()
);
page.IsBlockedWithoutProxy.ShouldBeFalse();
}
[Fact]
public void Switching_away_from_a_network_parser_lifts_the_gate()
{
var (page, _) = Build(
new AppSettings { LastParserId = "network" },
new ProxyPool([], new FakeProxyProbe(), new ProxyOptions()),
new NetworkParser()
);
page.SelectedParser = page.Parsers.Single(parser => parser.Id == "delimited");
page.IsBlockedWithoutProxy.ShouldBeFalse();
}
[Theory]
[InlineData("", false)]
[InlineData(" ", false)]