first commit
This commit is contained in:
Leonid Pershin
@@ -0,0 +1,309 @@
|
||||
@tool
|
||||
extends RefCounted
|
||||
|
||||
## Discovers and runs McpTestSuite scripts from res://tests/.
|
||||
## Exposes run_tests and get_test_results as MCP commands.
|
||||
##
|
||||
## Live MCP runs service the WebSocket transport between tests
|
||||
## (McpConnection.service_transport_during_exclusive_run) so a long suite
|
||||
## can no longer starve the server's keepalive, and abort at a per-call
|
||||
## ceiling derived from the server-provided time budget. Direct callers,
|
||||
## unit-test fixtures, and batch contexts (no request id / no connection)
|
||||
## keep the legacy fully-synchronous behavior with no ceiling.
|
||||
## See docs/test-run-transport-starvation-plan.md.
|
||||
|
||||
const ErrorCodes := preload("res://addons/godot_ai/utils/error_codes.gd")
|
||||
|
||||
## Clamp bounds for the server-provided ``timeout_budget_sec`` param. The
|
||||
## floor is purely defensive (a malformed or buggy server value must not
|
||||
## abort every run instantly); the param is not user-facing.
|
||||
const BUDGET_MIN_SEC := 30.0
|
||||
const BUDGET_MAX_SEC := 3600.0
|
||||
## Conservative default when the server sent no (or an invalid) budget: an
|
||||
## old server's own test_run timeout is 120s, and the plugin must abort
|
||||
## and reply before that future expires.
|
||||
const BUDGET_DEFAULT_SEC := 110.0
|
||||
## Abort this long before the server would time the call out, so the
|
||||
## partial-results reply beats the server-side timeout.
|
||||
const CEILING_MARGIN_SEC := 10.0
|
||||
|
||||
var _runner: McpTestRunner
|
||||
var _undo_redo: EditorUndoRedoManager
|
||||
var _log_buffer: McpLogBuffer
|
||||
## Live plugin dispatcher, exposed to suites via ctx so tests can prove the
|
||||
## lazy handler registrations (#736) materialize with their real ctor args.
|
||||
## Optional third arg keeps old two-arg fixtures working; untyped because
|
||||
## the dispatcher constructs this handler (avoids a load-time type cycle).
|
||||
var _dispatcher
|
||||
## Live connection for exclusive-run transport servicing. Null in unit-test
|
||||
## fixtures and batch contexts, which keep the legacy synchronous path.
|
||||
var _connection: McpConnection
|
||||
|
||||
|
||||
func _init(
|
||||
undo_redo: EditorUndoRedoManager,
|
||||
log_buffer: McpLogBuffer,
|
||||
dispatcher = null,
|
||||
connection: McpConnection = null,
|
||||
) -> void:
|
||||
_runner = McpTestRunner.new()
|
||||
_undo_redo = undo_redo
|
||||
_log_buffer = log_buffer
|
||||
_dispatcher = dispatcher
|
||||
_connection = connection
|
||||
|
||||
|
||||
func run_tests(params: Dictionary) -> Dictionary:
|
||||
var suite_filter: String = params.get("suite", "")
|
||||
var test_filter: String = params.get("test_name", "")
|
||||
var exclude_test_filter: String = params.get("exclude_test_name", "")
|
||||
var verbose: bool = params.get("verbose", false)
|
||||
|
||||
var request_id: String = params.get("_request_id", "")
|
||||
var live := _connection != null and not request_id.is_empty()
|
||||
var service_cb := Callable()
|
||||
var deadline_ticks_ms := 0
|
||||
var budget_sec := 0.0
|
||||
var started_ms := Time.get_ticks_msec()
|
||||
var run_state := {}
|
||||
if live:
|
||||
budget_sec = _validated_budget_sec(params)
|
||||
service_cb = Callable(_connection, "service_transport_during_exclusive_run")
|
||||
deadline_ticks_ms = started_ms + int((budget_sec - CEILING_MARGIN_SEC) * 1000.0)
|
||||
|
||||
## Clear the previous run's results BEFORE discovery so an abort at any
|
||||
## later point can never expose a stale prior run via get_test_results.
|
||||
_runner.clear()
|
||||
|
||||
var discovery := _discover_suites(service_cb, deadline_ticks_ms, run_state)
|
||||
var discovery_outcome: String = discovery.get("outcome", "")
|
||||
if not discovery_outcome.is_empty():
|
||||
## Aborted during discovery: no suite has begun, so there is no
|
||||
## suite teardown to run. Same outcome mapping as the run itself.
|
||||
var empty_results: Dictionary = _runner.get_results(verbose)
|
||||
if not discovery.errors.is_empty():
|
||||
empty_results["load_errors"] = discovery.errors
|
||||
return _map_outcome(
|
||||
discovery_outcome, "discovery", empty_results, 0, started_ms, budget_sec
|
||||
)
|
||||
|
||||
var suites: Array = discovery.suites
|
||||
if suites.is_empty():
|
||||
var msg := "No test suites found in res://tests/"
|
||||
if not discovery.errors.is_empty():
|
||||
msg += " (%d script(s) failed to load: %s)" % [
|
||||
discovery.errors.size(),
|
||||
", ".join(discovery.errors),
|
||||
]
|
||||
var no_suites := {"error": msg, "total": 0, "load_errors": discovery.errors}
|
||||
## Keep the edited_scene annotation on the no-suites error payload too,
|
||||
## so the response contract is consistent across every return path.
|
||||
_annotate_edited_scene(no_suites)
|
||||
return {"data": no_suites}
|
||||
|
||||
var ctx := {
|
||||
"undo_redo": _undo_redo,
|
||||
"log_buffer": _log_buffer,
|
||||
"dispatcher": _dispatcher,
|
||||
}
|
||||
|
||||
var run: Dictionary = _runner.run_suites_serviced(
|
||||
suites, suite_filter, test_filter, ctx, verbose, exclude_test_filter,
|
||||
service_cb, deadline_ticks_ms, run_state
|
||||
)
|
||||
var results: Dictionary = run["results"]
|
||||
if not discovery.errors.is_empty():
|
||||
results["load_errors"] = discovery.errors
|
||||
return _map_outcome(
|
||||
run["outcome"], "run", results, run["tests_not_run"], started_ms, budget_sec
|
||||
)
|
||||
|
||||
|
||||
## Map a runner/discovery outcome onto the response envelope. Ownership is
|
||||
## deliberately here, not in the runner: the runner reports WHAT happened,
|
||||
## the handler decides how it goes over the wire (plan D2).
|
||||
func _map_outcome(
|
||||
outcome: String,
|
||||
phase: String,
|
||||
results: Dictionary,
|
||||
tests_not_run: int,
|
||||
started_ms: int,
|
||||
budget_sec: float,
|
||||
) -> Dictionary:
|
||||
var elapsed_ms := Time.get_ticks_msec() - started_ms
|
||||
match outcome:
|
||||
"completed":
|
||||
_annotate_edited_scene(results)
|
||||
return {"data": results}
|
||||
"transport_lost":
|
||||
## The peer is gone (or flood-closed); the send will fail against
|
||||
## the dead socket regardless, but a sync handler must return an
|
||||
## envelope. Partials stay retrievable via get_test_results after
|
||||
## the plugin reconnects.
|
||||
results["aborted"] = "transport_lost"
|
||||
results["tests_not_run"] = tests_not_run
|
||||
_annotate_edited_scene(results)
|
||||
return {"data": results}
|
||||
"paused":
|
||||
var depth := _connection.pause_depth() if _connection != null else 0
|
||||
if _log_buffer != null:
|
||||
_log_buffer.log(
|
||||
"[error] test run aborted in %s: transport paused at checkpoint (depth %d)"
|
||||
% [phase, depth]
|
||||
)
|
||||
var paused_err := ErrorCodes.make(
|
||||
ErrorCodes.INTERNAL_ERROR,
|
||||
(
|
||||
"Test run aborted: the MCP transport was paused at a between-test "
|
||||
+ "checkpoint (pause depth %d) — a paused transport cannot service "
|
||||
+ "the WebSocket heartbeat, so continuing would starve the session. "
|
||||
+ "Partial results: test_manage(op=\"results_get\")."
|
||||
) % depth
|
||||
)
|
||||
paused_err["error"]["data"] = _abort_data(
|
||||
phase, results, tests_not_run, elapsed_ms, budget_sec, {"pause_depth": depth}
|
||||
)
|
||||
return paused_err
|
||||
"timeout":
|
||||
var timeout_err := ErrorCodes.make(
|
||||
ErrorCodes.TEST_RUN_TIMEOUT,
|
||||
(
|
||||
"Test run hit its abort ceiling after %.1fs (budget %.0fs, ceiling = "
|
||||
+ "budget - %.0fs): %d passed, %d failed, %d of the selected tests "
|
||||
+ "never ran. Narrow the run with suite=/test_name= filters, or fetch "
|
||||
+ "the partial results with test_manage(op=\"results_get\")."
|
||||
) % [
|
||||
elapsed_ms / 1000.0, budget_sec, CEILING_MARGIN_SEC,
|
||||
int(results.get("passed", 0)), int(results.get("failed", 0)),
|
||||
tests_not_run,
|
||||
]
|
||||
)
|
||||
timeout_err["error"]["data"] = _abort_data(
|
||||
phase, results, tests_not_run, elapsed_ms, budget_sec, {}
|
||||
)
|
||||
return timeout_err
|
||||
## Unknown outcome is a runner bug — surface it loudly.
|
||||
return ErrorCodes.make(
|
||||
ErrorCodes.INTERNAL_ERROR, "Unknown test run outcome '%s'" % outcome
|
||||
)
|
||||
|
||||
|
||||
func _abort_data(
|
||||
phase: String,
|
||||
results: Dictionary,
|
||||
tests_not_run: int,
|
||||
elapsed_ms: int,
|
||||
budget_sec: float,
|
||||
extra: Dictionary,
|
||||
) -> Dictionary:
|
||||
var data := {
|
||||
"phase": phase,
|
||||
"elapsed_ms": elapsed_ms,
|
||||
"budget_sec": budget_sec,
|
||||
"passed": int(results.get("passed", 0)),
|
||||
"failed": int(results.get("failed", 0)),
|
||||
"skipped": int(results.get("skipped", 0)),
|
||||
"total": int(results.get("total", 0)),
|
||||
"tests_not_run": tests_not_run,
|
||||
}
|
||||
data.merge(extra)
|
||||
return data
|
||||
|
||||
|
||||
## Strict validation of the server-provided per-call budget: numeric,
|
||||
## finite, positive, then clamped to [BUDGET_MIN_SEC, BUDGET_MAX_SEC].
|
||||
## Everything else (missing, wrong type, NaN/inf, non-positive) falls back
|
||||
## to BUDGET_DEFAULT_SEC. typeof() so bool never sneaks through as int.
|
||||
func _validated_budget_sec(params: Dictionary) -> float:
|
||||
var raw: Variant = params.get("timeout_budget_sec", null)
|
||||
var t := typeof(raw)
|
||||
if t == TYPE_FLOAT or t == TYPE_INT:
|
||||
var v := float(raw)
|
||||
if is_finite(v) and v > 0.0:
|
||||
return clampf(v, BUDGET_MIN_SEC, BUDGET_MAX_SEC)
|
||||
return BUDGET_DEFAULT_SEC
|
||||
|
||||
|
||||
## Many suites assume the project's main scene is the edited scene (they read
|
||||
## /Main/... nodes directly). Running with another scene open produces a flood
|
||||
## of phantom failures that look like real regressions. Surface the edited
|
||||
## scene and a warning when it differs from run/main_scene so the failures are
|
||||
## attributable at a glance instead of costing a debugging round (#635).
|
||||
func _annotate_edited_scene(results: Dictionary) -> void:
|
||||
var scene_root := EditorInterface.get_edited_scene_root()
|
||||
var edited := scene_root.scene_file_path if scene_root else ""
|
||||
results["edited_scene"] = edited
|
||||
var main_scene := str(ProjectSettings.get_setting("application/run/main_scene", ""))
|
||||
if main_scene.is_empty() or edited == main_scene:
|
||||
return
|
||||
if int(results.get("failed", 0)) <= 0:
|
||||
return
|
||||
results["scene_warning"] = (
|
||||
"Edited scene is '%s' but the project main scene is '%s'. Many suites "
|
||||
% [edited if not edited.is_empty() else "<none>", main_scene]
|
||||
+ "assume the main scene is open and will report phantom failures "
|
||||
+ "otherwise. If these failures are unexpected, scene_open('%s') and re-run." % main_scene
|
||||
)
|
||||
|
||||
|
||||
func get_test_results(params: Dictionary) -> Dictionary:
|
||||
var verbose: bool = params.get("verbose", false)
|
||||
return {"data": _runner.get_results(verbose)}
|
||||
|
||||
|
||||
## Returns {"suites": Array, "errors": Array[String], "outcome": String}.
|
||||
## Resilient: a broken script doesn't kill discovery of the rest. A
|
||||
## non-empty outcome ("timeout" / "transport_lost" / "paused") means a
|
||||
## between-load checkpoint aborted discovery — script loading is itself an
|
||||
## atomic phase, and a directory of heavy scripts must neither starve the
|
||||
## heartbeat nor escape the run budget.
|
||||
func _discover_suites(
|
||||
service_cb: Callable = Callable(),
|
||||
deadline_ticks_ms: int = 0,
|
||||
run_state: Dictionary = {},
|
||||
) -> Dictionary:
|
||||
var suites := []
|
||||
var errors: Array[String] = []
|
||||
var dir := DirAccess.open("res://tests")
|
||||
if dir == null:
|
||||
return {
|
||||
"suites": suites,
|
||||
"errors": ["DirAccess.open('res://tests') returned null — directory may not exist"],
|
||||
"outcome": "",
|
||||
}
|
||||
|
||||
dir.list_dir_begin()
|
||||
var file_name := dir.get_next()
|
||||
while not file_name.is_empty():
|
||||
if file_name.begins_with("test_") and file_name.ends_with(".gd"):
|
||||
var stop := _discovery_checkpoint(service_cb, deadline_ticks_ms, run_state)
|
||||
if not stop.is_empty():
|
||||
return {"suites": suites, "errors": errors, "outcome": stop}
|
||||
var path := "res://tests/" + file_name
|
||||
var script = ResourceLoader.load(path, "", ResourceLoader.CACHE_MODE_IGNORE)
|
||||
if script == null:
|
||||
errors.append("%s (load failed — check for parse errors or duplicate methods)" % file_name)
|
||||
elif script.can_instantiate():
|
||||
var instance = script.new()
|
||||
if instance is McpTestSuite:
|
||||
suites.append(instance)
|
||||
else:
|
||||
errors.append("%s (not a McpTestSuite subclass)" % file_name)
|
||||
else:
|
||||
errors.append("%s (cannot instantiate — abstract or broken)" % file_name)
|
||||
file_name = dir.get_next()
|
||||
|
||||
## Sort by suite name for deterministic order.
|
||||
suites.sort_custom(func(a, b) -> bool:
|
||||
return a.suite_name() < b.suite_name()
|
||||
)
|
||||
return {"suites": suites, "errors": errors, "outcome": ""}
|
||||
|
||||
|
||||
## Discovery-phase twin of McpTestRunner._checkpoint. Both delegate to the
|
||||
## shared McpConnection.exclusive_run_checkpoint so the outcome mapping
|
||||
## cannot drift between the discovery and between-test paths.
|
||||
func _discovery_checkpoint(
|
||||
service_cb: Callable, deadline_ticks_ms: int, run_state: Dictionary
|
||||
) -> String:
|
||||
return McpConnection.exclusive_run_checkpoint(service_cb, deadline_ticks_ms, run_state)
|
||||
Reference in New Issue
Block a user