@tool extends RefCounted ## Discovers and runs McpTestSuite scripts from res://tests/. ## Exposes run_tests and get_test_results as MCP commands. ## ## Live MCP runs service the WebSocket transport between tests ## (McpConnection.service_transport_during_exclusive_run) so a long suite ## can no longer starve the server's keepalive, and abort at a per-call ## ceiling derived from the server-provided time budget. Direct callers, ## unit-test fixtures, and batch contexts (no request id / no connection) ## keep the legacy fully-synchronous behavior with no ceiling. ## See docs/test-run-transport-starvation-plan.md. const ErrorCodes := preload("res://addons/godot_ai/utils/error_codes.gd") ## Clamp bounds for the server-provided ``timeout_budget_sec`` param. The ## floor is purely defensive (a malformed or buggy server value must not ## abort every run instantly); the param is not user-facing. const BUDGET_MIN_SEC := 30.0 const BUDGET_MAX_SEC := 3600.0 ## Conservative default when the server sent no (or an invalid) budget: an ## old server's own test_run timeout is 120s, and the plugin must abort ## and reply before that future expires. const BUDGET_DEFAULT_SEC := 110.0 ## Abort this long before the server would time the call out, so the ## partial-results reply beats the server-side timeout. const CEILING_MARGIN_SEC := 10.0 var _runner: McpTestRunner var _undo_redo: EditorUndoRedoManager var _log_buffer: McpLogBuffer ## Live plugin dispatcher, exposed to suites via ctx so tests can prove the ## lazy handler registrations (#736) materialize with their real ctor args. ## Optional third arg keeps old two-arg fixtures working; untyped because ## the dispatcher constructs this handler (avoids a load-time type cycle). var _dispatcher ## Live connection for exclusive-run transport servicing. Null in unit-test ## fixtures and batch contexts, which keep the legacy synchronous path. var _connection: McpConnection func _init( undo_redo: EditorUndoRedoManager, log_buffer: McpLogBuffer, dispatcher = null, connection: McpConnection = null, ) -> void: _runner = McpTestRunner.new() _undo_redo = undo_redo _log_buffer = log_buffer _dispatcher = dispatcher _connection = connection func run_tests(params: Dictionary) -> Dictionary: var suite_filter: String = params.get("suite", "") var test_filter: String = params.get("test_name", "") var exclude_test_filter: String = params.get("exclude_test_name", "") var verbose: bool = params.get("verbose", false) var request_id: String = params.get("_request_id", "") var live := _connection != null and not request_id.is_empty() var service_cb := Callable() var deadline_ticks_ms := 0 var budget_sec := 0.0 var started_ms := Time.get_ticks_msec() var run_state := {} if live: budget_sec = _validated_budget_sec(params) service_cb = Callable(_connection, "service_transport_during_exclusive_run") deadline_ticks_ms = started_ms + int((budget_sec - CEILING_MARGIN_SEC) * 1000.0) ## Clear the previous run's results BEFORE discovery so an abort at any ## later point can never expose a stale prior run via get_test_results. _runner.clear() var discovery := _discover_suites(service_cb, deadline_ticks_ms, run_state) var discovery_outcome: String = discovery.get("outcome", "") if not discovery_outcome.is_empty(): ## Aborted during discovery: no suite has begun, so there is no ## suite teardown to run. Same outcome mapping as the run itself. var empty_results: Dictionary = _runner.get_results(verbose) if not discovery.errors.is_empty(): empty_results["load_errors"] = discovery.errors return _map_outcome( discovery_outcome, "discovery", empty_results, 0, started_ms, budget_sec ) var suites: Array = discovery.suites if suites.is_empty(): var msg := "No test suites found in res://tests/" if not discovery.errors.is_empty(): msg += " (%d script(s) failed to load: %s)" % [ discovery.errors.size(), ", ".join(discovery.errors), ] var no_suites := {"error": msg, "total": 0, "load_errors": discovery.errors} ## Keep the edited_scene annotation on the no-suites error payload too, ## so the response contract is consistent across every return path. _annotate_edited_scene(no_suites) return {"data": no_suites} var ctx := { "undo_redo": _undo_redo, "log_buffer": _log_buffer, "dispatcher": _dispatcher, } var run: Dictionary = _runner.run_suites_serviced( suites, suite_filter, test_filter, ctx, verbose, exclude_test_filter, service_cb, deadline_ticks_ms, run_state ) var results: Dictionary = run["results"] if not discovery.errors.is_empty(): results["load_errors"] = discovery.errors return _map_outcome( run["outcome"], "run", results, run["tests_not_run"], started_ms, budget_sec ) ## Map a runner/discovery outcome onto the response envelope. Ownership is ## deliberately here, not in the runner: the runner reports WHAT happened, ## the handler decides how it goes over the wire (plan D2). func _map_outcome( outcome: String, phase: String, results: Dictionary, tests_not_run: int, started_ms: int, budget_sec: float, ) -> Dictionary: var elapsed_ms := Time.get_ticks_msec() - started_ms match outcome: "completed": _annotate_edited_scene(results) return {"data": results} "transport_lost": ## The peer is gone (or flood-closed); the send will fail against ## the dead socket regardless, but a sync handler must return an ## envelope. Partials stay retrievable via get_test_results after ## the plugin reconnects. results["aborted"] = "transport_lost" results["tests_not_run"] = tests_not_run _annotate_edited_scene(results) return {"data": results} "paused": var depth := _connection.pause_depth() if _connection != null else 0 if _log_buffer != null: _log_buffer.log( "[error] test run aborted in %s: transport paused at checkpoint (depth %d)" % [phase, depth] ) var paused_err := ErrorCodes.make( ErrorCodes.INTERNAL_ERROR, ( "Test run aborted: the MCP transport was paused at a between-test " + "checkpoint (pause depth %d) — a paused transport cannot service " + "the WebSocket heartbeat, so continuing would starve the session. " + "Partial results: test_manage(op=\"results_get\")." ) % depth ) paused_err["error"]["data"] = _abort_data( phase, results, tests_not_run, elapsed_ms, budget_sec, {"pause_depth": depth} ) return paused_err "timeout": var timeout_err := ErrorCodes.make( ErrorCodes.TEST_RUN_TIMEOUT, ( "Test run hit its abort ceiling after %.1fs (budget %.0fs, ceiling = " + "budget - %.0fs): %d passed, %d failed, %d of the selected tests " + "never ran. Narrow the run with suite=/test_name= filters, or fetch " + "the partial results with test_manage(op=\"results_get\")." ) % [ elapsed_ms / 1000.0, budget_sec, CEILING_MARGIN_SEC, int(results.get("passed", 0)), int(results.get("failed", 0)), tests_not_run, ] ) timeout_err["error"]["data"] = _abort_data( phase, results, tests_not_run, elapsed_ms, budget_sec, {} ) return timeout_err ## Unknown outcome is a runner bug — surface it loudly. return ErrorCodes.make( ErrorCodes.INTERNAL_ERROR, "Unknown test run outcome '%s'" % outcome ) func _abort_data( phase: String, results: Dictionary, tests_not_run: int, elapsed_ms: int, budget_sec: float, extra: Dictionary, ) -> Dictionary: var data := { "phase": phase, "elapsed_ms": elapsed_ms, "budget_sec": budget_sec, "passed": int(results.get("passed", 0)), "failed": int(results.get("failed", 0)), "skipped": int(results.get("skipped", 0)), "total": int(results.get("total", 0)), "tests_not_run": tests_not_run, } data.merge(extra) return data ## Strict validation of the server-provided per-call budget: numeric, ## finite, positive, then clamped to [BUDGET_MIN_SEC, BUDGET_MAX_SEC]. ## Everything else (missing, wrong type, NaN/inf, non-positive) falls back ## to BUDGET_DEFAULT_SEC. typeof() so bool never sneaks through as int. func _validated_budget_sec(params: Dictionary) -> float: var raw: Variant = params.get("timeout_budget_sec", null) var t := typeof(raw) if t == TYPE_FLOAT or t == TYPE_INT: var v := float(raw) if is_finite(v) and v > 0.0: return clampf(v, BUDGET_MIN_SEC, BUDGET_MAX_SEC) return BUDGET_DEFAULT_SEC ## Many suites assume the project's main scene is the edited scene (they read ## /Main/... nodes directly). Running with another scene open produces a flood ## of phantom failures that look like real regressions. Surface the edited ## scene and a warning when it differs from run/main_scene so the failures are ## attributable at a glance instead of costing a debugging round (#635). func _annotate_edited_scene(results: Dictionary) -> void: var scene_root := EditorInterface.get_edited_scene_root() var edited := scene_root.scene_file_path if scene_root else "" results["edited_scene"] = edited var main_scene := str(ProjectSettings.get_setting("application/run/main_scene", "")) if main_scene.is_empty() or edited == main_scene: return if int(results.get("failed", 0)) <= 0: return results["scene_warning"] = ( "Edited scene is '%s' but the project main scene is '%s'. Many suites " % [edited if not edited.is_empty() else "", main_scene] + "assume the main scene is open and will report phantom failures " + "otherwise. If these failures are unexpected, scene_open('%s') and re-run." % main_scene ) func get_test_results(params: Dictionary) -> Dictionary: var verbose: bool = params.get("verbose", false) return {"data": _runner.get_results(verbose)} ## Returns {"suites": Array, "errors": Array[String], "outcome": String}. ## Resilient: a broken script doesn't kill discovery of the rest. A ## non-empty outcome ("timeout" / "transport_lost" / "paused") means a ## between-load checkpoint aborted discovery — script loading is itself an ## atomic phase, and a directory of heavy scripts must neither starve the ## heartbeat nor escape the run budget. func _discover_suites( service_cb: Callable = Callable(), deadline_ticks_ms: int = 0, run_state: Dictionary = {}, ) -> Dictionary: var suites := [] var errors: Array[String] = [] var dir := DirAccess.open("res://tests") if dir == null: return { "suites": suites, "errors": ["DirAccess.open('res://tests') returned null — directory may not exist"], "outcome": "", } dir.list_dir_begin() var file_name := dir.get_next() while not file_name.is_empty(): if file_name.begins_with("test_") and file_name.ends_with(".gd"): var stop := _discovery_checkpoint(service_cb, deadline_ticks_ms, run_state) if not stop.is_empty(): return {"suites": suites, "errors": errors, "outcome": stop} var path := "res://tests/" + file_name var script = ResourceLoader.load(path, "", ResourceLoader.CACHE_MODE_IGNORE) if script == null: errors.append("%s (load failed — check for parse errors or duplicate methods)" % file_name) elif script.can_instantiate(): var instance = script.new() if instance is McpTestSuite: suites.append(instance) else: errors.append("%s (not a McpTestSuite subclass)" % file_name) else: errors.append("%s (cannot instantiate — abstract or broken)" % file_name) file_name = dir.get_next() ## Sort by suite name for deterministic order. suites.sort_custom(func(a, b) -> bool: return a.suite_name() < b.suite_name() ) return {"suites": suites, "errors": errors, "outcome": ""} ## Discovery-phase twin of McpTestRunner._checkpoint. Both delegate to the ## shared McpConnection.exclusive_run_checkpoint so the outcome mapping ## cannot drift between the discovery and between-test paths. func _discovery_checkpoint( service_cb: Callable, deadline_ticks_ms: int, run_state: Dictionary ) -> String: return McpConnection.exclusive_run_checkpoint(service_cb, deadline_ticks_ms, run_state)