626 lines
18 KiB
Python
626 lines
18 KiB
Python
"""Software-only tests for the stable versioned RX-agent launcher."""
|
|
|
|
from __future__ import annotations
|
|
|
|
from dataclasses import replace
|
|
from datetime import datetime, timedelta, timezone
|
|
import json
|
|
from pathlib import Path
|
|
import subprocess
|
|
|
|
import pytest
|
|
|
|
import tools.rf001_agent_launcher as launcher
|
|
|
|
|
|
class FakeProcess:
|
|
def __init__(self, process_id: int, returncode: int | None = None) -> None:
|
|
self.pid = process_id
|
|
self.returncode = returncode
|
|
|
|
def poll(self) -> int | None:
|
|
return self.returncode
|
|
|
|
def terminate(self) -> None:
|
|
self.returncode = 0
|
|
|
|
def kill(self) -> None:
|
|
self.returncode = -9
|
|
|
|
def wait(self, timeout: float | None = None) -> int:
|
|
del timeout
|
|
if self.returncode is None:
|
|
self.returncode = 0
|
|
return self.returncode
|
|
|
|
|
|
NOW = datetime(2026, 9, 29, 13, 0, tzinfo=timezone.utc)
|
|
WRAPPER = r"C:\runtime\.venv\Scripts\pythonw.exe"
|
|
CHILD = r"C:\runtime\Python313\python.exe"
|
|
AGENT = r"C:\release\app\tools\rf001_rx_http_agent.py"
|
|
|
|
|
|
def _attempt(
|
|
*,
|
|
process_id: int = 200,
|
|
nonce: str = "current-launch-nonce",
|
|
returncode: int | None = None,
|
|
) -> launcher.LaunchAttempt:
|
|
command = (
|
|
WRAPPER,
|
|
"-u",
|
|
"-B",
|
|
"-X",
|
|
"utf8",
|
|
AGENT,
|
|
"--launch-nonce",
|
|
nonce,
|
|
)
|
|
return launcher.LaunchAttempt(
|
|
process=FakeProcess(process_id, returncode), # type: ignore[arg-type]
|
|
command=command,
|
|
wrapper_pid=process_id,
|
|
launch_boundary=NOW.isoformat(),
|
|
wrapper_executable=WRAPPER,
|
|
child_executable=CHILD,
|
|
launch_nonce=nonce,
|
|
)
|
|
|
|
|
|
def _identity(
|
|
pid: int,
|
|
parent_pid: int,
|
|
*,
|
|
executable: str,
|
|
command: tuple[str, ...] | None = None,
|
|
seconds: float = 1.0,
|
|
) -> launcher.ProcessIdentity:
|
|
return launcher.ProcessIdentity(
|
|
pid=pid,
|
|
parent_pid=parent_pid,
|
|
executable=executable,
|
|
command_line=subprocess.list2cmdline(command or _attempt().command),
|
|
created_at=(NOW + timedelta(seconds=seconds)).isoformat(),
|
|
)
|
|
|
|
|
|
def _wrapper(attempt: launcher.LaunchAttempt | None = None) -> launcher.ProcessIdentity:
|
|
value = attempt or _attempt()
|
|
return _identity(
|
|
value.wrapper_pid,
|
|
100,
|
|
executable=value.wrapper_executable,
|
|
command=value.command,
|
|
)
|
|
|
|
|
|
def _child(attempt: launcher.LaunchAttempt | None = None) -> launcher.ProcessIdentity:
|
|
value = attempt or _attempt()
|
|
return _identity(
|
|
value.wrapper_pid + 1,
|
|
value.wrapper_pid,
|
|
executable=value.child_executable,
|
|
command=value.command,
|
|
)
|
|
|
|
|
|
def _health(
|
|
*,
|
|
pid: int = 200,
|
|
release_id: str = "rf001-agent-2026.09.29.1",
|
|
version: str = "2026.09.29.1",
|
|
nonce: str = "current-launch-nonce",
|
|
state: str = "RUNNING",
|
|
updated_at: str | None = None,
|
|
) -> dict[str, object]:
|
|
return {
|
|
"release_id": release_id,
|
|
"agent_version": version,
|
|
"launch_nonce": nonce,
|
|
"state": state,
|
|
"pid": pid,
|
|
"updated_at": updated_at or (NOW + timedelta(seconds=2)).isoformat(),
|
|
}
|
|
|
|
|
|
def _config(tmp_path: Path) -> dict[str, object]:
|
|
install = tmp_path / "install"
|
|
config: dict[str, object] = {
|
|
"install_root": str(install),
|
|
"releases_root": str(install / "versions"),
|
|
"active_release_file": str(install / "active_release.json"),
|
|
"launcher_status": str(install / "launcher_status.json"),
|
|
"launcher_request": str(install / "launcher_request.json"),
|
|
"health_file": str(install / "agent_health.json"),
|
|
"agent_log": str(install / "logs" / "agent.log"),
|
|
"agent_arguments": [],
|
|
}
|
|
return config
|
|
|
|
|
|
def _ready(config: dict[str, object], release_id: str, version: str) -> None:
|
|
root = Path(str(config["releases_root"])) / release_id
|
|
(root / "app" / "tools").mkdir(parents=True)
|
|
(root / "app" / "tools" / "rf001_rx_http_agent.py").write_text(
|
|
"# fixture\n", encoding="utf-8"
|
|
)
|
|
(root / "READY.json").write_text(
|
|
json.dumps(
|
|
{
|
|
"status": "READY",
|
|
"release_id": release_id,
|
|
"agent_version": version,
|
|
}
|
|
),
|
|
encoding="utf-8",
|
|
)
|
|
|
|
|
|
def _request(
|
|
config: dict[str, object],
|
|
current: str,
|
|
target: str,
|
|
version: str,
|
|
) -> dict[str, object]:
|
|
value = {
|
|
"task_id": "UPDATE_TEST_001",
|
|
"from_release_id": current,
|
|
"release_id": target,
|
|
"agent_version": version,
|
|
"safe_to_switch": True,
|
|
}
|
|
path = Path(str(config["launcher_request"]))
|
|
path.parent.mkdir(parents=True, exist_ok=True)
|
|
path.write_text(json.dumps(value), encoding="utf-8")
|
|
Path(str(config["health_file"])).write_text(
|
|
json.dumps({"busy": False, "active_task_id": None}),
|
|
encoding="utf-8",
|
|
)
|
|
return value
|
|
|
|
|
|
def test_launcher_switches_only_to_ready_release(
|
|
tmp_path: Path,
|
|
monkeypatch: pytest.MonkeyPatch,
|
|
) -> None:
|
|
config = _config(tmp_path)
|
|
old = "rf001-agent-2026.09.29.1"
|
|
new = "rf001-agent-2026.09.29.2"
|
|
_ready(config, old, "2026.09.29.1")
|
|
_ready(config, new, "2026.09.29.2")
|
|
request = _request(config, old, new, "2026.09.29.2")
|
|
terminated: list[FakeProcess] = []
|
|
monkeypatch.setattr(
|
|
launcher,
|
|
"terminate_process",
|
|
lambda process: terminated.append(process),
|
|
)
|
|
monkeypatch.setattr(
|
|
launcher,
|
|
"start_release",
|
|
lambda *_args, **_kwargs: (FakeProcess(2), "2026.09.29.2"),
|
|
)
|
|
process, release_id, version = launcher.switch_release(
|
|
config,
|
|
FakeProcess(1),
|
|
old,
|
|
"2026.09.29.1",
|
|
request,
|
|
)
|
|
assert process.pid == 2
|
|
assert release_id == new
|
|
assert version == "2026.09.29.2"
|
|
assert len(terminated) == 1
|
|
active = json.loads(Path(str(config["active_release_file"])).read_text())
|
|
assert active["release_id"] == new
|
|
status = json.loads(Path(str(config["launcher_status"])).read_text())
|
|
assert status["state"] == "RUNNING"
|
|
assert status["phase"] == "TARGET_HEALTHY"
|
|
assert status["rf_tx"] == 0
|
|
|
|
|
|
def test_launcher_rolls_back_after_new_version_start_failure(
|
|
tmp_path: Path,
|
|
monkeypatch: pytest.MonkeyPatch,
|
|
) -> None:
|
|
config = _config(tmp_path)
|
|
old = "rf001-agent-2026.09.29.1"
|
|
new = "rf001-agent-2026.09.29.2"
|
|
_ready(config, old, "2026.09.29.1")
|
|
_ready(config, new, "2026.09.29.2")
|
|
request = _request(config, old, new, "2026.09.29.2")
|
|
calls: list[str] = []
|
|
|
|
def fake_start(
|
|
_config_value: object,
|
|
release_id: str,
|
|
*,
|
|
expected_version: str | None = None,
|
|
) -> tuple[FakeProcess, str]:
|
|
del expected_version
|
|
calls.append(release_id)
|
|
if release_id == new:
|
|
raise RuntimeError("fixed startup failure")
|
|
return FakeProcess(3), "2026.09.29.1"
|
|
|
|
monkeypatch.setattr(launcher, "terminate_process", lambda _process: None)
|
|
monkeypatch.setattr(launcher, "start_release", fake_start)
|
|
process, release_id, version = launcher.switch_release(
|
|
config,
|
|
FakeProcess(1),
|
|
old,
|
|
"2026.09.29.1",
|
|
request,
|
|
)
|
|
assert process.pid == 3
|
|
assert release_id == old
|
|
assert version == "2026.09.29.1"
|
|
assert calls == [new, old]
|
|
active = json.loads(Path(str(config["active_release_file"])).read_text())
|
|
assert active["release_id"] == old
|
|
status = json.loads(Path(str(config["launcher_status"])).read_text())
|
|
assert status["state"] == "ROLLED_BACK"
|
|
assert status["phase"] == "LEGACY_RESTORED"
|
|
assert "fixed startup failure" in status["error"]
|
|
|
|
|
|
def test_launcher_rejects_busy_agent_without_stopping_it(tmp_path: Path) -> None:
|
|
config = _config(tmp_path)
|
|
old = "rf001-agent-2026.09.29.1"
|
|
new = "rf001-agent-2026.09.29.2"
|
|
_ready(config, old, "2026.09.29.1")
|
|
_ready(config, new, "2026.09.29.2")
|
|
request = _request(config, old, new, "2026.09.29.2")
|
|
Path(str(config["health_file"])).write_text(
|
|
json.dumps({"busy": True, "active_task_id": "STAIRCASE_TEST"}),
|
|
encoding="utf-8",
|
|
)
|
|
with pytest.raises(RuntimeError, match="not idle"):
|
|
launcher.switch_release(
|
|
config,
|
|
FakeProcess(1),
|
|
old,
|
|
"2026.09.29.1",
|
|
request,
|
|
)
|
|
|
|
|
|
def test_direct_exact_identity_passes() -> None:
|
|
attempt = _attempt()
|
|
wrapper = _wrapper(attempt)
|
|
result = launcher._validate_target_identity(
|
|
attempt,
|
|
wrapper,
|
|
attempt.wrapper_pid,
|
|
[wrapper],
|
|
)
|
|
assert result.mode == "DIRECT"
|
|
assert result.target == wrapper
|
|
|
|
|
|
def test_unique_expected_wrapper_child_passes() -> None:
|
|
attempt = _attempt()
|
|
wrapper = _wrapper(attempt)
|
|
child = _child(attempt)
|
|
result = launcher._validate_target_identity(
|
|
attempt,
|
|
wrapper,
|
|
child.pid,
|
|
[wrapper, child],
|
|
)
|
|
assert result.mode == "WINDOWS_VENV_WRAPPER"
|
|
assert result.target == child
|
|
|
|
|
|
@pytest.mark.parametrize(
|
|
("field", "value"),
|
|
[
|
|
("launch_nonce", "wrong"),
|
|
("release_id", "rf001-agent-2026.09.29.9"),
|
|
("agent_version", "2026.09.29.9"),
|
|
("state", "STARTING"),
|
|
],
|
|
)
|
|
def test_wrong_health_field_fails(field: str, value: str) -> None:
|
|
health = _health()
|
|
health[field] = value
|
|
assert not launcher._health_fields_match(
|
|
health,
|
|
release_id="rf001-agent-2026.09.29.1",
|
|
agent_version="2026.09.29.1",
|
|
launch_nonce="current-launch-nonce",
|
|
launch_boundary=NOW.isoformat(),
|
|
)
|
|
|
|
|
|
def test_stale_health_fails() -> None:
|
|
health = _health(updated_at=(NOW - timedelta(seconds=1)).isoformat())
|
|
assert not launcher._health_fields_match(
|
|
health,
|
|
release_id="rf001-agent-2026.09.29.1",
|
|
agent_version="2026.09.29.1",
|
|
launch_nonce="current-launch-nonce",
|
|
launch_boundary=NOW.isoformat(),
|
|
)
|
|
|
|
|
|
def test_unrelated_pid_fails() -> None:
|
|
attempt = _attempt()
|
|
wrapper = _wrapper(attempt)
|
|
unrelated = _identity(300, 999, executable=CHILD)
|
|
with pytest.raises(RuntimeError, match="not a direct wrapper child"):
|
|
launcher._validate_target_identity(
|
|
attempt,
|
|
wrapper,
|
|
unrelated.pid,
|
|
[wrapper, unrelated],
|
|
)
|
|
|
|
|
|
def test_unrelated_child_fails() -> None:
|
|
attempt = _attempt()
|
|
wrapper = _wrapper(attempt)
|
|
unrelated = replace(
|
|
_child(attempt),
|
|
command_line="python.exe unrelated.py",
|
|
)
|
|
with pytest.raises(RuntimeError, match="absent or ambiguous"):
|
|
launcher._validate_target_identity(
|
|
attempt,
|
|
wrapper,
|
|
unrelated.pid,
|
|
[wrapper, unrelated],
|
|
)
|
|
|
|
|
|
def test_unexpected_grandchild_fails() -> None:
|
|
attempt = _attempt()
|
|
wrapper = _wrapper(attempt)
|
|
child = _child(attempt)
|
|
grandchild = replace(child, pid=202, parent_pid=child.pid)
|
|
with pytest.raises(RuntimeError, match="not a direct wrapper child"):
|
|
launcher._validate_target_identity(
|
|
attempt,
|
|
wrapper,
|
|
grandchild.pid,
|
|
[wrapper, child, grandchild],
|
|
)
|
|
|
|
|
|
def test_two_matching_children_fail() -> None:
|
|
attempt = _attempt()
|
|
wrapper = _wrapper(attempt)
|
|
first = _child(attempt)
|
|
second = replace(first, pid=first.pid + 1)
|
|
with pytest.raises(RuntimeError, match="absent or ambiguous"):
|
|
launcher._validate_target_identity(
|
|
attempt,
|
|
wrapper,
|
|
first.pid,
|
|
[wrapper, first, second],
|
|
)
|
|
|
|
|
|
def test_creation_time_mismatch_fails() -> None:
|
|
attempt = _attempt()
|
|
wrapper = _wrapper(attempt)
|
|
stale_child = replace(
|
|
_child(attempt),
|
|
created_at=(NOW - timedelta(seconds=30)).isoformat(),
|
|
)
|
|
with pytest.raises(RuntimeError, match="absent or ambiguous"):
|
|
launcher._validate_target_identity(
|
|
attempt,
|
|
wrapper,
|
|
stale_child.pid,
|
|
[wrapper, stale_child],
|
|
)
|
|
|
|
|
|
def test_pid_reuse_between_validations_fails(
|
|
monkeypatch: pytest.MonkeyPatch,
|
|
) -> None:
|
|
attempt = _attempt()
|
|
wrapper = _wrapper(attempt)
|
|
child = _child(attempt)
|
|
reused = replace(
|
|
child,
|
|
created_at=(NOW + timedelta(seconds=3)).isoformat(),
|
|
)
|
|
snapshots = iter(([wrapper, child], [wrapper, reused]))
|
|
monkeypatch.setattr(
|
|
launcher,
|
|
"_query_process_identities",
|
|
lambda *_args, **_kwargs: next(snapshots),
|
|
)
|
|
with pytest.raises(RuntimeError, match="changed during validation"):
|
|
launcher._validate_target_twice(
|
|
attempt,
|
|
wrapper,
|
|
child.pid,
|
|
)
|
|
|
|
|
|
def test_wrapper_dies_too_early() -> None:
|
|
attempt = _attempt(returncode=1)
|
|
wrapper = _wrapper(attempt)
|
|
with pytest.raises(RuntimeError, match="wrapper died"):
|
|
launcher._validate_target_identity(
|
|
attempt,
|
|
wrapper,
|
|
wrapper.pid,
|
|
[wrapper],
|
|
)
|
|
|
|
|
|
def test_child_dies_before_final_validation(
|
|
monkeypatch: pytest.MonkeyPatch,
|
|
) -> None:
|
|
attempt = _attempt()
|
|
wrapper = _wrapper(attempt)
|
|
child = _child(attempt)
|
|
snapshots = iter(([wrapper, child], [wrapper]))
|
|
monkeypatch.setattr(
|
|
launcher,
|
|
"_query_process_identities",
|
|
lambda *_args, **_kwargs: next(snapshots),
|
|
)
|
|
with pytest.raises(RuntimeError, match="health process is absent"):
|
|
launcher._validate_target_twice(
|
|
attempt,
|
|
wrapper,
|
|
child.pid,
|
|
)
|
|
|
|
|
|
def test_wrong_executable_fails() -> None:
|
|
attempt = _attempt()
|
|
wrapper = _wrapper(attempt)
|
|
child = replace(_child(attempt), executable=r"C:\Other\python.exe")
|
|
with pytest.raises(RuntimeError, match="absent or ambiguous"):
|
|
launcher._validate_target_identity(
|
|
attempt,
|
|
wrapper,
|
|
child.pid,
|
|
[wrapper, child],
|
|
)
|
|
|
|
|
|
def test_wrong_command_line_fails() -> None:
|
|
attempt = _attempt()
|
|
wrapper = _wrapper(attempt)
|
|
child = replace(_child(attempt), command_line="python.exe other_agent.py")
|
|
with pytest.raises(RuntimeError, match="absent or ambiguous"):
|
|
launcher._validate_target_identity(
|
|
attempt,
|
|
wrapper,
|
|
child.pid,
|
|
[wrapper, child],
|
|
)
|
|
|
|
|
|
def test_cleanup_terminates_verified_child_and_wrapper_only(
|
|
monkeypatch: pytest.MonkeyPatch,
|
|
) -> None:
|
|
attempt = _attempt()
|
|
wrapper = _wrapper(attempt)
|
|
child = _child(attempt)
|
|
unrelated = _identity(300, 999, executable=CHILD)
|
|
alive = {wrapper.pid: wrapper, child.pid: child, unrelated.pid: unrelated}
|
|
terminated: list[int] = []
|
|
|
|
def query(wrapper_pid: int, health_pid: int | None = None) -> list[launcher.ProcessIdentity]:
|
|
del health_pid
|
|
return [
|
|
identity
|
|
for identity in alive.values()
|
|
if identity.pid == wrapper_pid or identity.parent_pid == wrapper_pid
|
|
]
|
|
|
|
def terminate_identity(identity: launcher.ProcessIdentity) -> str:
|
|
terminated.append(identity.pid)
|
|
alive.pop(identity.pid)
|
|
return f"PID {identity.pid}: terminated"
|
|
|
|
def terminate_wrapper(process: FakeProcess) -> None:
|
|
terminated.append(process.pid)
|
|
alive.pop(process.pid)
|
|
process.returncode = 0
|
|
|
|
monkeypatch.setattr(launcher, "_query_process_identities", query)
|
|
monkeypatch.setattr(launcher, "_terminate_verified_identity", terminate_identity)
|
|
monkeypatch.setattr(launcher, "terminate_process", terminate_wrapper)
|
|
|
|
result = launcher.strict_failure_cleanup(attempt, wrapper)
|
|
|
|
assert result.state == "COMPLETE"
|
|
assert result.orphan_verified_target_processes == ()
|
|
assert terminated == [child.pid, wrapper.pid]
|
|
assert unrelated.pid in alive
|
|
|
|
|
|
def test_cleanup_unknown_identity_kills_nothing(
|
|
monkeypatch: pytest.MonkeyPatch,
|
|
) -> None:
|
|
attempt = _attempt()
|
|
wrapper = _wrapper(attempt)
|
|
child = _child(attempt)
|
|
killed: list[int] = []
|
|
monkeypatch.setattr(
|
|
launcher,
|
|
"_query_process_identities",
|
|
lambda *_args, **_kwargs: [wrapper, child],
|
|
)
|
|
monkeypatch.setattr(
|
|
launcher,
|
|
"_terminate_verified_identity",
|
|
lambda identity: killed.append(identity.pid),
|
|
)
|
|
|
|
result = launcher.strict_failure_cleanup(attempt, None)
|
|
|
|
assert result.state == "CLEANUP_INCOMPLETE"
|
|
assert killed == []
|
|
|
|
|
|
def test_two_sequential_successful_starts_use_distinct_nonces(
|
|
tmp_path: Path,
|
|
monkeypatch: pytest.MonkeyPatch,
|
|
) -> None:
|
|
config = _config(tmp_path)
|
|
release_id = "rf001-agent-2026.09.29.1"
|
|
_ready(config, release_id, "2026.09.29.1")
|
|
nonces = iter(("first-nonce", "second-nonce"))
|
|
process_ids = iter((200, 300))
|
|
observed: list[tuple[int, str]] = []
|
|
|
|
monkeypatch.setattr(launcher.secrets, "token_hex", lambda _length: next(nonces))
|
|
monkeypatch.setattr(
|
|
launcher,
|
|
"_runtime_executables",
|
|
lambda _value: (WRAPPER, CHILD),
|
|
)
|
|
|
|
def spawn(
|
|
command: list[str],
|
|
*,
|
|
cwd: Path,
|
|
log_stream: object,
|
|
) -> FakeProcess:
|
|
del cwd, log_stream
|
|
process = FakeProcess(next(process_ids))
|
|
nonce = command[command.index("--launch-nonce") + 1]
|
|
Path(str(config["health_file"])).write_text(
|
|
json.dumps(_health(pid=process.pid, nonce=nonce, updated_at="2099-01-01T00:00:00+00:00")),
|
|
encoding="utf-8",
|
|
)
|
|
return process
|
|
|
|
def capture(attempt: launcher.LaunchAttempt, *, timeout_seconds: float) -> launcher.ProcessIdentity:
|
|
del timeout_seconds
|
|
return _identity(
|
|
attempt.wrapper_pid,
|
|
100,
|
|
executable=WRAPPER,
|
|
command=attempt.command,
|
|
)
|
|
|
|
def validate(
|
|
attempt: launcher.LaunchAttempt,
|
|
wrapper: launcher.ProcessIdentity,
|
|
health_pid: int,
|
|
) -> launcher.ValidatedTarget:
|
|
observed.append((health_pid, attempt.launch_nonce))
|
|
return launcher.ValidatedTarget("DIRECT", wrapper, wrapper)
|
|
|
|
monkeypatch.setattr(launcher, "_spawn_agent", spawn)
|
|
monkeypatch.setattr(launcher, "_capture_wrapper_identity", capture)
|
|
monkeypatch.setattr(launcher, "_validate_target_twice", validate)
|
|
|
|
first, _ = launcher.start_release(config, release_id)
|
|
second, _ = launcher.start_release(config, release_id)
|
|
|
|
assert first.pid == 200
|
|
assert second.pid == 300
|
|
assert observed == [(200, "first-nonce"), (300, "second-nonce")]
|