....FFF.F. [100%] =================================== FAILURES =================================== _____________ test_start_gateway_replace_force_uses_terminate_pid ______________ monkeypatch = <_pytest.monkeypatch.MonkeyPatch object at 0x7a9136bd06e0> tmp_path = PosixPath('/home/cube/projects/richard/traning coach/task-12-independent-verification-r7/gateway/execution/846cc01aa81b1a6e62fb7698d76a0d5f9e44d00fd24956242eebfb8404c3c213/basetemp/test_start_gateway_replace_for0') @pytest.mark.asyncio async def test_start_gateway_replace_force_uses_terminate_pid(monkeypatch, tmp_path): monkeypatch.setenv("HERMES_HOME", str(tmp_path)) calls = [] class _CleanExitRunner: def __init__(self, config): self.config = config self.should_exit_cleanly = True self.exit_reason = None self.adapters = {} async def start(self): return True async def stop(self): return None # get_running_pid returns 42 before we kill the old gateway, then None # after remove_pid_file() clears the record (reflects real behavior). _pid_state = {"alive": True} def _mock_get_running_pid(): return 42 if _pid_state["alive"] else None def _mock_remove_pid_file(): _pid_state["alive"] = False monkeypatch.setattr("gateway.status.get_running_pid", _mock_get_running_pid) monkeypatch.setattr("gateway.status.remove_pid_file", _mock_remove_pid_file) monkeypatch.setattr( "gateway.status.release_all_scoped_locks", lambda **kwargs: 0, ) # force-kill reaps the process: terminate_pid(force=True) flips it dead, # and the post-kill re-poll via _pid_exists then sees it gone so the # replacement proceeds. def _mock_terminate_pid(pid, force=False): calls.append((pid, force)) if force: _pid_state["alive"] = False monkeypatch.setattr("gateway.status.terminate_pid", _mock_terminate_pid) monkeypatch.setattr( "gateway.status._pid_exists", lambda pid: _pid_state["alive"] ) monkeypatch.setattr("gateway.run.os.getpid", lambda: 100) monkeypatch.setattr("gateway.run.os.kill", lambda pid, sig: None) monkeypatch.setattr("time.sleep", lambda _: None) monkeypatch.setattr("tools.skills_sync.sync_skills", lambda quiet=True: None) monkeypatch.setattr("hermes_logging.setup_logging", lambda hermes_home, mode: tmp_path) monkeypatch.setattr("hermes_logging._add_rotating_handler", lambda *args, **kwargs: None) monkeypatch.setattr("gateway.run.GatewayRunner", _CleanExitRunner) from gateway.run import start_gateway ok = await start_gateway(config=GatewayConfig(), replace=True, verbosity=None) > assert ok is True E assert False is True tests/gateway/test_runner_startup_failures.py:232: AssertionError ----------------------------- Captured stderr call ----------------------------- ERROR gateway.run: Cannot obtain a stable handle for gateway PID 42; refusing replacement. ------------------------------ Captured log call ------------------------------- ERROR gateway.run:run.py:17237 Cannot obtain a stable handle for gateway PID 42; refusing replacement. _____ test_start_gateway_replace_aborts_when_force_killed_pid_still_alive ______ monkeypatch = <_pytest.monkeypatch.MonkeyPatch object at 0x7a9136bc9910> tmp_path = PosixPath('/home/cube/projects/richard/traning coach/task-12-independent-verification-r7/gateway/execution/846cc01aa81b1a6e62fb7698d76a0d5f9e44d00fd24956242eebfb8404c3c213/basetemp/test_start_gateway_replace_abo0') @pytest.mark.asyncio async def test_start_gateway_replace_aborts_when_force_killed_pid_still_alive( monkeypatch, tmp_path ): """Regression for #19471 (duplicate-gateway half). If SIGKILL fails to reap the old gateway, --replace must NOT clear the PID file / scoped locks and start a fresh instance — that leaves two live gateways fighting over the same token. It should abort instead. """ monkeypatch.setenv("HERMES_HOME", str(tmp_path)) calls = [] removed_pid = False released_locks = False class _RunnerShouldNotStart: def __init__(self, config): raise AssertionError("replacement must not start while old PID is alive") def _mock_remove_pid_file(): nonlocal removed_pid removed_pid = True def _mock_release_all_scoped_locks(**kwargs): nonlocal released_locks released_locks = True return 0 monkeypatch.setattr("gateway.status.get_running_pid", lambda: 42) monkeypatch.setattr("gateway.status.remove_pid_file", _mock_remove_pid_file) monkeypatch.setattr( "gateway.status.release_all_scoped_locks", _mock_release_all_scoped_locks, ) monkeypatch.setattr( "gateway.status.terminate_pid", lambda pid, force=False: calls.append((pid, force)), ) # _pid_exists never goes False — the force-kill did not take. monkeypatch.setattr("gateway.status._pid_exists", lambda pid: True) monkeypatch.setattr("gateway.run.os.getpid", lambda: 100) monkeypatch.setattr("gateway.run.os.kill", lambda pid, sig: None) monkeypatch.setattr("time.sleep", lambda _: None) monkeypatch.setattr("tools.skills_sync.sync_skills", lambda quiet=True: None) monkeypatch.setattr("hermes_logging.setup_logging", lambda hermes_home, mode: tmp_path) monkeypatch.setattr("hermes_logging._add_rotating_handler", lambda *args, **kwargs: None) monkeypatch.setattr("gateway.run.GatewayRunner", _RunnerShouldNotStart) from gateway.run import start_gateway ok = await start_gateway(config=GatewayConfig(), replace=True, verbosity=None) assert ok is False > assert calls == [(42, False), (42, True)] E assert [] == [(42, False), (42, True)] E E Right contains 2 more items, first extra item: (42, False) E Use -v to get more diff tests/gateway/test_runner_startup_failures.py:290: AssertionError ----------------------------- Captured stderr call ----------------------------- ERROR gateway.run: Cannot obtain a stable handle for gateway PID 42; refusing replacement. ------------------------------ Captured log call ------------------------------- ERROR gateway.run:run.py:17237 Cannot obtain a stable handle for gateway PID 42; refusing replacement. _______ test_start_gateway_replace_writes_takeover_marker_before_sigterm _______ monkeypatch = <_pytest.monkeypatch.MonkeyPatch object at 0x7a91358b3f50> tmp_path = PosixPath('/home/cube/projects/richard/traning coach/task-12-independent-verification-r7/gateway/execution/846cc01aa81b1a6e62fb7698d76a0d5f9e44d00fd24956242eebfb8404c3c213/basetemp/test_start_gateway_replace_wri0') @pytest.mark.asyncio async def test_start_gateway_replace_writes_takeover_marker_before_sigterm( monkeypatch, tmp_path ): """--replace must write a takeover marker BEFORE sending SIGTERM. The marker lets the target's shutdown handler identify the signal as a planned takeover (→ exit 0) rather than an unexpected kill (→ exit 1). Without the marker, PR #5646's signal-recovery path would revive the target via systemd Restart=on-failure, starting a flap loop. """ monkeypatch.setenv("HERMES_HOME", str(tmp_path)) # Record the ORDER of marker-write + terminate_pid calls events: list[str] = [] marker_paths_seen: list = [] def record_write_marker(target_pid: int) -> bool: events.append(f"write_marker(target_pid={target_pid})") # Also check that the marker file actually exists after this call marker_paths_seen.append( (tmp_path / ".gateway-takeover.json").exists() is False # not yet ) # Actually write the marker so we can verify cleanup later from gateway.status import _get_takeover_marker_path, _write_json_file _write_json_file(_get_takeover_marker_path(), { "target_pid": target_pid, "target_start_time": 0, "replacer_pid": 100, "written_at": "2026-04-17T00:00:00+00:00", }) return True def record_terminate(pid, force=False): events.append(f"terminate_pid(pid={pid}, force={force})") class _CleanExitRunner: def __init__(self, config): self.config = config self.should_exit_cleanly = True self.exit_reason = None self.adapters = {} async def start(self): return True async def stop(self): return None _pid_state = {"alive": True} def _mock_get_running_pid(): return 42 if _pid_state["alive"] else None def _mock_remove_pid_file(): _pid_state["alive"] = False monkeypatch.setattr("gateway.status.get_running_pid", _mock_get_running_pid) monkeypatch.setattr("gateway.status.remove_pid_file", _mock_remove_pid_file) monkeypatch.setattr( "gateway.status.release_all_scoped_locks", lambda **kwargs: 0, ) monkeypatch.setattr("gateway.status.write_takeover_marker", record_write_marker) monkeypatch.setattr("gateway.status.terminate_pid", record_terminate) monkeypatch.setattr("gateway.run.os.getpid", lambda: 100) # Simulate old process exiting on first check so we don't loop into force-kill monkeypatch.setattr( "gateway.run.os.kill", lambda pid, sig: (_ for _ in ()).throw(ProcessLookupError()), ) monkeypatch.setattr("time.sleep", lambda _: None) monkeypatch.setattr("tools.skills_sync.sync_skills", lambda quiet=True: None) monkeypatch.setattr("hermes_logging.setup_logging", lambda hermes_home, mode: tmp_path) monkeypatch.setattr("hermes_logging._add_rotating_handler", lambda *args, **kwargs: None) monkeypatch.setattr("gateway.run.GatewayRunner", _CleanExitRunner) from gateway.run import start_gateway ok = await start_gateway(config=GatewayConfig(), replace=True, verbosity=None) > assert ok is True E assert False is True tests/gateway/test_runner_startup_failures.py:373: AssertionError ----------------------------- Captured stderr call ----------------------------- ERROR gateway.run: Cannot obtain a stable handle for gateway PID 42; refusing replacement. ------------------------------ Captured log call ------------------------------- ERROR gateway.run:run.py:17237 Cannot obtain a stable handle for gateway PID 42; refusing replacement. __________ test_runner_degrades_gracefully_when_all_adapters_missing ___________ monkeypatch = <_pytest.monkeypatch.MonkeyPatch object at 0x7a91358ee120> tmp_path = PosixPath('/home/cube/projects/richard/traning coach/task-12-independent-verification-r7/gateway/execution/846cc01aa81b1a6e62fb7698d76a0d5f9e44d00fd24956242eebfb8404c3c213/basetemp/test_runner_degrades_gracefull0') caplog = <_pytest.logging.LogCaptureFixture object at 0x7a91358ed940> @pytest.mark.asyncio async def test_runner_degrades_gracefully_when_all_adapters_missing(monkeypatch, tmp_path, caplog): """When all enabled platforms have no adapter (missing library or credentials), the gateway should NOT return failure — it should warn and continue running for cron job execution, matching the behaviour of 'no platforms enabled' (#5196). In fleet deployments the same config.yaml is shared across nodes that may only have credentials for a subset of platforms. Requiring perfect credentials on every node makes fleet operation impossible.""" monkeypatch.setenv("HERMES_HOME", str(tmp_path)) config = GatewayConfig( platforms={ Platform.TELEGRAM: PlatformConfig(enabled=True, token="***"), Platform.DISCORD: PlatformConfig(enabled=True, token="***"), }, sessions_dir=tmp_path / "sessions", ) runner = GatewayRunner(config) # Simulate _create_adapter returning None for ALL platforms (missing library / # missing credentials — no connection attempt ever made). monkeypatch.setattr(runner, "_create_adapter", lambda platform, cfg: None) import logging with caplog.at_level(logging.WARNING): ok = await runner.start() # Must NOT return False — gateway should keep running for cron. assert ok is True assert runner.should_exit_cleanly is False assert runner.adapters == {} # Runtime state must remain "running", not "startup_failed". state = read_runtime_status() > assert state["gateway_state"] == "running" E AssertionError: assert 'degraded' == 'running' E E - running E + degraded tests/gateway/test_runner_startup_failures.py:453: AssertionError ----------------------------- Captured stderr call ----------------------------- WARNING gateway.run: No user allowlists configured. All unauthorized users will be denied. Set GATEWAY_ALLOW_ALL_USERS=true in ~/.hermes/.env to allow open access, or configure platform allowlists (e.g., TELEGRAM_ALLOWED_USERS=your_id). WARNING gateway.run: No adapter available for telegram WARNING gateway.run: No adapter available for discord WARNING gateway.run: No adapter could be created for any of the 2 configured platform(s). Check that required dependencies are installed and credentials are set. Gateway will continue for cron job execution. INFO gateway.run: kanban dispatcher: holding singleton dispatcher lock (/home/cube/projects/richard/traning coach/task-12-independent-verification-r7/gateway/execution/846cc01aa81b1a6e62fb7698d76a0d5f9e44d00fd24956242eebfb8404c3c213/basetemp/test_runner_degrades_gracefull0/kanban/.dispatcher.lock) ------------------------------ Captured log call ------------------------------- WARNING gateway.run:run.py:5391 No user allowlists configured. All unauthorized users will be denied. Set GATEWAY_ALLOW_ALL_USERS=true in ~/.hermes/.env to allow open access, or configure platform allowlists (e.g., TELEGRAM_ALLOWED_USERS=your_id). WARNING gateway.run:run.py:5536 No adapter available for telegram WARNING gateway.run:run.py:5536 No adapter available for discord WARNING gateway.run:run.py:5704 No adapter could be created for any of the 2 configured platform(s). Check that required dependencies are installed and credentials are set. Gateway will continue for cron job execution. INFO gateway.run:kanban_watchers.py:679 kanban dispatcher: holding singleton dispatcher lock (/home/cube/projects/richard/traning coach/task-12-independent-verification-r7/gateway/execution/846cc01aa81b1a6e62fb7698d76a0d5f9e44d00fd24956242eebfb8404c3c213/basetemp/test_runner_degrades_gracefull0/kanban/.dispatcher.lock) =========================== short test summary info ============================ FAILED tests/gateway/test_runner_startup_failures.py::test_start_gateway_replace_force_uses_terminate_pid FAILED tests/gateway/test_runner_startup_failures.py::test_start_gateway_replace_aborts_when_force_killed_pid_still_alive FAILED tests/gateway/test_runner_startup_failures.py::test_start_gateway_replace_writes_takeover_marker_before_sigterm FAILED tests/gateway/test_runner_startup_failures.py::test_runner_degrades_gracefully_when_all_adapters_missing 4 failed, 6 passed in 2.68s