Implement idle-killer enhancements and swarm management improvements

- Marked critical bugs as resolved in the review documentation, including changes to the `arm_idle_killer` function to raise errors on credential creation failures and ensure proper file permissions for JSON credentials.
- Introduced a new `_try_arm_idle_killer` function in `provision.py` to manage idle-killer state more effectively, ensuring it arms correctly during provisioning.
- Updated the `swarm_busy` function in `remote/idle_killer.py` to allow idle state after a specified duration of Swarm unavailability, preventing unnecessary billing.
- Enhanced performance tuning logic in `tune_swarm_perf.py` to ensure proper handling of pip installation success before applying extra arguments.
- Added tests to validate the new idle-killer behavior and swarm management logic, ensuring robustness in handling idle states and error conditions.

tests/test_hold_killer.py

@@ -109,3 +109,34 @@ def test_classify_busy_queue(monkeypatch):
     busy, detail = mod.swarm_busy("http://127.0.0.1:7801")
     assert busy is True
     assert "waiting=2" in detail
+
+
+def test_swarm_unreachable_starts_busy_then_allows_idle(tmp_path, monkeypatch):
+    mod = _load_remote()
+    monkeypatch.setattr(mod, "DATA", tmp_path)
+    monkeypatch.setattr(mod, "SWARM_DOWN_SINCE", tmp_path / ".gpu-rent-swarm-down-since")
+    monkeypatch.setattr(mod, "SWARM_UNREACHABLE_IDLE_SEC", 100)
+
+    def boom(*_a, **_k):
+        raise mod.urllib.error.URLError("down")
+
+    monkeypatch.setattr(mod.urllib.request, "urlopen", boom)
+
+    busy1, d1 = mod.swarm_busy("http://127.0.0.1:7801")
+    assert busy1 is True
+    assert "clock start" in d1
+    assert mod.SWARM_DOWN_SINCE.is_file()
+
+    # Still within window
+    since = mod.read_ts(mod.SWARM_DOWN_SINCE)
+    assert since is not None
+    mod.write_ts(mod.SWARM_DOWN_SINCE, since - 50)
+    busy2, d2 = mod.swarm_busy("http://127.0.0.1:7801")
+    assert busy2 is True
+    assert "50s" in d2 or "/ 100s" in d2
+
+    # Past window → not busy so idle clock can run
+    mod.write_ts(mod.SWARM_DOWN_SINCE, since - 120)
+    busy3, d3 = mod.swarm_busy("http://127.0.0.1:7801")
+    assert busy3 is False
+    assert "allow idle" in d3