diff --git a/docs/llm.md b/docs/llm.md index c7ce134..7fb7d2e 100644 --- a/docs/llm.md +++ b/docs/llm.md @@ -100,14 +100,14 @@ $env:OLLAMA_HOST = "http://127.0.0.1:17811" Unit `gpu-rent-ollama` читает `/mnt/swarm_data/.gpu-rent-gpu.json`: -| Tier (VRAM) | Flash Attn | KEEP_ALIVE | KV cache | GPU_OVERHEAD | -| --- | --- | --- | --- | --- | -| low (<16 GiB) | off | 2m | q4_0 | 6 GiB | -| mid (16–23) | on* | 5m | q8_0 | 10 GiB | -| high (24–47) | on* | 15m | q8_0 | 14 GiB | -| ultra (≥48) | on* | 30m | q8_0 | 20 GiB | +| Tier (VRAM) | Flash Attn | KEEP_ALIVE | KV cache | GPU_OVERHEAD | CONTEXT | +| --- | --- | --- | --- | --- | --- | +| low (<16 GiB) | off | 2m | q4_0 | 6 GiB | 8k | +| mid (16–23) | on* | 5m | q8_0 | 10 GiB | 16k | +| high (24–47) | on* | 15m | q8_0 | 14 GiB | 16k | +| ultra (≥48) | on* | 30m | q8_0 | 20 GiB | 32k | -\*Flash на Ampere+ (compute ≥ 8.0). `NUM_PARALLEL=1`, `MAX_LOADED_MODELS=1`. Env: `/mnt/swarm_data/.gpu-rent-ollama.env`. +\*Flash на Ampere+ (compute ≥ 8.0). `NUM_PARALLEL=1`, `MAX_LOADED_MODELS=1`. Ollama default `num_ctx` is 4096; we set `OLLAMA_CONTEXT_LENGTH` so Assistent + vision fits. Env: `/mnt/swarm_data/.gpu-rent-ollama.env`. --- diff --git a/src/gpu_rent/perf_tiers.py b/src/gpu_rent/perf_tiers.py index 02afeaa..9f3877f 100644 --- a/src/gpu_rent/perf_tiers.py +++ b/src/gpu_rent/perf_tiers.py @@ -30,6 +30,7 @@ class OllamaTune: max_loaded_models: int kv_cache_type: str | None gpu_overhead_bytes: int + context_length: int notes: str @@ -72,23 +73,27 @@ def ollama_tune_for(info: GpuInfo) -> OllamaTune: overhead = 20 * 1024**3 keep = "30m" kv = "q8_0" - note = "ultra: flash+q8 KV, 20GiB reserved for Swarm, keep 30m" + ctx = 32768 + note = "ultra: flash+q8 KV, 20GiB reserved for Swarm, ctx 32k, keep 30m" elif info.tier == TIER_HIGH: overhead = 14 * 1024**3 keep = "15m" kv = "q8_0" - note = "high: flash+q8 KV, 14GiB reserved for Swarm, keep 15m" + ctx = 16384 + note = "high: flash+q8 KV, 14GiB reserved for Swarm, ctx 16k, keep 15m" elif info.tier == TIER_MID: overhead = 10 * 1024**3 keep = "5m" kv = "q8_0" - note = "mid: flash+q8 KV, 10GiB reserved for Swarm, keep 5m" + ctx = 16384 + note = "mid: flash+q8 KV, 10GiB reserved for Swarm, ctx 16k, keep 5m" else: overhead = 6 * 1024**3 keep = "2m" kv = "q4_0" + ctx = 8192 flash = False # prefer stability on tiny cards - note = "low: conservative, 6GiB reserved, short keep-alive" + note = "low: conservative, 6GiB reserved, ctx 8k, short keep-alive" return OllamaTune( flash_attention=flash, @@ -97,6 +102,7 @@ def ollama_tune_for(info: GpuInfo) -> OllamaTune: max_loaded_models=1, kv_cache_type=kv, gpu_overhead_bytes=overhead, + context_length=ctx, notes=note, ) @@ -139,6 +145,7 @@ def ollama_env_lines(tune: OllamaTune) -> list[str]: f"Environment=OLLAMA_MAX_LOADED_MODELS={tune.max_loaded_models}", f"Environment=OLLAMA_KEEP_ALIVE={tune.keep_alive}", f"Environment=OLLAMA_GPU_OVERHEAD={tune.gpu_overhead_bytes}", + f"Environment=OLLAMA_CONTEXT_LENGTH={tune.context_length}", ] if tune.flash_attention: lines.append("Environment=OLLAMA_FLASH_ATTENTION=1") diff --git a/src/gpu_rent/remote/install_ollama.sh b/src/gpu_rent/remote/install_ollama.sh index 8f47793..0b438af 100644 --- a/src/gpu_rent/remote/install_ollama.sh +++ b/src/gpu_rent/remote/install_ollama.sh @@ -45,13 +45,13 @@ try: except ValueError: ampere = False if gib >= 48: - tier, overhead, keep, kv, flash = "ultra", 20 * 1024**3, "30m", "q8_0", True + tier, overhead, keep, kv, flash, ctx = "ultra", 20 * 1024**3, "30m", "q8_0", True, 32768 elif gib >= 24: - tier, overhead, keep, kv, flash = "high", 14 * 1024**3, "15m", "q8_0", True + tier, overhead, keep, kv, flash, ctx = "high", 14 * 1024**3, "15m", "q8_0", True, 16384 elif gib >= 16: - tier, overhead, keep, kv, flash = "mid", 10 * 1024**3, "5m", "q8_0", True + tier, overhead, keep, kv, flash, ctx = "mid", 10 * 1024**3, "5m", "q8_0", True, 16384 else: - tier, overhead, keep, kv, flash = "low", 6 * 1024**3, "2m", "q4_0", False + tier, overhead, keep, kv, flash, ctx = "low", 6 * 1024**3, "2m", "q4_0", False, 8192 flash = bool(flash and (ampere or "A100" in name.upper() or "H100" in name.upper() or gib >= 16)) lines = [ f"# auto gpu-rent ollama tune tier={tier} gpu={name!r} vram_mib={vram}", @@ -59,13 +59,14 @@ lines = [ "OLLAMA_MAX_LOADED_MODELS=1", f"OLLAMA_KEEP_ALIVE={keep}", f"OLLAMA_GPU_OVERHEAD={overhead}", + f"OLLAMA_CONTEXT_LENGTH={ctx}", ] if flash: lines.append("OLLAMA_FLASH_ATTENTION=1") if kv: lines.append(f"OLLAMA_KV_CACHE_TYPE={kv}") out.write_text("\n".join(lines) + "\n", encoding="utf-8") -print(f"tier={tier} flash={int(flash)} keep={keep} overhead_gib={overhead/1024**3:.0f}") +print(f"tier={tier} flash={int(flash)} keep={keep} ctx={ctx} overhead_gib={overhead/1024**3:.0f}") PY } diff --git a/tests/test_perf_tiers.py b/tests/test_perf_tiers.py index 6acc4c9..68c6789 100644 --- a/tests/test_perf_tiers.py +++ b/tests/test_perf_tiers.py @@ -41,6 +41,8 @@ def test_ollama_high_reserves_vram_for_swarm(): env = "\n".join(ollama_env_lines(tune)) assert "OLLAMA_FLASH_ATTENTION=1" in env assert "OLLAMA_GPU_OVERHEAD=" in env + assert "OLLAMA_CONTEXT_LENGTH=16384" in env + assert tune.context_length == 16384 def test_swarm_sage_on_ampere_mid(): @@ -68,3 +70,16 @@ def test_swarm_no_sage_on_low(): st = swarm_tune_for(info) assert not st.use_sage_attention assert st.comfy_extra_args == "" + + +def test_ollama_mid_4090_context_16k(): + info = GpuInfo( + name="NVIDIA GeForce RTX 4090", + vram_mib=24564, + compute_cap="8.9", + uuid="GPU-4", + tier="mid", + ) + tune = ollama_tune_for(info) + assert tune.context_length == 16384 + assert "OLLAMA_CONTEXT_LENGTH=16384" in "\n".join(ollama_env_lines(tune))