mrleo1nid/gpu-rent
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Update GPU performance tuning and documentation for context length

Browse Source 
- Enhanced the `OllamaTune` class to include a new `context_length` attribute, improving the configuration for different GPU tiers.
- Updated performance tuning logic to set appropriate context lengths for low, mid, high, and ultra tiers, ensuring optimal resource allocation.
- Modified installation scripts to reflect the new context length settings, enhancing the installation process for Ollama.
- Revised documentation to include context length details in the GPU performance table, providing clearer guidance for users.
- Added tests to validate the correct context length settings in various scenarios, ensuring robustness in performance tuning.
This commit is contained in:
Leonid Pershin
2026-08-21 20:05:33 +03:00
parent 5832c5cf75
commit a882964ce0
4 changed files with 39 additions and 16 deletions
Download Patch File Download Diff File Expand all files Collapse all files
docs/llm.md
+7 -7
View File
@@ -100,14 +100,14 @@ $env:OLLAMA_HOST = "http://127.0.0.1:17811"
Unit `gpu-rent-ollama` читает `/mnt/swarm_data/.gpu-rent-gpu.json`:
| Tier (VRAM) | Flash Attn | KEEP_ALIVE | KV cache | GPU_OVERHEAD |
| --- | --- | --- | --- | --- |
| low (<16 GiB) | off | 2m | q4_0 | 6 GiB |
| mid (16–23) | on* | 5m | q8_0 | 10 GiB |
| high (24–47) | on* | 15m | q8_0 | 14 GiB |
| ultra (≥48) | on* | 30m | q8_0 | 20 GiB |
| Tier (VRAM) | Flash Attn | KEEP_ALIVE | KV cache | GPU_OVERHEAD | CONTEXT |
| --- | --- | --- | --- | --- | --- |
| low (<16 GiB) | off | 2m | q4_0 | 6 GiB | 8k |
| mid (16–23) | on* | 5m | q8_0 | 10 GiB | 16k |
| high (24–47) | on* | 15m | q8_0 | 14 GiB | 16k |
| ultra (≥48) | on* | 30m | q8_0 | 20 GiB | 32k |
\*Flash на Ampere+ (compute ≥ 8.0). `NUM_PARALLEL=1`, `MAX_LOADED_MODELS=1`. Env: `/mnt/swarm_data/.gpu-rent-ollama.env`.
\*Flash на Ampere+ (compute ≥ 8.0). `NUM_PARALLEL=1`, `MAX_LOADED_MODELS=1`. Ollama default `num_ctx` is 4096; we set `OLLAMA_CONTEXT_LENGTH` so Assistent + vision fits. Env: `/mnt/swarm_data/.gpu-rent-ollama.env`.
---
src/gpu_rent/perf_tiers.py
+11 -4
View File
@@ -30,6 +30,7 @@ class OllamaTune:
max_loaded_models: int
kv_cache_type: str | None
gpu_overhead_bytes: int
context_length: int
notes: str
@@ -72,23 +73,27 @@ def ollama_tune_for(info: GpuInfo) -> OllamaTune:
overhead = 20 * 1024**3
keep = "30m"
kv = "q8_0"
note = "ultra: flash+q8 KV, 20GiB reserved for Swarm, keep 30m"
ctx = 32768
note = "ultra: flash+q8 KV, 20GiB reserved for Swarm, ctx 32k, keep 30m"
elif info.tier == TIER_HIGH:
overhead = 14 * 1024**3
keep = "15m"
kv = "q8_0"
note = "high: flash+q8 KV, 14GiB reserved for Swarm, keep 15m"
ctx = 16384
note = "high: flash+q8 KV, 14GiB reserved for Swarm, ctx 16k, keep 15m"
elif info.tier == TIER_MID:
overhead = 10 * 1024**3
keep = "5m"
kv = "q8_0"
note = "mid: flash+q8 KV, 10GiB reserved for Swarm, keep 5m"
ctx = 16384
note = "mid: flash+q8 KV, 10GiB reserved for Swarm, ctx 16k, keep 5m"
else:
overhead = 6 * 1024**3
keep = "2m"
kv = "q4_0"
ctx = 8192
flash = False # prefer stability on tiny cards
note = "low: conservative, 6GiB reserved, short keep-alive"
note = "low: conservative, 6GiB reserved, ctx 8k, short keep-alive"
return OllamaTune(
flash_attention=flash,
@@ -97,6 +102,7 @@ def ollama_tune_for(info: GpuInfo) -> OllamaTune:
max_loaded_models=1,
kv_cache_type=kv,
gpu_overhead_bytes=overhead,
context_length=ctx,
notes=note,
)
@@ -139,6 +145,7 @@ def ollama_env_lines(tune: OllamaTune) -> list[str]:
f"Environment=OLLAMA_MAX_LOADED_MODELS={tune.max_loaded_models}",
f"Environment=OLLAMA_KEEP_ALIVE={tune.keep_alive}",
f"Environment=OLLAMA_GPU_OVERHEAD={tune.gpu_overhead_bytes}",
f"Environment=OLLAMA_CONTEXT_LENGTH={tune.context_length}",
]
if tune.flash_attention:
lines.append("Environment=OLLAMA_FLASH_ATTENTION=1")
src/gpu_rent/remote/install_ollama.sh
+6 -5
View File
@@ -45,13 +45,13 @@ try:
except ValueError:
ampere = False
if gib >= 48:
tier, overhead, keep, kv, flash = "ultra", 20 * 1024**3, "30m", "q8_0", True
tier, overhead, keep, kv, flash, ctx = "ultra", 20 * 1024**3, "30m", "q8_0", True, 32768
elif gib >= 24:
tier, overhead, keep, kv, flash = "high", 14 * 1024**3, "15m", "q8_0", True
tier, overhead, keep, kv, flash, ctx = "high", 14 * 1024**3, "15m", "q8_0", True, 16384
elif gib >= 16:
tier, overhead, keep, kv, flash = "mid", 10 * 1024**3, "5m", "q8_0", True
tier, overhead, keep, kv, flash, ctx = "mid", 10 * 1024**3, "5m", "q8_0", True, 16384
else:
tier, overhead, keep, kv, flash = "low", 6 * 1024**3, "2m", "q4_0", False
tier, overhead, keep, kv, flash, ctx = "low", 6 * 1024**3, "2m", "q4_0", False, 8192
flash = bool(flash and (ampere or "A100" in name.upper() or "H100" in name.upper() or gib >= 16))
lines = [
f"# auto gpu-rent ollama tune tier={tier} gpu={name!r} vram_mib={vram}",
@@ -59,13 +59,14 @@ lines = [
"OLLAMA_MAX_LOADED_MODELS=1",
f"OLLAMA_KEEP_ALIVE={keep}",
f"OLLAMA_GPU_OVERHEAD={overhead}",
f"OLLAMA_CONTEXT_LENGTH={ctx}",
]
if flash:
lines.append("OLLAMA_FLASH_ATTENTION=1")
if kv:
lines.append(f"OLLAMA_KV_CACHE_TYPE={kv}")
out.write_text("\n".join(lines) + "\n", encoding="utf-8")
print(f"tier={tier} flash={int(flash)} keep={keep} overhead_gib={overhead/1024**3:.0f}")
print(f"tier={tier} flash={int(flash)} keep={keep} ctx={ctx} overhead_gib={overhead/1024**3:.0f}")
PY
}
tests/test_perf_tiers.py
+15
View File
@@ -41,6 +41,8 @@ def test_ollama_high_reserves_vram_for_swarm():
env = "\n".join(ollama_env_lines(tune))
assert "OLLAMA_FLASH_ATTENTION=1" in env
assert "OLLAMA_GPU_OVERHEAD=" in env
assert "OLLAMA_CONTEXT_LENGTH=16384" in env
assert tune.context_length == 16384
def test_swarm_sage_on_ampere_mid():
@@ -68,3 +70,16 @@ def test_swarm_no_sage_on_low():
st = swarm_tune_for(info)
assert not st.use_sage_attention
assert st.comfy_extra_args == ""
def test_ollama_mid_4090_context_16k():
info = GpuInfo(
name="NVIDIA GeForce RTX 4090",
vram_mib=24564,
compute_cap="8.9",
uuid="GPU-4",
tier="mid",
)
tune = ollama_tune_for(info)
assert tune.context_length == 16384
assert "OLLAMA_CONTEXT_LENGTH=16384" in "\n".join(ollama_env_lines(tune))