Update GPU performance tuning and documentation for context length

- Enhanced the `OllamaTune` class to include a new `context_length` attribute, improving the configuration for different GPU tiers.
- Updated performance tuning logic to set appropriate context lengths for low, mid, high, and ultra tiers, ensuring optimal resource allocation.
- Modified installation scripts to reflect the new context length settings, enhancing the installation process for Ollama.
- Revised documentation to include context length details in the GPU performance table, providing clearer guidance for users.
- Added tests to validate the correct context length settings in various scenarios, ensuring robustness in performance tuning.

src/gpu_rent/perf_tiers.py

@@ -30,6 +30,7 @@ class OllamaTune:
     max_loaded_models: int
     kv_cache_type: str | None
     gpu_overhead_bytes: int
+    context_length: int
     notes: str
 
 
@@ -72,23 +73,27 @@ def ollama_tune_for(info: GpuInfo) -> OllamaTune:
         overhead = 20 * 1024**3
         keep = "30m"
         kv = "q8_0"
-        note = "ultra: flash+q8 KV, 20GiB reserved for Swarm, keep 30m"
+        ctx = 32768
+        note = "ultra: flash+q8 KV, 20GiB reserved for Swarm, ctx 32k, keep 30m"
     elif info.tier == TIER_HIGH:
         overhead = 14 * 1024**3
         keep = "15m"
         kv = "q8_0"
-        note = "high: flash+q8 KV, 14GiB reserved for Swarm, keep 15m"
+        ctx = 16384
+        note = "high: flash+q8 KV, 14GiB reserved for Swarm, ctx 16k, keep 15m"
     elif info.tier == TIER_MID:
         overhead = 10 * 1024**3
         keep = "5m"
         kv = "q8_0"
-        note = "mid: flash+q8 KV, 10GiB reserved for Swarm, keep 5m"
+        ctx = 16384
+        note = "mid: flash+q8 KV, 10GiB reserved for Swarm, ctx 16k, keep 5m"
     else:
         overhead = 6 * 1024**3
         keep = "2m"
         kv = "q4_0"
+        ctx = 8192
         flash = False  # prefer stability on tiny cards
-        note = "low: conservative, 6GiB reserved, short keep-alive"
+        note = "low: conservative, 6GiB reserved, ctx 8k, short keep-alive"
 
     return OllamaTune(
         flash_attention=flash,
@@ -97,6 +102,7 @@ def ollama_tune_for(info: GpuInfo) -> OllamaTune:
         max_loaded_models=1,
         kv_cache_type=kv,
         gpu_overhead_bytes=overhead,
+        context_length=ctx,
         notes=note,
     )
 
@@ -139,6 +145,7 @@ def ollama_env_lines(tune: OllamaTune) -> list[str]:
         f"Environment=OLLAMA_MAX_LOADED_MODELS={tune.max_loaded_models}",
         f"Environment=OLLAMA_KEEP_ALIVE={tune.keep_alive}",
         f"Environment=OLLAMA_GPU_OVERHEAD={tune.gpu_overhead_bytes}",
+        f"Environment=OLLAMA_CONTEXT_LENGTH={tune.context_length}",
     ]
     if tune.flash_attention:
         lines.append("Environment=OLLAMA_FLASH_ATTENTION=1")

src/gpu_rent/remote/install_ollama.sh

@@ -45,13 +45,13 @@ try:
 except ValueError:
     ampere = False
 if gib >= 48:
-    tier, overhead, keep, kv, flash = "ultra", 20 * 1024**3, "30m", "q8_0", True
+    tier, overhead, keep, kv, flash, ctx = "ultra", 20 * 1024**3, "30m", "q8_0", True, 32768
 elif gib >= 24:
-    tier, overhead, keep, kv, flash = "high", 14 * 1024**3, "15m", "q8_0", True
+    tier, overhead, keep, kv, flash, ctx = "high", 14 * 1024**3, "15m", "q8_0", True, 16384
 elif gib >= 16:
-    tier, overhead, keep, kv, flash = "mid", 10 * 1024**3, "5m", "q8_0", True
+    tier, overhead, keep, kv, flash, ctx = "mid", 10 * 1024**3, "5m", "q8_0", True, 16384
 else:
-    tier, overhead, keep, kv, flash = "low", 6 * 1024**3, "2m", "q4_0", False
+    tier, overhead, keep, kv, flash, ctx = "low", 6 * 1024**3, "2m", "q4_0", False, 8192
 flash = bool(flash and (ampere or "A100" in name.upper() or "H100" in name.upper() or gib >= 16))
 lines = [
     f"# auto gpu-rent ollama tune tier={tier} gpu={name!r} vram_mib={vram}",
@@ -59,13 +59,14 @@ lines = [
     "OLLAMA_MAX_LOADED_MODELS=1",
     f"OLLAMA_KEEP_ALIVE={keep}",
     f"OLLAMA_GPU_OVERHEAD={overhead}",
+    f"OLLAMA_CONTEXT_LENGTH={ctx}",
 ]
 if flash:
     lines.append("OLLAMA_FLASH_ATTENTION=1")
 if kv:
     lines.append(f"OLLAMA_KV_CACHE_TYPE={kv}")
 out.write_text("\n".join(lines) + "\n", encoding="utf-8")
-print(f"tier={tier} flash={int(flash)} keep={keep} overhead_gib={overhead/1024**3:.0f}")
+print(f"tier={tier} flash={int(flash)} keep={keep} ctx={ctx} overhead_gib={overhead/1024**3:.0f}")
 PY
 }