Update GPU performance tuning and documentation for context length
- Enhanced the `OllamaTune` class to include a new `context_length` attribute, improving the configuration for different GPU tiers. - Updated performance tuning logic to set appropriate context lengths for low, mid, high, and ultra tiers, ensuring optimal resource allocation. - Modified installation scripts to reflect the new context length settings, enhancing the installation process for Ollama. - Revised documentation to include context length details in the GPU performance table, providing clearer guidance for users. - Added tests to validate the correct context length settings in various scenarios, ensuring robustness in performance tuning.
This commit is contained in:
@@ -41,6 +41,8 @@ def test_ollama_high_reserves_vram_for_swarm():
|
||||
env = "\n".join(ollama_env_lines(tune))
|
||||
assert "OLLAMA_FLASH_ATTENTION=1" in env
|
||||
assert "OLLAMA_GPU_OVERHEAD=" in env
|
||||
assert "OLLAMA_CONTEXT_LENGTH=16384" in env
|
||||
assert tune.context_length == 16384
|
||||
|
||||
|
||||
def test_swarm_sage_on_ampere_mid():
|
||||
@@ -68,3 +70,16 @@ def test_swarm_no_sage_on_low():
|
||||
st = swarm_tune_for(info)
|
||||
assert not st.use_sage_attention
|
||||
assert st.comfy_extra_args == ""
|
||||
|
||||
|
||||
def test_ollama_mid_4090_context_16k():
|
||||
info = GpuInfo(
|
||||
name="NVIDIA GeForce RTX 4090",
|
||||
vram_mib=24564,
|
||||
compute_cap="8.9",
|
||||
uuid="GPU-4",
|
||||
tier="mid",
|
||||
)
|
||||
tune = ollama_tune_for(info)
|
||||
assert tune.context_length == 16384
|
||||
assert "OLLAMA_CONTEXT_LENGTH=16384" in "\n".join(ollama_env_lines(tune))
|
||||
|
||||
Reference in New Issue
Block a user