98 lines
3.8 KiB
YAML
98 lines
3.8 KiB
YAML
|
# Qwen3-235B-A22B-Instruct-2507 (Unsloth UD-Q2_K_XL ~88.8 GB) via the
|
|||
|
|
# kyuz0 rocm-7.2.2 Strix Halo toolbox. Same image + unified-memory
|
|||
|
|
# recipe as compose/llama.yml; the only deltas are model path, port,
|
|||
|
|
# alias, context, and the no-coexist `restart: "no"`.
|
|||
|
|
# https://github.com/kyuz0/amd-strix-halo-toolboxes
|
|||
|
|
#
|
|||
|
|
# Coexistence. At ~88.8 GB weights this CANNOT coexist with the
|
|||
|
|
# 30B llama service or Kimi-Linear (vLLM) — the merged GPU arena is
|
|||
|
|
# only ~110 GB. Stop those before bringing this up. Same pattern as
|
|||
|
|
# compose/comfyui.yml: `restart: "no"`, manual start, swap workflow
|
|||
|
|
# documented in compose/qwen3-235b/README.md.
|
|||
|
|
#
|
|||
|
|
# Weights. UD-Q2_K_XL is Unsloth's "Dynamic" quant — important tensors
|
|||
|
|
# kept at higher precision; closer to Q3 quality than naive Q2. 2 shards
|
|||
|
|
# (~50 GB + 38.8 GB); llama.cpp auto-discovers shard 2 from shard 1.
|
|||
|
|
# Download path on the box (see compose/qwen3-235b/README.md):
|
|||
|
|
# hf download unsloth/Qwen3-235B-A22B-Instruct-2507-GGUF \
|
|||
|
|
# --include 'UD-Q2_K_XL/*' \
|
|||
|
|
# --local-dir /models/qwen/Qwen3-235B-A22B-Instruct-2507
|
|||
|
|
#
|
|||
|
|
# Port 8081 — distinct from llama 30B (8080) so opencode/curl/etc. can
|
|||
|
|
# address either explicitly even though only one runs at a time.
|
|||
|
|
#
|
|||
|
|
# Performance target. Bandwidth-bound: 256 GB/s ÷ ~22 GB active-bytes →
|
|||
|
|
# ~5-10 tok/s decode. This is the "overnight long-task" model, NOT the
|
|||
|
|
# interactive driver — see StrixHaloMemory.md for the bandwidth math.
|
|||
|
|
services:
|
|||
|
|
qwen3-235b:
|
|||
|
|
image: kyuz0/amd-strix-halo-toolboxes:rocm-7.2.2
|
|||
|
|
container_name: qwen3-235b
|
|||
|
|
# Manual start only — see header note about GPU contention.
|
|||
|
|
restart: "no"
|
|||
|
|
devices:
|
|||
|
|
- /dev/kfd:/dev/kfd
|
|||
|
|
- /dev/dri:/dev/dri
|
|||
|
|
cap_add:
|
|||
|
|
- SYS_PTRACE
|
|||
|
|
security_opt:
|
|||
|
|
- seccomp=unconfined
|
|||
|
|
# Numeric GIDs of host's video (44) and render (991) groups —
|
|||
|
|
# required for /dev/kfd + /dev/dri access from inside the container.
|
|||
|
|
group_add:
|
|||
|
|
- "44"
|
|||
|
|
- "991"
|
|||
|
|
shm_size: 8g
|
|||
|
|
ipc: host
|
|||
|
|
environment:
|
|||
|
|
# Unified-memory recipe (same as compose/llama.yml + kimi-linear).
|
|||
|
|
# BIOS UMA=0.5 GB + ttm.pages_limit cmdline → these flags merge the
|
|||
|
|
# rocminfo pools into one ~110 GB arena. kyuz0's image is native
|
|||
|
|
# gfx1151 so no HSA_OVERRIDE_GFX_VERSION.
|
|||
|
|
- HSA_XNACK=1
|
|||
|
|
- HSA_FORCE_FINE_GRAIN_PCIE=1
|
|||
|
|
volumes:
|
|||
|
|
- /models:/models:ro
|
|||
|
|
ports:
|
|||
|
|
- "8081:8081"
|
|||
|
|
entrypoint: ["llama-server"]
|
|||
|
|
command:
|
|||
|
|
- --model
|
|||
|
|
- /models/qwen/Qwen3-235B-A22B-Instruct-2507/UD-Q2_K_XL/Qwen3-235B-A22B-Instruct-2507-UD-Q2_K_XL-00001-of-00002.gguf
|
|||
|
|
# OpenAI-compatible served name. Provider-side name lives in
|
|||
|
|
# opencode.json once M0 perf is verified and we wire it up.
|
|||
|
|
- --alias
|
|||
|
|
- qwen3-235b
|
|||
|
|
- --host
|
|||
|
|
- 0.0.0.0
|
|||
|
|
- --port
|
|||
|
|
- "8081"
|
|||
|
|
- --n-gpu-layers
|
|||
|
|
- "999"
|
|||
|
|
# 64K — opencode auto-compaction triggers at ~75-80 % of the
|
|||
|
|
# stated context limit, so a small ctx fires the summarize-and-
|
|||
|
|
# rewind loop after only a few turns. 64K roughly doubles how
|
|||
|
|
# many turns fit. KV at q8_0 ≈ 8 GB (94 layers × 8 kv-heads × 128
|
|||
|
|
# head-dim × 2 × 65536 × 1 byte); arena headroom still ~11 GB.
|
|||
|
|
# Stretch goal 131072 documented in compose/qwen3-235b/README.md
|
|||
|
|
# but tight — verify allocator behaviour first.
|
|||
|
|
- --ctx-size
|
|||
|
|
- "65536"
|
|||
|
|
- --no-mmap
|
|||
|
|
# Flash attention is required for q8_0 KV cache in llama.cpp.
|
|||
|
|
- --flash-attn
|
|||
|
|
- "on"
|
|||
|
|
- --cache-type-k
|
|||
|
|
- q8_0
|
|||
|
|
- --cache-type-v
|
|||
|
|
- q8_0
|
|||
|
|
# Qwen3-235B-Instruct-2507 ships its own chat template; let
|
|||
|
|
# llama.cpp use it rather than the hardcoded default.
|
|||
|
|
- --jinja
|
|||
|
|
# Single sequence — KV pool isn't sliced across speculative
|
|||
|
|
# concurrent requests we'll never have (long-task model, one
|
|||
|
|
# request at a time).
|
|||
|
|
- --parallel
|
|||
|
|
- "1"
|
|||
|
|
- --metrics
|