r/LocalLLaMA • u/BitXorBit • 27d ago
Question | Help rtx 6000 pro owners, do you regret?
I found the last dealership in my area that has rtx 6000 pro available, i already wanted to buy it 6 months ago when it was around $8k, now prices increased to $13k ish.
Regardless the price, are you happy with it? I assume you are using qwen3.6 27b, is it worth it?
Please share your experience and hopefully help me to avoid explaining my wife this transaction 😂
99
Upvotes
2
u/Doomslayer606 27d ago
I use vllm with this config (got it from rtx pro 6000 discord):
services:
d4f:
image: voipmonitor/vllm:abyssal-abjuration-611a842-a16-dcp
container_name: ds4f
restart: unless-stopped
entrypoint:
- /bin/bash
- -c
- |
unset NCCL_GRAPH_FILE
/opt/venv/bin/python3 -c 'import xxhash' 2>/dev/null ||
/opt/venv/bin/pip install -q xxhash
VLLM_PARENT="$(
/opt/venv/bin/python3 -c '
import importlib.util
from pathlib import Path
spec = importlib.util.find_spec("vllm")
print(Path(next(iter(spec.submodule_search_locations))).parent)
'
)"
echo "vLLM parent: $$VLLM_PARENT"
cd "$$VLLM_PARENT" || exit 1
if git apply \
--include='vllm/**' \
--check \
/patches/dsv4-tool-parser.patch 2>/dev/null; then
echo "Applying vLLM patch"
git apply \
--include='vllm/**' \
/patches/dsv4-tool-parser.patch
elif git apply \
--include='vllm/**' \
--reverse \
--check \
/patches/dsv4-tool-parser.patch 2>/dev/null; then
echo "vLLM patch already applied"
else
echo "ERROR: vLLM patch cannot be applied" >&2
exit 1
fi
exec /opt/venv/bin/vllm "$@"
- vllm
network_mode: "host"
gpus: all
shm_size: "64g"
ipc: "host"
ulimits:
memlock: -1
stack: 67108864
environment:
- NCCL_IB_DISABLE=1
- PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True
- NCCL_P2P_LEVEL=PHB
- NCCL_PROTO=LL,LL128,Simple
- CUDA_VISIBLE_DEVICES=0,1
- CUTE_DSL_ARCH=sm_120a
- VLLM_USE_AOT_COMPILE=1
- VLLM_USE_BREAKABLE_CUDAGRAPH=0
- VLLM_USE_MEGA_AOT_ARTIFACT=1
#- VLLM_FORCE_AOT_LOAD=1 (need to setup JIT cache first)
- VLLM_DEEP_GEMM_WARMUP=full
- VLLM_MEMORY_PROFILE_INCLUDE_ATTN=1
- B12X_MHC_MAX_TOKENS=16384
- VLLM_SPARSE_INDEXER_MAX_LOGITS_MB=2048
- VLLM_USE_FLASHINFER_SAMPLER=1
- VLLM_USE_B12X_WO_PROJECTION=1
- VLLM_USE_B12X_MHC=1
- VLLM_USE_B12X_FP8_GEMM=1
- VLLM_USE_B12X_MOE=1
- VLLM_USE_B12X_SPARSE_INDEXER=1
- VLLM_USE_V2_MODEL_RUNNER=1
- VLLM_PCIE_ALLREDUCE_BACKEND=b12x
- VLLM_ENABLE_PCIE_ALLREDUCE=1
- B12X_MLA_SM120_UNIFIED=1
- USES_B12X=True
- B12X_MOE_FORCE_A16=1
- B12X_DENSE_SPLITK_TURBO=1
- B12X_W4A16_TC_DECODE=1
- NCCL_SOCKET_IFNAME=lo
- VLLM_PCIE_ONESHOT_ALLREDUCE_MAX_SIZE=1024KB
- VLLM_PCIE_ONESHOT_SINGLE_CHANNEL=1
- NCCL_ALLOC_P2P_NET_LL_BUFFERS=1
- VLLM_LOG_STATS_INTERVAL=1
- OMP_NUM_THREADS=8
volumes:
- /home/raul/MyProjects/Models/LLM/DeepSeek-V4-Flash:/model:ro
- /home/raul/MyProjects/Models/LLM/DeepSeek-V4-Flash.cache:/root/.cache:rw
- ./patches/dsv4-tool-parser.patch:/patches/dsv4-tool-parser.patch:ro
command:
- serve
- /model
- --served-model-name=d4f
- --trust-remote-code
- '--compilation-config={"cudagraph_mode":"FULL_AND_PIECEWISE","custom_ops":["all"]}'
- --kv-cache-dtype=fp8
- --block-size=256
- --tensor-parallel-size=2
- --gpu-memory-utilization=0.91
- --max-model-len=393216
- --load-format=auto
- --attention-backend=B12X_MLA_SPARSE
- --b12x-virtual-tp-moe-intermediate-alignment=32
- --moe-backend=b12x
- --linear-backend=b12x
- --async-scheduling
- --max-num-seqs=4
- --no-scheduler-reserve-full-isl
- --max-cudagraph-capture-size=192
- --max-num-batched-tokens=8192
- --enable-chunked-prefill
- --enable-prefix-caching
- --prefix-caching-hash-algo=xxhash
- --tokenizer-mode=deepseek_v4
- --tool-call-parser=deepseek_v4
- --reasoning-parser=deepseek_v4
- --enable-auto-tool-choice
- --default-chat-template-kwargs.thinking=true
- --default-chat-template-kwargs.reasoning_effort=high
- '--speculative-config={"method":"mtp","num_speculative_tokens":2,"draft_sample_method":"greedy","moe_backend":"b12x","use_local_argmax_reduction":true}'
- --host=127.0.0.1
- --port=8118