ModelTC · sufubao · Aug 28, 2026 · Aug 28, 2026 · Aug 28, 2026 · Aug 28, 2026
diff --git a/.dockerignore b/.dockerignore
@@ -0,0 +1,46 @@
+.git
+.git/**
+.github
+.idea
+.vscode
+.pytest_cache
+.mypy_cache
+.ruff_cache
+**/__pycache__
+**/*.pyc
+**/*.pyo
+
+# Local environments, credentials, and host configuration.
+.env
+.env.*
+.aws
+.sco
+.netrc
+*.key
+*.pem
+id_rsa*
+venv
+.venv
+
+# Models, caches, traces, and experiment output never belong in an image.
+models
+checkpoints
+cache*
+.cache
+traces
+trace-*
+*.log
+*.jsonl
+*.pt
+*.pth
+*.safetensors
+
+# Runtime images only need the package and release Docker inputs.
+assets
+demos
+docs
+skills
+test
+unit_tests
+tools
+format_out
diff --git a/GLM53_H100_DEPLOY.md b/GLM53_H100_DEPLOY.md
@@ -0,0 +1,91 @@
+# GLM-5.3-Flash H100/H200 部署与验证
+
+该配置使用 TP8、BF16 KV cache、32K batch token、1.12M token cache 和
+CUDA Graph 128，面向 16K 输入、256 输出的高并发文本服务。
+
+## 镜像
+
+私有仓库标签：
+
+```text
+registry.ms-sc-01.maoshanwangtech.com/ms-ccr/lightllm:v1.5.0-glm53-16k256-kpool-996cef93
+```
+
+不可变镜像：
+
+```text
+registry.ms-sc-01.maoshanwangtech.com/ms-ccr/lightllm@sha256:0260e9884e46de899f4b845aa3796d0603b7b6ee7593c1fded35b7cee4462169
+```
+
+本机镜像为 `lightllm-glm53:16k256-kpool`，镜像 ID 为
+`sha256:8fbb91ee50dde5af4289d9e3cc75dddf93a8dd6fc2a201222a28612a5679cf0c`。
+同一不可变镜像已拉取到 H100 节点。
+
+## H100 部署命令
+
+镜像已内置服务参数；启动时显式开启 K-pool decode 快路径：
+
+```bash
+IMAGE="registry.ms-sc-01.maoshanwangtech.com/ms-ccr/lightllm@sha256:0260e9884e46de899f4b845aa3796d0603b7b6ee7593c1fded35b7cee4462169"
+
+sudo docker pull "$IMAGE"
+sudo docker run -d \
+  --name glm53-lightllm-16k256-kpool \
+  --restart unless-stopped \
+  --network host \
+  --ipc host \
+  --gpus all \
+  -e LIGHTLLM_ENABLE_KPOOL_DECODE_FASTPATH=1 \
+  -e PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True \
+  --ulimit memlock=-1 \
+  --ulimit nofile=1048576:1048576 \
+  -v /home/devsft/models/GLM-5.3-Flash:/model:ro \
+  -v /home/devsft/cache-glm53-lightllm:/root/.cache \
+  -v /home/devsft/cache-glm53-triton:/root/.triton \
+  -v /home/devsft/cache-glm53-deep-gemm:/root/.deep_gemm \
+  "$IMAGE"
+```
+
+接口为 `http://127.0.0.1:8002/v1`。该快路径要求请求 prompt 长度按
+16 token 对齐，并使用镜像内置的关闭 prompt cache、关闭 MTP 配置；其他负载可移除
+`LIGHTLLM_ENABLE_KPOOL_DECODE_FASTPATH`，使用保守路径。
+
+本机 H200 可直接运行：
+
+```bash
+LIGHTLLM_GLM53_MODEL_DIR=/nvme/sufubao/models/GLM-5.3-Flash \
+LIGHTLLM_GLM53_CACHE_DIR=/nvme/sufubao/m39-home/cache/glm53-lightllm-h200 \
+LIGHTLLM_GLM53_TRITON_CACHE_DIR=/nvme/sufubao/m39-home/cache/glm53-triton-h200 \
+LIGHTLLM_GLM53_DEEP_GEMM_CACHE_DIR=/nvme/sufubao/m39-home/cache/glm53-deep-gemm-h200 \
+tools/run_glm53_h100_container.sh
+```
+
+## 16K/256 性能
+
+LightLLM 与 vLLM 在同一台空闲的 8×H100 80GB 节点顺序测试，使用同一 FP8
+checkpoint、TP8、BF16 KV cache、32K batch token、关闭 prompt/prefix cache、
+seed 42、temperature 0、无限请求速率和一次不计入结果的 warmup。每档请求数等于
+并发数，每个请求输入 16,384 token、输出 256 token。
+
+| 并发 | LightLLM 总吞吐 tok/s | vLLM 总吞吐 tok/s | LightLLM / vLLM |
+| ---: | ---: | ---: | ---: |
+| 1 | 5,373.86 | 6,542.89 | 82.13% |
+| 8 | 16,444.11 | 17,635.33 | 93.25% |
+| 16 | 20,669.61 | 25,246.99 | 81.87% |
+| 64 | 25,614.05 | 31,485.12 | 81.35% |
+| 128 | 25,588.22 | 27,988.09 | 91.43% |
+| 256 | 25,623.93 | 31,206.94 | 82.11% |
+
+结果文件位于
+`/nvme/sufubao/m39-home/results/glm53_h100_16k_256_kpool_decode_ab/` 和
+`/nvme/sufubao/m39-home/results/glm53_h100_16k_256_vllm_opt/`；所有启动、性能与
+精度实验均由 `exp` 归档到 `~/experiments/runs/`。
+
+## 精度
+
+| 验证 | 结果 |
+| --- | --- |
+| GSM8K 固定前 100 题、5-shot、greedy | **99/100**；100/100 完成；无截断 |
+| 精确 16,384-token needle | **PASS**；API 与本地 tokenizer 均计数 16,384；找回 `ZEBRA-4821` |
+
+相关实验记录为 `260901-020939` 和 `260901-021013`。
diff --git a/docker/Dockerfile.glm53-h100 b/docker/Dockerfile.glm53-h100
@@ -0,0 +1,188 @@
+# This is the exact SGLang GLM-5.3 runtime used for the H200 validation.
+ARG BASE_IMAGE=lmsysorg/sglang@sha256:92afb4c878eef9cbb17ca9a2c1d15d5cda58585f90bbf5915a79f0f6284aad10
+FROM ${BASE_IMAGE} AS prepared
+
+ARG PIP_INDEX_URL
+
+WORKDIR /opt/lightllm
+
+COPY docker/requirements-glm53-runtime.txt /tmp/requirements-glm53-runtime.txt
+RUN --mount=type=cache,id=glm53-pip,target=/root/.cache/pip \
+    python -m pip install --no-deps --require-hashes \
+      -r /tmp/requirements-glm53-runtime.txt && \
+    rm /tmp/requirements-glm53-runtime.txt
+
+COPY setup.py LICENSE README.md ./
+COPY lightllm ./lightllm
+COPY docker/glm53-h100.openvex.json /usr/share/doc/lightllm/glm53-h100.openvex.json
+RUN --mount=type=cache,id=glm53-pip,target=/root/.cache/pip \
+    python -m pip install --no-deps . && \
+    rm -rf \
+      /etc/ssh/ssh_host_*_key* \
+      /opt/nvidia/nsight-compute/2025.3.1/host/target-linux-x64/plugins/efa_metrics \
+      /opt/nvidia/nsight-systems-cli/2026.4.1/target-linux-x64/plugins/efa_metrics \
+      /root/.cargo/registry \
+      /sgl-workspace/sglang/python/sglang/multimodal_gen \
+      /sgl-workspace/sglang/python/sglang/srt/disaggregation \
+      /sgl-workspace/sglang/scripts/playground/replay_request_dump.py \
+      /opt/sglang/bin/pip \
+      /opt/sglang/bin/pip3 \
+      /opt/sglang/bin/pip3.12 \
+      /opt/sglang/lib/python3.12/site-packages/pip \
+      /opt/sglang/lib/python3.12/site-packages/pip-*.dist-info \
+      /opt/sglang/lib/python3.12/site-packages/setuptools \
+      /opt/sglang/lib/python3.12/site-packages/setuptools-*.dist-info \
+      /opt/sglang/lib/python3.12/site-packages/wheel \
+      /opt/sglang/lib/python3.12/site-packages/wheel-*.dist-info && \
+    python -c "import frozendict, hypercorn, importlib.metadata as metadata, importlib.util, lightllm.server.api_start, msgpack, redis, rpyc, ujson; from sglang.srt.layers.moe.moe_runner.triton_utils import fused_moe; assert metadata.version('hypercorn') == '0.18.0'; assert msgpack.__version__ == '1.2.2'; assert redis.__version__ == '7.3.0'; assert rpyc.__version__ == '6.0.2'; assert importlib.util.find_spec('pip') is None; assert importlib.util.find_spec('setuptools') is None; print('Hardened LightLLM GLM-5.3 runtime import OK')" && \
+    test ! -e /etc/ssh/ssh_host_rsa_key && \
+    test ! -e /sgl-workspace/sglang/python/sglang/multimodal_gen && \
+    test ! -e /sgl-workspace/sglang/python/sglang/srt/disaggregation
+
+# Flatten the prepared root filesystem so deleted base-image host keys and
+# developer-only vulnerable components are absent from published layers,
+# rather than merely hidden by an OCI whiteout. Size-bound the clean content
+# layers because the private registry's cross-region gateway cannot reliably
+# accept one multi-gigabyte upload stream; the merged filesystem is unchanged.
+FROM prepared AS partitioned
+
+COPY docker/split_glm53_rootfs.py /tmp/split_glm53_rootfs.py
+RUN /usr/bin/python3 /tmp/split_glm53_rootfs.py
+
+FROM prepared AS prepared_remainder
+
+RUN rm -rf \
+      /opt \
+      /root \
+      /sgl-workspace \
+      /usr/include \
+      /usr/lib/x86_64-linux-gnu \
+      /usr/libexec \
+      /usr/local \
+      /usr/share
+
+FROM scratch AS runtime
+
+ARG OCI_CREATED
+ARG OCI_REVISION
+ARG OCI_SOURCE=https://github.com/sufubao/LightLLM
+ARG OCI_VERSION
+ARG BASE_NAME=lmsysorg/sglang:glm-5.3-flash
+ARG BASE_DIGEST=sha256:92afb4c878eef9cbb17ca9a2c1d15d5cda58585f90bbf5915a79f0f6284aad10
+
+COPY --from=prepared_remainder / /
+COPY --from=partitioned /__image_layers/00/ /
+COPY --from=partitioned /__image_layers/01/ /
+COPY --from=partitioned /__image_layers/02/ /
+COPY --from=partitioned /__image_layers/03/ /
+COPY --from=partitioned /__image_layers/04/ /
+COPY --from=partitioned /__image_layers/05/ /
+COPY --from=partitioned /__image_layers/06/ /
+COPY --from=partitioned /__image_layers/07/ /
+COPY --from=partitioned /__image_layers/08/ /
+COPY --from=partitioned /__image_layers/09/ /
+COPY --from=partitioned /__image_layers/10/ /
+COPY --from=partitioned /__image_layers/11/ /
+COPY --from=partitioned /__image_layers/12/ /
+COPY --from=partitioned /__image_layers/13/ /
+COPY --from=partitioned /__image_layers/14/ /
+COPY --from=partitioned /__image_layers/15/ /
+COPY --from=partitioned /__image_layers/16/ /
+COPY --from=partitioned /__image_layers/17/ /
+COPY --from=partitioned /__image_layers/18/ /
+COPY --from=partitioned /__image_layers/19/ /
+COPY --from=partitioned /__image_layers/20/ /
+COPY --from=partitioned /__image_layers/21/ /
+COPY --from=partitioned /__image_layers/22/ /
+COPY --from=partitioned /__image_layers/23/ /
+COPY --from=partitioned /__image_layers/24/ /
+COPY --from=partitioned /__image_layers/25/ /
+COPY --from=partitioned /__image_layers/26/ /
+COPY --from=partitioned /__image_layers/27/ /
+COPY --from=partitioned /__image_layers/28/ /
+COPY --from=partitioned /__image_layers/29/ /
+COPY --from=partitioned /__image_layers/30/ /
+COPY --from=partitioned /__image_layers/31/ /
+
+LABEL org.opencontainers.image.created="${OCI_CREATED}" \
+      org.opencontainers.image.revision="${OCI_REVISION}" \
+      org.opencontainers.image.source="${OCI_SOURCE}" \
+      org.opencontainers.image.version="${OCI_VERSION}" \
+      org.opencontainers.image.title="LightLLM GLM-5.3-Flash H100/H200 TP8 optimized text" \
+      org.opencontainers.image.base.name="${BASE_NAME}" \
+      org.opencontainers.image.base.digest="${BASE_DIGEST}" \
+      ai.lightllm.model="GLM-5.3-Flash" \
+      ai.lightllm.accelerator="NVIDIA H100/H200" \
+      ai.lightllm.tensor-parallel-size="8" \
+      ai.lightllm.profile="text-16k256-tp8-kpool-c256-no-prompt-cache" \
+      ai.lightllm.security-profile="flattened-no-sglang-server-components"
+
+ENV PATH=/opt/sglang/bin:/usr/local/nvidia/bin:/usr/local/cuda/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin \
+    LD_LIBRARY_PATH=/usr/local/nvidia/lib:/usr/local/nvidia/lib64:/usr/local/cuda/lib64 \
+    LIBRARY_PATH=/usr/local/cuda/lib64/stubs \
+    CUDA_HOME=/usr/local/cuda \
+    CUDA_VERSION=13.0.3 \
+    NVARCH=x86_64 \
+    NVIDIA_REQUIRE_CUDA="cuda>=13.0" \
+    NVIDIA_VISIBLE_DEVICES=all \
+    NVIDIA_DRIVER_CAPABILITIES=compute,utility \
+    LANG=en_US.UTF-8 \
+    LANGUAGE=en_US:en \
+    LC_ALL=en_US.UTF-8 \
+    PYTHONUNBUFFERED=1 \
+    PYTHONDONTWRITEBYTECODE=1 \
+    LOADWORKER=18 \
+    LIGHTLLM_USE_SGLANG_TRITON_MOE=1 \
+    LIGHTLLM_SGLANG_FUSED_MOE_SUM=1 \
+    LIGHTLLM_VOCAB_PARALLEL_GREEDY=1 \
+    LIGHTLLM_SYMM_MEM_OUT_OF_PLACE=1 \
+    LIGHTLLM_LOG_LEVEL=warning \
+    NCCL_CUMEM_ENABLE=1 \
+    NCCL_NVLS_ENABLE=1 \
+    CUDA_DEVICE_MAX_CONNECTIONS=8 \
+    TRTLLM_ENABLE_PDL=1 \
+    NCCL_GRAPH_MIXING_SUPPORT=0
+
+WORKDIR /opt/lightllm
+
+RUN test "$(command -v python)" = /opt/sglang/bin/python && \
+    python -c "import hypercorn, importlib.metadata as metadata, lightllm.server.api_start, redis, rpyc, torch; assert metadata.version('hypercorn') == '0.18.0'; assert redis.__version__ == '7.3.0'; assert rpyc.__version__ == '6.0.2'; print(torch.__version__)"
+
+EXPOSE 8002
+STOPSIGNAL SIGTERM
+ENTRYPOINT ["/opt/nvidia/nvidia_entrypoint.sh"]
+
+HEALTHCHECK --interval=30s --timeout=5s --start-period=15m --retries=3 \
+    CMD /opt/sglang/bin/python -c "import urllib.request; urllib.request.urlopen('http://127.0.0.1:8002/v1/models', timeout=3).read()" || exit 1
+
+# Exact 16K-input/256-output profile validated on an eight-H100 80 GB node.
+# The larger KV capacity keeps a 64-request prompt wave resident, and decode
+# CUDA graphs cover batches through 128.
+CMD ["/opt/sglang/bin/python", "-m", "lightllm.server.api_server", \
+     "--model_dir", "/model", \
+     "--model_name", "glm-5.3-flash", \
+     "--tp", "8", \
+     "--host", "0.0.0.0", \
+     "--port", "8002", \
+     "--httpserver_workers", "16", \
+     "--mem_fraction", ".90", \
+     "--max_total_token_num", "1120000", \
+     "--running_max_req_size", "256", \
+     "--max_req_total_len", "532480", \
+     "--batch_max_tokens", "32768", \
+     "--chunked_prefill_size", "32768", \
+     "--linear_att_ssm_data_type", "bfloat16", \
+     "--linear_att_cache_size", "256", \
+     "--graph_max_batch_size", "128", \
+     "--graph_split_batch_size", "8", \
+     "--graph_grow_step_size", "8", \
+     "--graph_max_len_in_batch", "32768", \
+     "--disable_flashinfer_allreduce", \
+     "--enable_fused_shared_experts", \
+     "--disable_dynamic_prompt_cache", \
+     "--router_max_wait_tokens", "64", \
+     "--disable_vision", \
+     "--cache_capacity", "64", \
+     "--schedule_time_interval", "0.001", \
+     "--reasoning_parser", "glm45", \
+     "--tool_call_parser", "glm47"]
diff --git a/docker/Dockerfile.glm53-h100-overlay b/docker/Dockerfile.glm53-h100-overlay
@@ -0,0 +1,66 @@
+# Fast release path for environments where the original Docker Hub base is
+# unavailable. The pinned base is the already-flattened GLM-5.3 runtime; this
+# layer replaces only the public LightLLM source and the validated default CMD.
+ARG BASE_IMAGE=registry.ms-sc-01.maoshanwangtech.com/ms-ccr/lightllm@sha256:3801951c8697edcab68ac5d38efc1807fcda0cba3d7946125a04103c302a3845
+FROM ${BASE_IMAGE}
+
+ARG OCI_CREATED
+ARG OCI_REVISION
+ARG OCI_SOURCE=https://github.com/sufubao/LightLLM
+ARG OCI_VERSION
+ARG BASE_NAME=registry.ms-sc-01.maoshanwangtech.com/ms-ccr/lightllm
+ARG BASE_DIGEST=sha256:3801951c8697edcab68ac5d38efc1807fcda0cba3d7946125a04103c302a3845
+
+WORKDIR /opt/lightllm
+
+COPY lightllm ./lightllm
+
+RUN test -f 'lightllm/common/triton_utils/autotune_kernel_configs/triton_3.7.1/NVIDIA_H100_80GB_HBM3/chunk_gated_delta_rule_fwd_h/{BT=64,H=8,K=128,V=128}_NVIDIA_H100_80GB_HBM3.json' && \
+    test -f 'lightllm/common/triton_utils/autotune_kernel_configs/triton_3.7.1/NVIDIA_H100_80GB_HBM3/silu_and_mul_fwd:v1/{N=1536,out_dtype=torch.bfloat16}_NVIDIA_H100_80GB_HBM3.json' && \
+    test -f 'lightllm/common/triton_utils/autotune_kernel_configs/triton_3.7.1/NVIDIA_H100_80GB_HBM3/silu_and_mul_fwd:v1/{N=2048,out_dtype=torch.bfloat16}_NVIDIA_H100_80GB_HBM3.json' && \
+    python -c "import lightllm.server.api_start; print('Updated LightLLM GLM-5.3 runtime import OK')"
+
+LABEL org.opencontainers.image.created="${OCI_CREATED}" \
+      org.opencontainers.image.revision="${OCI_REVISION}" \
+      org.opencontainers.image.source="${OCI_SOURCE}" \
+      org.opencontainers.image.version="${OCI_VERSION}" \
+      org.opencontainers.image.title="LightLLM GLM-5.3-Flash H100/H200 TP8 optimized text" \
+      org.opencontainers.image.base.name="${BASE_NAME}" \
+      org.opencontainers.image.base.digest="${BASE_DIGEST}" \
+      ai.lightllm.model="GLM-5.3-Flash" \
+      ai.lightllm.accelerator="NVIDIA H100/H200" \
+      ai.lightllm.tensor-parallel-size="8" \
+      ai.lightllm.profile="text-16k256-tp8-kpool-c256-no-prompt-cache" \
+      ai.lightllm.security-profile="flattened-base-plus-source-overlay"
+
+# Exact 16K-input/256-output profile validated on an eight-H100 80 GB node.
+# The larger KV capacity keeps a 64-request prompt wave resident, and decode
+# CUDA graphs cover batches through 128.
+CMD ["/opt/sglang/bin/python", "-m", "lightllm.server.api_server", \
+     "--model_dir", "/model", \
+     "--model_name", "glm-5.3-flash", \
+     "--tp", "8", \
+     "--host", "0.0.0.0", \
+     "--port", "8002", \
+     "--httpserver_workers", "16", \
+     "--mem_fraction", ".90", \
+     "--max_total_token_num", "1120000", \
+     "--running_max_req_size", "256", \
+     "--max_req_total_len", "532480", \
+     "--batch_max_tokens", "32768", \
+     "--chunked_prefill_size", "32768", \
+     "--linear_att_ssm_data_type", "bfloat16", \
+     "--linear_att_cache_size", "256", \
+     "--graph_max_batch_size", "128", \
+     "--graph_split_batch_size", "8", \
+     "--graph_grow_step_size", "8", \
+     "--graph_max_len_in_batch", "32768", \
+     "--disable_flashinfer_allreduce", \
+     "--enable_fused_shared_experts", \
+     "--disable_dynamic_prompt_cache", \
+     "--router_max_wait_tokens", "64", \
+     "--disable_vision", \
+     "--cache_capacity", "64", \
+     "--schedule_time_interval", "0.001", \
+     "--reasoning_parser", "glm45", \
+     "--tool_call_parser", "glm47"]