starvla: add Qwen3-VL and Qwen2.5-VL policy support

.gitattributes

@@ -0,0 +1 @@
+patches/**/*.patch -whitespace

.gitignore

@@ -20,6 +20,9 @@ dist/
 checkpoint/
 checkpoints/
 *.safetensors
+eval/results/
+/goldens/
+/Testing/
 
 # macOS resource forks
 ._*

CMakeLists.txt

@@ -1,9 +1,12 @@
 cmake_minimum_required(VERSION 3.16)
 project(robotcpp VERSION 0.1.0 LANGUAGES C CXX)
 
+include(CTest)
+
 option(ROBOT_CPP_BUILD_ROBOT_SERVER "Build model-server target" ON)
 option(ROBOT_CPP_BUILD_MODEL_CLI "Build model-cli target" OFF)
 option(ROBOT_CPP_BUILD_ROBOT_CLIENT "Build C++ robot client targets" OFF)
+option(ROBOT_CPP_BUILD_STARVLA "Build the StarVLA runtime (requires llama.cpp overlay)" OFF)
 
 set(CMAKE_CXX_STANDARD 17)
 set(CMAKE_CXX_STANDARD_REQUIRED ON)
@@ -12,6 +15,27 @@ set(CMAKE_CXX_EXTENSIONS OFF)
 if(NOT EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/third_party/llama.cpp/CMakeLists.txt")
     message(FATAL_ERROR "third_party/llama.cpp is required; run `git submodule update --init --recursive`")
 endif()
+if(ROBOT_CPP_BUILD_STARVLA)
+    file(READ
+        "${CMAKE_CURRENT_SOURCE_DIR}/third_party/llama.cpp/tools/mtmd/models/qwen3vl.cpp"
+        ROBOT_CPP_QWEN3VL_MTMD_SOURCE)
+    file(READ
+        "${CMAKE_CURRENT_SOURCE_DIR}/third_party/llama.cpp/include/llama.h"
+        ROBOT_CPP_LLAMA_PUBLIC_HEADER)
+    string(FIND "${ROBOT_CPP_QWEN3VL_MTMD_SOURCE}"
+        "FFN_GELU_ERF" ROBOT_CPP_QWEN3VL_PARITY_PATCH_INDEX)
+    string(FIND "${ROBOT_CPP_LLAMA_PUBLIC_HEADER}"
+        "llama_set_backend_native_graphs_enabled" ROBOT_CPP_LLAMA_GRAPH_PATCH_INDEX)
+    if(ROBOT_CPP_QWEN3VL_PARITY_PATCH_INDEX EQUAL -1 OR
+       ROBOT_CPP_LLAMA_GRAPH_PATCH_INDEX EQUAL -1)
+        message(FATAL_ERROR
+            "StarVLA requires the pinned llama.cpp overlay. Run "
+            "`./tools/apply_patches.sh` from the repository root, "
+            "then configure again.")
+    endif()
+    unset(ROBOT_CPP_QWEN3VL_MTMD_SOURCE)
+    unset(ROBOT_CPP_LLAMA_PUBLIC_HEADER)
+endif()
 set(LLAMA_BUILD_COMMON ON CACHE BOOL "" FORCE)
 set(LLAMA_BUILD_TOOLS OFF CACHE BOOL "" FORCE)
 set(LLAMA_BUILD_EXAMPLES OFF CACHE BOOL "" FORCE)
@@ -20,7 +44,16 @@ add_subdirectory(third_party/llama.cpp EXCLUDE_FROM_ALL)
 if(NOT TARGET ggml OR NOT TARGET llama)
     message(FATAL_ERROR "llama.cpp must provide ggml and llama targets")
 endif()
-
+if(ROBOT_CPP_BUILD_STARVLA)
+    # mtmd normally inherits this variable when llama.cpp builds all tools.
+    if(NOT LLAMA_INSTALL_VERSION)
+        set(LLAMA_INSTALL_VERSION ${PROJECT_VERSION})
+    endif()
+    add_subdirectory(third_party/llama.cpp/tools/mtmd EXCLUDE_FROM_ALL)
+    if(NOT TARGET mtmd)
+        message(FATAL_ERROR "llama.cpp must provide the mtmd target for Qwen-VL")
+    endif()
+endif()
 set(ROBOT_CPP_LLAMA_INCLUDE_DIRS
     ${CMAKE_CURRENT_SOURCE_DIR}/third_party/llama.cpp
     ${CMAKE_CURRENT_SOURCE_DIR}/third_party/llama.cpp/include
@@ -30,14 +63,29 @@ set(ROBOT_CPP_LLAMA_INCLUDE_DIRS
 )
 
 set(SMOLVLA_DIR ${CMAKE_CURRENT_SOURCE_DIR}/src/models/smolvla)
+set(STARVLA_DIR ${CMAKE_CURRENT_SOURCE_DIR}/src/models/starvla)
 set(ROBOT_SERVER_DIR ${CMAKE_CURRENT_SOURCE_DIR}/robot_server)
 set(ROBOT_CLIENT_DIR ${CMAKE_CURRENT_SOURCE_DIR}/robot_client)
 
-add_library(smolvla_runtime STATIC
+add_library(robotcpp_model_common STATIC
     src/models/ggml_backend.cpp
     src/models/ggml_backend.h
     src/models/gguf_loader.cpp
     src/models/gguf_loader.h
+    src/models/model_type.cpp
+)
+target_include_directories(robotcpp_model_common
+    PUBLIC
+        ${CMAKE_CURRENT_SOURCE_DIR}/src
+        ${ROBOT_CPP_LLAMA_INCLUDE_DIRS}
+)
+target_link_libraries(robotcpp_model_common PUBLIC ggml)
+target_compile_features(robotcpp_model_common PUBLIC cxx_std_17)
+if(NOT MSVC)
+    target_compile_options(robotcpp_model_common PRIVATE -Wno-cast-qual)
+endif()
+
+add_library(smolvla_runtime STATIC
     ${SMOLVLA_DIR}/smolvla_engine.cpp
     ${SMOLVLA_DIR}/smolvla_engine.h
     ${SMOLVLA_DIR}/state_proj.cpp
@@ -53,17 +101,13 @@ target_include_directories(smolvla_runtime
         ${SMOLVLA_DIR}
         ${ROBOT_CPP_LLAMA_INCLUDE_DIRS}
 )
-target_link_libraries(smolvla_runtime PUBLIC ggml llama)
+target_link_libraries(smolvla_runtime PUBLIC robotcpp_model_common llama)
 target_compile_features(smolvla_runtime PUBLIC cxx_std_17)
 if(NOT MSVC)
     target_compile_options(smolvla_runtime PRIVATE -Wno-cast-qual)
 endif()
 
 add_library(pi0_engine STATIC
-    src/models/ggml_backend.cpp
-    src/models/ggml_backend.h
-    src/models/gguf_loader.cpp
-    src/models/gguf_loader.h
     src/models/pi0/types.h
     src/models/pi0/action.cpp
     src/models/pi0/action.h
@@ -87,13 +131,65 @@ target_include_directories(pi0_engine
         ${CMAKE_CURRENT_SOURCE_DIR}/src
         ${ROBOT_CPP_LLAMA_INCLUDE_DIRS}
 )
-target_link_libraries(pi0_engine PUBLIC ggml llama)
+target_link_libraries(pi0_engine PUBLIC robotcpp_model_common llama)
 target_compile_features(pi0_engine PUBLIC cxx_std_17)
 if(NOT MSVC)
     target_compile_options(pi0_engine PRIVATE -Wno-cast-qual)
 endif()
 
+if(ROBOT_CPP_BUILD_STARVLA)
+add_library(starvla_runtime STATIC
+    ${STARVLA_DIR}/fast_codec.cpp
+    ${STARVLA_DIR}/fast_codec.h
+    ${STARVLA_DIR}/fast_policy.cpp
+    ${STARVLA_DIR}/fast_policy.h
+    ${STARVLA_DIR}/groot_policy.cpp
+    ${STARVLA_DIR}/groot_policy.h
+    ${STARVLA_DIR}/groot_prompt.cpp
+    ${STARVLA_DIR}/groot_prompt.h
+    ${STARVLA_DIR}/normalization.cpp
+    ${STARVLA_DIR}/normalization.h
+    ${STARVLA_DIR}/oft_image_preprocess.cpp
+    ${STARVLA_DIR}/oft_image_preprocess.h
+    ${STARVLA_DIR}/oft_prompt.cpp
+    ${STARVLA_DIR}/oft_prompt.h
+    ${STARVLA_DIR}/oft_policy.cpp
+    ${STARVLA_DIR}/oft_policy.h
+    ${STARVLA_DIR}/pi_policy.cpp
+    ${STARVLA_DIR}/pi_policy.h
+    ${STARVLA_DIR}/pi_v3_policy.cpp
+    ${STARVLA_DIR}/pi_v3_policy.h
+    ${STARVLA_DIR}/policy_gguf.h
+    ${STARVLA_DIR}/qwen3vl_bridge.cpp
+    ${STARVLA_DIR}/qwen3vl_bridge.h
+    ${STARVLA_DIR}/starvla_engine.cpp
+    ${STARVLA_DIR}/starvla_engine.h
+    third_party/llama.cpp/examples/gguf-hash/deps/sha256/sha256.c
+)
+target_include_directories(starvla_runtime
+    PUBLIC
+        ${CMAKE_CURRENT_SOURCE_DIR}/src
+        ${ROBOT_CPP_LLAMA_INCLUDE_DIRS}
+    PRIVATE
+        ${CMAKE_CURRENT_SOURCE_DIR}/third_party/llama.cpp/vendor
+        ${CMAKE_CURRENT_SOURCE_DIR}/third_party/llama.cpp/examples/gguf-hash/deps
+        ${CMAKE_CURRENT_SOURCE_DIR}/third_party/llama.cpp/examples/gguf-hash/deps/sha256
+)
+target_link_libraries(starvla_runtime PUBLIC robotcpp_model_common llama mtmd)
+target_compile_features(starvla_runtime PUBLIC cxx_std_17)
+if(GGML_CUDA)
+    enable_language(CUDA)
+    target_sources(starvla_runtime PRIVATE ${STARVLA_DIR}/qwen_bf16_round_cuda.cu)
+    target_compile_definitions(starvla_runtime PRIVATE ROBOTCPP_STARVLA_CUDA=1)
+    set_property(TARGET starvla_runtime PROPERTY CUDA_STANDARD 17)
+endif()
+if(NOT MSVC)
+    target_compile_options(starvla_runtime PRIVATE -Wno-cast-qual)
+endif()
+endif()
+
 add_library(robotcpp STATIC
+    src/models/argument_parse.h
     src/models/model.h
     src/models/model_factory.cpp
     src/models/pi0/pi0_model.cpp
@@ -108,6 +204,19 @@ target_include_directories(robotcpp
 )
 target_link_libraries(robotcpp PUBLIC smolvla_runtime pi0_engine)
 target_compile_features(robotcpp PUBLIC cxx_std_17)
+if(ROBOT_CPP_BUILD_STARVLA)
+    target_sources(robotcpp PRIVATE
+        ${STARVLA_DIR}/starvla_model.cpp
+        ${STARVLA_DIR}/starvla_model.h)
+    target_link_libraries(robotcpp PUBLIC starvla_runtime)
+    target_compile_definitions(robotcpp PUBLIC ROBOT_CPP_BUILD_STARVLA=1)
+endif()
+
+if(BUILD_TESTING AND ROBOT_CPP_BUILD_STARVLA)
+    add_executable(robotcpp-starvla-model-test tests/starvla/model_test.cpp)
+    target_link_libraries(robotcpp-starvla-model-test PRIVATE robotcpp)
+    add_test(NAME robotcpp-starvla-model-test COMMAND robotcpp-starvla-model-test)
+endif()
 
 if(ROBOT_CPP_BUILD_ROBOT_SERVER OR ROBOT_CPP_BUILD_ROBOT_CLIENT)
     add_library(robot_server_common STATIC
@@ -139,6 +248,7 @@ if(ROBOT_CPP_BUILD_ROBOT_CLIENT)
     set_target_properties(model-cpp-client-example PROPERTIES RUNTIME_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/bin)
     target_link_libraries(model-cpp-client-example PRIVATE model_client_cpp)
     target_compile_features(model-cpp-client-example PRIVATE cxx_std_17)
+
 endif()
 
 if(ROBOT_CPP_BUILD_ROBOT_SERVER)
@@ -165,7 +275,6 @@ if(ROBOT_CPP_BUILD_ROBOT_SERVER)
     )
     target_link_libraries(model-server PRIVATE robot_server_core robotcpp)
     target_compile_features(model-server PRIVATE cxx_std_17)
-
     add_executable(smolvla-raw-predict ${ROBOT_SERVER_DIR}/test/smolvla_raw_predict.cpp)
     set_target_properties(smolvla-raw-predict PROPERTIES RUNTIME_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/bin)
     target_include_directories(smolvla-raw-predict PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/src ${SMOLVLA_DIR})

README.md

@@ -54,8 +54,13 @@ We also provide two tools to support robot model development:
 git clone https://github.com/Robot-cpp/robot.cpp
 cd robot.cpp
 git submodule update --init --recursive
+./tools/apply_patches.sh
 ```
 
+The launch scripts below configure and build `model-server` automatically. For
+a manual StarVLA build, enable `ROBOT_CPP_BUILD_STARVLA`; see the
+[Robot Server build instructions](robot_server/README.md#manual-build).
+
 This section introduces three usage paths to help you quickly understand the repository:
 
 * Starting `model-server` and connecting it to a minimal dummy `model-client`.
@@ -95,6 +100,14 @@ After downloading, run `model-server` like this:
 
 For general local setups, we provide ready-to-use build-and-launch shells for three platforms. You can modify the environment variables inside the scripts, or override them directly with `export`. See [robot_server/README.md](robot_server/README.md) for details.
 
+For example, from the repository root on Linux with CUDA:
+
+```bash
+export ROBOT_CPP_ROOT="$PWD"
+export GGUF_DIR=/path/to/smolvla-so101-fp32
+bash robot_server/shell/launch_robot_server_linux_cuda.sh
+```
+
 | Backend | macOS                                                   | Linux                                                    | Windows                                                     |
 | ------- | ------------------------------------------------------- | -------------------------------------------------------- | ----------------------------------------------------------- |
 | CUDA    | -                                                       | `robot_server/shell/launch_robot_server_linux_cuda.sh` | `robot_server/shell/launch_robot_server_windows_cuda.bat` |
@@ -127,7 +140,7 @@ We provide a build-to-run example in `robot_client/shell/cpp_client_example.sh`.
 | `ROBOT_CPP_ROOT`   | unset; required                          | Repository root.                                                                               |
 | `BUILD_DIR`        | `${ROBOT_CPP_ROOT}/build_robot_client` | C++ client CMake build directory.                                                              |
 | `PORT`             | `5555`                                 | Server port used by the client.                                                                |
-| `BUILD_CLIENT`     | `0`                                    | Whether to force rebuild the client. Set to`1` to rebuild even if the binary already exists. |
+| `BUILD_CLIENT`     | `0`                                    | Whether to force rebuild the client. Set to `1` to rebuild even if the binary already exists. |
 | `CMAKE_BIN`        | `cmake`                                | CMake command path, useful for selecting a custom CMake binary.                                |
 
 Then run:
@@ -150,20 +163,33 @@ See the [SO-101 deployment guide](eval/lerobot_so101/README.md).
 
 ## ⚡ Performance
 
-We benchmark Robot.cpp on several platforms. Each measurement uses 5 warmup runs and 100 loop runs. The reported latency is the average time from receiving the image, through preprocessing and forward inference, to producing a usable action chunk, measured in milliseconds. All state projectors remain in f32 precision.
+We benchmark Robot.cpp on several platforms. Each measurement uses 5 warmup runs and 100 loop runs. The reported latency is the average time from receiving the image, through preprocessing and forward inference, to producing a usable action chunk, measured in milliseconds. State projectors, where present, remain in f32 precision.
 
 For the LIBERO setting, the input contains two 256x256 images and an 8-dimensional state. For the SO-101 real-robot setting, the input contains one 224x224 image and a 6-dimensional state.
 
 For SmolVLA preprocessing, we follow the official default setting: images are first resized to 512x512.
 
-| Model                  | Mac M4 Pro (CPU) | Mac M4 Pro (Metal) | RTX 4090 |    RTX 3060 | A100 | Jetson AGX Orin |
-| ---------------------- | ---------------: | -----------------: | -------: | ----------: | ---: | --------------: |
-| smolvla@libero (bf16*) |              527 |                216 |       28 |         116 |   43 |             282 |
-| smolvla@libero (f32)   |              577 |                236 |       32 |         142 |   42 |             299 |
-| smolvla@so-101 (bf16*) |              339 |                145 |       23 |          77 |   36 |             184 |
-| smolvla@so-101 (f32)   |              396 |                158 |       24 |          92 |   34 |             200 |
-| pi0@libero (f32)       |             1839 |                710 |       83 | OOM/offload |   71 |             956 |
-| pi0@libero (bf16*)     |             1954 |                635 |       57 |         267 |   66 |             498 |
+For StarVLA, the input contains one 224x224 image and no robot state. Qwen and
+the multimodal projector use bf16; OFT, GR00T, PI, and PI_v3 policies use f32.
+FAST stores its action codec in the policy GGUF.
+The StarVLA A100 results use an A100-PCIE-40GB with 8 CPU threads,
+`n_ctx=2048`, `n_batch=2048`, and noise seed 0.
+
+| Model                         | Mac M4 Pro (CPU) | Mac M4 Pro (Metal) | RTX 4090 |    RTX 3060 | A100 | Jetson AGX Orin |
+| ----------------------------- | ---------------: | -----------------: | -------: | ----------: | ---: | --------------: |
+| smolvla@libero (bf16*)        |              527 |                216 |       28 |         116 |   43 |             282 |
+| smolvla@libero (f32)          |              577 |                236 |       32 |         142 |   42 |             299 |
+| smolvla@so-101 (bf16*)        |              339 |                145 |       23 |          77 |   36 |             184 |
+| smolvla@so-101 (f32)          |              396 |                158 |       24 |          92 |   34 |             200 |
+| pi0@libero (f32)              |             1839 |                710 |       83 | OOM/offload |   71 |             956 |
+| pi0@libero (bf16*)            |             1954 |                635 |       57 |         267 |   66 |             498 |
+| starvla/oft@bridge            |                - |                  - |        - |           - |   50 |               - |
+| starvla/groot@bridge          |                - |                  - |        - |           - |   54 |               - |
+| starvla/pi_v3@bridge          |                - |                  - |        - |           - |  112 |               - |
+| starvla/qwen25_oft@bridge     |                - |                  - |        - |           - |   42 |               - |
+| starvla/qwen25_groot@bridge   |                - |                  - |        - |           - |   51 |               - |
+| starvla/qwen25_pi@bridge      |                - |                  - |        - |           - |  101 |               - |
+| starvla/qwen25_fast@bridge    |                - |                  - |        - |           - |  386 |               - |
 
 > `bf16*`: on Mac, f16 results are used in place of bf16 because current Mac bf16 support is not ideal.
 > `OOM/offload`: pi0@libero (f32) runs out of memory on RTX 3060 and triggers offload, so we do not report a latency number for now.
@@ -230,6 +256,55 @@ This section lists converted GGUF models that can be used directly with `model-s
       <td>f32</td>
       <td><a href="https://huggingface.co/robotcpp/pi0-libero-f32">pi0-libero-f32</a></td>
     </tr>
+    <tr>
+      <td>StarVLA Qwen3-VL OFT</td>
+      <td>Bridge</td>
+      <td><a href="https://huggingface.co/StarVLA/Qwen3VL-OFT-Bridge-RT-1">StarVLA/Qwen3VL-OFT-Bridge-RT-1</a></td>
+      <td>bf16 + f32 policy</td>
+      <td><a href="https://huggingface.co/robotcpp/starvla-qwen3-oft-bridge-bf16">starvla-qwen3-oft-bridge-bf16</a></td>
+    </tr>
+    <tr>
+      <td>StarVLA Qwen3-VL GR00T</td>
+      <td>Bridge</td>
+      <td><a href="https://huggingface.co/StarVLA/Qwen3VL-GR00T-Bridge-RT-1">StarVLA/Qwen3VL-GR00T-Bridge-RT-1</a></td>
+      <td>bf16 + f32 policy</td>
+      <td><a href="https://huggingface.co/robotcpp/starvla-qwen3-groot-bridge-bf16">starvla-qwen3-groot-bridge-bf16</a></td>
+    </tr>
+    <tr>
+      <td>StarVLA Qwen3-VL PI_v3</td>
+      <td>Bridge</td>
+      <td><a href="https://huggingface.co/StarVLA/Qwen3VL-PI_v3-Bridge-RT_1">StarVLA/Qwen3VL-PI_v3-Bridge-RT_1</a></td>
+      <td>bf16 + f32 policy</td>
+      <td><a href="https://huggingface.co/robotcpp/starvla-qwen3-pi-v3-bridge-bf16">starvla-qwen3-pi-v3-bridge-bf16</a></td>
+    </tr>
+    <tr>
+      <td>StarVLA Qwen2.5-VL OFT</td>
+      <td>Bridge</td>
+      <td><a href="https://huggingface.co/StarVLA/Qwen-OFT-Bridge-RT-1">StarVLA/Qwen-OFT-Bridge-RT-1</a></td>
+      <td>bf16 + f32 policy</td>
+      <td><a href="https://huggingface.co/robotcpp/starvla-qwen25-oft-bridge-bf16">starvla-qwen25-oft-bridge-bf16</a></td>
+    </tr>
+    <tr>
+      <td>StarVLA Qwen2.5-VL GR00T</td>
+      <td>Bridge</td>
+      <td><a href="https://huggingface.co/StarVLA/Qwen-GR00T-Bridge-RT-1">StarVLA/Qwen-GR00T-Bridge-RT-1</a></td>
+      <td>bf16 + f32 policy</td>
+      <td><a href="https://huggingface.co/robotcpp/starvla-qwen25-groot-bridge-bf16">starvla-qwen25-groot-bridge-bf16</a></td>
+    </tr>
+    <tr>
+      <td>StarVLA Qwen2.5-VL PI</td>
+      <td>Bridge</td>
+      <td><a href="https://huggingface.co/StarVLA/Qwen-PI-Bridge-RT-1">StarVLA/Qwen-PI-Bridge-RT-1</a></td>
+      <td>bf16 + f32 policy</td>
+      <td><a href="https://huggingface.co/robotcpp/starvla-qwen25-pi-bridge-bf16">starvla-qwen25-pi-bridge-bf16</a></td>
+    </tr>
+    <tr>
+      <td>StarVLA Qwen2.5-VL FAST</td>
+      <td>Bridge</td>
+      <td><a href="https://huggingface.co/StarVLA/Qwen-FAST-Bridge-RT-1">StarVLA/Qwen-FAST-Bridge-RT-1</a></td>
+      <td>bf16 + codec</td>
+      <td><a href="https://huggingface.co/robotcpp/starvla-qwen25-fast-bridge-bf16">starvla-qwen25-fast-bridge-bf16</a></td>
+    </tr>
   </tbody>
 </table>