[router] Implement gRPC SGLangSchedulerClient (#9364)

Author: CatherineSue

.github/workflows/release-pypi-router.yml

@@ -47,7 +47,7 @@ jobs:
         env:
           CIBW_BUILD: "cp38-manylinux_x86_64 cp39-manylinux_x86_64 cp310-manylinux_x86_64 cp311-manylinux_x86_64 cp312-manylinux_x86_64"
           CIBW_BEFORE_ALL: |
-            yum update && yum install -y openssl-devel && curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y
+            yum update && yum install -y openssl-devel protobuf-compiler && curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y
           CIBW_ENVIRONMENT: "PATH=$HOME/.cargo/bin:$PATH"
 
       - name: List built packages

docker/Dockerfile.router

@@ -39,13 +39,13 @@ ENV PATH="/root/.cargo/bin:${PATH}"
 
 # install dependencies
 RUN apt update -y \
-    && apt install -y git build-essential libssl-dev pkg-config \
+    && apt install -y git build-essential libssl-dev pkg-config protobuf-compiler \
     && rm -rf /var/lib/apt/lists/* \
     && apt clean
 
 # install rustup from rustup.rs
 RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y \
-    && rustc --version && cargo --version
+    && rustc --version && cargo --version && protoc --version
 
 # pull the github repository
 RUN cd /opt \

scripts/ci/ci_install_rust.sh

@@ -4,10 +4,10 @@ set -euxo pipefail
 # Check if sudo is available
 if command -v sudo >/dev/null 2>&1; then
     sudo apt-get update
-    sudo apt-get install -y libssl-dev pkg-config
+    sudo apt-get install -y libssl-dev pkg-config protobuf-compiler
 else
     apt-get update
-    apt-get install -y libssl-dev pkg-config
+    apt-get install -y libssl-dev pkg-config protobuf-compiler
 fi
 
 # Install rustup (Rust installer and version manager)
@@ -21,3 +21,4 @@ source $HOME/.cargo/env
 # Verify installation
 rustc --version
 cargo --version
+protoc --version

sgl-router/Cargo.toml

@@ -4,9 +4,11 @@ version = "0.0.0"
 edition = "2021"
 
 [features]
-default = ["huggingface"]
+default = ["huggingface", "grpc-client"]
 huggingface = ["tokenizers"]
 tiktoken = ["tiktoken-rs"]
+grpc-client = []
+grpc-server = []
 
 [lib]
 name = "sglang_router_rs"
@@ -52,6 +54,18 @@ anyhow = "1.0"
 tokenizers = { version = "0.21.4", optional = true }
 tiktoken-rs = { version = "0.5", optional = true }
 
+# gRPC and Protobuf dependencies
+tonic = { version = "0.12", features = ["tls", "gzip", "transport"] }
+prost = "0.13"
+prost-types = "0.13"
+deadpool = { version = "0.12", features = ["managed", "rt_tokio_1"] }
+backoff = { version = "0.4", features = ["tokio"] }
+strum = { version = "0.26", features = ["derive"] }
+
+[build-dependencies]
+tonic-build = "0.12"
+prost-build = "0.13"
+
 [dev-dependencies]
 criterion = { version = "0.5", features = ["html_reports"] }
 tower = { version = "0.5", features = ["util"] }

sgl-router/MANIFEST.in

@@ -1,3 +1,5 @@
 # Must include:
 include Cargo.toml        # Rust project configuration
+include build.rs          # Build script for protobuf generation
 recursive-include src *.rs  # Rust source files
+recursive-include src/proto *.proto  # Protobuf definitions

sgl-router/build.rs

@@ -0,0 +1,35 @@
+fn main() -> Result<(), Box<dyn std::error::Error>> {
+    // Only regenerate if the proto file changes
+    println!("cargo:rerun-if-changed=src/proto/sglang_scheduler.proto");
+
+    // Configure protobuf compilation with custom settings
+    let config = prost_build::Config::new();
+
+    // Skip serde for types that use prost_types::Struct
+    // These cause conflicts and we don't need serde for all generated types
+
+    // Configure tonic-build for gRPC code generation
+    tonic_build::configure()
+        // Generate both client and server code
+        .build_server(true)
+        .build_client(true)
+        // Add a module-level attribute for documentation and clippy warnings
+        .server_mod_attribute(
+            "sglang.grpc.scheduler",
+            "#[allow(unused, clippy::mixed_attributes_style)]",
+        )
+        .client_mod_attribute(
+            "sglang.grpc.scheduler",
+            "#[allow(unused, clippy::mixed_attributes_style)]",
+        )
+        // Compile the proto file with the custom config
+        .compile_protos_with_config(
+            config,
+            &["src/proto/sglang_scheduler.proto"],
+            &["src/proto"],
+        )?;
+
+    println!("cargo:warning=Protobuf compilation completed successfully");
+
+    Ok(())
+}