diff --git a/CMakeLists.txt b/CMakeLists.txt index 150ecde1c..b16a33e07 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -204,6 +204,12 @@ if(SD_WEBM) endif() endif() +if (SD_RPC) + message("-- Use RPC as backend stable-diffusion") + set(GGML_RPC ON) + add_definitions(-DSD_USE_RPC) +endif () + set(SD_LIB stable-diffusion) file(GLOB SD_LIB_SOURCES CONFIGURE_DEPENDS diff --git a/docs/rpc.md b/docs/rpc.md new file mode 100644 index 000000000..617a8b5a1 --- /dev/null +++ b/docs/rpc.md @@ -0,0 +1,220 @@ +# Building and Using the RPC Server with `stable-diffusion.cpp` + +This guide covers how to build a version of [the RPC server from `llama.cpp`](https://github.com/ggml-org/llama.cpp/blob/master/tools/rpc/README.md) that is compatible with your version of `stable-diffusion.cpp` to manage multi-backends setups. RPC allows you to offload specific model components to a remote server. + +> **Note on Model Location:** The model files (e.g., `.safetensors` or `.gguf`) remain on the **Client** machine. The client parses the file and transmits the necessary tensor data and computational graphs to the server. The server does not need to store the model files locally. + +## 1. Building `stable-diffusion.cpp` with RPC client + +First, you should build the client application from source. It requires `SD_RPC=ON` to include the RPC backend to your client. + +```bash +mkdir build +cd build +cmake .. \ + -DSD_RPC=ON \ + # Add other build flags here (e.g., -DSD_VULKAN=ON) +cmake --build . --config Release -j $(nproc) +``` + +> **Note:** Ensure you add the other flags you would normally use (e.g., `-DSD_VULKAN=ON`, `-DSD_CUDA=ON`, `-DSD_HIPBLAS=ON`, or `-DGGML_METAL=ON`), for more information about building `stable-diffusion.cpp` from source, please refer to the [build.md](build.md) documentation. + +## 2. Ensure `llama.cpp` is at the correct commit + +`stable-diffusion.cpp`'s RPC client is designed to work with a specific version of `llama.cpp` (compatible with the `ggml` submodule) to ensure API compatibility. The commit hash for `llama.cpp` is stored in `ggml/scripts/sync-llama.last`. + +> **Start from Root:** Perform these steps from the root of your `stable-diffusion.cpp` directory. + +1. Read the target commit hash from the submodule tracker: + + ```bash + # Linux / WSL / MacOS + HASH=$(cat ggml/scripts/sync-llama.last) + + # Windows (PowerShell) + $HASH = Get-Content -Path "ggml\scripts\sync-llama.last" + ``` + +2. Clone `llama.cpp` at the target commit . + ```bash + git clone https://github.com/ggml-org/llama.cpp.git + cd llama.cpp + git checkout $HASH + ``` + To save on download time and storage, you can use a shallow clone to download only the target commit: + ```bash + mkdir -p llama.cpp + cd llama.cpp + git init + git remote add origin https://github.com/ggml-org/llama.cpp.git + git fetch --depth 1 origin $HASH + git checkout FETCH_HEAD + ``` + +## 3. Build `llama.cpp` (RPC Server) + +The RPC server acts as the worker. You must explicitly enable the **backend** (the hardware interface, such as CUDA for Nvidia, Metal for Apple Silicon, or Vulkan) when building, otherwise the server will default to using only the CPU. + +To find the correct flags for your system, refer to the official documentation for the [`llama.cpp`](https://github.com/ggml-org/llama.cpp/blob/master/docs/build.md) repository. + +> **Crucial:** You must include the compiler flags required to satisfy the API compatibility with `stable-diffusion.cpp` (`-DGGML_MAX_NAME=128`). Without this flag, `GGML_MAX_NAME` will default to `64` for the server, and data transfers between the client and server will fail. Of course, `-DGGML_RPC` must also be enabled. +> +> I recommend disabling the `LLAMA_CURL` flag to avoid unnecessary dependencies, and disabling shared library builds to avoid potential conflicts. + +> **Build Target:** We are specifically building the `rpc-server` target. This prevents the build system from compiling the entire `llama.cpp` suite (like `llama-server`), making the build significantly faster. + +### Linux / WSL (Vulkan) + +```bash +mkdir build +cd build +cmake .. -DGGML_RPC=ON \ + -DGGML_VULKAN=ON \ # Ensure backend is enabled + -DGGML_BUILD_SHARED_LIBS=OFF \ + -DLLAMA_CURL=OFF \ + -DCMAKE_C_FLAGS=-DGGML_MAX_NAME=128 \ + -DCMAKE_CXX_FLAGS=-DGGML_MAX_NAME=128 +cmake --build . --config Release --target rpc-server -j $(nproc) +``` + +### macOS (Metal) + +```bash +mkdir build +cd build +cmake .. -DGGML_RPC=ON \ + -DGGML_METAL=ON \ + -DGGML_BUILD_SHARED_LIBS=OFF \ + -DLLAMA_CURL=OFF \ + -DCMAKE_C_FLAGS=-DGGML_MAX_NAME=128 \ + -DCMAKE_CXX_FLAGS=-DGGML_MAX_NAME=128 +cmake --build . --config Release --target rpc-server +``` + +### Windows (Visual Studio 2022, Vulkan) + +```powershell +mkdir build +cd build +cmake .. -G "Visual Studio 17 2022" -A x64 ` + -DGGML_RPC=ON ` + -DGGML_VULKAN=ON ` + -DGGML_BUILD_SHARED_LIBS=OFF ` + -DLLAMA_CURL=OFF ` + -DCMAKE_C_FLAGS=-DGGML_MAX_NAME=128 ` + -DCMAKE_CXX_FLAGS=-DGGML_MAX_NAME=128 +cmake --build . --config Release --target rpc-server +``` + +## 4. Usage + +Once both applications are built, you can run the server and the client to manage your GPU allocation. + +### Step A: Run the RPC Server + +Start the server. It listens for connections on the default address (usually `localhost:50052`). If your server is on a different machine, ensure the server binds to the correct interface and your firewall allows the connection. + +**On the Server :** +If running on the same machine, you can use the default address: + +```bash +./rpc-server +``` + +If you want to allow connections from other machines on the network: + +```bash +./rpc-server --host 0.0.0.0 +``` + +> **Security Warning:** The RPC server does not currently support authentication or encryption. **Only run the server on trusted local networks**. Never expose the RPC server directly to the open internet. + +> **Drivers & Hardware:** Ensure the Server machine has the necessary drivers installed and functional (e.g., Nvidia Drivers for CUDA, Vulkan SDK, or Metal). If no devices are found, the server will simply fallback to CPU usage. + + + +### Step B: Run with RPC device + +If everything is working correctly, you can now run the client while offloading some or all of the work to the RPC server. + +Example: Setting the main backend to the RPC0 device for doing all the work on the server. + +```bash +./sd-cli -m models/sd1.5.safetensors -p "A cat" --rpc-servers localhost:50052 --backend RPC0 +``` + +--- + +## 5. Scaling: Multiple RPC Servers + +You can connect the client to multiple RPC servers simultaneously to scale out your hardware usage. + +Example: A main machine (192.168.1.10) with 3 GPUs, with one GPU running CUDA and the other two running Vulkan, and a second machine (192.168.1.11) only one GPU. + +**On the first machine (Running two server instances):** + +**Terminal 1 (CUDA):** + +```bash +# Linux / WSL +export CUDA_VISIBLE_DEVICES=0 +cd ./build_cuda/bin/Release +./rpc-server --host 0.0.0.0 + +# Windows PowerShell +$env:CUDA_VISIBLE_DEVICES="0" +cd .\build_cuda\bin\Release +./rpc-server --host 0.0.0.0 +``` + +**Terminal 2 (Vulkan):** + +```bash +cd ./build_vulkan/bin/Release +# ignore the first GPU (used by CUDA server) +./rpc-server --host 0.0.0.0 --port 50053 -d Vulkan1,Vulkan2 +``` + +**On the second machine:** + +```bash +cd ./build/bin/Release +./rpc-server --host 0.0.0.0 +``` + +**On the Client:** +Pass multiple server addresses separated by commas. + +```bash +./sd-cli --rpc-servers 192.168.1.10:50052,192.168.1.10:50053,192.168.1.11:50052 [...] +``` + +The client will map these servers to sequential device IDs (e.g., RPC0 from the first server, RPC2, RPC3 from the second, and RPC4 from the third). With this setup, you could for example use RPC0 for the main backend, RPC1 and RPC2 for the text encoders, and RPC3 for the VAE. + +--- + +## 6. Performance Considerations + +RPC performance is heavily dependent on network bandwidth, as large weights and activations must be transferred back and forth over the network, especially for large models, or when using high resolutions. For best results, ensure your network connection is stable and has sufficient bandwidth (>1Gbps recommended). This shoumd not be a concern if you are running the server and client on the same machine, as the data transfer will happen over the loopback interface. \ No newline at end of file diff --git a/examples/cli/README.md b/examples/cli/README.md index 1b7c2731c..48f00ff48 100644 --- a/examples/cli/README.md +++ b/examples/cli/README.md @@ -43,7 +43,10 @@ Context Options: --high-noise-diffusion-model path to the standalone high noise diffusion model --uncond-diffusion-model path to the standalone unconditional diffusion model, currently used by Ideogram4 CFG + --embeddings-connectors path to LTXAV embeddings connectors --vae path to standalone vae model + --vae-format VAE latent format override: auto, flux, sd3, or flux2 (default: auto) + --audio-vae path to standalone LTX audio vae model --taesd path to taesd. Using Tiny AutoEncoder for fast decoding (low quality) --tae alias of --taesd --control-net path to control net model @@ -53,12 +56,18 @@ Context Options: --tensor-type-rules weight type per tensor pattern (example: "^vae\.=f16,model\.=q8_0") --photo-maker path to PHOTOMAKER model --upscale-model path to esrgan model. + --backend runtime backend assignment, e.g. cpu or clip=cpu,vae=cuda0,diffusion=vulkan0 + --params-backend parameter backend assignment, e.g. cpu or diffusion=cpu,clip=cpu + --rpc-servers comma-separated list of RPC servers to connect to for offloading, in the + format host:port, e.g. localhost:50052,192.168.1.3:50052 -t, --threads number of threads to use during computation (default: -1). If threads <= 0, then threads will be set to the number of CPU physical cores --chroma-t5-mask-pad t5 mask pad size of chroma --max-vram maximum VRAM budget in GiB for graph-cut segmented execution. 0 disables graph splitting; a negative value auto-detects free VRAM, sparing the specified value (e.g. -0.5 will keep at least 0.5 GiB free) + --stream-layers enable residency+prefetch streaming on top of --max-vram (no effect without + --max-vram; defaults to false) --force-sdxl-vae-conv-scale force use of conv scale on sdxl vae --offload-to-cpu place the weights in RAM to save VRAM, and automatically load them into VRAM when needed @@ -109,7 +118,8 @@ Generation Options: --extra-sample-args extra sampler/scheduler/guidance args, key=value list. APG supports apg_eta, apg_momentum, apg_norm_threshold, apg_norm_threshold_smoothing; SLG supports slg_uncond; lcm supports noise_clip_std, noise_scale_start, noise_scale_end; - ltx2 supports max_shift, base_shift, stretch, terminal; euler_ge supports gamma + ltx2 supports max_shift, base_shift, stretch, terminal; euler_ge supports + gamma --extra-tiling-args extra VAE tiling args, key=value list. LTX video VAE supports temporal_tile_frames (default: 4), temporal_tile_overlap (default: 1) -H, --height image height, in pixel space (default: 512) @@ -153,7 +163,7 @@ Generation Options: --high-noise-eta (high noise) noise multiplier (default: 0 for ddim_trailing, tcd, res_multistep and res_2s; 1 for euler_a, er_sde and dpm++2s_a) --strength strength for noising/unnoising (default: 0.75) - --pm-style-strength + --pm-style-strength --control-strength strength to apply Control Net (default: 0.9). 1.0 corresponds to full destruction of information in init image --moe-boundary timestep boundary for Wan2.2 MoE model. (default: 0.875). Only enabled if @@ -172,13 +182,15 @@ Generation Options: -s, --seed RNG seed (default: 42, use random seed for < 0) --sampling-method sampling method, one of [euler, euler_a, heun, dpm2, dpm++2s_a, dpm++2m, dpm++2mv2, ipndm, ipndm_v, lcm, ddim_trailing, tcd, res_multistep, res_2s, - er_sde, euler_cfg_pp, euler_a_cfg_pp] (default: euler for Flux/SD3/Wan, euler_a otherwise) + er_sde, euler_cfg_pp, euler_a_cfg_pp](default: euler for Flux/SD3/Wan, + euler_a otherwise) --high-noise-sampling-method (high noise) sampling method, one of [euler, euler_a, heun, dpm2, dpm++2s_a, dpm++2m, dpm++2mv2, ipndm, ipndm_v, lcm, ddim_trailing, tcd, res_multistep, - res_2s, er_sde, euler_cfg_pp, euler_a_cfg_pp] default: euler for Flux/SD3/Wan, euler_a otherwise + res_2s, er_sde, euler_cfg_pp, euler_a_cfg_pp] default: euler for + Flux/SD3/Wan, euler_a otherwise --scheduler denoiser sigma scheduler, one of [discrete, karras, exponential, ays, gits, - smoothstep, sgm_uniform, simple, kl_optimal, lcm, bong_tangent, ltx2], default: - model-specific + smoothstep, sgm_uniform, simple, kl_optimal, lcm, bong_tangent, ltx2], + default: model-specific --sigmas custom sigma values for the sampler, comma-separated (e.g., "14.61,7.8,3.5,0.0"). --hires-sigmas custom sigma values for the highres fix second pass, comma-separated (e.g., diff --git a/examples/common/common.cpp b/examples/common/common.cpp index 3ae5faba7..d80e68572 100644 --- a/examples/common/common.cpp +++ b/examples/common/common.cpp @@ -423,6 +423,10 @@ ArgOptions SDContextParams::get_options() { "--params-backend", "parameter backend assignment, e.g. cpu or diffusion=cpu,clip=cpu", ¶ms_backend}, + {"", + "--rpc-servers", + "comma-separated list of RPC servers to connect to for offloading, in the format host:port, e.g. localhost:50052,192.168.1.3:50052", + &rpc_servers}, }; options.int_options = { @@ -817,6 +821,7 @@ sd_ctx_params_t SDContextParams::to_sd_ctx_params_t(bool vae_decode_only, bool f stream_layers, backend.c_str(), params_backend.c_str(), + rpc_servers.c_str(), }; return sd_ctx_params; } diff --git a/examples/common/common.h b/examples/common/common.h index a90a33132..cc3fd2be2 100644 --- a/examples/common/common.h +++ b/examples/common/common.h @@ -148,6 +148,7 @@ struct SDContextParams { bool stream_layers = false; std::string backend; std::string params_backend; + std::string rpc_servers; bool enable_mmap = false; bool control_net_cpu = false; bool clip_on_cpu = false; diff --git a/examples/server/README.md b/examples/server/README.md index 16fb393c6..3f542a246 100644 --- a/examples/server/README.md +++ b/examples/server/README.md @@ -145,7 +145,10 @@ Context Options: --high-noise-diffusion-model path to the standalone high noise diffusion model --uncond-diffusion-model path to the standalone unconditional diffusion model, currently used by Ideogram4 CFG + --embeddings-connectors path to LTXAV embeddings connectors --vae path to standalone vae model + --vae-format VAE latent format override: auto, flux, sd3, or flux2 (default: auto) + --audio-vae path to standalone LTX audio vae model --taesd path to taesd. Using Tiny AutoEncoder for fast decoding (low quality) --tae alias of --taesd --control-net path to control net model @@ -155,12 +158,18 @@ Context Options: --tensor-type-rules weight type per tensor pattern (example: "^vae\.=f16,model\.=q8_0") --photo-maker path to PHOTOMAKER model --upscale-model path to esrgan model. + --backend runtime backend assignment, e.g. cpu or clip=cpu,vae=cuda0,diffusion=vulkan0 + --params-backend parameter backend assignment, e.g. cpu or diffusion=cpu,clip=cpu + --rpc-servers comma-separated list of RPC servers to connect to for offloading, in the + format host:port, e.g. localhost:50052,192.168.1.3:50052 -t, --threads number of threads to use during computation (default: -1). If threads <= 0, then threads will be set to the number of CPU physical cores --chroma-t5-mask-pad t5 mask pad size of chroma --max-vram maximum VRAM budget in GiB for graph-cut segmented execution. 0 disables graph splitting; a negative value auto-detects free VRAM, sparing the specified value (e.g. -0.5 will keep at least 0.5 GiB free) + --stream-layers enable residency+prefetch streaming on top of --max-vram (no effect without + --max-vram; defaults to false) --force-sdxl-vae-conv-scale force use of conv scale on sdxl vae --offload-to-cpu place the weights in RAM to save VRAM, and automatically load them into VRAM when needed @@ -211,7 +220,8 @@ Default Generation Options: --extra-sample-args extra sampler/scheduler/guidance args, key=value list. APG supports apg_eta, apg_momentum, apg_norm_threshold, apg_norm_threshold_smoothing; SLG supports slg_uncond; lcm supports noise_clip_std, noise_scale_start, noise_scale_end; - ltx2 supports max_shift, base_shift, stretch, terminal; euler_ge supports gamma + ltx2 supports max_shift, base_shift, stretch, terminal; euler_ge supports + gamma --extra-tiling-args extra VAE tiling args, key=value list. LTX video VAE supports temporal_tile_frames (default: 4), temporal_tile_overlap (default: 1) -H, --height image height, in pixel space (default: 512) @@ -255,7 +265,7 @@ Default Generation Options: --high-noise-eta (high noise) noise multiplier (default: 0 for ddim_trailing, tcd, res_multistep and res_2s; 1 for euler_a, er_sde and dpm++2s_a) --strength strength for noising/unnoising (default: 0.75) - --pm-style-strength + --pm-style-strength --control-strength strength to apply Control Net (default: 0.9). 1.0 corresponds to full destruction of information in init image --moe-boundary timestep boundary for Wan2.2 MoE model. (default: 0.875). Only enabled if @@ -274,13 +284,15 @@ Default Generation Options: -s, --seed RNG seed (default: 42, use random seed for < 0) --sampling-method sampling method, one of [euler, euler_a, heun, dpm2, dpm++2s_a, dpm++2m, dpm++2mv2, ipndm, ipndm_v, lcm, ddim_trailing, tcd, res_multistep, res_2s, - er_sde, euler_cfg_pp, euler_a_cfg_pp] (default: euler for Flux/SD3/Wan, euler_a otherwise) + er_sde, euler_cfg_pp, euler_a_cfg_pp](default: euler for Flux/SD3/Wan, + euler_a otherwise) --high-noise-sampling-method (high noise) sampling method, one of [euler, euler_a, heun, dpm2, dpm++2s_a, dpm++2m, dpm++2mv2, ipndm, ipndm_v, lcm, ddim_trailing, tcd, res_multistep, - res_2s, er_sde, euler_cfg_pp, euler_a_cfg_pp] default: euler for Flux/SD3/Wan, euler_a otherwise + res_2s, er_sde, euler_cfg_pp, euler_a_cfg_pp] default: euler for + Flux/SD3/Wan, euler_a otherwise --scheduler denoiser sigma scheduler, one of [discrete, karras, exponential, ays, gits, - smoothstep, sgm_uniform, simple, kl_optimal, lcm, bong_tangent, ltx2], default: - model-specific + smoothstep, sgm_uniform, simple, kl_optimal, lcm, bong_tangent, ltx2], + default: model-specific --sigmas custom sigma values for the sampler, comma-separated (e.g., "14.61,7.8,3.5,0.0"). --hires-sigmas custom sigma values for the highres fix second pass, comma-separated (e.g., diff --git a/include/stable-diffusion.h b/include/stable-diffusion.h index 17596f849..21b38d87f 100644 --- a/include/stable-diffusion.h +++ b/include/stable-diffusion.h @@ -226,6 +226,7 @@ typedef struct { bool stream_layers; // Enable residency+prefetch streaming on top of --max-vram (no effect without --max-vram) const char* backend; const char* params_backend; + const char* rpc_servers; } sd_ctx_params_t; typedef struct { diff --git a/src/core/ggml_extend_backend.cpp b/src/core/ggml_extend_backend.cpp index d085129db..2820af727 100644 --- a/src/core/ggml_extend_backend.cpp +++ b/src/core/ggml_extend_backend.cpp @@ -200,6 +200,36 @@ void ggml_ext_im_set_f32_1d(const struct ggml_tensor* tensor, int i, float value } } +bool add_rpc_devices(const std::string& servers) { + const std::string in = trim_copy(servers); + if (in.empty()) { + return true; + } + auto rpc_servers = split_copy(in, ','); + if (rpc_servers.empty()) { + LOG_ERROR("invalid RPC servers specification: '%s'", servers.c_str()); + return false; + } + ggml_backend_reg_t rpc_reg = ggml_backend_reg_by_name("RPC"); + if (!rpc_reg) { + LOG_ERROR("RPC backend not found, cannot add RPC servers"); + return false; + } + typedef ggml_backend_reg_t (*ggml_backend_rpc_add_server_t)(const char* endpoint); + ggml_backend_rpc_add_server_t ggml_backend_rpc_add_server_fn = (ggml_backend_rpc_add_server_t)ggml_backend_reg_get_proc_address(rpc_reg, "ggml_backend_rpc_add_server"); + if (!ggml_backend_rpc_add_server_fn) { + LOG_ERROR("RPC backend does not have ggml_backend_rpc_add_server function, cannot add RPC servers"); + return false; + } + for (const auto& server : rpc_servers) { + LOG_INFO("Adding RPC server: %s", server.c_str()); + auto reg = ggml_backend_rpc_add_server_fn(server.c_str()); + // no return value to check for success but should print errors from the RPC backend if it fails to add the server + ggml_backend_register(reg); + } + return true; +} + static void ggml_backend_load_all_once() { // If the registry already has devices and the CPU backend is present, // assume either static registration or explicit host-side preloading has diff --git a/src/core/ggml_extend_backend.h b/src/core/ggml_extend_backend.h index fc071ffda..5a8591eb6 100644 --- a/src/core/ggml_extend_backend.h +++ b/src/core/ggml_extend_backend.h @@ -76,4 +76,5 @@ ggml_backend_t sd_backend_cpu_init(); bool sd_backend_cpu_set_n_threads(ggml_backend_t backend_cpu, int n_threads); const char* sd_backend_module_name(SDBackendModule module); void ggml_ext_im_set_f32_1d(const struct ggml_tensor* tensor, int i, float value); +bool add_rpc_devices(const std::string& servers); #endif // __SD_CORE_GGML_EXTEND_BACKEND_H__ diff --git a/src/model_loader.cpp b/src/model_loader.cpp index 9c2d5cef1..a3eee1f8f 100644 --- a/src/model_loader.cpp +++ b/src/model_loader.cpp @@ -962,6 +962,7 @@ bool ModelLoader::load_tensors(on_new_tensor_cb_t on_new_tensor_cb, int n_thread std::atomic tensor_idx(0); std::atomic failed(false); std::vector workers; + std::mutex rpc_backend_mutex; for (int i = 0; i < n_threads; ++i) { workers.emplace_back([&, file_path, is_zip]() { @@ -1118,7 +1119,19 @@ bool ModelLoader::load_tensors(on_new_tensor_cb_t on_new_tensor_cb, int n_thread if (dst_tensor->buffer != nullptr && !ggml_backend_buffer_is_host(dst_tensor->buffer)) { t0 = ggml_time_ms(); - ggml_backend_tensor_set(dst_tensor, convert_buf, 0, ggml_nbytes(dst_tensor)); + + // RPC backends require serialized access to prevent concurrency issues + const char* buffer_type_name = ggml_backend_buft_name(ggml_backend_buffer_get_type(dst_tensor->buffer)); + bool is_rpc_buffer = buffer_type_name != nullptr && + std::string(buffer_type_name).find("RPC") != std::string::npos; + + if (is_rpc_buffer) { + std::lock_guard lock(rpc_backend_mutex); + ggml_backend_tensor_set(dst_tensor, convert_buf, 0, ggml_nbytes(dst_tensor)); + } else { + ggml_backend_tensor_set(dst_tensor, convert_buf, 0, ggml_nbytes(dst_tensor)); + } + t1 = ggml_time_ms(); copy_to_backend_time_ms.fetch_add(t1 - t0); } diff --git a/src/stable-diffusion.cpp b/src/stable-diffusion.cpp index 8ba4a463a..5f9dc3709 100644 --- a/src/stable-diffusion.cpp +++ b/src/stable-diffusion.cpp @@ -266,6 +266,10 @@ class StableDiffusionGGML { stream_layers = sd_ctx_params->stream_layers; backend_spec = SAFE_STR(sd_ctx_params->backend); params_backend_spec = SAFE_STR(sd_ctx_params->params_backend); + + std::string rpc_servers_spec = SAFE_STR(sd_ctx_params->rpc_servers); + add_rpc_devices(rpc_servers_spec); + if (stream_layers && max_vram == 0.f) { LOG_WARN("--stream-layers has no effect without --max-vram set; ignoring"); stream_layers = false;