[rqd/cicd] Replace python/rqd by rust/rqd on docker stack (AcademySoftwareFoundation#2227)

As previously communicated, python/rqd is now deprecated and it's being
officially replaced by rust/rqd on docker-composed and on the release
pipeline.

---------

Signed-off-by: Diego Tavares <dtavares@imageworks.com>

Author: DiegoTavares

.github/workflows/packaging-pipeline.yml

@@ -189,10 +189,12 @@ jobs:
         include:
           - component: cuebot
             NAME: Cuebot
+            DOCKERFILE: cuebot/Dockerfile
             ARTIFACTS: cuebot-${BUILD_ID}-all.jar opencue-cuebot-${BUILD_ID}-1.noarch.rpm
 
           - component: rqd
             NAME: RQD
+            DOCKERFILE: rust/Dockerfile.rqd
 
     name: Build ${{ matrix.NAME }}
     runs-on: ubuntu-22.04
@@ -243,7 +245,7 @@ jobs:
       - name: Build Docker image
         run: |
           image_name="opencuebuild/${{ matrix.component }}:${BUILD_ID}"
-          docker buildx build . -f ${{ matrix.component }}/Dockerfile -t ${image_name} \
+          docker buildx build . -f ${{ matrix.DOCKERFILE }} -t ${image_name} \
             --build-arg OPENCUE_PROTO_PACKAGE_PATH="${{ needs.build_opencue_packages.outputs.opencue_proto_path }}" \
             --build-arg OPENCUE_RQD_PACKAGE_PATH="${{ needs.build_opencue_packages.outputs.opencue_rqd_path }}" \
             --build-arg OPENCUE_PYCUE_PACKAGE_PATH="${{ needs.build_opencue_packages.outputs.opencue_pycue_path }}" \

ci/run_integration_test.sh

@@ -245,22 +245,12 @@ main() {
 
     log INFO "Building Cuebot image..."
     docker build -t opencue/cuebot -f cuebot/Dockerfile . &>"${TEST_LOGS}/docker-build-cuebot.log"
-    if [[ ! -e "${OPENCUE_PROTO_PACKAGE_PATH}" ]]; then
-      rm -rf proto/dist/*.*
-      python -m build proto
-      OPENCUE_PROTO_PACKAGE_PATH=$(ls -1 proto/dist/*.tar.gz)
-      export OPENCUE_PROTO_PACKAGE_PATH
-    fi
-    if [[ ! -e "${OPENCUE_RQD_PACKAGE_PATH}" ]]; then
-      rm -rf rqd/dist/*.*
-      python -m build rqd
-      OPENCUE_RQD_PACKAGE_PATH=$(ls -1 rqd/dist/*.tar.gz)
-      export OPENCUE_RQD_PACKAGE_PATH
-    fi
     log INFO "Building RQD image..."
-    docker build --build-arg OPENCUE_PROTO_PACKAGE_PATH="${OPENCUE_PROTO_PACKAGE_PATH}" \
-           --build-arg OPENCUE_RQD_PACKAGE_PATH="${OPENCUE_RQD_PACKAGE_PATH}" \
-           -t opencue/rqd -f rqd/Dockerfile . &>"${TEST_LOGS}/docker-build-rqd.log"
+    if ! docker build -t opencue/rqd -f rust/Dockerfile.rqd . &>"${TEST_LOGS}/docker-build-rqd.log"; then
+        log ERROR "RQD Docker build failed. Build log:"
+        cat "${TEST_LOGS}/docker-build-rqd.log"
+        exit 1
+    fi
 
     # Pre-create bind-mount directories so Docker doesn't create them as root
     mkdir -p "${HOME}/.opencue/sessions"

docker-compose.yml

@@ -1,6 +1,6 @@
 # OpenCue Docker Compose Configuration
 #
-# By default, only the core services are enabled (db, flyway, cuebot, rqd).
+# By default, only the core services are enabled (db, flyway, cuebot, rqd (Rust)).
 # Uncomment the optional sections to enable additional functionality:
 #
 # Profiles:
@@ -190,10 +190,10 @@ services:
   rqd:
     build:
       context: ./
-      dockerfile: ./rqd/Dockerfile
+      dockerfile: ./rust/Dockerfile.rqd
     environment:
-      - PYTHONUNBUFFERED=1
-      - CUEBOT_HOSTNAME=cuebot
+      - OPENRQD__GRPC__CUEBOT_ENDPOINTS=cuebot:8443
+      - OPENRQD__MACHINE__USE_IP_AS_HOSTNAME=true
     depends_on:
       cuebot:
         condition: service_healthy

docs/_docs/getting-started/deploying-rqd.md

@@ -86,30 +86,26 @@ Make sure you also complete the following steps:
 1.  On macOS you might also need to
     [increase Docker's RAM limit](https://docs.docker.com/docker-for-mac/#advanced).
 
-### Option 1: Downloading and running RQD from DockerHub
+### Option 1: Building and running Rust RQD from source (Recommended)
 
-To download and run the pre-built Docker image from DockerHub:
+To build and run the Rust RQD Docker image from source:
 
 ```shell
-docker pull opencue/rqd
-docker run -td --name rqd01 --env CUEBOT_HOSTNAME=${CUEBOT_HOSTNAME} --volume "${CUE_FS_ROOT}:${CUE_FS_ROOT}" --add-host host.docker.internal:host-gateway opencue/rqd
+docker build -t opencue/rqd -f rust/Dockerfile.rqd rust/
+docker run -td --name rqd01 --env OPENRQD__GRPC__CUEBOT_ENDPOINTS=${CUEBOT_HOSTNAME}:8443 --volume "${CUE_FS_ROOT}:${CUE_FS_ROOT}" --add-host host.docker.internal:host-gateway opencue/rqd
 ```
 
-### Option 2: Building and running RQD from source
+### Option 2 (Legacy): Downloading and running Python RQD from DockerHub
 
-To build and run the RQD Docker image from source:
+> **Note:** The Python RQD is deprecated. Consider using the Rust RQD (Option 1) for improved performance and resource efficiency.
+
+To download and run the pre-built Python RQD Docker image from DockerHub:
 
 ```shell
-docker build -t opencue/rqd -f rqd/Dockerfile .
+docker pull opencue/rqd
 docker run -td --name rqd01 --env CUEBOT_HOSTNAME=${CUEBOT_HOSTNAME} --volume "${CUE_FS_ROOT}:${CUE_FS_ROOT}" --add-host host.docker.internal:host-gateway opencue/rqd
 ```
 
-<!-- -   **In both Option 1 and 2**, if running the RQD container in your local machine, use the `--add-host` flag on the `docker run` command as follows:
-
-    ```shell
-    docker run -td --name rqd01 --env CUEBOT_HOSTNAME=${CUEBOT_HOSTNAME} --volume "${CUE_FS_ROOT}:${CUE_FS_ROOT}" --add-host host.docker.internal:host-gateway opencue/rqd
-    ``` -->
-
 ### Option 3: Installing from pypi
 
 To install from the published pypi release:

rust/Dockerfile.rqd

@@ -0,0 +1,58 @@
+# Multi-stage Dockerfile for OpenCue Rust RQD
+
+# Build stage
+FROM rust:1.88-bookworm AS builder
+
+# Install build dependencies
+RUN apt-get update && apt-get install -y \
+    protobuf-compiler \
+    libprotobuf-dev \
+    pkg-config \
+    && rm -rf /var/lib/apt/lists/*
+
+# Create app directory
+WORKDIR /build
+
+# Copy workspace files
+COPY rust/Cargo.toml ./
+COPY rust/Cargo.lock* ./
+
+# Copy all crate directories
+COPY rust/crates ./crates
+
+# Remove symlink and copy proto files from repo root
+RUN rm ./crates/opencue-proto/src/protos
+COPY proto/src ./crates/opencue-proto/src/protos
+
+# Build RQD in release mode
+RUN cargo build --release -p rqd
+
+# Runtime stage
+FROM debian:bookworm-slim
+
+# Install runtime dependencies
+RUN apt-get update && apt-get install -y \
+    ca-certificates \
+    libx11-6 \
+    && rm -rf /var/lib/apt/lists/*
+
+# Create app user
+RUN useradd -m -u 1000 -s /bin/bash rqd
+
+# Create necessary directories
+RUN mkdir -p /app/resources && chown -R rqd:rqd /app
+
+WORKDIR /app
+
+# Copy the built binary from builder stage
+COPY --from=builder /build/target/release/openrqd /app/openrqd
+
+# Copy resources
+COPY --from=builder /build/crates/rqd/resources /app/resources
+
+# RQD gRPC server
+EXPOSE 8444
+
+# Run RQD
+ENTRYPOINT ["/app/openrqd"]
+CMD []

rust/crates/rqd/src/config/mod.rs

@@ -16,7 +16,7 @@ use crate::config::error::RqdConfigError;
 use bytesize::ByteSize;
 use config::{Config as ConfigBase, Environment, File};
 use lazy_static::lazy_static;
-use serde::{Deserialize, Serialize};
+use serde::{Deserialize, Deserializer, Serialize};
 use std::{collections::HashMap, env, fs, path::Path, time::Duration};
 
 static DEFAULT_CONFIG_FILE: &str = "~/.local/share/rqd.yaml";
@@ -47,11 +47,30 @@ impl Default for LoggingConfig {
     }
 }
 
+/// Deserializes a value that can be either a single comma-separated string or a sequence of strings.
+fn string_or_vec<'de, D>(deserializer: D) -> Result<Vec<String>, D::Error>
+where
+    D: Deserializer<'de>,
+{
+    #[derive(Deserialize)]
+    #[serde(untagged)]
+    enum StringOrVec {
+        String(String),
+        Vec(Vec<String>),
+    }
+
+    match StringOrVec::deserialize(deserializer)? {
+        StringOrVec::String(s) => Ok(s.split(',').map(|item| item.trim().to_string()).collect()),
+        StringOrVec::Vec(v) => Ok(v),
+    }
+}
+
 #[derive(Debug, Deserialize, Clone)]
 #[serde(default)]
 pub struct GrpcConfig {
     pub rqd_port: u16,
     pub rqd_interface: Option<String>,
+    #[serde(deserialize_with = "string_or_vec")]
     pub cuebot_endpoints: Vec<String>,
     #[serde(with = "humantime_serde")]
     pub connection_expires_after: Duration,
@@ -68,7 +87,7 @@ impl Default for GrpcConfig {
         GrpcConfig {
             rqd_port: 8444,
             rqd_interface: None,
-            cuebot_endpoints: vec!["localhost:4343".to_string()],
+            cuebot_endpoints: vec!["localhost:8443".to_string()],
             connection_expires_after: Duration::from_secs(3600), // 1h. from_hour is experimental
             backoff_delay_min: Duration::from_millis(10),
             backoff_delay_max: Duration::from_secs(60),
@@ -250,7 +269,11 @@ impl Config {
 
         let config = ConfigBase::builder()
             .add_source(File::with_name(&config_file).required(required))
-            .add_source(Environment::with_prefix("OPENRQD").separator("_"))
+            .add_source(
+                Environment::with_prefix("OPENRQD")
+                    .separator("__")
+                    .list_separator(","),
+            )
             .build()
             .map_err(|err| {
                 RqdConfigError::LoadConfigError(format!(

rust/crates/rqd/src/system/linux.rs

@@ -339,7 +339,7 @@ impl LinuxSystem {
             let addr = addrs_iter
                 .next()
                 .ok_or_else(|| miette::miette!("Failed to find IP for {}", hostname))?;
-            Ok(addr.to_string())
+            Ok(addr.ip().to_string())
         } else {
             Ok(hostname)
         }
@@ -594,7 +594,6 @@ impl LinuxSystem {
         Ok(())
     }
 
-
     // Read stats from /proc/{pid}/stat file and return session_id
     // NSsid might be is missing from /proc/{pid}/status
     // In this case we fallback if not found and and then use stat instead
@@ -615,12 +614,12 @@ impl LinuxSystem {
 
         match end {
             Some(end) => {
-                let fields: Vec<&str> = stat[end+2..].split_whitespace().collect();
-                return fields[3].parse::<u32>().ok()
-            },
+                let fields: Vec<&str> = stat[end + 2..].split_whitespace().collect();
+                return fields[3].parse::<u32>().ok();
+            }
             None => {
                 let fields: Vec<&str> = stat.split_whitespace().collect();
-                return fields[5].parse::<u32>().ok()
+                return fields[5].parse::<u32>().ok();
             }
         }
     }