Merge remote-tracking branch 'upstream/master' into seiji/prometheus-autoscaling

Signed-off-by: Seiji Eicher <seiji@anyscale.com>

Author: eicherseiji

.buildkite/core.rayci.yml

@@ -524,7 +524,7 @@ steps:
         --python-version 3.10 --build-name corebuild-py3.10
         --install-mask all-ray-libraries
         --run-flaky-tests
-        --except-tags multi_gpu,cgroup
+        --except-tags multi_gpu,cgroup,runtime_env_container
     depends_on:
       - corebuild-multipy(python=3.10)
 

.buildkite/serve.rayci.yml

@@ -213,6 +213,7 @@ steps:
     parallelism: 3
     tags:
       - serve
+      - skip-on-microcheck
     instance_type: large
     commands:
       # Full serve suite with HAProxy, minus tags owned by their own steps or images.

.github/CODEOWNERS

@@ -21,7 +21,7 @@
 /python/ray/ @ray-project/ray-core
 
 # Java worker.
-/java/ @kfstorm @raulchen @WangTaoTheTonic @SongGuyang @ray-project/ray-core
+/java/ @ray-project/ray-core
 
 # C++ worker
 /cpp/ @SongGuyang @raulchen @kfstorm @ray-project/ray-core
@@ -32,7 +32,7 @@
 /doc/source/cluster/kubernetes/ @andrewsykim @ray-project/ray-core @ray-project/ray-docs
 
 # Public protobuf files.
-/src/ray/protobuf/public/ @edoakes @dayshah @MengjinYan
+/src/ray/protobuf/public/ @edoakes @MengjinYan
 
 # Azure autoscaler
 /python/ray/autoscaler/azure/ @ray-project/ray-core @marosset @jackfrancis @alimaazamat
@@ -45,7 +45,7 @@
 
 # Common directory shared by core and the libraries.
 # Stricter enforcement because this defines the public boundary supported by core.
-/python/ray/_common/ @edoakes @dayshah @MengjinYan
+/python/ray/_common/ @edoakes @MengjinYan
 
 # Ray data.
 /python/ray/data/ @ray-project/ray-data
@@ -78,8 +78,8 @@
 # LLM
 /python/ray/llm/ @ray-project/ray-llm
 /python/ray/data/llm.py @ray-project/ray-llm
-/python/ray/dashboard/modules/metrics/dashboards/serve_llm_dashboard_panels.py @ray-project/ray-llm
-/python/ray/dashboard/modules/metrics/dashboards/serve_llm_grafana_dashboard_base.json @ray-project/ray-llm
+/python/ray/dashboard/modules/metrics/dashboards/serve_llm* @ray-project/ray-llm
+/python/ray/dashboard/modules/metrics/dashboards/data_llm* @ray-project/ray-llm
 /python/ray/serve/llm/ @ray-project/ray-llm
 /doc/source/serve/llm/ @ray-project/ray-llm @ray-project/ray-docs
 /doc/source/data/working-with-llms.rst @ray-project/ray-llm @ray-project/ray-docs

bazel/ray_deps_setup.bzl

@@ -186,6 +186,14 @@ def ray_deps_setup():
         build_file = "@io_ray//bazel:rocksdb.BUILD",
         url = "https://github.com/facebook/rocksdb/archive/refs/tags/v10.6.2.tar.gz",
         sha256 = "14c619b8a10f994aa6061bd12182b20270c4c27c2f3d9cb4376d57f3cd1c5d7f",
+        patches = [
+            # WITH_TSAN=ON injects `-Wl,-pie`, which breaks every CMake
+            # try_compile under the rules_foreign_cc crosstool (clang +
+            # static libc++), aborting configure at find_package(Threads).
+            # `-fsanitize=thread` alone is sufficient for TSan.
+            "@io_ray//thirdparty/patches:rocksdb-tsan-no-pie.patch",
+        ],
+        patch_args = ["-p1"],
     )
 
     auto_http_archive(
@@ -219,10 +227,10 @@ def ray_deps_setup():
         sha256 = "06327c2ddc81e126a6d9a78b0be5014b976a2c0832f492dcfc4755d7facf6d33",
         strip_prefix = "xz-5.2.7",
         urls = [
-            "https://cfhcable.dl.sourceforge.net/project/lzmautils/xz-5.2.7.tar.gz",
-            "https://superb-sea2.dl.sourceforge.net/project/lzmautils/xz-5.2.7.tar.gz",
-            "https://ayera.dl.sourceforge.net/project/lzmautils/xz-5.2.7.tar.gz",
-            "https://astuteinternet.dl.sourceforge.net/project/lzmautils/xz-5.2.7.tar.gz",
+            # Use the SourceForge redirector, which picks a live mirror,
+            # instead of pinning specific mirrors that can go offline.
+            "https://downloads.sourceforge.net/project/lzmautils/xz-5.2.7.tar.gz",
+            "https://tukaani.org/xz/xz-5.2.7.tar.gz",
         ],
         build_file = "//thirdparty/patches:org_lzma_lzma.BUILD.bazel",
     )

ci/lint/pyrefly-excluded-files.txt

@@ -288,6 +288,7 @@ python/ray/data/tests/test_hash_shuffle_v2.py
 python/ray/data/tests/test_issue_detection.py
 python/ray/data/tests/test_issue_detection_manager.py
 python/ray/data/tests/test_iter_torch_batches.py
+python/ray/data/tests/test_iter_torch_batches_gpu.py
 python/ray/data/tests/test_iterator.py
 python/ray/data/tests/test_join.py
 python/ray/data/tests/test_jumbo_arrow_block.py

ci/ray_ci/serve_di_test_names.txt

@@ -1,40 +1,2 @@
-//python/ray/serve/tests:test_api
-//python/ray/serve/tests:test_callback
-//python/ray/serve/tests:test_cli
-//python/ray/serve/tests:test_cli_2
-//python/ray/serve/tests:test_deploy
-//python/ray/serve/tests:test_deploy_2
-//python/ray/serve/tests:test_deploy_app
-//python/ray/serve/tests:test_deploy_app_2
-//python/ray/serve/tests:test_fastapi
-//python/ray/serve/tests:test_failure
-//python/ray/serve/tests:test_gcs_failure
-//python/ray/serve/tests:test_grpc
-//python/ray/serve/tests:test_grpc_e2e
-//python/ray/serve/tests:test_grpc_replica_wrapper
-//python/ray/serve/tests:test_handle_1
-//python/ray/serve/tests:test_handle_2
-//python/ray/serve/tests:test_handle_cancellation
-//python/ray/serve/tests:test_handle_streaming
-//python/ray/serve/tests:test_healthcheck
-//python/ray/serve/tests:test_logging
-//python/ray/serve/tests:test_metrics
-//python/ray/serve/tests:test_model_composition
-//python/ray/serve/tests:test_websockets
-//python/ray/serve/tests:test_request_timeout
-//python/ray/serve/tests:test_streaming_response
-//python/ray/serve/tests:test_target_capacity
-//python/ray/serve/tests:test_deployment_version
-//python/ray/serve/tests:test_http_cancellation
-//python/ray/serve/tests:test_http_headers
-//python/ray/serve/tests:test_controller_recovery
-//python/ray/serve/tests:test_http_routes
-//python/ray/serve/tests:test_backpressure
-//python/ray/serve/tests:test_batching
-//python/ray/serve/tests:test_replica_sync_methods_with_run_sync_in_threadpool
-//python/ray/serve/tests:test_replica_request_context
 //python/ray/serve/tests:test_direct_ingress
-//python/ray/serve/tests:test_per_request_headers
 //python/ray/serve/tests:test_direct_ingress_with_grpc
-//python/ray/serve/tests:test_per_request_headers_with_grpc
-//python/ray/serve/tests:test_handle

doc/source/_static/css/custom.css

@@ -378,6 +378,16 @@ table.autosummary tr > td:first-child > p > a > code > span {
   overflow-wrap: break-word;
 }
 
+/* The theme bolds the whole signature via `dt { font-weight: 700 }`, flattening
+   its hierarchy. Reset to normal weight and re-bold only the object name so it
+   stays the scannable anchor. */
+dt.sig {
+  font-weight: normal;
+}
+dt.sig .sig-name {
+  font-weight: 700;
+}
+
 /* RTD footer container makes the parent */
 /* #main-sidebar always scrollable if you don't remove negative margin. */
 /* Restrict width to 30% of the window */

doc/source/cluster/kubernetes/user-guides/config.md

@@ -116,6 +116,20 @@ It's ideal to size each Ray pod to take up the entire Kubernetes node. In other
 #### nodeSelector and tolerations
 You can control the scheduling of worker groups' Ray pods by setting the `nodeSelector` and `tolerations` fields of the pod spec. Specifically, these fields determine on which Kubernetes nodes the pods may be scheduled. See the [Kubernetes docs](https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/) for more about Pod-to-Node assignment.
 
+(kuberay-config-safe-to-evict)=
+#### Preventing eviction of the Ray head Pod by the Cluster Autoscaler
+The Ray head Pod is a single point of failure for the Ray cluster. To prevent the Kubernetes Cluster Autoscaler from evicting it during node consolidation, set the [`cluster-autoscaler.kubernetes.io/safe-to-evict`](https://kubernetes.io/docs/reference/labels-annotations-taints/#cluster-autoscaler-kubernetes-io-safe-to-evict) annotation to `"false"` on the head Pod template:
+
+```yaml
+headGroupSpec:
+  template:
+    metadata:
+      annotations:
+        cluster-autoscaler.kubernetes.io/safe-to-evict: "false"
+```
+
+This setting blocks eviction during autoscaling events. This doesn't prevent Ray head Pod errors such events as node failure, manual Pod deletion, or reclamation of the underlying node (such as spot instance preemption).
+
 #### image
 The Ray container images specified in the `RayCluster` CR should carry the same Ray version as the CR's `spec.rayVersion`. If you are using a nightly or development Ray image, you can specify Ray's latest release version under `spec.rayVersion`.
 

doc/source/cluster/kubernetes/user-guides/k8s-autoscaler.md

@@ -21,6 +21,8 @@ Horizontal Pod Autoscaling is centrally controlled by a manager in the Kubernete
 ## Ray Autoscaler with Kubernetes Cluster Autoscaler
 The Ray Autoscaler and the [Kubernetes Cluster Autoscaler](https://github.com/kubernetes/autoscaler/tree/master/cluster-autoscaler) complement each other. After the Ray autoscaler decides to create a Ray Pod, the Kubernetes Cluster Autoscaler can provision a Kubernetes node so that the Pod can be placed. Similarly, after the Ray autoscaler decides to delete an idle Pod, the Kubernetes Cluster Autoscaler can clean up the idle Kubernetes node that remains. It is recommended to configure your RayCluster so that only one Ray Pod fits per Kubernetes node. If you follow this pattern, Ray Autoscaler Pod scaling events correspond roughly one-to-one with cluster autoscaler node scaling events. (We say "roughly" because it is possible for a Ray Pod be deleted and replaced with a new Ray Pod before the underlying Kubernetes node is scaled down.)
 
+Because the Ray head Pod is a single point of failure, consider setting `cluster-autoscaler.kubernetes.io/safe-to-evict: "false"` on its Pod template to prevent the Cluster Autoscaler from evicting it; see {ref}`RayCluster Configuration <kuberay-config-safe-to-evict>`.
+
 
 ## Vertical Pod Autoscaler
 There is no relationship between the Ray Autoscaler and the Kubernetes [Vertical Pod Autoscaler](https://github.com/kubernetes/autoscaler/tree/master/vertical-pod-autoscaler) (VPA), which is meant to size individual Pods to the appropriate size based on current and past usage. If you find that the load on your individual Ray Pods is too high, there are a number of manual techniques to decrease the load. One method is to schedule fewer tasks/actors per node by increasing the resource requirements specified in the `ray.remote` annotation. For example, changing `@ray.remote(num_cpus=2)` to `@ray.remote(num_cpus=4)` will halve the quantity of that task or actor that can fit in a given Ray Pod.

doc/source/cluster/kubernetes/user-guides/tls.md

@@ -125,19 +125,19 @@ For more information on how to configure Ray with TLS authentication, please ref
 # Log in to the worker Pod
 kubectl exec -it ${WORKER_POD} -- bash
 
-# Since the head Pod has the certificate of $FQ_RAY_IP, the connection to the worker Pods
-# will be established successfully, and the exit code of the ray health-check command
-# should be 0.
+# The worker connects to the head through the head Service address.
+# The head certificate includes $FQ_RAY_IP in its SAN entries, so TLS verification succeeds.
+# The exit code of the ray health-check command should be 0.
 ray health-check --address $FQ_RAY_IP:6379
 echo $? # 0
 
-# Since the head Pod has the certificate of $RAY_IP, the connection will fail and an error
-# message similar to the following will be displayed: "Peer name raycluster-tls-head-svc is
-# not in peer certificate".
+# The head certificate doesn't include $RAY_IP in its SAN entries, so TLS verification fails.
+# You should see a TLS verification error.
+# The error will indicate that the pod IP ($RAY_IP) is not in the certificate.
 ray health-check --address $RAY_IP:6379
 
 # If you add `DNS.3 = $RAY_IP` to the [alt_names] section in `gencert_head.sh`,
-# the head Pod will generate the certificate of $RAY_IP.
+# the generated head certificate will include a SAN entry for $RAY_IP.
 #
 # For KubeRay versions prior to 0.5.0, this step is necessary because Ray workers in earlier
 # versions use $RAY_IP to connect with Ray head.