Merge upstream/master into master (sync fork with upstream)

.buildkite/build-start-operator.sh

@@ -1,20 +1,27 @@
 #!/bin/bash
+set -eo pipefail
 
 # This script is used to start the operator in the buildkite test-e2e steps.
 
 # When starting from the ray ci release automation, we want to install the latest
 # released version from helm as actual users might. Ray ci is also always expected
 # to kick off from the release branch so tests should match up accordingly.
+#
+# NOTE: Do not enable `set -u` (nounset) here. This script is sourced by the
+# Buildkite steps, so shell options propagate to the caller; downstream steps
+# reference $KUBERAY_TEST_RAY_IMAGE without a default and would fail under -u
+# when the variable is not set (e.g. in the non-release path).
 
 if [ "$IS_FROM_RAY_RELEASE_AUTOMATION" = 1 ]; then
     helm repo update
     echo "Installing helm chart with test override values (feature gates enabled as needed)"
     # NOTE: The override file is CI/test-only. It is NOT part of the released chart defaults.
     helm install kuberay-operator kuberay/kuberay-operator -f ../.buildkite/values-kuberay-operator-override.yaml
-    KUBERAY_TEST_RAY_IMAGE="rayproject/ray:nightly-extra-py310-cpu" && export KUBERAY_TEST_RAY_IMAGE
+    KUBERAY_TEST_RAY_IMAGE="rayproject/ray:nightly-extra-py310-cpu"
+    export KUBERAY_TEST_RAY_IMAGE
 else
-    IMG=kuberay/operator:nightly make docker-image &&
-    kind load docker-image kuberay/operator:nightly &&
+    IMG=kuberay/operator:nightly make docker-image
+    kind load docker-image kuberay/operator:nightly
     echo "Deploying operator with test overrides (feature gates via test-overrides overlay)"
     IMG=kuberay/operator:nightly make deploy-with-override
 fi

.buildkite/setup-env.sh

@@ -7,7 +7,7 @@ export PATH=$PATH:/usr/local/go/bin
 export DOCKER_API_VERSION=1.43
 
 # Install kind
-curl -Lo ./kind https://kind.sigs.k8s.io/dl/v0.22.0/kind-linux-amd64
+curl -Lo ./kind https://kind.sigs.k8s.io/dl/v0.31.0/kind-linux-amd64
 chmod +x ./kind
 mv ./kind /usr/local/bin/kind
 
@@ -35,7 +35,7 @@ apt-get update
 apt-get install -y python3.11 python3-pip python3-venv python3-dev build-essential
 
 # Install requirements
-pip install --break-system-packages ray[default]==2.52.0
+pip install --break-system-packages ray[default]==2.55.0
 
 # Bypass Git's ownership check due to unconventional user IDs in Docker containers
 git config --global --add safe.directory /workdir

.buildkite/test-e2e.yml

@@ -55,15 +55,35 @@
     - set -o pipefail
     - mkdir -p "$(pwd)/tmp" && export KUBERAY_TEST_OUTPUT_DIR=$(pwd)/tmp
     - echo "KUBERAY_TEST_OUTPUT_DIR=$$KUBERAY_TEST_OUTPUT_DIR"
-    - KUBERAY_TEST_TIMEOUT_SHORT=1m KUBERAY_TEST_TIMEOUT_MEDIUM=5m KUBERAY_TEST_TIMEOUT_LONG=10m go test -timeout 30m -v ./test/e2erayservice 2>&1 | awk -f ../.buildkite/format.awk | tee $$KUBERAY_TEST_OUTPUT_DIR/gotest.log || (kubectl logs --tail -1 -l app.kubernetes.io/name=kuberay | tee $$KUBERAY_TEST_OUTPUT_DIR/kuberay-operator.log && cd $$KUBERAY_TEST_OUTPUT_DIR && find . -name "*.log" | tar -cf /artifact-mount/e2e-rayservice-log.tar -T - && exit 1)
+    - KUBERAY_TEST_TIMEOUT_SHORT=1m KUBERAY_TEST_TIMEOUT_MEDIUM=5m KUBERAY_TEST_TIMEOUT_LONG=10m go test -timeout 30m -v -skip Suspend ./test/e2erayservice 2>&1 | awk -f ../.buildkite/format.awk | tee $$KUBERAY_TEST_OUTPUT_DIR/gotest.log || (kubectl logs --tail -1 -l app.kubernetes.io/name=kuberay | tee $$KUBERAY_TEST_OUTPUT_DIR/kuberay-operator.log && cd $$KUBERAY_TEST_OUTPUT_DIR && find . -name "*.log" | tar -cf /artifact-mount/e2e-rayservice-log.tar -T - && exit 1)
     - echo "--- END:e2e rayservice (nightly operator) tests finished"
 
+- label: 'Test E2E rayservice suspend (nightly operator)'
+  instance_size: large
+  image: golang:1.26-bookworm
+  commands:
+    - source .buildkite/setup-env.sh
+    - kind create cluster --wait 900s --config ./ci/kind-config-buildkite.yml
+    - kubectl config set clusters.kind-kind.server https://docker:6443
+    # Build nightly KubeRay operator image
+    - pushd ray-operator
+    - source ../.buildkite/build-start-operator.sh
+    - kubectl wait --timeout=90s --for=condition=Available=true deployment kuberay-operator
+    # Run suspend e2e tests and print KubeRay operator logs if tests fail
+    - echo "--- START:Running e2e rayservice suspend (nightly operator) tests"
+    - if [ -n "$${KUBERAY_TEST_RAY_IMAGE}" ]; then echo "Using Ray Image $${KUBERAY_TEST_RAY_IMAGE}"; fi
+    - set -o pipefail
+    - mkdir -p "$(pwd)/tmp" && export KUBERAY_TEST_OUTPUT_DIR=$(pwd)/tmp
+    - echo "KUBERAY_TEST_OUTPUT_DIR=$$KUBERAY_TEST_OUTPUT_DIR"
+    - KUBERAY_TEST_TIMEOUT_SHORT=1m KUBERAY_TEST_TIMEOUT_MEDIUM=5m KUBERAY_TEST_TIMEOUT_LONG=10m go test -timeout 30m -v -run Suspend ./test/e2erayservice 2>&1 | awk -f ../.buildkite/format.awk | tee $$KUBERAY_TEST_OUTPUT_DIR/gotest.log || (kubectl logs --tail -1 -l app.kubernetes.io/name=kuberay | tee $$KUBERAY_TEST_OUTPUT_DIR/kuberay-operator.log && cd $$KUBERAY_TEST_OUTPUT_DIR && find . -name "*.log" | tar -cf /artifact-mount/e2e-rayservice-suspend-log.tar -T - && exit 1)
+    - echo "--- END:e2e rayservice suspend (nightly operator) tests finished"
+
 - label: 'Test RayService Incremental Upgrade E2E (nightly operator)'
   instance_size: large
   image: golang:1.26-bookworm
   commands:
     - source .buildkite/setup-env.sh
-    - kind create cluster --wait 900s --config ./ci/kind-config-buildkite-1-29.yml
+    - kind create cluster --wait 900s --config ./ci/kind-config-buildkite.yml
     - kubectl config set clusters.kind-kind.server https://docker:6443
 
     # Install MetalLB for LoadBalancer IPs on Kind

.buildkite/test-python-client.yml

@@ -7,8 +7,27 @@
     - kubectl config set clusters.kind-kind.server https://docker:6443
     # Build nightly KubeRay operator image
     - pushd ray-operator
-    - source ../.buildkite/build-start-operator.sh
-    - kubectl wait --timeout=90s --for=condition=Available=true deployment kuberay-operator
+    - |
+      for attempt in 1 2; do
+        if source ../.buildkite/build-start-operator.sh; then
+          break
+        fi
+        if [ "$attempt" -eq 2 ]; then
+          echo "--- ERROR: build-start-operator.sh failed twice"
+          exit 1
+        fi
+        echo "--- WARN: build-start-operator.sh failed; retrying in 20s"
+        sleep 20
+      done
+    - |
+      if ! kubectl wait --timeout=180s --for=condition=Available=true deployment kuberay-operator; then
+        echo "--- kuberay-operator not ready within 180s; collecting diagnostics"
+        kubectl get pods -A -o wide || true
+        kubectl describe deployment kuberay-operator || true
+        kubectl logs --tail=200 -l app.kubernetes.io/name=kuberay || true
+        echo "--- retrying kuberay-operator readiness wait once"
+        kubectl wait --timeout=180s --for=condition=Available=true deployment kuberay-operator
+      fi
     - popd
     # Setup Python environment and install Python client
     - echo "--- START:Setting up Python environment"

.buildkite/values-kuberay-operator-override.yaml

@@ -20,3 +20,5 @@ featureGates:
     enabled: true
   - name: RayServiceIncrementalUpgrade
     enabled: true
+  - name: SidecarSubmitterRestart
+    enabled: true

.github/dependabot.yml

@@ -4,7 +4,6 @@ updates:
   # Maintain go dependencies for sub-projects
   - package-ecosystem: "gomod"
     directories:
-      - "/experimental"
       - "/ray-operator"
       - "/apiserver"
       - "/kubectl-plugin"
@@ -25,3 +24,13 @@ updates:
       all-dependencies: # for all other dependencies not listed above
         patterns:
           - "*"
+
+  # Maintain GitHub Actions referenced in .github/workflows/
+  - package-ecosystem: "github-actions"
+    directory: "/"
+    schedule:
+      interval: "weekly"
+    groups:
+      all-actions:
+        patterns:
+          - "*"

.github/workflows/consistency-check.yaml

@@ -16,12 +16,12 @@ jobs:
     runs-on: ubuntu-22.04
     timeout-minutes: 10
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v6
         with:
           fetch-depth: 0
 
       - name: Set up Go
-        uses: actions/setup-go@v3
+        uses: actions/setup-go@v6
         with:
           # Use the same go version with build job
           go-version: v1.26
@@ -43,12 +43,12 @@ jobs:
     runs-on: ubuntu-22.04
     timeout-minutes: 10
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v6
         with:
           fetch-depth: 0
 
       - name: Set up Go
-        uses: actions/setup-go@v3
+        uses: actions/setup-go@v6
         with:
           # Use the same go version with build job
           go-version: v1.26
@@ -67,12 +67,12 @@ jobs:
     runs-on: ubuntu-22.04
     timeout-minutes: 10
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v6
         with:
           fetch-depth: 0
 
       - name: Set up Go
-        uses: actions/setup-go@v3
+        uses: actions/setup-go@v6
         with:
           # Use the same go version with build job
           go-version: v1.26
@@ -83,7 +83,7 @@ jobs:
 
       - name: Verify Changed files
         id: verify-changed-files
-        uses: tj-actions/verify-changed-files@v17
+        uses: tj-actions/verify-changed-files@v20
         with:
           files: |
             ./config/crd/bases/*.yaml
@@ -91,7 +91,7 @@ jobs:
 
       - name: Check changed files
         if: steps.verify-changed-files.outputs.files_changed == 'true'
-        uses: actions/github-script@v3
+        uses: actions/github-script@v9
         with:
           script: |
             core.setFailed('Please run \"make manifests\" to synchronize CRD and RBAC!')
@@ -102,7 +102,7 @@ jobs:
     runs-on: ubuntu-22.04
     timeout-minutes: 10
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v6
         with:
           fetch-depth: 0
 
@@ -113,14 +113,14 @@ jobs:
 
       - name: Verify Changed files
         id: verify-changed-files
-        uses: tj-actions/verify-changed-files@v17
+        uses: tj-actions/verify-changed-files@v20
         with:
           files: |
             ./helm-chart/kuberay-operator/crds/*.yaml
 
       - name: Check changed files
         if: steps.verify-changed-files.outputs.files_changed == 'true'
-        uses: actions/github-script@v3
+        uses: actions/github-script@v9
         with:
           script: |
             core.setFailed('Please run \"make sync\" to synchronize CRDs!')
@@ -131,16 +131,16 @@ jobs:
     runs-on: ubuntu-22.04
     timeout-minutes: 10
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v6
         with:
           fetch-depth: 0
 
       - name: Set up Helm
-        uses: azure/setup-helm@v3.3
+        uses: azure/setup-helm@v5.0.0
         with:
           version: v3.9.4
 
-      - uses: actions/setup-python@v4
+      - uses: actions/setup-python@v6
         with:
           python-version: 3.7
 

.github/workflows/helm.yaml

@@ -43,16 +43,16 @@ jobs:
           echo "BRANCH=$BRANCH" >> "$GITHUB_OUTPUT"
 
       - name: Checkout Code
-        uses: actions/checkout@v4
+        uses: actions/checkout@v6
         with:
           fetch-depth: 0
 
       - name: Set up Helm
-        uses: azure/setup-helm@v4.3.0
+        uses: azure/setup-helm@v5.0.0
         with:
           version: v3.17.3
 
-      - uses: actions/setup-python@v5.3.0
+      - uses: actions/setup-python@v6
         with:
           python-version: 3.13
 
@@ -63,7 +63,7 @@ jobs:
         run: helm unittest ${{ env.CHART_DIR }}/${{ matrix.chart }} --file "tests/**/*_test.yaml" --strict --debug
 
       - name: Set up chart-testing
-        uses: helm/chart-testing-action@v2.7.0
+        uses: helm/chart-testing-action@v2.8.0
 
       - name: Run chart-testing (list-changed)
         id: list-changed
@@ -87,7 +87,7 @@ jobs:
 
       - name: Create kind cluster
         if: steps.list-changed.outputs.changed == 'true'
-        uses: helm/kind-action@v1.12.0
+        uses: helm/kind-action@v1.14.0
         with:
           cluster_name: kind
 
@@ -101,11 +101,6 @@ jobs:
         run: |
           cd apiserver && make docker-image -e IMG=kuberay/apiserver:local
 
-      - name: Build Docker image (security-proxy)
-        if: steps.list-changed.outputs.changed == 'true' && matrix.chart == 'kuberay-apiserver'
-        run: |
-          cd experimental && make docker-image -e IMG=kuberay/security-proxy:local
-
       - name: Load image to kind cluster (kuberay-operator)
         if: steps.list-changed.outputs.changed == 'true' && matrix.chart == 'kuberay-operator'
         run: |
@@ -116,11 +111,6 @@ jobs:
         run: |
           kind load docker-image kuberay/apiserver:local
 
-      - name: Load image to kind cluster (security-proxy)
-        if: steps.list-changed.outputs.changed == 'true' && matrix.chart == 'kuberay-apiserver'
-        run: |
-          kind load docker-image kuberay/security-proxy:local
-
       - name: Install Custom Resource Definitions to kind cluster
         if: steps.list-changed.outputs.changed == 'true' && matrix.chart == 'ray-cluster'
         working-directory: helm-chart/kuberay-operator

.github/workflows/image-release.yaml

@@ -12,18 +12,18 @@ jobs:
     steps:
 
     - name: Error if not a tag
-      uses: actions/github-script@v7
+      uses: actions/github-script@v9
       if: ${{ ! startsWith(github.ref, 'refs/tags/') }}
       with:
         script: core.setFailed('This action can only be run on tags')
 
     - name: Set up Go
-      uses: actions/setup-go@v3
+      uses: actions/setup-go@v6
       with:
         go-version: v1.26
 
     - name: Check out code into the Go module directory
-      uses: actions/checkout@v2
+      uses: actions/checkout@v6
       with:
         fetch-depth: 0
 
@@ -53,7 +53,7 @@ jobs:
       working-directory: ${{env.working-directory}}
 
     - name: Log in to Quay.io
-      uses: docker/login-action@v2
+      uses: docker/login-action@v4
       with:
         registry: quay.io
         username: ${{ secrets.QUAY_USERNAME }}
@@ -79,10 +79,10 @@ jobs:
       working-directory: ${{env.working-directory}}
 
     - name: Set up Docker Buildx
-      uses: docker/setup-buildx-action@v3
+      uses: docker/setup-buildx-action@v4
 
     - name: Build MultiArch Image
-      uses: docker/build-push-action@v5
+      uses: docker/build-push-action@v7
       env:
         PUSH: true
         REPO_ORG: kuberay
@@ -105,18 +105,18 @@ jobs:
     steps:
 
     - name: Error if not a tag
-      uses: actions/github-script@v7
+      uses: actions/github-script@v9
       if: ${{ ! startsWith(github.ref, 'refs/tags/') }}
       with:
         script: core.setFailed('This action can only be run on tags')
 
     - name: Set up Go
-      uses: actions/setup-go@v3
+      uses: actions/setup-go@v6
       with:
         go-version: v1.26
 
     - name: Check out code into the Go module directory
-      uses: actions/checkout@v2
+      uses: actions/checkout@v6
       with:
         fetch-depth: 0
 
@@ -146,7 +146,7 @@ jobs:
       working-directory: ${{env.working-directory}}
 
     - name: Log in to Quay.io
-      uses: docker/login-action@v2
+      uses: docker/login-action@v4
       with:
         registry: quay.io
         username: ${{ secrets.QUAY_USERNAME }}
@@ -175,10 +175,10 @@ jobs:
       working-directory: ${{env.working-directory}}
 
     - name: Set up Docker Buildx
-      uses: docker/setup-buildx-action@v3
+      uses: docker/setup-buildx-action@v4
 
     - name: Build MultiArch Image
-      uses: docker/build-push-action@v5
+      uses: docker/build-push-action@v7
       env:
         PUSH: true
         REPO_ORG: kuberay
@@ -194,7 +194,7 @@ jobs:
           quay.io/${{env.REPO_ORG}}/${{env.REPO_NAME}}:${{ env.tag }}
 
     - name: Create ray-operator tag
-      uses: actions/github-script@v6
+      uses: actions/github-script@v9
       with:
         script: |
           await github.rest.git.createRef({
@@ -212,13 +212,13 @@ jobs:
     steps:
 
     - name: Error if not a tag
-      uses: actions/github-script@v7
+      uses: actions/github-script@v9
       if: ${{ ! startsWith(github.ref, 'refs/tags/') }}
       with:
         script: core.setFailed('This action can only be run on tags')
 
     - name: Check out code into dashboard directory
-      uses: actions/checkout@v2
+      uses: actions/checkout@v6
       with:
         fetch-depth: 0
 
@@ -231,14 +231,14 @@ jobs:
       run: echo "::set-output name=sha_short::$(git rev-parse --short HEAD)"
 
     - name: Log in to Quay.io
-      uses: docker/login-action@v2
+      uses: docker/login-action@v4
       with:
         registry: quay.io
         username: ${{ secrets.QUAY_USERNAME }}
         password: ${{ secrets.QUAY_ROBOT_TOKEN }}
 
     - name: Build and push Dashboard image
-      uses: docker/build-push-action@v4
+      uses: docker/build-push-action@v7
       with:
         context: ${{ env.working-directory }}
         file: ${{ env.working-directory }}/Dockerfile
@@ -258,18 +258,18 @@ jobs:
     steps:
 
     - name: Error if not a tag
-      uses: actions/github-script@v7
+      uses: actions/github-script@v9
       if: ${{ ! startsWith(github.ref, 'refs/tags/') }}
       with:
         script: core.setFailed('This action can only be run on tags')
 
     - name: Set up Go
-      uses: actions/setup-go@v3
+      uses: actions/setup-go@v6
       with:
         go-version: v1.26
 
     - name: Check out code into the Go module directory
-      uses: actions/checkout@v2
+      uses: actions/checkout@v6
       with:
         fetch-depth: 0
 
@@ -286,7 +286,7 @@ jobs:
       working-directory: ${{env.working-directory}}
 
     - name: Log in to Quay.io
-      uses: docker/login-action@v2
+      uses: docker/login-action@v4
       with:
         registry: quay.io
         username: ${{ secrets.QUAY_USERNAME }}
@@ -311,10 +311,10 @@ jobs:
       working-directory: ${{env.working-directory}}
 
     - name: Set up Docker Buildx
-      uses: docker/setup-buildx-action@v3
+      uses: docker/setup-buildx-action@v4
 
     - name: Build MultiArch Image
-      uses: docker/build-push-action@v5
+      uses: docker/build-push-action@v7
       env:
         PUSH: true
         REPO_ORG: kuberay

.github/workflows/kubectl-plugin-release.yaml

@@ -7,11 +7,11 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout
-        uses: actions/checkout@v4
+        uses: actions/checkout@v6
         with:
           fetch-depth: 0
       - name: Error if not a tag
-        uses: actions/github-script@v7
+        uses: actions/github-script@v9
         if: ${{ ! startsWith(github.ref, 'refs/tags/') }}
         with:
           script: core.setFailed('This action can only be run on tags')
@@ -25,11 +25,11 @@ jobs:
           fi
 
       - name: Setup Go
-        uses: actions/setup-go@v5
+        uses: actions/setup-go@v6
         with:
           go-version: "1.26"
       - name: GoReleaser
-        uses: goreleaser/goreleaser-action@v6
+        uses: goreleaser/goreleaser-action@v7
         with:
           distribution: "goreleaser"
           version: latest
@@ -39,4 +39,4 @@ jobs:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
       - name: Update new version in krew-index
         if: ${{ steps.check_stable.outputs.is_stable == 'true' }}
-        uses: rajatjindal/krew-release-bot@v0.0.46
+        uses: rajatjindal/krew-release-bot@v0.0.51

.github/workflows/site.yaml

@@ -8,7 +8,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout main
-        uses: actions/checkout@v2
+        uses: actions/checkout@v6
       - name: Deploy docs
         uses: mhausenblas/mkdocs-deploy-gh-pages@master
         env:

.github/workflows/test-job.yaml

@@ -15,9 +15,9 @@ jobs:
     name: Lint (pre-commit)
     runs-on: ubuntu-22.04
     steps:
-      - uses: actions/checkout@v3
-      - uses: actions/setup-python@v3
-      - uses: actions/setup-node@v4
+      - uses: actions/checkout@v6
+      - uses: actions/setup-python@v6
+      - uses: actions/setup-node@v6
         with:
           node-version: lts/*
           cache: yarn
@@ -36,12 +36,12 @@ jobs:
     runs-on: ubuntu-22.04
     steps:
       - name: Set up Go
-        uses: actions/setup-go@v3
+        uses: actions/setup-go@v6
         with:
           go-version: v1.26
 
       - name: Check out code into the Go module directory
-        uses: actions/checkout@v2
+        uses: actions/checkout@v6
         with:
           # When checking out the repository that
           # triggered a workflow, this defaults to the reference or SHA for that event.
@@ -76,7 +76,7 @@ jobs:
         working-directory: ${{env.working-directory}}
 
       - name: Set up Docker
-        uses: docker/setup-docker-action@v4
+        uses: docker/setup-docker-action@v5
 
       - name: Build Docker Image - Apiserver
         run: |
@@ -84,13 +84,13 @@ jobs:
           docker save -o /tmp/apiserver.tar kuberay/apiserver:${{ steps.vars.outputs.sha_short }}
 
       - name: Upload Artifact Apiserver
-        uses: actions/upload-artifact@v4
+        uses: actions/upload-artifact@v7
         with:
           name: apiserver_img
           path: /tmp/apiserver.tar
 
       - name: Log in to Quay.io
-        uses: docker/login-action@v2
+        uses: docker/login-action@v4
         with:
           registry: quay.io
           username: ${{ secrets.QUAY_USERNAME }}
@@ -117,10 +117,10 @@ jobs:
         working-directory: ${{env.working-directory}}
 
       - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v3
+        uses: docker/setup-buildx-action@v4
 
       - name: Build MultiArch Docker Image and Push on Merge
-        uses: docker/build-push-action@v5
+        uses: docker/build-push-action@v7
         env:
           REPO_ORG: kuberay
           REPO_NAME: apiserver
@@ -141,12 +141,12 @@ jobs:
     runs-on: ubuntu-22.04
     steps:
       - name: Set up Go
-        uses: actions/setup-go@v3
+        uses: actions/setup-go@v6
         with:
           go-version: v1.26
 
       - name: Check out code into the Go module directory
-        uses: actions/checkout@v2
+        uses: actions/checkout@v6
         with:
           ref: ${{github.event.pull_request.head.sha}}
 
@@ -162,109 +162,6 @@ jobs:
         run: make test
         working-directory: ${{env.working-directory}}
 
-  build_security_proxy:
-    env:
-      working-directory: ./experimental
-    name: Build security proxy Binaries and Docker Images
-    runs-on: ubuntu-22.04
-    steps:
-
-      - name: Set up Go
-        uses: actions/setup-go@v3
-        with:
-          go-version: v1.26
-
-      - name: Check out code into the Go module directory
-        uses: actions/checkout@v2
-        with:
-          # When checking out the repository that
-          # triggered a workflow, this defaults to the reference or SHA for that event.
-          # Default value should work for both pull_request and merge(push) event.
-          ref: ${{github.event.pull_request.head.sha}}
-
-      - name: list directories
-        working-directory: ${{env.working-directory}}
-        run: |
-          pwd
-          ls -R
-
-      - name: install kubebuilder
-        run: |
-          wget https://github.com/kubernetes-sigs/kubebuilder/releases/download/v3.0.0/kubebuilder_$(go env GOOS)_$(go env GOARCH)
-          sudo mv kubebuilder_$(go env GOOS)_$(go env GOARCH) /usr/local/bin/kubebuilder
-
-      - name: Get revision SHA
-        id: vars
-        run: echo "::set-output name=sha_short::$(git rev-parse --short HEAD)"
-
-      - name: Get dependencies
-        run: go mod download
-        working-directory: ${{env.working-directory}}
-
-      - name: Build
-        run: make build
-        working-directory: ${{env.working-directory}}
-
-      - name: Set up Docker
-        uses: docker/setup-docker-action@v4
-
-      - name: Build Docker Image - security proxy
-        run: |
-          IMG=kuberay/security-proxy:${{ steps.vars.outputs.sha_short }} make docker-image
-          docker save -o /tmp/security-proxy.tar kuberay/security-proxy:${{ steps.vars.outputs.sha_short }}
-        working-directory: ${{env.working-directory}}
-
-      - name: Upload security proxy artifact
-        uses: actions/upload-artifact@v4
-        with:
-          name: security-proxy_img
-          path: /tmp/security-proxy.tar
-
-      - name: Log in to Quay.io
-        uses: docker/login-action@v2
-        with:
-          registry: quay.io
-          username: ${{ secrets.QUAY_USERNAME }}
-          password: ${{ secrets.QUAY_ROBOT_TOKEN }}
-        if: contains(fromJson('["refs/heads/master"]'), github.ref)
-
-      # Build security proxy binaries inside the gh runner vm directly and then copy the go binaries to docker images using the Dockerfile.buildx
-      - name: Build linux/amd64 security proxy go binary
-        env:
-          CGO_ENABLED: 0
-          GOOS: linux
-          GOARCH: amd64
-        run: |
-          CGO_ENABLED=$CGO_ENABLED GOOS=$GOOS GOARCH=$GOARCH go build -a -o security_proxy-$GOARCH cmd/main.go
-        working-directory: ${{env.working-directory}}
-
-      - name: Build linux/arm64 security proxy binary
-        env:
-          CGO_ENABLED: 0
-          GOOS: linux
-          GOARCH: arm64
-        run: |
-          CGO_ENABLED=$CGO_ENABLED GOOS=$GOOS GOARCH=$GOARCH go build -a -o security_proxy-$GOARCH cmd/main.go
-        working-directory: ${{env.working-directory}}
-
-      - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v3
-
-      - name: Build MultiArch Docker Image and Push on Merge
-        uses: docker/build-push-action@v5
-        env:
-          REPO_ORG: kuberay
-          REPO_NAME: security-proxy
-        with:
-          platforms: linux/amd64,linux/arm64
-          context: .
-          file: ./experimental/Dockerfile.buildx
-          push: ${{ contains(fromJson('["refs/heads/master"]'), github.ref) }}
-          provenance: false
-          tags: |
-            quay.io/${{env.REPO_ORG}}/${{env.REPO_NAME}}:${{ steps.vars.outputs.sha_short }}
-            quay.io/${{env.REPO_ORG}}/${{env.REPO_NAME}}:nightly
-
   build_operator:
     env:
       working-directory: ./ray-operator
@@ -273,12 +170,12 @@ jobs:
     steps:
 
     - name: Set up Go
-      uses: actions/setup-go@v3
+      uses: actions/setup-go@v6
       with:
         go-version: v1.26
 
     - name: Check out code into the Go module directory
-      uses: actions/checkout@v2
+      uses: actions/checkout@v6
       with:
         # When checking out the repository that
         # triggered a workflow, this defaults to the reference or SHA for that event.
@@ -313,7 +210,7 @@ jobs:
       working-directory: ${{env.working-directory}}
 
     - name: Set up Docker
-      uses: docker/setup-docker-action@v4
+      uses: docker/setup-docker-action@v5
 
     - name: Build Docker Image - Operator
       run: |
@@ -322,13 +219,13 @@ jobs:
       working-directory: ${{env.working-directory}}
 
     - name: Upload Artifact Operator
-      uses: actions/upload-artifact@v4
+      uses: actions/upload-artifact@v7
       with:
         name: operator_img
         path: /tmp/operator.tar
 
     - name: Log in to Quay.io
-      uses: docker/login-action@v2
+      uses: docker/login-action@v4
       with:
         registry: quay.io
         username: ${{ secrets.QUAY_USERNAME }}
@@ -360,11 +257,11 @@ jobs:
       if: contains(fromJson('["refs/heads/master"]'), github.ref)
 
     - name: Set up Docker Buildx
-      uses: docker/setup-buildx-action@v3
+      uses: docker/setup-buildx-action@v4
       if: contains(fromJson('["refs/heads/master"]'), github.ref)
 
     - name: Build MultiArch Docker Image and Push on Merge
-      uses: docker/build-push-action@v5
+      uses: docker/build-push-action@v7
       env:
         REPO_ORG: kuberay
         REPO_NAME: operator
@@ -386,12 +283,12 @@ jobs:
     runs-on: ubuntu-22.04
     steps:
       - name: Set up Go
-        uses: actions/setup-go@v3
+        uses: actions/setup-go@v6
         with:
           go-version: v1.26
 
       - name: Check out code into the Go module directory
-        uses: actions/checkout@v2
+        uses: actions/checkout@v6
         with:
           # When checking out the repository that
           # triggered a workflow, this defaults to the reference or SHA for that event.
@@ -417,12 +314,12 @@ jobs:
     runs-on: ubuntu-22.04
     steps:
       - name: Set up Go
-        uses: actions/setup-go@v3
+        uses: actions/setup-go@v6
         with:
           go-version: v1.26
 
       - name: Check out code into the Go module directory
-        uses: actions/checkout@v2
+        uses: actions/checkout@v6
         with:
           ref: ${{github.event.pull_request.head.sha}}
 

.golangci.yml

@@ -112,8 +112,6 @@ linters:
       # These require non-trivial migration and should be addressed separately.
       - linters: [staticcheck]
         text: "SA1019:.*GetEventRecorderFor is deprecated"
-      - linters: [staticcheck]
-        text: "SA1019:.*scheme\\.Builder is deprecated"
     generated: strict
     presets:
       - comments

apiserver/Makefile

@@ -156,9 +156,6 @@ load-image: ## Load the api server image to the kind cluster created with create
 operator-image: ## Build the operator image to be loaded in your kind cluster.
 	cd ../ray-operator && $(MAKE) docker-image -e IMG=quay.io/kuberay/operator:$(OPERATOR_IMAGE_TAG)
 
-.PHONY: security-proxy-image
-security-proxy-image: ## Build the security proxy image to be loaded in your kind cluster.
-	cd ../experimental && $(MAKE) docker-image -e IMG=quay.io/kuberay/security-proxy:$(SECURITY_IMAGE_TAG)
 
 .PHONY: deploy-operator
 deploy-operator: ## Deploy operator via helm into the K8s cluster specified in ~/.kube/config.

apiserver/SecuringImplementation.md

@@ -17,10 +17,6 @@ There is also a wealth of open-source implementations, for example,
 [oauth2-proxy](https://github.com/oauth2-proxy/oauth2-proxy) and many commercial
 offerings.
 
-## Basic token-based authentication reverse proxy
-
-A simple token-based authentication reverse proxy [implementation](../experimental/cmd/main.go) is provided in the project, along with make targets to build a Docker image for it and push this image to the image repository. There is also a pre-built image `kuberay/security-proxy:nightly` that you can use for experimenting with security.
-
 ## Installation
 
 ### Set `security.proxy.tag`

apiserversdk/Makefile

@@ -43,7 +43,7 @@ test: envtest ## Run tests.
 ENVTEST = $(LOCALBIN)/setup-envtest
 $(ENVTEST): $(LOCALBIN)
 envtest: $(ENVTEST) ## Download envtest locally if necessary.
-	test -s $(ENVTEST) || GOBIN=$(LOCALBIN) go install sigs.k8s.io/controller-runtime/tools/setup-envtest@release-0.23
+	test -s $(ENVTEST) || GOBIN=$(LOCALBIN) go install sigs.k8s.io/controller-runtime/tools/setup-envtest@release-0.24
 
 .PHONY: clean
 clean:

ci/kind-config-buildkite.yml

@@ -9,7 +9,7 @@ networking:
 # https://blog.scottlowe.org/2019/07/30/adding-a-name-to-kubernetes-api-server-certificate/
 nodes:
 - role: control-plane
-  image: kindest/node:v1.29.0
+  image: kindest/node:v1.35.0
   kubeadmConfigPatches:
   - |
     kind: ClusterConfiguration

clients/python-client/poetry.lock

@@ -1,4 +1,4 @@
-# This file is automatically @generated by Poetry 2.2.1 and should not be changed by hand.
+# This file is automatically @generated by Poetry 2.4.1 and should not be changed by hand.
 
 [[package]]
 name = "cachetools"
@@ -167,18 +167,18 @@ urllib3 = ["packaging", "urllib3"]
 
 [[package]]
 name = "idna"
-version = "3.10"
+version = "3.15"
 description = "Internationalized Domain Names in Applications (IDNA)"
 optional = false
-python-versions = ">=3.6"
+python-versions = ">=3.8"
 groups = ["main"]
 files = [
-    {file = "idna-3.10-py3-none-any.whl", hash = "sha256:946d195a0d259cbba61165e88e65941f16e9b36ea6ddb97f00452bae8b1287d3"},
-    {file = "idna-3.10.tar.gz", hash = "sha256:12f65c9b470abda6dc35cf8e63cc574b1c52b11df2c86030af0ac09b01b13ea9"},
+    {file = "idna-3.15-py3-none-any.whl", hash = "sha256:048adeaf8c2d788c40fee287673ccaa74c24ffd8dcf09ffa555a2fbb59f10ac8"},
+    {file = "idna-3.15.tar.gz", hash = "sha256:ca962446ea538f7092a95e057da437618e886f4d349216d2b1e294abfdb65fdc"},
 ]
 
 [package.extras]
-all = ["flake8 (>=7.1.1)", "mypy (>=1.11.2)", "pytest (>=8.3.2)", "ruff (>=0.6.2)"]
+all = ["mypy (>=1.11.2)", "pytest (>=8.3.2)", "ruff (>=0.6.2)"]
 
 [[package]]
 name = "kubernetes"
@@ -193,7 +193,7 @@ files = [
 ]
 
 [package.dependencies]
-certifi = ">=14.05.14"
+certifi = ">=14.5.14"
 durationpy = ">=0.7"
 google-auth = ">=1.0.1"
 oauthlib = ">=3.2.2"

dashboard/package.json

@@ -17,7 +17,7 @@
     "@mui/joy": "^5.0.0-beta.32",
     "@mui/material": "^5.15.14",
     "dayjs": "^1.11.10",
-    "next": "15.4.10",
+    "next": "15.5.18",
     "react": "^18",
     "react-dom": "^18",
     "swr": "^2.2.5"

dashboard/yarn.lock

@@ -1000,10 +1000,10 @@ __metadata:
   languageName: node
   linkType: hard
 
-"@next/env@npm:15.4.10":
-  version: 15.4.10
-  resolution: "@next/env@npm:15.4.10"
-  checksum: 10c0/f455f9630f56691800441333bf1462135f64eba65cc81a34fceedb42fcc62fd22fb2dfaa8de49f65b378fc46c5cf35795c0a32363006b989806223856be221fa
+"@next/env@npm:15.5.18":
+  version: 15.5.18
+  resolution: "@next/env@npm:15.5.18"
+  checksum: 10c0/cfb628f88b903b2593a778658abd2eed8a35ca91807ef19c08461543ed3ba9143e6f0ea4f29b98693f806f9498d1e6ca7c1efd1e0c3a0595991e2bac202e545a
   languageName: node
   linkType: hard
 
@@ -1016,58 +1016,58 @@ __metadata:
   languageName: node
   linkType: hard
 
-"@next/swc-darwin-arm64@npm:15.4.8":
-  version: 15.4.8
-  resolution: "@next/swc-darwin-arm64@npm:15.4.8"
+"@next/swc-darwin-arm64@npm:15.5.18":
+  version: 15.5.18
+  resolution: "@next/swc-darwin-arm64@npm:15.5.18"
   conditions: os=darwin & cpu=arm64
   languageName: node
   linkType: hard
 
-"@next/swc-darwin-x64@npm:15.4.8":
-  version: 15.4.8
-  resolution: "@next/swc-darwin-x64@npm:15.4.8"
+"@next/swc-darwin-x64@npm:15.5.18":
+  version: 15.5.18
+  resolution: "@next/swc-darwin-x64@npm:15.5.18"
   conditions: os=darwin & cpu=x64
   languageName: node
   linkType: hard
 
-"@next/swc-linux-arm64-gnu@npm:15.4.8":
-  version: 15.4.8
-  resolution: "@next/swc-linux-arm64-gnu@npm:15.4.8"
+"@next/swc-linux-arm64-gnu@npm:15.5.18":
+  version: 15.5.18
+  resolution: "@next/swc-linux-arm64-gnu@npm:15.5.18"
   conditions: os=linux & cpu=arm64 & libc=glibc
   languageName: node
   linkType: hard
 
-"@next/swc-linux-arm64-musl@npm:15.4.8":
-  version: 15.4.8
-  resolution: "@next/swc-linux-arm64-musl@npm:15.4.8"
+"@next/swc-linux-arm64-musl@npm:15.5.18":
+  version: 15.5.18
+  resolution: "@next/swc-linux-arm64-musl@npm:15.5.18"
   conditions: os=linux & cpu=arm64 & libc=musl
   languageName: node
   linkType: hard
 
-"@next/swc-linux-x64-gnu@npm:15.4.8":
-  version: 15.4.8
-  resolution: "@next/swc-linux-x64-gnu@npm:15.4.8"
+"@next/swc-linux-x64-gnu@npm:15.5.18":
+  version: 15.5.18
+  resolution: "@next/swc-linux-x64-gnu@npm:15.5.18"
   conditions: os=linux & cpu=x64 & libc=glibc
   languageName: node
   linkType: hard
 
-"@next/swc-linux-x64-musl@npm:15.4.8":
-  version: 15.4.8
-  resolution: "@next/swc-linux-x64-musl@npm:15.4.8"
+"@next/swc-linux-x64-musl@npm:15.5.18":
+  version: 15.5.18
+  resolution: "@next/swc-linux-x64-musl@npm:15.5.18"
   conditions: os=linux & cpu=x64 & libc=musl
   languageName: node
   linkType: hard
 
-"@next/swc-win32-arm64-msvc@npm:15.4.8":
-  version: 15.4.8
-  resolution: "@next/swc-win32-arm64-msvc@npm:15.4.8"
+"@next/swc-win32-arm64-msvc@npm:15.5.18":
+  version: 15.5.18
+  resolution: "@next/swc-win32-arm64-msvc@npm:15.5.18"
   conditions: os=win32 & cpu=arm64
   languageName: node
   linkType: hard
 
-"@next/swc-win32-x64-msvc@npm:15.4.8":
-  version: 15.4.8
-  resolution: "@next/swc-win32-x64-msvc@npm:15.4.8"
+"@next/swc-win32-x64-msvc@npm:15.5.18":
+  version: 15.5.18
+  resolution: "@next/swc-win32-x64-msvc@npm:15.5.18"
   conditions: os=win32 & cpu=x64
   languageName: node
   linkType: hard
@@ -3357,9 +3357,9 @@ __metadata:
   linkType: hard
 
 "ip-address@npm:^10.0.1":
-  version: 10.0.1
-  resolution: "ip-address@npm:10.0.1"
-  checksum: 10c0/1634d79dae18394004775cb6d699dc46b7c23df6d2083164025a2b15240c1164fccde53d0e08bd5ee4fc53913d033ab6b5e395a809ad4b956a940c446e948843
+  version: 10.2.0
+  resolution: "ip-address@npm:10.2.0"
+  checksum: 10c0/5a00aada6e922c9c69dfc800ed5d0fa3348675ebdeed0e1575f503f27ca385b5f534363c9af7ad1daf64c1f1409388cdd3cc2e9b9b0fe1c924a431378d55075a
   languageName: node
   linkType: hard
 
@@ -4101,19 +4101,19 @@ __metadata:
   languageName: node
   linkType: hard
 
-"next@npm:15.4.10":
-  version: 15.4.10
-  resolution: "next@npm:15.4.10"
+"next@npm:15.5.18":
+  version: 15.5.18
+  resolution: "next@npm:15.5.18"
   dependencies:
-    "@next/env": "npm:15.4.10"
-    "@next/swc-darwin-arm64": "npm:15.4.8"
-    "@next/swc-darwin-x64": "npm:15.4.8"
-    "@next/swc-linux-arm64-gnu": "npm:15.4.8"
-    "@next/swc-linux-arm64-musl": "npm:15.4.8"
-    "@next/swc-linux-x64-gnu": "npm:15.4.8"
-    "@next/swc-linux-x64-musl": "npm:15.4.8"
-    "@next/swc-win32-arm64-msvc": "npm:15.4.8"
-    "@next/swc-win32-x64-msvc": "npm:15.4.8"
+    "@next/env": "npm:15.5.18"
+    "@next/swc-darwin-arm64": "npm:15.5.18"
+    "@next/swc-darwin-x64": "npm:15.5.18"
+    "@next/swc-linux-arm64-gnu": "npm:15.5.18"
+    "@next/swc-linux-arm64-musl": "npm:15.5.18"
+    "@next/swc-linux-x64-gnu": "npm:15.5.18"
+    "@next/swc-linux-x64-musl": "npm:15.5.18"
+    "@next/swc-win32-arm64-msvc": "npm:15.5.18"
+    "@next/swc-win32-x64-msvc": "npm:15.5.18"
     "@swc/helpers": "npm:0.5.15"
     caniuse-lite: "npm:^1.0.30001579"
     postcss: "npm:8.4.31"
@@ -4156,7 +4156,7 @@ __metadata:
       optional: true
   bin:
     next: dist/bin/next
-  checksum: 10c0/67101363146d48f6e532b7b4d15df6d6a02601782037e4f9013f620a3cb046231a0db986bd983e1726b95aa30d10d02f673f9f123188582fe7c42d1fe94d74d9
+  checksum: 10c0/94c3be1ee04240913ab9d46e70fde3b26a73cedf6c76e946cfd2580d7ddd494b7da66260fe3a9a2be187652f5a10b1d9a6a555f10744d350be7f6ae05157d893
   languageName: node
   linkType: hard
 
@@ -4677,7 +4677,7 @@ __metadata:
     dayjs: "npm:^1.11.10"
     eslint: "npm:^9.33.0"
     eslint-config-next: "npm:^15.4.6"
-    next: "npm:15.4.10"
+    next: "npm:15.5.18"
     postcss: "npm:^8"
     prettier: "npm:3.6.2"
     react: "npm:^18"

docs/reference/api.md

@@ -75,6 +75,8 @@ _Appears in:_
 | `env` _[EnvVar](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.28/#envvar-v1-core) array_ | Optional list of environment variables to set in the autoscaler container. |  |  |
 | `envFrom` _[EnvFromSource](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.28/#envfromsource-v1-core) array_ | Optional list of sources to populate environment variables in the autoscaler container. |  |  |
 | `volumeMounts` _[VolumeMount](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.28/#volumemount-v1-core) array_ | Optional list of volumeMounts.  This is needed for enabling TLS for the autoscaler container. |  |  |
+| `command` _string array_ | Optional list overwrite the default command of the autoscaler container. |  |  |
+| `args` _string array_ | Optional to overwrite the default args of the autoscaler container. |  |  |
 
 
 #### AutoscalerVersion
@@ -285,6 +287,63 @@ _Appears in:_
 | `SidecarMode` |  |
 
 
+#### NetworkIsolationConfig
+
+
+
+NetworkIsolationConfig defines network isolation settings for Ray cluster.
+All modes permit intra-cluster pod-to-pod traffic.
+DNS egress is not included automatically; see NetworkPolicyRules.EgressRules
+for why it must be added under DenyAll/DenyAllEgress.
+
+
+
+_Appears in:_
+- [RayClusterSpec](#rayclusterspec)
+
+| Field | Description | Default | Validation |
+| --- | --- | --- | --- |
+| `mode` _[NetworkIsolationMode](#networkisolationmode)_ | Mode controls the security level. All modes permit intra-cluster pod-to-pod<br />traffic (DNS egress excluded, see EgressRules).<br />- "DenyAll": Denies all Ingress and Egress.<br />- "DenyAllIngress": Denies all Ingress.<br />- "DenyAllEgress": Denies all Egress. | DenyAll | Enum: [DenyAll DenyAllIngress DenyAllEgress] <br /> |
+| `head` _[NetworkPolicyRules](#networkpolicyrules)_ | Head specifies custom NetworkPolicy rules applied only to the head pod's policy.<br />The base head policy always allows intra-cluster traffic and (for K8sJobMode<br />RayJob-owned clusters) the submitter pod. Rules here are appended to those<br />base rules. Platforms that need operator dashboard access should add it here<br />(e.g. via a mutating webhook). |  |  |
+| `worker` _[NetworkPolicyRules](#networkpolicyrules)_ | Worker specifies custom NetworkPolicy rules applied only to worker pods' policy.<br />The base worker policy always allows intra-cluster traffic.<br />Rules here are appended to that base rule. |  |  |
+
+
+#### NetworkIsolationMode
+
+_Underlying type:_ _string_
+
+NetworkIsolationMode is the type for network isolation mode constants.
+
+_Validation:_
+- Enum: [DenyAll DenyAllIngress DenyAllEgress]
+
+_Appears in:_
+- [NetworkIsolationConfig](#networkisolationconfig)
+
+| Field | Description |
+| --- | --- |
+| `DenyAll` | NetworkIsolationDenyAll denies all ingress and egress traffic.<br /> |
+| `DenyAllIngress` | NetworkIsolationDenyAllIngress denies all ingress traffic.<br /> |
+| `DenyAllEgress` | NetworkIsolationDenyAllEgress denies all egress traffic.<br /> |
+
+
+#### NetworkPolicyRules
+
+
+
+NetworkPolicyRules defines custom ingress and egress rules for a NetworkPolicy.
+
+
+
+_Appears in:_
+- [NetworkIsolationConfig](#networkisolationconfig)
+
+| Field | Description | Default | Validation |
+| --- | --- | --- | --- |
+| `ingressRules` _[NetworkPolicyIngressRule](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.28/#networkpolicyingressrule-v1-networking) array_ | IngressRules specifies custom ingress rules appended to the base policy.<br />Only meaningful when the mode includes ingress denial (DenyAll or DenyAllIngress). |  |  |
+| `egressRules` _[NetworkPolicyEgressRule](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.28/#networkpolicyegressrule-v1-networking) array_ | EgressRules specifies custom egress rules appended to the base policy.<br />Only meaningful when the mode includes egress denial (DenyAll or DenyAllEgress).<br />DNS egress is NOT added automatically: under DenyAll/DenyAllEgress you MUST<br />add a DNS rule here (e.g. to kube-system pods labeled k8s-app=kube-dns on<br />port 53), because Ray workers reach the head via its service FQDN and cannot<br />resolve it without DNS. See the network-isolation-deny-all sample. |  |  |
+
+
 #### RayCluster
 
 
@@ -328,6 +387,7 @@ _Appears in:_
 | `headServiceAnnotations` _object (keys:string, values:string)_ |  |  |  |
 | `enableInTreeAutoscaling` _boolean_ | EnableInTreeAutoscaling indicates whether operator should create in tree autoscaling configs |  |  |
 | `gcsFaultToleranceOptions` _[GcsFaultToleranceOptions](#gcsfaulttoleranceoptions)_ | GcsFaultToleranceOptions for enabling GCS FT |  |  |
+| `networkIsolation` _[NetworkIsolationConfig](#networkisolationconfig)_ | NetworkIsolation specifies optional configuration for network isolation.<br />When set, separate NetworkPolicies are created for head and worker pods.<br />The reconciler always permits intra-cluster pod-to-pod traffic.<br />Note: under DenyAll/DenyAllEgress, DNS egress is not added<br />automatically; since Ray pods reach the head via its service FQDN, you must<br />allow DNS egress via Head/Worker EgressRules or the cluster will fail to start. |  |  |
 | `headGroupSpec` _[HeadGroupSpec](#headgroupspec)_ | HeadGroupSpec is the spec for the head pod |  |  |
 | `rayVersion` _string_ | RayVersion is used to determine the command for the Kubernetes Job managed by RayJob |  |  |
 | `workerGroupSpecs` _[WorkerGroupSpec](#workergroupspec) array_ | WorkerGroupSpecs are the specs for the worker pods |  |  |
@@ -400,6 +460,7 @@ _Appears in:_
 | --- | --- | --- | --- |
 | `jobTemplate` _[RayJobSpec](#rayjobspec)_ | JobTemplate defines the job spec that will be created by cron scheduling |  |  |
 | `schedule` _string_ | Schedule is the cron schedule string |  |  |
+| `timeZone` _string_ | TimeZone is the time zone name for the given schedule. If not specified, default to the local time zone of the<br />Kuberay Operator. Empty string is not allowed.<br />The bundled version of the time zone database is used. |  | MinLength: 1 <br /> |
 | `suspend` _boolean_ | Suspend tells the controller to suspend the scheduling, it does not apply to<br />scheduled RayJob. |  |  |
 
 
@@ -505,6 +566,7 @@ _Appears in:_
 | `serveConfigV2` _string_ | Important: Run "make" to regenerate code after modifying this file<br />Defines the applications and deployments to deploy, should be a YAML multi-line scalar string. |  |  |
 | `rayClusterConfig` _[RayClusterSpec](#rayclusterspec)_ |  |  |  |
 | `excludeHeadPodFromServeSvc` _boolean_ | If the field is set to true, the value of the label `ray.io/serve` on the head Pod should always be false.<br />Therefore, the head Pod's endpoint will not be added to the Kubernetes Serve service. |  |  |
+| `suspend` _boolean_ | Suspend indicates whether the RayService should suspend its execution. When set to true,<br />all Kubernetes resources owned by the RayService controller will be deleted. Setting it<br />back to false will allow the RayService controller to recreate the resources. |  |  |
 
 
 
@@ -590,7 +652,7 @@ _Appears in:_
 
 | Field | Description | Default | Validation |
 | --- | --- | --- | --- |
-| `backoffLimit` _integer_ | BackoffLimit of the submitter k8s job. |  |  |
+| `backoffLimit` _integer_ | BackoffLimit of the submitter. In K8sJobMode, this is the K8s Job backoffLimit.<br />In SidecarMode with SidecarSubmitterRestart enabled, this is the maximum container restart count. |  |  |
 
 
 #### UpscalingMode

experimental/Dockerfile

@@ -1,27 +0,0 @@
-# Build security proxy
-FROM golang:1.26-bookworm AS builder
-
-WORKDIR /workspace
-# Copy the Go Modules manifests
-COPY go.mod go.sum ./
-COPY ray-operator/go.mod ray-operator/go.mod
-COPY ray-operator/go.sum ray-operator/go.sum
-
-# cache deps before building and copying source so that we don't need to re-download as much
-# and so that source changes don't invalidate our downloaded layer
-RUN go mod download
-
-# Copy the go source
-COPY experimental/ experimental/
-
-WORKDIR /workspace/experimental
-
-# Build
-RUN CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -a -o security_proxy cmd/main.go
-
-FROM scratch
-WORKDIR /workspace
-
-COPY --from=builder /workspace/experimental/security_proxy /usr/local/bin/security_proxy
-
-ENTRYPOINT ["/usr/local/bin/security_proxy"]

experimental/Dockerfile.buildx

@@ -1,7 +0,0 @@
-FROM scratch
-ARG TARGETARCH
-WORKDIR /workspace
-COPY ./experimental/security_proxy-${TARGETARCH} /usr/local/bin/security_proxy
-USER 65532:65532
-
-ENTRYPOINT ["/usr/local/bin/security_proxy"]

experimental/Makefile

@@ -1,120 +0,0 @@
-BUILD_TIME      := $(shell date "+%F %T")
-COMMIT_SHA1     := $(shell git rev-parse HEAD )
-REPO_ROOT	    := $(shell dirname ${PWD})
-REPO_ROOT_BIN	:= $(REPO_ROOT)/bin
-
-# Image URL to use all building/pushing image targets
-IMG_TAG ?=latest
-IMG ?= kuberay/security-proxy:$(IMG_TAG)
-
-# Get the currently used golang install path (in GOPATH/bin, unless GOBIN is set)
-ifeq (,$(shell go env GOBIN))
-GOBIN=$(shell go env GOPATH)/bin
-else
-GOBIN=$(shell go env GOBIN)
-endif
-
-# Setting SHELL to bash allows bash commands to be executed by recipes.
-# This is a requirement for 'setup-envtest.sh' in the test target.
-# Options are set to exit when a recipe line exits non-zero or a piped command fails.
-SHELL = /usr/bin/env bash -o pipefail
-.SHELLFLAGS = -ec
-
-# Container Engine to be used for building images
-ENGINE ?= docker
-
-all: build
-
-##@ General
-
-# The help target prints out all targets with their descriptions organized
-# beneath their categories. The categories are represented by '##@' and the
-# target descriptions by '##'. The awk commands is responsible for reading the
-# entire set of makefiles included in this invocation, looking for lines of the
-# file as xyz: ## something, and then pretty-format the target and help. Then,
-# if there's a line with ##@ something, that gets pretty-printed as a category.
-# More info on the usage of ANSI control characters for terminal formatting:
-# https://en.wikipedia.org/wiki/ANSI_escape_code#SGR_parameters
-# More info on the awk command:
-# http://linuxcommand.org/lc3_adv_awk.php
-
-help: ## Display this help.
-	@awk 'BEGIN {FS = ":.*##"; printf "\nUsage:\n  make \033[36m<target>\033[0m\n"} /^[a-zA-Z_0-9-]+:.*?##/ { printf "  \033[36m%-20s\033[0m %s\n", $$1, $$2 } /^##@/ { printf "\n\033[1m%s\033[0m\n", substr($$0, 5) } ' $(MAKEFILE_LIST)
-
-##@ Development
-
-.PHONY:
-fmt: ## Run go fmt against code.
-	go fmt ./...
-
-.PHONY: vet
-vet: ## Run go vet against code.
-	go vet ./...
-
-.PHONY: fumpt
-fumpt: gofumpt ## Run gofmtumpt against code.
-	$(GOFUMPT) -l -w .
-
-.PHONY: lint
-lint: golangci-lint fmt vet fumpt ## Run the linter.
-	$(GOLANGCI_LINT) run  --timeout=3m
-
-build: fmt vet fumpt lint ## Build api server binary.
-	go build  -o ${REPO_ROOT_BIN}/kuberay-apiserver-proxy cmd/main.go
-
-##@ Docker Build
-
-docker-image: ## Build image for the security proxy.
-	$(ENGINE) build -t ${IMG} -f Dockerfile ..
-
-docker-push: ## Push image for the api server.
-	$(ENGINE) push ${IMG}
-
-docker-multi-arch-build: ## Build multi arch image, amd64 and arm64 currently.
-	$(ENGINE) buildx build --platform linux/amd64,linux/arm64 -t ${IMG} -f Dockerfile.buildx ..
-
-docker-multi-arch-push: ## Build and Push multi arch image, amd64 and arm64 currently.
-	$(ENGINE) buildx build --push --platform linux/amd64,linux/arm64 -t ${IMG} -f Dockerfile.buildx ..
-
-##@ Development Tools Setup
-
-## Location to install dependencies to
-$(REPO_ROOT_BIN):
-	mkdir -p $(REPO_ROOT_BIN)
-
-## Tool Binaries
-GOFUMPT ?= $(REPO_ROOT_BIN)/gofumpt
-GOLANGCI_LINT ?= $(REPO_ROOT_BIN)/golangci-lint
-GOBINDATA ?= $(REPO_ROOT_BIN)/go-bindata
-
-
-## Tool Versions
-GOFUMPT_VERSION ?= v0.3.1
-GOIMPORTS_VERSION ?= v0.14.0
-GOLANGCI_LINT_VERSION ?= v2.11.4
-KIND_VERSION ?= v0.19.0
-GOBINDATA_VERSION ?= v4.0.2
-
-.PHONY: gofumpt
-gofumpt: $(GOFUMPT) ## Download gofumpt locally if necessary.
-$(GOFUMPT): $(REPO_ROOT_BIN)
-	test -s $(GOFUMPT) || GOBIN=$(REPO_ROOT_BIN) go install mvdan.cc/gofumpt@$(GOFUMPT_VERSION)
-
-.PHONY: golangci-lint
-golangci-lint: $(GOLANGCI_LINT) ## Download golangci_lint locally if necessary.
-$(GOLANGCI_LINT): $(REPO_ROOT_BIN)
-	test -s $(GOLANGCI_LINT) || (curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | bash -s -- -b $(REPO_ROOT_BIN)/ $(GOLANGCI_LINT_VERSION))
-
-.PHONY: go-bindata
-go-bindata: $(GOBINDATA) ## Download the go-bindata executable if necessary.
-$(GOBINDATA): $(REPO_ROOT_BIN)
-	test -s $(GOBINDATA) || GOBIN=$(REPO_ROOT_BIN) go install github.com/kevinburke/go-bindata/v4/...@$(GOBINDATA_VERSION)
-
-.PHONY: dev-tools
-dev-tools: golangci-lint gofumpt go-bindata ## Install all development tools
-
-.PHONY: clean-dev-tools
-clean-dev-tools: ## Remove all development tools
-	rm -f $(REPO_ROOT_BIN)/golangci-lint
-	rm -f $(REPO_ROOT_BIN)/gofumpt
-	rm -f $(REPO_ROOT_BIN)/go-bindata

experimental/cmd/main.go

@@ -1,123 +0,0 @@
-package main
-
-import (
-	"context"
-	"net"
-	"net/http"
-	"net/http/httputil"
-	"net/url"
-	"os"
-	"strings"
-
-	"google.golang.org/grpc"
-	"google.golang.org/grpc/credentials/insecure"
-	"google.golang.org/grpc/metadata"
-	"k8s.io/klog/v2"
-
-	"github.com/ray-project/kuberay/experimental/pkg/grpcproxy"
-	"github.com/ray-project/kuberay/experimental/pkg/httpproxy"
-)
-
-func main() {
-	// Ports
-	httpRemotePort := os.Getenv("HTTP_REMOTE_PORT")
-	httpLocalPort := os.Getenv("HTTP_LOCAL_PORT")
-	grpcRemotePort := os.Getenv("GRPC_REMOTE_PORT")
-	grpcLocalPort := os.Getenv("GRPC_LOCAL_PORT")
-	// Uncomment below for local testing
-	/*	httpRemotePort := "9091"
-		httpLocalPort := "9090"
-		grpcRemotePort := "9091"
-		grpcLocalPort := "9090" */
-
-	// Security info
-	securePrefix := os.Getenv("SECURITY_PREFIX")
-	securityToken := os.Getenv("SECURITY_TOKEN")
-	// Uncomment below for local testing
-	/*	securePrefix := "/" // Only used for HTTP to define secure prefix
-		securityToken := "12345"*/
-
-	// Enabling GRPC
-	enableGrpc := strings.ToLower(os.Getenv("ENABLE_GRPC")) == "true"
-	// Uncomment below for local testing
-	//	enableGrpc := true
-
-	if httpRemotePort == "" || httpLocalPort == "" || securePrefix == "" || securityToken == "" {
-		klog.Fatal("Failing to get execution parameters - http remote port: ", httpRemotePort, " http local port: ", httpLocalPort, " security prefix: ", securePrefix)
-	}
-	if enableGrpc && (grpcRemotePort == "" || grpcLocalPort == "") {
-		klog.Fatal("Failing to get execution parameters - grpc remote port: ", grpcRemotePort, " grpc local port: ", grpcLocalPort)
-	}
-	klog.Info("Starting reverse proxy with parameters - http remote port: ", httpRemotePort, " http local port: ", httpLocalPort, " security prefix: ", securePrefix)
-	if enableGrpc {
-		klog.Info(" grpc remote port: ", grpcRemotePort, " grpc local port: ", grpcLocalPort)
-	}
-
-	// GRPC proxy
-	if enableGrpc {
-		go func() {
-			remoteURL := "http://localhost:" + httpRemotePort
-			// Client connection
-			cc, err := grpc.NewClient(remoteURL, grpc.WithTransportCredentials(insecure.NewCredentials()))
-			if err != nil {
-				klog.Fatal("cannot dial server: ", err)
-			}
-			defer cc.Close()
-			klog.Info("connecting to GRPC server ", cc.Target())
-
-			// set up listener
-			lc := net.ListenConfig{}
-			lis, err := lc.Listen(context.Background(), "tcp", "localhost:"+grpcLocalPort)
-			if err != nil {
-				klog.Fatal("failed to listen: ", err)
-			}
-			klog.Info("GRPC listening on ", lis.Addr())
-
-			// Director function
-			directorFn := func(ctx context.Context, _ string) (context.Context, *grpc.ClientConn, error) {
-				md, _ := metadata.FromIncomingContext(ctx)
-				outCtx := metadata.NewOutgoingContext(ctx, md.Copy())
-				return outCtx, cc, nil
-			}
-
-			// Set up the proxy server and then serve from it.
-			sh, h := grpcproxy.TransparentHandler(directorFn)
-			proxySrv := grpc.NewServer(grpc.UnknownServiceHandler(sh))
-			// Add security header checking
-			h.AddSecurityHeaderToHandler(map[string]string{"Authorization": securityToken})
-			klog.Info("Security header added ")
-
-			// run GRPC proxy backend
-			klog.Info("Running GRPC proxy")
-			if err := proxySrv.Serve(lis); err != nil {
-				klog.Fatal("server stopped unexpectedly")
-			}
-		}()
-	}
-
-	// HTTP proxy. Start it as a goroutine so that we can start GRPC one as well
-	remoteURL := "http://localhost:" + httpRemotePort
-	remote, err := url.Parse(remoteURL)
-	if err != nil {
-		klog.Fatal("Failed to parse remote url ", remoteURL, " - error: ", err)
-	}
-
-	// Create reverse proxy
-	proxy := httputil.NewSingleHostReverseProxy(remote) //nolint:gosec // URL is constructed from localhost + config port, not user input
-	klog.Info("Connected to remote HTTP ", remoteURL)
-	// Create token authorization
-	token := httpproxy.NewTokenAuth(securityToken, proxy, securePrefix, remote)
-	// Create handler
-	http.HandleFunc("/", token.AuthFunc())
-	// Run HTTP proxy with timeouts
-	server := &http.Server{
-		Addr:         ":" + httpLocalPort,
-		ReadTimeout:  0, // No timeout
-		WriteTimeout: 0, // No timeout
-		IdleTimeout:  0, // No timeout
-	}
-	err = server.ListenAndServe()
-	if err != nil {
-		klog.Fatal("HTTP server died unexpectedly, error - ", err)
-	}
-}

experimental/pkg/grpcproxy/director.go

@@ -1,22 +0,0 @@
-package grpcproxy
-
-import (
-	"context"
-
-	"google.golang.org/grpc"
-)
-
-// StreamDirector returns a gRPC ClientConn to be used to forward the call to.
-//
-// The presence of the `Context` allows for rich filtering, e.g. based on Metadata (headers).
-// If no handling is meant to be done, a `codes.NotImplemented` gRPC error should be returned.
-//
-// The context returned from this function should be the context for the *outgoing* (to backend) call. In case you want
-// to forward any Metadata between the inbound request and outbound requests, you should do it manually. However, you
-// *must* propagate the cancel function (`context.WithCancel`) of the inbound context to the one returned.
-//
-// It is worth noting that the StreamDirector will be fired *after* all server-side stream interceptors
-// are invoked. So decisions around authorization, monitoring etc. are better to be handled there.
-//
-// See the rather rich example.
-type StreamDirector func(ctx context.Context, fullMethodName string) (context.Context, *grpc.ClientConn, error)

experimental/pkg/grpcproxy/handler.go

@@ -1,188 +0,0 @@
-package grpcproxy
-
-import (
-	"context"
-	"errors"
-	"io"
-	"strings"
-
-	"google.golang.org/grpc"
-	"google.golang.org/grpc/codes"
-	"google.golang.org/grpc/metadata"
-	"google.golang.org/grpc/status"
-	"google.golang.org/protobuf/types/known/emptypb"
-	"k8s.io/klog/v2"
-)
-
-var clientStreamDescForProxying = &grpc.StreamDesc{
-	ServerStreams: true,
-	ClientStreams: true,
-}
-
-// RegisterService sets up a proxy handler for a particular gRPC service and method.
-// The behavior is the same as if you were registering a handler method, e.g. from a generated pb.go file.
-func RegisterService(server *grpc.Server, director StreamDirector, serviceName string, methodNames ...string) *Handler {
-	streamer := &Handler{director: director, securityheader: nil}
-	fakeDesc := &grpc.ServiceDesc{
-		ServiceName: serviceName,
-		HandlerType: (*any)(nil),
-	}
-	for _, m := range methodNames {
-		streamDesc := grpc.StreamDesc{
-			StreamName:    m,
-			Handler:       streamer.handler,
-			ServerStreams: true,
-			ClientStreams: true,
-		}
-		fakeDesc.Streams = append(fakeDesc.Streams, streamDesc)
-	}
-	server.RegisterService(fakeDesc, streamer)
-	return streamer
-}
-
-// TransparentHandler returns a Handler that attempts to proxy all requests that are not registered in the server.
-// The indented use here is as a transparent proxy, where the server doesn't know about the services implemented by the
-// backends. It should be used as a `grpc.UnknownServiceHandler`.
-func TransparentHandler(director StreamDirector) (grpc.StreamHandler, *Handler) {
-	streamer := &Handler{director: director, securityheader: nil}
-	return streamer.handler, streamer
-}
-
-type Handler struct {
-	director       StreamDirector
-	securityheader map[string]string
-}
-
-// Adding security header handler
-func (s *Handler) AddSecurityHeaderToHandler(securityheader map[string]string) {
-	s.securityheader = securityheader
-}
-
-// handler is where the real magic of proxying happens.
-// It is invoked like any gRPC server stream and uses the emptypb.Empty type server
-// to proxy calls between the input and output streams.
-func (s *Handler) handler(_ any, serverStream grpc.ServerStream) error {
-	// little bit of gRPC internals never hurt anyone
-	fullMethodName, ok := grpc.MethodFromServerStream(serverStream)
-	if !ok {
-		return status.Errorf(codes.Internal, "lowLevelServerStream not exists in context")
-	}
-	if s.securityheader != nil {
-		header, ok := metadata.FromIncomingContext(serverStream.Context())
-		if !ok {
-			return status.Errorf(codes.Internal, "lowLevelServerStream not exists in context")
-		}
-		for headerName, headerValue := range s.securityheader {
-			hName := strings.ToLower(headerName)
-			if v, exists := header[hName]; exists {
-				// Authentication header exists
-				if v[0] != strings.ToLower(headerValue) {
-					return status.Error(codes.Unauthenticated, "Request unauthorized")
-				}
-			} else {
-				// Authentication header does not exist
-				return status.Error(codes.Unauthenticated, "Request unauthorized")
-			}
-		}
-	}
-
-	// We require that the director's returned context inherits from the serverStream.Context().
-	outgoingCtx, backendConn, err := s.director(serverStream.Context(), fullMethodName)
-	if err != nil {
-		return err
-	}
-
-	clientCtx, clientCancel := context.WithCancel(outgoingCtx)
-	defer clientCancel()
-	// TODO(mwitkow): Add a `forwarded` header to metadata, https://en.wikipedia.org/wiki/X-Forwarded-For.
-	clientStream, err := grpc.NewClientStream(clientCtx, clientStreamDescForProxying, backendConn, fullMethodName)
-	if err != nil {
-		return err
-	}
-	// Explicitly *do not close* s2cErrChan and c2sErrChan, otherwise the select below will not terminate.
-	// Channels do not have to be closed, it is just a control flow mechanism, see
-	// https://groups.google.com/forum/#!msg/golang-nuts/pZwdYRGxCIk/qpbHxRRPJdUJ
-	s2cErrChan := s.forwardServerToClient(serverStream, clientStream)
-	c2sErrChan := s.forwardClientToServer(clientStream, serverStream)
-	// We don't know which side is going to stop sending first, so we need a select between the two.
-	for range 2 {
-		select {
-		case s2cErr := <-s2cErrChan:
-			if errors.Is(s2cErr, io.EOF) {
-				// this is the happy case where the sender has encountered io.EOF, and won't be sending anymore./
-				// the clientStream>serverStream may continue pumping though.
-				err := clientStream.CloseSend()
-				if err != nil {
-					klog.Info("send direction of the stream closing error ", err)
-				}
-			} else {
-				// however, we may have gotten a receive error (stream disconnected, a read error etc) in which case we need
-				// to cancel the clientStream to the backend, let all of its goroutines be freed up by the CancelFunc and
-				// exit with an error to the stack
-				clientCancel()
-				return status.Errorf(codes.Internal, "failed proxying s2c: %v", s2cErr)
-			}
-		case c2sErr := <-c2sErrChan:
-			// This happens when the clientStream has nothing else to offer (io.EOF), returned a gRPC error. In those two
-			// cases we may have received Trailers as part of the call. In case of other errors (stream closed) the trailers
-			// will be nil.
-			serverStream.SetTrailer(clientStream.Trailer())
-			// c2sErr will contain RPC error from client code. If not io.EOF return the RPC error as server stream error.
-			if !errors.Is(c2sErr, io.EOF) {
-				return c2sErr
-			}
-			return nil
-		}
-	}
-	return status.Errorf(codes.Internal, "gRPC proxying should never reach this stage.")
-}
-
-func (s *Handler) forwardClientToServer(src grpc.ClientStream, dst grpc.ServerStream) chan error {
-	ret := make(chan error, 1)
-	go func() {
-		f := &emptypb.Empty{}
-		for i := 0; ; i++ {
-			if err := src.RecvMsg(f); err != nil {
-				ret <- err // this can be io.EOF which is happy case
-				break
-			}
-			if i == 0 {
-				// This is a bit of a hack, but client to server headers are only readable after first client msg is
-				// received but must be written to server stream before the first msg is flushed.
-				// This is the only place to do it nicely.
-				md, err := src.Header()
-				if err != nil {
-					ret <- err
-					break
-				}
-				if err := dst.SendHeader(md); err != nil {
-					ret <- err
-					break
-				}
-			}
-			if err := dst.SendMsg(f); err != nil {
-				ret <- err
-				break
-			}
-		}
-	}()
-	return ret
-}
-
-func (s *Handler) forwardServerToClient(src grpc.ServerStream, dst grpc.ClientStream) chan error {
-	ret := make(chan error, 1)
-	go func() {
-		f := &emptypb.Empty{}
-		for i := 0; ; i++ {
-			if err := src.RecvMsg(f); err != nil {
-				ret <- err // this can be io.EOF which is happy case
-				break
-			}
-			if err := dst.SendMsg(f); err != nil {
-				ret <- err
-				break
-			}
-		}
-	}()
-	return ret
-}

experimental/pkg/grpcproxy/proxy.go

@@ -1,32 +0,0 @@
-package grpcproxy
-
-import (
-	"context"
-
-	"google.golang.org/grpc"
-	"google.golang.org/grpc/metadata"
-)
-
-// NewProxy sets up a simple proxy that forwards all requests to dst.
-func NewProxy(dst *grpc.ClientConn, opts ...grpc.ServerOption) (*grpc.Server, *Handler) {
-	option, h := DefaultProxyOpt(dst)
-	opts = append(opts, option)
-	// Set up the proxy server and then serve from it like in step one.
-	return grpc.NewServer(opts...), h
-}
-
-// DefaultProxyOpt returns an grpc.UnknownServiceHandler with a DefaultDirector.
-func DefaultProxyOpt(cc *grpc.ClientConn) (grpc.ServerOption, *Handler) {
-	sh, h := TransparentHandler(DefaultDirector(cc))
-	return grpc.UnknownServiceHandler(sh), h
-}
-
-// DefaultDirector returns a very simple forwarding StreamDirector that forwards all
-// calls.
-func DefaultDirector(cc *grpc.ClientConn) StreamDirector {
-	return func(ctx context.Context, _ string) (context.Context, *grpc.ClientConn, error) {
-		md, _ := metadata.FromIncomingContext(ctx)
-		ctx = metadata.NewOutgoingContext(ctx, md.Copy())
-		return ctx, cc, nil
-	}
-}

experimental/pkg/httpproxy/support.go

@@ -1,58 +0,0 @@
-package httpproxy
-
-import (
-	"bytes"
-	"io"
-	"net/http"
-	"net/http/httputil"
-	"net/url"
-
-	"k8s.io/klog/v2"
-)
-
-type authorization struct {
-	proxy    *httputil.ReverseProxy
-	upstream *url.URL
-	prefix   string
-}
-
-// Create Unauthorized response
-func WriteUnauthorisedResponse(w http.ResponseWriter) {
-	w.WriteHeader(http.StatusUnauthorized)
-	_, err := w.Write([]byte("Unauthorized\n"))
-	if err != nil {
-		klog.Info("failed writing unauthorized response ", err)
-	}
-}
-
-// Create bad request response
-func WriteBadRequestResponse(w http.ResponseWriter) {
-	w.WriteHeader(http.StatusBadRequest)
-	_, err := w.Write([]byte("Bad Request\n"))
-	if err != nil {
-		klog.Info("failed writing bad request response ", err)
-	}
-}
-
-// Create internal error response
-func WriteInternalErrorResponse(w http.ResponseWriter) {
-	w.WriteHeader(http.StatusInternalServerError)
-	_, err := w.Write([]byte("Internal Server Error\n"))
-	if err != nil {
-		klog.Info("failed writing internal error response ", err)
-	}
-}
-
-// Modify request upstream URL
-func modifyRequest(r *http.Request, upstream *url.URL) {
-	r.URL.Host = upstream.Host
-	r.URL.Scheme = upstream.Scheme
-	r.Header.Set("X-Forwarded-Host", r.Host)
-	r.Host = upstream.Host
-	body, err := io.ReadAll(r.Body)
-	if err != nil {
-		klog.Info("failed reading request body ", err)
-	}
-	defer r.Body.Close()
-	r.Body = io.NopCloser(bytes.NewReader(body))
-}

experimental/pkg/httpproxy/token_auth.go

@@ -1,37 +0,0 @@
-package httpproxy
-
-import (
-	"net/http"
-	"net/http/httputil"
-	"net/url"
-	"strings"
-)
-
-const (
-	tokenHeader = "Authorization"
-)
-
-type TokenAuth struct {
-	authorization
-	token string
-}
-
-func NewTokenAuth(token string, proxy *httputil.ReverseProxy, prefix string, upstream *url.URL) TokenAuth {
-	auth := authorization{proxy: proxy, prefix: prefix, upstream: upstream}
-	return TokenAuth{token: token, authorization: auth}
-}
-
-func (ta TokenAuth) AuthFunc() func(http.ResponseWriter, *http.Request) {
-	return func(w http.ResponseWriter, r *http.Request) {
-		if strings.HasPrefix(r.URL.String(), ta.prefix) {
-			auth := r.Header.Get(tokenHeader)
-			if auth != ta.token {
-				// Wrong token
-				WriteUnauthorisedResponse(w)
-				return
-			}
-		}
-		modifyRequest(r, ta.upstream)
-		ta.proxy.ServeHTTP(w, r)
-	}
-}

go.mod

@@ -35,7 +35,7 @@ require (
 	k8s.io/klog/v2 v2.140.0
 	k8s.io/kubectl v0.36.0
 	k8s.io/utils v0.0.0-20260210185600-b8788abfbbc2
-	sigs.k8s.io/controller-runtime v0.23.1-0.20260424122448-c8b4b9d61fbd
+	sigs.k8s.io/controller-runtime v0.24.1
 	sigs.k8s.io/yaml v1.6.0
 )
 

go.sum

@@ -2198,8 +2198,8 @@ rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8
 rsc.io/pdf v0.1.1/go.mod h1:n8OzWcQ6Sp37PL01nO98y4iUCRdTGarVfzxY20ICaU4=
 rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0=
 rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA=
-sigs.k8s.io/controller-runtime v0.23.1-0.20260424122448-c8b4b9d61fbd h1:GFs6wqtAt4TORkbECY+vCvLzEykTIh9uBRiPARKu704=
-sigs.k8s.io/controller-runtime v0.23.1-0.20260424122448-c8b4b9d61fbd/go.mod h1:vFkfY5fGt5xAC/sKb8IBFKgWPNKG9OUG29dR8Y2wImw=
+sigs.k8s.io/controller-runtime v0.24.1 h1:miPEwrmirImAvgME1L9qebGHrOnGJoVmVdtOU9fRfo4=
+sigs.k8s.io/controller-runtime v0.24.1/go.mod h1:vFkfY5fGt5xAC/sKb8IBFKgWPNKG9OUG29dR8Y2wImw=
 sigs.k8s.io/gateway-api v1.4.1 h1:NPxFutNkKNa8UfLd2CMlEuhIPMQgDQ6DXNKG9sHbJU8=
 sigs.k8s.io/gateway-api v1.4.1/go.mod h1:AR5RSqciWP98OPckEjOjh2XJhAe2Na4LHyXD2FUY7Qk=
 sigs.k8s.io/json v0.0.0-20250730193827-2d320260d730 h1:IpInykpT6ceI+QxKBbEflcR5EXP7sU1kvOlxwZh5txg=

helm-chart/kuberay-apiserver/README.md

@@ -133,4 +133,4 @@ kubectl get pods
 | singleNamespaceInstall | bool | `false` | The chart can be installed by users with permissions to a single namespace only |
 | enableAPIServerV2 | bool | `true` | If set to true, APIServer v2 would be served on the same port as the APIServer v1. |
 | security.proxy | object | `{"pullPolicy":"IfNotPresent","repository":"quay.io/kuberay/security-proxy","tag":"nightly"}` | security proxy image. |
-| security.env | object | `{"ENABLE_GRPC":"true","GRPC_LOCAL_PORT":8987,"HTTP_LOCAL_PORT":8988,"SECURITY_PREFIX":"/","SECURITY_TOKEN":"12345"}` | security proxy environment variables. |
+| security.env | object | `{"ENABLE_GRPC":"true","GRPC_LOCAL_PORT":8987,"HTTP_LOCAL_PORT":8988,"SECURITY_PREFIX":"/"}` | security proxy environment variables. |

helm-chart/kuberay-apiserver/ci/ci-values.yaml

@@ -4,7 +4,5 @@ image:
   pullPolicy: Never
 
 security:
-  proxy:
-    repository: kuberay/security-proxy
-    tag: local
-    pullPolicy: Never
+  env:
+    SECURITY_TOKEN: "12345"

helm-chart/kuberay-apiserver/values.yaml

@@ -116,6 +116,7 @@ security:
   env:
     HTTP_LOCAL_PORT: 8988
     GRPC_LOCAL_PORT: 8987
-    SECURITY_TOKEN: "12345"
+    # SECURITY_TOKEN must be provided explicitly by the user or fetched from a Secret.
+    # SECURITY_TOKEN: "12345"
     SECURITY_PREFIX: "/"
     ENABLE_GRPC: "true"

helm-chart/kuberay-operator/crds/ray.io_rayclusters.yaml

@@ -76,6 +76,14 @@ spec:
                 type: object
               autoscalerOptions:
                 properties:
+                  args:
+                    items:
+                      type: string
+                    type: array
+                  command:
+                    items:
+                      type: string
+                    type: array
                   env:
                     items:
                       properties:
@@ -4541,6 +4549,386 @@ spec:
                 - message: the managedBy field value must be either 'ray.io/kuberay-operator'
                     or 'kueue.x-k8s.io/multikueue'
                   rule: self in ['ray.io/kuberay-operator', 'kueue.x-k8s.io/multikueue']
+              networkIsolation:
+                properties:
+                  head:
+                    properties:
+                      egressRules:
+                        items:
+                          properties:
+                            ports:
+                              items:
+                                properties:
+                                  endPort:
+                                    format: int32
+                                    type: integer
+                                  port:
+                                    anyOf:
+                                    - type: integer
+                                    - type: string
+                                    x-kubernetes-int-or-string: true
+                                  protocol:
+                                    type: string
+                                type: object
+                              type: array
+                              x-kubernetes-list-type: atomic
+                            to:
+                              items:
+                                properties:
+                                  ipBlock:
+                                    properties:
+                                      cidr:
+                                        type: string
+                                      except:
+                                        items:
+                                          type: string
+                                        type: array
+                                        x-kubernetes-list-type: atomic
+                                    required:
+                                    - cidr
+                                    type: object
+                                  namespaceSelector:
+                                    properties:
+                                      matchExpressions:
+                                        items:
+                                          properties:
+                                            key:
+                                              type: string
+                                            operator:
+                                              type: string
+                                            values:
+                                              items:
+                                                type: string
+                                              type: array
+                                              x-kubernetes-list-type: atomic
+                                          required:
+                                          - key
+                                          - operator
+                                          type: object
+                                        type: array
+                                        x-kubernetes-list-type: atomic
+                                      matchLabels:
+                                        additionalProperties:
+                                          type: string
+                                        type: object
+                                    type: object
+                                    x-kubernetes-map-type: atomic
+                                  podSelector:
+                                    properties:
+                                      matchExpressions:
+                                        items:
+                                          properties:
+                                            key:
+                                              type: string
+                                            operator:
+                                              type: string
+                                            values:
+                                              items:
+                                                type: string
+                                              type: array
+                                              x-kubernetes-list-type: atomic
+                                          required:
+                                          - key
+                                          - operator
+                                          type: object
+                                        type: array
+                                        x-kubernetes-list-type: atomic
+                                      matchLabels:
+                                        additionalProperties:
+                                          type: string
+                                        type: object
+                                    type: object
+                                    x-kubernetes-map-type: atomic
+                                type: object
+                              type: array
+                              x-kubernetes-list-type: atomic
+                          type: object
+                        type: array
+                      ingressRules:
+                        items:
+                          properties:
+                            from:
+                              items:
+                                properties:
+                                  ipBlock:
+                                    properties:
+                                      cidr:
+                                        type: string
+                                      except:
+                                        items:
+                                          type: string
+                                        type: array
+                                        x-kubernetes-list-type: atomic
+                                    required:
+                                    - cidr
+                                    type: object
+                                  namespaceSelector:
+                                    properties:
+                                      matchExpressions:
+                                        items:
+                                          properties:
+                                            key:
+                                              type: string
+                                            operator:
+                                              type: string
+                                            values:
+                                              items:
+                                                type: string
+                                              type: array
+                                              x-kubernetes-list-type: atomic
+                                          required:
+                                          - key
+                                          - operator
+                                          type: object
+                                        type: array
+                                        x-kubernetes-list-type: atomic
+                                      matchLabels:
+                                        additionalProperties:
+                                          type: string
+                                        type: object
+                                    type: object
+                                    x-kubernetes-map-type: atomic
+                                  podSelector:
+                                    properties:
+                                      matchExpressions:
+                                        items:
+                                          properties:
+                                            key:
+                                              type: string
+                                            operator:
+                                              type: string
+                                            values:
+                                              items:
+                                                type: string
+                                              type: array
+                                              x-kubernetes-list-type: atomic
+                                          required:
+                                          - key
+                                          - operator
+                                          type: object
+                                        type: array
+                                        x-kubernetes-list-type: atomic
+                                      matchLabels:
+                                        additionalProperties:
+                                          type: string
+                                        type: object
+                                    type: object
+                                    x-kubernetes-map-type: atomic
+                                type: object
+                              type: array
+                              x-kubernetes-list-type: atomic
+                            ports:
+                              items:
+                                properties:
+                                  endPort:
+                                    format: int32
+                                    type: integer
+                                  port:
+                                    anyOf:
+                                    - type: integer
+                                    - type: string
+                                    x-kubernetes-int-or-string: true
+                                  protocol:
+                                    type: string
+                                type: object
+                              type: array
+                              x-kubernetes-list-type: atomic
+                          type: object
+                        type: array
+                    type: object
+                  mode:
+                    default: DenyAll
+                    enum:
+                    - DenyAll
+                    - DenyAllIngress
+                    - DenyAllEgress
+                    type: string
+                  worker:
+                    properties:
+                      egressRules:
+                        items:
+                          properties:
+                            ports:
+                              items:
+                                properties:
+                                  endPort:
+                                    format: int32
+                                    type: integer
+                                  port:
+                                    anyOf:
+                                    - type: integer
+                                    - type: string
+                                    x-kubernetes-int-or-string: true
+                                  protocol:
+                                    type: string
+                                type: object
+                              type: array
+                              x-kubernetes-list-type: atomic
+                            to:
+                              items:
+                                properties:
+                                  ipBlock:
+                                    properties:
+                                      cidr:
+                                        type: string
+                                      except:
+                                        items:
+                                          type: string
+                                        type: array
+                                        x-kubernetes-list-type: atomic
+                                    required:
+                                    - cidr
+                                    type: object
+                                  namespaceSelector:
+                                    properties:
+                                      matchExpressions:
+                                        items:
+                                          properties:
+                                            key:
+                                              type: string
+                                            operator:
+                                              type: string
+                                            values:
+                                              items:
+                                                type: string
+                                              type: array
+                                              x-kubernetes-list-type: atomic
+                                          required:
+                                          - key
+                                          - operator
+                                          type: object
+                                        type: array
+                                        x-kubernetes-list-type: atomic
+                                      matchLabels:
+                                        additionalProperties:
+                                          type: string
+                                        type: object
+                                    type: object
+                                    x-kubernetes-map-type: atomic
+                                  podSelector:
+                                    properties:
+                                      matchExpressions:
+                                        items:
+                                          properties:
+                                            key:
+                                              type: string
+                                            operator:
+                                              type: string
+                                            values:
+                                              items:
+                                                type: string
+                                              type: array
+                                              x-kubernetes-list-type: atomic
+                                          required:
+                                          - key
+                                          - operator
+                                          type: object
+                                        type: array
+                                        x-kubernetes-list-type: atomic
+                                      matchLabels:
+                                        additionalProperties:
+                                          type: string
+                                        type: object
+                                    type: object
+                                    x-kubernetes-map-type: atomic
+                                type: object
+                              type: array
+                              x-kubernetes-list-type: atomic
+                          type: object
+                        type: array
+                      ingressRules:
+                        items:
+                          properties:
+                            from:
+                              items:
+                                properties:
+                                  ipBlock:
+                                    properties:
+                                      cidr:
+                                        type: string
+                                      except:
+                                        items:
+                                          type: string
+                                        type: array
+                                        x-kubernetes-list-type: atomic
+                                    required:
+                                    - cidr
+                                    type: object
+                                  namespaceSelector:
+                                    properties:
+                                      matchExpressions:
+                                        items:
+                                          properties:
+                                            key:
+                                              type: string
+                                            operator:
+                                              type: string
+                                            values:
+                                              items:
+                                                type: string
+                                              type: array
+                                              x-kubernetes-list-type: atomic
+                                          required:
+                                          - key
+                                          - operator
+                                          type: object
+                                        type: array
+                                        x-kubernetes-list-type: atomic
+                                      matchLabels:
+                                        additionalProperties:
+                                          type: string
+                                        type: object
+                                    type: object
+                                    x-kubernetes-map-type: atomic
+                                  podSelector:
+                                    properties:
+                                      matchExpressions:
+                                        items:
+                                          properties:
+                                            key:
+                                              type: string
+                                            operator:
+                                              type: string
+                                            values:
+                                              items:
+                                                type: string
+                                              type: array
+                                              x-kubernetes-list-type: atomic
+                                          required:
+                                          - key
+                                          - operator
+                                          type: object
+                                        type: array
+                                        x-kubernetes-list-type: atomic
+                                      matchLabels:
+                                        additionalProperties:
+                                          type: string
+                                        type: object
+                                    type: object
+                                    x-kubernetes-map-type: atomic
+                                type: object
+                              type: array
+                              x-kubernetes-list-type: atomic
+                            ports:
+                              items:
+                                properties:
+                                  endPort:
+                                    format: int32
+                                    type: integer
+                                  port:
+                                    anyOf:
+                                    - type: integer
+                                    - type: string
+                                    x-kubernetes-int-or-string: true
+                                  protocol:
+                                    type: string
+                                type: object
+                              type: array
+                              x-kubernetes-list-type: atomic
+                          type: object
+                        type: array
+                    type: object
+                type: object
               rayVersion:
                 type: string
               suspend:

helm-chart/kuberay-operator/crds/ray.io_raycronjobs.yaml

@@ -20,9 +20,12 @@ spec:
     - jsonPath: .spec.schedule
       name: schedule
       type: string
+    - jsonPath: .spec.timeZone
+      name: timezone
+      type: string
     - jsonPath: .status.lastScheduleTime
       name: last schedule
-      type: string
+      type: date
     - jsonPath: .metadata.creationTimestamp
       name: age
       type: date
@@ -167,6 +170,14 @@ spec:
                         type: object
                       autoscalerOptions:
                         properties:
+                          args:
+                            items:
+                              type: string
+                            type: array
+                          command:
+                            items:
+                              type: string
+                            type: array
                           env:
                             items:
                               properties:
@@ -4632,6 +4643,386 @@ spec:
                         - message: the managedBy field value must be either 'ray.io/kuberay-operator'
                             or 'kueue.x-k8s.io/multikueue'
                           rule: self in ['ray.io/kuberay-operator', 'kueue.x-k8s.io/multikueue']
+                      networkIsolation:
+                        properties:
+                          head:
+                            properties:
+                              egressRules:
+                                items:
+                                  properties:
+                                    ports:
+                                      items:
+                                        properties:
+                                          endPort:
+                                            format: int32
+                                            type: integer
+                                          port:
+                                            anyOf:
+                                            - type: integer
+                                            - type: string
+                                            x-kubernetes-int-or-string: true
+                                          protocol:
+                                            type: string
+                                        type: object
+                                      type: array
+                                      x-kubernetes-list-type: atomic
+                                    to:
+                                      items:
+                                        properties:
+                                          ipBlock:
+                                            properties:
+                                              cidr:
+                                                type: string
+                                              except:
+                                                items:
+                                                  type: string
+                                                type: array
+                                                x-kubernetes-list-type: atomic
+                                            required:
+                                            - cidr
+                                            type: object
+                                          namespaceSelector:
+                                            properties:
+                                              matchExpressions:
+                                                items:
+                                                  properties:
+                                                    key:
+                                                      type: string
+                                                    operator:
+                                                      type: string
+                                                    values:
+                                                      items:
+                                                        type: string
+                                                      type: array
+                                                      x-kubernetes-list-type: atomic
+                                                  required:
+                                                  - key
+                                                  - operator
+                                                  type: object
+                                                type: array
+                                                x-kubernetes-list-type: atomic
+                                              matchLabels:
+                                                additionalProperties:
+                                                  type: string
+                                                type: object
+                                            type: object
+                                            x-kubernetes-map-type: atomic
+                                          podSelector:
+                                            properties:
+                                              matchExpressions:
+                                                items:
+                                                  properties:
+                                                    key:
+                                                      type: string
+                                                    operator:
+                                                      type: string
+                                                    values:
+                                                      items:
+                                                        type: string
+                                                      type: array
+                                                      x-kubernetes-list-type: atomic
+                                                  required:
+                                                  - key
+                                                  - operator
+                                                  type: object
+                                                type: array
+                                                x-kubernetes-list-type: atomic
+                                              matchLabels:
+                                                additionalProperties:
+                                                  type: string
+                                                type: object
+                                            type: object
+                                            x-kubernetes-map-type: atomic
+                                        type: object
+                                      type: array
+                                      x-kubernetes-list-type: atomic
+                                  type: object
+                                type: array
+                              ingressRules:
+                                items:
+                                  properties:
+                                    from:
+                                      items:
+                                        properties:
+                                          ipBlock:
+                                            properties:
+                                              cidr:
+                                                type: string
+                                              except:
+                                                items:
+                                                  type: string
+                                                type: array
+                                                x-kubernetes-list-type: atomic
+                                            required:
+                                            - cidr
+                                            type: object
+                                          namespaceSelector:
+                                            properties:
+                                              matchExpressions:
+                                                items:
+                                                  properties:
+                                                    key:
+                                                      type: string
+                                                    operator:
+                                                      type: string
+                                                    values:
+                                                      items:
+                                                        type: string
+                                                      type: array
+                                                      x-kubernetes-list-type: atomic
+                                                  required:
+                                                  - key
+                                                  - operator
+                                                  type: object
+                                                type: array
+                                                x-kubernetes-list-type: atomic
+                                              matchLabels:
+                                                additionalProperties:
+                                                  type: string
+                                                type: object
+                                            type: object
+                                            x-kubernetes-map-type: atomic
+                                          podSelector:
+                                            properties:
+                                              matchExpressions:
+                                                items:
+                                                  properties:
+                                                    key:
+                                                      type: string
+                                                    operator:
+                                                      type: string
+                                                    values:
+                                                      items:
+                                                        type: string
+                                                      type: array
+                                                      x-kubernetes-list-type: atomic
+                                                  required:
+                                                  - key
+                                                  - operator
+                                                  type: object
+                                                type: array
+                                                x-kubernetes-list-type: atomic
+                                              matchLabels:
+                                                additionalProperties:
+                                                  type: string
+                                                type: object
+                                            type: object
+                                            x-kubernetes-map-type: atomic
+                                        type: object
+                                      type: array
+                                      x-kubernetes-list-type: atomic
+                                    ports:
+                                      items:
+                                        properties:
+                                          endPort:
+                                            format: int32
+                                            type: integer
+                                          port:
+                                            anyOf:
+                                            - type: integer
+                                            - type: string
+                                            x-kubernetes-int-or-string: true
+                                          protocol:
+                                            type: string
+                                        type: object
+                                      type: array
+                                      x-kubernetes-list-type: atomic
+                                  type: object
+                                type: array
+                            type: object
+                          mode:
+                            default: DenyAll
+                            enum:
+                            - DenyAll
+                            - DenyAllIngress
+                            - DenyAllEgress
+                            type: string
+                          worker:
+                            properties:
+                              egressRules:
+                                items:
+                                  properties:
+                                    ports:
+                                      items:
+                                        properties:
+                                          endPort:
+                                            format: int32
+                                            type: integer
+                                          port:
+                                            anyOf:
+                                            - type: integer
+                                            - type: string
+                                            x-kubernetes-int-or-string: true
+                                          protocol:
+                                            type: string
+                                        type: object
+                                      type: array
+                                      x-kubernetes-list-type: atomic
+                                    to:
+                                      items:
+                                        properties:
+                                          ipBlock:
+                                            properties:
+                                              cidr:
+                                                type: string
+                                              except:
+                                                items:
+                                                  type: string
+                                                type: array
+                                                x-kubernetes-list-type: atomic
+                                            required:
+                                            - cidr
+                                            type: object
+                                          namespaceSelector:
+                                            properties:
+                                              matchExpressions:
+                                                items:
+                                                  properties:
+                                                    key:
+                                                      type: string
+                                                    operator:
+                                                      type: string
+                                                    values:
+                                                      items:
+                                                        type: string
+                                                      type: array
+                                                      x-kubernetes-list-type: atomic
+                                                  required:
+                                                  - key
+                                                  - operator
+                                                  type: object
+                                                type: array
+                                                x-kubernetes-list-type: atomic
+                                              matchLabels:
+                                                additionalProperties:
+                                                  type: string
+                                                type: object
+                                            type: object
+                                            x-kubernetes-map-type: atomic
+                                          podSelector:
+                                            properties:
+                                              matchExpressions:
+                                                items:
+                                                  properties:
+                                                    key:
+                                                      type: string
+                                                    operator:
+                                                      type: string
+                                                    values:
+                                                      items:
+                                                        type: string
+                                                      type: array
+                                                      x-kubernetes-list-type: atomic
+                                                  required:
+                                                  - key
+                                                  - operator
+                                                  type: object
+                                                type: array
+                                                x-kubernetes-list-type: atomic
+                                              matchLabels:
+                                                additionalProperties:
+                                                  type: string
+                                                type: object
+                                            type: object
+                                            x-kubernetes-map-type: atomic
+                                        type: object
+                                      type: array
+                                      x-kubernetes-list-type: atomic
+                                  type: object
+                                type: array
+                              ingressRules:
+                                items:
+                                  properties:
+                                    from:
+                                      items:
+                                        properties:
+                                          ipBlock:
+                                            properties:
+                                              cidr:
+                                                type: string
+                                              except:
+                                                items:
+                                                  type: string
+                                                type: array
+                                                x-kubernetes-list-type: atomic
+                                            required:
+                                            - cidr
+                                            type: object
+                                          namespaceSelector:
+                                            properties:
+                                              matchExpressions:
+                                                items:
+                                                  properties:
+                                                    key:
+                                                      type: string
+                                                    operator:
+                                                      type: string
+                                                    values:
+                                                      items:
+                                                        type: string
+                                                      type: array
+                                                      x-kubernetes-list-type: atomic
+                                                  required:
+                                                  - key
+                                                  - operator
+                                                  type: object
+                                                type: array
+                                                x-kubernetes-list-type: atomic
+                                              matchLabels:
+                                                additionalProperties:
+                                                  type: string
+                                                type: object
+                                            type: object
+                                            x-kubernetes-map-type: atomic
+                                          podSelector:
+                                            properties:
+                                              matchExpressions:
+                                                items:
+                                                  properties:
+                                                    key:
+                                                      type: string
+                                                    operator:
+                                                      type: string
+                                                    values:
+                                                      items:
+                                                        type: string
+                                                      type: array
+                                                      x-kubernetes-list-type: atomic
+                                                  required:
+                                                  - key
+                                                  - operator
+                                                  type: object
+                                                type: array
+                                                x-kubernetes-list-type: atomic
+                                              matchLabels:
+                                                additionalProperties:
+                                                  type: string
+                                                type: object
+                                            type: object
+                                            x-kubernetes-map-type: atomic
+                                        type: object
+                                      type: array
+                                      x-kubernetes-list-type: atomic
+                                    ports:
+                                      items:
+                                        properties:
+                                          endPort:
+                                            format: int32
+                                            type: integer
+                                          port:
+                                            anyOf:
+                                            - type: integer
+                                            - type: string
+                                            x-kubernetes-int-or-string: true
+                                          protocol:
+                                            type: string
+                                        type: object
+                                      type: array
+                                      x-kubernetes-list-type: atomic
+                                  type: object
+                                type: array
+                            type: object
+                        type: object
                       rayVersion:
                         type: string
                       suspend:
@@ -12359,6 +12750,9 @@ spec:
                 type: string
               suspend:
                 type: boolean
+              timeZone:
+                minLength: 1
+                type: string
             required:
             - jobTemplate
             - schedule

helm-chart/kuberay-operator/crds/ray.io_rayjobs.yaml

@@ -171,6 +171,14 @@ spec:
                     type: object
                   autoscalerOptions:
                     properties:
+                      args:
+                        items:
+                          type: string
+                        type: array
+                      command:
+                        items:
+                          type: string
+                        type: array
                       env:
                         items:
                           properties:
@@ -4636,6 +4644,386 @@ spec:
                     - message: the managedBy field value must be either 'ray.io/kuberay-operator'
                         or 'kueue.x-k8s.io/multikueue'
                       rule: self in ['ray.io/kuberay-operator', 'kueue.x-k8s.io/multikueue']
+                  networkIsolation:
+                    properties:
+                      head:
+                        properties:
+                          egressRules:
+                            items:
+                              properties:
+                                ports:
+                                  items:
+                                    properties:
+                                      endPort:
+                                        format: int32
+                                        type: integer
+                                      port:
+                                        anyOf:
+                                        - type: integer
+                                        - type: string
+                                        x-kubernetes-int-or-string: true
+                                      protocol:
+                                        type: string
+                                    type: object
+                                  type: array
+                                  x-kubernetes-list-type: atomic
+                                to:
+                                  items:
+                                    properties:
+                                      ipBlock:
+                                        properties:
+                                          cidr:
+                                            type: string
+                                          except:
+                                            items:
+                                              type: string
+                                            type: array
+                                            x-kubernetes-list-type: atomic
+                                        required:
+                                        - cidr
+                                        type: object
+                                      namespaceSelector:
+                                        properties:
+                                          matchExpressions:
+                                            items:
+                                              properties:
+                                                key:
+                                                  type: string
+                                                operator:
+                                                  type: string
+                                                values:
+                                                  items:
+                                                    type: string
+                                                  type: array
+                                                  x-kubernetes-list-type: atomic
+                                              required:
+                                              - key
+                                              - operator
+                                              type: object
+                                            type: array
+                                            x-kubernetes-list-type: atomic
+                                          matchLabels:
+                                            additionalProperties:
+                                              type: string
+                                            type: object
+                                        type: object
+                                        x-kubernetes-map-type: atomic
+                                      podSelector:
+                                        properties:
+                                          matchExpressions:
+                                            items:
+                                              properties:
+                                                key:
+                                                  type: string
+                                                operator:
+                                                  type: string
+                                                values:
+                                                  items:
+                                                    type: string
+                                                  type: array
+                                                  x-kubernetes-list-type: atomic
+                                              required:
+                                              - key
+                                              - operator
+                                              type: object
+                                            type: array
+                                            x-kubernetes-list-type: atomic
+                                          matchLabels:
+                                            additionalProperties:
+                                              type: string
+                                            type: object
+                                        type: object
+                                        x-kubernetes-map-type: atomic
+                                    type: object
+                                  type: array
+                                  x-kubernetes-list-type: atomic
+                              type: object
+                            type: array
+                          ingressRules:
+                            items:
+                              properties:
+                                from:
+                                  items:
+                                    properties:
+                                      ipBlock:
+                                        properties:
+                                          cidr:
+                                            type: string
+                                          except:
+                                            items:
+                                              type: string
+                                            type: array
+                                            x-kubernetes-list-type: atomic
+                                        required:
+                                        - cidr
+                                        type: object
+                                      namespaceSelector:
+                                        properties:
+                                          matchExpressions:
+                                            items:
+                                              properties:
+                                                key:
+                                                  type: string
+                                                operator:
+                                                  type: string
+                                                values:
+                                                  items:
+                                                    type: string
+                                                  type: array
+                                                  x-kubernetes-list-type: atomic
+                                              required:
+                                              - key
+                                              - operator
+                                              type: object
+                                            type: array
+                                            x-kubernetes-list-type: atomic
+                                          matchLabels:
+                                            additionalProperties:
+                                              type: string
+                                            type: object
+                                        type: object
+                                        x-kubernetes-map-type: atomic
+                                      podSelector:
+                                        properties:
+                                          matchExpressions:
+                                            items:
+                                              properties:
+                                                key:
+                                                  type: string
+                                                operator:
+                                                  type: string
+                                                values:
+                                                  items:
+                                                    type: string
+                                                  type: array
+                                                  x-kubernetes-list-type: atomic
+                                              required:
+                                              - key
+                                              - operator
+                                              type: object
+                                            type: array
+                                            x-kubernetes-list-type: atomic
+                                          matchLabels:
+                                            additionalProperties:
+                                              type: string
+                                            type: object
+                                        type: object
+                                        x-kubernetes-map-type: atomic
+                                    type: object
+                                  type: array
+                                  x-kubernetes-list-type: atomic
+                                ports:
+                                  items:
+                                    properties:
+                                      endPort:
+                                        format: int32
+                                        type: integer
+                                      port:
+                                        anyOf:
+                                        - type: integer
+                                        - type: string
+                                        x-kubernetes-int-or-string: true
+                                      protocol:
+                                        type: string
+                                    type: object
+                                  type: array
+                                  x-kubernetes-list-type: atomic
+                              type: object
+                            type: array
+                        type: object
+                      mode:
+                        default: DenyAll
+                        enum:
+                        - DenyAll
+                        - DenyAllIngress
+                        - DenyAllEgress
+                        type: string
+                      worker:
+                        properties:
+                          egressRules:
+                            items:
+                              properties:
+                                ports:
+                                  items:
+                                    properties:
+                                      endPort:
+                                        format: int32
+                                        type: integer
+                                      port:
+                                        anyOf:
+                                        - type: integer
+                                        - type: string
+                                        x-kubernetes-int-or-string: true
+                                      protocol:
+                                        type: string
+                                    type: object
+                                  type: array
+                                  x-kubernetes-list-type: atomic
+                                to:
+                                  items:
+                                    properties:
+                                      ipBlock:
+                                        properties:
+                                          cidr:
+                                            type: string
+                                          except:
+                                            items:
+                                              type: string
+                                            type: array
+                                            x-kubernetes-list-type: atomic
+                                        required:
+                                        - cidr
+                                        type: object
+                                      namespaceSelector:
+                                        properties:
+                                          matchExpressions:
+                                            items:
+                                              properties:
+                                                key:
+                                                  type: string
+                                                operator:
+                                                  type: string
+                                                values:
+                                                  items:
+                                                    type: string
+                                                  type: array
+                                                  x-kubernetes-list-type: atomic
+                                              required:
+                                              - key
+                                              - operator
+                                              type: object
+                                            type: array
+                                            x-kubernetes-list-type: atomic
+                                          matchLabels:
+                                            additionalProperties:
+                                              type: string
+                                            type: object
+                                        type: object
+                                        x-kubernetes-map-type: atomic
+                                      podSelector:
+                                        properties:
+                                          matchExpressions:
+                                            items:
+                                              properties:
+                                                key:
+                                                  type: string
+                                                operator:
+                                                  type: string
+                                                values:
+                                                  items:
+                                                    type: string
+                                                  type: array
+                                                  x-kubernetes-list-type: atomic
+                                              required:
+                                              - key
+                                              - operator
+                                              type: object
+                                            type: array
+                                            x-kubernetes-list-type: atomic
+                                          matchLabels:
+                                            additionalProperties:
+                                              type: string
+                                            type: object
+                                        type: object
+                                        x-kubernetes-map-type: atomic
+                                    type: object
+                                  type: array
+                                  x-kubernetes-list-type: atomic
+                              type: object
+                            type: array
+                          ingressRules:
+                            items:
+                              properties:
+                                from:
+                                  items:
+                                    properties:
+                                      ipBlock:
+                                        properties:
+                                          cidr:
+                                            type: string
+                                          except:
+                                            items:
+                                              type: string
+                                            type: array
+                                            x-kubernetes-list-type: atomic
+                                        required:
+                                        - cidr
+                                        type: object
+                                      namespaceSelector:
+                                        properties:
+                                          matchExpressions:
+                                            items:
+                                              properties:
+                                                key:
+                                                  type: string
+                                                operator:
+                                                  type: string
+                                                values:
+                                                  items:
+                                                    type: string
+                                                  type: array
+                                                  x-kubernetes-list-type: atomic
+                                              required:
+                                              - key
+                                              - operator
+                                              type: object
+                                            type: array
+                                            x-kubernetes-list-type: atomic
+                                          matchLabels:
+                                            additionalProperties:
+                                              type: string
+                                            type: object
+                                        type: object
+                                        x-kubernetes-map-type: atomic
+                                      podSelector:
+                                        properties:
+                                          matchExpressions:
+                                            items:
+                                              properties:
+                                                key:
+                                                  type: string
+                                                operator:
+                                                  type: string
+                                                values:
+                                                  items:
+                                                    type: string
+                                                  type: array
+                                                  x-kubernetes-list-type: atomic
+                                              required:
+                                              - key
+                                              - operator
+                                              type: object
+                                            type: array
+                                            x-kubernetes-list-type: atomic
+                                          matchLabels:
+                                            additionalProperties:
+                                              type: string
+                                            type: object
+                                        type: object
+                                        x-kubernetes-map-type: atomic
+                                    type: object
+                                  type: array
+                                  x-kubernetes-list-type: atomic
+                                ports:
+                                  items:
+                                    properties:
+                                      endPort:
+                                        format: int32
+                                        type: integer
+                                      port:
+                                        anyOf:
+                                        - type: integer
+                                        - type: string
+                                        x-kubernetes-int-or-string: true
+                                      protocol:
+                                        type: string
+                                    type: object
+                                  type: array
+                                  x-kubernetes-list-type: atomic
+                              type: object
+                            type: array
+                        type: object
+                    type: object
                   rayVersion:
                     type: string
                   suspend:
@@ -12376,6 +12764,9 @@ spec:
                 type: string
               jobStatus:
                 type: string
+              jobStatusCheckFailureStartTime:
+                format: date-time
+                type: string
               message:
                 type: string
               observedGeneration:

helm-chart/kuberay-operator/crds/ray.io_rayservices.yaml

@@ -64,6 +64,14 @@ spec:
                     type: object
                   autoscalerOptions:
                     properties:
+                      args:
+                        items:
+                          type: string
+                        type: array
+                      command:
+                        items:
+                          type: string
+                        type: array
                       env:
                         items:
                           properties:
@@ -4529,6 +4537,386 @@ spec:
                     - message: the managedBy field value must be either 'ray.io/kuberay-operator'
                         or 'kueue.x-k8s.io/multikueue'
                       rule: self in ['ray.io/kuberay-operator', 'kueue.x-k8s.io/multikueue']
+                  networkIsolation:
+                    properties:
+                      head:
+                        properties:
+                          egressRules:
+                            items:
+                              properties:
+                                ports:
+                                  items:
+                                    properties:
+                                      endPort:
+                                        format: int32
+                                        type: integer
+                                      port:
+                                        anyOf:
+                                        - type: integer
+                                        - type: string
+                                        x-kubernetes-int-or-string: true
+                                      protocol:
+                                        type: string
+                                    type: object
+                                  type: array
+                                  x-kubernetes-list-type: atomic
+                                to:
+                                  items:
+                                    properties:
+                                      ipBlock:
+                                        properties:
+                                          cidr:
+                                            type: string
+                                          except:
+                                            items:
+                                              type: string
+                                            type: array
+                                            x-kubernetes-list-type: atomic
+                                        required:
+                                        - cidr
+                                        type: object
+                                      namespaceSelector:
+                                        properties:
+                                          matchExpressions:
+                                            items:
+                                              properties:
+                                                key:
+                                                  type: string
+                                                operator:
+                                                  type: string
+                                                values:
+                                                  items:
+                                                    type: string
+                                                  type: array
+                                                  x-kubernetes-list-type: atomic
+                                              required:
+                                              - key
+                                              - operator
+                                              type: object
+                                            type: array
+                                            x-kubernetes-list-type: atomic
+                                          matchLabels:
+                                            additionalProperties:
+                                              type: string
+                                            type: object
+                                        type: object
+                                        x-kubernetes-map-type: atomic
+                                      podSelector:
+                                        properties:
+                                          matchExpressions:
+                                            items:
+                                              properties:
+                                                key:
+                                                  type: string
+                                                operator:
+                                                  type: string
+                                                values:
+                                                  items:
+                                                    type: string
+                                                  type: array
+                                                  x-kubernetes-list-type: atomic
+                                              required:
+                                              - key
+                                              - operator
+                                              type: object
+                                            type: array
+                                            x-kubernetes-list-type: atomic
+                                          matchLabels:
+                                            additionalProperties:
+                                              type: string
+                                            type: object
+                                        type: object
+                                        x-kubernetes-map-type: atomic
+                                    type: object
+                                  type: array
+                                  x-kubernetes-list-type: atomic
+                              type: object
+                            type: array
+                          ingressRules:
+                            items:
+                              properties:
+                                from:
+                                  items:
+                                    properties:
+                                      ipBlock:
+                                        properties:
+                                          cidr:
+                                            type: string
+                                          except:
+                                            items:
+                                              type: string
+                                            type: array
+                                            x-kubernetes-list-type: atomic
+                                        required:
+                                        - cidr
+                                        type: object
+                                      namespaceSelector:
+                                        properties:
+                                          matchExpressions:
+                                            items:
+                                              properties:
+                                                key:
+                                                  type: string
+                                                operator:
+                                                  type: string
+                                                values:
+                                                  items:
+                                                    type: string
+                                                  type: array
+                                                  x-kubernetes-list-type: atomic
+                                              required:
+                                              - key
+                                              - operator
+                                              type: object
+                                            type: array
+                                            x-kubernetes-list-type: atomic
+                                          matchLabels:
+                                            additionalProperties:
+                                              type: string
+                                            type: object
+                                        type: object
+                                        x-kubernetes-map-type: atomic
+                                      podSelector:
+                                        properties:
+                                          matchExpressions:
+                                            items:
+                                              properties:
+                                                key:
+                                                  type: string
+                                                operator:
+                                                  type: string
+                                                values:
+                                                  items:
+                                                    type: string
+                                                  type: array
+                                                  x-kubernetes-list-type: atomic
+                                              required:
+                                              - key
+                                              - operator
+                                              type: object
+                                            type: array
+                                            x-kubernetes-list-type: atomic
+                                          matchLabels:
+                                            additionalProperties:
+                                              type: string
+                                            type: object
+                                        type: object
+                                        x-kubernetes-map-type: atomic
+                                    type: object
+                                  type: array
+                                  x-kubernetes-list-type: atomic
+                                ports:
+                                  items:
+                                    properties:
+                                      endPort:
+                                        format: int32
+                                        type: integer
+                                      port:
+                                        anyOf:
+                                        - type: integer
+                                        - type: string
+                                        x-kubernetes-int-or-string: true
+                                      protocol:
+                                        type: string
+                                    type: object
+                                  type: array
+                                  x-kubernetes-list-type: atomic
+                              type: object
+                            type: array
+                        type: object
+                      mode:
+                        default: DenyAll
+                        enum:
+                        - DenyAll
+                        - DenyAllIngress
+                        - DenyAllEgress
+                        type: string
+                      worker:
+                        properties:
+                          egressRules:
+                            items:
+                              properties:
+                                ports:
+                                  items:
+                                    properties:
+                                      endPort:
+                                        format: int32
+                                        type: integer
+                                      port:
+                                        anyOf:
+                                        - type: integer
+                                        - type: string
+                                        x-kubernetes-int-or-string: true
+                                      protocol:
+                                        type: string
+                                    type: object
+                                  type: array
+                                  x-kubernetes-list-type: atomic
+                                to:
+                                  items:
+                                    properties:
+                                      ipBlock:
+                                        properties:
+                                          cidr:
+                                            type: string
+                                          except:
+                                            items:
+                                              type: string
+                                            type: array
+                                            x-kubernetes-list-type: atomic
+                                        required:
+                                        - cidr
+                                        type: object
+                                      namespaceSelector:
+                                        properties:
+                                          matchExpressions:
+                                            items:
+                                              properties:
+                                                key:
+                                                  type: string
+                                                operator:
+                                                  type: string
+                                                values:
+                                                  items:
+                                                    type: string
+                                                  type: array
+                                                  x-kubernetes-list-type: atomic
+                                              required:
+                                              - key
+                                              - operator
+                                              type: object
+                                            type: array
+                                            x-kubernetes-list-type: atomic
+                                          matchLabels:
+                                            additionalProperties:
+                                              type: string
+                                            type: object
+                                        type: object
+                                        x-kubernetes-map-type: atomic
+                                      podSelector:
+                                        properties:
+                                          matchExpressions:
+                                            items:
+                                              properties:
+                                                key:
+                                                  type: string
+                                                operator:
+                                                  type: string
+                                                values:
+                                                  items:
+                                                    type: string
+                                                  type: array
+                                                  x-kubernetes-list-type: atomic
+                                              required:
+                                              - key
+                                              - operator
+                                              type: object
+                                            type: array
+                                            x-kubernetes-list-type: atomic
+                                          matchLabels:
+                                            additionalProperties:
+                                              type: string
+                                            type: object
+                                        type: object
+                                        x-kubernetes-map-type: atomic
+                                    type: object
+                                  type: array
+                                  x-kubernetes-list-type: atomic
+                              type: object
+                            type: array
+                          ingressRules:
+                            items:
+                              properties:
+                                from:
+                                  items:
+                                    properties:
+                                      ipBlock:
+                                        properties:
+                                          cidr:
+                                            type: string
+                                          except:
+                                            items:
+                                              type: string
+                                            type: array
+                                            x-kubernetes-list-type: atomic
+                                        required:
+                                        - cidr
+                                        type: object
+                                      namespaceSelector:
+                                        properties:
+                                          matchExpressions:
+                                            items:
+                                              properties:
+                                                key:
+                                                  type: string
+                                                operator:
+                                                  type: string
+                                                values:
+                                                  items:
+                                                    type: string
+                                                  type: array
+                                                  x-kubernetes-list-type: atomic
+                                              required:
+                                              - key
+                                              - operator
+                                              type: object
+                                            type: array
+                                            x-kubernetes-list-type: atomic
+                                          matchLabels:
+                                            additionalProperties:
+                                              type: string
+                                            type: object
+                                        type: object
+                                        x-kubernetes-map-type: atomic
+                                      podSelector:
+                                        properties:
+                                          matchExpressions:
+                                            items:
+                                              properties:
+                                                key:
+                                                  type: string
+                                                operator:
+                                                  type: string
+                                                values:
+                                                  items:
+                                                    type: string
+                                                  type: array
+                                                  x-kubernetes-list-type: atomic
+                                              required:
+                                              - key
+                                              - operator
+                                              type: object
+                                            type: array
+                                            x-kubernetes-list-type: atomic
+                                          matchLabels:
+                                            additionalProperties:
+                                              type: string
+                                            type: object
+                                        type: object
+                                        x-kubernetes-map-type: atomic
+                                    type: object
+                                  type: array
+                                  x-kubernetes-list-type: atomic
+                                ports:
+                                  items:
+                                    properties:
+                                      endPort:
+                                        format: int32
+                                        type: integer
+                                      port:
+                                        anyOf:
+                                        - type: integer
+                                        - type: string
+                                        x-kubernetes-int-or-string: true
+                                      protocol:
+                                        type: string
+                                    type: object
+                                  type: array
+                                  x-kubernetes-list-type: atomic
+                              type: object
+                            type: array
+                        type: object
+                    type: object
                   rayVersion:
                     type: string
                   suspend:
@@ -8618,6 +9006,8 @@ spec:
               serviceUnhealthySecondThreshold:
                 format: int32
                 type: integer
+              suspend:
+                type: boolean
               upgradeStrategy:
                 properties:
                   clusterUpgradeOptions:

helm-chart/kuberay-operator/templates/_helpers.tpl

@@ -266,6 +266,7 @@ rules:
   - httproutes
   verbs:
   - create
+  - delete
   - get
   - list
   - update
@@ -278,6 +279,17 @@ rules:
   - get
   - list
   - watch
+- apiGroups:
+  - networking.k8s.io
+  resources:
+  - networkpolicies
+  verbs:
+  - create
+  - delete
+  - get
+  - list
+  - update
+  - watch
 - apiGroups:
   - ray.io
   resources:

helm-chart/kuberay-operator/values.yaml

@@ -267,6 +267,10 @@ env:
 # KubeRay will update JobDeploymentStatus directly.
 # - name: RAYJOB_DEPLOYMENT_STATUS_TRANSITION_GRACE_PERIOD_SECONDS
 #   value: "300"
+# If job status checks keep failing for longer than this timeout in seconds,
+# KubeRay will transition the RayJob's JobDeploymentStatus to Failed. Default is 300.
+# - name: RAYJOB_STATUS_CHECK_TIMEOUT_SECONDS
+#   value: "300"
 
 # -- Resource requests and limits for containers.
 resources:

historyserver/Dockerfile.collector

@@ -17,6 +17,7 @@ COPY pkg/collector/ pkg/collector/
 COPY pkg/storage/ pkg/storage/
 COPY pkg/utils/ pkg/utils/
 COPY pkg/eventserver/ pkg/eventserver/
+COPY pkg/compression/ pkg/compression/
 
 # Build the collector binary.
 COPY Makefile Makefile

historyserver/Dockerfile.historyserver

@@ -20,6 +20,7 @@ COPY pkg/historyserver/ pkg/historyserver/
 COPY pkg/storage/ pkg/storage/
 COPY pkg/utils/ pkg/utils/
 COPY pkg/eventserver/ pkg/eventserver/
+COPY pkg/compression/ pkg/compression/
 
 # Build the historyserver binary.
 COPY Makefile Makefile

historyserver/README.md

@@ -11,7 +11,7 @@ It provides a web interface to explore the history of Ray jobs, tasks, actors, a
 The History Server consists of two main components:
 
 1. **Collector**: Runs as a sidecar container in Ray clusters to collect logs and metadata
-2. **History Server**: Central service that aggregates data from collectors and provides a web UI
+2. **History Server**: Serves a Ray Dashboard compatible HTTP API and ingests cluster sessions' events on demand
 
 ## Building
 
@@ -139,6 +139,87 @@ To run lint checks:
 make alllint
 ```
 
+## Smoke Tests
+
+### 1. Deploy History Server
+
+Apply MinIO and the History Server manifests:
+
+```bash
+kubectl apply -f historyserver/config/minio.yaml
+kubectl apply -f historyserver/config/service_account.yaml
+kubectl apply -f historyserver/config/historyserver.yaml
+```
+
+Port-forward the HS service:
+
+```bash
+kubectl port-forward svc/historyserver 8080:30080
+```
+
+### 2. Generate a Dead Session
+
+Deploy the sample RayCluster, run a deterministic workload, then delete the CR:
+
+```bash
+kubectl apply -f historyserver/config/raycluster.yaml
+kubectl wait pod -l ray.io/node-type=head --for=condition=Ready --timeout=180s
+
+# Run a workload so events are written to MinIO
+kubectl exec $(kubectl get pod -l ray.io/node-type=head -o name) \
+  -c ray-head -- python -c "
+import ray
+ray.init(address='auto')
+
+@ray.remote
+def add(x, y): return x + y
+
+print('tasks:', ray.get([add.remote(i, i) for i in range(5)]))
+"
+
+# Delete the cluster — produces a 'dead' session
+kubectl delete -f historyserver/config/raycluster.yaml
+
+# Discover the session name. /clusters lists both live and dead sessions;
+# dead sessions carry the `session_*` name you'll feed into /enter_cluster.
+curl -sS http://localhost:8080/clusters
+```
+
+### 3. Cold Path (first visit)
+
+Trigger the lazy load synchronously. Replace `<session>` with the session name from §2:
+
+```bash
+time curl -s -o /dev/null \
+  http://localhost:8080/enter_cluster/default/raycluster-historyserver/<session>
+```
+
+> [!NOTE]
+> Cold path runs synchronously: K8s probe + event parse. Expect this to take seconds. Subsequent endpoint calls
+> (`/api/v0/jobs`, `/api/v0/tasks/...`) read from in-memory state populated by this call.
+
+### 4. Warm Path (subsequent visits on same replica)
+
+Re-enter the same cluster:
+
+```bash
+time curl -s -o /dev/null \
+  http://localhost:8080/enter_cluster/default/raycluster-historyserver/<session>
+```
+
+> [!NOTE]
+> Warm path returns immediately — SessionLoader's loaded-session set fast-paths the request without re-parsing. If the HS
+> process restarts, the next visit returns to cold-path latency.
+
+### 5. Tail Logs
+
+```bash
+kubectl logs -f -l app=historyserver --tail=50
+```
+
+You should see one `current eventFileList for cluster ...` line (and the per-file `Reading event file: ...` entries)
+per first-time cold-path call. Warm-path calls produce no parse log lines.
+
 ## Deployment
 
 History Server can be deployed in Kubernetes using the manifests in the `config/samples/` directory.

historyserver/cmd/collector/main.go

@@ -13,6 +13,7 @@ import (
 	"time"
 
 	"github.com/sirupsen/logrus"
+	"k8s.io/apimachinery/pkg/util/validation"
 
 	"github.com/ray-project/kuberay/historyserver/pkg/collector"
 	"github.com/ray-project/kuberay/historyserver/pkg/collector/eventcollector"
@@ -33,6 +34,8 @@ func main() {
 	eventsPort := 8080
 	pushInterval := time.Minute
 	runtimeClassConfigPath := "/var/collector-config/data"
+	ownerKind := ""
+	ownerName := ""
 
 	flag.StringVar(&role, "role", "Worker", "")
 	flag.StringVar(&runtimeClassName, "runtime-class-name", "", "")
@@ -43,9 +46,15 @@ func main() {
 	flag.IntVar(&eventsPort, "events-port", 8080, "")
 	flag.StringVar(&runtimeClassConfigPath, "runtime-class-config-path", "", "") //"/var/collector-config/data"
 	flag.DurationVar(&pushInterval, "push-interval", time.Minute, "")
+	flag.StringVar(&ownerKind, "owner-kind", "", "")
+	flag.StringVar(&ownerName, "owner-name", "", "")
 
 	flag.Parse()
 
+	if err := validateFlags(&rayClusterName, &rayClusterNamespace, &ownerKind, &ownerName); err != nil {
+		logrus.Fatalf("Failed to validate flags: %v", err)
+	}
+
 	var additionalEndpoints []string
 	if epStr := os.Getenv("RAY_COLLECTOR_ADDITIONAL_ENDPOINTS"); epStr != "" {
 		for _, ep := range strings.Split(epStr, ",") {
@@ -108,6 +117,8 @@ func main() {
 		PushInterval:        pushInterval,
 		LogBatching:         logBatching,
 		DashboardAddress:    os.Getenv("RAY_DASHBOARD_ADDRESS"),
+		OwnerKind:           ownerKind,
+		OwnerName:           ownerName,
 
 		AdditionalEndpoints:  additionalEndpoints,
 		EndpointPollInterval: endpointPollInterval,
@@ -152,3 +163,30 @@ func main() {
 	wg.Wait()
 	logrus.Info("Graceful shutdown complete")
 }
+
+func validateFlags(rayClusterName, rayClusterNamespace, ownerKind, ownerName *string) error {
+	*rayClusterName = strings.TrimSpace(*rayClusterName)
+	*rayClusterNamespace = strings.TrimSpace(*rayClusterNamespace)
+
+	if errs := validation.IsDNS1123Subdomain(*rayClusterName); len(errs) > 0 {
+		return fmt.Errorf("invalid ray-cluster-name %q: %s", *rayClusterName, strings.Join(errs, ", "))
+	}
+	if errs := validation.IsDNS1123Subdomain(*rayClusterNamespace); len(errs) > 0 {
+		return fmt.Errorf("invalid ray-cluster-namespace %q: %s", *rayClusterNamespace, strings.Join(errs, ", "))
+	}
+
+	*ownerKind = strings.ToLower(strings.TrimSpace(*ownerKind))
+	*ownerName = strings.TrimSpace(*ownerName)
+	if (*ownerName != "" && *ownerKind == "") || (*ownerName == "" && *ownerKind != "") {
+		return fmt.Errorf("both --owner-name and --owner-kind must be provided together, or both omitted")
+	}
+	if *ownerKind != "" && *ownerKind != utils.RayJobKind && *ownerKind != utils.RayServiceKind {
+		return fmt.Errorf("unsupported owner-kind: %q. Supported kinds are %q or %q", *ownerKind, utils.RayJobKind, utils.RayServiceKind)
+	}
+	if *ownerName != "" {
+		if errs := validation.IsDNS1123Subdomain(*ownerName); len(errs) > 0 {
+			return fmt.Errorf("invalid owner-name %q: %s", *ownerName, strings.Join(errs, ", "))
+		}
+	}
+	return nil
+}

historyserver/cmd/historyserver/main.go

@@ -1,6 +1,7 @@
 package main
 
 import (
+	"context"
 	"encoding/json"
 	"flag"
 	"os"
@@ -8,50 +9,79 @@ import (
 	"sync"
 	"syscall"
 
+	"github.com/sirupsen/logrus"
+
 	"github.com/ray-project/kuberay/historyserver/pkg/collector"
 	"github.com/ray-project/kuberay/historyserver/pkg/collector/types"
-	"github.com/ray-project/kuberay/historyserver/pkg/eventserver"
 	"github.com/ray-project/kuberay/historyserver/pkg/historyserver"
-	"github.com/sirupsen/logrus"
 )
 
 func main() {
 	runtimeClassName := ""
 	rayRootDir := ""
 	kubeconfigs := ""
-	runtimeClassConfigPath := "/var/collector-config/data"
+	runtimeClassConfigPath := ""
 	dashboardDir := ""
 	useKubernetesProxy := false
-	flag.StringVar(&runtimeClassName, "runtime-class-name", "", "")
-	flag.StringVar(&rayRootDir, "ray-root-dir", "", "")
-	flag.StringVar(&kubeconfigs, "kubeconfigs", "", "")
-	flag.StringVar(&dashboardDir, "dashboard-dir", "/dashboard", "")
-	flag.StringVar(&runtimeClassConfigPath, "runtime-class-config-path", "", "") //"/var/collector-config/data"
-	flag.BoolVar(&useKubernetesProxy, "use-kubernetes-proxy", false, "")
+	qps := historyserver.DefaultKubeAPIQPS
+	burst := historyserver.DefaultKubeAPIBurst
+	sessionProcessTimeout := historyserver.DefaultSessionProcessTimeout
+	sessionCacheSize := historyserver.DefaultSessionCacheSize
+	sessionCacheTTL := historyserver.DefaultSessionCacheTTL
+	flag.StringVar(&runtimeClassName, "runtime-class-name", "", "Storage backend: s3 / gcs / azureblob / aliyunoss / localtest")
+	flag.StringVar(&rayRootDir, "ray-root-dir", "", "Root dir inside the bucket")
+	flag.StringVar(&kubeconfigs, "kubeconfigs", "", "Kubeconfig path; empty = in-cluster")
+	flag.StringVar(&dashboardDir, "dashboard-dir", "/dashboard", "Path to Ray Dashboard static assets")
+	flag.StringVar(&runtimeClassConfigPath, "runtime-class-config-path", "", "Path to backend config JSON")
+	flag.BoolVar(&useKubernetesProxy, "use-kubernetes-proxy", false, "Use local kubeconfig instead of in-cluster config")
+	flag.Float64Var(&qps, "kube-api-qps", historyserver.DefaultKubeAPIQPS, "The QPS value for the client communicating with the Kubernetes API server.")
+	flag.IntVar(&burst, "kube-api-burst", historyserver.DefaultKubeAPIBurst, "The maximum burst for throttling requests from this client to the Kubernetes API server.")
+	flag.DurationVar(&sessionProcessTimeout, "session-process-timeout", historyserver.DefaultSessionProcessTimeout, "Timeout duration for processing and loading a single Ray cluster session.")
+	flag.IntVar(&sessionCacheSize, "session-cache-size", historyserver.DefaultSessionCacheSize, "Max number of dead-session snapshots held in the LRU cache.")
+	flag.DurationVar(&sessionCacheTTL, "session-cache-ttl", historyserver.DefaultSessionCacheTTL, "How long a dead-session snapshot stays cached after last access. 0 disables TTL.")
 	flag.Parse()
 
-	cliMgr, err := historyserver.NewClientManager(kubeconfigs, useKubernetesProxy)
+	if runtimeClassName == "" {
+		logrus.Fatal("--runtime-class-name is required")
+	}
+	if qps <= 0 {
+		logrus.Fatalf("--kube-api-qps must be > 0, got %v", qps)
+	}
+	if burst <= 0 {
+		logrus.Fatalf("--kube-api-burst must be > 0, got %d", burst)
+	}
+	if sessionCacheSize <= 0 {
+		logrus.Fatalf("--session-cache-size must be > 0, got %d", sessionCacheSize)
+	}
+	if sessionCacheTTL < 0 {
+		logrus.Fatalf("--session-cache-ttl must be >= 0, got %s", sessionCacheTTL)
+	}
+
+	cliMgr, err := historyserver.NewClientManager(historyserver.ClientManagerConfig{
+		Kubeconfigs:        kubeconfigs,
+		UseKubernetesProxy: useKubernetesProxy,
+		QPS:                float32(qps),
+		Burst:              burst,
+	})
 	if err != nil {
-		logrus.Errorf("Failed to create client manager: %v", err)
-		os.Exit(1)
+		logrus.Fatalf("Failed to create client manager: %v", err)
 	}
 
 	jsonData := make(map[string]interface{})
 	if runtimeClassConfigPath != "" {
 		data, err := os.ReadFile(runtimeClassConfigPath)
 		if err != nil {
-			panic("Failed to read runtime class config " + err.Error())
+			logrus.Fatalf("Failed to read runtime-class-config-path from %s: %v", runtimeClassConfigPath, err)
 		}
-		err = json.Unmarshal(data, &jsonData)
-		if err != nil {
-			panic("Failed to parse runtime class config: " + err.Error())
+		if err := json.Unmarshal(data, &jsonData); err != nil {
+			logrus.Fatalf("Failed to parse runtime-class-config-path from %s: %v", runtimeClassConfigPath, err)
 		}
 	}
 
 	registry := collector.GetReaderRegistry()
 	factory, ok := registry[runtimeClassName]
 	if !ok {
-		panic("Not supported runtime class name: " + runtimeClassName + ".")
+		logrus.Fatalf("Unsupported runtime-class-name for reader: %s", runtimeClassName)
 	}
 
 	globalConfig := types.RayHistoryServerConfig{
@@ -60,34 +90,30 @@ func main() {
 
 	reader, err := factory(&globalConfig, jsonData)
 	if err != nil {
-		panic("Failed to create reader for runtime class name: " + runtimeClassName + ".")
+		logrus.Fatalf("Failed to create reader for runtime class name %s: %v", runtimeClassName, err)
 	}
 
-	// Create EventHandler with storage reader
-	eventHandler := eventserver.NewEventHandler(reader)
+	serverCtx, serverCancel := signal.NotifyContext(
+		context.Background(),
+		syscall.SIGINT, syscall.SIGTERM,
+	)
+	defer serverCancel()
 
-	// WaitGroup to track goroutine completion
+	processor := historyserver.NewSessionProcessor(reader, cliMgr.Client())
+	sessionLoader := historyserver.NewSessionLoader(processor, serverCtx, sessionProcessTimeout, sessionCacheSize, sessionCacheTTL)
+
+	// ServerHandler.Run consumes a stop chan; bridge serverCtx into it.
 	var wg sync.WaitGroup
-
-	sigChan := make(chan os.Signal, 1)
-	stop := make(chan struct{}, 1)
-	signal.Notify(sigChan, syscall.SIGINT, syscall.SIGTERM)
-
-	// Start EventHandler in background goroutine
-	wg.Add(1)
+	stop := make(chan struct{})
 	go func() {
-		defer wg.Done()
-		logrus.Info("Starting EventHandler in background...")
-		if err := eventHandler.Run(stop, 2); err != nil {
-			logrus.Errorf("EventHandler stopped with error: %v", err)
-		}
-		logrus.Info("EventHandler shutdown complete")
+		<-serverCtx.Done()
+		logrus.Info("Received shutdown signal, initiating graceful shutdown...")
+		close(stop)
 	}()
 
-	handler, err := historyserver.NewServerHandler(&globalConfig, dashboardDir, reader, cliMgr, eventHandler, useKubernetesProxy)
+	handler, err := historyserver.NewServerHandler(&globalConfig, dashboardDir, reader, cliMgr, sessionLoader, useKubernetesProxy)
 	if err != nil {
-		logrus.Errorf("Failed to create server handler: %v", err)
-		os.Exit(1)
+		logrus.Fatalf("Failed to create server handler: %v", err)
 	}
 
 	wg.Add(1)
@@ -97,13 +123,6 @@ func main() {
 		logrus.Info("HTTP server shutdown complete")
 	}()
 
-	<-sigChan
-	logrus.Info("Received shutdown signal, initiating graceful shutdown...")
-
-	// Stop both the server and the event handler
-	close(stop)
-
-	// Wait for both goroutines to complete
 	wg.Wait()
 	logrus.Info("Graceful shutdown complete")
 }

historyserver/go.mod

@@ -14,6 +14,7 @@ require (
 	github.com/emicklei/go-restful/v3 v3.13.0
 	github.com/fsnotify/fsnotify v1.9.0
 	github.com/fsouza/fake-gcs-server v1.53.1
+	github.com/hashicorp/golang-lru/v2 v2.0.7
 	github.com/onsi/gomega v1.38.2
 	github.com/ray-project/kuberay/ray-operator v1.5.1
 	github.com/sirupsen/logrus v1.9.3
@@ -67,7 +68,7 @@ require (
 	go.yaml.in/yaml/v3 v3.0.4 // indirect
 	golang.org/x/net v0.49.0 // indirect
 	golang.org/x/oauth2 v0.34.0 // indirect
-	golang.org/x/sync v0.19.0 // indirect
+	golang.org/x/sync v0.19.0
 	golang.org/x/sys v0.40.0 // indirect
 	golang.org/x/term v0.39.0 // indirect
 	golang.org/x/text v0.33.0 // indirect
@@ -106,6 +107,7 @@ require (
 	github.com/cncf/xds/go v0.0.0-20251022180443-0feb69152e9f // indirect
 	github.com/envoyproxy/go-control-plane/envoy v1.35.0 // indirect
 	github.com/envoyproxy/protoc-gen-validate v1.2.1 // indirect
+	github.com/evanphx/json-patch v5.6.0+incompatible // indirect
 	github.com/felixge/httpsnoop v1.0.4 // indirect
 	github.com/go-jose/go-jose/v4 v4.1.3 // indirect
 	github.com/go-logr/stdr v1.2.2 // indirect

historyserver/go.sum

@@ -95,8 +95,8 @@ github.com/envoyproxy/go-control-plane/ratelimit v0.1.0/go.mod h1:Wk+tMFAFbCXaJP
 github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c=
 github.com/envoyproxy/protoc-gen-validate v1.2.1 h1:DEo3O99U8j4hBFwbJfrz9VtgcDfUKS7KJ7spH3d86P8=
 github.com/envoyproxy/protoc-gen-validate v1.2.1/go.mod h1:d/C80l/jxXLdfEIhX1W2TmLfsJ31lvEjwamM4DxlWXU=
-github.com/evanphx/json-patch v0.5.2 h1:xVCHIVMUu1wtM/VkR9jVZ45N3FhZfYMMYGorLCR8P3k=
-github.com/evanphx/json-patch v0.5.2/go.mod h1:ZWS5hhDbVDyob71nXKNL0+PWn6ToqBHMikGIFbs31qQ=
+github.com/evanphx/json-patch v5.6.0+incompatible h1:jBYDEEiFBPxA0v50tFdvOzQQTCvpL6mnFh5mB2/l16U=
+github.com/evanphx/json-patch v5.6.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk=
 github.com/evanphx/json-patch/v5 v5.9.11 h1:/8HVnzMq13/3x9TPvjG08wUGqBTmZBsCWzjTM0wiaDU=
 github.com/evanphx/json-patch/v5 v5.9.11/go.mod h1:3j+LviiESTElxA4p3EMKAB9HXj3/XEtnUf6OZxqIQTM=
 github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg=
@@ -190,6 +190,8 @@ github.com/gorilla/mux v1.8.1 h1:TuBL49tXwgrFYWhqrNgrUNEY92u81SPhu7sTdzQEiWY=
 github.com/gorilla/mux v1.8.1/go.mod h1:AKf9I4AEqPTmMytcMc0KkNouC66V3BtZ4qD5fmWSiMQ=
 github.com/gorilla/websocket v1.5.4-0.20250319132907-e064f32e3674 h1:JeSE6pjso5THxAzdVpqr6/geYxZytqFMBCOtn/ujyeo=
 github.com/gorilla/websocket v1.5.4-0.20250319132907-e064f32e3674/go.mod h1:r4w70xmWCQKmi1ONH4KIaBptdivuRPyosB9RmPlGEwA=
+github.com/hashicorp/golang-lru/v2 v2.0.7 h1:a+bsQ5rvGLjzHuww6tVxozPZFVghXaHOwFs4luLUK2k=
+github.com/hashicorp/golang-lru/v2 v2.0.7/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyfM2/ZepoAG6RGpeM=
 github.com/jarcoal/httpmock v1.4.0 h1:BvhqnH0JAYbNudL2GMJKgOHe2CtKlzJ/5rWKyp+hc2k=
 github.com/jarcoal/httpmock v1.4.0/go.mod h1:ftW1xULwo+j0R0JJkJIIi7UKigZUXCLLanykgjwBXL0=
 github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg=

historyserver/pkg/collector/eventcollector/eventcollector.go

@@ -1,7 +1,6 @@
 package eventcollector
 
 import (
-	"bytes"
 	"encoding/json"
 	"fmt"
 	"io"
@@ -16,6 +15,7 @@ import (
 	"github.com/fsnotify/fsnotify"
 	"github.com/sirupsen/logrus"
 
+	"github.com/ray-project/kuberay/historyserver/pkg/compression"
 	"github.com/ray-project/kuberay/historyserver/pkg/storage"
 	"github.com/ray-project/kuberay/historyserver/pkg/utils"
 )
@@ -437,8 +437,6 @@ func (ec *EventCollector) flushNodeEventsForHour(hourKey string, events []Event)
 		return fmt.Errorf("failed to marshal node events: %w", err)
 	}
 
-	reader := bytes.NewReader(data)
-
 	// Use sessionName from event, not config
 	sessionNameToUse := ec.sessionName // Default to configured sessionName
 	if len(events) > 0 && events[0].SessionName != "" {
@@ -452,12 +450,9 @@ func (ec *EventCollector) flushNodeEventsForHour(hourKey string, events []Event)
 	}
 
 	// Build node event storage path using event's nodeID
-	basePath := path.Join(
-		ec.root,
-		fmt.Sprintf("%s_%s", ec.clusterName, ec.clusterNamespace),
-		sessionNameToUse,
-		"node_events",
-		fmt.Sprintf("%s-%s", nodeIDToUse, hourKey))
+	sessionPath := path.Clean(path.Join(ec.root, utils.AppendRayClusterNameNamespace(ec.clusterName, ec.clusterNamespace), sessionNameToUse))
+
+	basePath := path.Join(sessionPath, "node_events", fmt.Sprintf("%s-%s.gz", nodeIDToUse, hourKey))
 
 	// Ensure storage directory exists
 	dir := path.Dir(basePath)
@@ -466,8 +461,9 @@ func (ec *EventCollector) flushNodeEventsForHour(hourKey string, events []Event)
 	}
 
 	// Write event file
-	if err := ec.storageWriter.WriteFile(basePath, reader); err != nil {
-		return fmt.Errorf("failed to write node events file %s: %w", basePath, err)
+	writeErr := compression.WriteCompressedBytes(ec.storageWriter, basePath, data)
+	if writeErr != nil {
+		return fmt.Errorf("failed to write node events file %s: %w", basePath, writeErr)
 	}
 
 	logrus.Infof("Successfully flushed %d node events for hour %s to %s, context: %s", len(events), hourKey, basePath, string(data))
@@ -487,8 +483,6 @@ func (ec *EventCollector) flushJobEventsForHour(jobID, hourKey string, events []
 		return fmt.Errorf("failed to marshal job events: %w", err)
 	}
 
-	reader := bytes.NewReader(data)
-
 	// Use sessionName from event, not config
 	sessionNameToUse := ec.sessionName // Default to configured sessionName
 	if len(events) > 0 && events[0].SessionName != "" {
@@ -502,13 +496,9 @@ func (ec *EventCollector) flushJobEventsForHour(jobID, hourKey string, events []
 	}
 
 	// Build job event storage path using event's nodeID
-	basePath := path.Join(
-		ec.root,
-		fmt.Sprintf("%s_%s", ec.clusterName, ec.clusterNamespace),
-		sessionNameToUse,
-		"job_events",
-		jobID,
-		fmt.Sprintf("%s-%s", nodeIDToUse, hourKey))
+	sessionPath := path.Clean(path.Join(ec.root, utils.AppendRayClusterNameNamespace(ec.clusterName, ec.clusterNamespace), sessionNameToUse))
+
+	basePath := path.Join(sessionPath, "job_events", jobID, fmt.Sprintf("%s-%s.gz", nodeIDToUse, hourKey))
 
 	// Ensure storage directory exists
 	dir := path.Dir(basePath)
@@ -517,8 +507,9 @@ func (ec *EventCollector) flushJobEventsForHour(jobID, hourKey string, events []
 	}
 
 	// Write event file
-	if err := ec.storageWriter.WriteFile(basePath, reader); err != nil {
-		return fmt.Errorf("failed to write job events file %s: %w", basePath, err)
+	writeErr := compression.WriteCompressedBytes(ec.storageWriter, basePath, data)
+	if writeErr != nil {
+		return fmt.Errorf("failed to write job events file %s: %w", basePath, writeErr)
 	}
 
 	logrus.Infof("Successfully flushed %d job events for job %s hour %s to %s", len(events), jobID, hourKey, basePath)

historyserver/pkg/collector/logcollector/runtime/logcollector/collector.go

@@ -16,6 +16,7 @@ import (
 	"github.com/sirupsen/logrus"
 
 	"github.com/ray-project/kuberay/historyserver/pkg/storage"
+	"github.com/ray-project/kuberay/historyserver/pkg/storage/clustermetadata"
 	"github.com/ray-project/kuberay/historyserver/pkg/utils"
 )
 
@@ -30,6 +31,8 @@ type RayLogHandler struct {
 	LogDir                 string
 	RayNodeName            string
 	RayClusterNamespace    string
+	OwnerKind              string
+	OwnerName              string
 	RootDir                string
 	SessionDir             string
 	prevLogsDir            string
@@ -99,17 +102,21 @@ func (r *RayLogHandler) processSessionLatestLogs() {
 	// Extract the real session ID from the resolved path
 	sessionID := filepath.Base(sessionRealDir)
 	if r.IsHead {
-		metadir := path.Join(r.RootDir, "metadir")
-		metafile := path.Clean(metadir + "/" + fmt.Sprintf("%s/%v",
-			utils.AppendRayClusterNameNamespace(r.RayClusterName, r.RayClusterNamespace),
-			path.Base(sessionID),
-		))
+		metafile := clustermetadata.EncodePath(
+			utils.ClusterInfo{
+				Name:      r.RayClusterName,
+				Namespace: r.RayClusterNamespace,
+				OwnerKind: r.OwnerKind,
+				OwnerName: r.OwnerName},
+			r.RootDir,
+			sessionID,
+		)
 		if err := r.Writer.CreateDirectory(path.Dir(metafile)); err != nil {
-			logrus.Errorf("CreateObjectIfNotExist %s error %v", metadir, err)
+			logrus.Errorf("Failed to create directory %s error %v", path.Dir(metafile), err)
 			return
 		}
 		if err := r.Writer.WriteFile(metafile, strings.NewReader("")); err != nil {
-			logrus.Errorf("CreateObjectIfNotExist %s error %v", metafile, err)
+			logrus.Errorf("Failed to write session file %s error %v", metafile, err)
 			return
 		}
 	}
@@ -448,17 +455,20 @@ func (r *RayLogHandler) processSessionPrevLogs(sessionDir string) {
 	sessionID := parts[0]
 	logrus.Infof("Processing all node logs for session: %s", sessionID)
 	if r.IsHead {
-		metadir := path.Join(r.RootDir, "metadir")
-		metafile := path.Clean(metadir + "/" + fmt.Sprintf("%s/%v",
-			utils.AppendRayClusterNameNamespace(r.RayClusterName, r.RayClusterNamespace),
-			path.Base(sessionID),
-		))
+		metafile := clustermetadata.EncodePath(
+			utils.ClusterInfo{
+				Name:      r.RayClusterName,
+				Namespace: r.RayClusterNamespace,
+				OwnerKind: r.OwnerKind,
+				OwnerName: r.OwnerName},
+			r.RootDir,
+			sessionID)
 		if err := r.Writer.CreateDirectory(path.Dir(metafile)); err != nil {
-			logrus.Errorf("CreateObjectIfNotExist %s error %v", metadir, err)
+			logrus.Errorf("Failed to create directory %s error %v", path.Dir(metafile), err)
 			return
 		}
 		if err := r.Writer.WriteFile(metafile, strings.NewReader("")); err != nil {
-			logrus.Errorf("CreateObjectIfNotExist %s error %v", metafile, err)
+			logrus.Errorf("Failed to write session file %s error %v", metafile, err)
 			return
 		}
 	}
@@ -689,14 +699,9 @@ func (r *RayLogHandler) processPrevLogFile(absoluteLogPathName, localLogDir, ses
 	return nil
 }
 
-// meta-dir only stores metadata indicating which clusters have been saved.
-// As long as worker logs are uploaded normally and head writes the metadata,
-// the cluster can be viewed.
 // Any session change triggers sessiondir updates on all head and worker nodes,
 // so we only need to update from one node.
 // for example:
-// metadir/
-//
 //	my-cluster_abc123/
 //		session_2024-12-15_10-30-45_123456    ← Empty file! The path itself is the information
 //		session_2024-12-15_14-20-10_789012
@@ -745,17 +750,21 @@ func (r *RayLogHandler) WatchSessionLatestLoops() {
 			// Handle changes to the symlink
 			if event.Op&(fsnotify.Create|fsnotify.Write) != 0 {
 				sessionID := filepath.Base(event.Name)
-				metadir := path.Join(r.RootDir, "metadir")
-				metafile := path.Clean(metadir + "/" + fmt.Sprintf("%s/%v",
-					utils.AppendRayClusterNameNamespace(r.RayClusterName, r.RayClusterNamespace),
-					path.Base(sessionID),
-				))
+				metafile := clustermetadata.EncodePath(
+					utils.ClusterInfo{
+						Name:      r.RayClusterName,
+						Namespace: r.RayClusterNamespace,
+						OwnerKind: r.OwnerKind,
+						OwnerName: r.OwnerName},
+					r.RootDir,
+					sessionID,
+				)
 				if err := r.Writer.CreateDirectory(path.Dir(metafile)); err != nil {
-					logrus.Errorf("CreateObjectIfNotExist %s error %v", metadir, err)
+					logrus.Errorf("Failed to create directory %s error %v", path.Dir(metafile), err)
 					return
 				}
 				if err := r.Writer.WriteFile(metafile, strings.NewReader("")); err != nil {
-					logrus.Errorf("CreateObjectIfNotExist %s error %v", metafile, err)
+					logrus.Errorf("Failed to write session file %s error %v", metafile, err)
 					return
 				}
 			}

historyserver/pkg/collector/logcollector/runtime/runtime.go

@@ -8,6 +8,8 @@ import (
 	"strings"
 	"time"
 
+	"github.com/sirupsen/logrus"
+
 	"github.com/ray-project/kuberay/historyserver/pkg/collector/logcollector/runtime/logcollector"
 	"github.com/ray-project/kuberay/historyserver/pkg/collector/types"
 	"github.com/ray-project/kuberay/historyserver/pkg/storage"
@@ -42,10 +44,15 @@ func NewCollector(config *types.RayCollectorConfig, writer storage.StorageWriter
 		Writer:       writer,
 		ShutdownChan: make(chan struct{}),
 	}
+
+	handler.OwnerKind = config.OwnerKind
+	handler.OwnerName = config.OwnerName
+	if handler.OwnerKind != "" && handler.OwnerName != "" {
+		logrus.Infof("The associated owner resource is: %s/%s", handler.OwnerKind, handler.OwnerName)
+	}
+
 	logDir := strings.TrimSpace(filepath.Join(config.SessionDir, utils.RAY_SESSIONDIR_LOGDIR_NAME))
 	handler.LogDir = logDir
-	// clusterRootDir uses flat key format (name_id) for S3/OSS performance optimization.
-	// See utils.connector for the design rationale.
 	clusterRootDir := fmt.Sprintf("%s/", path.Clean(path.Join(handler.RootDir, utils.AppendRayClusterNameNamespace(handler.RayClusterName, handler.RayClusterNamespace))))
 	handler.ClusterDir = clusterRootDir
 

historyserver/pkg/collector/types/types.go

@@ -18,6 +18,8 @@ type RayCollectorConfig struct {
 	Role                string
 	RayClusterName      string
 	RayClusterNamespace string
+	OwnerKind           string
+	OwnerName           string
 	LogBatching         int
 	PushInterval        time.Duration
 	DashboardAddress    string

historyserver/pkg/compression/compression.go

@@ -0,0 +1,158 @@
+package compression
+
+import (
+	"bytes"
+	"compress/gzip"
+	"errors"
+	"fmt"
+	"io"
+	"os"
+
+	"github.com/sirupsen/logrus"
+)
+
+// CompressStream stream data in fixed chunks from src to dst through gzip,
+// ensuring constant memory usage regardless of payload size.
+func CompressStream(dst io.Writer, src io.Reader) error {
+	w := gzip.NewWriter(dst)
+	if _, err := io.Copy(w, src); err != nil {
+		w.Close()
+		return err
+	}
+	return w.Close()
+}
+
+// DecompressStream stream data in fixed chunks from src to dst through gzip.
+func DecompressStream(dst io.Writer, src io.Reader) error {
+	r, err := gzip.NewReader(src)
+	if err != nil {
+		return err
+	}
+	defer func() {
+		if err := r.Close(); err != nil {
+			logrus.Warnf("compression: failed to close gzip reader: %v", err)
+		}
+	}()
+
+	_, err = io.Copy(dst, r)
+	return err
+}
+
+// StorageWriter defines the duck-typed interface for storage handlers to avoid cyclic dependencies
+type StorageWriter interface {
+	WriteFile(file string, reader io.ReadSeeker) error
+}
+
+// WriteCompressedFile compresses a local file on the fly and uploads it to any active cloud provider.
+// It utilizes disk-based spooling to maintain constant memory usage (zero RAM expansion) regardless
+// of file size.
+func WriteCompressedFile(writer StorageWriter, remotePath string, localFilePath string) error {
+	// 1. Opens the uncompressed local file on disk.
+	rawFile, err := os.Open(localFilePath)
+	if err != nil {
+		return fmt.Errorf("compression: failed to open local file %q: %w", localFilePath, err)
+	}
+	defer func() {
+		if err := rawFile.Close(); err != nil {
+			logrus.Warnf("compression: failed to close local file %q: %v", localFilePath, err)
+		}
+	}()
+
+	// 2. Spools compressed output into an ephemeral temporary file via CompressStream.
+	tmpFile, err := os.CreateTemp("", "upload-*.gz")
+	if err != nil {
+		return fmt.Errorf("compression: failed to create temp file: %w", err)
+	}
+	defer func() {
+		if err := tmpFile.Close(); err != nil {
+			logrus.Warnf("compression: failed to close temp file %q: %v", tmpFile.Name(), err)
+		}
+		if err := os.Remove(tmpFile.Name()); err != nil {
+			logrus.Warnf("compression: failed to remove temp file %q: %v", tmpFile.Name(), err)
+		}
+	}()
+
+	if err := CompressStream(tmpFile, rawFile); err != nil {
+		return fmt.Errorf("compression: stream compression failed: %w", err)
+	}
+
+	// 3. Seeks the temp file to the beginning to satisfy io.ReadSeeker, which is required by cloud SDKs.
+	if _, err := tmpFile.Seek(0, io.SeekStart); err != nil {
+		return fmt.Errorf("compression: failed to seek temp file: %w", err)
+	}
+
+	// 4. Delegates the upload to the duck-typed StorageWriter.
+	if err := writer.WriteFile(remotePath, tmpFile); err != nil {
+		return fmt.Errorf("compression: failed to upload compressed file: %w", err)
+	}
+
+	return nil
+}
+
+// WriteCompressedBytes compresses an in-memory byte slice and uploads it to any active cloud provider.
+func WriteCompressedBytes(writer StorageWriter, remotePath string, data []byte) error {
+	var buf bytes.Buffer
+	if err := CompressStream(&buf, bytes.NewReader(data)); err != nil {
+		return fmt.Errorf("compression: in-memory compression failed: %w", err)
+	}
+
+	if err := writer.WriteFile(remotePath, bytes.NewReader(buf.Bytes())); err != nil {
+		return fmt.Errorf("compression: failed to upload compressed bytes: %w", err)
+	}
+
+	return nil
+}
+
+// StorageContentReader defines the duck-typed interface for storage readers to avoid cyclic dependencies.
+type StorageContentReader interface {
+	GetContent(clusterId string, fileName string) io.Reader
+}
+
+// gzipReadCloser wraps a *gzip.Reader and the underlying reader so both are closed properly.
+type gzipReadCloser struct {
+	gzipReader *gzip.Reader
+	underlying io.Reader
+}
+
+func (g *gzipReadCloser) Read(p []byte) (n int, err error) {
+	return g.gzipReader.Read(p)
+}
+
+func (g *gzipReadCloser) Close() error {
+	var errs []error
+
+	if err := g.gzipReader.Close(); err != nil {
+		errs = append(errs, fmt.Errorf("gzip close: %w", err))
+	}
+
+	if closer, ok := g.underlying.(io.Closer); ok {
+		if err := closer.Close(); err != nil {
+			errs = append(errs, fmt.Errorf("source close: %w", err))
+		}
+	}
+
+	return errors.Join(errs...)
+}
+
+// ReadCompressedContent reads a gzip-compressed file from cloud storage and returns an io.ReadCloser.
+func ReadCompressedContent(reader StorageContentReader, clusterId string, fileName string) (io.ReadCloser, error) {
+	contentReader := reader.GetContent(clusterId, fileName)
+	if contentReader == nil {
+		return nil, errors.New("compression: file not found or nil reader returned by storage")
+	}
+
+	r, err := gzip.NewReader(contentReader)
+	if err != nil {
+		if closer, ok := contentReader.(io.Closer); ok {
+			if err := closer.Close(); err != nil {
+				logrus.Warnf("compression: failed to close underlying stream after gzip reader creation error: %v", err)
+			}
+		}
+		return nil, fmt.Errorf("compression: failed to create gzip reader: %w", err)
+	}
+
+	return &gzipReadCloser{
+		gzipReader: r,
+		underlying: contentReader,
+	}, nil
+}

historyserver/pkg/compression/compression_test.go

@@ -0,0 +1,129 @@
+package compression
+
+import (
+	"bytes"
+	"io"
+	"os"
+	"testing"
+)
+
+func compressBytesForTest(data []byte) ([]byte, error) {
+	var buf bytes.Buffer
+	err := CompressStream(&buf, bytes.NewReader(data))
+	return buf.Bytes(), err
+}
+
+func decompressBytesForTest(data []byte) ([]byte, error) {
+	var buf bytes.Buffer
+	err := DecompressStream(&buf, bytes.NewReader(data))
+	return buf.Bytes(), err
+}
+
+func TestCompressDecompressStream(t *testing.T) {
+	originalData := []byte("Streaming log compression payload testing across io.Reader and io.Writer")
+
+	var compressedBuf bytes.Buffer
+	if err := CompressStream(&compressedBuf, bytes.NewReader(originalData)); err != nil {
+		t.Fatalf("CompressStream failed: %v", err)
+	}
+
+	var decompressedBuf bytes.Buffer
+	if err := DecompressStream(&decompressedBuf, &compressedBuf); err != nil {
+		t.Fatalf("DecompressStream failed: %v", err)
+	}
+
+	if !bytes.Equal(originalData, decompressedBuf.Bytes()) {
+		t.Fatalf("Stream decompressed data mismatch.\nExpected: %s\nGot: %s", originalData, decompressedBuf.Bytes())
+	}
+}
+
+type mockStorageWriter struct {
+	writtenData []byte
+}
+
+func (m *mockStorageWriter) WriteFile(file string, reader io.ReadSeeker) error {
+	var err error
+	m.writtenData, err = io.ReadAll(reader)
+	return err
+}
+
+func TestWriteCompressedFile(t *testing.T) {
+	tmpFile, err := os.CreateTemp("", "test-raw-*.txt")
+	if err != nil {
+		t.Fatalf("CreateTemp failed: %v", err)
+	}
+	defer os.Remove(tmpFile.Name())
+
+	rawData := []byte("upload compression payload test")
+	if _, err := tmpFile.Write(rawData); err != nil {
+		t.Fatalf("Write failed: %v", err)
+	}
+	tmpFile.Close()
+
+	mockWriter := &mockStorageWriter{}
+	if err := WriteCompressedFile(mockWriter, "test/remote.gz", tmpFile.Name()); err != nil {
+		t.Fatalf("WriteCompressedFile failed: %v", err)
+	}
+
+	decompressed, err := decompressBytesForTest(mockWriter.writtenData)
+	if err != nil {
+		t.Fatalf("decompressBytesForTest failed: %v", err)
+	}
+
+	if !bytes.Equal(rawData, decompressed) {
+		t.Fatalf("Decompressed written data mismatch.\nExpected: %s\nGot: %s", rawData, decompressed)
+	}
+}
+
+func TestWriteCompressedBytes(t *testing.T) {
+	rawData := []byte("upload in-memory bytes payload test")
+	mockWriter := &mockStorageWriter{}
+
+	if err := WriteCompressedBytes(mockWriter, "test/bytes.gz", rawData); err != nil {
+		t.Fatalf("WriteCompressedBytes failed: %v", err)
+	}
+
+	decompressed, err := decompressBytesForTest(mockWriter.writtenData)
+	if err != nil {
+		t.Fatalf("decompressBytesForTest failed: %v", err)
+	}
+
+	if !bytes.Equal(rawData, decompressed) {
+		t.Fatalf("Decompressed written bytes mismatch.\nExpected: %s\nGot: %s", rawData, decompressed)
+	}
+}
+
+type mockStorageReader struct {
+	content []byte
+}
+
+func (m *mockStorageReader) GetContent(clusterId string, fileName string) io.Reader {
+	if m.content == nil {
+		return nil
+	}
+	return bytes.NewReader(m.content)
+}
+
+func TestReadCompressedContent(t *testing.T) {
+	originalText := []byte("Universal cloud storage reader decompression test")
+	compressed, err := compressBytesForTest(originalText)
+	if err != nil {
+		t.Fatalf("compressBytesForTest failed: %v", err)
+	}
+
+	mockReader := &mockStorageReader{content: compressed}
+	rc, err := ReadCompressedContent(mockReader, "cluster-1", "log.gz")
+	if err != nil {
+		t.Fatalf("ReadCompressedContent failed: %v", err)
+	}
+	defer rc.Close()
+
+	decompressed, err := io.ReadAll(rc)
+	if err != nil {
+		t.Fatalf("io.ReadAll failed: %v", err)
+	}
+
+	if !bytes.Equal(originalText, decompressed) {
+		t.Fatalf("Decompressed read content mismatch.\nExpected: %s\nGot: %s", originalText, decompressed)
+	}
+}

historyserver/pkg/eventserver/eventprocessor_interface.go

@@ -1,7 +0,0 @@
-package eventserver
-
-import "context"
-
-type EventProcessor[T any] interface {
-	ProcessEvents(ctx context.Context, ch <-chan T) error
-}

historyserver/pkg/eventserver/eventserver.go

@@ -9,13 +9,14 @@ import (
 	"sort"
 	"strconv"
 	"strings"
-	"sync"
 	"time"
 
+	"github.com/sirupsen/logrus"
+
+	"github.com/ray-project/kuberay/historyserver/pkg/compression"
 	"github.com/ray-project/kuberay/historyserver/pkg/eventserver/types"
 	"github.com/ray-project/kuberay/historyserver/pkg/storage"
 	"github.com/ray-project/kuberay/historyserver/pkg/utils"
-	"github.com/sirupsen/logrus"
 )
 
 type EventHandler struct {
@@ -28,7 +29,7 @@ type EventHandler struct {
 	ClusterLogEventMap *types.ClusterLogEventMap // For /events API (Log Events from logs/events/)
 }
 
-var eventFilePattern = regexp.MustCompile(`-\d{4}-\d{2}-\d{2}-\d{2}$`)
+var eventFilePattern = regexp.MustCompile(`-\d{4}-\d{2}-\d{2}-\d{2}(\.gz)?$`)
 
 // taskPrefix is extracted to avoid hard-coded "task::" usage
 const taskPrefix = "task::"
@@ -61,155 +62,8 @@ func NewEventHandler(reader storage.StorageReader) *EventHandler {
 	}
 }
 
-// ProcessEvents func reads the channel and then processes the event received
-func (h *EventHandler) ProcessEvents(ctx context.Context, ch <-chan map[string]any) error {
-	logrus.Infof("Starting a event processor channel")
-	for {
-		select {
-		case <-ctx.Done():
-			// TODO: The context was cancelled, either stop here or process the rest of the events and return
-			// Currently, it will just stop.
-			logrus.Warnf("Event processor context was cancelled")
-			return ctx.Err()
-		case currEventData, ok := <-ch:
-			if !ok {
-				logrus.Warnf("Channel was closed")
-				return nil
-			}
-			if err := h.storeEvent(currEventData); err != nil {
-				logrus.Errorf("Failed to store event: %v", err)
-				continue
-			}
-		}
-	}
-}
-
-// Run will start numOfEventProcessors (default to 5) processing functions and the event reader. The event reader will run once an hr,
-// which is currently how often the collector flushes.
-func (h *EventHandler) Run(stop chan struct{}, numOfEventProcessors int) error {
-	var wg sync.WaitGroup
-
-	if numOfEventProcessors == 0 {
-		numOfEventProcessors = 5
-	}
-	eventProcessorChannels := make([]chan map[string]any, numOfEventProcessors)
-	cctx := make([]context.CancelFunc, numOfEventProcessors)
-
-	for i := range numOfEventProcessors {
-		eventProcessorChannels[i] = make(chan map[string]any, 100)
-	}
-
-	for i, currEventChannel := range eventProcessorChannels {
-		wg.Add(1)
-		ctx, cancel := context.WithCancel(context.Background())
-		cctx[i] = cancel
-		go func() {
-			defer wg.Done()
-			var processor EventProcessor[map[string]any] = h
-			err := processor.ProcessEvents(ctx, currEventChannel)
-			if err == ctx.Err() {
-				logrus.Warnf("Event processor go routine %d is now closed", i)
-				return
-			}
-			if err != nil {
-				logrus.Errorf("event processor %d go routine failed %v", i, err)
-				return
-			}
-		}()
-	}
-
-	// Start reading files and sending events for processing
-	wg.Add(1)
-	go func() {
-		defer wg.Done()
-		logrus.Info("Starting event file reader loop")
-
-		// Create a LogEventReader for reading logs/events/event_*.log files
-		logEventReader := NewLogEventReader(h.reader)
-
-		// Helper function to process all events
-		processAllEvents := func() {
-			clusterList := h.reader.List()
-			for _, clusterInfo := range clusterList {
-				clusterNameNamespace := clusterInfo.Name + "_" + clusterInfo.Namespace
-				clusterSessionKey := utils.BuildClusterSessionKey(clusterInfo.Name, clusterInfo.Namespace, clusterInfo.SessionName)
-
-				// Read Log Events from logs/{nodeId}/events/event_*.log
-				// This is the format used by Ray Dashboard's /events API
-				if err := logEventReader.ReadLogEvents(clusterInfo, clusterSessionKey, h.ClusterLogEventMap); err != nil {
-					logrus.Errorf("Failed to read Log Events for %s: %v", clusterSessionKey, err)
-				}
-
-				// Also read RayEvents (Export Events) from node_events/ and job_events/ for backward compatibility
-				// These are used for task/actor/job/node data APIs
-				eventFileList := append(h.getAllJobEventFiles(clusterInfo), h.getAllNodeEventFiles(clusterInfo)...)
-
-				logrus.Infof("current eventFileList for cluster %s is: %v", clusterInfo.Name, eventFileList)
-				for _, eventFile := range eventFileList {
-					// TODO: Filter out ones that have already been read
-					logrus.Infof("Reading event file: %s", eventFile)
-
-					eventioReader := h.reader.GetContent(clusterNameNamespace, eventFile)
-					if eventioReader == nil {
-						logrus.Errorf("Failed to get content for event file: %s, skipping", eventFile)
-						continue
-					}
-					eventbytes, err := io.ReadAll(eventioReader)
-					if err != nil {
-						logrus.Errorf("Failed to read event file: %v", err)
-						continue
-					}
-
-					var eventList []map[string]any
-					if err := json.Unmarshal(eventbytes, &eventList); err != nil {
-						logrus.Errorf("Failed to unmarshal event: %v", err)
-						continue
-					}
-
-					// Evenly distribute events to each channel
-					for i, curr := range eventList {
-						// Skip nil events (can occur with corrupted event files containing null elements)
-						if curr == nil {
-							continue
-						}
-						curr["clusterName"] = clusterSessionKey
-						eventProcessorChannels[i%numOfEventProcessors] <- curr
-					}
-				}
-			}
-		}
-
-		// Process events immediately on startup
-		processAllEvents()
-
-		// Create a ticker for hourly processing
-		ticker := time.NewTicker(1 * time.Hour)
-		defer ticker.Stop()
-
-		for {
-			logrus.Info("Finished reading files, waiting for next cycle...")
-			select {
-			case <-stop:
-				// Received stop signal, clean up and exit
-				for i, currChan := range eventProcessorChannels {
-					close(currChan)
-					cctx[i]()
-				}
-				logrus.Info("Event processor received stop signal, exiting.")
-				return
-			case <-ticker.C:
-				// Process events every hour
-				processAllEvents()
-			}
-		}
-	}()
-
-	wg.Wait()
-	return nil
-}
-
-// storeEvent unmarshals the event map into the correct actor/task struct and then stores it into the corresonding list
-func (h *EventHandler) storeEvent(eventMap map[string]any) error {
+// storeEvent unmarshals the event map into the correct actor/task struct and then stores it into the corresonding list.
+func (h *EventHandler) storeEvent(clusterSessionKey string, eventMap map[string]any) error {
 	eventTypeVal, ok := eventMap["eventType"]
 	if !ok {
 		return fmt.Errorf("event missing 'eventType' field")
@@ -220,17 +74,6 @@ func (h *EventHandler) storeEvent(eventMap map[string]any) error {
 	}
 	eventType := types.EventType(eventTypeStr)
 
-	// clusterSessionKey is injected during event reading (Run function) and contains
-	// the full key: "{clusterName}_{namespace}_{sessionName}"
-	clusterSessionKeyVal, ok := eventMap["clusterName"]
-	if !ok {
-		return fmt.Errorf("event missing 'clusterName' field (clusterSessionKey)")
-	}
-	clusterSessionKey, ok := clusterSessionKeyVal.(string)
-	if !ok {
-		return fmt.Errorf("clusterName is not a string, got %T", clusterSessionKeyVal)
-	}
-
 	logrus.Infof("current eventType: %v", eventType)
 	switch eventType {
 	case types.TASK_DEFINITION_EVENT:
@@ -661,13 +504,13 @@ func (h *EventHandler) getAllNodeEventFiles(clusterInfo utils.ClusterInfo) []str
 
 // GetTasks returns a slice of thread-safe deep copies of all task attempts for a given cluster session.
 // Deep copy ensures the returned data is safe to use after the lock is released.
-func (h *EventHandler) GetTasks(clusterSessionKey string) []types.Task {
+func (h *EventHandler) getTasks(clusterSessionKey string) []types.Task {
 	h.ClusterTaskMap.RLock()
 	defer h.ClusterTaskMap.RUnlock()
 
 	taskMap, ok := h.ClusterTaskMap.ClusterTaskMap[clusterSessionKey]
 	if !ok {
-		// TODO(jwj): Add error handling.
+		// TODO(jiangjiawei1103): Add error handling.
 		logrus.Errorf("Task map not found for cluster session: %s", clusterSessionKey)
 		return []types.Task{}
 	}
@@ -686,100 +529,8 @@ func (h *EventHandler) GetTasks(clusterSessionKey string) []types.Task {
 	return allTasks
 }
 
-// GetTaskByID returns all attempts for a specific task ID in a given cluster.
-// Returns a slice of tasks representing all attempts, sorted by attempt number is not guaranteed.
-func (h *EventHandler) GetTaskByID(clusterName, taskID string) ([]types.Task, bool) {
-	h.ClusterTaskMap.RLock()
-	defer h.ClusterTaskMap.RUnlock()
-
-	taskMap, ok := h.ClusterTaskMap.ClusterTaskMap[clusterName]
-	if !ok {
-		return nil, false
-	}
-
-	taskMap.Lock()
-	defer taskMap.Unlock()
-
-	attempts, ok := taskMap.TaskMap[taskID]
-	if !ok || len(attempts) == 0 {
-		return nil, false
-	}
-	// Return a deep copy to avoid data race
-	result := make([]types.Task, len(attempts))
-	for i, task := range attempts {
-		result[i] = task.DeepCopy()
-	}
-	return result, true
-}
-
-// GetTasksByJobID returns all tasks (including all attempts) for a given job ID in a cluster.
-func (h *EventHandler) GetTasksByJobID(clusterName, jobID string) []types.Task {
-	h.ClusterTaskMap.RLock()
-	defer h.ClusterTaskMap.RUnlock()
-
-	taskMap, ok := h.ClusterTaskMap.ClusterTaskMap[clusterName]
-	if !ok {
-		return []types.Task{}
-	}
-
-	taskMap.Lock()
-	defer taskMap.Unlock()
-
-	var tasks []types.Task
-	for _, attempts := range taskMap.TaskMap {
-		for _, task := range attempts {
-			if task.JobID == jobID {
-				tasks = append(tasks, task.DeepCopy())
-			}
-		}
-	}
-	return tasks
-}
-
-// GetActors returns a thread-safe deep copy of all actors for a given cluster
-func (h *EventHandler) GetActors(clusterName string) []types.Actor {
-	h.ClusterActorMap.RLock()
-	defer h.ClusterActorMap.RUnlock()
-
-	actorMap, ok := h.ClusterActorMap.ClusterActorMap[clusterName]
-	if !ok {
-		return []types.Actor{}
-	}
-
-	actorMap.Lock()
-	defer actorMap.Unlock()
-
-	actors := make([]types.Actor, 0, len(actorMap.ActorMap))
-	for _, actor := range actorMap.ActorMap {
-		actors = append(actors, actor.DeepCopy())
-	}
-	return actors
-}
-
-// GetActorByID returns a specific actor by ID for a given cluster
-func (h *EventHandler) GetActorByID(clusterName, actorID string) (types.Actor, bool) {
-	h.ClusterActorMap.RLock()
-	defer h.ClusterActorMap.RUnlock()
-
-	actorMap, ok := h.ClusterActorMap.ClusterActorMap[clusterName]
-	if !ok {
-		return types.Actor{}, false
-	}
-
-	actorMap.Lock()
-	defer actorMap.Unlock()
-
-	// Actor IDs are normalized to hex at ingestion time (normalizeActorIDsToHex),
-	// so direct lookup by hex ID always succeeds.
-	actor, ok := actorMap.ActorMap[actorID]
-	if !ok {
-		return types.Actor{}, false
-	}
-	return actor.DeepCopy(), true
-}
-
 // GetActorsMap returns a thread-safe deep copy of all actors as a map for a given cluster
-func (h *EventHandler) GetActorsMap(clusterName string) map[string]types.Actor {
+func (h *EventHandler) getActorsMap(clusterName string) map[string]types.Actor {
 	h.ClusterActorMap.RLock()
 	defer h.ClusterActorMap.RUnlock()
 
@@ -798,7 +549,7 @@ func (h *EventHandler) GetActorsMap(clusterName string) map[string]types.Actor {
 	return actors
 }
 
-func (h *EventHandler) GetJobsMap(clusterName string) map[string]types.Job {
+func (h *EventHandler) getJobsMap(clusterName string) map[string]types.Job {
 	h.ClusterJobMap.RLock()
 	defer h.ClusterJobMap.RUnlock()
 
@@ -817,23 +568,9 @@ func (h *EventHandler) GetJobsMap(clusterName string) map[string]types.Job {
 	return jobs
 }
 
-func (h *EventHandler) GetJobByJobID(clusterName, jobID string) (types.Job, bool) {
-	h.ClusterJobMap.RLock()
-	defer h.ClusterJobMap.RUnlock()
-
-	jobMap, ok := h.ClusterJobMap.ClusterJobMap[clusterName]
-	if !ok {
-		return types.Job{}, false
-	}
-
-	jobMap.Lock()
-	defer jobMap.Unlock()
-
-	job, ok := jobMap.JobMap[jobID]
-	if !ok {
-		return types.Job{}, false
-	}
-	return job.DeepCopy(), true
+// getLogEventsByJobID returns a thread-safe deep copy of log events grouped by job ID.
+func (h *EventHandler) getLogEventsByJobID(clusterSessionKey string) map[string][]types.LogEvent {
+	return h.ClusterLogEventMap.GetLogEventsByJobID(clusterSessionKey)
 }
 
 // handleTaskDefinitionEvent processes TASK_DEFINITION_EVENT or ACTOR_TASK_DEFINITION_EVENT and preserves the task attempt ordering.
@@ -938,7 +675,7 @@ func (h *EventHandler) handleTaskLifecycleEvent(eventMap map[string]any, cluster
 	}
 	normalizeTaskIDsToHex(&currTask)
 
-	// TODO(jwj): Clarify if there must be at least one state transition. Can one task have more than one state transition?
+	// TODO(jiangjiawei1103): Clarify if there must be at least one state transition. Can one task have more than one state transition?
 	if len(currTask.StateTransitions) == 0 {
 		return fmt.Errorf("TASK_LIFECYCLE_EVENT must have at least one state transition")
 	}
@@ -974,7 +711,7 @@ func (h *EventHandler) handleTaskLifecycleEvent(eventMap map[string]any, cluster
 			return
 		}
 
-		// TODO(jwj): Before beta, the lifecycle-related fields are overwritten.
+		// TODO(jiangjiawei1103): Before beta, the lifecycle-related fields are overwritten.
 		// In beta, the complete historical replay will be supported.
 		task.RayErrorInfo = currTask.RayErrorInfo
 		if currTask.JobID != "" {
@@ -1255,8 +992,8 @@ func normalizeActorIDsToHex(actor *types.Actor) {
 	actor.Address.WorkerID = normalizeIDToHex(actor.Address.WorkerID)
 }
 
-// GetNodeMap returns a thread-safe deep copy of all nodes for a given cluster session.
-func (h *EventHandler) GetNodeMap(clusterSessionID string) map[string]types.Node {
+// getNodeMap returns a thread-safe deep copy of all nodes for a given cluster session.
+func (h *EventHandler) getNodeMap(clusterSessionID string) map[string]types.Node {
 	h.ClusterNodeMap.RLock()
 	defer h.ClusterNodeMap.RUnlock()
 
@@ -1275,259 +1012,84 @@ func (h *EventHandler) GetNodeMap(clusterSessionID string) map[string]types.Node
 	return nodes
 }
 
-// GetNodeByNodeID returns a node by node ID for a given cluster session.
-func (h *EventHandler) GetNodeByNodeID(clusterSessionID, nodeID string) (types.Node, bool) {
-	h.ClusterNodeMap.RLock()
-	defer h.ClusterNodeMap.RUnlock()
+// ProcessSingleSession reads all event files for a single session synchronously
+// and populates the handler's in-memory maps.
+//
+// TODO(jiangjiawei1103): Empty event file list vs ListFiles outage is ambiguous without
+// StorageReader interface surfacing errors.
+func (h *EventHandler) ProcessSingleSession(ctx context.Context, clusterInfo utils.ClusterInfo) error {
+	clusterNameNamespace := clusterInfo.Name + "_" + clusterInfo.Namespace
+	clusterSessionKey := utils.BuildClusterSessionKey(clusterInfo.Name, clusterInfo.Namespace, clusterInfo.SessionName)
 
-	nodeMap, ok := h.ClusterNodeMap.ClusterNodeMap[clusterSessionID]
-	if !ok {
-		return types.Node{}, false
+	// ClusterLogEventMap backs only the /events endpoint, so log event read failures must not
+	// block marking the session as loaded and force subsequent Ray event re-processing.
+	logEventReader := NewLogEventReader(h.reader)
+	if err := logEventReader.ReadLogEvents(clusterInfo, clusterSessionKey, h.ClusterLogEventMap); err != nil {
+		logrus.Errorf("Incomplete Log Events read for %s: %v. /events endpoint may serve partial data.",
+			clusterSessionKey, err)
 	}
 
-	nodeMap.Lock()
-	defer nodeMap.Unlock()
+	eventFileList := append(h.getAllJobEventFiles(clusterInfo), h.getAllNodeEventFiles(clusterInfo)...)
+	logrus.Debugf("current eventFileList for cluster %s is: %v", clusterInfo.Name, eventFileList)
 
-	node, ok := nodeMap.NodeMap[nodeID]
-	if !ok {
-		return types.Node{}, false
-	}
-	return node.DeepCopy(), true
-}
-
-// GetTasksTimeline returns timeline data in Chrome Tracing Format
-// Output format matches Ray Dashboard's /api/v0/tasks/timeline endpoint
-func (h *EventHandler) GetTasksTimeline(clusterName string, jobID string) []types.ChromeTraceEvent {
-	var tasks []types.Task
-	if jobID != "" {
-		tasks = h.GetTasksByJobID(clusterName, jobID)
-	} else {
-		tasks = h.GetTasks(clusterName)
-	}
-
-	if len(tasks) == 0 {
-		return []types.ChromeTraceEvent{}
-	}
-
-	events := []types.ChromeTraceEvent{}
-
-	filteredTasks := make([]types.Task, 0, len(tasks))
-	for _, task := range tasks {
-		if task.ProfileData == nil || len(task.ProfileData.Events) == 0 {
-			continue
+	rayEventsTotal := len(eventFileList)
+	rayEventsRead := 0
+	for _, eventFile := range eventFileList {
+		if err := ctx.Err(); err != nil {
+			return err
 		}
-		// Only include worker and driver components (consistent with Ray's profiling implementation in profiling.py)
-		componentType := task.ProfileData.ComponentType
-		if componentType != "worker" && componentType != "driver" {
-			continue
-		}
-		if task.ProfileData.NodeIPAddress == "" {
-			continue
-		}
-		filteredTasks = append(filteredTasks, task)
-	}
+		logrus.Debugf("Reading event file: %s", eventFile)
 
-	// Build PID/TID mappings
-	// PID: Node IP -> numeric ID
-	// TID: clusterID (componentType:componentId) -> globally unique numeric ID
-	nodeIPToPID := make(map[string]int)
-	nodeIPToClusterIDToTID := make(map[string]map[string]int) // nodeIP -> clusterID (componentType:componentId) -> tid
-	pidCounter := 0
-	tidCounter := 0
-
-	// First pass: collect all unique nodes and workers
-	for _, task := range filteredTasks {
-		nodeIP := task.ProfileData.NodeIPAddress
-		clusterID := task.ProfileData.ComponentType + ":" + task.ProfileData.ComponentID
-		if _, exists := nodeIPToPID[nodeIP]; !exists {
-			nodeIPToPID[nodeIP] = pidCounter
-			pidCounter++
-			nodeIPToClusterIDToTID[nodeIP] = make(map[string]int)
-		}
-		if _, exists := nodeIPToClusterIDToTID[nodeIP][clusterID]; !exists {
-			nodeIPToClusterIDToTID[nodeIP][clusterID] = tidCounter
-			tidCounter++
-		}
-	}
-
-	// Generate process_name and thread_name metadata events
-	for nodeIP, pid := range nodeIPToPID {
-		events = append(events, types.ChromeTraceEvent{
-			Name:  "process_name",
-			PID:   pid,
-			TID:   nil,
-			Phase: "M",
-			Args: map[string]interface{}{
-				"name": "Node " + nodeIP,
-			},
-		})
-
-		for clusterID, tid := range nodeIPToClusterIDToTID[nodeIP] {
-			tidVal := tid
-			events = append(events, types.ChromeTraceEvent{
-				Name:  "thread_name",
-				PID:   pid,
-				TID:   &tidVal,
-				Phase: "M",
-				Args: map[string]interface{}{
-					"name": clusterID,
-				},
-			})
-		}
-	}
-
-	// Generate trace events from ProfileData
-	for _, task := range filteredTasks {
-		nodeIP := task.ProfileData.NodeIPAddress
-		clusterID := task.ProfileData.ComponentType + ":" + task.ProfileData.ComponentID
-
-		pid, ok := nodeIPToPID[nodeIP]
-		if !ok {
-			continue
-		}
-		var tidPtr *int
-		if tid, ok := nodeIPToClusterIDToTID[nodeIP][clusterID]; ok {
-			tidVal := tid
-			tidPtr = &tidVal
+		var eventioReader io.Reader
+		if strings.HasSuffix(eventFile, ".gz") {
+			rc, err := compression.ReadCompressedContent(h.reader, clusterNameNamespace, eventFile)
+			if err != nil {
+				logrus.Errorf("Failed to decompress event file %s: %v", eventFile, err)
+				continue
+			}
+			eventioReader = rc
 		} else {
-			// This shouldn't happen if first pass worked correctly,
-			// but skip to avoid null TID
+			eventioReader = h.reader.GetContent(clusterNameNamespace, eventFile)
+		}
+
+		if eventioReader == nil {
+			logrus.Errorf("Failed to get content for event file: %s, skipping", eventFile)
 			continue
 		}
 
-		for _, profEvent := range task.ProfileData.Events {
-			// Convert nanoseconds to microseconds
-			startTimeUs := float64(profEvent.StartTime) / 1000.0
-			durationUs := float64(profEvent.EndTime-profEvent.StartTime) / 1000.0
+		eventbytes, err := io.ReadAll(eventioReader)
+		if closer, ok := eventioReader.(io.Closer); ok {
+			closer.Close()
+		}
 
-			// Parse extraData for additional fields
-			var extraData map[string]interface{}
-			if profEvent.ExtraData != "" {
-				json.Unmarshal([]byte(profEvent.ExtraData), &extraData)
+		if err != nil {
+			logrus.Errorf("Failed to read events for file %s: %v", eventFile, err)
+			continue
+		}
+		rayEventsRead++
+
+		// json.Unmarshal and storeEvent failures are treated as corrupt-data errors:
+		// retrying won't fix bad bytes, accepting partial loss.
+		var eventList []map[string]any
+		if err := json.Unmarshal(eventbytes, &eventList); err != nil {
+			logrus.Errorf("Failed to unmarshal events for file %s: %v", eventFile, err)
+			continue
+		}
+
+		for _, event := range eventList {
+			if event == nil {
+				continue
 			}
-
-			// Determine task_id and func_or_class_name
-			taskIDForArgs := task.TaskID
-			funcOrClassName := task.FuncOrClassName
-
-			// Fallback to GetFuncName() if FuncOrClassName is empty
-			// This ensures consistency with Ray's profiling.py which uses task["func_or_class_name"]
-			// from TASK_DEFINITION_EVENT, and handles cases where TASK_PROFILE_EVENT is missing
-			if funcOrClassName == "" {
-				funcOrClassName = task.GetFuncName()
+			if err := h.storeEvent(clusterSessionKey, event); err != nil {
+				logrus.Errorf("Failed to store events for file %s: %v", eventFile, err)
+				continue
 			}
-
-			// Try to get from extraData if available (for hex format task_id)
-			if extraData != nil {
-				if tid, ok := extraData["task_id"].(string); ok && tid != "" {
-					taskIDForArgs = tid
-				}
-			}
-
-			// Build args
-			actorID := extractActorIDFromTaskID(taskIDForArgs)
-			args := map[string]interface{}{
-				"task_id":            taskIDForArgs,
-				"job_id":             task.JobID,
-				"attempt_number":     task.TaskAttempt,
-				"func_or_class_name": funcOrClassName,
-				"actor_id":           nil,
-			}
-
-			if actorID != "" {
-				args["actor_id"] = actorID
-			}
-
-			// Determine event name for display
-			eventName := profEvent.EventName
-			displayName := profEvent.EventName
-
-			// For overall task events like "task::slow_task", use the full name from extraData
-			if strings.HasPrefix(profEvent.EventName, taskPrefix) && extraData != nil {
-				if name, ok := extraData["name"].(string); ok && name != "" {
-					displayName = name
-					args["name"] = name
-				}
-			}
-
-			traceEvent := types.ChromeTraceEvent{
-				Category:  eventName,
-				Name:      displayName,
-				PID:       pid,
-				TID:       tidPtr,
-				Timestamp: &startTimeUs,
-				Duration:  &durationUs,
-				Color:     getChromeTraceColor(eventName),
-				Args:      args,
-				Phase:     "X",
-			}
-
-			events = append(events, traceEvent)
 		}
 	}
 
-	return events
-}
-
-// getChromeTraceColor maps event names to Chrome trace colors
-// Based on Ray's _default_color_mapping in profiling.py
-func getChromeTraceColor(eventName string) string {
-	// Handle task::xxx pattern (overall task event)
-	if strings.HasPrefix(eventName, taskPrefix) {
-		return "generic_work"
+	if rayEventsTotal > 0 && rayEventsRead == 0 {
+		return fmt.Errorf("read 0 of %d event files for %s: likely transient storage outage",
+			rayEventsTotal, clusterSessionKey)
 	}
-
-	// Direct mapping for known event names
-	// This logic follows Ray's profiling implementation:
-	// https://github.com/ray-project/ray/blob/68d01c4c48a59c7768ec9c2359a1859966c446b6/python/ray/_private/profiling.py#L25
-	switch eventName {
-	case "task:deserialize_arguments":
-		return "rail_load"
-	case "task:execute":
-		return "rail_animation"
-	case "task:store_outputs":
-		return "rail_idle"
-	case "task:submit_task", "task":
-		return "rail_response"
-	case "worker_idle":
-		return "cq_build_abandoned"
-	case "ray.get":
-		return "good"
-	case "ray.put":
-		return "terrible"
-	case "ray.wait":
-		return "vsync_highlight_color"
-	case "submit_task":
-		return "background_memory_dump"
-	case "wait_for_function", "fetch_and_run_function", "register_remote_function":
-		return "detailed_memory_dump"
-	default:
-		return "generic_work"
-	}
-}
-
-// extractActorIDFromTaskID extracts the ActorID from a TaskID following Ray's ID specification.
-//
-// Design doc: src/ray/design_docs/id_specification.md
-// - TaskID: 8B unique + 16B ActorID (total 24 bytes = 48 hex chars)
-// - ActorID: 12B unique + 4B JobID (total 16 bytes = 32 hex chars)
-//
-// For a 48-character hex TaskID, the last 32 hex characters (bytes 16–48)
-// correspond to the ActorID. This function further checks the "unique" portion
-// of the ActorID (first 24 hex chars) and returns an empty string if it is all Fs,
-// which indicates normal/driver tasks with no associated actor.
-func extractActorIDFromTaskID(taskIDHex string) string {
-	if len(taskIDHex) != 48 {
-		return "" // can't process if encoded in base64
-	}
-
-	actorPortion := taskIDHex[16:40] // 24 chars for actor id (12 bytes)
-	jobPortion := taskIDHex[40:48]   // 8 chars for job id (4 bytes)
-
-	// Check if all Fs (no actor)
-	if strings.ToLower(actorPortion) == "ffffffffffffffffffffffff" {
-		return ""
-	}
-
-	return actorPortion + jobPortion
+	return nil
 }

historyserver/pkg/eventserver/eventserver_test.go

@@ -1,20 +1,23 @@
 package eventserver
 
 import (
+	"bytes"
+	"compress/gzip"
 	"context"
-	"errors"
 	"testing"
 	"time"
 
 	"github.com/google/go-cmp/cmp"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+
 	"github.com/ray-project/kuberay/historyserver/pkg/eventserver/types"
 	"github.com/ray-project/kuberay/historyserver/pkg/utils"
 )
 
-func makeTaskEventMap(taskName, nodeID, taskID, cluster string, attempt int) map[string]any {
+func makeTaskEventMap(taskName, nodeID, taskID string, attempt int) map[string]any {
 	return map[string]any{
-		"eventType":   string(types.TASK_DEFINITION_EVENT),
-		"clusterName": cluster,
+		"eventType": string(types.TASK_DEFINITION_EVENT),
 		"taskDefinitionEvent": map[string]any{
 			"taskId":      taskID,
 			"taskName":    taskName,
@@ -24,179 +27,10 @@ func makeTaskEventMap(taskName, nodeID, taskID, cluster string, attempt int) map
 	}
 }
 
-func TestEventProcessor(t *testing.T) {
+func TestStoreEvent(t *testing.T) {
 	// IDs follow Ray's ID specification:
 	// ref: https://github.com/ray-project/ray/blob/f229d5376eb87b09a3fa0b991323450de84df890/src/ray/design_docs/id_specification.md
 	// Sizes: JobID=4B, ActorID=12B unique+JobID (16B), TaskID=8B unique+ActorID (24B), NodeID/WorkerID=28B
-	const (
-		// Pure-hex IDs to verify normalizeIDToHex is identity.
-		// Use lowercase to match production hex output and avoid map-key collision.
-		testJobID   = "aaaabbbb"                                                 // 4B
-		testActorID = "aaaabbbb1234aaaabbbb1234" + testJobID                     // 12B unique + JobID
-		testTaskID1 = "ccccdddd5678cccc" + testActorID                           // 8B unique + ActorID
-		testTaskID2 = "ccccdddd9012dddd" + testActorID                           // 8B unique + ActorID
-		testNodeID1 = "eeeeffff1234eeeeffff1234eeeeffff1234eeeeffff1234eeeeffff" // 28B
-		testNodeID2 = "eeeeffff9012eeeeffff9012eeeeffff9012eeeeffff9012eeeeffff" // 28B
-
-		testClusterName = "cluster1"
-		testTaskName1   = "Name_12345"
-		testTaskName2   = "Name_54321"
-	)
-	tests := []struct {
-		name string
-		// Setup
-		eventsToSend []map[string]any
-		cancelAfter  time.Duration // Time after which to cancel context (0 for no cancel)
-		closeChan    bool          // Whether to close the channel after sending events
-
-		// Expectations
-		wantErr          bool
-		expectedErrType  error // Specific error type to check (e.g., context.Canceled)
-		wantStoredEvents map[string][]types.Task
-	}{
-		{
-			name: "process multiple events then close channel",
-			eventsToSend: []map[string]any{
-				{
-					"clusterName": testClusterName,
-					"eventType":   "TASK_DEFINITION_EVENT",
-					"taskDefinitionEvent": map[string]any{
-						"taskId":      testTaskID1,
-						"taskName":    testTaskName1,
-						"nodeId":      testNodeID1,
-						"taskAttempt": 2,
-					},
-				},
-				{
-					"clusterName": testClusterName,
-					"eventType":   "TASK_DEFINITION_EVENT",
-					"taskDefinitionEvent": map[string]any{
-						"taskId":      testTaskID2,
-						"taskName":    testTaskName2,
-						"nodeId":      testNodeID2,
-						"taskAttempt": 1,
-					},
-				},
-			},
-			closeChan: true,
-			wantStoredEvents: map[string][]types.Task{
-				testTaskID1: {
-					{
-						TaskID:      testTaskID1,
-						TaskName:    testTaskName1,
-						NodeID:      testNodeID1,
-						TaskAttempt: 2,
-					},
-				},
-				testTaskID2: {
-					{
-						TaskID:      testTaskID2,
-						TaskName:    testTaskName2,
-						NodeID:      testNodeID2,
-						TaskAttempt: 1,
-					},
-				},
-			},
-		},
-		{
-			name:      "channel closed immediately",
-			closeChan: true,
-			wantErr:   false,
-		},
-		{
-			name: "context canceled",
-			eventsToSend: []map[string]any{
-				{
-					"clusterName": testClusterName,
-					"eventType":   "TASK_DEFINITION_EVENT",
-					"taskDefinitionEvent": map[string]any{
-						"taskId":      testTaskID1,
-						"taskName":    testTaskName1,
-						"nodeId":      testNodeID1,
-						"taskAttempt": 2,
-					},
-				},
-			},
-			cancelAfter:     50 * time.Millisecond,
-			wantErr:         true,
-			expectedErrType: context.Canceled,
-			// Event might be processed before cancellation is detected
-			wantStoredEvents: map[string][]types.Task{
-				testTaskID1: {
-					{
-						TaskID:      testTaskID1,
-						TaskName:    testTaskName1,
-						NodeID:      testNodeID1,
-						TaskAttempt: 2,
-					},
-				},
-			},
-		},
-		{
-			name:            "no events, context canceled",
-			cancelAfter:     10 * time.Millisecond,
-			wantErr:         true,
-			expectedErrType: context.Canceled,
-		},
-	}
-
-	for _, tt := range tests {
-		t.Run(tt.name, func(t *testing.T) {
-			// Sending nil for reader since it won't be used anyways
-			h := NewEventHandler(nil)
-
-			// Channel buffer size a bit larger than events to avoid blocking sender in test setup
-			ch := make(chan map[string]any, len(tt.eventsToSend)+2)
-			ctx, cancel := context.WithCancel(context.Background())
-			defer cancel()
-
-			// Send events into the channel
-			go func() {
-				for _, event := range tt.eventsToSend {
-					select {
-					case ch <- event:
-					case <-ctx.Done(): // Stop sending if context is cancelled
-						return
-					}
-				}
-				if tt.closeChan {
-					close(ch)
-				}
-			}()
-
-			// Handle context cancellation if specified
-			if tt.cancelAfter > 0 {
-				go func() {
-					time.Sleep(tt.cancelAfter)
-					cancel()
-				}()
-			}
-
-			// Run the ProcessEvent
-			err := h.ProcessEvents(ctx, ch)
-
-			// Check error expectations
-			if (err != nil) != tt.wantErr {
-				t.Errorf("ProcessEvents() error = %v, wantErr %v", err, tt.wantErr)
-			}
-			if tt.expectedErrType != nil {
-				if !errors.Is(err, tt.expectedErrType) {
-					t.Errorf("ProcessEvents() error type = %T, want type %T (err: %v)", err, tt.expectedErrType, err)
-				}
-			}
-
-			// Check stored events
-			if tt.wantStoredEvents != nil {
-				if diff := cmp.Diff(tt.wantStoredEvents, h.ClusterTaskMap.ClusterTaskMap[testClusterName].TaskMap); diff != "" {
-					t.Errorf("storeEventCalls diff (-want +got):\n%s", diff)
-				}
-			}
-		})
-	}
-}
-
-func TestStoreEvent(t *testing.T) {
-	// IDs follow Ray's ID spec; see TestEventProcessor for rationale.
 	const (
 		testJobID        = "aaaabbbb"                                                 // 4B
 		testActorID      = "aaaabbbb1234aaaabbbb1234" + testJobID                     // 12B unique + JobID
@@ -242,8 +76,7 @@ func TestStoreEvent(t *testing.T) {
 				ClusterTaskMap: make(map[string]*types.TaskMap),
 			},
 			eventMap: map[string]any{
-				"eventType":   "UNKNOWN_TYPE",
-				"clusterName": testClusterName,
+				"eventType": "UNKNOWN_TYPE",
 			},
 			wantErr:          false,
 			wantClusterCount: 0,
@@ -253,7 +86,7 @@ func TestStoreEvent(t *testing.T) {
 			initialState: &types.ClusterTaskMap{
 				ClusterTaskMap: make(map[string]*types.TaskMap),
 			},
-			eventMap:          makeTaskEventMap(testTaskName1, testNodeID1, testTaskID1, testClusterName, 0),
+			eventMap:          makeTaskEventMap(testTaskName1, testNodeID1, testTaskID1, 0),
 			wantErr:           false,
 			wantClusterCount:  1,
 			wantTaskInCluster: testClusterName,
@@ -274,7 +107,7 @@ func TestStoreEvent(t *testing.T) {
 					testClusterName: types.NewTaskMap(),
 				},
 			},
-			eventMap:          makeTaskEventMap(testTaskName2, testNodeID2, testTaskID2, testClusterName, 1),
+			eventMap:          makeTaskEventMap(testTaskName2, testNodeID2, testTaskID2, 1),
 			wantErr:           false,
 			wantClusterCount:  1,
 			wantTaskInCluster: testClusterName,
@@ -299,7 +132,7 @@ func TestStoreEvent(t *testing.T) {
 					},
 				},
 			},
-			eventMap:          makeTaskEventMap(testTaskName1, testNodeID1, testTaskID1, testClusterName, 2),
+			eventMap:          makeTaskEventMap(testTaskName1, testNodeID1, testTaskID1, 2),
 			wantErr:           false,
 			wantClusterCount:  1,
 			wantTaskInCluster: testClusterName,
@@ -325,7 +158,7 @@ func TestStoreEvent(t *testing.T) {
 			initialState: &types.ClusterTaskMap{
 				ClusterTaskMap: make(map[string]*types.TaskMap),
 			},
-			eventMap:          makeTaskEventMap(testTaskName1, testBase64ID1, testBase64ID2, testClusterName, 0),
+			eventMap:          makeTaskEventMap(testTaskName1, testBase64ID1, testBase64ID2, 0),
 			wantErr:           false,
 			wantClusterCount:  1,
 			wantTaskInCluster: testClusterName,
@@ -344,7 +177,7 @@ func TestStoreEvent(t *testing.T) {
 			initialState: &types.ClusterTaskMap{
 				ClusterTaskMap: make(map[string]*types.TaskMap),
 			},
-			eventMap:          makeTaskEventMap(testTaskName1, testUpperNodeID1, testUpperTaskID1, testClusterName, 0),
+			eventMap:          makeTaskEventMap(testTaskName1, testUpperNodeID1, testUpperTaskID1, 0),
 			wantErr:           false,
 			wantClusterCount:  1,
 			wantTaskInCluster: testClusterName,
@@ -363,7 +196,7 @@ func TestStoreEvent(t *testing.T) {
 			initialState: &types.ClusterTaskMap{
 				ClusterTaskMap: make(map[string]*types.TaskMap),
 			},
-			eventMap:          makeTaskEventMap(testTaskName1, testLowerNodeID1, testLowerTaskID1, testClusterName, 0),
+			eventMap:          makeTaskEventMap(testTaskName1, testLowerNodeID1, testLowerTaskID1, 0),
 			wantErr:           false,
 			wantClusterCount:  1,
 			wantTaskInCluster: testClusterName,
@@ -383,8 +216,7 @@ func TestStoreEvent(t *testing.T) {
 				ClusterTaskMap: make(map[string]*types.TaskMap),
 			},
 			eventMap: map[string]any{
-				"eventType":   string(types.TASK_DEFINITION_EVENT),
-				"clusterName": testClusterName,
+				"eventType": string(types.TASK_DEFINITION_EVENT),
 			},
 			wantErr: true,
 		},
@@ -395,7 +227,6 @@ func TestStoreEvent(t *testing.T) {
 			},
 			eventMap: map[string]any{
 				"eventType":           string(types.TASK_DEFINITION_EVENT),
-				"clusterName":         testClusterName,
 				"taskDefinitionEvent": "not a map",
 			},
 			wantErr: true, // Marshal will fail
@@ -406,8 +237,7 @@ func TestStoreEvent(t *testing.T) {
 				ClusterTaskMap: make(map[string]*types.TaskMap),
 			},
 			eventMap: map[string]any{
-				"eventType":   string(types.TASK_DEFINITION_EVENT),
-				"clusterName": testClusterName,
+				"eventType": string(types.TASK_DEFINITION_EVENT),
 				"taskDefinitionEvent": map[string]any{
 					"taskId":      123, // Should be string
 					"taskAttempt": 0,
@@ -428,7 +258,7 @@ func TestStoreEvent(t *testing.T) {
 				}
 			}
 
-			err := h.storeEvent(tt.eventMap)
+			err := h.storeEvent(testClusterName, tt.eventMap)
 
 			if (err != nil) != tt.wantErr {
 				t.Fatalf("storeEvent() error = %v, wantErr %v", err, tt.wantErr)
@@ -468,7 +298,7 @@ func TestStoreEvent(t *testing.T) {
 // TestTaskLifecycleEventDeduplication verifies that duplicate events are correctly filtered
 // and out-of-order events are properly sorted
 func TestTaskLifecycleEventDeduplication(t *testing.T) {
-	// IDs follow Ray's ID spec; see TestEventProcessor for rationale.
+	// IDs follow Ray's ID spec; see TestStoreEvent for rationale.
 	const (
 		testJobID       = "aaaabbbb"                                                 // 4B
 		testActorID     = "aaaabbbb1234aaaabbbb1234" + testJobID                     // 12B unique + JobID
@@ -494,8 +324,7 @@ func TestTaskLifecycleEventDeduplication(t *testing.T) {
 			transitionsAny[i] = t
 		}
 		return map[string]any{
-			"eventType":   string(types.TASK_LIFECYCLE_EVENT),
-			"clusterName": testClusterName,
+			"eventType": string(types.TASK_LIFECYCLE_EVENT),
 			"taskLifecycleEvent": map[string]any{
 				"taskId":           testTaskID,
 				"taskAttempt":      float64(0),
@@ -651,7 +480,7 @@ func TestTaskLifecycleEventDeduplication(t *testing.T) {
 
 			// Process the lifecycle event
 			eventMap := makeLifecycleEvent(tt.newTransitions)
-			err := h.storeEvent(eventMap)
+			err := h.storeEvent(testClusterName, eventMap)
 			if err != nil {
 				t.Fatalf("storeEvent() unexpected error: %v", err)
 			}
@@ -688,7 +517,7 @@ func TestTaskLifecycleEventDeduplication(t *testing.T) {
 
 // TestActorLifecycleEventDeduplication verifies that duplicate actor events are correctly filtered
 func TestActorLifecycleEventDeduplication(t *testing.T) {
-	// IDs follow Ray's ID spec; see TestEventProcessor for rationale.
+	// IDs follow Ray's ID spec; see TestStoreEvent for rationale.
 	const (
 		testJobID       = "aaaabbbb"                             // 4B
 		testActorID     = "aaaabbbb1234aaaabbbb1234" + testJobID // 12B unique + JobID
@@ -711,8 +540,7 @@ func TestActorLifecycleEventDeduplication(t *testing.T) {
 			transitionsAny[i] = t
 		}
 		return map[string]any{
-			"eventType":   string(types.ACTOR_LIFECYCLE_EVENT),
-			"clusterName": testClusterName,
+			"eventType": string(types.ACTOR_LIFECYCLE_EVENT),
 			"actorLifecycleEvent": map[string]any{
 				"actorId":          testActorID,
 				"stateTransitions": transitionsAny,
@@ -816,13 +644,13 @@ func TestActorLifecycleEventDeduplication(t *testing.T) {
 
 			// Process the lifecycle event
 			eventMap := makeActorLifecycleEvent(tt.newTransitions)
-			err := h.storeEvent(eventMap)
+			err := h.storeEvent(testClusterName, eventMap)
 			if err != nil {
 				t.Fatalf("storeEvent() unexpected error: %v", err)
 			}
 
 			// Get the actor and verify
-			actor, found := h.GetActorByID(testClusterName, testActorID)
+			actor, found := h.getActorsMap(testClusterName)[testActorID]
 			if !found {
 				t.Fatal("Actor not found after processing")
 			}
@@ -843,7 +671,7 @@ func TestActorLifecycleEventDeduplication(t *testing.T) {
 // TestDriverJobLifeCycleEventDuplication tests that duplicate events are properly filtered and sorted
 // TODO(chiayi): Update once more fields are added to driver job event
 func TestDriverJobLifecycleEventDuplication(t *testing.T) {
-	// IDs follow Ray's ID spec; see TestEventProcessor for rationale.
+	// IDs follow Ray's ID spec; see TestStoreEvent for rationale.
 	const (
 		testJobID       = "aaaabbbb" // 4B
 		testClusterName = "cluster1"
@@ -862,8 +690,7 @@ func TestDriverJobLifecycleEventDuplication(t *testing.T) {
 			transitionsAny[i] = t
 		}
 		return map[string]any{
-			"eventType":   string(types.DRIVER_JOB_LIFECYCLE_EVENT),
-			"clusterName": testClusterName,
+			"eventType": string(types.DRIVER_JOB_LIFECYCLE_EVENT),
 			"driverJobLifecycleEvent": map[string]any{
 				"jobId":            testJobID,
 				"stateTransitions": transitionsAny,
@@ -985,12 +812,12 @@ func TestDriverJobLifecycleEventDuplication(t *testing.T) {
 			}
 
 			eventMap := makeDriverJobLifecycleEvent(tt.newTransitions)
-			err := h.storeEvent(eventMap)
+			err := h.storeEvent(testClusterName, eventMap)
 			if err != nil {
 				t.Fatalf("storeEvent() unexpected error: %v", err)
 			}
 
-			job, exists := h.GetJobByJobID(testClusterName, testJobID)
+			job, exists := h.getJobsMap(testClusterName)[testJobID]
 			if !exists {
 				t.Fatal("Job not found after processing")
 			}
@@ -1010,7 +837,7 @@ func TestDriverJobLifecycleEventDuplication(t *testing.T) {
 func TestMultipleReprocessingCycles(t *testing.T) {
 	h := NewEventHandler(nil)
 
-	// IDs follow Ray's ID spec; see TestEventProcessor for rationale.
+	// IDs follow Ray's ID spec; see TestStoreEvent for rationale.
 	const (
 		testJobID       = "aaaabbbb"                                                 // 4B
 		testActorID     = "aaaabbbb1234aaaabbbb1234" + testJobID                     // 12B unique + JobID
@@ -1029,8 +856,7 @@ func TestMultipleReprocessingCycles(t *testing.T) {
 	}
 
 	eventMap := map[string]any{
-		"eventType":   string(types.TASK_LIFECYCLE_EVENT),
-		"clusterName": testClusterName,
+		"eventType": string(types.TASK_LIFECYCLE_EVENT),
 		"taskLifecycleEvent": map[string]any{
 			"taskId":           testTaskID,
 			"taskAttempt":      float64(0),
@@ -1042,7 +868,7 @@ func TestMultipleReprocessingCycles(t *testing.T) {
 
 	// Simulate 10 hourly reprocessing cycles
 	for cycle := 0; cycle < 10; cycle++ {
-		err := h.storeEvent(eventMap)
+		err := h.storeEvent(testClusterName, eventMap)
 		if err != nil {
 			t.Fatalf("Cycle %d: storeEvent() error = %v", cycle, err)
 		}
@@ -1126,3 +952,136 @@ func TestNormalizeActorIDsToHex(t *testing.T) {
 		t.Errorf("Address.WorkerID = %q, want %q", actor.Address.WorkerID, expectedHex)
 	}
 }
+
+func TestProcessSingleSession(t *testing.T) {
+	clusterInfo := utils.ClusterInfo{Name: "cluster", Namespace: "ns", SessionName: "session1"}
+
+	t.Run("returns error when every listed file fails I/O", func(t *testing.T) {
+		mock := newLogEventMockReader()
+		mock.addDir("cluster_ns", "session1/job_events/", []string{"job-01000000/"})
+		mock.addDir("cluster_ns", "session1/job_events/job-01000000/",
+			[]string{"01000000-2024-01-01-00.gz"})
+		mock.addDir("cluster_ns", "session1/node_events/",
+			[]string{"node1-2024-01-01-00.gz"})
+		mock.addDir("cluster_ns", "session1/logs", []string{})
+
+		h := NewEventHandler(mock)
+		err := h.ProcessSingleSession(context.Background(), clusterInfo)
+		require.Error(t, err)
+		assert.Contains(t, err.Error(), "read 0 of 2")
+	})
+
+	t.Run("empty file list returns nil (legit empty session)", func(t *testing.T) {
+		mock := newLogEventMockReader()
+		mock.addDir("cluster_ns", "session1/job_events/", []string{})
+		mock.addDir("cluster_ns", "session1/node_events/", []string{})
+		mock.addDir("cluster_ns", "session1/logs", []string{})
+
+		h := NewEventHandler(mock)
+		err := h.ProcessSingleSession(context.Background(), clusterInfo)
+		require.NoError(t, err)
+	})
+
+	t.Run("partial success does not return error", func(t *testing.T) {
+		mock := newLogEventMockReader()
+		mock.addDir("cluster_ns", "session1/node_events/",
+			[]string{"node1-2024-01-01-00.gz", "node2-2024-01-01-00.gz"})
+
+		var buf bytes.Buffer
+		gw := gzip.NewWriter(&buf)
+		_, err := gw.Write([]byte(`[{"eventType":"NODE_DEFINITION_EVENT","nodeDefinitionEvent":{"nodeId":"YWJjZA=="}}]`))
+		require.NoError(t, err)
+		require.NoError(t, gw.Close())
+
+		mock.addFile("cluster_ns", "session1/node_events/node1-2024-01-01-00.gz", buf.String())
+		mock.addDir("cluster_ns", "session1/job_events/", []string{})
+		mock.addDir("cluster_ns", "session1/logs", []string{})
+
+		h := NewEventHandler(mock)
+		err = h.ProcessSingleSession(context.Background(), clusterInfo)
+		require.NoError(t, err)
+	})
+
+	t.Run("all corrupt JSON does not return error", func(t *testing.T) {
+		mock := newLogEventMockReader()
+		mock.addDir("cluster_ns", "session1/node_events/",
+			[]string{"node1-2024-01-01-00.gz", "node2-2024-01-01-00.gz"})
+
+		var buf bytes.Buffer
+		gw := gzip.NewWriter(&buf)
+		_, err := gw.Write([]byte("this is not json"))
+		require.NoError(t, err)
+		require.NoError(t, gw.Close())
+
+		mock.addFile("cluster_ns", "session1/node_events/node1-2024-01-01-00.gz", buf.String())
+		mock.addFile("cluster_ns", "session1/node_events/node2-2024-01-01-00.gz", buf.String())
+		mock.addDir("cluster_ns", "session1/job_events/", []string{})
+		mock.addDir("cluster_ns", "session1/logs", []string{})
+
+		h := NewEventHandler(mock)
+		err = h.ProcessSingleSession(context.Background(), clusterInfo)
+		require.NoError(t, err)
+	})
+
+	t.Run("log events failure alone does not surface as error", func(t *testing.T) {
+		mock := newLogEventMockReader()
+		mock.addDir("cluster_ns", "session1/job_events/", []string{})
+		mock.addDir("cluster_ns", "session1/node_events/", []string{})
+		mock.addDir("cluster_ns", "session1/logs", []string{"node1/"})
+		mock.addDir("cluster_ns", "session1/logs/node1/events", []string{"event_GCS.log"})
+
+		h := NewEventHandler(mock)
+		err := h.ProcessSingleSession(context.Background(), clusterInfo)
+		require.NoError(t, err)
+	})
+
+	t.Run("ray events failure surfaces; log events failure stays silent", func(t *testing.T) {
+		mock := newLogEventMockReader()
+		mock.addDir("cluster_ns", "session1/node_events/", []string{"node1-2024-01-01-00.gz"})
+		mock.addDir("cluster_ns", "session1/job_events/", []string{})
+		mock.addDir("cluster_ns", "session1/logs", []string{"node1/"})
+		mock.addDir("cluster_ns", "session1/logs/node1/events", []string{"event_GCS.log"})
+
+		h := NewEventHandler(mock)
+		err := h.ProcessSingleSession(context.Background(), clusterInfo)
+		require.Error(t, err)
+		assert.Contains(t, err.Error(), "read 0 of 1 event files")
+		assert.NotContains(t, err.Error(), "log event")
+	})
+
+	t.Run("transparent decompression of compressed .gz files and reading uncompressed legacy files", func(t *testing.T) {
+		mock := newLogEventMockReader()
+		mock.addDir("cluster_ns", "session1/node_events/",
+			[]string{"node1-2024-01-01-00.gz", "node2-2024-01-01-00"})
+
+		// 1. Compress node1 event file
+		var buf bytes.Buffer
+		gw := gzip.NewWriter(&buf)
+		_, err := gw.Write([]byte(`[{"eventType":"NODE_DEFINITION_EVENT","nodeDefinitionEvent":{"nodeId":"YWJjZA=="}}]`))
+		require.NoError(t, err)
+		require.NoError(t, gw.Close())
+		mock.addFile("cluster_ns", "session1/node_events/node1-2024-01-01-00.gz", buf.String())
+
+		// 2. Keep node2 event file uncompressed
+		mock.addFile("cluster_ns", "session1/node_events/node2-2024-01-01-00",
+			`[{"eventType":"NODE_DEFINITION_EVENT","nodeDefinitionEvent":{"nodeId":"ZWZnaA=="}}]`)
+
+		mock.addDir("cluster_ns", "session1/job_events/", []string{})
+		mock.addDir("cluster_ns", "session1/logs", []string{})
+
+		h := NewEventHandler(mock)
+		err = h.ProcessSingleSession(context.Background(), clusterInfo)
+		require.NoError(t, err)
+
+		nodeMap := h.getNodeMap("cluster_ns_session1")
+		assert.Len(t, nodeMap, 2)
+
+		// "YWJjZA==" -> hex "61626364" (abcd)
+		_, ok := nodeMap["61626364"]
+		assert.True(t, ok, "node1 (compressed) should be successfully loaded")
+
+		// "ZWZnaA==" -> hex "65666768" (efgh)
+		_, ok = nodeMap["65666768"]
+		assert.True(t, ok, "node2 (uncompressed) should be successfully loaded")
+	})
+}

historyserver/pkg/eventserver/log_event_reader.go

@@ -49,7 +49,8 @@ func NewLogEventReader(reader storage.StorageReader) *LogEventReader {
 // and stores them in the provided ClusterLogEventMap.
 //
 // Path structure in storage: {clusterName}_{namespace}/{sessionName}/logs/{nodeId}/events/event_*.log
-// This is called from eventserver.go Run() to populate events for a cluster session.
+//
+// Return an error if any listed file fails to read (total or partial).
 func (r *LogEventReader) ReadLogEvents(clusterInfo utils.ClusterInfo, clusterSessionKey string, eventStore *types.ClusterLogEventMap) error {
 	// Build cluster ID used by StorageReader
 	clusterID := clusterInfo.Name + "_" + clusterInfo.Namespace
@@ -80,6 +81,8 @@ func (r *LogEventReader) ReadLogEvents(clusterInfo utils.ClusterInfo, clusterSes
 	}
 	logrus.Debugf("Found %d node directories for cluster %s: %v", len(nodeIDs), clusterSessionKey, nodeIDs)
 
+	total := 0
+	read := 0
 	for _, nodeID := range nodeIDs {
 		// Path: {sessionName}/logs/{nodeId}/events/
 		eventsDir := path.Join(clusterInfo.SessionName, utils.RAY_SESSIONDIR_LOGDIR_NAME, nodeID, "events")
@@ -98,13 +101,18 @@ func (r *LogEventReader) ReadLogEvents(clusterInfo utils.ClusterInfo, clusterSes
 			// Read and parse the event file
 			// Note: Duplicate events are handled by JobEventMap's deduplication using event_id as key.
 			// This matches the design of existing RayEvents reading in eventserver.go.
+			total++
 			if err := r.readEventFile(clusterID, eventFilePath, jobEventMap); err != nil {
 				logrus.Warnf("Failed to read event file %s: %v", eventFilePath, err)
-				// Continue with other files - failed files will be retried in the next cycle
+				continue
 			}
+			read++
 		}
 	}
 
+	if total != read {
+		return fmt.Errorf("read %d of %d log event files for %s", read, total, clusterSessionKey)
+	}
 	return nil
 }
 

historyserver/pkg/eventserver/log_event_reader_test.go

@@ -220,4 +220,37 @@ func TestReadLogEvents(t *testing.T) {
 		require.NoError(t, err)
 		assert.Empty(t, store.GetAllEvents("cluster_ns_session1"))
 	})
+
+	t.Run("returns error when every listed file fails to read", func(t *testing.T) {
+		// Intentionally skip addFile.
+		mock := newLogEventMockReader()
+		mock.addDir("cluster_ns", "session1/logs", []string{"node1/"})
+		mock.addDir("cluster_ns", "session1/logs/node1/events", []string{"event_GCS.log", "event_RAYLET.log"})
+
+		reader := NewLogEventReader(mock)
+		store := types.NewClusterLogEventMap()
+		clusterInfo := utils.ClusterInfo{Name: "cluster", Namespace: "ns", SessionName: "session1"}
+
+		err := reader.ReadLogEvents(clusterInfo, "cluster_ns_session1", store)
+		require.Error(t, err)
+		assert.Contains(t, err.Error(), "read 0 of 2")
+	})
+
+	t.Run("partial read surfaces error but preserves successful events", func(t *testing.T) {
+		mock := newLogEventMockReader()
+		mock.addDir("cluster_ns", "session1/logs", []string{"node1/"})
+		mock.addDir("cluster_ns", "session1/logs/node1/events", []string{"event_GCS.log", "event_RAYLET.log"})
+		mock.addFile("cluster_ns", "session1/logs/node1/events/event_GCS.log",
+			`{"event_id":"e1","source_type":"GCS","severity":"INFO","message":"ok","timestamp":"1770635700"}`+"\n")
+
+		reader := NewLogEventReader(mock)
+		store := types.NewClusterLogEventMap()
+		clusterInfo := utils.ClusterInfo{Name: "cluster", Namespace: "ns", SessionName: "session1"}
+
+		err := reader.ReadLogEvents(clusterInfo, "cluster_ns_session1", store)
+		require.Error(t, err)
+		assert.Contains(t, err.Error(), "read 1 of 2")
+		events := store.GetAllEvents("cluster_ns_session1")
+		assert.Len(t, events["global"], 1)
+	})
 }

historyserver/pkg/eventserver/snapshot.go

@@ -0,0 +1,32 @@
+package eventserver
+
+import (
+	"github.com/ray-project/kuberay/historyserver/pkg/eventserver/types"
+	"github.com/ray-project/kuberay/historyserver/pkg/utils"
+)
+
+// SessionSnapshot is the in-memory representation of a dead session's processed
+// event state. The same *SessionSnapshot is shared by all concurrent handlers.
+// To avoid races, handlers MUST treat all fields as read-only.
+type SessionSnapshot struct {
+	SessionKey string `json:"sessionKey"`
+
+	Tasks            []types.Task                `json:"tasks"`
+	Actors           map[string]types.Actor      `json:"actors"`
+	Jobs             map[string]types.Job        `json:"jobs"`
+	Nodes            map[string]types.Node       `json:"nodes"`
+	LogEventsByJobID map[string][]types.LogEvent `json:"logEventsByJobId"`
+}
+
+// BuildSnapshot builds a SessionSnapshot from the handler's per-session state.
+func (h *EventHandler) BuildSnapshot(session utils.ClusterInfo) *SessionSnapshot {
+	clusterSessionKey := utils.BuildClusterSessionKey(session.Name, session.Namespace, session.SessionName)
+	return &SessionSnapshot{
+		SessionKey:       clusterSessionKey,
+		Tasks:            h.getTasks(clusterSessionKey),
+		Actors:           h.getActorsMap(clusterSessionKey),
+		Jobs:             h.getJobsMap(clusterSessionKey),
+		Nodes:            h.getNodeMap(clusterSessionKey),
+		LogEventsByJobID: h.getLogEventsByJobID(clusterSessionKey),
+	}
+}

historyserver/pkg/eventserver/snapshot_test.go

@@ -0,0 +1,71 @@
+package eventserver
+
+import (
+	"encoding/json"
+	"reflect"
+	"testing"
+
+	"github.com/ray-project/kuberay/historyserver/pkg/eventserver/types"
+)
+
+// TestEmptyRoundtrip verifies that an empty SessionSnapshot marshals and
+// unmarshals back to an equal struct.
+func TestEmptyRoundtrip(t *testing.T) {
+	original := SessionSnapshot{
+		SessionKey: "raycluster-historyserver_default_session_2026-01-11_19-38-40",
+	}
+
+	data, err := json.Marshal(original)
+	if err != nil {
+		t.Fatalf("json.Marshal failed: %v", err)
+	}
+
+	var decoded SessionSnapshot
+	if err := json.Unmarshal(data, &decoded); err != nil {
+		t.Fatalf("json.Unmarshal failed: %v", err)
+	}
+
+	if !reflect.DeepEqual(original, decoded) {
+		t.Fatalf("roundtrip mismatch\noriginal=%+v\ndecoded =%+v", original, decoded)
+	}
+}
+
+// TestFullRoundtrip populates all five maps with at least one entry and
+// verifies the snapshot survives a JSON roundtrip intact.
+func TestFullRoundtrip(t *testing.T) {
+	original := SessionSnapshot{
+		SessionKey: "raycluster-historyserver_default_session_2026-01-11_19-38-40",
+		Tasks: []types.Task{
+			{TaskID: "task-1", TaskAttempt: 0, JobID: "job-1"},
+			{TaskID: "task-1", TaskAttempt: 1, JobID: "job-1"},
+		},
+		Actors: map[string]types.Actor{
+			"actor-1": {ActorID: "actor-1", JobID: "job-1"},
+		},
+		Jobs: map[string]types.Job{
+			"job-1": {JobID: "job-1", EntryPoint: "python main.py"},
+		},
+		Nodes: map[string]types.Node{
+			"node-1": {NodeID: "node-1", NodeIPAddress: "10.0.0.1"},
+		},
+		LogEventsByJobID: map[string][]types.LogEvent{
+			"job-1": {
+				{EventID: "evt-1", Message: "hello", Severity: "INFO"},
+			},
+		},
+	}
+
+	data, err := json.Marshal(original)
+	if err != nil {
+		t.Fatalf("json.Marshal failed: %v", err)
+	}
+
+	var decoded SessionSnapshot
+	if err := json.Unmarshal(data, &decoded); err != nil {
+		t.Fatalf("json.Unmarshal failed: %v", err)
+	}
+
+	if !reflect.DeepEqual(original, decoded) {
+		t.Fatalf("roundtrip mismatch\noriginal=%+v\ndecoded =%+v", original, decoded)
+	}
+}

historyserver/pkg/eventserver/types/error.go

@@ -44,7 +44,7 @@ const (
 // RayErrorInfo is the information per Ray error type.
 type RayErrorInfo struct {
 	// More detailed error context for various error types.
-	// TODO(jwj): Add error context.
+	// TODO(jiangjiawei1103): Add error context.
 	Error interface{} `json:"error"`
 
 	ErrorMessage string `json:"errorMessage"`

historyserver/pkg/eventserver/types/log_event.go

@@ -52,6 +52,38 @@ func (e *LogEvent) GetJobID() string {
 	return "global"
 }
 
+func (e LogEvent) DeepCopy() LogEvent {
+	cp := e
+	cp.CustomFields = cloneMapAny(e.CustomFields)
+	return cp
+}
+
+func cloneMapAny(m map[string]any) map[string]any {
+	if m == nil {
+		return nil
+	}
+	out := make(map[string]any, len(m))
+	for k, v := range m {
+		out[k] = cloneAny(v)
+	}
+	return out
+}
+
+func cloneAny(v any) any {
+	switch typed := v.(type) {
+	case map[string]any:
+		return cloneMapAny(typed)
+	case []any:
+		out := make([]any, len(typed))
+		for i, item := range typed {
+			out[i] = cloneAny(item)
+		}
+		return out
+	default:
+		return v
+	}
+}
+
 // RestoreNewline restores escaped newlines in the message field.
 // This matches Ray Dashboard's _restore_newline() function in event_utils.py.
 func (e *LogEvent) RestoreNewline() {
@@ -332,3 +364,30 @@ func (s *LogEventStore) GetEventsByJobID(clusterSessionKey, jobID string) []map[
 	}
 	return m.GetEventsByJobID(jobID)
 }
+
+// GetLogEventsByJobID returns a deep copy of stored log events grouped by job ID.
+func (s *LogEventStore) GetLogEventsByJobID(clusterSessionKey string) map[string][]LogEvent {
+	s.mu.RLock()
+	m, ok := s.sessions[clusterSessionKey]
+	s.mu.RUnlock()
+
+	if !ok {
+		return map[string][]LogEvent{}
+	}
+
+	m.mu.RLock()
+	defer m.mu.RUnlock()
+
+	out := make(map[string][]LogEvent, len(m.events))
+	for jobID, eventMap := range m.events {
+		events := make([]LogEvent, 0, len(eventMap))
+		for _, e := range eventMap {
+			events = append(events, e.DeepCopy())
+		}
+		sort.Slice(events, func(i, j int) bool {
+			return timestampLess(events[i].Timestamp, events[j].Timestamp)
+		})
+		out[jobID] = events
+	}
+	return out
+}

historyserver/pkg/eventserver/types/log_event_test.go

@@ -126,6 +126,28 @@ func TestLogEventGetJobID(t *testing.T) {
 	}
 }
 
+func TestLogEventStoreGetLogEventsByJobIDDeepCopy(t *testing.T) {
+	store := NewLogEventStore()
+	const clusterSessionKey = "raycluster-test_default_session_2026-04-22_10-00-00_000000_1"
+
+	m := store.GetOrCreateJobEventMap(clusterSessionKey)
+	m.AddEvent(&LogEvent{
+		EventID:      "e1",
+		Timestamp:    "1770635705",
+		CustomFields: map[string]any{"job_id": "01000000"},
+	})
+
+	copied := store.GetLogEventsByJobID(clusterSessionKey)
+	require.Len(t, copied["01000000"], 1)
+
+	copied["01000000"][0].Message = "mutated"
+	copied["01000000"][0].CustomFields["job_id"] = "changed"
+
+	again := store.GetLogEventsByJobID(clusterSessionKey)
+	assert.NotEqual(t, "mutated", again["01000000"][0].Message)
+	assert.Equal(t, "01000000", again["01000000"][0].CustomFields["job_id"])
+}
+
 func TestLogEventRestoreNewline(t *testing.T) {
 	event := LogEvent{Message: "line1\\nline2\\rline3"}
 	event.RestoreNewline()

historyserver/pkg/eventserver/types/task.go

@@ -66,7 +66,7 @@ type TaskStatus string
 // The following statuses follow a rough chronological order of transition.
 // For typical order of states, please refer to:
 // https://github.com/ray-project/ray/blob/d0b1d151d8ea964a711e451d0ae736f8bf95b629/src/ray/protobuf/common.proto#L884-L899.
-// TODO(jwj): Each entity (actor, task, job, node) should have its own const def with entity name prepended to avoid conflicts.
+// TODO(jiangjiawei1103): Each entity (actor, task, job, node) should have its own const def with entity name prepended to avoid conflicts.
 const (
 	NIL                                        TaskStatus = "NIL"
 	PENDING_ARGS_AVAIL                         TaskStatus = "PENDING_ARGS_AVAIL"
@@ -154,7 +154,7 @@ type Task struct {
 	ActorReprName *string `json:"actorReprName,omitempty"`
 
 	// TaskLogInfo is just added at https://github.com/ray-project/ray/pull/60287.
-	// TODO(jwj): Add support for TaskLogInfo.
+	// TODO(jiangjiawei1103): Add support for TaskLogInfo.
 	TaskLogInfo *TaskLogInfo `json:"taskLogInfo,omitempty"`
 
 	State        TaskStatus
@@ -332,7 +332,7 @@ func (t *Task) GetLastState() TaskStatus {
 // Ref: https://github.com/ray-project/ray/blob/d0b1d151d8ea964a711e451d0ae736f8bf95b629/src/ray/protobuf/gcs_service.proto#L795-L859.
 // For all filterable fields, please refer to:
 // Ref: https://github.com/ray-project/ray/blob/d0b1d151d8ea964a711e451d0ae736f8bf95b629/python/ray/util/state/common.py#L730-L819.
-// TODO(jwj): Define object-specific filterable fields (e.g., Actor, Node).
+// TODO(jiangjiawei1103): Define object-specific filterable fields (e.g., Actor, Node).
 func (t *Task) GetFilterableFieldValue(filterKey string) string {
 	switch filterKey {
 	case "task_type":

historyserver/pkg/historyserver/clientmanager.go

@@ -16,11 +16,23 @@ import (
 	"sigs.k8s.io/controller-runtime/pkg/client"
 )
 
+const (
+	// Aligned with ray-operator defaults:
+	// https://github.com/ray-project/kuberay/blob/178e6c91/ray-operator/apis/config/v1alpha1/defaults.go#L12-L13
+	DefaultKubeAPIQPS   = float64(100)
+	DefaultKubeAPIBurst = 200
+)
+
 type ClientManager struct {
 	configs []*rest.Config
 	clients []client.Client
 }
 
+// Client returns the primary controller-runtime client.
+func (c *ClientManager) Client() client.Client {
+	return c.clients[0]
+}
+
 func (c *ClientManager) ListRayClusters(ctx context.Context) ([]*rayv1.RayCluster, error) {
 	list := []*rayv1.RayCluster{}
 	for _, c := range c.clients {
@@ -37,7 +49,18 @@ func (c *ClientManager) ListRayClusters(ctx context.Context) ([]*rayv1.RayCluste
 	return list, nil
 }
 
-func NewClientManager(kubeconfigs string, useKubernetesProxy bool) (*ClientManager, error) {
+type ClientManagerConfig struct {
+	Kubeconfigs        string
+	UseKubernetesProxy bool
+	QPS                float32
+	Burst              int
+}
+
+func NewClientManager(cfg ClientManagerConfig) (*ClientManager, error) {
+	kubeconfigs := cfg.Kubeconfigs
+
+	var c *rest.Config
+	var err error
 	kubeconfigList := []*rest.Config{}
 	if len(kubeconfigs) > 0 {
 		stringList := strings.Split(kubeconfigs, ",")
@@ -51,17 +74,12 @@ func NewClientManager(kubeconfigs string, useKubernetesProxy bool) (*ClientManag
 			return nil, fmt.Errorf("kubeconfig is empty")
 		}
 
-		c, err := clientcmd.BuildConfigFromFlags("", stringList[0])
+		c, err = clientcmd.BuildConfigFromFlags("", stringList[0])
 		if err != nil {
 			return nil, fmt.Errorf("failed to build config from kubeconfig: %w", err)
 		}
-		c.QPS = 50
-		c.Burst = 100
-		kubeconfigList = append(kubeconfigList, c)
 	} else {
-		var c *rest.Config
-		var err error
-		if useKubernetesProxy {
+		if cfg.UseKubernetesProxy {
 			// Load Kubernetes REST config from default kubeconfig locations (KUBECONFIG environment variable or ~/.kube/config)
 			// without interactive prompts.
 			loadingRules := clientcmd.NewDefaultClientConfigLoadingRules()
@@ -77,10 +95,11 @@ func NewClientManager(kubeconfigs string, useKubernetesProxy bool) (*ClientManag
 				return nil, fmt.Errorf("failed to build config from in-cluster kubeconfig: %w", err)
 			}
 		}
-		c.QPS = 50
-		c.Burst = 100
-		kubeconfigList = append(kubeconfigList, c)
 	}
+	c.QPS = cfg.QPS
+	c.Burst = cfg.Burst
+	kubeconfigList = append(kubeconfigList, c)
+
 	scheme := runtime.NewScheme()
 	utilruntime.Must(rayv1.AddToScheme(scheme))
 	clientList := []client.Client{}

historyserver/pkg/historyserver/enter_cluster_test.go

@@ -0,0 +1,176 @@
+package historyserver
+
+import (
+	"context"
+	"fmt"
+	"net/http"
+	"net/http/httptest"
+	"sort"
+	"testing"
+
+	"github.com/emicklei/go-restful/v3"
+	"github.com/ray-project/kuberay/historyserver/pkg/eventserver"
+	"github.com/ray-project/kuberay/historyserver/pkg/utils"
+)
+
+func TestEnterCluster(t *testing.T) {
+	// Reset default container to avoid polluting or using duplicate services across runs
+	restful.DefaultContainer = restful.NewContainer()
+
+	// Create ServerHandler with fake sessionLoader
+	handler := &ServerHandler{
+		maxClusters: 100,
+		clustersMap: make(map[utils.ClusterKey][]utils.ClusterInfo),
+	}
+
+	// Setup fake processor for sessionLoader
+	fp := &fakeProcessor{
+		fn: func(ctx context.Context, info utils.ClusterInfo) (SessionStatus, *eventserver.SessionSnapshot, error) {
+			if info.SessionName == "session_2026-04-22_10-00-00_000000_1" {
+				return SessionStatusProcessed, &eventserver.SessionSnapshot{}, nil
+			}
+			if info.SessionName == "session_2026-04-22_10-00-00_000000_2_live" {
+				return SessionStatusLive, nil, nil
+			}
+			return SessionStatusEventsErr, nil, fmt.Errorf("unknown session")
+		},
+	}
+	handler.sessionLoader = NewSessionLoader(fp, context.Background(), DefaultSessionProcessTimeout, DefaultSessionCacheSize, DefaultSessionCacheTTL)
+
+	// Single session cluster
+	keyA := utils.ClusterKey{
+		Namespace: "default",
+		Name:      "cluster-a",
+	}
+	handler.clustersMap[keyA] = []utils.ClusterInfo{
+		{
+			Namespace:   "default",
+			Name:        "cluster-a",
+			SessionName: "session_2026-04-22_10-00-00_000000_1",
+			OwnerKind:   "RayJob",
+			OwnerName:   "job-a",
+		},
+	}
+
+	// Multi-session cluster (past session AND live session)
+	keyB := utils.ClusterKey{
+		Namespace: "default",
+		Name:      "cluster-b",
+	}
+	handler.clustersMap[keyB] = []utils.ClusterInfo{
+		{
+			Namespace:       "default",
+			Name:            "cluster-b",
+			SessionName:     "session_2026-04-22_10-00-00_000000_1",
+			OwnerKind:       "RayService",
+			OwnerName:       "svc-b",
+			CreateTimeStamp: 1000, // Older
+		},
+		{
+			Namespace:       "default",
+			Name:            "cluster-b",
+			SessionName:     "live",
+			OwnerKind:       "RayService",
+			OwnerName:       "svc-b",
+			CreateTimeStamp: 2000, // Newer (latest)
+		},
+	}
+
+	// Cluster with session that is resolved but will trigger live resolution
+	keyC := utils.ClusterKey{
+		Namespace: "default",
+		Name:      "cluster-c",
+	}
+	handler.clustersMap[keyC] = []utils.ClusterInfo{
+		{
+			Namespace:   "default",
+			Name:        "cluster-c",
+			SessionName: "session_2026-04-22_10-00-00_000000_2_live",
+			OwnerKind:   "RayJob",
+			OwnerName:   "job-c",
+		},
+	}
+
+	// Cluster with invalid session format
+	keyD := utils.ClusterKey{
+		Namespace: "default",
+		Name:      "cluster-d",
+	}
+	handler.clustersMap[keyD] = []utils.ClusterInfo{
+		{
+			Namespace:   "default",
+			Name:        "cluster-d",
+			SessionName: "invalid-session-name",
+		},
+	}
+
+	// Explicitly sort `Multi-session cluster` to simulate listClusters post-sorting logic
+	sort.Sort(utils.ClusterInfoList(handler.clustersMap[keyB]))
+
+	// Register actual router
+	routerRayClusterSet(handler)
+
+	container := restful.DefaultContainer
+
+	t.Run("Verify sorted order of multi-session slice puts latest first", func(t *testing.T) {
+		sessions := handler.clustersMap[keyB]
+		if len(sessions) != 2 {
+			t.Fatalf("Expected 2 sessions, got %d", len(sessions))
+		}
+		// Index 0 must be the newer session (live, timestamp 2000)
+		if sessions[0].SessionName != "live" {
+			t.Errorf("Expected latest session 'live' at index 0, got %s", sessions[0].SessionName)
+		}
+	})
+
+	t.Run("Enter existing single-session cluster with explicit session (Successful Dead Session Loading)", func(t *testing.T) {
+		req := httptest.NewRequest("GET", "/enter_cluster/default/cluster-a/session_2026-04-22_10-00-00_000000_1", nil)
+		resp := httptest.NewRecorder()
+		container.ServeHTTP(resp, req)
+
+		if resp.Code != http.StatusOK {
+			t.Fatalf("Expected status 200, got %d: %s", resp.Code, resp.Body.String())
+		}
+
+		cookies := resp.Result().Cookies()
+		cookieMap := make(map[string]*http.Cookie)
+		for _, cookie := range cookies {
+			cookieMap[cookie.Name] = cookie
+		}
+
+		if c, ok := cookieMap[COOKIE_SESSION_NAME_KEY]; !ok || c.Value != "session_2026-04-22_10-00-00_000000_1" {
+			t.Errorf("Expected cookie %s to be 'session_2026-04-22_10-00-00_000000_1', got %v", COOKIE_SESSION_NAME_KEY, c)
+		}
+	})
+
+	t.Run("Enter cluster with session that is resolved but triggers live resolution (Line 326 path)", func(t *testing.T) {
+		req := httptest.NewRequest("GET", "/enter_cluster/default/cluster-c/session_2026-04-22_10-00-00_000000_2_live", nil)
+		resp := httptest.NewRecorder()
+		container.ServeHTTP(resp, req)
+
+		if resp.Code != http.StatusOK {
+			t.Fatalf("Expected status 200, got %d: %s", resp.Code, resp.Body.String())
+		}
+
+		cookies := resp.Result().Cookies()
+		cookieMap := make(map[string]*http.Cookie)
+		for _, cookie := range cookies {
+			cookieMap[cookie.Name] = cookie
+		}
+
+		// Since the fake processor returns SessionStatusLive for this session, resolvedSession is set to "live"
+		if c, ok := cookieMap[COOKIE_SESSION_NAME_KEY]; !ok || c.Value != "live" {
+			t.Errorf("Expected cookie %s to be 'live' (resolved from timestamp because cluster is live), got %v", COOKIE_SESSION_NAME_KEY, c)
+		}
+	})
+
+	t.Run("Enter cluster with invalid session name format (Line 310 path)", func(t *testing.T) {
+		req := httptest.NewRequest("GET", "/enter_cluster/default/cluster-d/invalid-session-name", nil)
+		resp := httptest.NewRecorder()
+		container.ServeHTTP(resp, req)
+
+		if resp.Code != http.StatusBadRequest {
+			t.Fatalf("Expected status 400 (BadRequest), got %d", resp.Code)
+		}
+	})
+}

historyserver/pkg/historyserver/reader.go

@@ -14,6 +14,7 @@ import (
 	"strings"
 
 	"github.com/bmatcuk/doublestar/v4"
+	"github.com/ray-project/kuberay/historyserver/pkg/compression"
 	eventtypes "github.com/ray-project/kuberay/historyserver/pkg/eventserver/types"
 	"github.com/sirupsen/logrus"
 
@@ -67,9 +68,47 @@ func (s *ServerHandler) listClusters(limit int) []utils.ClusterInfo {
 		clusters = clusters[:limit]
 	}
 	clusters = append(liveClusterInfos, clusters...)
+
+	clustersMap := make(map[utils.ClusterKey][]utils.ClusterInfo)
+	for _, c := range clusters {
+		key := utils.ClusterKey{
+			Namespace: c.Namespace,
+			Name:      c.Name,
+		}
+		clustersMap[key] = append(clustersMap[key], c)
+	}
+
+	for key := range clustersMap {
+		sort.Sort(utils.ClusterInfoList(clustersMap[key]))
+	}
+
+	s.mu.Lock()
+	s.clustersMap = clustersMap
+	s.mu.Unlock()
+
 	return clusters
 }
 
+func (s *ServerHandler) findSessionInMap(namespace, name, session string) (string, bool) {
+	s.mu.RLock()
+	defer s.mu.RUnlock()
+	key := utils.ClusterKey{
+		Namespace: namespace,
+		Name:      name,
+	}
+	if list, ok := s.clustersMap[key]; ok {
+		if len(list) == 0 {
+			return "", false
+		}
+		for _, c := range list {
+			if c.SessionName == session {
+				return c.SessionName, true
+			}
+		}
+	}
+	return "", false
+}
+
 func (s *ServerHandler) _getNodeLogs(rayClusterNameNamespace, sessionId, nodeId, folder, glob string) ([]byte, error) {
 	logPath := path.Join(sessionId, utils.RAY_SESSIONDIR_LOGDIR_NAME, nodeId)
 	if folder != "" {
@@ -170,9 +209,9 @@ func categorizeLogFiles(files []string) map[string][]string {
 	return result
 }
 
-func (s *ServerHandler) _getNodeLogFile(rayClusterNameNamespace, sessionID string, options GetLogFileOptions) ([]byte, error) {
+func (s *ServerHandler) _getNodeLogFile(clusterSessionKey, rayClusterNameNamespace, sessionID string, options GetLogFileOptions) ([]byte, error) {
 	// Resolve node_id and filename based on options
-	nodeID, filename, err := s.resolveLogFilename(rayClusterNameNamespace, sessionID, options)
+	nodeID, filename, err := s.resolveLogFilename(clusterSessionKey, rayClusterNameNamespace, sessionID, options)
 	if err != nil {
 		// Preserve HTTPError status code if already set, otherwise use BadRequest
 		var httpErr *utils.HTTPError
@@ -259,7 +298,7 @@ func (s *ServerHandler) _getNodeLogFile(rayClusterNameNamespace, sessionID strin
 // resolveLogFilename resolves the log file node_id and filename based on the provided options.
 // This mirrors Ray Dashboard's resolve_filename logic.
 // The sessionID parameter is required for task_id resolution to search worker log files.
-func (s *ServerHandler) resolveLogFilename(clusterNameID, sessionID string, options GetLogFileOptions) (nodeID, filename string, err error) {
+func (s *ServerHandler) resolveLogFilename(clusterSessionKey, clusterNameID, sessionID string, options GetLogFileOptions) (nodeID, filename string, err error) {
 	// If filename is explicitly provided, use it and ignore suffix
 	if options.Filename != "" {
 		if options.NodeID == "" {
@@ -275,12 +314,12 @@ func (s *ServerHandler) resolveLogFilename(clusterNameID, sessionID string, opti
 
 	// If task_id is provided, resolve from task events
 	if options.TaskID != "" {
-		return s.resolveTaskLogFilename(clusterNameID, sessionID, options.TaskID, options.AttemptNumber, options.Suffix)
+		return s.resolveTaskLogFilename(clusterSessionKey, clusterNameID, sessionID, options.TaskID, options.AttemptNumber, options.Suffix)
 	}
 
 	// If actor_id is provided, resolve from actor events
 	if options.ActorID != "" {
-		return s.resolveActorLogFilename(clusterNameID, sessionID, options.ActorID, options.Suffix)
+		return s.resolveActorLogFilename(clusterSessionKey, clusterNameID, sessionID, options.ActorID, options.Suffix)
 	}
 
 	// If pid is provided, resolve worker log file
@@ -318,17 +357,29 @@ func (s *ServerHandler) resolvePidLogFilename(clusterNameID, sessionID, nodeID s
 	return "", "", utils.NewHTTPError(fmt.Errorf("log file not found for pid %d in path %s", pid, logPath), http.StatusNotFound)
 }
 
+func getTasksByID(tasks []eventtypes.Task, taskID string) ([]eventtypes.Task, bool) {
+	var attempts []eventtypes.Task
+	for _, t := range tasks {
+		if t.TaskID == taskID {
+			attempts = append(attempts, t)
+		}
+	}
+	if len(attempts) == 0 {
+		return nil, false
+	}
+	return attempts, true
+}
+
 // resolveTaskLogFilename resolves log file for a task by querying task events.
 // This mirrors Ray Dashboard's _resolve_task_filename logic.
 // The sessionID parameter is required for searching worker log files when task_log_info is not available.
-func (s *ServerHandler) resolveTaskLogFilename(clusterNameID, sessionID, taskID string, attemptNumber int, suffix string) (nodeID, filename string, err error) {
-	// Construct full cluster session key for event lookup
-	// We append the sessionID to the clusterNameID (which is "name_namespace")
-	// to match the key format used by utils.BuildClusterSessionKey.
-	fullKey := fmt.Sprintf("%s_%s", clusterNameID, sessionID)
+func (s *ServerHandler) resolveTaskLogFilename(clusterSessionKey, clusterNameID, sessionID, taskID string, attemptNumber int, suffix string) (nodeID, filename string, err error) {
+	snap, ok := s.sessionLoader.GetSnapshot(clusterSessionKey)
+	if !ok {
+		return "", "", fmt.Errorf("snapshot not found for %s", clusterSessionKey)
+	}
 
-	// Get task attempts by task ID
-	taskAttempts, found := s.eventHandler.GetTaskByID(fullKey, taskID)
+	taskAttempts, found := getTasksByID(snap.Tasks, taskID)
 	if !found {
 		return "", "", fmt.Errorf("task not found: task_id=%s", taskID)
 	}
@@ -402,14 +453,13 @@ func (s *ServerHandler) resolveTaskLogFilename(clusterNameID, sessionID, taskID
 
 // resolveActorLogFilename resolves log file for an actor by querying actor events.
 // This mirrors Ray Dashboard's _resolve_actor_filename logic.
-func (s *ServerHandler) resolveActorLogFilename(clusterNameID, sessionID, actorID, suffix string) (nodeID, filename string, err error) {
-	// Construct full cluster session key for event lookup
-	// We append the sessionID to the clusterNameID (which is "name_namespace")
-	// to match the key format used by utils.BuildClusterSessionKey.
-	fullKey := fmt.Sprintf("%s_%s", clusterNameID, sessionID)
+func (s *ServerHandler) resolveActorLogFilename(clusterSessionKey, clusterNameID, sessionID, actorID, suffix string) (nodeID, filename string, err error) {
+	snap, ok := s.sessionLoader.GetSnapshot(clusterSessionKey)
+	if !ok {
+		return "", "", fmt.Errorf("snapshot not found for %s", clusterSessionKey)
+	}
 
-	// Get actor by actor ID
-	actor, found := s.eventHandler.GetActorByID(fullKey, actorID)
+	actor, found := snap.Actors[actorID]
 	if !found {
 		return "", "", fmt.Errorf("actor not found: actor_id=%s", actorID)
 	}
@@ -520,10 +570,26 @@ func (s *ServerHandler) ipToNodeId(rayClusterNameNamespace, sessionID, nodeIP st
 // searchNodeIDHexInEventFile searches for a node with the given IP in a single event file.
 // Returns (nodeIDHex, true) if found, ("", false) otherwise.
 func (s *ServerHandler) searchNodeIDHexInEventFile(rayClusterNameNamespace, filePath, nodeIP string) (string, bool) {
-	reader := s.reader.GetContent(rayClusterNameNamespace, filePath)
+	var reader io.Reader
+	var err error
+	if strings.HasSuffix(filePath, ".gz") {
+		var rc io.ReadCloser
+		rc, err = compression.ReadCompressedContent(s.reader, rayClusterNameNamespace, filePath)
+		if err != nil {
+			logrus.Warnf("Failed to decompress node event file %s: %v", filePath, err)
+			return "", false
+		}
+		reader = rc
+	} else {
+		reader = s.reader.GetContent(rayClusterNameNamespace, filePath)
+	}
+
 	if reader == nil {
 		return "", false
 	}
+	if closer, ok := reader.(io.Closer); ok {
+		defer closer.Close()
+	}
 
 	data, err := io.ReadAll(reader)
 	if err != nil {

historyserver/pkg/historyserver/router.go

@@ -23,6 +23,7 @@ import (
 	"k8s.io/apimachinery/pkg/types"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 
+	"github.com/ray-project/kuberay/historyserver/pkg/eventserver"
 	eventtypes "github.com/ray-project/kuberay/historyserver/pkg/eventserver/types"
 	"github.com/ray-project/kuberay/historyserver/pkg/utils"
 )
@@ -36,6 +37,12 @@ const (
 	ATTRIBUTE_SERVICE_NAME = "cluster_service_name"
 )
 
+// handleMissingSnapshot responds 503 when the session snapshot is not in the cache.
+func (s *ServerHandler) handleMissingSnapshot(resp *restful.Response) {
+	resp.WriteErrorString(http.StatusServiceUnavailable,
+		"session snapshot not in cache; reload via /enter_cluster")
+}
+
 type ServiceInfo struct {
 	ServiceName string
 	Namespace   string
@@ -294,19 +301,58 @@ func routerRayClusterSet(s *ServerHandler) {
 	defer restful.Add(ws)
 
 	ws.Path("/enter_cluster").Consumes(restful.MIME_JSON).Produces(restful.MIME_JSON).Filter(RequestLogFilter)
-	ws.Route(ws.GET("/{namespace}/{name}/{session}").To(func(r1 *restful.Request, r2 *restful.Response) {
-		name := r1.PathParameter("name")
-		namespace := r1.PathParameter("namespace")
-		session := r1.PathParameter("session")
+	enterHandler := func(r1 *restful.Request, r2 *restful.Response, namespace, name, session string) {
+		resolvedSession, found := s.findSessionInMap(namespace, name, session)
+		if !found {
+			if s.clientManager != nil && s.reader != nil {
+				s.listClusters(s.maxClusters)
+			}
+			resolvedSession, found = s.findSessionInMap(namespace, name, session)
+		}
+
+		if !found {
+			r2.WriteErrorString(http.StatusNotFound, fmt.Sprintf("cluster %s/%s with session %s not found", namespace, name, session))
+			return
+		}
+
+		if resolvedSession != "live" {
+			if ParseSessionTimestamp(resolvedSession).IsZero() {
+				logrus.Warnf("Rejecting invalid session name: %s/%s/%s", namespace, name, resolvedSession)
+				r2.WriteErrorString(http.StatusBadRequest, fmt.Sprintf("invalid session name: %q", resolvedSession))
+				return
+			}
+			info := utils.ClusterInfo{Name: name, Namespace: namespace, SessionName: resolvedSession}
+			live, err := s.sessionLoader.LoadSession(r1.Request.Context(), info)
+			if err != nil {
+				logrus.Errorf("Failed to load session %s/%s/%s: %v", namespace, name, resolvedSession, err)
+				r2.WriteErrorString(http.StatusInternalServerError, err.Error())
+				return
+			}
+
+			// Users might use the complete session name to enter a live cluster,
+			// so "live" sentinel is set to avoid querying empty in-memory state.
+			if live {
+				resolvedSession = "live"
+			}
+		}
+
 		http.SetCookie(r2, &http.Cookie{MaxAge: 600, Path: "/", Name: COOKIE_CLUSTER_NAME_KEY, Value: name})
 		http.SetCookie(r2, &http.Cookie{MaxAge: 600, Path: "/", Name: COOKIE_CLUSTER_NAMESPACE_KEY, Value: namespace})
-		http.SetCookie(r2, &http.Cookie{MaxAge: 600, Path: "/", Name: COOKIE_SESSION_NAME_KEY, Value: session})
+		http.SetCookie(r2, &http.Cookie{MaxAge: 600, Path: "/", Name: COOKIE_SESSION_NAME_KEY, Value: resolvedSession})
+
 		r2.WriteJson(map[string]interface{}{
 			"result":    "success",
 			"name":      name,
 			"namespace": namespace,
-			"session":   session,
+			"session":   resolvedSession,
 		}, "application/json")
+	}
+
+	ws.Route(ws.GET("/{namespace}/{name}/{session}").To(func(r1 *restful.Request, r2 *restful.Response) {
+		name := r1.PathParameter("name")
+		namespace := r1.PathParameter("namespace")
+		session := r1.PathParameter("session")
+		enterHandler(r1, r2, namespace, name, session)
 	}).
 		Doc("set cookie for cluster").
 		Param(ws.PathParameter("namespace", "namespace")).
@@ -415,19 +461,22 @@ func (s *ServerHandler) getNodes(req *restful.Request, resp *restful.Response) {
 	// Parse query parameters.
 	viewParam := req.QueryParameter("view")
 
-	// Get nodes from the cluster session.
 	clusterName := req.Attribute(COOKIE_CLUSTER_NAME_KEY).(string)
 	clusterNamespace := req.Attribute(COOKIE_CLUSTER_NAMESPACE_KEY).(string)
 	clusterSessionKey := utils.BuildClusterSessionKey(clusterName, clusterNamespace, sessionName)
-	nodeMap := s.eventHandler.GetNodeMap(clusterSessionKey)
+	snap, ok := s.sessionLoader.GetSnapshot(clusterSessionKey)
+	if !ok {
+		s.handleMissingSnapshot(resp)
+		return
+	}
 
 	// Handle different view types.
 	switch viewParam {
 	case "hostNameList":
-		s.getNodesHostNameList(nodeMap, resp)
+		s.getNodesHostNameList(snap.Nodes, resp)
 	case "summary", "":
 		// Default to summary view
-		s.getNodesSummary(nodeMap, sessionName, resp)
+		s.getNodesSummary(snap.Nodes, sessionName, resp)
 	default:
 		resp.WriteErrorString(http.StatusBadRequest, fmt.Sprintf("unsupported view parameter: %s", viewParam))
 	}
@@ -549,12 +598,15 @@ func (s *ServerHandler) getNode(req *restful.Request, resp *restful.Response) {
 		return
 	}
 
-	// Get the specified node from the cluster session.
-	// A cluster lifecycle is identified by a cluster session.
 	clusterName := req.Attribute(COOKIE_CLUSTER_NAME_KEY).(string)
 	clusterNamespace := req.Attribute(COOKIE_CLUSTER_NAMESPACE_KEY).(string)
 	clusterSessionKey := utils.BuildClusterSessionKey(clusterName, clusterNamespace, sessionName)
-	targetNode, found := s.eventHandler.GetNodeByNodeID(clusterSessionKey, targetNodeId)
+	snap, ok := s.sessionLoader.GetSnapshot(clusterSessionKey)
+	if !ok {
+		s.handleMissingSnapshot(resp)
+		return
+	}
+	targetNode, found := snap.Nodes[targetNodeId]
 	if !found {
 		resp.WriteErrorString(http.StatusNotFound, fmt.Sprintf("node %s not found", targetNodeId))
 		return
@@ -574,9 +626,8 @@ func (s *ServerHandler) getNode(req *restful.Request, resp *restful.Response) {
 	// Fill actors for this node.
 	// Frontend expects actors as {[actorId]: ActorDetail}, not an empty array.
 	// Ref: https://github.com/ray-project/ray/blob/8a7b47bc5c/python/ray/dashboard/client/src/pages/node/NodeDetail.tsx#L233
-	actorsMap := s.eventHandler.GetActorsMap(clusterSessionKey)
 	nodeActors := make(map[string]interface{})
-	for _, actor := range actorsMap {
+	for _, actor := range snap.Actors {
 		if actor.Address.NodeID == targetNodeId {
 			nodeActors[actor.ActorID] = formatActorForResponse(actor)
 		}
@@ -616,6 +667,11 @@ func (s *ServerHandler) getEvents(req *restful.Request, resp *restful.Response)
 	}
 
 	clusterSessionKey := utils.BuildClusterSessionKey(clusterName, clusterNamespace, sessionName)
+	snap, ok := s.sessionLoader.GetSnapshot(clusterSessionKey)
+	if !ok {
+		s.handleMissingSnapshot(resp)
+		return
+	}
 
 	// Check if job_id parameter exists in query string (even if empty)
 	// This aligns with Ray Dashboard behavior:
@@ -628,9 +684,10 @@ func (s *ServerHandler) getEvents(req *restful.Request, resp *restful.Response)
 	var response map[string]any
 
 	if jobIDExists {
-		// Return events for a specific job
-		// Response format matches Ray Dashboard: {"result": true, "msg": "...", "data": {"jobId": "...", "events": [...]}}
-		events := s.eventHandler.ClusterLogEventMap.GetEventsByJobID(clusterSessionKey, jobID)
+		events := make([]map[string]any, 0)
+		for _, logEvent := range snap.LogEventsByJobID[jobID] {
+			events = append(events, logEvent.ToAPIResponse())
+		}
 		response = map[string]any{
 			"result": true,
 			"msg":    "Job events fetched.",
@@ -640,9 +697,14 @@ func (s *ServerHandler) getEvents(req *restful.Request, resp *restful.Response)
 			},
 		}
 	} else {
-		// Return all events grouped by job_id
-		// Response format matches Ray Dashboard: {"result": true, "msg": "...", "data": {"events": {job_id: [...], ...}}}
-		events := s.eventHandler.ClusterLogEventMap.GetAllEvents(clusterSessionKey)
+		events := make(map[string][]map[string]any, len(snap.LogEventsByJobID))
+		for jobID, logEvents := range snap.LogEventsByJobID {
+			eventList := make([]map[string]any, 0, len(logEvents))
+			for _, logEvent := range logEvents {
+				eventList = append(eventList, logEvent.ToAPIResponse())
+			}
+			events[jobID] = eventList
+		}
 		response = map[string]any{
 			"result": true,
 			"msg":    "All events fetched.",
@@ -692,10 +754,14 @@ func (s *ServerHandler) getJobs(req *restful.Request, resp *restful.Response) {
 	}
 
 	clusterSessionKey := utils.BuildClusterSessionKey(clusterName, clusterNamespace, sessionName)
-	jobsMap := s.eventHandler.GetJobsMap(clusterSessionKey)
+	snap, ok := s.sessionLoader.GetSnapshot(clusterSessionKey)
+	if !ok {
+		s.handleMissingSnapshot(resp)
+		return
+	}
 
-	jobs := make([]eventtypes.Job, 0, len(jobsMap))
-	for _, job := range jobsMap {
+	jobs := make([]eventtypes.Job, 0, len(snap.Jobs))
+	for _, job := range snap.Jobs {
 		jobs = append(jobs, job)
 	}
 
@@ -797,8 +863,13 @@ func (s *ServerHandler) getJob(req *restful.Request, resp *restful.Response) {
 	jobID := req.PathParameter("job_id")
 
 	clusterSessionKey := utils.BuildClusterSessionKey(clusterName, clusterNamespace, sessionName)
-	job, found := s.eventHandler.GetJobByJobID(clusterSessionKey, jobID)
+	snap, ok := s.sessionLoader.GetSnapshot(clusterSessionKey)
+	if !ok {
+		s.handleMissingSnapshot(resp)
+		return
+	}
 
+	job, found := snap.Jobs[jobID]
 	if !found {
 		responseString := fmt.Sprintf("Job %s does not exist", jobID)
 		resp.Write([]byte(responseString))
@@ -831,8 +902,15 @@ func (s *ServerHandler) getClusterStatus(req *restful.Request, resp *restful.Res
 	var err error
 
 	if format == "1" {
-		// Build cluster status from debug_state.txt and task/actor data
-		statusString := s.buildFormattedClusterStatus(clusterName, clusterNamespace, sessionName)
+		clusterSessionKey := utils.BuildClusterSessionKey(clusterName, clusterNamespace, sessionName)
+		snap, ok := s.sessionLoader.GetSnapshot(clusterSessionKey)
+		if !ok {
+			s.handleMissingSnapshot(resp)
+			return
+		}
+
+		// Build cluster status from debug_state.txt and snapshot data
+		statusString := s.buildFormattedClusterStatus(snap, clusterName, clusterNamespace, sessionName)
 
 		response := FormattedClusterStatusResponse{
 			Result: true,
@@ -864,7 +942,7 @@ func (s *ServerHandler) getClusterStatus(req *restful.Request, resp *restful.Res
 }
 
 // buildFormattedClusterStatus reconstructs the cluster status from debug_state.txt and pending tasks and actors
-func (s *ServerHandler) buildFormattedClusterStatus(clusterName, clusterNamespace, sessionName string) string {
+func (s *ServerHandler) buildFormattedClusterStatus(snap *eventserver.SessionSnapshot, clusterName, clusterNamespace, sessionName string) string {
 	builder := NewClusterStatusBuilder()
 	clusterNameID := clusterName + "_" + clusterNamespace
 	logsPath := path.Join(sessionName, utils.RAY_SESSIONDIR_LOGDIR_NAME)
@@ -894,10 +972,12 @@ func (s *ServerHandler) buildFormattedClusterStatus(clusterName, clusterNamespac
 		logrus.Debugf("Found %d nodes but failed to parse any debug_state.txt for cluster %s session %s", len(nodeIDs), clusterName, sessionName)
 	}
 
-	clusterSessionKey := utils.BuildClusterSessionKey(clusterName, clusterNamespace, sessionName)
-	tasks := s.eventHandler.GetTasks(clusterSessionKey)
-	actors := s.eventHandler.GetActors(clusterSessionKey)
-	nodes := s.eventHandler.GetNodeMap(clusterSessionKey)
+	nodes := snap.Nodes
+	tasks := snap.Tasks
+	actors := make([]eventtypes.Actor, 0, len(snap.Actors))
+	for _, a := range snap.Actors {
+		actors = append(actors, a)
+	}
 
 	// Use the last timestamp from tasks/actors to represent when the cluster was last active.
 	// Fallback to session timestamp if no task/actor timestamps are available.
@@ -1110,13 +1190,16 @@ func (s *ServerHandler) getLogicalActors(req *restful.Request, resp *restful.Res
 		return
 	}
 
-	// Get actors from EventHandler's in-memory map
 	clusterSessionKey := utils.BuildClusterSessionKey(clusterName, clusterNamespace, sessionName)
-	actorsMap := s.eventHandler.GetActorsMap(clusterSessionKey)
+	snap, ok := s.sessionLoader.GetSnapshot(clusterSessionKey)
+	if !ok {
+		s.handleMissingSnapshot(resp)
+		return
+	}
 
 	// Format response to match Ray Dashboard API format
-	formattedActors := make(map[string]interface{})
-	for _, actor := range actorsMap {
+	formattedActors := make(map[string]interface{}, len(snap.Actors))
+	for _, actor := range snap.Actors {
 		formattedActors[actor.ActorID] = formatActorForResponse(actor)
 	}
 
@@ -1189,10 +1272,15 @@ func (s *ServerHandler) getLogicalActor(req *restful.Request, resp *restful.Resp
 
 	actorID := req.PathParameter("single_actor")
 
-	// Get actor from EventHandler's in-memory map.
-	// Actor IDs are normalized to hex at ingestion time, so lookup is by hex ID.
 	clusterSessionKey := utils.BuildClusterSessionKey(clusterName, clusterNamespace, sessionName)
-	actor, found := s.eventHandler.GetActorByID(clusterSessionKey, actorID)
+	snap, ok := s.sessionLoader.GetSnapshot(clusterSessionKey)
+	if !ok {
+		s.handleMissingSnapshot(resp)
+		return
+	}
+
+	// Actor IDs are normalized to hex at ingestion time, so lookup is by hex ID.
+	actor, found := snap.Actors[actorID]
 
 	replyActorInfo := ReplyActorInfo{
 		Data: ActorInfoData{},
@@ -1258,6 +1346,12 @@ func (s *ServerHandler) getNodeLogFile(req *restful.Request, resp *restful.Respo
 		return
 	}
 
+	clusterSessionKey := utils.BuildClusterSessionKey(clusterNameID, clusterNamespace, sessionName)
+	if _, ok := s.sessionLoader.GetSnapshot(clusterSessionKey); !ok {
+		s.handleMissingSnapshot(resp)
+		return
+	}
+
 	// Only resolve node_ip to node_id from stored events for dead cluster
 	if options.NodeID == "" && options.NodeIP != "" {
 		nodeID, err := s.ipToNodeId(clusterNameID+"_"+clusterNamespace, sessionName, options.NodeIP)
@@ -1269,7 +1363,7 @@ func (s *ServerHandler) getNodeLogFile(req *restful.Request, resp *restful.Respo
 		options.NodeID = nodeID
 	}
 
-	content, err := s._getNodeLogFile(clusterNameID+"_"+clusterNamespace, sessionName, options)
+	content, err := s._getNodeLogFile(clusterSessionKey, clusterNameID+"_"+clusterNamespace, sessionName, options)
 	if err != nil {
 		var httpErr *utils.HTTPError
 		if errors.As(err, &httpErr) {
@@ -1429,9 +1523,13 @@ func (s *ServerHandler) getTaskSummarize(req *restful.Request, resp *restful.Res
 	// Ref: https://github.com/ray-project/ray/blob/ad1b87448fec4db7ef11f1697f9bc02ae6a7ba09/python/ray/dashboard/state_aggregator.py#L569-L582
 	listAPIOptions.Limit = utils.RayMaxLimitFromAPIServer
 
-	// Get all tasks
 	clusterSessionKey := utils.BuildClusterSessionKey(clusterName, clusterNamespace, sessionName)
-	tasks := s.eventHandler.GetTasks(clusterSessionKey)
+	snap, ok := s.sessionLoader.GetSnapshot(clusterSessionKey)
+	if !ok {
+		s.handleMissingSnapshot(resp)
+		return
+	}
+	tasks := snap.Tasks
 
 	// Calculate the number of tasks after GCS source truncation.
 	// Since we can't access the GCS and num_status_task_events_dropped, we use num_after_truncation to approximate the total number of tasks.
@@ -1452,7 +1550,10 @@ func (s *ServerHandler) getTaskSummarize(req *restful.Request, resp *restful.Res
 	// Ref: https://github.com/ray-project/ray/blob/master/python/ray/dashboard/routes.py
 	var response interface{}
 	if summaryBy == "lineage" {
-		actors := s.eventHandler.GetActors(clusterSessionKey)
+		actors := make([]eventtypes.Actor, 0, len(snap.Actors))
+		for _, a := range snap.Actors {
+			actors = append(actors, a)
+		}
 		lineageSummary := utils.ToSummaryByLineage(tasks, actors)
 
 		response = map[string]interface{}{
@@ -1592,11 +1693,15 @@ func (s *ServerHandler) getTasks(req *restful.Request, resp *restful.Response) {
 		return
 	}
 
-	// Get tasks from the cluster session.
 	clusterName := req.Attribute(COOKIE_CLUSTER_NAME_KEY).(string)
 	clusterNamespace := req.Attribute(COOKIE_CLUSTER_NAMESPACE_KEY).(string)
 	clusterSessionKey := utils.BuildClusterSessionKey(clusterName, clusterNamespace, sessionName)
-	tasks := s.eventHandler.GetTasks(clusterSessionKey)
+	snap, ok := s.sessionLoader.GetSnapshot(clusterSessionKey)
+	if !ok {
+		s.handleMissingSnapshot(resp)
+		return
+	}
+	tasks := snap.Tasks
 
 	// Calculate the number of tasks after GCS source truncation.
 	// Since we can't access the GCS and num_status_task_events_dropped, we use num_after_truncation to approximate the total number of tasks.
@@ -1643,7 +1748,7 @@ func (s *ServerHandler) getTasks(req *restful.Request, resp *restful.Response) {
 // The schema aligns with the Ray Dashboard API.
 // Ref: https://github.com/ray-project/ray/blob/d0b1d151d8ea964a711e451d0ae736f8bf95b629/python/ray/util/state/common.py#L730-L819.
 func formatTaskForResponse(task eventtypes.Task, detail bool) map[string]interface{} {
-	// TODO(jwj): Maybe define result schema in types.go.
+	// TODO(jiangjiawei1103): Maybe define result schema in types.go.
 	result := map[string]interface{}{
 		"task_id":            task.TaskID,
 		"attempt_number":     task.TaskAttempt,
@@ -1697,7 +1802,7 @@ func formatTaskForResponse(task eventtypes.Task, detail bool) map[string]interfa
 			})
 		}
 		result["events"] = events
-		// TODO(jwj): Support profiling_data after TASK_PROFILE_EVENT is supported.
+		// TODO(jiangjiawei1103): Support profiling_data after TASK_PROFILE_EVENT is supported.
 		// Ref: https://github.com/ray-project/ray/blob/d0b1d151d8ea964a711e451d0ae736f8bf95b629/python/ray/util/state/common.py#L1616-L1622.
 		// result["profiling_data"] = task.ProfilingData
 		result["task_log_info"] = task.TaskLogInfo
@@ -2053,7 +2158,12 @@ func (s *ServerHandler) getTasksTimeline(req *restful.Request, resp *restful.Res
 	download := req.QueryParameter("download")
 
 	clusterSessionKey := utils.BuildClusterSessionKey(clusterName, clusterNamespace, sessionName)
-	timeline := s.eventHandler.GetTasksTimeline(clusterSessionKey, jobID)
+	snap, ok := s.sessionLoader.GetSnapshot(clusterSessionKey)
+	if !ok {
+		s.handleMissingSnapshot(resp)
+		return
+	}
+	timeline := getTasksTimeline(snap, jobID)
 
 	respData, err := json.Marshal(timeline)
 	if err != nil {

historyserver/pkg/historyserver/server.go

@@ -5,11 +5,12 @@ import (
 	"fmt"
 	"log"
 	"net/http"
+	"sync"
 	"time"
 
 	"github.com/ray-project/kuberay/historyserver/pkg/collector/types"
-	"github.com/ray-project/kuberay/historyserver/pkg/eventserver"
 	"github.com/ray-project/kuberay/historyserver/pkg/storage"
+	"github.com/ray-project/kuberay/historyserver/pkg/utils"
 	"github.com/sirupsen/logrus"
 	"k8s.io/client-go/transport"
 )
@@ -21,17 +22,20 @@ type ServerHandler struct {
 
 	reader        storage.StorageReader
 	clientManager *ClientManager
-	eventHandler  *eventserver.EventHandler
+	sessionLoader *SessionLoader
 	httpClient    *http.Client
 
 	useKubernetesProxy bool
+
+	mu          sync.RWMutex
+	clustersMap map[utils.ClusterKey][]utils.ClusterInfo
 }
 
-func NewServerHandler(c *types.RayHistoryServerConfig, dashboardDir string, reader storage.StorageReader, clientManager *ClientManager, eventHandler *eventserver.EventHandler, useKubernetesProxy bool) (*ServerHandler, error) {
+func NewServerHandler(c *types.RayHistoryServerConfig, dashboardDir string, reader storage.StorageReader, clientManager *ClientManager, sessionLoader *SessionLoader, useKubernetesProxy bool) (*ServerHandler, error) {
 	handler := &ServerHandler{
 		reader:        reader,
 		clientManager: clientManager,
-		eventHandler:  eventHandler,
+		sessionLoader: sessionLoader,
 
 		rootDir:      c.RootDir,
 		dashboardDir: dashboardDir,

historyserver/pkg/historyserver/session_loader.go

@@ -0,0 +1,137 @@
+package historyserver
+
+import (
+	"context"
+	"fmt"
+	"time"
+
+	"github.com/hashicorp/golang-lru/v2/expirable"
+	"golang.org/x/sync/singleflight"
+
+	"github.com/ray-project/kuberay/historyserver/pkg/eventserver"
+	eventtypes "github.com/ray-project/kuberay/historyserver/pkg/eventserver/types"
+	"github.com/ray-project/kuberay/historyserver/pkg/utils"
+)
+
+const (
+	// DefaultSessionProcessTimeout caps how long cold-load for a single session can run.
+	DefaultSessionProcessTimeout = 2 * time.Minute
+	// DefaultSessionCacheSize is the LRU capacity for dead-session snapshots.
+	DefaultSessionCacheSize = 100
+	// DefaultSessionCacheTTL is how long a dead-session snapshot stays cached after last access.
+	// 0 disables expiry, leaving LRU capacity (cacheSize) as the only bound.
+	DefaultSessionCacheTTL time.Duration = 0
+)
+
+// processor is an interface to enable mocking SessionProcessor in tests.
+type processor interface {
+	ProcessSession(ctx context.Context, info utils.ClusterInfo) (SessionStatus, *eventserver.SessionSnapshot, error)
+}
+
+// SessionLoader caches dead-session snapshots in a size-bounded LRU with optional
+// TTL expiry and triggers session processing on cache miss. Concurrent callers
+// for the same session are coalesced via singleflight.
+type SessionLoader struct {
+	processor      processor
+	cache          *expirable.LRU[string, *eventserver.SessionSnapshot]
+	sf             singleflight.Group
+	serverCtx      context.Context
+	processTimeout time.Duration
+}
+
+// NewSessionLoader wires a SessionLoader.
+func NewSessionLoader(p processor, serverCtx context.Context, processTimeout time.Duration, cacheSize int, cacheTTL time.Duration) *SessionLoader {
+	return &SessionLoader{
+		processor:      p,
+		cache:          expirable.NewLRU[string, *eventserver.SessionSnapshot](cacheSize, nil, cacheTTL),
+		serverCtx:      serverCtx,
+		processTimeout: processTimeout,
+	}
+}
+
+// GetSnapshot returns a per-request view of the cached snapshot.
+func (s *SessionLoader) GetSnapshot(clusterSessionKey string) (*eventserver.SessionSnapshot, bool) {
+	cached, ok := s.cache.Get(clusterSessionKey)
+	if !ok {
+		return nil, false
+	}
+	// Renew the TTL so active debug sessions are not evicted.
+	s.cache.Add(clusterSessionKey, cached)
+	out := *cached
+	out.Tasks = append([]eventtypes.Task(nil), cached.Tasks...)
+	return &out, true
+}
+
+// LoadSession blocks until a dead session is processed and cached or an
+// unrecoverable error is observed.
+func (s *SessionLoader) LoadSession(ctx context.Context, info utils.ClusterInfo) (live bool, err error) {
+	// Fast pre-flight: skip singleflight entirely if ctx is already dead.
+	if err := ctx.Err(); err != nil {
+		return false, err
+	}
+
+	clusterSessionKey := utils.BuildClusterSessionKey(info.Name, info.Namespace, info.SessionName)
+	if _, ok := s.cache.Get(clusterSessionKey); ok {
+		return false, nil
+	}
+
+	// TODO(jiangjiawei1103): No graceful drain on shutdown. When the pod receives
+	// SIGTERM, serverCtx is cancelled immediately, causing any in-flight cold-load
+	// requests to return ctx.Err() and clients to receive HTTP 500.
+	ch := s.sf.DoChan(clusterSessionKey, func() (interface{}, error) {
+		loadCtx, cancel := context.WithTimeout(s.serverCtx, s.processTimeout)
+		defer cancel()
+		return s.doLoadSession(loadCtx, info, clusterSessionKey)
+	})
+
+	select {
+	case <-ctx.Done():
+		// Release the caller; the singleflight winner keeps running and its
+		// result will be cached for the next caller for this session.
+		//
+		// Do NOT sf.Forget(clusterSessionKey) here: a racing new call would kick off a second
+		// processor execution in parallel with the still-running one.
+		return false, ctx.Err()
+	case result := <-ch:
+		if result.Err != nil {
+			return false, result.Err
+		}
+		live, _ := result.Val.(bool)
+		return live, nil
+	}
+}
+
+// doLoadSession is the singleflight body invoked by LoadSession.
+// live is true when the cluster is still alive.
+func (s *SessionLoader) doLoadSession(ctx context.Context, info utils.ClusterInfo, clusterSessionKey string) (live bool, err error) {
+	if _, ok := s.cache.Get(clusterSessionKey); ok {
+		return false, nil
+	}
+
+	status, snap, err := s.processor.ProcessSession(ctx, info)
+	if err != nil {
+		return false, err
+	}
+
+	switch status {
+	case SessionStatusProcessed:
+		if snap == nil {
+			return false, fmt.Errorf("unexpected nil snapshot for session status %v", status)
+		}
+		s.putSnapshot(clusterSessionKey, snap)
+		return false, nil
+
+	case SessionStatusLive:
+		return true, nil
+
+	default:
+		// The zero-value guard prevents an uninitialized status from being silently
+		// treated as Live or Processed.
+		return false, fmt.Errorf("unexpected session status %v", status)
+	}
+}
+
+// putSnapshot stores a dead-session snapshot in the LRU cache.
+func (s *SessionLoader) putSnapshot(clusterSessionKey string, snap *eventserver.SessionSnapshot) {
+	s.cache.Add(clusterSessionKey, snap)
+}

historyserver/pkg/historyserver/session_loader_test.go

@@ -0,0 +1,379 @@
+package historyserver
+
+import (
+	"context"
+	"errors"
+	"sort"
+	"sync"
+	"sync/atomic"
+	"testing"
+	"time"
+
+	"github.com/ray-project/kuberay/historyserver/pkg/eventserver"
+	eventtypes "github.com/ray-project/kuberay/historyserver/pkg/eventserver/types"
+	"github.com/ray-project/kuberay/historyserver/pkg/utils"
+)
+
+// fakeProcessor is a configurable test double for processor.
+type fakeProcessor struct {
+	calls int32
+	fn    func(ctx context.Context, info utils.ClusterInfo) (SessionStatus, *eventserver.SessionSnapshot, error)
+}
+
+func (f *fakeProcessor) ProcessSession(ctx context.Context, info utils.ClusterInfo) (SessionStatus, *eventserver.SessionSnapshot, error) {
+	atomic.AddInt32(&f.calls, 1)
+	if f.fn == nil {
+		return SessionStatusProcessed, &eventserver.SessionSnapshot{}, nil
+	}
+	return f.fn(ctx, info)
+}
+
+func (f *fakeProcessor) callCount() int32 { return atomic.LoadInt32(&f.calls) }
+
+func (f *fakeProcessor) setFn(fn func(ctx context.Context, info utils.ClusterInfo) (SessionStatus, *eventserver.SessionSnapshot, error)) {
+	f.fn = fn
+}
+
+func newTestSessionLoader(t *testing.T, p processor, cacheSize int) *SessionLoader {
+	t.Helper()
+	if cacheSize <= 0 {
+		cacheSize = DefaultSessionCacheSize
+	}
+	return NewSessionLoader(p, context.Background(), DefaultSessionProcessTimeout, cacheSize, DefaultSessionCacheTTL)
+}
+
+func testEnterClusterInfo() utils.ClusterInfo {
+	return utils.ClusterInfo{
+		Name:        "raycluster-test",
+		Namespace:   "default",
+		SessionName: "session_2026-04-22_10-00-00_000000_1",
+	}
+}
+
+// testSnapshot builds a minimal snapshot.
+func testSnapshot(clusterSessionKey string) *eventserver.SessionSnapshot {
+	return &eventserver.SessionSnapshot{
+		SessionKey: clusterSessionKey,
+	}
+}
+
+// TestLoadSession_ProcessorError_PropagatesAndCleans verifies that on the error
+// path, all coalesced callers see the same error and the dedup group is cleared.
+func TestLoadSession_ProcessorError_PropagatesAndCleans(t *testing.T) {
+	info := testEnterClusterInfo()
+	processorErr := errors.New("simulated parse failure")
+
+	// Phase 1: error path
+	release := make(chan struct{})
+	fp := &fakeProcessor{
+		fn: func(_ context.Context, _ utils.ClusterInfo) (SessionStatus, *eventserver.SessionSnapshot, error) {
+			<-release
+			return SessionStatusEventsErr, nil, processorErr
+		},
+	}
+	sessionLoader := newTestSessionLoader(t, fp, 0)
+
+	const n = 5
+	var wg sync.WaitGroup
+	errs := make([]error, n)
+	for i := 0; i < n; i++ {
+		wg.Add(1)
+		go func(idx int) {
+			defer wg.Done()
+			_, errs[idx] = sessionLoader.LoadSession(context.Background(), info)
+		}(i)
+	}
+
+	// 50ms >> scheduling latency, well under a second.
+	time.Sleep(50 * time.Millisecond)
+	close(release)
+	wg.Wait()
+
+	if got := fp.callCount(); got != 1 {
+		t.Fatalf("expected exactly 1 processor call (dedup applies on error path), got %d", got)
+	}
+	for i, err := range errs {
+		if !errors.Is(err, processorErr) {
+			t.Errorf("caller %d: expected error wrapping processorErr, got %v", i, err)
+		}
+	}
+
+	// Phase 2: dedup group cleared
+	fp.setFn(func(_ context.Context, _ utils.ClusterInfo) (SessionStatus, *eventserver.SessionSnapshot, error) {
+		return SessionStatusProcessed, &eventserver.SessionSnapshot{}, nil
+	})
+	if _, err := sessionLoader.LoadSession(context.Background(), info); err != nil {
+		t.Fatalf("post-error retry: expected nil, got %v", err)
+	}
+	if got := fp.callCount(); got != 2 {
+		t.Fatalf("expected processor called 2x total (1 error + 1 retry), got %d", got)
+	}
+}
+
+// TestLoadSession_ProcessorError_NoInternalRetry verifies a processor error
+// is returned to the caller exactly once; SessionLoader does not re-invoke
+// processor within the same LoadSession call.
+func TestLoadSession_ProcessorError_NoInternalRetry(t *testing.T) {
+	info := testEnterClusterInfo()
+	processorErr := errors.New("simulated parse failure")
+	fp := &fakeProcessor{
+		fn: func(_ context.Context, _ utils.ClusterInfo) (SessionStatus, *eventserver.SessionSnapshot, error) {
+			return SessionStatusEventsErr, nil, processorErr
+		},
+	}
+	sessionLoader := newTestSessionLoader(t, fp, 0)
+
+	_, err := sessionLoader.LoadSession(context.Background(), info)
+	if !errors.Is(err, processorErr) {
+		t.Fatalf("expected LoadSession to return processorErr, got %v", err)
+	}
+	if got := fp.callCount(); got != 1 {
+		t.Fatalf("expected processor called exactly 1x (no internal retry), got %d", got)
+	}
+
+	fp.setFn(func(_ context.Context, _ utils.ClusterInfo) (SessionStatus, *eventserver.SessionSnapshot, error) {
+		return SessionStatusProcessed, &eventserver.SessionSnapshot{}, nil
+	})
+	if _, err := sessionLoader.LoadSession(context.Background(), info); err != nil {
+		t.Fatalf("client-driven retry: expected nil, got %v", err)
+	}
+	if got := fp.callCount(); got != 2 {
+		t.Fatalf("expected processor called 2x total (1 error + 1 client retry), got %d", got)
+	}
+}
+
+// TestLoadSession_LiveAndProcessed verifies LoadSession surfaces the live/processed distinction.
+func TestLoadSession_LiveAndProcessed(t *testing.T) {
+	tests := []struct {
+		name     string
+		status   SessionStatus
+		wantLive bool
+	}{
+		{"processed -> live=false", SessionStatusProcessed, false},
+		{"live -> live=true", SessionStatusLive, true},
+	}
+
+	for _, tc := range tests {
+		t.Run(tc.name, func(t *testing.T) {
+			fp := &fakeProcessor{
+				fn: func(_ context.Context, _ utils.ClusterInfo) (SessionStatus, *eventserver.SessionSnapshot, error) {
+					var built *eventserver.SessionSnapshot
+					if tc.status == SessionStatusProcessed {
+						built = &eventserver.SessionSnapshot{}
+					}
+					return tc.status, built, nil
+				},
+			}
+			sessionLoader := newTestSessionLoader(t, fp, 0)
+
+			live, err := sessionLoader.LoadSession(context.Background(), testEnterClusterInfo())
+			if err != nil {
+				t.Fatalf("unexpected error: %v", err)
+			}
+			if live != tc.wantLive {
+				t.Fatalf("live = %v, want %v", live, tc.wantLive)
+			}
+		})
+	}
+}
+
+// TestLoadSession_ZeroValueStatus_DoesNotSilentlyMatchLive verifies the zero-value
+// SessionStatus (SessionStatusUnknown) surfaces an error.
+func TestLoadSession_ZeroValueStatus_DoesNotSilentlyMatchLive(t *testing.T) {
+	fp := &fakeProcessor{
+		fn: func(_ context.Context, _ utils.ClusterInfo) (SessionStatus, *eventserver.SessionSnapshot, error) {
+			var zero SessionStatus // SessionStatusUnknown
+			return zero, nil, nil
+		},
+	}
+	sessionLoader := newTestSessionLoader(t, fp, 0)
+
+	live, err := sessionLoader.LoadSession(context.Background(), testEnterClusterInfo())
+	if err == nil {
+		t.Fatalf("expected error from zero-value status, got nil (live=%v)", live)
+	}
+	if live {
+		t.Fatalf("zero-value status must not produce live=true (got live=%v, err=%v)", live, err)
+	}
+}
+
+// TestLoadSession_FastPath_SkipsSingleflight verifies the fast-path:
+// once a session is loaded, repeat LoadSession calls must not invoke processor again.
+func TestLoadSession_FastPath_SkipsSingleflight(t *testing.T) {
+	fp := &fakeProcessor{
+		fn: func(_ context.Context, _ utils.ClusterInfo) (SessionStatus, *eventserver.SessionSnapshot, error) {
+			return SessionStatusProcessed, &eventserver.SessionSnapshot{}, nil
+		},
+	}
+	sessionLoader := newTestSessionLoader(t, fp, 0)
+	info := testEnterClusterInfo()
+
+	// First call: cold path, processor runs, session marked loaded.
+	if _, err := sessionLoader.LoadSession(context.Background(), info); err != nil {
+		t.Fatalf("cold-path LoadSession: %v", err)
+	}
+	if got := fp.callCount(); got != 1 {
+		t.Fatalf("after cold path: callCount = %d, want 1", got)
+	}
+
+	// Subsequent calls: fast path, processor must NOT be invoked again.
+	for i := 0; i < 5; i++ {
+		if _, err := sessionLoader.LoadSession(context.Background(), info); err != nil {
+			t.Fatalf("fast path LoadSession #%d: %v", i, err)
+		}
+	}
+	if got := fp.callCount(); got != 1 {
+		t.Fatalf("after fast path: callCount = %d, want 1 (processor must not be invoked again)", got)
+	}
+}
+
+// TestGetSnapshot_PutThenGet verifies the canonical hot path:
+// putSnapshot, then GetSnapshot returns a snapshot with matching content.
+func TestGetSnapshot_PutThenGet(t *testing.T) {
+	info := testEnterClusterInfo()
+	clusterSessionKey := utils.BuildClusterSessionKey(info.Name, info.Namespace, info.SessionName)
+
+	sl := newTestSessionLoader(t, &fakeProcessor{}, 0)
+	stored := testSnapshot(clusterSessionKey)
+	sl.putSnapshot(clusterSessionKey, stored)
+
+	got, ok := sl.GetSnapshot(clusterSessionKey)
+	if !ok {
+		t.Fatal("GetSnapshot: ok=false")
+	}
+	if got.SessionKey != stored.SessionKey {
+		t.Fatalf("SessionKey: got %q, want %q", got.SessionKey, stored.SessionKey)
+	}
+}
+
+// TestGetSnapshot_ColdMiss verifies that a miss returns (nil, false).
+func TestGetSnapshot_ColdMiss(t *testing.T) {
+	info := testEnterClusterInfo()
+	clusterSessionKey := utils.BuildClusterSessionKey(info.Name, info.Namespace, info.SessionName)
+
+	sl := newTestSessionLoader(t, &fakeProcessor{}, 0)
+	snap, ok := sl.GetSnapshot(clusterSessionKey)
+	if ok || snap != nil {
+		t.Fatalf("expected (nil, false) on cold miss, got (%v, %v)", snap, ok)
+	}
+}
+
+// TestGetSnapshot_PutOverwrites verifies that putSnapshot replaces any prior entry.
+func TestGetSnapshot_PutOverwrites(t *testing.T) {
+	info := testEnterClusterInfo()
+	clusterSessionKey := utils.BuildClusterSessionKey(info.Name, info.Namespace, info.SessionName)
+
+	stale := testSnapshot(clusterSessionKey)
+	stale.Tasks = []eventtypes.Task{{TaskID: "stale-task"}}
+	fresh := testSnapshot(clusterSessionKey)
+	fresh.Tasks = []eventtypes.Task{{TaskID: "fresh-task"}}
+
+	sl := newTestSessionLoader(t, &fakeProcessor{}, 0)
+	sl.putSnapshot(clusterSessionKey, stale)
+	sl.putSnapshot(clusterSessionKey, fresh)
+
+	got, ok := sl.GetSnapshot(clusterSessionKey)
+	if !ok {
+		t.Fatal("Get: ok=false")
+	}
+	if len(got.Tasks) != 1 || got.Tasks[0].TaskID != "fresh-task" {
+		t.Fatalf("putSnapshot did not overwrite; got tasks = %#v, want fresh snapshot", got.Tasks)
+	}
+}
+
+// TestGetSnapshot_TasksIsPerRequest verifies that mutating the slice returned
+// by one GetSnapshot call must not affect another concurrent reader of the same
+// cached entry.
+func TestGetSnapshot_TasksIsPerRequest(t *testing.T) {
+	info := testEnterClusterInfo()
+	key := utils.BuildClusterSessionKey(info.Name, info.Namespace, info.SessionName)
+
+	sl := newTestSessionLoader(t, &fakeProcessor{}, 0)
+	sl.putSnapshot(key, &eventserver.SessionSnapshot{
+		SessionKey: key,
+		Tasks:      []eventtypes.Task{{TaskID: "c"}, {TaskID: "a"}, {TaskID: "b"}},
+	})
+
+	first, _ := sl.GetSnapshot(key)
+	second, _ := sl.GetSnapshot(key)
+
+	sort.Slice(first.Tasks, func(i, j int) bool {
+		return first.Tasks[i].TaskID < first.Tasks[j].TaskID
+	})
+
+	wantOriginal := []string{"c", "a", "b"}
+	for i, task := range second.Tasks {
+		if task.TaskID != wantOriginal[i] {
+			t.Fatalf("second.Tasks[%d].TaskID = %q, want %q (mutation leaked)", i, task.TaskID, wantOriginal[i])
+		}
+	}
+}
+
+// TestGetSnapshot_LRUEviction verifies that once capacity is exceeded, the
+// oldest cached entry is evicted and GetSnapshot for that key returns ok=false.
+func TestGetSnapshot_LRUEviction(t *testing.T) {
+	info := testEnterClusterInfo()
+	k1 := utils.BuildClusterSessionKey(info.Name, info.Namespace, "session_2026-04-22_10-00-00_000000_1")
+	k2 := utils.BuildClusterSessionKey(info.Name, info.Namespace, "session_2026-04-22_11-00-00_000000_1")
+	k3 := utils.BuildClusterSessionKey(info.Name, info.Namespace, "session_2026-04-22_12-00-00_000000_1")
+
+	sl := newTestSessionLoader(t, &fakeProcessor{}, 2)
+	sl.putSnapshot(k1, testSnapshot(k1))
+	sl.putSnapshot(k2, testSnapshot(k2))
+	// Third session evicts the first (LRU).
+	sl.putSnapshot(k3, testSnapshot(k3))
+
+	if _, ok := sl.GetSnapshot(k1); ok {
+		t.Fatal("expected first session to be evicted")
+	}
+	if _, ok := sl.GetSnapshot(k2); !ok {
+		t.Fatal("expected second session to still be cached")
+	}
+	if _, ok := sl.GetSnapshot(k3); !ok {
+		t.Fatal("expected third session to still be cached")
+	}
+}
+
+// TestGetSnapshot_TTLExpiry verifies that when a TTL is set, a cached snapshot
+// expires after the TTL elapses and GetSnapshot returns ok=false.
+func TestGetSnapshot_TTLExpiry(t *testing.T) {
+	info := testEnterClusterInfo()
+	key := utils.BuildClusterSessionKey(info.Name, info.Namespace, info.SessionName)
+
+	const ttl = 30 * time.Millisecond
+	sl := NewSessionLoader(&fakeProcessor{}, context.Background(), DefaultSessionProcessTimeout, DefaultSessionCacheSize, ttl)
+	sl.putSnapshot(key, testSnapshot(key))
+
+	if _, ok := sl.GetSnapshot(key); !ok {
+		t.Fatal("expected snapshot to be cached right after put")
+	}
+
+	time.Sleep(80 * time.Millisecond)
+	if _, ok := sl.GetSnapshot(key); ok {
+		t.Fatal("expected snapshot to be evicted after TTL expiry")
+	}
+}
+
+// TestGetSnapshot_SlidingTTLRenewal verifies that repeated GetSnapshot calls
+// keep a cached entry alive until idle TTL expiry.
+func TestGetSnapshot_SlidingTTLRenewal(t *testing.T) {
+	info := testEnterClusterInfo()
+	key := utils.BuildClusterSessionKey(info.Name, info.Namespace, info.SessionName)
+
+	const ttl = 30 * time.Millisecond
+	sl := NewSessionLoader(&fakeProcessor{}, context.Background(), DefaultSessionProcessTimeout, DefaultSessionCacheSize, ttl)
+	sl.putSnapshot(key, testSnapshot(key))
+
+	deadline := time.Now().Add(100 * time.Millisecond)
+	for time.Now().Before(deadline) {
+		if _, ok := sl.GetSnapshot(key); !ok {
+			t.Fatal("expected snapshot to stay cached while being accessed")
+		}
+		time.Sleep(10 * time.Millisecond)
+	}
+
+	time.Sleep(80 * time.Millisecond)
+	if _, ok := sl.GetSnapshot(key); ok {
+		t.Fatal("expected snapshot to expire after idle period")
+	}
+}

historyserver/pkg/historyserver/session_processor.go

@@ -0,0 +1,103 @@
+package historyserver
+
+import (
+	"context"
+	"errors"
+	"fmt"
+
+	apierrors "k8s.io/apimachinery/pkg/api/errors"
+	k8stypes "k8s.io/apimachinery/pkg/types"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+
+	rayv1 "github.com/ray-project/kuberay/ray-operator/apis/ray/v1"
+
+	"github.com/ray-project/kuberay/historyserver/pkg/eventserver"
+	"github.com/ray-project/kuberay/historyserver/pkg/storage"
+	"github.com/ray-project/kuberay/historyserver/pkg/utils"
+)
+
+// SessionStatus is ProcessSession's outcome classification. Success statuses
+// (Live, Processed) return a nil error; all others pair with a non-nil error.
+type SessionStatus int
+
+const (
+	// SessionStatusUnknown is the zero value, reserved as a defensive guard.
+	SessionStatusUnknown SessionStatus = iota
+	// SessionStatusLive means the RayCluster CR is still present and the
+	// session is intentionally skipped.
+	SessionStatusLive
+	// SessionStatusProcessed means events were ingested into EventHandler's
+	// in-memory state.
+	SessionStatusProcessed
+	// SessionStatusClusterStateUnknown means the cluster state could not be
+	// determined (e.g., transient API failure).
+	SessionStatusClusterStateUnknown
+	// SessionStatusEventsErr means event parsing failed.
+	SessionStatusEventsErr
+	// SessionStatusCanceled means ctx was canceled mid-processing.
+	SessionStatusCanceled
+)
+
+// SessionProcessor processes a single session end-to-end: dead detection and
+// event parsing. It is stateless across sessions and safe for concurrent use.
+type SessionProcessor struct {
+	reader    storage.StorageReader
+	k8sClient client.Client
+}
+
+// NewSessionProcessor constructs a SessionProcessor.
+func NewSessionProcessor(reader storage.StorageReader, k8sClient client.Client) *SessionProcessor {
+	return &SessionProcessor{
+		reader:    reader,
+		k8sClient: k8sClient,
+	}
+}
+
+// ProcessSession processes one session end-to-end.
+func (p *SessionProcessor) ProcessSession(ctx context.Context, session utils.ClusterInfo) (SessionStatus, *eventserver.SessionSnapshot, error) {
+	dead, err := p.isDead(ctx, session)
+	if err != nil {
+		if isCtxCanceled(err) {
+			return SessionStatusCanceled, nil, err
+		}
+		return SessionStatusClusterStateUnknown, nil, fmt.Errorf("check cluster state for %s/%s: %w", session.Namespace, session.Name, err)
+	}
+	if !dead {
+		return SessionStatusLive, nil, nil
+	}
+
+	// Use per-call EventHandler so ingestion maps become GC-eligible once the snapshot is built.
+	h := eventserver.NewEventHandler(p.reader)
+	if err := h.ProcessSingleSession(ctx, session); err != nil {
+		if isCtxCanceled(err) {
+			return SessionStatusCanceled, nil, err
+		}
+		return SessionStatusEventsErr, nil, fmt.Errorf("process events for %s/%s: %w", session.Namespace, session.Name, err)
+	}
+
+	return SessionStatusProcessed, h.BuildSnapshot(session), nil
+}
+
+// isDead determines if the RayCluster CR is absent.
+//
+// Known limit: An old session of a still-running RayCluster will be misclassified as live.
+func (p *SessionProcessor) isDead(ctx context.Context, session utils.ClusterInfo) (bool, error) {
+	rc := &rayv1.RayCluster{}
+	err := p.k8sClient.Get(ctx, k8stypes.NamespacedName{
+		Namespace: session.Namespace,
+		Name:      session.Name,
+	}, rc)
+	if apierrors.IsNotFound(err) {
+		return true, nil
+	}
+	if err != nil {
+		return false, err
+	}
+	return false, nil
+}
+
+// isCtxCanceled checks if the error is caused by context cancellation
+// or deadline exceeded.
+func isCtxCanceled(err error) bool {
+	return errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded)
+}

historyserver/pkg/historyserver/session_processor_test.go

@@ -0,0 +1,74 @@
+package historyserver
+
+import (
+	"context"
+	"testing"
+
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/runtime"
+	utilruntime "k8s.io/apimachinery/pkg/util/runtime"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+	"sigs.k8s.io/controller-runtime/pkg/client/fake"
+
+	rayv1 "github.com/ray-project/kuberay/ray-operator/apis/ray/v1"
+
+	"github.com/ray-project/kuberay/historyserver/pkg/utils"
+)
+
+func newFakeK8sClient(t *testing.T, rc *rayv1.RayCluster) client.Client {
+	t.Helper()
+	scheme := runtime.NewScheme()
+	utilruntime.Must(rayv1.AddToScheme(scheme))
+	b := fake.NewClientBuilder().WithScheme(scheme)
+	if rc != nil {
+		b = b.WithObjects(rc)
+	}
+	return b.Build()
+}
+
+func rayCluster(namespace, name string) *rayv1.RayCluster {
+	return &rayv1.RayCluster{
+		ObjectMeta: metav1.ObjectMeta{Namespace: namespace, Name: name},
+	}
+}
+
+func TestIsDead(t *testing.T) {
+	const (
+		ns             = "default"
+		name           = "raycluster-test"
+		queriedSession = "session_2026-04-22_10-00-00_000000_1"
+	)
+
+	tests := []struct {
+		name     string
+		cr       *rayv1.RayCluster // nil = RayCluster CR absent
+		wantDead bool
+	}{
+		{
+			name:     "RayCluster CR absent -> dead",
+			cr:       nil,
+			wantDead: true,
+		},
+		{
+			name:     "RayCluster CR present -> alive",
+			cr:       rayCluster(ns, name),
+			wantDead: false,
+		},
+	}
+
+	for _, tc := range tests {
+		t.Run(tc.name, func(t *testing.T) {
+			k := newFakeK8sClient(t, tc.cr)
+			p := NewSessionProcessor(nil, k)
+
+			info := utils.ClusterInfo{Namespace: ns, Name: name, SessionName: queriedSession}
+			gotDead, err := p.isDead(context.Background(), info)
+			if err != nil {
+				t.Fatalf("unexpected error: %v", err)
+			}
+			if gotDead != tc.wantDead {
+				t.Fatalf("isDead = %v, want %v", gotDead, tc.wantDead)
+			}
+		})
+	}
+}

historyserver/pkg/historyserver/timeline.go

@@ -0,0 +1,252 @@
+package historyserver
+
+import (
+	"encoding/json"
+	"strings"
+
+	"github.com/ray-project/kuberay/historyserver/pkg/eventserver"
+	eventtypes "github.com/ray-project/kuberay/historyserver/pkg/eventserver/types"
+)
+
+// getTasksTimeline returns timeline data in Chrome Tracing Format.
+// Output format matches Ray Dashboard's /api/v0/tasks/timeline endpoint.
+func getTasksTimeline(snap *eventserver.SessionSnapshot, jobID string) []eventtypes.ChromeTraceEvent {
+	var tasks []eventtypes.Task
+	if snap != nil {
+		if jobID != "" {
+			for _, task := range snap.Tasks {
+				if task.JobID == jobID {
+					tasks = append(tasks, task)
+				}
+			}
+		} else {
+			tasks = snap.Tasks
+		}
+	}
+
+	if len(tasks) == 0 {
+		return []eventtypes.ChromeTraceEvent{}
+	}
+
+	events := []eventtypes.ChromeTraceEvent{}
+
+	filteredTasks := make([]eventtypes.Task, 0, len(tasks))
+	for _, task := range tasks {
+		if task.ProfileData == nil || len(task.ProfileData.Events) == 0 {
+			continue
+		}
+		// Only include worker and driver components (consistent with Ray's profiling implementation in profiling.py)
+		componentType := task.ProfileData.ComponentType
+		if componentType != "worker" && componentType != "driver" {
+			continue
+		}
+		if task.ProfileData.NodeIPAddress == "" {
+			continue
+		}
+		filteredTasks = append(filteredTasks, task)
+	}
+
+	// Build PID/TID mappings
+	// PID: Node IP -> numeric ID
+	// TID: clusterID (componentType:componentId) -> globally unique numeric ID
+	nodeIPToPID := make(map[string]int)
+	nodeIPToClusterIDToTID := make(map[string]map[string]int) // nodeIP -> clusterID (componentType:componentId) -> tid
+	pidCounter := 0
+	tidCounter := 0
+
+	// First pass: collect all unique nodes and workers
+	for _, task := range filteredTasks {
+		nodeIP := task.ProfileData.NodeIPAddress
+		clusterID := task.ProfileData.ComponentType + ":" + task.ProfileData.ComponentID
+		if _, exists := nodeIPToPID[nodeIP]; !exists {
+			nodeIPToPID[nodeIP] = pidCounter
+			pidCounter++
+			nodeIPToClusterIDToTID[nodeIP] = make(map[string]int)
+		}
+		if _, exists := nodeIPToClusterIDToTID[nodeIP][clusterID]; !exists {
+			nodeIPToClusterIDToTID[nodeIP][clusterID] = tidCounter
+			tidCounter++
+		}
+	}
+
+	// Generate process_name and thread_name metadata events
+	for nodeIP, pid := range nodeIPToPID {
+		events = append(events, eventtypes.ChromeTraceEvent{
+			Name:  "process_name",
+			PID:   pid,
+			TID:   nil,
+			Phase: "M",
+			Args: map[string]interface{}{
+				"name": "Node " + nodeIP,
+			},
+		})
+
+		for clusterID, tid := range nodeIPToClusterIDToTID[nodeIP] {
+			tidVal := tid
+			events = append(events, eventtypes.ChromeTraceEvent{
+				Name:  "thread_name",
+				PID:   pid,
+				TID:   &tidVal,
+				Phase: "M",
+				Args: map[string]interface{}{
+					"name": clusterID,
+				},
+			})
+		}
+	}
+
+	// Generate trace events from ProfileData
+	for _, task := range filteredTasks {
+		nodeIP := task.ProfileData.NodeIPAddress
+		clusterID := task.ProfileData.ComponentType + ":" + task.ProfileData.ComponentID
+
+		pid, ok := nodeIPToPID[nodeIP]
+		if !ok {
+			continue
+		}
+		var tidPtr *int
+		if tid, ok := nodeIPToClusterIDToTID[nodeIP][clusterID]; ok {
+			tidVal := tid
+			tidPtr = &tidVal
+		} else {
+			// This shouldn't happen if first pass worked correctly,
+			// but skip to avoid null TID
+			continue
+		}
+
+		for _, profEvent := range task.ProfileData.Events {
+			// Convert nanoseconds to microseconds
+			startTimeUs := float64(profEvent.StartTime) / 1000.0
+			durationUs := float64(profEvent.EndTime-profEvent.StartTime) / 1000.0
+
+			// Parse extraData for additional fields
+			var extraData map[string]interface{}
+			if profEvent.ExtraData != "" {
+				json.Unmarshal([]byte(profEvent.ExtraData), &extraData)
+			}
+
+			// Determine task_id and func_or_class_name
+			taskIDForArgs := task.TaskID
+			funcOrClassName := task.FuncOrClassName
+
+			// Fallback to GetFuncName() if FuncOrClassName is empty
+			// This ensures consistency with Ray's profiling.py which uses task["func_or_class_name"]
+			// from TASK_DEFINITION_EVENT, and handles cases where TASK_PROFILE_EVENT is missing
+			if funcOrClassName == "" {
+				funcOrClassName = task.GetFuncName()
+			}
+
+			// Try to get from extraData if available (for hex format task_id)
+			if extraData != nil {
+				if tid, ok := extraData["task_id"].(string); ok && tid != "" {
+					taskIDForArgs = tid
+				}
+			}
+
+			// Build args
+			actorID := extractActorIDFromTaskID(taskIDForArgs)
+			args := map[string]interface{}{
+				"task_id":            taskIDForArgs,
+				"job_id":             task.JobID,
+				"attempt_number":     task.TaskAttempt,
+				"func_or_class_name": funcOrClassName,
+				"actor_id":           nil,
+			}
+
+			if actorID != "" {
+				args["actor_id"] = actorID
+			}
+
+			// Determine event name for display
+			eventName := profEvent.EventName
+			displayName := profEvent.EventName
+
+			// For overall task events like "task::slow_task", use the full name from extraData
+			if strings.HasPrefix(profEvent.EventName, "task::") && extraData != nil {
+				if name, ok := extraData["name"].(string); ok && name != "" {
+					displayName = name
+					args["name"] = name
+				}
+			}
+
+			traceEvent := eventtypes.ChromeTraceEvent{
+				Category:  eventName,
+				Name:      displayName,
+				PID:       pid,
+				TID:       tidPtr,
+				Timestamp: &startTimeUs,
+				Duration:  &durationUs,
+				Color:     getChromeTraceColor(eventName),
+				Args:      args,
+				Phase:     "X",
+			}
+
+			events = append(events, traceEvent)
+		}
+	}
+
+	return events
+}
+
+// getChromeTraceColor maps event names to Chrome trace colors
+// Based on Ray's _default_color_mapping in profiling.py
+func getChromeTraceColor(eventName string) string {
+	// Handle task::xxx pattern (overall task event)
+	if strings.HasPrefix(eventName, "task::") {
+		return "generic_work"
+	}
+
+	// Direct mapping for known event names
+	// This logic follows Ray's profiling implementation:
+	// https://github.com/ray-project/ray/blob/68d01c4c48a59c7768ec9c2359a1859966c446b6/python/ray/_private/profiling.py#L25
+	switch eventName {
+	case "task:deserialize_arguments":
+		return "rail_load"
+	case "task:execute":
+		return "rail_animation"
+	case "task:store_outputs":
+		return "rail_idle"
+	case "task:submit_task", "task":
+		return "rail_response"
+	case "worker_idle":
+		return "cq_build_abandoned"
+	case "ray.get":
+		return "good"
+	case "ray.put":
+		return "terrible"
+	case "ray.wait":
+		return "vsync_highlight_color"
+	case "submit_task":
+		return "background_memory_dump"
+	case "wait_for_function", "fetch_and_run_function", "register_remote_function":
+		return "detailed_memory_dump"
+	default:
+		return "generic_work"
+	}
+}
+
+// extractActorIDFromTaskID extracts the ActorID from a TaskID following Ray's ID specification.
+//
+// Design doc: src/ray/design_docs/id_specification.md
+// - TaskID: 8B unique + 16B ActorID (total 24 bytes = 48 hex chars)
+// - ActorID: 12B unique + 4B JobID (total 16 bytes = 32 hex chars)
+//
+// For a 48-character hex TaskID, the last 32 hex characters (bytes 16–48)
+// correspond to the ActorID. This function further checks the "unique" portion
+// of the ActorID (first 24 hex chars) and returns an empty string if it is all Fs,
+// which indicates normal/driver tasks with no associated actor.
+func extractActorIDFromTaskID(taskIDHex string) string {
+	if len(taskIDHex) != 48 {
+		return "" // can't process if encoded in base64
+	}
+
+	actorPortion := taskIDHex[16:40] // 24 chars for actor id (12 bytes)
+	jobPortion := taskIDHex[40:48]   // 8 chars for job id (4 bytes)
+
+	// Check if all Fs (no actor)
+	if strings.ToLower(actorPortion) == "ffffffffffffffffffffffff" {
+		return ""
+	}
+
+	return actorPortion + jobPortion
+}

historyserver/pkg/historyserver/types.go

@@ -30,7 +30,7 @@ type GetLogFileOptions struct {
 	Suffix string
 }
 
-// TODO(jwj): Can be extracted to a task-specific interface, e.g., TaskSummaryProvider.
+// TODO(jiangjiawei1103): Can be extracted to a task-specific interface, e.g., TaskSummaryProvider.
 type TaskDataResult struct {
 	Total                 int                      `json:"total"`
 	NumAfterTruncation    int                      `json:"num_after_truncation"`

historyserver/pkg/storage/aliyunoss/ray/ray.go

@@ -18,6 +18,7 @@ import (
 	"github.com/ray-project/kuberay/historyserver/pkg/collector/types"
 	"github.com/ray-project/kuberay/historyserver/pkg/storage"
 	"github.com/ray-project/kuberay/historyserver/pkg/storage/aliyunoss/rrsa"
+	"github.com/ray-project/kuberay/historyserver/pkg/storage/clustermetadata"
 	"github.com/ray-project/kuberay/historyserver/pkg/utils"
 )
 
@@ -147,10 +148,12 @@ func (r *RayLogsHandler) List() (res []utils.ClusterInfo) {
 	clusters := make(utils.ClusterInfoList, 0, 10)
 	logrus.Debugf("Prepare to get list clusters info ...")
 
+	prefix := clustermetadata.Prefix(r.OssRootDir)
+
 	getClusters := func() {
 		p := r.OssClient.NewListObjectsV2Paginator(&oss.ListObjectsV2Request{
 			Bucket:    oss.Ptr(r.OssBucket),
-			Prefix:    oss.Ptr(path.Join(r.OssRootDir, "metadir") + "/"),
+			Prefix:    oss.Ptr(prefix),
 			Delimiter: oss.Ptr(""),
 			MaxKeys:   100,
 		})
@@ -158,34 +161,18 @@ func (r *RayLogsHandler) List() (res []utils.ClusterInfo) {
 		for p.HasNext() {
 			page, err := p.NextPage(ctx)
 			if err != nil {
-				logrus.Errorf("Failed to list objects from %s: %v", path.Join(r.OssRootDir, "metadir")+"/", err)
+				logrus.Errorf("Failed to list objects from %s: %v", prefix, err)
 				return
 			}
-			logrus.Infof("[List]Returned objects in %v. length of Contents: %v, length of CommonPrefixes: %v", path.Join(r.OssRootDir, "metadir")+"/", len(page.Contents),
+			logrus.Infof("[List]Returned objects in %v. length of Contents: %v, length of CommonPrefixes: %v", prefix, len(page.Contents),
 				len(page.CommonPrefixes))
 			for _, objects := range page.Contents {
-				c := &utils.ClusterInfo{}
-				metaInfo := strings.Trim(strings.TrimPrefix(*objects.Key, path.Join(r.OssRootDir, "metadir/")), "/")
-				metas := strings.Split(metaInfo, "/")
-				if len(metas) < 2 {
-					continue
-				}
-				logrus.Infof("Process %++v", metas)
-				namespaceName := strings.Split(metas[0], "_")
-				c.Name = namespaceName[0]
-				c.Namespace = namespaceName[1]
-				c.SessionName = metas[1]
-				sessionInfo := strings.Split(metas[1], "_")
-				date := sessionInfo[1]
-				dataTime := sessionInfo[2]
-				createTime, err := time.Parse("2006-01-02_15-04-05", date+"_"+dataTime)
+				c, err := clustermetadata.DecodePath(*objects.Key, r.OssRootDir)
 				if err != nil {
-					logrus.Errorf("Failed to parse time %s: %v", date+"_"+dataTime, err)
+					logrus.Errorf("Failed to parse meta file path: %s, error: %v", *objects.Key, err)
 					continue
 				}
-				c.CreateTimeStamp = createTime.Unix()
-				c.CreateTime = createTime.UTC().Format(("2006-01-02T15:04:05Z"))
-				clusters = append(clusters, *c)
+				clusters = append(clusters, c)
 			}
 		}
 	}

historyserver/pkg/storage/azureblob/azureblob.go

@@ -20,6 +20,7 @@ import (
 
 	"github.com/ray-project/kuberay/historyserver/pkg/collector/types"
 	"github.com/ray-project/kuberay/historyserver/pkg/storage"
+	"github.com/ray-project/kuberay/historyserver/pkg/storage/clustermetadata"
 	"github.com/ray-project/kuberay/historyserver/pkg/utils"
 )
 
@@ -164,52 +165,30 @@ func (r *RayLogsHandler) List() (res []utils.ClusterInfo) {
 	ctx, cancel := context.WithTimeout(context.Background(), listTimeout)
 	defer cancel()
 
-	metadirPrefix := path.Join(r.RootDir, "metadir") + "/"
+	prefix := clustermetadata.Prefix(r.RootDir)
 
 	pager := r.ContainerClient.NewListBlobsFlatPager(&container.ListBlobsFlatOptions{
-		Prefix:     &metadirPrefix,
+		Prefix:     &prefix,
 		MaxResults: to32(100),
 	})
 
 	for pager.More() {
 		resp, err := pager.NextPage(ctx)
 		if err != nil {
-			logrus.Errorf("Failed to list blobs from %s: %v", metadirPrefix, err)
+			logrus.Errorf("Failed to list blobs from %s: %v", prefix, err)
 			break
 		}
 
 		logrus.Infof("[List]Returned blobs in %v. length of Segment.BlobItems: %v",
-			metadirPrefix, len(resp.Segment.BlobItems))
+			prefix, len(resp.Segment.BlobItems))
 
 		for _, blob := range resp.Segment.BlobItems {
-			c := &utils.ClusterInfo{}
-			metaInfo := strings.Trim(strings.TrimPrefix(*blob.Name, path.Join(r.RootDir, "metadir/")), "/")
-			metas := strings.Split(metaInfo, "/")
-			if len(metas) < 2 {
-				continue
-			}
-			logrus.Infof("Process %++v", metas)
-			namespaceName := strings.Split(metas[0], "_")
-			if len(namespaceName) < 2 {
-				continue
-			}
-			c.Name = namespaceName[0]
-			c.Namespace = namespaceName[1]
-			c.SessionName = metas[1]
-			sessionInfo := strings.Split(metas[1], "_")
-			if len(sessionInfo) < 3 {
-				continue
-			}
-			date := sessionInfo[1]
-			dataTime := sessionInfo[2]
-			createTime, err := time.Parse("2006-01-02_15-04-05", date+"_"+dataTime)
+			c, err := clustermetadata.DecodePath(*blob.Name, r.RootDir)
 			if err != nil {
-				logrus.Errorf("Failed to parse time %s: %v", date+"_"+dataTime, err)
+				logrus.Errorf("Failed to parse meta file path: %s, error: %v", *blob.Name, err)
 				continue
 			}
-			c.CreateTimeStamp = createTime.Unix()
-			c.CreateTime = createTime.UTC().Format("2006-01-02T15:04:05Z")
-			clusters = append(clusters, *c)
+			clusters = append(clusters, c)
 		}
 	}
 

historyserver/pkg/storage/clustermetadata/clustermetadata.go

@@ -0,0 +1,100 @@
+package clustermetadata
+
+import (
+	"fmt"
+	"path"
+	"strings"
+
+	"github.com/ray-project/kuberay/historyserver/pkg/utils"
+)
+
+const (
+	ClusterMetadataDir = "cluster-metadata"
+	// Without owner: cluster-metadata/raycluster/{namespace}_{cluster_name}/{session_id}
+	metaPartsCountWithoutOwner = 2
+	// With owner:    cluster-metadata/{rayjob|rayservice}/{namespace}_{owner_name}_{cluster_name}/{session_id}
+	metaPartsCountWithOwner    = 3
+)
+
+// EncodePath builds the hierarchical path for a given session ID.
+func EncodePath(info utils.ClusterInfo, rootDir, sessionID string) string {
+	prefix := Prefix(rootDir)
+	ownerKind := strings.ToLower(info.OwnerKind)
+	hasOwner := (ownerKind == utils.RayJobKind || ownerKind == utils.RayServiceKind) && info.OwnerName != ""
+
+	resourceSubDir := utils.RayClusterKind
+	if hasOwner {
+		resourceSubDir = ownerKind
+	}
+
+	var parts []string
+	parts = append(parts, info.Namespace)
+	if hasOwner {
+		parts = append(parts, info.OwnerName)
+	}
+	parts = append(parts, info.Name)
+	p := path.Join(prefix, resourceSubDir, strings.Join(parts, utils.Connector))
+	return path.Clean(path.Join(p, path.Base(sessionID)))
+}
+
+// DecodePath parses the hierarchical path and returns the cluster info.
+func DecodePath(filePath string, rootDir string) (utils.ClusterInfo, error) {
+	prefix := Prefix(rootDir)
+	cleanFilePath := strings.Trim(filePath, "/")
+	cleanPrefix := strings.Trim(prefix, "/") + "/"
+	if !strings.HasPrefix(cleanFilePath, cleanPrefix) {
+		return utils.ClusterInfo{}, fmt.Errorf("invalid path %q: expected prefix %q", filePath, prefix)
+	}
+	relativePath := strings.Trim(strings.TrimPrefix(cleanFilePath, cleanPrefix), "/")
+
+	pathSegments := strings.Split(relativePath, "/")
+
+	if len(pathSegments) != 3 {
+		return utils.ClusterInfo{}, fmt.Errorf("invalid path segment count for cluster metadata structure: %s", filePath)
+	}
+
+	resourceSubDir := strings.ToLower(pathSegments[0])
+	if resourceSubDir != utils.RayClusterKind && resourceSubDir != utils.RayJobKind && resourceSubDir != utils.RayServiceKind {
+		return utils.ClusterInfo{}, fmt.Errorf("unsupported cluster metadata subdirectory %q in path %q", resourceSubDir, filePath)
+	}
+
+	c := utils.ClusterInfo{
+		SessionName: pathSegments[2],
+	}
+
+	metaParts := strings.Split(pathSegments[1], utils.Connector)
+	c.Namespace = metaParts[0]
+	switch len(metaParts) {
+	case metaPartsCountWithoutOwner:
+		if resourceSubDir != utils.RayClusterKind {
+			return utils.ClusterInfo{}, fmt.Errorf("mismatched subdirectory %q for metadata structure without owner in path %q", resourceSubDir, filePath)
+		}
+		c.Name = metaParts[1]
+	case metaPartsCountWithOwner:
+		if resourceSubDir != utils.RayJobKind && resourceSubDir != utils.RayServiceKind {
+			return utils.ClusterInfo{}, fmt.Errorf("unsupported subdirectory %q with owner metadata in path %q", resourceSubDir, filePath)
+		}
+		c.OwnerKind = resourceSubDir
+		c.OwnerName = metaParts[1]
+		c.Name = metaParts[2]
+	default:
+		return utils.ClusterInfo{}, fmt.Errorf("invalid cluster-metadata segment count %d in path %q", len(metaParts), filePath)
+	}
+
+	createTime, err := utils.GetDateTimeFromSessionID(c.SessionName)
+	if err != nil {
+		return utils.ClusterInfo{}, fmt.Errorf("parse session time from %s: %w", c.SessionName, err)
+	}
+	c.CreateTimeStamp = createTime.Unix()
+	c.CreateTime = createTime.UTC().Format("2006-01-02T15:04:05Z")
+
+	return c, nil
+}
+
+// Prefix returns the root directory prefix.
+func Prefix(rootDir string) string {
+	if rootDir == "" {
+		return ClusterMetadataDir + "/"
+	}
+	return strings.TrimRight(path.Join(rootDir, ClusterMetadataDir), "/") + "/"
+}

historyserver/pkg/storage/clustermetadata/clustermetadata_test.go

@@ -0,0 +1,227 @@
+package clustermetadata
+
+import (
+	"testing"
+	"time"
+
+	"github.com/ray-project/kuberay/historyserver/pkg/utils"
+)
+
+func TestDecodePath(t *testing.T) {
+	tests := []struct {
+		name        string
+		filePath    string
+		rootDir     string
+		expectErr   bool
+		expected    utils.ClusterInfo
+	}{
+		{
+			name:        "valid hierarchical path job",
+			filePath:    "cluster-metadata/rayjob/defaultns_myrayjob_mycluster3/session_2024-05-15_10-30-55_123456",
+			expectErr:   false,
+			expected: utils.ClusterInfo{
+				Namespace:       "defaultns",
+				OwnerKind:       "rayjob",
+				OwnerName:       "myrayjob",
+				Name:            "mycluster3",
+				SessionName:     "session_2024-05-15_10-30-55_123456",
+				CreateTimeStamp: time.Date(2024, time.May, 15, 10, 30, 55, 123456000, time.UTC).Unix(),
+				CreateTime:      "2024-05-15T10:30:55Z",
+			},
+		},
+		{
+			name:        "valid hierarchical path standalone",
+			filePath:    "cluster-metadata/raycluster/defaultns_mycluster1/session_2024-05-15_10-30-55_123456",
+			expectErr:   false,
+			expected: utils.ClusterInfo{
+				Namespace:       "defaultns",
+				Name:            "mycluster1",
+				SessionName:     "session_2024-05-15_10-30-55_123456",
+				CreateTimeStamp: time.Date(2024, time.May, 15, 10, 30, 55, 123456000, time.UTC).Unix(),
+				CreateTime:      "2024-05-15T10:30:55Z",
+			},
+		},
+		{
+			name:        "invalid path prefix",
+			filePath:    "wrong-prefix/rayjob/defaultns_myrayjob_mycluster3/session_2024-05-15_10-30-55_123456",
+			expectErr:   true,
+		},
+		{
+			name:        "invalid path prefix with custom rootDir",
+			filePath:    "cluster-metadata/rayjob/defaultns_myrayjob_mycluster3/session_2024-05-15_10-30-55_123456",
+			rootDir:     "custom-root",
+			expectErr:   true,
+		},
+		{
+			name:        "invalid path structure - missing session",
+			filePath:    "cluster-metadata/rayjob/defaultns_myrayjob_mycluster3",
+			expectErr:   true,
+		},
+		{
+			name:        "invalid path structure - too many parts",
+			filePath:    "cluster-metadata/rayjob/defaultns_myrayjob_mycluster3/session/extra",
+			expectErr:   true,
+		},
+		{
+			name:        "invalid metadir segment - only one part",
+			filePath:    "cluster-metadata/raycluster/defaultns/session_2024-05-15_10-30-55_123456",
+			expectErr:   true,
+		},
+		{
+			name:        "invalid metadir segment with owner - too few parts",
+			filePath:    "cluster-metadata/rayjob/defaultns/session_2024-05-15_10-30-55_123456",
+			expectErr:   true,
+		},
+		{
+			name:        "session id with bad timestamp pattern",
+			filePath:    "cluster-metadata/raycluster/defaultns_mycluster/session_not-a-real-timestamp",
+			expectErr:   true,
+		},
+		{
+			name:      "invalid decode - raycluster with 3 meta parts",
+			filePath:  "cluster-metadata/raycluster/defaultns_myservice_mycluster3/session_2024-05-15_10-30-55_123456",
+			expectErr: true,
+		},
+		{
+			name:        "valid owner kind - mixed case",
+			filePath:    "cluster-metadata/RaYSerVice/defaultns_myservice_mycluster3/session_2024-05-15_10-30-55_123456",
+			expectErr:   false,
+			expected: utils.ClusterInfo{
+				Namespace:       "defaultns",
+				OwnerKind:       "rayservice",
+				OwnerName:       "myservice",
+				Name:            "mycluster3",
+				SessionName:     "session_2024-05-15_10-30-55_123456",
+				CreateTimeStamp: time.Date(2024, time.May, 15, 10, 30, 55, 123456000, time.UTC).Unix(),
+				CreateTime:      "2024-05-15T10:30:55Z",
+			},
+		},
+		{
+			name:      "unsupported owner kind",
+			filePath:  "cluster-metadata/wrongowner/defaultns_myservice_name/session_2024-05-15_10-30-55_123456",
+			expectErr: true,
+		},
+	}
+
+	for _, tc := range tests {
+		t.Run(tc.name, func(t *testing.T) {
+			c, err := DecodePath(tc.filePath, tc.rootDir)
+
+			if tc.expectErr {
+				if err == nil {
+					t.Errorf("DecodePath(%q) succeeded unexpectedly", tc.filePath)
+				}
+			} else {
+				if err != nil {
+					t.Fatalf("DecodePath(%q) failed unexpectedly: %v", tc.filePath, err)
+				}
+				if c.Namespace != tc.expected.Namespace {
+					t.Errorf("Namespace = %q, want %q", c.Namespace, tc.expected.Namespace)
+				}
+				if c.Name != tc.expected.Name {
+					t.Errorf("Name = %q, want %q", c.Name, tc.expected.Name)
+				}
+				if c.SessionName != tc.expected.SessionName {
+					t.Errorf("SessionName = %q, want %q", c.SessionName, tc.expected.SessionName)
+				}
+				if c.OwnerKind != tc.expected.OwnerKind {
+					t.Errorf("OwnerKind = %q, want %q", c.OwnerKind, tc.expected.OwnerKind)
+				}
+				if c.OwnerName != tc.expected.OwnerName {
+					t.Errorf("OwnerName = %q, want %q", c.OwnerName, tc.expected.OwnerName)
+				}
+				if c.CreateTimeStamp != tc.expected.CreateTimeStamp {
+					t.Errorf("CreateTimeStamp = %d, want %d", c.CreateTimeStamp, tc.expected.CreateTimeStamp)
+				}
+				if c.CreateTime != tc.expected.CreateTime {
+					t.Errorf("CreateTime = %q, want %q", c.CreateTime, tc.expected.CreateTime)
+				}
+			}
+		})
+	}
+}
+
+func TestMetadirPathRoundTrip(t *testing.T) {
+	const sessionID = "session_2026-05-15_10-30-55_123456"
+	sessionTime, _ := utils.GetDateTimeFromSessionID(sessionID)
+
+	cases := []utils.ClusterInfo{
+		{Namespace: "default", Name: "raycluster1"},
+		{Namespace: "testns", Name: "raycluster2"},
+		{OwnerKind: "rayjob", OwnerName: "rayjob1", Namespace: "default", Name: "raycluster3"},
+		{OwnerKind: "rayservice", OwnerName: "rayservice1", Namespace: "testns", Name: "raycluster3"},
+	}
+
+	rootDirs := []string{"", "rootdir"}
+	for _, rootDir := range rootDirs {
+		for _, in := range cases {
+			t.Run(rootDir+"/"+in.Namespace+"/"+in.Name, func(t *testing.T) {
+				fullKey := EncodePath(in, rootDir, sessionID)
+
+				got, err := DecodePath(fullKey, rootDir)
+				if err != nil {
+					t.Fatalf("DecodePath(%q) failed: %v", fullKey, err)
+				}
+
+				if got.Namespace != in.Namespace || got.Name != in.Name ||
+					got.OwnerKind != in.OwnerKind || got.OwnerName != in.OwnerName {
+					t.Errorf("Round-trip structure mismatch:\n  in : %+v\n  got: %+v", in, got)
+				}
+				if got.SessionName != sessionID {
+					t.Errorf("SessionName = %q, want %q", got.SessionName, sessionID)
+				}
+				if got.CreateTimeStamp != sessionTime.Unix() {
+					t.Errorf("CreateTimeStamp = %d, want %d", got.CreateTimeStamp, sessionTime.Unix())
+				}
+			})
+		}
+	}
+}
+
+func TestEncodePath(t *testing.T) {
+	tests := []struct {
+		name      string
+		info      utils.ClusterInfo
+		rootDir   string
+		sessionID string
+		want      string
+	}{
+		{
+			name:      "standalone without owner kind",
+			info:      utils.ClusterInfo{Namespace: "ns", Name: "cluster"},
+			rootDir:   "myroot",
+			sessionID: "session-123",
+			want:      "myroot/cluster-metadata/raycluster/ns_cluster/session-123",
+		},
+		{
+			name:      "job with owner kind details",
+			info:      utils.ClusterInfo{OwnerKind: "rayjob", OwnerName: "job1", Namespace: "ns", Name: "cluster"},
+			rootDir:   "",
+			sessionID: "session-456",
+			want:      "cluster-metadata/rayjob/ns_job1_cluster/session-456",
+		},
+		{
+			name:      "inconsistent owner details (OwnerKind set but OwnerName empty)",
+			info:      utils.ClusterInfo{OwnerKind: "rayjob", OwnerName: "", Namespace: "ns", Name: "cluster"},
+			rootDir:   "",
+			sessionID: "session-789",
+			want:      "cluster-metadata/raycluster/ns_cluster/session-789",
+		},
+		{
+			name:      "job with owner kind details in inconsistent case",
+			info:      utils.ClusterInfo{OwnerKind: "RaYjOb", OwnerName: "job1", Namespace: "ns", Name: "cluster"},
+			rootDir:   "",
+			sessionID: "session-456",
+			want:      "cluster-metadata/rayjob/ns_job1_cluster/session-456",
+		},
+	}
+
+	for _, tc := range tests {
+		t.Run(tc.name, func(t *testing.T) {
+			got := EncodePath(tc.info, tc.rootDir, tc.sessionID)
+			if got != tc.want {
+				t.Errorf("EncodePath() = %q, want %q", got, tc.want)
+			}
+		})
+	}
+}

historyserver/pkg/storage/gcs/gcs_handler.go

@@ -15,6 +15,7 @@ import (
 	gstorage "cloud.google.com/go/storage"
 	"github.com/ray-project/kuberay/historyserver/pkg/collector/types"
 	"github.com/ray-project/kuberay/historyserver/pkg/storage"
+	"github.com/ray-project/kuberay/historyserver/pkg/storage/clustermetadata"
 	"github.com/ray-project/kuberay/historyserver/pkg/utils"
 	"github.com/sirupsen/logrus"
 	gIterator "google.golang.org/api/iterator"
@@ -140,14 +141,14 @@ func (h *RayLogsHandler) List() []utils.ClusterInfo {
 
 	clusterList := make(utils.ClusterInfoList, 0, 20)
 	bucket := h.StorageClient.Bucket(h.GCSBucket)
-	pathPrefix := strings.TrimPrefix(path.Join(h.RootDir, "metadir"), "/") + "/"
+	prefix := clustermetadata.Prefix(h.RootDir)
+	pathPrefix := strings.TrimPrefix(prefix, "/")
 	query := &gstorage.Query{
 		// Match with only non-directory objects
 		MatchGlob: pathPrefix + "**/*[!/]",
 	}
 	objectIterator := bucket.Objects(ctx, query)
 	for {
-		cluster := &utils.ClusterInfo{}
 		objectAttr, err := objectIterator.Next()
 		if err == gIterator.Done {
 			logrus.Infof("Finished iterating through gcs objects")
@@ -158,31 +159,14 @@ func (h *RayLogsHandler) List() []utils.ClusterInfo {
 			return nil
 		}
 
-		fullObjectPath := objectAttr.Name
-		metaInfo := strings.Split(strings.TrimPrefix(fullObjectPath, pathPrefix), "/")
-		if len(metaInfo) != 2 {
-			logrus.Errorf("Unable to properly parse cluster metadir path with fullpath: %s", fullObjectPath)
-			continue
-		}
-		clusterMeta := strings.Split(metaInfo[0], "_")
-		if len(clusterMeta) != 2 {
-			logrus.Errorf("Unable to get cluster name and namespace from directory: %s", metaInfo[0])
-			continue
-		}
-		cluster.Name = clusterMeta[0]
-		cluster.Namespace = clusterMeta[1]
-
-		cluster.SessionName = metaInfo[1]
-		datetime, err := utils.GetDateTimeFromSessionID(metaInfo[1])
+		c, err := clustermetadata.DecodePath(objectAttr.Name, h.RootDir)
 		if err != nil {
-			logrus.Errorf("Failed to get date time from the given sessionID: %s, error: %v", metaInfo[1], err)
+			logrus.Errorf("Failed to parse meta file path: %s, error: %v", objectAttr.Name, err)
 			continue
 		}
-		cluster.CreateTimeStamp = datetime.Unix()
-		cluster.CreateTime = datetime.UTC().Format(("2006-01-02T15:04:05Z"))
 
-		logrus.Infof("Parsed cluster %s for session %s to list", cluster.Name, cluster.SessionName)
-		clusterList = append(clusterList, *cluster)
+		logrus.Infof("Parsed cluster %s for session %s to list", c.Name, c.SessionName)
+		clusterList = append(clusterList, c)
 	}
 
 	sort.Sort(clusterList)

historyserver/pkg/storage/gcs/gcs_handler_test.go

@@ -215,7 +215,7 @@ func TestListFiles(t *testing.T) {
 
 func TestList(t *testing.T) {
 	// RootDir is "ray_historyserver"
-	// Path format: {RootDir}/metadir/{ClusterName}_{Namespace}/{SessionName}
+	// Path format: {RootDir}/cluster-metadata/{ClusterKey}/{SessionName}
 	ts := time.Now().UTC()
 	sessionID := "session_" + ts.Format("2006-01-02_15-04-05_000000")
 
@@ -223,14 +223,28 @@ func TestList(t *testing.T) {
 		{
 			ObjectAttrs: fakestorage.ObjectAttrs{
 				BucketName: "test-bucket",
-				Name:       "ray_historyserver/metadir/mycluster1_default/" + sessionID,
+				Name:       "ray_historyserver/cluster-metadata/raycluster/defaultns_mycluster1/" + sessionID,
 			},
 			Content: []byte(""),
 		},
 		{
 			ObjectAttrs: fakestorage.ObjectAttrs{
 				BucketName: "test-bucket",
-				Name:       "ray_historyserver/metadir/mycluster2_testns/" + sessionID,
+				Name:       "ray_historyserver/cluster-metadata/raycluster/testns_mycluster2/" + sessionID,
+			},
+			Content: []byte(""),
+		},
+		{
+			ObjectAttrs: fakestorage.ObjectAttrs{
+				BucketName: "test-bucket",
+				Name:       "ray_historyserver/cluster-metadata/rayjob/defaultns_myrayjob_mycluster3/" + sessionID,
+			},
+			Content: []byte(""),
+		},
+		{
+			ObjectAttrs: fakestorage.ObjectAttrs{
+				BucketName: "test-bucket",
+				Name:       "ray_historyserver/cluster-metadata/rayservice/defaultns_myraysvc_mycluster4/" + sessionID,
 			},
 			Content: []byte(""),
 		},
@@ -239,8 +253,10 @@ func TestList(t *testing.T) {
 	handler := createRayLogsHandler(client, bucketName)
 
 	expected := []utils.ClusterInfo{
-		{Name: "mycluster1", Namespace: "default", SessionName: sessionID, CreateTimeStamp: ts.Unix(), CreateTime: ts.UTC().Format("2006-01-02T15:04:05Z")},
+		{Name: "mycluster1", Namespace: "defaultns", SessionName: sessionID, CreateTimeStamp: ts.Unix(), CreateTime: ts.UTC().Format("2006-01-02T15:04:05Z")},
 		{Name: "mycluster2", Namespace: "testns", SessionName: sessionID, CreateTimeStamp: ts.Unix(), CreateTime: ts.UTC().Format("2006-01-02T15:04:05Z")},
+		{Name: "mycluster3", OwnerKind: "rayjob", OwnerName: "myrayjob", Namespace: "defaultns", SessionName: sessionID, CreateTimeStamp: ts.Unix(), CreateTime: ts.UTC().Format("2006-01-02T15:04:05Z")},
+		{Name: "mycluster4", OwnerKind: "rayservice", OwnerName: "myraysvc", Namespace: "defaultns", SessionName: sessionID, CreateTimeStamp: ts.Unix(), CreateTime: ts.UTC().Format("2006-01-02T15:04:05Z")},
 	}
 
 	result := handler.List()

historyserver/pkg/storage/localtest/reader.go

@@ -20,13 +20,13 @@ func NewMockReader() *MockReader {
 	clusters := []utils.ClusterInfo{
 		{
 			Name:            "cluster-1",
-			SessionName:     "session-1",
+			SessionName:     "session_2023-01-01_00-00-00_000000",
 			CreateTime:      "2023-01-01T00:00:00Z",
 			CreateTimeStamp: 1672531200000,
 		},
 		{
 			Name:            "cluster-2",
-			SessionName:     "session-2",
+			SessionName:     "session_2023-01-02_00-00-00_000000",
 			CreateTime:      "2023-01-02T00:00:00Z",
 			CreateTimeStamp: 1672617600000,
 		},
@@ -35,11 +35,11 @@ func NewMockReader() *MockReader {
 	data := map[string]map[string]string{
 		"cluster-1": {
 			"log.txt":       "This is log content for cluster-1\nMultiple lines\nof log content",
-			"metadata.json": "{\n  \"name\": \"cluster-1\",\n  \"sessionName\": \"session-1\",\n  \"createTime\": \"2023-01-01T00:00:00Z\"\n}",
+			"metadata.json": "{\n  \"name\": \"cluster-1\",\n  \"sessionName\": \"session_2023-01-01_00-00-00_000000\",\n  \"createTime\": \"2023-01-01T00:00:00Z\"\n}",
 		},
 		"cluster-2": {
 			"log.txt":       "This is log content for cluster-2\nMultiple lines\nof log content",
-			"metadata.json": "{\n  \"name\": \"cluster-2\",\n  \"sessionName\": \"session-2\",\n  \"createTime\": \"2023-01-02T00:00:00Z\"\n}",
+			"metadata.json": "{\n  \"name\": \"cluster-2\",\n  \"sessionName\": \"session_2023-01-02_00-00-00_000000\",\n  \"createTime\": \"2023-01-02T00:00:00Z\"\n}",
 		},
 	}
 

historyserver/pkg/storage/s3/s3.go

@@ -36,6 +36,7 @@ import (
 
 	"github.com/ray-project/kuberay/historyserver/pkg/collector/types"
 	"github.com/ray-project/kuberay/historyserver/pkg/storage"
+	"github.com/ray-project/kuberay/historyserver/pkg/storage/clustermetadata"
 	"github.com/ray-project/kuberay/historyserver/pkg/utils"
 )
 
@@ -152,10 +153,12 @@ func (r *RayLogsHandler) List() (res []utils.ClusterInfo) {
 	clusters := make(utils.ClusterInfoList, 0, 10)
 	logrus.Debugf("Prepare to get list clusters info ...")
 
+	prefix := clustermetadata.Prefix(r.S3RootDir)
+
 	getClusters := func() {
 		listInput := &s3.ListObjectsV2Input{
 			Bucket:    aws.String(r.S3Bucket),
-			Prefix:    aws.String(path.Join(r.S3RootDir, "metadir") + "/"),
+			Prefix:    aws.String(prefix),
 			MaxKeys:   aws.Int64(100),
 			Delimiter: aws.String(""),
 		}
@@ -163,39 +166,21 @@ func (r *RayLogsHandler) List() (res []utils.ClusterInfo) {
 		err := r.S3Client.ListObjectsV2Pages(listInput,
 			func(page *s3.ListObjectsV2Output, lastPage bool) bool {
 				logrus.Infof("[List]Returned objects in %v. length of page.Contents: %v, length of page.CommonPrefixes: %v",
-					path.Join(r.S3RootDir, "metadir")+"/", len(page.Contents), len(page.CommonPrefixes))
+					prefix, len(page.Contents), len(page.CommonPrefixes))
 
 				for _, object := range page.Contents {
-					c := &utils.ClusterInfo{}
-					metaInfo := strings.Trim(strings.TrimPrefix(*object.Key, path.Join(r.S3RootDir, "metadir/")), "/")
-					metas := strings.Split(metaInfo, "/")
-					if len(metas) < 2 {
-						continue
-					}
-					logrus.Infof("Process %++v", metas)
-					namespaceName := strings.Split(metas[0], "_")
-					if len(namespaceName) < 2 {
-						continue
-					}
-					c.Name = namespaceName[0]
-					c.Namespace = namespaceName[1]
-					c.SessionName = metas[1]
-					sessionInfo := strings.Split(metas[1], "_")
-					date := sessionInfo[1]
-					dataTime := sessionInfo[2]
-					createTime, err := time.Parse("2006-01-02_15-04-05", date+"_"+dataTime)
+					c, err := clustermetadata.DecodePath(*object.Key, r.S3RootDir)
 					if err != nil {
-						logrus.Errorf("Failed to parse time %s: %v", date+"_"+dataTime, err)
+						logrus.Errorf("Failed to parse meta file path: %s, error: %v", *object.Key, err)
 						continue
 					}
-					c.CreateTimeStamp = createTime.Unix()
-					c.CreateTime = createTime.UTC().Format(("2006-01-02T15:04:05Z"))
-					clusters = append(clusters, *c)
+					logrus.Infof("Parsed cluster %s for session %s to list", c.Name, c.SessionName)
+					clusters = append(clusters, c)
 				}
 				return true
 			})
 		if err != nil {
-			logrus.Errorf("Failed to list objects from %s: %v", path.Join(r.S3RootDir, "metadir")+"/", err)
+			logrus.Errorf("Failed to list objects from %s: %v", prefix, err)
 			return
 		}
 	}

historyserver/pkg/utils/constant.go

@@ -5,7 +5,15 @@ import (
 	"path/filepath"
 )
 
-const defaultTmpRayRoot = "/tmp/ray"
+const (
+	defaultTmpRayRoot = "/tmp/ray"
+
+	// Lowercase, normalized CRD kinds. Used both as the comparison key
+	// for ToLower(OwnerKind) and as the cluster-metadata path subdir segment.
+	RayJobKind     = "rayjob"
+	RayServiceKind = "rayservice"
+	RayClusterKind = "raycluster"
+)
 
 func GetTmpRayRoot() string {
 	if tmpRoot := os.Getenv("RAY_TMP_ROOT"); tmpRoot != "" {

historyserver/pkg/utils/filter.go

@@ -120,7 +120,7 @@ func getFiltersFromReq(req *restful.Request) ([]Filter, error) {
 
 	filters := make([]Filter, len(filterKeys))
 	for i := range filterKeys {
-		// TODO(jwj): Add error handling for invalid filter keys based on filterable fields.
+		// TODO(jiangjiawei1103): Add error handling for invalid filter keys based on filterable fields.
 		predicate, err := parsePredicate(string(filterPredicates[i]))
 		if err != nil {
 			return nil, fmt.Errorf("invalid predicate: %w", err)
@@ -206,7 +206,7 @@ func filterTasks(tasks []eventtypes.Task, filter Filter) []eventtypes.Task {
 	return filteredTasks
 }
 
-// TODO(jwj): ApplyFilters and helpers like sortByIdAndAttempt and limitByLimit should be shared among different objects, e.g., actors, nodes.
+// TODO(jiangjiawei1103): ApplyFilters and helpers like sortByIdAndAttempt and limitByLimit should be shared among different objects, e.g., actors, nodes.
 // The following functions are for compatibility for other endpoints other than tasks.
 type PredicateFunc func(fieldValue, filterValue string) bool
 

historyserver/pkg/utils/types.go

@@ -6,6 +6,13 @@ type ClusterInfo struct {
 	SessionName     string `json:"sessionName"`
 	CreateTime      string `json:"createTime"`
 	CreateTimeStamp int64  `json:"createTimeStamp"`
+	OwnerKind       string `json:"ownerKind,omitempty"`
+	OwnerName       string `json:"ownerName,omitempty"`
+}
+
+type ClusterKey struct {
+	Namespace string
+	Name      string
 }
 
 type ClusterInfoList []ClusterInfo

historyserver/pkg/utils/utils.go

@@ -66,11 +66,11 @@ const (
 	// - Namespace CANNOT contain "_", so we can unambiguously split from the LAST "_"
 	//
 	// DO NOT CHANGE: Would break existing stored data paths
-	connector = "_"
+	Connector = "_"
 )
 
 func AppendRayClusterNameNamespace(rayClusterName, rayClusterNamespace string) string {
-	return fmt.Sprintf("%s%s%s", rayClusterName, connector, rayClusterNamespace)
+	return fmt.Sprintf("%s%s%s", rayClusterName, Connector, rayClusterNamespace)
 }
 
 func GetSessionDir() (string, error) {
@@ -152,7 +152,7 @@ func IsHexNil(hexStr string) (bool, error) {
 // Format: "{clusterName}_{namespace}_{sessionName}"
 // Example: "raycluster-historyserver_default_session_2026-01-11_19-38-40"
 func BuildClusterSessionKey(clusterName, namespace, sessionName string) string {
-	return clusterName + connector + namespace + connector + sessionName
+	return clusterName + Connector + namespace + Connector + sessionName
 }
 
 // GetDateTimeFromSessionID will convert sessionID string i.e. `session_2026-01-27_10-52-59_373533_1` to time.Time

historyserver/test/e2e/azureblob_collector_test.go

@@ -1,9 +1,11 @@
 package e2e
 
 import (
+	"compress/gzip"
 	"context"
 	"encoding/json"
 	"fmt"
+	"io"
 	"path/filepath"
 	"strings"
 	"testing"
@@ -293,7 +295,22 @@ func loadRayEventsFromAzureBlob(containerClient *container.Client, prefix string
 			}
 
 			var fileEvents []rayEvent
-			decodeErr := json.NewDecoder(downloadResp.Body).Decode(&fileEvents)
+			var reader io.Reader = downloadResp.Body
+			var gzReader *gzip.Reader
+			if strings.HasSuffix(*blob.Name, ".gz") {
+				var err error
+				gzReader, err = gzip.NewReader(downloadResp.Body)
+				if err != nil {
+					downloadResp.Body.Close()
+					return nil, fmt.Errorf("failed to create gzip reader for %s: %w", *blob.Name, err)
+				}
+				reader = gzReader
+			}
+
+			decodeErr := json.NewDecoder(reader).Decode(&fileEvents)
+			if gzReader != nil {
+				gzReader.Close()
+			}
 			downloadResp.Body.Close()
 
 			if decodeErr != nil {

historyserver/test/e2e/collector_test.go

@@ -1,6 +1,7 @@
 package e2e
 
 import (
+	"compress/gzip"
 	"encoding/json"
 	"fmt"
 	"io"
@@ -593,12 +594,28 @@ func loadRayEventsFromS3(s3Client *s3.S3, bucket string, prefix string) ([]rayEv
 		}
 
 		var fileEvents []rayEvent
-		if err := json.NewDecoder(content.Body).Decode(&fileEvents); err != nil {
-			content.Body.Close()
-			return nil, fmt.Errorf("failed to decode Ray events from %s: %w", fileKey, err)
+		var reader io.Reader = content.Body
+		var gzReader *gzip.Reader
+		if strings.HasSuffix(fileKey, ".gz") {
+			var err error
+			gzReader, err = gzip.NewReader(content.Body)
+			if err != nil {
+				content.Body.Close()
+				return nil, fmt.Errorf("failed to create gzip reader for %s: %w", fileKey, err)
+			}
+			reader = gzReader
+		}
+
+		decodeErr := json.NewDecoder(reader).Decode(&fileEvents)
+		if gzReader != nil {
+			gzReader.Close()
 		}
 		content.Body.Close()
 
+		if decodeErr != nil {
+			return nil, fmt.Errorf("failed to decode Ray events from %s: %w", fileKey, decodeErr)
+		}
+
 		events = append(events, fileEvents...)
 	}
 

historyserver/test/e2e/historyserver_test.go

@@ -2700,7 +2700,7 @@ func verifySingleEndpoint(test Test, g *WithT, client *http.Client, endpointURL
 	verifySchema(test, g, respData)
 }
 
-// TODO(jwj): Make verification for node-related endpoints more robust.
+// TODO(jiangjiawei1103): Make verification for node-related endpoints more robust.
 // verifyNodesRespSchema verifies that the /nodes response is valid according to the API schema.
 // Both live and dead clusters now return the same format (flat array of latest snapshots).
 // isLive is kept for signature compatibility but no longer affects validation.

ray-operator/Makefile

@@ -81,8 +81,12 @@ test-e2e-autoscaler: manifests fmt vet ## Run e2e autoscaler tests.
 	go test -timeout 30m -v $(WHAT)
 
 test-e2e-rayservice: WHAT ?= ./test/e2erayservice
-test-e2e-rayservice: manifests fmt vet ## Run e2e RayService tests.
-	go test -timeout 30m -v $(WHAT)
+test-e2e-rayservice: manifests fmt vet ## Run e2e RayService tests (excluding suspend tests, which have their own target).
+	go test -timeout 30m -v -skip Suspend $(WHAT)
+
+test-e2e-rayservice-suspend: WHAT ?= ./test/e2erayservice
+test-e2e-rayservice-suspend: manifests fmt vet ## Run e2e RayService suspend tests.
+	go test -timeout 30m -v -run Suspend $(WHAT)
 
 test-e2e-upgrade: WHAT ?= ./test/e2eupgrade
 test-e2e-upgrade: manifests fmt vet ## Run e2e operator upgrade tests.
@@ -205,7 +209,7 @@ kustomize: $(KUSTOMIZE) ## Download kustomize locally if necessary.
 ENVTEST = $(LOCALBIN)/setup-envtest
 $(ENVTEST): $(LOCALBIN)
 envtest: $(ENVTEST) ## Download envtest locally if necessary.
-	test -s $(ENVTEST) || GOBIN=$(LOCALBIN) go install sigs.k8s.io/controller-runtime/tools/setup-envtest@release-0.23
+	test -s $(ENVTEST) || GOBIN=$(LOCALBIN) go install sigs.k8s.io/controller-runtime/tools/setup-envtest@release-0.24
 
 GOFUMPT = $(LOCALBIN)/gofumpt
 $(GOFUMPT): $(LOCALBIN)

ray-operator/apis/config/v1alpha1/groupversion_info.go

@@ -4,8 +4,9 @@
 package v1alpha1
 
 import (
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/apimachinery/pkg/runtime/schema"
-	"sigs.k8s.io/controller-runtime/pkg/scheme"
 )
 
 var (
@@ -13,8 +14,16 @@ var (
 	GroupVersion = schema.GroupVersion{Group: "config.ray.io", Version: "v1alpha1"}
 
 	// SchemeBuilder is used to add go types to the GroupVersionKind scheme
-	SchemeBuilder = &scheme.Builder{GroupVersion: GroupVersion}
+	SchemeBuilder = runtime.NewSchemeBuilder(addKnownTypes)
 
 	// AddToScheme adds the types in this group-version to the given scheme.
 	AddToScheme = SchemeBuilder.AddToScheme
 )
+
+func addKnownTypes(s *runtime.Scheme) error {
+	s.AddKnownTypes(GroupVersion,
+		&Configuration{},
+	)
+	metav1.AddToGroupVersion(s, GroupVersion)
+	return nil
+}

ray-operator/apis/config/v1alpha1/register.go

@@ -5,8 +5,7 @@ import (
 )
 
 func init() {
-	SchemeBuilder.Register(&Configuration{})
-	SchemeBuilder.SchemeBuilder.Register(addDefaultingFuncs)
+	SchemeBuilder.Register(addDefaultingFuncs)
 }
 
 // SchemeGroupVersion is group version used to register these objects.

ray-operator/apis/ray/v1/groupversion_info.go

@@ -4,8 +4,9 @@
 package v1
 
 import (
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/apimachinery/pkg/runtime/schema"
-	"sigs.k8s.io/controller-runtime/pkg/scheme"
 )
 
 var (
@@ -13,8 +14,19 @@ var (
 	GroupVersion = schema.GroupVersion{Group: "ray.io", Version: "v1"}
 
 	// SchemeBuilder is used to add go types to the GroupVersionKind scheme
-	SchemeBuilder = &scheme.Builder{GroupVersion: GroupVersion}
+	SchemeBuilder = runtime.NewSchemeBuilder(addKnownTypes)
 
 	// AddToScheme adds the types in this group-version to the given scheme.
 	AddToScheme = SchemeBuilder.AddToScheme
 )
+
+func addKnownTypes(s *runtime.Scheme) error {
+	s.AddKnownTypes(GroupVersion,
+		&RayCluster{}, &RayClusterList{},
+		&RayJob{}, &RayJobList{},
+		&RayService{}, &RayServiceList{},
+		&RayCronJob{}, &RayCronJobList{},
+	)
+	metav1.AddToGroupVersion(s, GroupVersion)
+	return nil
+}

ray-operator/apis/ray/v1/raycluster_types.go

@@ -2,6 +2,7 @@ package v1
 
 import (
 	corev1 "k8s.io/api/core/v1"
+	networkingv1 "k8s.io/api/networking/v1"
 	"k8s.io/apimachinery/pkg/api/resource"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 )
@@ -42,6 +43,14 @@ type RayClusterSpec struct {
 	// GcsFaultToleranceOptions for enabling GCS FT
 	// +optional
 	GcsFaultToleranceOptions *GcsFaultToleranceOptions `json:"gcsFaultToleranceOptions,omitempty"`
+	// NetworkIsolation specifies optional configuration for network isolation.
+	// When set, separate NetworkPolicies are created for head and worker pods.
+	// The reconciler always permits intra-cluster pod-to-pod traffic.
+	// Note: under DenyAll/DenyAllEgress, DNS egress is not added
+	// automatically; since Ray pods reach the head via its service FQDN, you must
+	// allow DNS egress via Head/Worker EgressRules or the cluster will fail to start.
+	// +optional
+	NetworkIsolation *NetworkIsolationConfig `json:"networkIsolation,omitempty"`
 	// HeadGroupSpec is the spec for the head pod
 	HeadGroupSpec HeadGroupSpec `json:"headGroupSpec"`
 	// RayVersion is used to determine the command for the Kubernetes Job managed by RayJob
@@ -125,6 +134,66 @@ type RedisCredential struct {
 	Value string `json:"value,omitempty"`
 }
 
+// NetworkIsolationMode is the type for network isolation mode constants.
+// +kubebuilder:validation:Enum=DenyAll;DenyAllIngress;DenyAllEgress
+type NetworkIsolationMode string
+
+// Network isolation mode constants for NetworkIsolationConfig.Mode.
+const (
+	// NetworkIsolationDenyAll denies all ingress and egress traffic.
+	NetworkIsolationDenyAll NetworkIsolationMode = "DenyAll"
+	// NetworkIsolationDenyAllIngress denies all ingress traffic.
+	NetworkIsolationDenyAllIngress NetworkIsolationMode = "DenyAllIngress"
+	// NetworkIsolationDenyAllEgress denies all egress traffic.
+	NetworkIsolationDenyAllEgress NetworkIsolationMode = "DenyAllEgress"
+)
+
+// NetworkIsolationConfig defines network isolation settings for Ray cluster.
+// All modes permit intra-cluster pod-to-pod traffic.
+// DNS egress is not included automatically; see NetworkPolicyRules.EgressRules
+// for why it must be added under DenyAll/DenyAllEgress.
+type NetworkIsolationConfig struct {
+	// Mode controls the security level. All modes permit intra-cluster pod-to-pod
+	// traffic (DNS egress excluded, see EgressRules).
+	// - "DenyAll": Denies all Ingress and Egress.
+	// - "DenyAllIngress": Denies all Ingress.
+	// - "DenyAllEgress": Denies all Egress.
+	// +optional
+	// +kubebuilder:default=DenyAll
+	Mode *NetworkIsolationMode `json:"mode,omitempty"`
+
+	// Head specifies custom NetworkPolicy rules applied only to the head pod's policy.
+	// The base head policy always allows intra-cluster traffic and (for K8sJobMode
+	// RayJob-owned clusters) the submitter pod. Rules here are appended to those
+	// base rules. Platforms that need operator dashboard access should add it here
+	// (e.g. via a mutating webhook).
+	// +optional
+	Head *NetworkPolicyRules `json:"head,omitempty"`
+
+	// Worker specifies custom NetworkPolicy rules applied only to worker pods' policy.
+	// The base worker policy always allows intra-cluster traffic.
+	// Rules here are appended to that base rule.
+	// +optional
+	Worker *NetworkPolicyRules `json:"worker,omitempty"`
+}
+
+// NetworkPolicyRules defines custom ingress and egress rules for a NetworkPolicy.
+type NetworkPolicyRules struct {
+	// IngressRules specifies custom ingress rules appended to the base policy.
+	// Only meaningful when the mode includes ingress denial (DenyAll or DenyAllIngress).
+	// +optional
+	IngressRules []networkingv1.NetworkPolicyIngressRule `json:"ingressRules,omitempty"`
+
+	// EgressRules specifies custom egress rules appended to the base policy.
+	// Only meaningful when the mode includes egress denial (DenyAll or DenyAllEgress).
+	// DNS egress is NOT added automatically: under DenyAll/DenyAllEgress you MUST
+	// add a DNS rule here (e.g. to kube-system pods labeled k8s-app=kube-dns on
+	// port 53), because Ray workers reach the head via its service FQDN and cannot
+	// resolve it without DNS. See the network-isolation-deny-all sample.
+	// +optional
+	EgressRules []networkingv1.NetworkPolicyEgressRule `json:"egressRules,omitempty"`
+}
+
 // HeadGroupSpec are the spec for the head pod
 type HeadGroupSpec struct {
 	// Template is the exact pod template used in K8s deployments, statefulsets, etc.
@@ -250,6 +319,12 @@ type AutoscalerOptions struct {
 	// Optional list of volumeMounts.  This is needed for enabling TLS for the autoscaler container.
 	// +optional
 	VolumeMounts []corev1.VolumeMount `json:"volumeMounts,omitempty"`
+	// Optional list overwrite the default command of the autoscaler container.
+	// +optional
+	Command []string `json:"command,omitempty"`
+	// Optional to overwrite the default args of the autoscaler container.
+	// +optional
+	Args []string `json:"args,omitempty"`
 }
 
 // +kubebuilder:validation:Enum=Default;Aggressive;Conservative
@@ -432,10 +507,6 @@ type RayClusterList struct {
 	Items           []RayCluster `json:"items"`
 }
 
-func init() {
-	SchemeBuilder.Register(&RayCluster{}, &RayClusterList{})
-}
-
 type EventReason string
 
 const (

ray-operator/apis/ray/v1/raycronjob_types.go

@@ -12,10 +12,16 @@ type RayCronJobSpec struct {
 	JobTemplate RayJobSpec `json:"jobTemplate"`
 	// Schedule is the cron schedule string
 	Schedule string `json:"schedule"`
+	// TimeZone is the time zone name for the given schedule. If not specified, default to the local time zone of the
+	// Kuberay Operator. Empty string is not allowed.
+	// The bundled version of the time zone database is used.
+	// +optional
+	// +kubebuilder:validation:MinLength=1
+	TimeZone *string `json:"timeZone,omitempty"`
 	// Suspend tells the controller to suspend the scheduling, it does not apply to
 	// scheduled RayJob.
 	// +optional
-	Suspend bool `json:"suspend,omitempty"`
+	Suspend *bool `json:"suspend,omitempty"`
 }
 
 // RayCronJobStatus defines the observed state of RayCronJob
@@ -26,7 +32,8 @@ type RayCronJobStatus struct {
 //+kubebuilder:object:root=true
 //+kubebuilder:subresource:status
 //+kubebuilder:printcolumn:name="schedule",type=string,JSONPath=".spec.schedule",priority=0
-//+kubebuilder:printcolumn:name="last schedule",type=string,JSONPath=".status.lastScheduleTime",priority=0
+//+kubebuilder:printcolumn:name="timezone",type=string,JSONPath=".spec.timeZone",priority=0
+//+kubebuilder:printcolumn:name="last schedule",type=date,JSONPath=".status.lastScheduleTime",priority=0
 //+kubebuilder:printcolumn:name="age",type="date",JSONPath=".metadata.creationTimestamp",priority=0
 //+kubebuilder:printcolumn:name="suspend",type=boolean,JSONPath=".spec.suspend",priority=0
 
@@ -50,7 +57,3 @@ type RayCronJobList struct {
 	metav1.ListMeta `json:"metadata,omitempty"`
 	Items           []RayCronJob `json:"items"`
 }
-
-func init() {
-	SchemeBuilder.Register(&RayCronJob{}, &RayCronJobList{})
-}

ray-operator/apis/ray/v1/rayjob_types.go

@@ -73,6 +73,7 @@ const (
 	PreRunningDeadlineExceeded                       JobFailedReason = "PreRunningDeadlineExceeded"
 	AppFailed                                        JobFailedReason = "AppFailed"
 	JobDeploymentStatusTransitionGracePeriodExceeded JobFailedReason = "JobDeploymentStatusTransitionGracePeriodExceeded"
+	JobStatusCheckTimeoutExceeded                    JobFailedReason = "JobStatusCheckTimeoutExceeded"
 	ValidationFailed                                 JobFailedReason = "ValidationFailed"
 )
 
@@ -187,7 +188,8 @@ const (
 )
 
 type SubmitterConfig struct {
-	// BackoffLimit of the submitter k8s job.
+	// BackoffLimit of the submitter. In K8sJobMode, this is the K8s Job backoffLimit.
+	// In SidecarMode with SidecarSubmitterRestart enabled, this is the maximum container restart count.
 	// +optional
 	BackoffLimit *int32 `json:"backoffLimit,omitempty"`
 }
@@ -338,6 +340,10 @@ type RayJobStatus struct {
 	// RayClusterStatus is the status of the RayCluster running the job.
 	// +optional
 	RayClusterStatus RayClusterStatus `json:"rayClusterStatus,omitempty"`
+	// JobStatusCheckFailureStartTime is set when job status checks via the Ray Dashboard started failing.
+	// Reset on a successful check. JobDeploymentStatus transitions to 'Failed' after RAYJOB_STATUS_CHECK_TIMEOUT_SECONDS
+	// +optional
+	JobStatusCheckFailureStartTime *metav1.Time `json:"jobStatusCheckFailureStartTime,omitempty"`
 
 	// observedGeneration is the most recent generation observed for this RayJob. It corresponds to the
 	// RayJob's generation, which is updated on mutation by the API Server.
@@ -374,7 +380,3 @@ type RayJobList struct {
 	metav1.ListMeta `json:"metadata,omitempty"`
 	Items           []RayJob `json:"items"`
 }
-
-func init() {
-	SchemeBuilder.Register(&RayJob{}, &RayJobList{})
-}

ray-operator/apis/ray/v1/rayservice_types.go

@@ -122,6 +122,11 @@ type RayServiceSpec struct {
 	// Therefore, the head Pod's endpoint will not be added to the Kubernetes Serve service.
 	// +optional
 	ExcludeHeadPodFromServeSvc bool `json:"excludeHeadPodFromServeSvc,omitempty"`
+	// Suspend indicates whether the RayService should suspend its execution. When set to true,
+	// all Kubernetes resources owned by the RayService controller will be deleted. Setting it
+	// back to false will allow the RayService controller to recreate the resources.
+	// +optional
+	Suspend bool `json:"suspend,omitempty"`
 }
 
 // RayServiceStatuses defines the observed state of RayService
@@ -209,6 +214,11 @@ const (
 	UpgradeInProgress RayServiceConditionType = "UpgradeInProgress"
 	// RollbackInProgress means the RayService is currently rolling back an in-progress upgrade to the original cluster state.
 	RollbackInProgress RayServiceConditionType = "RollbackInProgress"
+	// RayServiceSuspending means the RayService is in the middle of deleting its owned resources in response to Spec.Suspend.
+	// Once entered, the suspend operation completes atomically regardless of later changes to Spec.Suspend.
+	RayServiceSuspending RayServiceConditionType = "Suspending"
+	// RayServiceSuspended means all resources owned by the RayService controller have been deleted and the RayService is suspended.
+	RayServiceSuspended RayServiceConditionType = "Suspended"
 )
 
 const (
@@ -221,6 +231,10 @@ const (
 	NoActiveCluster                RayServiceConditionReason = "NoActiveCluster"
 	RayServiceValidationFailed     RayServiceConditionReason = "ValidationFailed"
 	TargetClusterChanged           RayServiceConditionReason = "TargetClusterChanged"
+	SuspendRequested               RayServiceConditionReason = "SuspendRequested"
+	SuspendInProgress              RayServiceConditionReason = "SuspendInProgress"
+	SuspendComplete                RayServiceConditionReason = "SuspendComplete"
+	RayServiceResumed              RayServiceConditionReason = "RayServiceResumed"
 )
 
 // +kubebuilder:object:root=true
@@ -247,7 +261,3 @@ type RayServiceList struct {
 	metav1.ListMeta `json:"metadata,omitempty"`
 	Items           []RayService `json:"items"`
 }
-
-func init() {
-	SchemeBuilder.Register(&RayService{}, &RayServiceList{})
-}

ray-operator/apis/ray/v1/zz_generated.deepcopy.go

@@ -6,8 +6,9 @@ package v1
 
 import (
 	corev1 "k8s.io/api/core/v1"
+	networkingv1 "k8s.io/api/networking/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
-	runtime "k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/apimachinery/pkg/runtime"
 )
 
 // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
@@ -116,6 +117,16 @@ func (in *AutoscalerOptions) DeepCopyInto(out *AutoscalerOptions) {
 			(*in)[i].DeepCopyInto(&(*out)[i])
 		}
 	}
+	if in.Command != nil {
+		in, out := &in.Command, &out.Command
+		*out = make([]string, len(*in))
+		copy(*out, *in)
+	}
+	if in.Args != nil {
+		in, out := &in.Args, &out.Args
+		*out = make([]string, len(*in))
+		copy(*out, *in)
+	}
 }
 
 // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new AutoscalerOptions.
@@ -338,6 +349,65 @@ func (in *HeadInfo) DeepCopy() *HeadInfo {
 	return out
 }
 
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *NetworkIsolationConfig) DeepCopyInto(out *NetworkIsolationConfig) {
+	*out = *in
+	if in.Mode != nil {
+		in, out := &in.Mode, &out.Mode
+		*out = new(NetworkIsolationMode)
+		**out = **in
+	}
+	if in.Head != nil {
+		in, out := &in.Head, &out.Head
+		*out = new(NetworkPolicyRules)
+		(*in).DeepCopyInto(*out)
+	}
+	if in.Worker != nil {
+		in, out := &in.Worker, &out.Worker
+		*out = new(NetworkPolicyRules)
+		(*in).DeepCopyInto(*out)
+	}
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new NetworkIsolationConfig.
+func (in *NetworkIsolationConfig) DeepCopy() *NetworkIsolationConfig {
+	if in == nil {
+		return nil
+	}
+	out := new(NetworkIsolationConfig)
+	in.DeepCopyInto(out)
+	return out
+}
+
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *NetworkPolicyRules) DeepCopyInto(out *NetworkPolicyRules) {
+	*out = *in
+	if in.IngressRules != nil {
+		in, out := &in.IngressRules, &out.IngressRules
+		*out = make([]networkingv1.NetworkPolicyIngressRule, len(*in))
+		for i := range *in {
+			(*in)[i].DeepCopyInto(&(*out)[i])
+		}
+	}
+	if in.EgressRules != nil {
+		in, out := &in.EgressRules, &out.EgressRules
+		*out = make([]networkingv1.NetworkPolicyEgressRule, len(*in))
+		for i := range *in {
+			(*in)[i].DeepCopyInto(&(*out)[i])
+		}
+	}
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new NetworkPolicyRules.
+func (in *NetworkPolicyRules) DeepCopy() *NetworkPolicyRules {
+	if in == nil {
+		return nil
+	}
+	out := new(NetworkPolicyRules)
+	in.DeepCopyInto(out)
+	return out
+}
+
 // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
 func (in *RayCluster) DeepCopyInto(out *RayCluster) {
 	*out = *in
@@ -442,6 +512,11 @@ func (in *RayClusterSpec) DeepCopyInto(out *RayClusterSpec) {
 		*out = new(GcsFaultToleranceOptions)
 		(*in).DeepCopyInto(*out)
 	}
+	if in.NetworkIsolation != nil {
+		in, out := &in.NetworkIsolation, &out.NetworkIsolation
+		*out = new(NetworkIsolationConfig)
+		(*in).DeepCopyInto(*out)
+	}
 	in.HeadGroupSpec.DeepCopyInto(&out.HeadGroupSpec)
 	if in.WorkerGroupSpecs != nil {
 		in, out := &in.WorkerGroupSpecs, &out.WorkerGroupSpecs
@@ -590,6 +665,16 @@ func (in *RayCronJobList) DeepCopyObject() runtime.Object {
 func (in *RayCronJobSpec) DeepCopyInto(out *RayCronJobSpec) {
 	*out = *in
 	in.JobTemplate.DeepCopyInto(&out.JobTemplate)
+	if in.TimeZone != nil {
+		in, out := &in.TimeZone, &out.TimeZone
+		*out = new(string)
+		**out = **in
+	}
+	if in.Suspend != nil {
+		in, out := &in.Suspend, &out.Suspend
+		*out = new(bool)
+		**out = **in
+	}
 }
 
 // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new RayCronJobSpec.
@@ -772,6 +857,10 @@ func (in *RayJobStatus) DeepCopyInto(out *RayJobStatus) {
 		**out = **in
 	}
 	in.RayClusterStatus.DeepCopyInto(&out.RayClusterStatus)
+	if in.JobStatusCheckFailureStartTime != nil {
+		in, out := &in.JobStatusCheckFailureStartTime, &out.JobStatusCheckFailureStartTime
+		*out = (*in).DeepCopy()
+	}
 }
 
 // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new RayJobStatus.