Upgrade alpine to 3.20.x (#3710 )

* upgrade alpine Signed-off-by: Joe Elliott <number101010@gmail.com> * changelog Signed-off-by: Joe Elliott <number101010@gmail.com> --------- Signed-off-by: Joe Elliott <number101010@gmail.com>
2.5.0-rc1 changelog (#3709 )
2024-05-23 11:49:50 -04:00 · 2024-05-23 10:31:52 -04:00 · 2024-05-23 08:51:24 +02:00 · 2024-05-22 20:39:00 -05:00 · 2024-05-22 18:59:39 -04:00 · 2024-05-23 08:58:55 +10:00
7488 changed files with 753508 additions and 220030 deletions
@@ -3,4 +3,3 @@ example/
 integration/
 operations/
 opentelemetry-proto/
-tools/
@@ -105,7 +105,7 @@ local image_tag_for_cd() = {

 local build_binaries(arch) = {
  name: 'build-tempo-binaries',
-  image: 'golang:1.20.4-alpine',
+  image: 'golang:1.21.3-alpine',
  commands: [
    'apk --update --no-cache add make git bash',
  ] + [
@@ -114,14 +114,15 @@ local build_binaries(arch) = {
  ],
 };

-local docker_build(arch, app) = {
+local docker_build(arch, app, dockerfile='') = {
  name: 'build-%s-image' % app,
  image: 'plugins/docker',
  settings: {
-    dockerfile: 'cmd/%s/Dockerfile' % app,
+    dockerfile: if dockerfile != '' then dockerfile else 'cmd/%s/Dockerfile' % app,
    repo: 'grafana/%s' % app,
    username: { from_secret: docker_username_secret.name },
    password: { from_secret: docker_password_secret.name },
+    platform: '%s/%s' % ['linux', arch],
    build_args: [
      'TARGETARCH=' + arch,
    ],
@@ -199,6 +200,29 @@ local deploy_to_dev() = {
      for arch in archs
    ],
  },
+] + [
+  // Publish tools Docker manifests
+  (
+    pipeline('docker-ci-tools-%s' % arch, arch) {
+      steps+: [
+        image_tag(arch),
+        docker_build(arch, 'tempo-ci-tools', dockerfile='tools/Dockerfile'),
+      ],
+    }
+  )
+  for arch in archs
+] + [
+  // Publish Docker manifests
+  pipeline('manifest-ci-tools') {
+    steps+: [
+      image_tag(),
+      docker_manifest('tempo-ci-tools'),
+    ],
+    depends_on+: [
+      'docker-ci-tools-%s' % arch
+      for arch in archs
+    ],
+  },
 ] + [
  // Continuously Deploy to dev env
  pipeline('cd-to-dev-env') {
@@ -227,7 +251,7 @@ local deploy_to_dev() = {
    steps+: [
              {
                name: 'build-tempo-serverless',
-                image: 'golang:1.20.4-alpine',
+                image: 'golang:1.21.3-alpine',
                commands: [
                  'apk add make git zip bash',
                  './tools/image-tag | cut -d, -f 1 | tr A-Z a-z > .tags',  // values in .tags are used by the next step when pushing the image
@@ -324,7 +348,7 @@ local deploy_to_dev() = {
      },
      {
        name: 'write-key',
-        image: 'golang:1.20.4',
+        image: 'golang:1.21.3',
        commands: ['printf "%s" "$NFPM_SIGNING_KEY" > $NFPM_SIGNING_KEY_FILE'],
        environment: {
          NFPM_SIGNING_KEY: { from_secret: gpg_private_key.name },
@@ -333,7 +357,7 @@ local deploy_to_dev() = {
      },
      {
        name: 'test release',
-        image: 'golang:1.20.4',
+        image: 'golang:1.21.3',
        commands: ['make release-snapshot'],
        environment: {
          NFPM_DEFAULT_PASSPHRASE: { from_secret: gpg_passphrase.name },
@@ -366,7 +390,7 @@ local deploy_to_dev() = {
      },
      {
        name: 'release',
-        image: 'golang:1.20.4',
+        image: 'golang:1.21.3',
        commands: ['make release'],
        environment: {
          GITHUB_TOKEN: { from_secret: gh_token_secret.name },
@@ -17,7 +17,7 @@ steps:
  - COMPONENT=tempo GOARCH=amd64 make exe
  - COMPONENT=tempo-vulture GOARCH=amd64 make exe
  - COMPONENT=tempo-query GOARCH=amd64 make exe
-  image: golang:1.20.4-alpine
+  image: golang:1.21.3-alpine
  name: build-tempo-binaries
 - image: plugins/docker
  name: build-tempo-image
@@ -27,6 +27,7 @@ steps:
    dockerfile: cmd/tempo/Dockerfile
    password:
      from_secret: docker_password
+    platform: linux/amd64
    repo: grafana/tempo
    username:
      from_secret: docker_username
@@ -38,6 +39,7 @@ steps:
    dockerfile: cmd/tempo-vulture/Dockerfile
    password:
      from_secret: docker_password
+    platform: linux/amd64
    repo: grafana/tempo-vulture
    username:
      from_secret: docker_username
@@ -49,6 +51,7 @@ steps:
    dockerfile: cmd/tempo-query/Dockerfile
    password:
      from_secret: docker_password
+    platform: linux/amd64
    repo: grafana/tempo-query
    username:
      from_secret: docker_username
@@ -78,7 +81,7 @@ steps:
  - COMPONENT=tempo GOARCH=arm64 make exe
  - COMPONENT=tempo-vulture GOARCH=arm64 make exe
  - COMPONENT=tempo-query GOARCH=arm64 make exe
-  image: golang:1.20.4-alpine
+  image: golang:1.21.3-alpine
  name: build-tempo-binaries
 - image: plugins/docker
  name: build-tempo-image
@@ -88,6 +91,7 @@ steps:
    dockerfile: cmd/tempo/Dockerfile
    password:
      from_secret: docker_password
+    platform: linux/arm64
    repo: grafana/tempo
    username:
      from_secret: docker_username
@@ -99,6 +103,7 @@ steps:
    dockerfile: cmd/tempo-vulture/Dockerfile
    password:
      from_secret: docker_password
+    platform: linux/arm64
    repo: grafana/tempo-vulture
    username:
      from_secret: docker_username
@@ -110,6 +115,7 @@ steps:
    dockerfile: cmd/tempo-query/Dockerfile
    password:
      from_secret: docker_password
+    platform: linux/arm64
    repo: grafana/tempo-query
    username:
      from_secret: docker_username
@@ -171,6 +177,104 @@ trigger:
  - refs/heads/r??
  - refs/heads/r???
 ---
+depends_on: []
+kind: pipeline
+name: docker-ci-tools-amd64
+platform:
+  arch: amd64
+  os: linux
+steps:
+- commands:
+  - apk --update --no-cache add bash
+  - git fetch origin --tags
+  - echo $(./tools/image-tag)-amd64 > .tags
+  image: alpine/git:v2.30.2
+  name: image-tag
+- image: plugins/docker
+  name: build-tempo-ci-tools-image
+  settings:
+    build_args:
+    - TARGETARCH=amd64
+    dockerfile: tools/Dockerfile
+    password:
+      from_secret: docker_password
+    platform: linux/amd64
+    repo: grafana/tempo-ci-tools
+    username:
+      from_secret: docker_username
+trigger:
+  ref:
+  - refs/heads/main
+  - refs/tags/v*
+  - refs/heads/r?
+  - refs/heads/r??
+  - refs/heads/r???
+---
+depends_on: []
+kind: pipeline
+name: docker-ci-tools-arm64
+platform:
+  arch: arm64
+  os: linux
+steps:
+- commands:
+  - apk --update --no-cache add bash
+  - git fetch origin --tags
+  - echo $(./tools/image-tag)-arm64 > .tags
+  image: alpine/git:v2.30.2
+  name: image-tag
+- image: plugins/docker
+  name: build-tempo-ci-tools-image
+  settings:
+    build_args:
+    - TARGETARCH=arm64
+    dockerfile: tools/Dockerfile
+    password:
+      from_secret: docker_password
+    platform: linux/arm64
+    repo: grafana/tempo-ci-tools
+    username:
+      from_secret: docker_username
+trigger:
+  ref:
+  - refs/heads/main
+  - refs/tags/v*
+  - refs/heads/r?
+  - refs/heads/r??
+  - refs/heads/r???
+---
+depends_on:
+- docker-ci-tools-amd64
+- docker-ci-tools-arm64
+kind: pipeline
+name: manifest-ci-tools
+platform:
+  arch: amd64
+  os: linux
+steps:
+- commands:
+  - apk --update --no-cache add bash
+  - git fetch origin --tags
+  - echo $(./tools/image-tag) > .tags
+  image: alpine/git:v2.30.2
+  name: image-tag
+- image: plugins/manifest:1.4.0
+  name: manifest-tempo-ci-tools
+  settings:
+    password:
+      from_secret: docker_password
+    spec: .drone/docker-manifest.tmpl
+    target: tempo-ci-tools
+    username:
+      from_secret: docker_username
+trigger:
+  ref:
+  - refs/heads/main
+  - refs/tags/v*
+  - refs/heads/r?
+  - refs/heads/r??
+  - refs/heads/r???
+---
 depends_on:
 - manifest
 image_pull_secrets:
@@ -238,7 +342,7 @@ steps:
  - cd ./cmd/tempo-serverless
  - make build-docker-gcr-binary
  - make build-lambda-zip
-  image: golang:1.20.4-alpine
+  image: golang:1.21.3-alpine
  name: build-tempo-serverless
 - image: plugins/gcr
  name: deploy-tempo-serverless-gcr
@@ -308,7 +412,7 @@ steps:
    NFPM_SIGNING_KEY:
      from_secret: gpg_private_key
    NFPM_SIGNING_KEY_FILE: /drone/src/private-key.key
-  image: golang:1.20.4
+  image: golang:1.21.3
  name: write-key
 - commands:
  - make release-snapshot
@@ -316,7 +420,7 @@ steps:
    NFPM_DEFAULT_PASSPHRASE:
      from_secret: gpg_passphrase
    NFPM_SIGNING_KEY_FILE: /drone/src/private-key.key
-  image: golang:1.20.4
+  image: golang:1.21.3
  name: test release
 - commands:
  - ./tools/packaging/verify-deb-install.sh
@@ -342,7 +446,7 @@ steps:
    NFPM_DEFAULT_PASSPHRASE:
      from_secret: gpg_passphrase
    NFPM_SIGNING_KEY_FILE: /drone/src/private-key.key
-  image: golang:1.20.4
+  image: golang:1.21.3
  name: release
  when:
    event:
@@ -426,6 +530,6 @@ kind: secret
 name: gpg_passphrase
 ---
 kind: signature
-hmac: beb2593e403f909da214e906b27317f8df0ed24dea4d98d04ba27e985b1ea701
+hmac: fd0d2fe13ee75cfc3a721fee180a293121c8aad04419e22e47e16793b45a2e33

 ...
@@ -1,4 +0,0 @@
-// the following functions will be excluded from errcheck
-// https://github.com/kisielk/errcheck#excluding-functions
-(github.com/go-kit/kit/log.Logger).Log
-(github.com/go-kit/log.Logger).Log
@@ -16,7 +16,7 @@ jobs:
    runs-on: ubuntu-latest
    steps:
      - name: Checkout Actions
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
        with:
          repository: "grafana/grafana-github-actions"
          path: ./actions
@@ -22,20 +22,15 @@ jobs:
    name: Lint
    runs-on: ubuntu-latest
    steps:
-      - name: Set up Go 1.20
-        uses: actions/setup-go@v3
+      - name: Set up Go 1.21
+        uses: actions/setup-go@v5
        with:
-          go-version: 1.20.x
-
-      - name: Install jsonnetfmt and goimports
-        run: |
-          curl -fSL -o jsonnet.tar.gz https://github.com/google/jsonnet/releases/download/v0.17.0/jsonnet-bin-v0.17.0-linux.tar.gz
-          tar -xvf jsonnet.tar.gz -C /usr/local/bin/
-          chmod a+x /usr/local/bin/jsonnetfmt
-          go install golang.org/x/tools/cmd/goimports@v0.1.12
+          go-version: 1.21.x

      - name: Check out code
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: '0'

      - name: check-fmt
        run: make check-fmt
@@ -43,22 +38,20 @@ jobs:
      - name: check-jsonnetfmt
        run: make check-jsonnetfmt

-      - name: Lint
-        uses: golangci/golangci-lint-action@v3
-        with:
-          version: v1.53.3
+      - name: lint
+        run: make lint base=origin/${{github.base_ref}}

  unit-tests-pkg:
    name: Test packages - pkg
    runs-on: ubuntu-latest
    steps:
-      - name: Set up Go 1.20
-        uses: actions/setup-go@v3
+      - name: Set up Go 1.21
+        uses: actions/setup-go@v5
        with:
-          go-version: 1.20.x
+          go-version: 1.21.x

      - name: Check out code
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4

      - name: Test
        run: make test-with-cover-pkg
@@ -67,13 +60,13 @@ jobs:
    name: Test packages - tempodb
    runs-on: ubuntu-latest
    steps:
-      - name: Set up Go 1.20
-        uses: actions/setup-go@v3
+      - name: Set up Go 1.21
+        uses: actions/setup-go@v5
        with:
-          go-version: 1.20.x
+          go-version: 1.21.x

      - name: Check out code
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4

      - name: Test
        run: make test-with-cover-tempodb
@@ -82,13 +75,13 @@ jobs:
    name: Test packages - tempodb/wal
    runs-on: ubuntu-latest
    steps:
-      - name: Set up Go 1.20
-        uses: actions/setup-go@v3
+      - name: Set up Go 1.21
+        uses: actions/setup-go@v5
        with:
-          go-version: 1.20.x
+          go-version: 1.21.x

      - name: Check out code
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4

      - name: Test
        run: make test-with-cover-tempodb-wal
@@ -97,13 +90,13 @@ jobs:
    name: Test packages - others
    runs-on: ubuntu-latest
    steps:
-      - name: Set up Go 1.20
-        uses: actions/setup-go@v3
+      - name: Set up Go 1.21
+        uses: actions/setup-go@v5
        with:
-          go-version: 1.20.x
+          go-version: 1.21.x

      - name: Check out code
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4

      - name: Test
        run: make test-with-cover-others
@@ -112,28 +105,31 @@ jobs:
    name: Test integration e2e suite
    runs-on: ubuntu-latest
    steps:
-      - name: Set up Go 1.20
-        uses: actions/setup-go@v3
+      - name: Set up Go 1.21
+        uses: actions/setup-go@v5
        with:
-          go-version: 1.20.x
+          go-version: 1.21.x

      - name: Check out code
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4

      - name: Test
        run: make test-e2e

+      - name: Poller
+        run: make test-integration-poller
+
  integration-tests-serverless:
    name: Test serverless integration e2e suite
    runs-on: ubuntu-latest
    steps:
-      - name: Set up Go 1.20
-        uses: actions/setup-go@v3
+      - name: Set up Go 1.21
+        uses: actions/setup-go@v5
        with:
-          go-version: 1.20.x
+          go-version: 1.21.x

      - name: Check out code
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4

      - name: Test
        run: make test-e2e-serverless
@@ -142,13 +138,13 @@ jobs:
    name: Build
    runs-on: ubuntu-latest
    steps:
-      - name: Set up Go 1.20
-        uses: actions/setup-go@v3
+      - name: Set up Go 1.21
+        uses: actions/setup-go@v5
        with:
-          go-version: 1.20.x
+          go-version: 1.21.x

      - name: Check out code
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4

      - name: Build Tempo
        run: make tempo
@@ -166,13 +162,13 @@ jobs:
    name: Benchmark
    runs-on: ubuntu-latest
    steps:
-      - name: Set up Go 1.20
-        uses: actions/setup-go@v3
+      - name: Set up Go 1.21
+        uses: actions/setup-go@v5
        with:
-          go-version: 1.20.x
+          go-version: 1.21.x

      - name: Check out code
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4

      - name: Bench
        run: make test-bench
@@ -181,18 +177,17 @@ jobs:
    name: Vendor check
    runs-on: ubuntu-latest
    steps:
-      - name: Set up Go 1.20
-        uses: actions/setup-go@v3
+      - name: Set up Go 1.21
+        uses: actions/setup-go@v5
        with:
-          go-version: 1.20.x
+          go-version: 1.21.x

      - name: Check out code
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4

      - name: Install Protoc
-        uses: arduino/setup-protoc@v1
+        uses: arduino/setup-protoc@v3
        with:
-          version: '3.11.2'
          repo-token: ${{ secrets.GITHUB_TOKEN }}

      - name: Check vendor
@@ -202,24 +197,13 @@ jobs:
    name: Check jsonnet & tempo-mixin
    runs-on: ubuntu-latest
    steps:
-      - name: Set up Go 1.20
-        uses: actions/setup-go@v3
+      - name: Set up Go 1.21
+        uses: actions/setup-go@v5
        with:
-          go-version: 1.20.x
-
-      - name: Install jsonnet, jsonnet-bundler & tanka
-        run: |
-          curl -fSL -o jsonnet.tar.gz https://github.com/google/jsonnet/releases/download/v0.17.0/jsonnet-bin-v0.17.0-linux.tar.gz
-          tar -xvf jsonnet.tar.gz -C /usr/local/bin/
-          chmod a+x /usr/local/bin/jsonnet
-
-          go install github.com/jsonnet-bundler/jsonnet-bundler/cmd/jb@v0.4.0
-
-          curl -fSL -o /usr/local/bin/tk https://github.com/grafana/tanka/releases/download/v0.19.0/tk-linux-amd64
-          chmod a+x /usr/local/bin/tk
+          go-version: 1.21.x

      - name: Check out code
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4

      - name: Check jsonnet
        run: make jsonnet-check
@@ -235,6 +219,6 @@ jobs:
    runs-on: ubuntu-latest
    steps:
      - name: Check out code
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
      - name: Build Website
        run: docker run -v ${PWD}/docs/sources:/hugo/content/docs/tempo/latest --rm grafana/docs-base:latest make prod
@@ -1,5 +1,7 @@
-name: Dependabot serverless gomod update
+name: Dependabot
 on: pull_request
+permissions:
+  contents: write

 # NOTE: For this job, we are unable to use the GITHUB_TOKEN from the action
 # itself.  The reason for this is that the tokens handed out for actions dont'
@@ -11,16 +13,17 @@ on: pull_request

 jobs:
  dependabot:
+    name: Serverless gomod update
    runs-on: ubuntu-latest
    if: ${{ github.event.pull_request.user.login == 'dependabot[bot]' }}
    steps:
-      - name: Set up Go 1.20
-        uses: actions/setup-go@v3
+      - name: Set up Go 1.21
+        uses: actions/setup-go@v5
        with:
-          go-version: 1.20.x
+          go-version: 1.21.x

      - name: Check out code
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
        with:
          ref: ${{ github.head_ref }}
          token: ${{ secrets.GH_BOT_ACCESS_TOKEN }}
@@ -29,7 +32,11 @@ jobs:
        run: make -C cmd/tempo-serverless update-mod

      - name: Commit serverless gomod changes
+        env:
+          USER: grafanabot
+          TOKEN: ${{ secrets.GH_BOT_ACCESS_TOKEN }}
        run: |
+          git config --global url."https://${USER}:${TOKEN}@github.com/grafana/tempo".insteadOf "https://github.com/grafana/tempo"
          git config user.name grafanabot
          git config user.email bot@grafana.com
          git add cmd/tempo-serverless/lambda/go.mod
@@ -7,10 +7,10 @@ jobs:
  doc-validator:
    runs-on: "ubuntu-latest"
    container:
-      image: "grafana/doc-validator:v3.0.0"
+      image: "grafana/doc-validator:v4.1.1"
    steps:
      - name: "Checkout code"
-        uses: "actions/checkout@v3"
+        uses: "actions/checkout@v4"
      - name: "Run doc-validator"
        run: |
          rm -rf /hugo/content/docs
@@ -9,7 +9,7 @@ jobs:
    runs-on: ubuntu-latest
    steps:
      - name: Checkout Actions
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
        with:
          repository: "grafana/grafana-github-actions"
          path: ./actions
@@ -11,7 +11,7 @@ jobs:
    runs-on: ubuntu-latest
    steps:
      - name: Checkout Actions
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
        with:
          repository: "grafana/grafana-github-actions"
          path: ./actions
@@ -12,7 +12,7 @@ jobs:
    runs-on: ubuntu-latest

    steps:
-      - uses: actions/add-to-project@v0.4.0
+      - uses: actions/add-to-project@v1.0.0
        with:
          project-url: https://github.com/orgs/grafana/projects/135
          github-token: ${{ secrets.GH_BOT_ACCESS_TOKEN }}
@@ -11,10 +11,10 @@ jobs:

    steps:
    - name: Checkout
-      uses: actions/checkout@v2
+      uses: actions/checkout@v4
        
    - name: Anchore SBOM Action
-      uses: anchore/sbom-action@v0.12.0
+      uses: anchore/sbom-action@v0.15.10
      with:
         artifact-name: ${{ github.event.repository.name }}-spdx.json

@@ -12,7 +12,7 @@ jobs:
    if: github.repository == 'grafana/tempo'
    runs-on: ubuntu-latest
    steps:
-      - uses: actions/stale@v6.0.1
+      - uses: actions/stale@v9.0.0
        with:
          close-issue-reason: not_planned
          operations-per-run: 100
@@ -0,0 +1,12 @@
+name: Update `make docs` procedure
+on:
+  schedule:
+    - cron: '0 7 * * 1-5'
+  workflow_dispatch:
+jobs:
+  main:
+    if: github.repository == 'grafana/tempo'
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: grafana/writers-toolkit/update-make-docs@update-make-docs/v1
@@ -16,7 +16,7 @@ jobs:
    runs-on: ubuntu-latest
    steps:
      - name: Check out code
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
      - name: Build Website
        run: |
          docker run \
@@ -33,7 +33,7 @@ jobs:
    runs-on: ubuntu-latest
    needs: test
    steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
      - name: "Clone website-sync Action"
        # WEBSITE_SYNC_TOKEN is a fine-grained GitHub Personal Access Token that expires.
        # It must be regenerated in the grafanabot GitHub account and requires a Grafana organization
@@ -18,7 +18,7 @@ jobs:
    runs-on: ubuntu-latest
    steps:
      - name: Check out code
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
      - name: Build Website
        run: docker run -v ${PWD}/docs/sources/tempo:/hugo/content/docs/tempo/latest --rm grafana/docs-base:latest make prod
  sync:
@@ -28,13 +28,13 @@ jobs:
    needs: test
    steps:
      - name: Checkout this repo
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
        with:
          # Fetch tags
          fetch-depth: 0

      - name: Checkout Actions library
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
        with:
          repository: "grafana/grafana-github-actions"
          path: ./actions
@@ -1,6 +1,8 @@
 .DS_Store
+.envrc
 .idea
 .vscode
+*.test
 /bin
 /cmd/tempo-cli/tempo-cli
 /cmd/tempo-serverless/vendor/
@@ -14,4 +16,5 @@
 /tempo-query
 /tempo-vulture
 /tempodb/encoding/benchmark_block
-private-key.key
+private-key.key
+integration/e2e/e2e_integration_test[0-9]*
@@ -32,7 +32,8 @@ run:
 # output configuration options
 output:
  # colored-line-number|line-number|json|tab|checkstyle, default is "colored-line-number"
-  format: colored-line-number
+  formats:
+    - format: colored-line-number

  # print lines of code with issue, default is true
  print-issued-lines: true
@@ -43,6 +44,7 @@ output:
 linters:
  enable:
    - errcheck
+    - errorlint
    - goconst
    - gofmt
    - goimports
@@ -59,7 +61,13 @@ linters:

 linters-settings:
  errcheck:
-    exclude: ./.errcheck-exclude.txt
+    exclude-functions:
+      - (github.com/go-kit/kit/log.Logger).Log
+      - (github.com/go-kit/log.Logger).Log
+  errorlint:
+    errorf: false # don't check whether fmt.Errorf uses %w to wrap errors
+  goconst:
+    ignore-tests: true

 issues:
  exclude:
@@ -1,2 +1,3 @@
 * Grafana Labs
 * [Megvii](https://en.megvii.com/)
+* [Nuro](https://nuro.ai/)
@@ -1,6 +1,274 @@
 ## main / unreleased

-## v2.2.0-rc0 / 2023-07-21
+## v2.5.0-rc.1
+
+* [CHANGE] Update Alpine image version to 3.20 [#3710](https://github.com/grafana/tempo/pull/3710) (@joe-elliott)
+* [ENHANCEMENT] TraceQL - Add support for trace:id and span:id [#3670](https://github.com/grafana/tempo/pull/3670) (@ie-pham)
+* [BUGFIX] Fix TraceQL queries involving non boolean operations between statics and attributes. [#3698](https://github.com/grafana/tempo/pull/3698) (@joe-elliott)
+
+## v2.5.0-rc.0
+
+* [CHANGE] Align metrics query time ranges to the step parameter [#3490](https://github.com/grafana/tempo/pull/3490) (@mdisibio)
+* [CHANGE] Change the UID and GID of the `tempo` user to avoid root [#2265](https://github.com/grafana/tempo/pull/2265) (@zalegrala)
+  **BREAKING CHANGE** Ownership of /var/tempo is changing.  Historically, this
+  has been owned by root:root. With this change, it will now be owned by
+  tempo:tempo with the UID/GID of 10001.  The `ingester` and
+  `metrics-generator` statefulsets may need to be `chown`'d in order to start
+  properly.  A jsonnet example of an init container is included with the PR.
+  This impacts all users of the `grafana/tempo` Docker image.
+* [CHANGE] Remove vParquet encoding [#3663](https://github.com/grafana/tempo/pull/3663) (@mdisibio)
+  **BREAKING CHANGE** In the last release vParquet (the first version) was deprecated and blocked from writes. Now, it's 
+  removed entirely.  It will no longer be recognized as a valid encoding and cannot read any remaining vParquet blocks. Installations
+  running with historical defaults should not require any changes as the default has been migrated for several releases. Installations
+  with storage settings pinned to vParquet must run a previous release configured for vParquet2 or higher until all existing vParquet (1) blocks
+  have expired and been deleted from the backend, or else will encounter read errors after upgrading to this release.
+* [CHANGE] Return a less confusing error message to the client when refusing spans due to ingestion rates. [#3485](https://github.com/grafana/tempo/pull/3485) (@ie-pham)
+* [CHANGE] Clean Metrics Generator's Prometheus wal before creating instance [#3548](https://github.com/grafana/tempo/pull/3548) (@ie-pham)
+* [CHANGE] Update docker examples for permissions, deprecations, and clean-up [#3603](https://github.com/grafana/tempo/pull/3603) (@zalegrala)
+* [CHANGE] Update debian and rpm packages to grant required permissions to default storage path after installation [#3657](https://github.com/grafana/tempo/pull/3657) (@mdisibio)
+* [CHANGE] Delete any remaining objects for empty tenants after a configurable duration, requires config enable [#3611](https://github.com/grafana/tempo/pull/3611) (@zalegrala)
+* [CHANGE] Add golangci to the tools image and update `lint` make target [#3610](https://github.com/grafana/tempo/pull/3610) (@zalegrala)
+* [FEATURE] Add TLS support for Memcached Client [#3585](https://github.com/grafana/tempo/pull/3585) (@sonisr)
+* [FEATURE] TraceQL metrics queries: add quantile_over_time [#3605](https://github.com/grafana/tempo/pull/3605) [#3633](https://github.com/grafana/tempo/pull/3633) (@mdisibio) 
+* [FEATURE] TraceQL metrics queries: add histogram_over_time [#3644](https://github.com/grafana/tempo/pull/3644) (@mdisibio)
+* [FEATURE] Added gRPC streaming endpoints for Tempo APIs.
+  * Added gRPC streaming endpoints for all tag queries. [#3460](https://github.com/grafana/tempo/pull/3460) (@joe-elliott)
+  * Added gRPC streaming endpoints for metrics. [#3584](https://github.com/grafana/tempo/pull/3584) (@joe-elliott)
+  * Reduced memory consumption in the frontend for large traces. [#3522](https://github.com/grafana/tempo/pull/3522) (@joe-elliott)
+  * **Breaking Change** Remove trace by id hedging from the frontend. [#3522](https://github.com/grafana/tempo/pull/3522) (@joe-elliott)
+  * **Breaking Change** Dropped meta-tag for tenant from trace by id multitenant. [#3522](https://github.com/grafana/tempo/pull/3522) (@joe-elliott)
+* [FEATURE] New block encoding vParquet4 with support for links, events, and arrays [#3368](https://github.com/grafana/tempo/pull/3368) (@stoewer @ie-pham @andreasgerstmayr)
+* [ENHANCEMENT] Remove hardcoded delay in distributor shutdown [#3687](https://github.com/grafana/tempo/pull/3687) (@chodges15)
+* [ENHANCEMENT] Tempo CLI - add percentage support for query blocks #3697 [#3697](https://github.com/grafana/tempo/pull/3697) (@edgarkz)
+* [ENHANCEMENT] Update OTLP and add attributes to instrumentation scope in vParquet4 [#3649](https://github.com/grafana/tempo/pull/3649) (@stoewer)
+  **Breaking Change** The update to OTLP 1.3.0 removes the deprecated `InstrumentationLibrary`
+  and `InstrumentationLibrarySpan` from the OTLP receivers
+* [ENHANCEMENT] Surface new labels for uninstrumented services and systems [#3543](https://github.com/grafana/tempo/pull/3543) (@t00mas)
+* [ENHANCEMENT] Add querier metrics for requests executed [#3524](https://github.com/grafana/tempo/pull/3524) (@electron0zero)
+* [ENHANCEMENT] Add messaging-system latency histogram to service-graph [#3453](https://github.com/grafana/tempo/pull/3453) (@adirmatzkin)
+* [ENHANCEMENT] Add string interning to TraceQL queries [#3411](https://github.com/grafana/tempo/pull/3411) (@mapno)
+* [ENHANCEMENT] Add new (unsafe) query hints for metrics queries [#3396](https://github.com/grafana/tempo/pull/3396) (@mdisibio)
+* [ENHANCEMENT] Add nestedSetLeft/Right/Parent instrinsics to TraceQL. [#3497](https://github.com/grafana/tempo/pull/3497) (@joe-elliott)
+* [ENHANCEMENT] Add tenant to frontend job cache key. [#3527](https://github.com/grafana/tempo/pull/3527) (@joe-elliott)
+* [ENHANCEMENT] Better compaction throughput and memory usage [#3579](https://github.com/grafana/tempo/pull/3579) (@mdisibio)
+* [ENHANCEMENT] Add support for sharded ingester queries  [#3574](https://github.com/grafana/tempo/pull/3574) (@zalegrala)
+* [ENHANCEMENT] TraceQL - Add support for scoped intrinsics using `:` [#3629](https://github.com/grafana/tempo/pull/3629) (@ie-pham)
+  available scoped intrinsics: trace:duration, trace:rootName, trace:rootService, span:duration, span:kind, span:name, span:status, span:statusMessage
+* [ENHANCEMENT] Performance improvements on TraceQL and tag value search. [#3650](https://github.com/grafana/tempo/pull/3650),[#3667](https://github.com/grafana/tempo/pull/3667) (@joe-elliott)
+* [BUGFIX] Fix handling of regex matchers in autocomplete endpoints [#3641](https://github.com/grafana/tempo/pull/3641) (@sd2k)
+* [BUGFIX] Update golang.org/x/net package to 0.24.0 to fix CVE-2023-45288 [#3613](https://github.com/grafana/tempo/pull/3613) (@pavolloffay)
+* [BUGFIX] Fix metrics query results when filtering and rating on the same attribute [#3428](https://github.com/grafana/tempo/issues/3428) (@mdisibio)
+* [BUGFIX] Fix metrics query results when series contain empty strings or nil values [#3429](https://github.com/grafana/tempo/issues/3429) (@mdisibio)
+* [BUGFIX] Fix metrics query duration check, add per-tenant override for max metrics query duration [#3479](https://github.com/grafana/tempo/issues/3479) (@mdisibio)
+* [BUGFIX] Fix metrics query panic "index out of range [-1]" when a trace has zero-length ID [](https://github.com/grafana/tempo/pull/3668) (@mdisibio)
+* [BUGFIX] Return unfiltered results when a bad TraceQL query is provided in autocomplete. [#3426](https://github.com/grafana/tempo/pull/3426) (@mapno)
+* [BUGFIX] Add support for dashes, quotes and spaces in attribute names in autocomplete [#3458](https://github.com/grafana/tempo/pull/3458) (@mapno)
+* [BUGFIX] Correctly handle 429s in GRPC search streaming. [#3469](https://github.com/grafana/tempo/pull/3469) (@joe-ellitot)
+* [BUGFIX] Correctly cancel GRPC and HTTP contexts in the frontend to prevent having to rely on http write timeout. [#3443](https://github.com/grafana/tempo/pull/3443) (@joe-elliott)
+* [BUGFIX] Add spss and limit to the frontend cache key to prevent the return of incorrect results. [#3557](https://github.com/grafana/tempo/pull/3557) (@joe-elliott)
+* [BUGFIX] Use os path separator to split blocks path. [#3552](https://github.com/grafana/tempo/issues/3552) (@teyyubismayil)
+* [BUGFIX] Correctly parse traceql queries with > 1024 character attribute names or static values. [#3571](https://github.com/grafana/tempo/issues/3571) (@joe-elliott)
+* [BUGFIX] Fix span-metrics' subprocessors bug that applied wrong configs when running multiple tenants. [#3612](https://github.com/grafana/tempo/pull/3612) (@mapno)
+* [BUGFIX] Fix panic in query-frontend when combining results [#3683](https://github.com/grafana/tempo/pull/3683) (@mapno)
+
+## v2.4.2
+
+* [BUGFIX] Update golang.org/x/net package to 0.24.0 to fix CVE-2023-45288 [#3613](https://github.com/grafana/tempo/pull/3613) (@pavolloffay)
+
+## v2.4.1
+
+* [BUGFIX] Fix compaction/retention in AWS S3 and GCS when a prefix is configured. [#3465](https://github.com/grafana/tempo/issues/3465) (@bpfoster)
+
+## v2.4.0
+
+* [CHANGE] Merge the processors overrides set through runtime overrides and user-configurable overrides [#3125](https://github.com/grafana/tempo/pull/3125) (@kvrhdn)
+* [CHANGE] Make vParquet3 the default block encoding [#2526](https://github.com/grafana/tempo/pull/3134) (@stoewer)
+* [CHANGE] Set `autocomplete_filtering_enabled` to `true` by default [#3178](https://github.com/grafana/tempo/pull/3178) (@mapno)
+* [CHANGE] Update Alpine image version to 3.19 [#3289](https://github.com/grafana/tempo/pull/3289) (@zalegrala)
+* [CHANGE] **Breaking Change** Fix issue where tempo drops the entire batch if one trace triggers an error [#2571](https://github.com/grafana/tempo/pull/2571) (@ie-pham)
+  Distributor now returns 200 for any batch containing only trace_too_large and max_live_traces errors
+  The number of discarded spans are still reflected in the tempo_discarded_spans_total metrics
+* [CHANGE] Remove experimental websockets support for search streaming. GRPC is the supported method of streaming results [#3307](https://github.com/grafana/tempo/pull/3307) (@joe-elliott)
+* [CHANGE] **Breaking Change** Deprecating vParquet v1 [#3377](https://github.com/grafana/tempo/pull/3377) (@ie-pham)
+* [FEATURE] TraceQL metrics queries [#3227](https://github.com/grafana/tempo/pull/3227) [#3252](https://github.com/grafana/tempo/pull/3252) [#3258](https://github.com/grafana/tempo/pull/3258) (@mdisibio @zalegrala)
+* [FEATURE] Add support for multi-tenant queries. [#3087](https://github.com/grafana/tempo/pull/3087) (@electron0zero)
+* [FEATURE] Major cache refactor to allow multiple role based caches to be configured [#3166](https://github.com/grafana/tempo/pull/3166).
+  **BREAKING CHANGE** Deprecate the following fields. These have all been migrated to a top level "cache:" field.
+  ```
+  storage:
+    trace:
+      cache:
+      search:
+        cache_control:
+      background_cache:
+      memcached:
+      redis:
+  ```
+* [ENHANCEMENT] Add support for multi-tenant queries in streaming search [#3262](https://github.com/grafana/tempo/pull/3262) (@electron0zero)
+* [ENHANCEMENT] Add configuration on tempo-query plugin for fetch services older than complete_block_timeout [#3262](https://github.com/grafana/tempo/pull/3350) (@rubenvp8510)
+* [ENHANCEMENT] Add tracing integration to profiling endpoints [#3276](https://github.com/grafana/tempo/pull/3276) (@cyriltovena)
+* [ENHANCEMENT] Introduced `AttributePolicyMatch` & `IntrinsicPolicyMatch` structures to match span attributes based on strongly typed values & precompiled regexp [#3025](https://github.com/grafana/tempo/pull/3025) (@andriusluk)
+* [ENHANCEMENT] Make the trace ID label name configurable for remote written exemplars [#3074](https://github.com/grafana/tempo/pull/3074)
+* [ENHANCEMENT] Update poller to make use of previous results and reduce backend load. [#2652](https://github.com/grafana/tempo/pull/2652) (@zalegrala)
+* [ENHANCEMENT] Improve TraceQL regex performance in certain queries. [#3139](https://github.com/grafana/tempo/pull/3139) (@joe-elliott)
+* [ENHANCEMENT] Improve TraceQL performance in complex queries. [#3113](https://github.com/grafana/tempo/pull/3113) (@joe-elliott)
+* [ENHANCEMENT] Added a `frontend-search` cache role for job search caching. [#3225](https://github.com/grafana/tempo/pull/3225) (@joe-elliott)
+* [ENHANCEMENT] Added a `parquet-page` cache role for page level caching. [#3196](https://github.com/grafana/tempo/pull/3196) (@joe-elliott)
+* [ENHANCEMENT] Update opentelemetry-collector-contrib dependency to the latest version, v0.89.0 [#3148](https://github.com/grafana/tempo/pull/3148) (@gebn)
+* [ENHANCEMENT] Add `--max-start-time` and `--min-start-time` flag to tempo-cli command `analyse blocks` [#3250](https://github.com/grafana/tempo/pull/3250) (@mapno)
+* [ENHANCEMENT] Add per-tenant configurable remote_write headers to metrics-generator [#3175](https://github.com/grafana/tempo/pull/3175) (@mapno)
+* [ENHANCEMENT] Add variable expansion support to overrides configuration [#3175](https://github.com/grafana/tempo/pull/3175) (@mapno)
+* [ENHANCEMENT] Update memcached default image in jsonnet for multiple CVE [#3310](https://github.com/grafana/tempo/pull/3310) (@zalegrala)
+* [ENHANCEMENT] Add HTML pages /status/overrides and /status/overrides/{tenant} [#3244](https://github.com/grafana/tempo/pull/3244) [#3332](https://github.com/grafana/tempo/pull/3332) (@kvrhdn)
+* [ENHANCEMENT] Precalculate and reuse the vParquet3 schema before opening blocks [#3367](https://github.com/grafana/tempo/pull/3367) (@stoewer)
+* [ENHANCEMENT] Config: Adds `query-frontend.log-query-request-headers` to enable logging of request headers in query logs. [#3383](https://github.com/grafana/tempo/pull/3383) (@jmichalek132)
+* [ENHANCEMENT] Add `--shutdown-delay` to allow Tempo to cleanly drain connections. [#3395](https://github.com/grafana/tempo/pull/3395) (@joe-elliott)
+* [ENHANCEMENT] Introduce localblocks process config option to select only server spans 3303<https://github.com/grafana/tempo/pull/3303> (@zalegrala)
+* [ENHANCEMENT] TraceQL/Structural operators performance improvement. [#3088](https://github.com/grafana/tempo/pull/3088) (@joe-elliott)
+* [ENHANCEMENT] Localblocks processor honor tenant max trace size limit [3305](https://github.com/grafana/tempo/pull/3305) (@mdisibio)
+* [ENHANCEMENT] Introduce list_blocks_concurrency on GCS and S3 backends to control backend load and performance. [#2652](https://github.com/grafana/tempo/pull/2652) (@zalegrala)
+* [ENHANCEMENT] Add per-tenant compaction window [#3129](https://github.com/grafana/tempo/pull/3129) (@zalegrala)
+* [BUGFIX] Fix parsing of span.resource.xyz attributes in TraceQL. [#3284](https://github.com/grafana/tempo/pull/3284) (@mghildiy)
+* [BUGFIX] Change exit code if config is successfully verified [#3174](https://github.com/grafana/tempo/pull/3174) (@am3o @agrib-01)
+* [BUGFIX] The tempo-cli analyse blocks command no longer fails on compacted blocks [#3183](https://github.com/grafana/tempo/pull/3183) (@stoewer)
+* [BUGFIX] Move waitgroup handling for poller error condition [#3224](https://github.com/grafana/tempo/pull/3224) (@zalegrala)
+* [BUGFIX] Fix head block excessive locking in ingester search [#3328](https://github.com/grafana/tempo/pull/3328) (@mdisibio)
+* [BUGFIX] Fix issue with ingester failed to cut traces no such file or directory [#3346](https://github.com/grafana/tempo/issues/3346) (@mdisibio)
+* [BUGFIX] Restore `tempo_request_duration_seconds` metrics for `querier_api_*` requests [#3403](https://github.com/grafana/tempo/pull/3403) (@kvrhdn)
+* [BUGFIX] Prevent building parquet iterators that would loop forever. [#3159](https://github.com/grafana/tempo/pull/3159) (@mapno)
+* [BUGFIX] Sanitize name in mapped dimensions in span-metrics processor [#3171](https://github.com/grafana/tempo/pull/3171) (@mapno)
+* [BUGFIX] Fixed an issue where cached footers were requested then ignored. [#3196](https://github.com/grafana/tempo/pull/3196) (@joe-elliott)
+* [BUGFIX] Fix panic in autocomplete when query condition had wrong type [#3277](https://github.com/grafana/tempo/pull/3277) (@mapno)
+* [BUGFIX] Fix TLS when GRPC is enabled on HTTP [#3300](https://github.com/grafana/tempo/pull/3300) (@joe-elliott)
+* [BUGFIX] Correctly return 400 when max limit is requested on search. [#3340](https://github.com/grafana/tempo/pull/3340) (@joe-elliott)
+* [BUGFIX] Fix autocomplete filters sometimes returning erroneous results. [#3339](https://github.com/grafana/tempo/pull/3339) (@joe-elliott)
+* [BUGFIX] Fixes trace context propagation between query-frontend and querier. [#3387](https://github.com/grafana/tempo/pull/3387) (@mapno)
+* [BUGFIX] Fix some instances where spanmetrics histograms could be inconsistent [#3412](https://github.com/grafana/tempo/pull/3412) (@mdisibio)
+
+## v2.3.1 / 2023-11-28
+
+* [BUGFIX] Include statusMessage intrinsic attribute in tag search. [#3084](https://github.com/grafana/tempo/pull/3084) (@rcrowe)
+* [BUGFIX] Fix compactor ignore configured S3 headers [#3149](https://github.com/grafana/tempo/pull/3154) (@Batkilin)
+* [BUGFIX] Readd session token to s3 credentials. [#3144](https://github.com/grafana/tempo/pull/3144) (@farodin91)
+
+## v2.3.0 / 2023-10-30
+
+* [CHANGE] Update Go to 1.21 [#2486](https://github.com/grafana/tempo/pull/2829) (@zalegrala)
+* [CHANGE] Moved the tempo_ingester_traces_created_total metric to be incremented when a trace is cut to the wal [#2884](https://github.com/grafana/tempo/pull/2884) (@joe-elliott)
+* [CHANGE] Upgrade from deprecated [azure-storage-blob-go](https://github.com/Azure/azure-storage-blob-go) SDK to [azure-sdk-for-go](https://github.com/Azure/azure-sdk-for-go) [#2835](https://github.com/grafana/tempo/issues/2835) (@LasseHels)
+* [CHANGE] Metrics summary API validate the requested time range [#2902](https://github.com/grafana/tempo/pull/2902) (@mdisibio)
+* [CHANGE] Restructure Azure backends into versioned backends.  Introduce `use_v2_sdk` config option for switching. [#2952](https://github.com/grafana/tempo/issues/2952) (@zalegrala)
+    v1: [azure-storage-blob-go](https://github.com/Azure/azure-storage-blob-go) original (now deprecated) SDK
+    v2: [azure-sdk-for-go](https://github.com/Azure/azure-sdk-for-go)
+* [CHANGE] Adjust trace size estimation to better honor row group size settings. [#3038](https://github.com/grafana/tempo/pull/3038) (@joe-elliott)
+* [CHANGE] Update alpine image version to 3.18 to patch CVE-2022-48174. [#3046](https://github.com/grafana/tempo/pull/3046) (@joe-elliott)
+* [CHANGE] Overrides module refactor [#2688](https://github.com/grafana/tempo/pull/2688) (@mapno)
+    Added new `defaults` block to the overrides' module. Overrides change to indented syntax.
+    Old config:
+
+```
+overrides:
+  ingestion_rate_strategy: local
+  ingestion_rate_limit_bytes: 12345
+  ingestion_burst_size_bytes: 67890
+  max_search_duration: 17s
+  forwarders: ['foo']
+  metrics_generator_processors: [service-graphs, span-metrics]
+```
+
+New config:
+
+```
+overrides:
+defaults:
+  ingestion:
+    rate_strategy: local
+    rate_limit_bytes: 12345
+    burst_size_bytes: 67890
+  read:
+    max_search_duration: 17s
+  forwarders: ['foo']
+  metrics_generator:
+    processors: [service-graphs, span-metrics]
+```
+
+* [CHANGE] Bump Jaeger query docker image to 1.50.0 [#2998](https://github.com/grafana/tempo/pull/2998) (@pavolloffay)
+* [FEATURE] New TraceQL structural operators ancestor (<<), parent (<) [#2877](https://github.com/grafana/tempo/pull/2877) (@kousikmitra)
+* [FEATURE] Add the `/api/status/buildinfo` endpoint [#2702](https://github.com/grafana/tempo/pull/2702) (@fabrizio-grafana)
+* [FEATURE] New encoding vParquet3 with support for dedicated attribute columns (@mapno, @stoewer) [#2649](https://github.com/grafana/tempo/pull/2649)
+* [FEATURE] Add filtering support to Generic Forwarding [#2742](https://github.com/grafana/tempo/pull/2742) (@Blinkuu)
+* [FEATURE] Add cli command to print out summary of large traces [#2775](https://github.com/grafana/tempo/pull/2775) (@ie-pham)
+* [FEATURE] Added not structural operators to TraceQL: !>, !<, and !~ [#2993](https://github.com/grafana/tempo/pull/2993) (@joe-elliott)
+* [ENHANCEMENT] Make metrics-generator ingestion slack per tenant [#2589](https://github.com/grafana/tempo/pull/2589) (@ie-pham)
+* [ENHANCEMENT] Support quoted attribute name in TraceQL [#3004](https://github.com/grafana/tempo/pull/3004) (@kousikmitra)
+* [ENHANCEMENT] Add support for searching by span status message using  `statusMessage` keyword [#2848](https://github.com/grafana/tempo/pull/2848) (@kousikmitra)
+* [ENHANCEMENT] Add block indexes to vParquet2 and vParquet3 to improve trace by ID lookup [#2697](https://github.com/grafana/tempo/pull/2697) (@mdisibio)
+* [ENHANCEMENT] Assert ingestion rate limits as early as possible [#2640](https://github.com/grafana/tempo/pull/2703) (@mghildiy)
+* [ENHANCEMENT] Add several metrics-generator fields to user-configurable overrides [#2711](https://github.com/grafana/tempo/pull/2711) (@kvrhdn)
+* [ENHANCEMENT] Update /api/metrics/summary to correctly handle missing attributes and improve performance of TraceQL `select()` queries. [#2765](https://github.com/grafana/tempo/pull/2765) (@mdisibio)
+* [ENHANCEMENT] Tempo CLI command to convert from vParquet2 -> 3. [#2828](https://github.com/grafana/tempo/pull/2828) (@joe-elliott)
+* [ENHANCEMENT] Add `TempoUserConfigurableOverridesReloadFailing` alert [#2784](https://github.com/grafana/tempo/pull/2784) (@kvrhdn)
+* [ENHANCEMENT] Add RootSpanName and RootServiceName to log about discarded spans [#2816](https://github.com/grafana/tempo/pull/2816) (@marcinginszt)
+* [ENHANCEMENT] Add `UserID` to log message about rate limiting [#2850](https://github.com/grafana/tempo/pull/2850) (@lshippy)
+* [ENHANCEMENT] Requests to Azure Blob Storage will now be retried once instead of zero times [#2835](https://github.com/grafana/tempo/issues/2835) (@LasseHels)
+* [ENHANCEMENT] Add span metrics filter policies to user-configurable overrides [#2906](https://github.com/grafana/tempo/pull/2906) (@rlankfo)
+* [ENHANCEMENT] Add collection-interval to metrics-generator config in user-configurable overrides [#2899](https://github.com/grafana/tempo/pull/2899) (@rlankfo)
+* [ENHANCEMENT] Enforce max trace size on the trace by id path. [#2935](https://github.com/grafana/tempo/issues/2935) (@joe-elliott)
+* [ENHANCEMENT] Add `target_info_excluded_dimensions` to user-config api [#2945](https://github.com/grafana/tempo/pull/2945) (@ie-pham)
+* [ENHANCEMENT] User-configurable overrides: add scope query parameter to return merged overrides for tenant [#2915](https://github.com/grafana/tempo/pull/2915) [#3018](https://github.com/grafana/tempo/pull/3018) (@kvrhdn)
+* [ENHANCEMENT] Add histogram buckets to metrics-generator config in user-configurable overrides [#2928](https://github.com/grafana/tempo/pull/2928) (@mar4uk)
+* [ENHANCEMENT] Adds websocket support for search streaming. [#2971](https://github.com/grafana/tempo/pull/2840) (@joe-elliott)
+* [ENHANCEMENT] Add new config block to distributors to produce debug metrics. [#3008](https://github.com/grafana/tempo/pull/3008) (@joe-elliott)
+   **Breaking Change** Removed deprecated config option: distributor.log_received_spans
+* [ENHANCEMENT] added a metrics generator config option to enable/disable X-Scope-OrgID headers on remote write. [#2974](https://github.com/grafana/tempo/pull/2974) (@vineetjp)
+* [ENHANCEMENT] Correctly return RetryInfo to Otel Collector/Grafana Agent on ResourceExhausted. This allows the agents to honor their own retry
+  settings. [#3019](https://github.com/grafana/tempo/pull/3019) (@joe-elliott)
+* [BUGFIX] Unescape tag names [#2894](https://github.com/grafana/tempo/pull/2894) (@fabrizio-grafana)
+* [BUGFIX] Load defaults for the internal server [#3041](https://github.com/grafana/tempo/pull/3041) (@rubenvp8510)
+* [BUGFIX] Fix pass-through to runtime overrides for FilterPolicies and TargetInfoExcludedDimensions [#3012](https://github.com/grafana/tempo/pull/3012) (@electron0zero)
+* [BUGFIX] Fix panic in metrics summary api [#2738](https://github.com/grafana/tempo/pull/2738) (@mdisibio)
+* [BUGFIX] Fix rare deadlock when uploading blocks to Azure Blob Storage [#2129](https://github.com/grafana/tempo/issues/2129) (@LasseHels)
+* [BUGFIX] Only search ingester blocks that fall within the request time range. [#2783](https://github.com/grafana/tempo/pull/2783) (@joe-elliott)
+* [BUGFIX] Align tempo_query_frontend_queries_total and tempo_query_frontend_queries_within_slo_total. [#2840](https://github.com/grafana/tempo/pull/2840) (@joe-elliott)
+  This query will now correctly tell you %age of requests that are within SLO:
+
+  ```
+  sum(rate(tempo_query_frontend_queries_within_slo_total{}[1m])) by (op)
+  /
+  sum(rate(tempo_query_frontend_queries_total{}[1m])) by (op)
+  ```
+
+  **BREAKING CHANGE** Removed: tempo_query_frontend_queries_total{op="searchtags|metrics"}.
+* [BUGFIX] To support blob storage in Azure Stack Hub as backend. [#2853](https://github.com/grafana/tempo/pull/2853) (@chlislb)
+* [BUGFIX] Respect spss on GRPC streaming. [#2971](https://github.com/grafana/tempo/pull/2840) (@joe-elliott)
+* [BUGFIX] Moved empty root span substitution from `querier` to `query-frontend`. [#2671](https://github.com/grafana/tempo/issues/2671) (@galalen)
+* [BUGFIX] Correctly propagate ingester errors on the query path [#2935](https://github.com/grafana/tempo/issues/2935) (@joe-elliott)
+* [BUGFIX] Fix issue where ingester doesn't stop query after timeout [#3031](https://github.com/grafana/tempo/pull/3031) (@mdisibio)
+* [BUGFIX] Fix cases where empty filter {} wouldn't return expected results [#2498](https://github.com/grafana/tempo/issues/2498) (@mdisibio)
+* [BUGFIX] Reorder S3 credential chain and upgrade minio-go. `native_aws_auth_enabled` is deprecated [#3006](https://github.com/grafana/tempo/pull/3006) (@ekristen, @mapno)
+* [BUGFIX] Update parquet-go dependency including a bugfix that prevents corrupted blocks from being written [#3068](https://github.com/grafana/tempo/pull/3068) (@stoewer)
+
+# v2.2.4 / 2023-10-25
+
+* [CHANGE] Update alpine image version to 3.18 to patch CVE-2022-48174. [#3046](https://github.com/grafana/tempo/pull/3046) (@joe-elliott)
+* [CHANGE] Bump Jaeger query docker image to 1.50.0 [#2998](https://github.com/grafana/tempo/pull/2998) (@pavolloffay)
+
+# v2.2.3 / 2023-09-13
+
+* [BUGFIX] Fix S3 credentials providers configuration [#2889](https://github.com/grafana/tempo/pull/2889) (@mapno)
+
+# v2.2.2 / 2023-08-30
+
+* [BUGFIX] Fix node role auth IDMSv1 [#2760](https://github.com/grafana/tempo/pull/2760) (@coufalja)
+
+# v2.2.1 / 2023-08-21
+
+* [BUGFIX] Fix incorrect metrics for index failures [#2781](https://github.com/grafana/tempo/pull/2781) (@zalegrala)
+* [BUGFIX] Fix panic in the metrics-generator when using multiple tenants with default overrides [#2786](https://github.com/grafana/tempo/pull/2786) (@kvrhdn)
+* [BUGFIX] Restore `tenant_header_key` removed in #2414. [#2795](https://github.com/grafana/tempo/pull/2795) (@joe-elliott)
+* [BUGFIX] Disable streaming over http by default. [#2803](https://github.com/grafana/tempo/pull/2803) (@joe-elliott)
+
+## v2.2.0 / 2023-07-31

 * [CHANGE] Make vParquet2 the default block format [#2526](https://github.com/grafana/tempo/pull/2526) (@stoewer)
 * [CHANGE] Disable tempo-query by default in Jsonnet libs. [#2462](https://github.com/grafana/tempo/pull/2462) (@electron0zero)
@@ -9,29 +277,33 @@
 * [CHANGE] Remove `tenant_header_key` option from `tempo-query` config [#2414](https://github.com/grafana/tempo/pull/2414) (@kousikmitra)
 * [CHANGE] **Breaking Change** Remove support tolerate_failed_blocks. [#2416](https://github.com/grafana/tempo/pull/2416) (@joe-elliott)
  Removed config option:
+
  ```
  query_frontend:
    tolerate_failed_blocks: <int>
  ```
+
 * [CHANGE] Upgrade memcached version in jsonnet microservices [#2466](https://github.com/grafana/tempo/pull/2466) (@zalegrala)
 * [CHANGE] Prefix service graph extra dimensions labels with `server_` and `client_` if `enable_client_server_prefix` is enabled [#2335](https://github.com/grafana/tempo/pull/2335) (@domasx2)
 * [CHANGE] **Breaking Change** Rename s3.insecure_skip_verify [#2407](https://github.com/grafana/tempo/pull/2407) (@zalegrala)
+
 ```yaml
 storage:
  trace:
    s3:
      insecure_skip_verify: true   // renamed to tls_insecure_skip_verify
 ```
+
 * [CHANGE] Ignore context canceled errors in the queriers [#2440](https://github.com/grafana/tempo/pull/2440) (@joe-elliott)
 * [CHANGE] Start flush queue worker after wal replay and block rediscovery [#2456](https://github.com/grafana/tempo/pull/2456) (@ie-pham)
 * [CHANGE] Update Go to 1.20.4 [#2486](https://github.com/grafana/tempo/pull/2486) (@ie-pham)
 * [CHANGE] **Breaking Change** Convert metrics generator from deployment to a statefulset in jsonnet. Refer to the PR for seamless migration instructions. [#2533](https://github.com/grafana/tempo/pull/2533) [#2467](https://github.com/grafana/tempo/pull/2647) (@zalegrala)
 * [FEATURE] New experimental API to derive on-demand RED metrics grouped by any attribute, and new metrics generator processor [#2368](https://github.com/grafana/tempo/pull/2368) [#2418](https://github.com/grafana/tempo/pull/2418) [#2424](https://github.com/grafana/tempo/pull/2424) [#2442](https://github.com/grafana/tempo/pull/2442) [#2480](https://github.com/grafana/tempo/pull/2480) [#2481](https://github.com/grafana/tempo/pull/2481) [#2501](https://github.com/grafana/tempo/pull/2501) [#2579](https://github.com/grafana/tempo/pull/2579) [#2582](https://github.com/grafana/tempo/pull/2582) (@mdisibio @zalegrala)
 * [FEATURE] New TraceQL structural operators descendant (>>), child (>), and sibling (~) [#2625](https://github.com/grafana/tempo/pull/2625) [#2660](https://github.com/grafana/tempo/pull/2660) (@mdisibio)
-* [FEATURE] Add user-configurable overrides module [#2543](https://github.com/grafana/tempo/pull/2543) (@electron0zero @kvrhdn)
+* [FEATURE] Add user-configurable overrides module [#2543](https://github.com/grafana/tempo/pull/2543) [#2682](https://github.com/grafana/tempo/pull/2682) [#2681](https://github.com/grafana/tempo/pull/2681) (@electron0zero @kvrhdn)
 * [FEATURE] Add support for `q` query param in `/api/v2/search/<tag.name>/values` to filter results based on a TraceQL query [#2253](https://github.com/grafana/tempo/pull/2253) (@mapno)
 To make use of filtering, configure `autocomplete_filtering_enabled`.
-* [FEATURE] Add support for `by()` and `coalesce()` to TraceQL. [#2490](https://github.com/grafana/tempo/pull/2490) 
+* [FEATURE] Add support for `by()` and `coalesce()` to TraceQL. [#2490](https://github.com/grafana/tempo/pull/2490)
 * [FEATURE] Add a GRPC streaming endpoint for traceql search [#2366](https://github.com/grafana/tempo/pull/2366) (@joe-elliott)
 * [FEATURE] Add new API to summarize span metrics from generators [#2481](https://github.com/grafana/tempo/pull/2481) (@zalegrala)
 * [FEATURE] Add `select()` to TraceQL [#2494](https://github.com/grafana/tempo/pull/2494) (@joe-elliott)
@@ -51,18 +323,22 @@ To make use of filtering, configure `autocomplete_filtering_enabled`.
 * [ENHANCEMENT] Add `prefix` configuration option to `storage.trace.azure` and `storage.trace.gcs` [#2386](https://github.com/grafana/tempo/pull/2386) (@kousikmitra)
 * [ENHANCEMENT] Add `prefix` configuration option to `storage.trace.s3` [#2362](https://github.com/grafana/tempo/pull/2362) (@kousikmitra)
 * [ENHANCEMENT] Add support for `concurrent_shards` under `trace_by_id` [#2416](https://github.com/grafana/tempo/pull/2416) (@joe-elliott)
+
  ```
  query_frontend:
    trace_by_id:
      concurrent_shards: 3
  ```
+
 * [ENHANCEMENT] Enable cross cluster querying by adding two config options. [#2598](https://github.com/grafana/tempo/pull/2598) (@joe-elliott)
+
  ```
  querier:
    secondary_ingester_ring: <string>
  metrics_generator:
    override_ring_key: <string>
  ```
+
 * [ENHANCEMENT] Add `scope` parameter to `/api/search/tags` [#2282](https://github.com/grafana/tempo/pull/2282) (@joe-elliott)
  Create new endpoint `/api/v2/search/tags` that returns all tags organized by scope.
 * [ENHANCEMENT] Ability to toggle off latency or count metrics in metrics-generator [#2070](https://github.com/grafana/tempo/pull/2070) (@AlexDHoffer)
@@ -78,28 +354,33 @@ To make use of filtering, configure `autocomplete_filtering_enabled`.
 * [BUGFIX] Fix Search SLO by routing tags to a new handler. [#2468](https://github.com/grafana/tempo/issues/2468) (@electron0zero)
 * [BUGFIX] tempodb integer divide by zero error [#2167](https://github.com/grafana/tempo/issues/2167) (@kroksys)
 * [BUGFIX] metrics-generator: ensure Prometheus will scale up shards when remote write is lagging behind [#2463](https://github.com/grafana/tempo/issues/2463) (@kvrhdn)
-* [BUGFIX] Fixes issue where matches and other spanset level attributes were not persisted to the TraceQL results. [#2490](https://github.com/grafana/tempo/pull/2490) 
+* [BUGFIX] Fixes issue where matches and other spanset level attributes were not persisted to the TraceQL results. [#2490](https://github.com/grafana/tempo/pull/2490)
 * [BUGFIX] Fixes issue where ingester search could occasionally fail with file does not exist error [#2534](https://github.com/grafana/tempo/issues/2534) (@mdisibio)
 * [BUGFIX] Tempo failed to find meta.json path after adding prefix in S3/GCS/Azure configuration. [#2585](https://github.com/grafana/tempo/issues/2585) (@WildCatFish)
 * [BUGFIX] Delay logging config warnings until the logger has been initialized [#2645](https://github.com/grafana/tempo/pull/2645) (@kvrhdn)
 * [BUGFIX] Fix issue where metrics-generator was setting wrong labels for traces_target_info [#2546](https://github.com/grafana/tempo/pull/2546) (@ie-pham)
+* [FEATURE] Add `tempo-cli` commands `analyse block` and `analyse blocks` to analyse parquet blocks and output summaries of generic attribute columns [#2622](https://github.com/grafana/tempo/pull/2622) (@mapno)

 ## v2.1.1 / 2023-04-28
+
 * [BUGFIX] Fix issue where Tempo sometimes flips booleans from false->true at storage time. [#2400](https://github.com/grafana/tempo/issues/2400) (@joe-elliott)

 ## v2.1.0 / 2023-04-26
+
 * [CHANGE] Capture and update search metrics for TraceQL [#2087](https://github.com/grafana/tempo/pull/2087) (@electron0zero)
 * [CHANGE] tempo-mixin: disable auto refresh every 10 seconds [#2290](https://github.com/grafana/tempo/pull/2290) (@electron0zero)
 * [CHANGE] Update tempo-mixin to show request in Resources dashboard [#2281](https://github.com/grafana/tempo/pull/2281) (@electron0zero)
 * [CHANGE] Add support for s3 session token in static config [#2093](https://github.com/grafana/tempo/pull/2093) (@farodin91)
 * [CHANGE] **Breaking Change** Remove support for search on v2 blocks. [#2159](https://github.com/grafana/tempo/pull/2159) (@joe-elliott)
  Removed config options:
+
  ```
  overrides:
    max_search_bytes_per_trace:
    search_tags_allow_list:
    search_tags_deny_list:
  ```
+
  Removed metrics:
  `tempo_ingester_trace_search_bytes_discarded_total`
 * [CHANGE] Stop caching parquet files for search [#2164](https://github.com/grafana/tempo/pull/2164) (@mapno)
@@ -113,7 +394,7 @@ To make use of filtering, configure `autocomplete_filtering_enabled`.
 * [FEATURE] Add support for span `kind` to TraceQL [#2217](https://github.com/grafana/tempo/pull/2217) (@joe-elliott)
 * [FEATURE] Add support for min/max/avg aggregates to TraceQL[#2255](https://github.com/grafana/tempo/pull/2255) (@joe-elliott)
 * [ENHANCEMENT] Add Throughput and SLO Metrics with SLOConfig in Query Frontend [#2008](https://github.com/grafana/tempo/pull/2008) (@electron0zero)
-  - **BREAKING CHANGE** `query_frontend_result_metrics_inspected_bytes` metric removed in favour of `query_frontend_bytes_processed_per_second`
+  * **BREAKING CHANGE** `query_frontend_result_metrics_inspected_bytes` metric removed in favour of `query_frontend_bytes_processed_per_second`
 * [ENHANCEMENT] Metrics generator to make use of counters earlier [#2068](https://github.com/grafana/tempo/pull/2068) (@zalegrala)
 * [ENHANCEMENT] Log when a trace is too large to compact [#2105](https://github.com/grafana/tempo/pull/2105) (@scalalang2)
 * [ENHANCEMENT] Add support for arbitrary arithemtic to TraceQL queries [#2146](https://github.com/grafana/tempo/pull/2146) (@joe-elliott)
@@ -147,6 +428,7 @@ To make use of filtering, configure `autocomplete_filtering_enabled`.

 * [CHANGE] **BREAKING CHANGE** Use snake case on Azure Storage config [#1879](https://github.com/grafana/tempo/issues/1879) (@faustodavid)
 Example of using snake case on Azure Storage config:
+
 ```
 # config.yaml
 storage:
@@ -155,6 +437,7 @@ storage:
    storage_account_key:
    container_name:
 ```
+
 * [CHANGE] Increase default values for `server.grpc_server_max_recv_msg_size` and `server.grpc_server_max_send_msg_size` from 4MB to 16MB [#1688](https://github.com/grafana/tempo/pull/1688) (@mapno)
 * [CHANGE] Propagate Ingesters search errors correctly [#2023](https://github.com/grafana/tempo/pull/2023) (@electron0zero)
 * [CHANGE] **BREAKING CHANGE** Use storage.trace.block.row_group_size_bytes to cut rows during compaction instead of
@@ -165,7 +448,7 @@ storage:
 Old config will still work but will be removed in a future release. [#1735](https://github.com/grafana/tempo/pull/1735) (@mapno)
 * [CHANGE] Update alpine image version to 3.16. [#1784](https://github.com/grafana/tempo/pull/1784) (@zalegrala)
 * [CHANGE] Delete TempoRequestErrors alert from mixin [#1810](https://github.com/grafana/tempo/pull/1810) (@zalegrala)
-  - **BREAKING CHANGE** Any jsonnet users relying on this alert should copy this into their own environment.
+  * **BREAKING CHANGE** Any jsonnet users relying on this alert should copy this into their own environment.
 * [CHANGE] Update and replace a few go modules [#1945](https://github.com/grafana/tempo/pull/1945) (@zalegrala)
  * Replace `github.com/thanos-io/thanos/pkg/discovery/dns` use with `github.com/grafana/dskit/dns`
  * Upgrade `github.com/grafana/dskit`
@@ -173,6 +456,7 @@ Old config will still work but will be removed in a future release. [#1735](http
  * Upgrade `github.com/minio/minio-go/v7`
 * [CHANGE] Config updates to prepare for Tempo 2.0. [#1978](https://github.com/grafana/tempo/pull/1978) (@joe-elliott)
  Defaults updated:
+
  ```
  query_frontend:
    max_oustanding_per_tenant: 2000
@@ -200,7 +484,9 @@ Old config will still work but will be removed in a future release. [#1735](http
              read_buffer_count: 32
              read_buffer_size_bytes: 1048576
  ```
+
  **BREAKING CHANGE** Renamed/removed/moved
+
  ```
  query_frontend:
    query_shards:                  // removed. use trace_by_id.query_shards
@@ -223,12 +509,14 @@ Old config will still work but will be removed in a future release. [#1735](http
          encoding:                // renamed to v2_encoding
          row_group_size_bytes:    // renamed to parquet_row_group_size_bytes
  ```
+
 * [CHANGE] **BREAKING CHANGE** Remove `search_enabled` and `metrics_generator_enabled`. Both default to true. [#2004](https://github.com/grafana/tempo/pull/2004) (@joe-elliott)
 * [CHANGE] Update OTel collector to v0.57.2 [#1757](https://github.com/grafana/tempo/pull/1757) (@mapno)
-* [FEATURE] TraceQL support https://grafana.com/docs/tempo/latest/traceql/
+* [FEATURE] TraceQL support <https://grafana.com/docs/tempo/latest/traceql/>
 * [FEATURE] Parquet backend is GA and default
 * [FEATURE] Add generic forwarder and implement otlpgrpc forwarder [#1775](https://github.com/grafana/tempo/pull/1775) (@Blinkuu)
    New config options and example configuration:
+
 ```
 # config.yaml
 distributor:
@@ -247,11 +535,13 @@ overrides:
  "example-tenant-2":
    forwarders: ['otel-forwarder']
 ```
+
 * [ENHANCEMENT] Add support for TraceQL in Parquet WAL and Local Blocks. [#1966](https://github.com/grafana/tempo/pull/1966) (@electron0zero)
 * [ENHANCEMENT] Add `/status/usage-stats` endpoint to show usage stats data [#1782](https://github.com/grafana/tempo/pull/1782) (@electron0zero)
 * [ENHANCEMENT] Add TLS support to jaeger query plugin. [#1999](https://github.com/grafana/tempo/pull/1999) (@rubenvp8510)
 * [ENHANCEMENT] Collect inspectedBytes from SearchMetrics [#1975](https://github.com/grafana/tempo/pull/1975) (@electron0zero)
 * [ENHANCEMENT] Add zone awareness replication for ingesters. [#1936](https://github.com/grafana/tempo/pull/1936) (@manohar-koukuntla)
+
 ```
 # use the following fields in _config field of jsonnet config, to enable zone aware ingester
    multi_zone_ingester_enabled: false,
@@ -259,12 +549,13 @@ overrides:
    multi_zone_ingester_replicas: 0,
    multi_zone_ingester_max_unavailable: 25,
 ```
+
 * [ENHANCEMENT] Support global and wildcard overrides in generic forwarder feature [#1871](https://github.com/grafana/tempo/pull/1871) (@Blinkuu)
 * [ENHANCEMENT] Add new data-type aware searchtagvalues v2 api [#1956](https://github.com/grafana/tempo/pull/1956) (@mdisibio)
 * [ENHANCEMENT] Refactor queueManager into generic queue.Queue [#1796](https://github.com/grafana/tempo/pull/1796) (@Blinkuu)
-  - **BREAKING CHANGE** Rename `tempo_distributor_forwarder_queue_length` metric to `tempo_distributor_queue_length`. New metric has two custom labels: `name` and  `tenant`.
-  - Deprecated `tempo_distributor_forwarder_pushes_total` metric in favor of `tempo_distributor_queue_pushes_total`.
-  - Deprecated `tempo_distributor_forwarder_pushes_failures_total` metric in favor of `tempo_distributor_queue_pushes_failures_total`.
+  * **BREAKING CHANGE** Rename `tempo_distributor_forwarder_queue_length` metric to `tempo_distributor_queue_length`. New metric has two custom labels: `name` and  `tenant`.
+  * Deprecated `tempo_distributor_forwarder_pushes_total` metric in favor of `tempo_distributor_queue_pushes_total`.
+  * Deprecated `tempo_distributor_forwarder_pushes_failures_total` metric in favor of `tempo_distributor_queue_pushes_failures_total`.
 * [ENHANCEMENT] Filter namespace by cluster in tempo dashboards variables [#1771](https://github.com/grafana/tempo/pull/1771) (@electron0zero)
 * [ENHANCEMENT] Exit early from sharded search requests [#1742](https://github.com/grafana/tempo/pull/1742) (@electron0zero)
 * [ENHANCEMENT] Upgrade prometheus/prometheus to `51a44e6657c3` [#1829](https://github.com/grafana/tempo/pull/1829) (@mapno)
@@ -276,7 +567,7 @@ overrides:
 * [ENHANCEMENT] Improve parquet compaction memory profile when dropping spans [#1692](https://github.com/grafana/tempo/pull/1692) (@joe-elliott)
 * [ENHANCEMENT] Use Parquet for local block search, tag search and tag value search instead of flatbuffers. A configuration value
  (`ingester.use_flatbuffer_search`) is provided to continue using flatbuffers.
-  - **BREAKING CHANGE** Makes Parquet the default encoding.
+  * **BREAKING CHANGE** Makes Parquet the default encoding.
 * [ENHANCEMENT] Return 200 instead of 206 when blocks failed is < tolerate_failed_blocks. [#1725](https://github.com/grafana/tempo/pull/1725) (@joe-elliott)
 * [ENHANCEMENT] Add GOMEMLIMIT variable to compactor jsonnet and set the value to equal compactor memory limit. [#1758](https://github.com/grafana/tempo/pull/1758/files) (@ie-pham)
 * [ENHANCEMENT] Add capability to configure the used S3 Storage Class [#1697](https://github.com/grafana/tempo/pull/1714) (@amitsetty)
@@ -289,14 +580,16 @@ overrides:
 * [ENHANCEMENT] metrics-generator: filter out older spans before metrics are aggregated [#1612](https://github.com/grafana/tempo/pull/1612) (@ie-pham)
 * [ENHANCEMENT] Add hedging to trace by ID lookups created by the frontend. [#1735](https://github.com/grafana/tempo/pull/1735) (@mapno)
    New config options and defaults:
+
 ```
 query_frontend:
  trace_by_id:
    hedge_requests_at: 5s
    hedge_requests_up_to: 3
 ```
+
 * [ENHANCEMENT] Vulture now has improved distribution of the random traces it searches. [#1763](https://github.com/grafana/tempo/pull/1763) (@rfratto)
-* [ENHANCEMENT] Upgrade opentelemetry-proto submodule to v0.18.0 Internal types are updated to use `scope` instead of `instrumentation_library`. 
+* [ENHANCEMENT] Upgrade opentelemetry-proto submodule to v0.18.0 Internal types are updated to use `scope` instead of `instrumentation_library`.
                This is a breaking change in trace by ID queries if JSON is requested. [#1754](https://github.com/grafana/tempo/pull/1754) (@mapno)
 * [ENHANCEMENT] Add TLS support to the vulture [#1874](https://github.com/grafana/tempo/pull/1874) (@zalegrala)
 * [ENHANCEMENT] metrics-generator: extract `status_message` field from spans [#1786](https://github.com/grafana/tempo/pull/1786), [#1794](https://github.com/grafana/tempo/pull/1794) (@stoewer)
@@ -332,24 +625,31 @@ query_frontend:
 * [CHANGE] **BREAKING CHANGE** Change spanmetrics metric names and labels to match OTel conventions. [#1478](https://github.com/grafana/tempo/pull/1478) (@mapno)
 * [FEATURE] Add support for time picker in jaeger query plugin. [#1631](https://github.com/grafana/tempo/pull/1631) (@rubenvp8510)
 Old metric names:
+
 ```
 traces_spanmetrics_duration_seconds_{sum,count,bucket}
 ```
+
 New metric names:
+
 ```
 traces_spanmetrics_latency_{sum,count,bucket}
 ```
+
 Additionally, default label `span_status` is renamed to `status_code`.
+
 * [CHANGE] Update to Go 1.18 [#1504](https://github.com/grafana/tempo/pull/1504) (@annanay25)
 * [CHANGE] Change tag/value lookups to return partial results when reaching response size limit instead of failing [#1517](https://github.com/grafana/tempo/pull/1517) (@mdisibio)
 * [CHANGE] Change search to be case-sensitive [#1547](https://github.com/grafana/tempo/issues/1547) (@mdisibio)
 * [CHANGE] Relax Hedged request defaults for external endpoints. [#1566](https://github.com/grafana/tempo/pull/1566) (@joe-elliott)
+
  ```
  querier:
    search:
      external_hedge_requests_at: 4s    -> 8s
      external_hedge_requests_up_to: 3  -> 2
  ```
+
 * [CHANGE] **BREAKING CHANGE** Include emptyDir for metrics generator wal storage in jsonnet [#1556](https://github.com/grafana/tempo/pull/1556) (@zalegrala)
 Jsonnet users will now need to specify a storage request and limit for the generator wal.
    _config+:: {
@@ -371,7 +671,7 @@ Jsonnet users will now need to specify a storage request and limit for the gener
 * [ENHANCEMENT] Add metric to track feature enablement [#1459](https://github.com/grafana/tempo/pull/1459) (@zalegrala)
 * [ENHANCEMENT] Added s3 config option `insecure_skip_verify` [#1470](https://github.com/grafana/tempo/pull/1470) (@zalegrala)
 * [ENHANCEMENT] Added polling option to reduce issues in Azure `blocklist_poll_jitter_ms` [#1518](https://github.com/grafana/tempo/pull/1518) (@joe-elliott)
-* [ENHANCEMENT] Add a config to query single ingester instance based on trace id hash for Trace By ID API. (1484)[https://github.com/grafana/tempo/pull/1484] (@sagarwala, @bikashmishra100, @ashwinidulams)
+* [ENHANCEMENT] Add a config to query single ingester instance based on trace id hash for Trace By ID API. [1484](https://github.com/grafana/tempo/pull/1484) (@sagarwala, @bikashmishra100, @ashwinidulams)
 * [ENHANCEMENT] Add blocklist metrics for total backend objects and total backend bytes [#1519](https://github.com/grafana/tempo/pull/1519) (@ie-pham)
 * [ENHANCEMENT] Adds `tempo_querier_external_endpoint_hedged_roundtrips_total` to count the total hedged requests [#1558](https://github.com/grafana/tempo/pull/1558) (@joe-elliott)
  **BREAKING CHANGE** Removed deprecated metrics `tempodb_(gcs|s3|azure)_request_duration_seconds` in favor of `tempodb_backend_request_duration_seconds`. These metrics
@@ -429,21 +729,26 @@ Jsonnet users will now need to specify a storage request and limit for the gener
 * [ENHANCEMENT] Add forwarding queue from distributor to metrics-generator. [#1331](https://github.com/grafana/tempo/pull/1331) (@mapno)
 * [ENHANCEMENT] Add hedging to queries to external endpoints. [#1350](https://github.com/grafana/tempo/pull/1350) (@joe-elliott)
  New config options and defaults:
+
  ```
  querier:
    search:
      external_hedge_requests_at: 5s
      external_hedge_requests_up_to: 3
  ```
-  ** BREAKING CHANGE **
+
+  **BREAKING CHANGE**
  Querier options related to search have moved under a `search` block:
+
  ```
  querier:
   search_query_timeout: 30s
   search_external_endpoints: []
   search_prefer_self: 2
  ```
+
  becomes
+
  ```
  querier:
    search:
@@ -451,6 +756,7 @@ Jsonnet users will now need to specify a storage request and limit for the gener
      prefer_self: 2
      external_endpoints: []
  ```
+
 * [ENHANCEMENT] Added tenant ID (instance ID) to `trace too large message`. [#1385](https://github.com/grafana/tempo/pull/1385) (@cristiangsp)
 * [ENHANCEMENT] Add a startTime and endTime parameter to the Trace by ID Tempo Query API to improve query performance [#1388](https://github.com/grafana/tempo/pull/1388) (@sagarwala, @bikashmishra100, @ashwinidulams)
 * [BUGFIX] Correct issue where Azure "Blob Not Found" errors were sometimes not handled correctly [#1390](https://github.com/grafana/tempo/pull/1390) (@joe-elliott)
@@ -460,24 +766,29 @@ Jsonnet users will now need to specify a storage request and limit for the gener
 * [BUGFIX] Update OTLP port in examples (docker-compose & kubernetes) from legacy ports (55680/55681) to new ports (4317/4318) [#1294](https://github.com/grafana/tempo/pull/1294) (@mapno)
 * [BUGFIX] Fixes min/max time on blocks to be based on span times instead of ingestion time. [#1314](https://github.com/grafana/tempo/pull/1314) (@joe-elliott)
  * Includes new configuration option to restrict the amount of slack around now to update the block start/end time. [#1332](https://github.com/grafana/tempo/pull/1332) (@joe-elliott)
+
    ```
    storage:
      trace:
        wal:
          ingestion_time_range_slack: 2m0s
    ```
+
  * Includes a new metric to determine how often this range is exceeded: `tempo_warnings_total{reason="outside_ingestion_time_slack"}`
 * [BUGFIX] Prevent data race / ingester crash during searching by trace id by using xxhash instance as a local variable. [#1387](https://github.com/grafana/tempo/pull/1387) (@bikashmishra100, @sagarwala, @ashwinidulams)
 * [BUGFIX] Fix spurious "failed to mark block compacted during retention" errors [#1372](https://github.com/grafana/tempo/issues/1372) (@mdisibio)
 * [BUGFIX] Fix error message "Writer is closed" by resetting compression writer correctly on the error path. [#1379](https://github.com/grafana/tempo/issues/1379) (@annanay25)

 ## v1.3.2 / 2022-02-23
+
 * [BUGFIX] Fixed an issue where the query-frontend would corrupt start/end time ranges on searches which included the ingesters [#1295] (@joe-elliott)

 ## v1.3.1 / 2022-02-02
+
 * [BUGFIX] Fixed panic when using etcd as ring's kvstore [#1260](https://github.com/grafana/tempo/pull/1260) (@mapno)

 ## v1.3.0 / 2022-01-24
+
 * [FEATURE]: Add support for [inline environments](https://tanka.dev/inline-environments). [#1184](https://github.com/grafana/tempo/pull/1184) @irizzant
 * [CHANGE] Search: Add new per-tenant limit `max_bytes_per_tag_values_query` to limit the size of tag-values response. [#1068](https://github.com/grafana/tempo/pull/1068) (@annanay25)
 * [CHANGE] Reduce MaxSearchBytesPerTrace `ingester.max-search-bytes-per-trace` default to 5KB [#1129](https://github.com/grafana/tempo/pull/1129) @annanay25
@@ -520,15 +831,17 @@ Jsonnet users will now need to specify a storage request and limit for the gener
 * [BUGFIX] tempo-mixin: remove TempoDB Access panel from `Tempo / Reads`, metrics don't exist anymore [#1218](https://github.com/grafana/tempo/issues/1218) (@kvrhdn)

 ## v1.2.1 / 2021-11-15
+
 * [BUGFIX] Fix defaults for MaxBytesPerTrace (ingester.max-bytes-per-trace) and MaxSearchBytesPerTrace (ingester.max-search-bytes-per-trace) [#1109](https://github.com/grafana/tempo/pull/1109) (@bitprocessor)
 * [BUGFIX] Ignore empty objects during compaction [#1113](https://github.com/grafana/tempo/pull/1113) (@mdisibio)

 ## v1.2.0 / 2021-11-05
+
 * [CHANGE] **BREAKING CHANGE** Drop support for v0 and v1 blocks. See [1.1 changelog](https://github.com/grafana/tempo/releases/tag/v1.1.0) for details [#919](https://github.com/grafana/tempo/pull/919) (@joe-elliott)
 * [CHANGE] Renamed CLI flag from `--storage.trace.maintenance-cycle` to `--storage.trace.blocklist_poll`. This is a **breaking change**  [#897](https://github.com/grafana/tempo/pull/897) (@mritunjaysharma394)
 * [CHANGE] update jsonnet alerts and recording rules to use `job_selectors` and `cluster_selectors` for configurable unique identifier labels [#935](https://github.com/grafana/tempo/pull/935) (@kevinschoonover)
 * [CHANGE] Modify generated tag keys in Vulture for easier filtering [#934](https://github.com/grafana/tempo/pull/934) (@zalegrala)
-* [CHANGE] **BREAKING CHANGE** Consolidate status information onto /status endpoint [ #952 ](https://github.com/grafana/tempo/pull/952) @zalegrala)
+* [CHANGE] **BREAKING CHANGE** Consolidate status information onto /status endpoint [#952](https://github.com/grafana/tempo/pull/952) @zalegrala)
  The following endpoints moved.
  `/runtime_config` moved to `/status/runtime_config`
  `/config` moved to `/status/config`
@@ -549,6 +862,7 @@ Jsonnet users will now need to specify a storage request and limit for the gener
 * [ENHANCEMENT] Make `overrides_config` block name consistent with Loki and Cortex in microservice mode. [#906](https://github.com/grafana/tempo/pull/906) (@kavirajk)
 * [ENHANCEMENT] Changes the metrics name from `cortex_runtime_config_last_reload_successful` to `tempo_runtime_config_last_reload_successful` [#945](https://github.com/grafana/tempo/pull/945) (@kavirajk)
 * [ENHANCEMENT] Updated config defaults to reflect better capture operational knowledge. [#913](https://github.com/grafana/tempo/pull/913) (@joe-elliott)
+
  ```
  ingester:
    trace_idle_period: 30s => 10s  # reduce ingester memory requirements with little impact on querying
@@ -567,6 +881,7 @@ Jsonnet users will now need to specify a storage request and limit for the gener
      chunk_size_bytes: 10 MiB => 5 MiB  # will reduce compactor memory needs
      compaction_window: 4h => 1h        # will allow more compactors to participate in compaction without substantially increasing blocks
  ```
+
 * [ENHANCEMENT] Make s3 backend readError logic more robust [#905](https://github.com/grafana/tempo/pull/905) (@wei840222)
 * [ENHANCEMENT] Include additional detail when searching for traces [#916](https://github.com/grafana/tempo/pull/916) (@zalegrala)
 * [ENHANCEMENT] Add `gen index` and `gen bloom` commands to tempo-cli. [#903](https://github.com/grafana/tempo/pull/903) (@annanay25)
@@ -609,6 +924,7 @@ Jsonnet users will now need to specify a storage request and limit for the gener
 * [BUGFIX] Update go-kit logger package to remove spurious debug logs [#1094](https://github.com/grafana/tempo/pull/1094) (@bboreham)

 ## v1.1.0 / 2021-08-26
+
 * [CHANGE] Upgrade Cortex from v1.9.0 to v1.9.0-131-ga4bf10354 [#841](https://github.com/grafana/tempo/pull/841) (@aknuds1)
 * [CHANGE] Change default tempo port from 3100 to 3200 [#770](https://github.com/grafana/tempo/pull/809) (@MurzNN)
 * [CHANGE] Jsonnet: use dedicated configmaps for distributors and ingesters [#775](https://github.com/grafana/tempo/pull/775) (@kvrhdn)
@@ -644,7 +960,6 @@ Jsonnet users will now need to specify a storage request and limit for the gener
 * [BUGFIX] Update port spec for GCS docker-compose example [#869](https://github.com/grafana/tempo/pull/869) (@zalegrala)
 * [BUGFIX] Cortex upgrade to fix an issue where unhealthy compactors can't be forgotten [#878](https://github.com/grafana/tempo/pull/878) (@joe-elliott)

-
 ## v1.0.1 / 2021-06-14

 * [BUGFIX] Guard against negative dataLength [#763](https://github.com/grafana/tempo/pull/763) (@joe-elliott)
@@ -1,2 +1,25 @@
+# Lines starting with '#' are comments.
+# Each line is a file pattern followed by one or more owners.
+
+# More details are here: https://help.github.com/articles/about-codeowners/
+
+# The '*' pattern is global owners.
+
+# Order is important. The last matching pattern has the most precedence.
+# The folders are ordered as follows:
+
+# In each subsection folders are ordered first by depth, then alphabetically.
+# This should make it easy to add new rules without breaking existing ones.
+
 * @joe-elliott @annanay25 @mdisibio @mapno @kvrhdn @zalegrala @electron0zero @ie-pham @stoewer
+
 /docs/ @knylander-grafana @joe-elliott @annanay25 @mdisibio @mapno @kvrhdn @zalegrala @electron0zero @ie-pham @stoewer
+
+/.github/backport.yml           @jdbaldry @joe-elliott @annanay25 @mdisibio @mapno @kvrhdn @zalegrala @electron0zero @ie-pham @stoewer
+/.github/update-make-docs.yml   @jdbaldry @joe-elliott @annanay25 @mdisibio @mapno @kvrhdn @zalegrala @electron0zero @ie-pham @stoewer
+/.github/website-next.yml       @jdbaldry @joe-elliott @annanay25 @mdisibio @mapno @kvrhdn @zalegrala @electron0zero @ie-pham @stoewer
+/.github/website-versioned.yml  @jdbaldry @joe-elliott @annanay25 @mdisibio @mapno @kvrhdn @zalegrala @electron0zero @ie-pham @stoewer
+/docs/docs.mk                   @jdbaldry @knylander-grafana @joe-elliott @annanay25 @mdisibio @mapno @kvrhdn @zalegrala @electron0zero @ie-pham @stoewer
+/docs/make-docs                 @jdbaldry @knylander-grafana @joe-elliott @annanay25 @mdisibio @mapno @kvrhdn @zalegrala @electron0zero @ie-pham @stoewer
+/docs/Makefile                  @jdbaldry @knylander-grafana @joe-elliott @annanay25 @mdisibio @mapno @kvrhdn @zalegrala @electron0zero @ie-pham @stoewer
+/docs/variables.mk              @jdbaldry @knylander-grafana @joe-elliott @annanay25 @mdisibio @mapno @kvrhdn @zalegrala @electron0zero @ie-pham @stoewer
@@ -164,7 +164,7 @@ Tempo documentation is located in the `docs` directory. The `docs` directory has
 Once you know what you would like to write, use the [Writer's Toolkit](https://grafana.com/docs/writers-toolkit/writing-guide/contribute-documentation/) for information on creating good documentation.
 The toolkit also provides [document templates](https://github.com/grafana/writers-toolkit/tree/main/docs/static/templates) to help get started.

-When you create a PR for documentation, add the `types/doc` label to identify the PR as contributing documentation.
+When you create a PR for documentation, add the `type/doc` label to identify the PR as contributing documentation.

 If your content needs to be added to a previous release, use the `backport` label for the version. When your PR is merged, the backport label triggers an automatic process to create an additional PR to merge the content into the version's branch. Check the PR for content that might not be appropriate for the version. For example, if you fix a broken link on a page and then backport to Tempo 1.5, you would not want any TraceQL information to appear.

@@ -173,7 +173,7 @@ If your content needs to be added to a previous release, use the `backport` labe
 To preview the documentation locally, run `make docs` from the root folder of the Tempo repository. This uses
 the `grafana/docs` image which internally uses Hugo to generate the static site. The site is available on `localhost:3002/docs/`.

-> **Note** The `make docs` command uses a lot of memory. If its crashing make sure to increase the memory allocated to Docker
+> **Note** The `make docs` command uses a lot of memory. If it is crashing, make sure to increase the memory allocated to Docker
 > and try again.

 ### Publishing process
@@ -11,7 +11,7 @@ GOPATH := $(shell go env GOPATH)
 GORELEASER := $(GOPATH)/bin/goreleaser

 # Build Images
-DOCKER_PROTOBUF_IMAGE ?= otel/build-protobuf:0.14.0
+DOCKER_PROTOBUF_IMAGE ?= otel/build-protobuf:0.23.0
 LOKI_BUILD_IMAGE ?= grafana/loki-build-image:0.21.0
 DOCS_IMAGE ?= grafana/docs-base:latest

@@ -45,7 +45,7 @@ ifeq ($(BUILD_DEBUG), 1)
 	GO_OPT+= -gcflags="all=-N -l"
 endif

-GOTEST_OPT?= -race -timeout 20m -count=1 -v
+GOTEST_OPT?= -race -timeout 25m -count=1 -v
 GOTEST_OPT_WITH_COVERAGE = $(GOTEST_OPT) -cover
 GOTEST=gotestsum --format=testname --
 LINT=golangci-lint
@@ -55,7 +55,7 @@ ifeq ($(UNAME), Darwin)
    SED_OPTS := ''
 endif

-FILES_TO_FMT=$(shell find . -type d \( -path ./vendor -o -path ./opentelemetry-proto -o -path ./vendor-fix \) -prune -o -name '*.go' -not -name "*.pb.go" -not -name '*.y.go' -print)
+FILES_TO_FMT=$(shell find . -type d \( -path ./vendor -o -path ./opentelemetry-proto -o -path ./vendor-fix \) -prune -o -name '*.go' -not -name "*.pb.go" -not -name '*.y.go' -not -name '*.gen.go' -print)
 FILES_TO_JSONNETFMT=$(shell find ./operations/jsonnet ./operations/tempo-mixin -type f \( -name '*.libsonnet' -o -name '*.jsonnet' \) -not -path "*/vendor/*" -print)

 ### Build
@@ -129,32 +129,40 @@ test-e2e: tools docker-tempo docker-tempo-query
 test-e2e-serverless: tools docker-tempo docker-serverless
 	$(GOTEST) -v $(GOTEST_OPT) ./integration/e2e/serverless

+.PHONY: test-integration-poller
+test-integration-poller: tools
+	$(GOTEST) -v $(GOTEST_OPT) ./integration/poller
+
 # test-all/bench use a docker image so build it first to make sure we're up to date
 .PHONY: test-all
-test-all: test-with-cover test-e2e test-e2e-serverless
+test-all: test-with-cover test-e2e test-e2e-serverless test-integration-poller

 .PHONY: test-bench
 test-bench: tools docker-tempo
 	$(GOTEST) -v $(GOTEST_OPT) ./integration/bench

 .PHONY: fmt check-fmt
-fmt: tools
-	@gofumpt -w $(FILES_TO_FMT)
-	@goimports -w $(FILES_TO_FMT)
+fmt: tools-image
+	@$(TOOLS_CMD) gofumpt -w $(FILES_TO_FMT)
+	@$(TOOLS_CMD) goimports -w $(FILES_TO_FMT)

 check-fmt: fmt
 	@git diff --exit-code -- $(FILES_TO_FMT)

 .PHONY: jsonnetfmt check-jsonnetfmt
-jsonnetfmt:
-	@jsonnetfmt -i $(FILES_TO_JSONNETFMT)
+jsonnetfmt: tools-image
+	@$(TOOLS_CMD) jsonnetfmt -i $(FILES_TO_JSONNETFMT)

 check-jsonnetfmt: jsonnetfmt
 	@git diff --exit-code -- $(FILES_TO_JSONNETFMT)

 .PHONY: lint
 lint:
-	$(LINT) run
+ifneq ($(base),)
+	$(TOOLS_CMD) $(LINT) run --config .golangci.yml --new-from-rev=$(base)
+else
+	$(TOOLS_CMD) $(LINT) run --config .golangci.yml
+endif

 ### Docker Images

@@ -260,9 +268,16 @@ gen-traceql:
 gen-traceql-local:
 	goyacc -o pkg/traceql/expr.y.go pkg/traceql/expr.y && rm y.output

+# ##############
+# Gen Parquet-Query
+# ##############
+.PHONY: gen-parquet-query
+gen-parquet-query:
+	go run ./pkg/parquetquerygen/predicates.go > ./pkg/parquetquery/predicates.gen.go
+
 ### Check vendored and generated files are up to date
 .PHONY: vendor-check
-vendor-check: gen-proto update-mod gen-traceql
+vendor-check: gen-proto update-mod gen-traceql gen-parquet-query
 	git diff --exit-code -- **/go.sum **/go.mod vendor/ pkg/tempopb/ pkg/traceql/

 ### Tidy dependencies for tempo and tempo-serverless modules
@@ -275,7 +290,7 @@ update-mod:

 ### Release (intended to be used in the .github/workflows/release.yml)
 $(GORELEASER):
-	go install github.com/goreleaser/goreleaser@latest
+	go install github.com/goreleaser/goreleaser@v1.25.1

 .PHONY: release
 release: $(GORELEASER)
@@ -298,14 +313,14 @@ docs-test:

 ### jsonnet
 .PHONY: jsonnet jsonnet-check jsonnet-test
-jsonnet:
-	$(MAKE) -C operations/jsonnet-compiled/util gen
+jsonnet: tools-image
+	$(TOOLS_CMD) $(MAKE) -C operations/jsonnet-compiled/util gen

-jsonnet-check:
-	$(MAKE) -C operations/jsonnet-compiled/util check
+jsonnet-check: tools-image
+	$(TOOLS_CMD) $(MAKE) -C operations/jsonnet-compiled/util check

-jsonnet-test:
-	$(MAKE) -C operations/jsonnet/microservices test
+jsonnet-test: tools-image
+	$(TOOLS_CMD) $(MAKE) -C operations/jsonnet/microservices test

 ### serverless
 .PHONY: docker-serverless test-serverless
@@ -317,11 +332,11 @@ test-serverless:

 ### tempo-mixin
 .PHONY: tempo-mixin tempo-mixin-check
-tempo-mixin:
-	$(MAKE) -C operations/tempo-mixin all
+tempo-mixin: tools-image
+	$(TOOLS_CMD) $(MAKE) -C operations/tempo-mixin all

-tempo-mixin-check:
-	$(MAKE) -C operations/tempo-mixin check
+tempo-mixin-check: tools-image
+	$(TOOLS_CMD) $(MAKE) -C operations/tempo-mixin check

 ### drone
 .PHONY: drone drone-jsonnet drone-signature
@@ -14,8 +14,10 @@

 # Releases

-This document details release procedures for Tempo.  Currently it's pretty dang easy.
+This document details release procedures for Tempo.

+- Create a release branch. This will likely be on the same commit as the release candidate above.
+  - Name the branch like `release-v2.2`
 - Follow all steps in [Release Candidates](#release-candidates) except:
  - Drop the `-rc.#` postfix from the tag. For instance use `v1.2.0` instead. Something like:
    - `git tag -a v1.2.0`
@@ -23,6 +25,8 @@ This document details release procedures for Tempo.  Currently it's pretty dang
  - Make sure that the "This is a pre-release" is unchecked when publishing the release.
 - Submit a PR cleaning up the changelog and moving everything under "main/unreleased" to be under
  the newly minted version.
+  - Given that the changelog was already cleaned up for the RC above. This will likely be simply
+    renaming the release candidate to the full version.
 - In [github releases](https://github.com/grafana/tempo/releases) there should be a "Draft" release.
  Pretty up the changelog, add it to the release notes and hit "Publish release". Make sure that
  "Set as the latest release" is checked.
@@ -11,7 +11,26 @@ GO_MOD_OUTDATED  ?= go-mod-outdated
 TOOL_DIR     ?= tools
 TOOL_CONFIG  ?= $(TOOL_DIR)/tools.go

-GOTOOLS ?= $(shell cd $(TOOL_DIR) && go list -f '{{ .Imports }}' -tags tools |tr -d '[]')
+TOOLS_IMAGE ?= grafana/tempo-ci-tools
+TOOLS_IMAGE_TAG ?= main-d8c69d4
+
+GOTOOLS ?= $(shell cd $(TOOL_DIR) && go list -e -f '{{ .Imports }}' -tags tools |tr -d '[]')
+
+TOOLS_CMD = docker run --rm -t -v ${PWD}:/tools $(TOOLS_IMAGE):$(TOOLS_IMAGE_TAG)
+
+.PHONY: tools-image-build
+tools-image-build:
+	@echo "=== [ tools-image-build]: Building tools image..."
+	@docker build -t $(TOOLS_IMAGE) -f ./tools/Dockerfile .
+
+.PHONY: tools-docker
+tools-docker:
+	@echo "=== [ tools-docker     ]: Running tools in docker..."
+	@docker run -it -v $(shell pwd):/var/tempo $(TOOLS_IMAGE_NAME) make -C /var/tempo tools
+
+tools-image:
+	@echo "=== [ tools-image     ]: Running tools in docker..."
+	@docker pull ${TOOLS_IMAGE}

 tools:
 	@echo "=== [ tools            ]: Installing tools required by the project..."
@@ -1,4 +1,4 @@
-FROM alpine:3.16 as certs
+FROM alpine:3.20 as certs
 RUN apk --update add ca-certificates
 ARG TARGETARCH
 COPY bin/linux/tempo-cli-${TARGETARCH} /tempo-cli
@@ -0,0 +1,362 @@
+package main
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"io"
+	"os"
+	"sort"
+	"strconv"
+	"text/tabwriter"
+	"time"
+
+	tempo_io "github.com/grafana/tempo/pkg/io"
+	"github.com/parquet-go/parquet-go"
+
+	pq "github.com/grafana/tempo/pkg/parquetquery"
+	"github.com/stoewer/parquet-cli/pkg/inspect"
+
+	"github.com/grafana/tempo/tempodb/encoding/vparquet2"
+	"github.com/grafana/tempo/tempodb/encoding/vparquet3"
+
+	"github.com/dustin/go-humanize"
+	"github.com/google/uuid"
+	"github.com/grafana/tempo/tempodb/backend"
+)
+
+var (
+	vparquet2SpanAttrs = []string{
+		vparquet2.FieldSpanAttrVal,
+	}
+	vparquet2ResourceAttrs = []string{
+		vparquet2.FieldResourceAttrVal,
+	}
+	vparquet3SpanAttrs = []string{
+		vparquet3.FieldSpanAttrVal,
+	}
+	vparquet3ResourceAttrs = []string{
+		vparquet3.FieldResourceAttrVal,
+	}
+)
+
+func spanPathsForVersion(v string) (string, []string) {
+	switch v {
+	case vparquet2.VersionString:
+		return vparquet2.FieldSpanAttrKey, vparquet2SpanAttrs
+	case vparquet3.VersionString:
+		return vparquet3.FieldSpanAttrKey, vparquet3SpanAttrs
+	}
+	return "", nil
+}
+
+func resourcePathsForVersion(v string) (string, []string) {
+	switch v {
+	case vparquet2.VersionString:
+		return vparquet2.FieldResourceAttrKey, vparquet2ResourceAttrs
+	case vparquet3.VersionString:
+		return vparquet3.FieldResourceAttrKey, vparquet3ResourceAttrs
+	}
+	return "", nil
+}
+
+func dedicatedColPathForVersion(i int, scope backend.DedicatedColumnScope, v string) string {
+	switch v {
+	case vparquet3.VersionString:
+		return vparquet3.DedicatedResourceColumnPaths[scope][backend.DedicatedColumnTypeString][i]
+	}
+	return ""
+}
+
+type analyseBlockCmd struct {
+	backendOptions
+
+	TenantID        string `arg:"" help:"tenant-id within the bucket"`
+	BlockID         string `arg:"" help:"block ID to list"`
+	NumAttr         int    `help:"Number of attributes to display" default:"15"`
+	GenerateJsonnet bool   `help:"Generate overrides Jsonnet for dedicated columns"`
+}
+
+func (cmd *analyseBlockCmd) Run(ctx *globalOptions) error {
+	r, _, _, err := loadBackend(&cmd.backendOptions, ctx)
+	if err != nil {
+		return err
+	}
+
+	blockSum, err := processBlock(r, cmd.TenantID, cmd.BlockID, time.Time{}, time.Time{}, 0)
+	if err != nil {
+		if errors.Is(err, backend.ErrDoesNotExist) {
+			return fmt.Errorf("unable to analyze block: block has no block.meta because it was compacted")
+		}
+		return err
+	}
+
+	if blockSum == nil {
+		return errors.New("failed to process block")
+	}
+
+	return blockSum.print(cmd.NumAttr, cmd.GenerateJsonnet)
+}
+
+func processBlock(r backend.Reader, tenantID, blockID string, maxStartTime, minStartTime time.Time, minCompactionLvl uint8) (*blockSummary, error) {
+	id := uuid.MustParse(blockID)
+
+	meta, err := r.BlockMeta(context.TODO(), id, tenantID)
+	if err != nil {
+		return nil, err
+	}
+	if meta.CompactionLevel < minCompactionLvl {
+		return nil, nil
+	}
+	if !maxStartTime.IsZero() && meta.StartTime.After(maxStartTime) {
+		// Block is newer than maxStartTime
+		return nil, nil
+	}
+	if !minStartTime.IsZero() && meta.StartTime.Before(minStartTime) {
+		// Block is older than minStartTime
+		return nil, nil
+	}
+
+	var reader io.ReaderAt
+	switch meta.Version {
+	case vparquet2.VersionString:
+		reader = vparquet2.NewBackendReaderAt(context.Background(), r, vparquet2.DataFileName, meta)
+	case vparquet3.VersionString:
+		reader = vparquet3.NewBackendReaderAt(context.Background(), r, vparquet3.DataFileName, meta)
+	default:
+		fmt.Println("Unsupported block version:", meta.Version)
+		return nil, nil
+	}
+
+	br := tempo_io.NewBufferedReaderAt(reader, int64(meta.Size), 2*1024*1024, 64) // 128 MB memory buffering
+
+	pf, err := parquet.OpenFile(br, int64(meta.Size), parquet.SkipBloomFilters(true), parquet.SkipPageIndex(true))
+	if err != nil {
+		return nil, err
+	}
+
+	fmt.Println("Scanning block contents.  Press CRTL+C to quit ...")
+
+	// Aggregate span attributes
+	spanKey, spanVals := spanPathsForVersion(meta.Version)
+	spanAttrsSummary, err := aggregateAttributes(pf, spanKey, spanVals)
+	if err != nil {
+		return nil, err
+	}
+
+	// add up dedicated span attribute columns
+	spanDedicatedSummary, err := aggregateDedicatedColumns(pf, backend.DedicatedColumnScopeSpan, meta)
+	if err != nil {
+		return nil, err
+	}
+	// merge dedicated with span attributes
+	for k, v := range spanDedicatedSummary.attributes {
+		spanAttrsSummary.attributes[k] = v
+	}
+	spanAttrsSummary.totalBytes += spanDedicatedSummary.totalBytes
+
+	// Aggregate resource attributes
+	resourceKey, resourceVals := resourcePathsForVersion(meta.Version)
+	resourceAttrsSummary, err := aggregateAttributes(pf, resourceKey, resourceVals)
+	if err != nil {
+		return nil, err
+	}
+
+	// add up dedicated resource attribute columns
+	resourceDedicatedSummary, err := aggregateDedicatedColumns(pf, backend.DedicatedColumnScopeResource, meta)
+	if err != nil {
+		return nil, err
+	}
+	// merge dedicated with span attributes
+	for k, v := range resourceDedicatedSummary.attributes {
+		resourceAttrsSummary.attributes[k] = v
+	}
+	resourceAttrsSummary.totalBytes += spanDedicatedSummary.totalBytes
+
+	return &blockSummary{
+		spanSummary:     spanAttrsSummary,
+		resourceSummary: resourceAttrsSummary,
+	}, nil
+}
+
+type blockSummary struct {
+	spanSummary, resourceSummary genericAttrSummary
+}
+
+func (s *blockSummary) print(maxAttr int, generateJsonnet bool) error {
+	if err := printSummary("span", maxAttr, s.spanSummary); err != nil {
+		return err
+	}
+
+	if err := printSummary("resource", maxAttr, s.resourceSummary); err != nil {
+		return err
+	}
+
+	if generateJsonnet {
+		printDedicatedColumnOverridesJsonnet(s.spanSummary, s.resourceSummary)
+	}
+
+	return nil
+}
+
+type genericAttrSummary struct {
+	totalBytes uint64
+	attributes map[string]uint64 // key: attribute name, value: total bytes
+}
+
+type attribute struct {
+	name  string
+	bytes uint64
+}
+
+func aggregateAttributes(pf *parquet.File, keyPath string, valuePaths []string) (genericAttrSummary, error) {
+	keyIdx, _ := pq.GetColumnIndexByPath(pf, keyPath)
+	valueIdxs := make([]int, 0, len(valuePaths))
+	for _, v := range valuePaths {
+		idx, _ := pq.GetColumnIndexByPath(pf, v)
+		valueIdxs = append(valueIdxs, idx)
+	}
+
+	opts := inspect.AggregateOptions{
+		GroupByColumn: keyIdx,
+		Columns:       valueIdxs,
+	}
+	rowStats, err := inspect.NewAggregateCalculator(pf, opts)
+	if err != nil {
+		return genericAttrSummary{}, err
+	}
+
+	attrMap := make(map[string]uint64)
+	totalBytes := uint64(0)
+
+	for {
+		row, err := rowStats.NextRow()
+		if err != nil {
+			if errors.Is(err, io.EOF) {
+				break
+			}
+			return genericAttrSummary{}, err
+		}
+
+		cells := row.Cells()
+
+		name := cells[0].(string)
+		bytes := uint64(cells[1].(int))
+		attrMap[name] = bytes
+		totalBytes += bytes
+	}
+
+	return genericAttrSummary{
+		totalBytes: totalBytes,
+		attributes: attrMap,
+	}, nil
+}
+
+func aggregateDedicatedColumns(pf *parquet.File, scope backend.DedicatedColumnScope, meta *backend.BlockMeta) (genericAttrSummary, error) {
+	attrMap := make(map[string]uint64)
+	totalBytes := uint64(0)
+
+	i := 0
+	for _, dedColumn := range meta.DedicatedColumns {
+		if dedColumn.Scope != scope {
+			continue
+		}
+
+		path := dedicatedColPathForVersion(i, scope, meta.Version)
+		sz, err := aggregateColumn(pf, path)
+		if err != nil {
+			return genericAttrSummary{}, err
+		}
+		i++
+
+		attrMap["dedicated: "+dedColumn.Name] = sz
+		totalBytes += sz
+	}
+
+	return genericAttrSummary{
+		totalBytes: totalBytes,
+		attributes: attrMap,
+	}, nil
+}
+
+func aggregateColumn(pf *parquet.File, colName string) (uint64, error) {
+	idx, _ := pq.GetColumnIndexByPath(pf, colName)
+	calc, err := inspect.NewRowStatCalculator(pf, inspect.RowStatOptions{
+		Columns: []int{idx},
+	})
+	if err != nil {
+		return 0, err
+	}
+
+	totalBytes := uint64(0)
+	for {
+		row, err := calc.NextRow()
+		if err != nil {
+			if errors.Is(err, io.EOF) {
+				break
+			}
+			return 0, err
+		}
+
+		cells := row.Cells()
+
+		bytes := uint64(cells[1].(int))
+		totalBytes += bytes
+	}
+
+	return totalBytes, nil
+}
+
+func printSummary(scope string, max int, summary genericAttrSummary) error {
+	// TODO: Support more output formats
+	w := tabwriter.NewWriter(os.Stdout, 0, 0, 2, ' ', 0)
+
+	if max > len(summary.attributes) {
+		max = len(summary.attributes)
+	}
+
+	fmt.Printf("Top %d %s attributes by size\n", max, scope)
+	attrList := topN(max, summary.attributes)
+	for _, a := range attrList {
+		percentage := float64(a.bytes) / float64(summary.totalBytes) * 100
+		_, err := fmt.Fprintf(w, "name: %s\t size: %s\t (%s%%)\n", a.name, humanize.Bytes(a.bytes), strconv.FormatFloat(percentage, 'f', 2, 64))
+		if err != nil {
+			return err
+		}
+	}
+
+	return w.Flush()
+}
+
+func printDedicatedColumnOverridesJsonnet(spanSummary, resourceSummary genericAttrSummary) {
+	fmt.Println("")
+	fmt.Printf("parquet_dedicated_columns: [\n")
+
+	// span attributes first
+	spanAttrList := topN(10, spanSummary.attributes)
+	for _, a := range spanAttrList {
+		fmt.Printf(" { scope: 'span', name: '%s', type: 'string' },\n", a.name)
+	}
+
+	// span attributes first
+	resourceAttrList := topN(10, resourceSummary.attributes)
+	for _, a := range resourceAttrList {
+		fmt.Printf(" { scope: 'resource', name: '%s', type: 'string' },\n", a.name)
+	}
+	fmt.Printf("], \n")
+	fmt.Println("")
+}
+
+func topN(n int, attrs map[string]uint64) []attribute {
+	top := make([]attribute, 0, len(attrs))
+	for name, bytes := range attrs {
+		top = append(top, attribute{name, bytes})
+	}
+	sort.Slice(top, func(i, j int) bool {
+		return top[i].bytes > top[j].bytes
+	})
+	if len(top) > n {
+		top = top[:n]
+	}
+	return top
+}
@@ -0,0 +1,104 @@
+package main
+
+import (
+	"context"
+	"errors"
+	"time"
+
+	"github.com/google/uuid"
+
+	"github.com/grafana/tempo/tempodb/backend"
+)
+
+type analyseBlocksCmd struct {
+	backendOptions
+
+	TenantID           string `arg:"" help:"tenant-id within the bucket"`
+	MinCompactionLevel int    `help:"Min compaction level to analyse" default:"3"`
+	MaxBlocks          int    `help:"Max number of blocks to analyse" default:"10"`
+	NumAttr            int    `help:"Number of attributes to display" default:"15"`
+	MaxStartTime       string `help:"Oldest start time for a block to be processed. RFC3339 format '2006-01-02T15:04:05Z07:00'" default:""`
+	MinStartTime       string `help:"Newest start time for a block to be processed. RFC3339 format '2006-01-02T15:04:05Z07:00'" default:""`
+}
+
+func (cmd *analyseBlocksCmd) Run(ctx *globalOptions) error {
+	r, _, _, err := loadBackend(&cmd.backendOptions, ctx)
+	if err != nil {
+		return err
+	}
+
+	// TODO: Parallelize this
+	blocks, _, err := r.Blocks(context.Background(), cmd.TenantID)
+	if err != nil {
+		return err
+	}
+
+	processedBlocks := map[uuid.UUID]struct{}{}
+	topSpanAttrs, topResourceAttrs := make(map[string]uint64), make(map[string]uint64)
+	totalSpanBytes, totalResourceBytes := uint64(0), uint64(0)
+
+	var maxStartTime, minStartTime time.Time
+	if cmd.MaxStartTime != "" {
+		maxStartTime, err = time.Parse(time.RFC3339, cmd.MaxStartTime)
+		if err != nil {
+			return err
+		}
+	}
+	if cmd.MinStartTime != "" {
+		minStartTime, err = time.Parse(time.RFC3339, cmd.MinStartTime)
+		if err != nil {
+			return err
+		}
+	}
+
+	for i := 0; i < len(blocks) && len(processedBlocks) < cmd.MaxBlocks; i++ {
+		block := blocks[i]
+		if _, ok := processedBlocks[block]; ok {
+			continue
+		}
+
+		blockSum, err := processBlock(r, cmd.TenantID, block.String(), maxStartTime, minStartTime, uint8(cmd.MinCompactionLevel))
+		if err != nil {
+			if !errors.Is(err, backend.ErrDoesNotExist) {
+				return err
+			}
+
+			// the block was already compacted and blocks might be outdated: refreshing blocks
+			blocks, _, err = r.Blocks(context.Background(), cmd.TenantID)
+			if err != nil {
+				return err
+			}
+			i = -1
+
+			continue
+		}
+
+		if blockSum == nil {
+			continue
+		}
+
+		for k, v := range blockSum.spanSummary.attributes {
+			topSpanAttrs[k] += v
+		}
+		totalSpanBytes += blockSum.spanSummary.totalBytes
+
+		for k, v := range blockSum.resourceSummary.attributes {
+			topResourceAttrs[k] += v
+		}
+		totalResourceBytes += blockSum.resourceSummary.totalBytes
+
+		processedBlocks[block] = struct{}{}
+	}
+
+	// Get top N attributes from map
+	return (&blockSummary{
+		spanSummary: genericAttrSummary{
+			totalBytes: totalSpanBytes,
+			attributes: topSpanAttrs,
+		},
+		resourceSummary: genericAttrSummary{
+			totalBytes: totalResourceBytes,
+			attributes: topResourceAttrs,
+		},
+	}).print(cmd.NumAttr, false)
+}
@@ -1,62 +0,0 @@
-package main
-
-import (
-	"fmt"
-	"os"
-
-	"github.com/grafana/tempo/tempodb/encoding/vparquet"
-	"github.com/segmentio/parquet-go"
-)
-
-type convertParquet struct {
-	In  string `arg:"" help:"The input parquet file to read from"`
-	Out string `arg:"" help:"The output parquet file to write to"`
-}
-
-func (cmd *convertParquet) Run() error {
-	// open In
-	fIn, err := os.Open(cmd.In)
-	if err != nil {
-		return err
-	}
-	s, err := fIn.Stat()
-	if err != nil {
-		return err
-	}
-	pf, err := parquet.OpenFile(fIn, s.Size())
-	if err != nil {
-		return err
-	}
-
-	// open Out
-	fOut, err := os.Create(cmd.Out)
-	if err != nil {
-		return err
-	}
-	sch := parquet.SchemaOf(new(vparquet.Trace))
-	writer := parquet.NewWriter(fOut, sch)
-
-	conversion, err := parquet.Convert(sch, pf.Schema())
-	if err != nil {
-		return err
-	}
-
-	// copy a rowgroup at a time
-	rgs := pf.RowGroups()
-	fmt.Println("Total Rowgroups: ", len(rgs))
-	for i, rg := range rgs {
-		fmt.Println("Converting ", i+1)
-		rg = parquet.ConvertRowGroup(rg, conversion)
-
-		_, err = writer.WriteRowGroup(rg)
-		if err != nil {
-			return err
-		}
-		err = writer.Flush()
-		if err != nil {
-			return err
-		}
-	}
-
-	return writer.Close()
-}
@@ -1,228 +0,0 @@
-package main
-
-import (
-	"encoding/binary"
-	"fmt"
-	"io"
-	"os"
-
-	"github.com/pkg/errors"
-	"github.com/segmentio/parquet-go"
-
-	"github.com/grafana/tempo/tempodb/encoding/vparquet"
-	"github.com/grafana/tempo/tempodb/encoding/vparquet2"
-)
-
-type convertParquet1to2 struct {
-	In  string `arg:"" help:"The input parquet file to read from"`
-	Out string `arg:"" help:"The output parquet file to write to"`
-}
-
-func (cmd *convertParquet1to2) Run() error {
-	err := cmd.convert()
-	if err != nil {
-		return err
-	}
-
-	return cmd.printFileInfo()
-}
-
-func (cmd *convertParquet1to2) printFileInfo() error {
-	fmt.Printf("Converted file: %s\n", cmd.Out)
-	out, err := os.Open(cmd.Out)
-	if err != nil {
-		return err
-	}
-	defer out.Close()
-
-	stat, err := out.Stat()
-	if err != nil {
-		return err
-	}
-	fmt.Printf("File size: %d\n", stat.Size())
-
-	buf := make([]byte, 8)
-	n, err := out.ReadAt(buf, stat.Size()-8)
-	if err != nil && !errors.Is(err, io.EOF) {
-		return err
-	}
-	if n < 4 {
-		return errors.New("not enough bytes read to determine footer size")
-	}
-	fmt.Printf("Footer size: %d\n", binary.LittleEndian.Uint32(buf[0:4]))
-
-	return nil
-}
-
-func (cmd *convertParquet1to2) convert() error {
-	in, err := os.Open(cmd.In)
-	if err != nil {
-		return err
-	}
-	defer in.Close()
-
-	inStat, err := in.Stat()
-	if err != nil {
-		return err
-	}
-
-	pf, err := parquet.OpenFile(in, inStat.Size())
-	if err != nil {
-		return err
-	}
-
-	out, err := os.OpenFile(cmd.Out, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, inStat.Mode())
-	if err != nil {
-		return err
-	}
-	defer out.Close()
-
-	writer := parquet.NewGenericWriter[vparquet2.Trace](out)
-	defer writer.Close()
-
-	readBuffer := make([]vparquet.Trace, 500)
-	writeBuffer := make([]vparquet2.Trace, 500)
-
-	rowGroups := pf.RowGroups()
-	fmt.Printf("Total rowgroups: %d\n", len(rowGroups))
-
-	for i, rowGroup := range rowGroups {
-		fmt.Printf("Converting rowgroup: %d\n", i+1)
-		reader := parquet.NewGenericRowGroupReader[vparquet.Trace](rowGroup)
-
-		for {
-			readCount, err := reader.Read(readBuffer)
-			if err != nil && !errors.Is(err, io.EOF) {
-				return err
-			}
-			if readCount == 0 {
-				err = writer.Flush()
-				if err != nil {
-					return err
-				}
-				break
-			}
-
-			vparquet1to2(readBuffer[:readCount], writeBuffer)
-
-			writeCount := 0
-			for writeCount < readCount {
-				n, err := writer.Write(writeBuffer[writeCount:readCount])
-				if err != nil {
-					return err
-				}
-				writeCount += n
-			}
-		}
-	}
-
-	return nil
-}
-
-func vparquet1to2(in []vparquet.Trace, out []vparquet2.Trace) {
-	for i, trace := range in {
-		vparquetTrace1to2(&trace, &out[i])
-	}
-}
-
-func vparquetTrace1to2(trace *vparquet.Trace, v2trace *vparquet2.Trace) {
-	v2trace.TraceID = trace.TraceID
-	v2trace.TraceIDText = trace.TraceIDText
-	v2trace.ResourceSpans = make([]vparquet2.ResourceSpans, len(trace.ResourceSpans))
-	v2trace.StartTimeUnixNano = trace.StartTimeUnixNano
-	v2trace.EndTimeUnixNano = trace.EndTimeUnixNano
-	v2trace.DurationNano = trace.DurationNanos
-	v2trace.RootServiceName = trace.RootServiceName
-	v2trace.RootSpanName = trace.RootSpanName
-
-	for i, rspan := range trace.ResourceSpans {
-		vparquetResourceSpans1to2(&rspan, &v2trace.ResourceSpans[i])
-	}
-}
-
-func vparquetResourceSpans1to2(rspan *vparquet.ResourceSpans, v2rspan *vparquet2.ResourceSpans) {
-	v2rspan.Resource.ServiceName = rspan.Resource.ServiceName
-	v2rspan.Resource.Cluster = rspan.Resource.Cluster
-	v2rspan.Resource.Namespace = rspan.Resource.Namespace
-	v2rspan.Resource.Pod = rspan.Resource.Pod
-	v2rspan.Resource.Container = rspan.Resource.Container
-	v2rspan.Resource.K8sClusterName = rspan.Resource.K8sClusterName
-	v2rspan.Resource.K8sNamespaceName = rspan.Resource.K8sNamespaceName
-	v2rspan.Resource.K8sPodName = rspan.Resource.K8sPodName
-	v2rspan.Resource.K8sContainerName = rspan.Resource.K8sContainerName
-
-	v2rspan.Resource.Test = rspan.Resource.Test
-
-	v2rspan.Resource.Attrs = make([]vparquet2.Attribute, len(rspan.Resource.Attrs))
-	for i, attr := range rspan.Resource.Attrs {
-		vparquetAttribute1to2(&attr, &v2rspan.Resource.Attrs[i])
-	}
-
-	v2rspan.ScopeSpans = make([]vparquet2.ScopeSpans, len(rspan.ScopeSpans))
-	for i, sspan := range rspan.ScopeSpans {
-		v2sspan := &v2rspan.ScopeSpans[i]
-		v2sspan.Scope.Name = sspan.Scope.Name
-		v2sspan.Scope.Version = sspan.Scope.Version
-
-		v2sspan.Spans = make([]vparquet2.Span, len(sspan.Spans))
-		for j, span := range sspan.Spans {
-			vparquetSpan1to2(&span, &v2sspan.Spans[j])
-		}
-	}
-}
-
-func vparquetSpan1to2(span *vparquet.Span, v2span *vparquet2.Span) {
-	v2span.SpanID = span.ID
-	v2span.Name = span.Name
-	v2span.Kind = span.Kind
-	v2span.ParentSpanID = span.ParentSpanID
-	v2span.TraceState = span.TraceState
-	v2span.StartTimeUnixNano = span.StartUnixNanos
-	v2span.DurationNano = span.EndUnixNanos - span.StartUnixNanos
-	v2span.StatusCode = span.StatusCode
-	v2span.StatusMessage = span.StatusMessage
-	v2span.DroppedAttributesCount = span.DroppedAttributesCount
-	v2span.DroppedEventsCount = span.DroppedEventsCount
-	v2span.Links = span.Links
-	v2span.DroppedLinksCount = span.DroppedLinksCount
-	v2span.HttpMethod = span.HttpMethod
-	v2span.HttpUrl = span.HttpUrl
-	v2span.HttpStatusCode = span.HttpStatusCode
-
-	v2span.Events = make([]vparquet2.Event, len(span.Events))
-	for i, event := range span.Events {
-		vparquetEvent1to2(&event, &v2span.Events[i])
-	}
-
-	v2span.Attrs = make([]vparquet2.Attribute, 0, len(span.Attrs))
-	for _, attr := range span.Attrs {
-		var v2attr vparquet2.Attribute
-		vparquetAttribute1to2(&attr, &v2attr)
-		v2span.Attrs = append(v2span.Attrs, v2attr)
-	}
-}
-
-func vparquetAttribute1to2(attr *vparquet.Attribute, v2attr *vparquet2.Attribute) {
-	v2attr.Key = attr.Key
-	v2attr.Value = attr.Value
-	v2attr.ValueInt = attr.ValueInt
-	v2attr.ValueDouble = attr.ValueDouble
-	v2attr.ValueBool = attr.ValueBool
-	v2attr.ValueKVList = attr.ValueKVList
-	v2attr.ValueArray = attr.ValueArray
-}
-
-func vparquetEvent1to2(event *vparquet.Event, v2event *vparquet2.Event) {
-	v2event.TimeUnixNano = event.TimeUnixNano
-	v2event.Name = event.Name
-	v2event.DroppedAttributesCount = event.DroppedAttributesCount
-
-	v2event.Test = event.Test
-
-	v2event.Attrs = make([]vparquet2.EventAttribute, len(event.Attrs))
-	for i, attr := range event.Attrs {
-		v2attr := &v2event.Attrs[i]
-		v2attr.Key = attr.Key
-		v2attr.Value = attr.Value
-	}
-}
@@ -0,0 +1,124 @@
+package main
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"io"
+
+	"github.com/parquet-go/parquet-go"
+
+	"github.com/grafana/tempo/pkg/tempopb"
+	"github.com/grafana/tempo/pkg/traceql"
+	"github.com/grafana/tempo/tempodb/backend"
+	"github.com/grafana/tempo/tempodb/backend/local"
+	"github.com/grafana/tempo/tempodb/encoding/common"
+	"github.com/grafana/tempo/tempodb/encoding/vparquet2"
+	"github.com/grafana/tempo/tempodb/encoding/vparquet3"
+)
+
+type convertParquet2to3 struct {
+	In               string   `arg:"" help:"The input parquet file to read from."`
+	Out              string   `arg:"" help:"The output folder to write block to." default:"./out" optional:""`
+	DedicatedColumns []string `arg:"" help:"List of dedicated columns to convert." optional:""`
+}
+
+func (cmd *convertParquet2to3) Run() error {
+	cmd.In = getPathToBlockDir(cmd.In)
+
+	// open the in file
+	in, pf, err := openParquetFile(cmd.In)
+	if err != nil {
+		return err
+	}
+	defer in.Close()
+
+	// open the input metadata file
+	meta, err := readBlockMeta(cmd.In)
+	if err != nil {
+		return err
+	}
+
+	// create out block
+	outR, outW, _, err := local.New(&local.Config{
+		Path: cmd.Out,
+	})
+	if err != nil {
+		return err
+	}
+
+	dedicatedCols := make([]backend.DedicatedColumn, 0, len(cmd.DedicatedColumns))
+	for _, col := range cmd.DedicatedColumns {
+		att, err := traceql.ParseIdentifier(col)
+		if err != nil {
+			return err
+		}
+
+		scope := backend.DedicatedColumnScopeSpan
+		if att.Scope == traceql.AttributeScopeResource {
+			scope = backend.DedicatedColumnScopeResource
+		}
+
+		fmt.Println("scope", scope, "name", att.Name)
+
+		dedicatedCols = append(dedicatedCols, backend.DedicatedColumn{
+			Scope: scope,
+			Name:  att.Name,
+			Type:  backend.DedicatedColumnTypeString,
+		})
+	}
+
+	blockCfg := &common.BlockConfig{
+		BloomFP:             0.99,
+		BloomShardSizeBytes: 1024 * 1024,
+		Version:             vparquet3.VersionString,
+		RowGroupSizeBytes:   100 * 1024 * 1024,
+		DedicatedColumns:    dedicatedCols,
+	}
+
+	newMeta := *meta
+	newMeta.Version = vparquet3.VersionString
+	newMeta.DedicatedColumns = dedicatedCols
+
+	// create iterator over in file
+	iter := &parquetIterator2{
+		r: parquet.NewGenericReader[*vparquet2.Trace](pf),
+	}
+
+	_, err = vparquet3.CreateBlock(context.Background(), blockCfg, &newMeta, iter, backend.NewReader(outR), backend.NewWriter(outW))
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
+type parquetIterator2 struct {
+	r *parquet.GenericReader[*vparquet2.Trace]
+	i int
+}
+
+func (i *parquetIterator2) Next(_ context.Context) (common.ID, *tempopb.Trace, error) {
+	traces := make([]*vparquet2.Trace, 1)
+
+	i.i++
+	if i.i%1000 == 0 {
+		fmt.Println(i.i)
+	}
+
+	_, err := i.r.Read(traces)
+	if errors.Is(err, io.EOF) {
+		return nil, nil, io.EOF
+	}
+	if err != nil {
+		return nil, nil, err
+	}
+
+	pqTrace := traces[0]
+	pbTrace := vparquet2.ParquetTraceToTempopbTrace(pqTrace)
+	return pqTrace.TraceID, pbTrace, nil
+}
+
+func (i *parquetIterator2) Close() {
+	_ = i.r.Close()
+}
@@ -0,0 +1,184 @@
+package main
+
+import (
+	"context"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"io"
+	"os"
+	"path/filepath"
+
+	"github.com/parquet-go/parquet-go"
+
+	"github.com/grafana/tempo/pkg/tempopb"
+	"github.com/grafana/tempo/pkg/traceql"
+	"github.com/grafana/tempo/tempodb/backend"
+	"github.com/grafana/tempo/tempodb/backend/local"
+	"github.com/grafana/tempo/tempodb/encoding/common"
+	"github.com/grafana/tempo/tempodb/encoding/vparquet3"
+	"github.com/grafana/tempo/tempodb/encoding/vparquet4"
+)
+
+type convertParquet3to4 struct {
+	In               string   `arg:"" help:"The input parquet block to read from."`
+	Out              string   `arg:"" help:"The output folder to write block to." default:"./out" optional:""`
+	DedicatedColumns []string `arg:"" help:"List of dedicated columns to convert. Overwrites existing dedicated columns" optional:""`
+}
+
+func (cmd *convertParquet3to4) Run() error {
+	cmd.In = getPathToBlockDir(cmd.In)
+
+	// open the input parquet file
+	in, pf, err := openParquetFile(cmd.In)
+	if err != nil {
+		return err
+	}
+	defer in.Close()
+
+	// open the input metadata file
+	meta, err := readBlockMeta(cmd.In)
+	if err != nil {
+		return err
+	}
+
+	// create output block
+	outR, outW, _, err := local.New(&local.Config{
+		Path: cmd.Out,
+	})
+	if err != nil {
+		return err
+	}
+
+	// calculate dedicated columns
+	var dedicatedCols backend.DedicatedColumns
+
+	if len(cmd.DedicatedColumns) > 0 {
+		dedicatedCols = make(backend.DedicatedColumns, 0, len(cmd.DedicatedColumns))
+
+		for _, col := range cmd.DedicatedColumns {
+			att, err := traceql.ParseIdentifier(col)
+			if err != nil {
+				return err
+			}
+
+			scope := backend.DedicatedColumnScopeSpan
+			if att.Scope == traceql.AttributeScopeResource {
+				scope = backend.DedicatedColumnScopeResource
+			}
+
+			fmt.Printf("add dedicated column scope=%s name=%s\n", scope, att.Name)
+
+			dedicatedCols = append(dedicatedCols, backend.DedicatedColumn{
+				Scope: scope,
+				Name:  att.Name,
+				Type:  backend.DedicatedColumnTypeString,
+			})
+		}
+	} else {
+		dedicatedCols = meta.DedicatedColumns
+	}
+
+	// copy block
+	blockCfg := &common.BlockConfig{
+		BloomFP:             0.99,
+		BloomShardSizeBytes: 1024 * 1024,
+		Version:             vparquet4.VersionString,
+		RowGroupSizeBytes:   100 * 1024 * 1024,
+	}
+
+	newMeta := *meta
+	newMeta.Version = vparquet4.VersionString
+	newMeta.DedicatedColumns = dedicatedCols
+
+	// create iterator over in file
+	iter := &parquetIterator3{
+		r: parquet.NewGenericReader[*vparquet3.Trace](pf),
+		m: meta,
+	}
+
+	fmt.Printf("Creating vParquet4 block in %s\n", filepath.Join(cmd.Out, meta.TenantID, newMeta.BlockID.String()))
+	fmt.Printf("Converting rows 0 to %d\n", pf.NumRows())
+	outMeta, err := vparquet4.CreateBlock(context.Background(), blockCfg, &newMeta, iter, backend.NewReader(outR), backend.NewWriter(outW))
+	if err != nil {
+		return err
+	}
+
+	fmt.Printf("Successfully created block with size=%d and footerSize=%d\n", outMeta.Size, outMeta.FooterSize)
+	return nil
+}
+
+func getPathToBlockDir(path string) string {
+	if filepath.Base(path) == "data.parquet" {
+		return filepath.Dir(path)
+	}
+	return path
+}
+
+func openParquetFile(blockPath string) (*os.File, *parquet.File, error) {
+	inFile := filepath.Join(blockPath, "data.parquet")
+	in, err := os.Open(inFile)
+	if err != nil {
+		return nil, nil, err
+	}
+
+	inStat, err := in.Stat()
+	if err != nil {
+		return nil, nil, err
+	}
+
+	pf, err := parquet.OpenFile(in, inStat.Size())
+	if err != nil {
+		return nil, nil, err
+	}
+
+	return in, pf, nil
+}
+
+func readBlockMeta(blockPath string) (*backend.BlockMeta, error) {
+	metaFile := filepath.Join(blockPath, "meta.json")
+	inMeta, err := os.Open(metaFile)
+	if err != nil {
+		return nil, err
+	}
+	defer inMeta.Close()
+
+	var meta backend.BlockMeta
+	err = json.NewDecoder(inMeta).Decode(&meta)
+	if err != nil {
+		return nil, err
+	}
+
+	return &meta, nil
+}
+
+type parquetIterator3 struct {
+	r *parquet.GenericReader[*vparquet3.Trace]
+	m *backend.BlockMeta
+	i int
+}
+
+func (i *parquetIterator3) Next(_ context.Context) (common.ID, *tempopb.Trace, error) {
+	traces := []*vparquet3.Trace{{}}
+
+	i.i++
+	if i.i%1000 == 0 {
+		fmt.Println(i.i)
+	}
+
+	_, err := i.r.Read(traces)
+	if errors.Is(err, io.EOF) {
+		return nil, nil, io.EOF
+	}
+	if err != nil {
+		return nil, nil, err
+	}
+
+	pqTrace := traces[0]
+	pbTrace := vparquet3.ParquetTraceToTempopbTrace(i.m, pqTrace)
+	return pqTrace.TraceID, pbTrace, nil
+}
+
+func (i *parquetIterator3) Close() {
+	_ = i.r.Close()
+}
@@ -3,6 +3,7 @@ package main
 import (
 	"bytes"
 	"context"
+	"errors"
 	"fmt"
 	"io"
 	"os"
@@ -43,7 +44,7 @@ func ReplayBlockAndDoForEachRecord(meta *backend.BlockMeta, filepath string, for
 	objectRW := v2.NewObjectReaderWriter()
 	for {
 		buffer, _, err := dataReader.NextPage(buffer)
-		if err == io.EOF {
+		if errors.Is(err, io.EOF) {
 			break
 		}
 		if err != nil {
@@ -64,7 +65,7 @@ func ReplayBlockAndDoForEachRecord(meta *backend.BlockMeta, filepath string, for
 			}
 		}

-		if iterErr != io.EOF {
+		if !errors.Is(iterErr, io.EOF) {
 			return iterErr
 		}
 	}
@@ -119,7 +120,7 @@ func (cmd *bloomCmd) Run(ctx *globalOptions) error {
 	}

 	for i := 0; i < len(bloomBytes); i++ {
-		err = w.Write(context.TODO(), bloomFilePrefix+strconv.Itoa(i), blockID, cmd.TenantID, bloomBytes[i], false)
+		err = w.Write(context.TODO(), bloomFilePrefix+strconv.Itoa(i), blockID, cmd.TenantID, bloomBytes[i], nil)
 		if err != nil {
 			fmt.Println("error writing bloom filter to backend", err)
 			return err
@@ -131,7 +132,7 @@ func (cmd *bloomCmd) Run(ctx *globalOptions) error {
 	// verify generated bloom
 	shardedBloomFilter := make([]*willf_bloom.BloomFilter, meta.BloomShardCount)
 	for i := 0; i < int(meta.BloomShardCount); i++ {
-		bloomBytes, err := r.Read(context.TODO(), bloomFilePrefix+strconv.Itoa(i), blockID, cmd.TenantID, false)
+		bloomBytes, err := r.Read(context.TODO(), bloomFilePrefix+strconv.Itoa(i), blockID, cmd.TenantID, nil)
 		if err != nil {
 			fmt.Println("error reading bloom from backend")
 			return nil
@@ -3,6 +3,7 @@ package main
 import (
 	"bytes"
 	"context"
+	"errors"
 	"fmt"
 	"io"
 	"os"
@@ -40,7 +41,7 @@ func ReplayBlockAndGetRecords(meta *backend.BlockMeta, filepath string) ([]v2.Re
 	currentOffset := uint64(0)
 	for {
 		buffer, pageLen, err := dataReader.NextPage(buffer)
-		if err == io.EOF {
+		if errors.Is(err, io.EOF) {
 			break
 		}
 		if err != nil {
@@ -60,7 +61,7 @@ func ReplayBlockAndGetRecords(meta *backend.BlockMeta, filepath string) ([]v2.Re
 			lastID = id
 		}

-		if iterErr != io.EOF {
+		if !errors.Is(iterErr, io.EOF) {
 			replayError = iterErr
 			break
 		}
@@ -139,7 +140,7 @@ func (cmd *indexCmd) Run(ctx *globalOptions) error {
 	}

 	// write to the local backend
-	err = w.Write(context.TODO(), "index", blockID, cmd.TenantID, indexBytes, false)
+	err = w.Write(context.TODO(), "index", blockID, cmd.TenantID, indexBytes, nil)
 	if err != nil {
 		fmt.Println("error writing index to backend", err)
 		return err
@@ -3,6 +3,7 @@ package main
 import (
 	"bytes"
 	"context"
+	"errors"
 	"fmt"
 	"io"
 	"os"
@@ -13,6 +14,7 @@ import (

 	"github.com/dustin/go-humanize"
 	"github.com/google/uuid"
+
 	"github.com/grafana/tempo/pkg/model"
 	"github.com/grafana/tempo/pkg/tempopb"
 	"github.com/grafana/tempo/pkg/util"
@@ -52,12 +54,12 @@ func dumpBlock(r tempodb_backend.Reader, c tempodb_backend.Compactor, tenantID s
 	id := uuid.MustParse(blockID)

 	meta, err := r.BlockMeta(context.TODO(), id, tenantID)
-	if err != nil && err != tempodb_backend.ErrDoesNotExist {
+	if err != nil && !errors.Is(err, tempodb_backend.ErrDoesNotExist) {
 		return err
 	}

 	compactedMeta, err := c.CompactedBlockMeta(id, tenantID)
-	if err != nil && err != tempodb_backend.ErrDoesNotExist {
+	if err != nil && !errors.Is(err, tempodb_backend.ErrDoesNotExist) {
 		return err
 	}

@@ -132,7 +134,7 @@ func dumpBlock(r tempodb_backend.Reader, c tempodb_backend.Compactor, tenantID s
 		prevID := make([]byte, 16)
 		for {
 			objID, obj, err := iter.NextBytes(ctx)
-			if err == io.EOF {
+			if errors.Is(err, io.EOF) {
 				break
 			} else if err != nil {
 				return err
@@ -242,7 +244,7 @@ func addKey(kvp kvPairs, key string, count int) {
 	kvp[key] = v
 }

-func addVal(kvp kvPairs, key string, val string, count int) {
+func addVal(kvp kvPairs, key, val string, count int) {
 	v := kvp[key]
 	stats, ok := v.all[val]
 	if !ok {
@@ -2,14 +2,15 @@ package main

 import (
 	"context"
+	"errors"
 	"fmt"
 	"io"

 	"github.com/google/uuid"
+	"github.com/parquet-go/parquet-go"

 	pq "github.com/grafana/tempo/pkg/parquetquery"
-	"github.com/grafana/tempo/tempodb/encoding/vparquet"
-	"github.com/segmentio/parquet-go"
+	"github.com/grafana/tempo/tempodb/encoding/vparquet3"
 )

 type listColumnCmd struct {
@@ -31,7 +32,7 @@ func (cmd *listColumnCmd) Run(ctx *globalOptions) error {
 		return err
 	}

-	rr := vparquet.NewBackendReaderAt(context.Background(), r, vparquet.DataFileName, meta.BlockID, meta.TenantID)
+	rr := vparquet3.NewBackendReaderAt(context.Background(), r, vparquet3.DataFileName, meta)
 	pf, err := parquet.OpenFile(rr, int64(meta.Size))
 	if err != nil {
 		return err
@@ -47,14 +48,18 @@ func (cmd *listColumnCmd) Run(ctx *globalOptions) error {
 		fmt.Printf("\n***************       rowgroup %d      ********************\n\n\n", i)

 		pages := cc.Pages()
-		numPages := cc.ColumnIndex().NumPages()
-		fmt.Println("Min Value of rowgroup", cc.ColumnIndex().MinValue(0).Bytes())
-		fmt.Println("Max Value of rowgroup", cc.ColumnIndex().MaxValue(numPages-1).Bytes())
+		idx, err := cc.ColumnIndex()
+		if err != nil {
+			return err
+		}
+		numPages := idx.NumPages()
+		fmt.Println("Min Value of rowgroup", idx.MinValue(0).Bytes())
+		fmt.Println("Max Value of rowgroup", idx.MaxValue(numPages-1).Bytes())

 		buffer := make([]parquet.Value, 10000)
 		for {
 			pg, err := pages.ReadPage()
-			if pg == nil || err == io.EOF {
+			if pg == nil || errors.Is(err, io.EOF) {
 				break
 			}

@@ -66,7 +71,7 @@ func (cmd *listColumnCmd) Run(ctx *globalOptions) error {
 				}

 				// check for EOF after processing any returned data
-				if err == io.EOF {
+				if errors.Is(err, io.EOF) {
 					break
 				}
 				// todo: better error handling
@@ -0,0 +1,102 @@
+package main
+
+import (
+	"bytes"
+	"context"
+	"flag"
+	"fmt"
+	"net/http"
+	"net/url"
+	"os"
+
+	"github.com/grafana/dskit/services"
+	"github.com/prometheus/client_golang/prometheus"
+	"gopkg.in/yaml.v2"
+
+	"github.com/grafana/tempo/cmd/tempo/app"
+	"github.com/grafana/tempo/modules/overrides"
+)
+
+type migrateOverridesConfigCmd struct {
+	ConfigFile string `arg:"" help:"Path to tempo config file"`
+
+	ConfigDest    string `type:"path" short:"d" help:"Path to tempo config file. If not specified, output to stdout"`
+	OverridesDest string `type:"path" short:"o" help:"Path to tempo overrides file. If not specified, output to stdout"`
+}
+
+func (cmd *migrateOverridesConfigCmd) Run(*globalOptions) error {
+	// Defaults
+	cfg := app.Config{}
+	cfg.RegisterFlagsAndApplyDefaults("", &flag.FlagSet{})
+
+	// Existing config
+	buff, err := os.ReadFile(cmd.ConfigFile)
+	if err != nil {
+		return fmt.Errorf("failed to read configFile %s: %w", cmd.ConfigFile, err)
+	}
+
+	if err := yaml.UnmarshalStrict(buff, &cfg); err != nil {
+		return fmt.Errorf("failed to parse configFile %s: %w", cmd.ConfigFile, err)
+	}
+
+	o, err := overrides.NewOverrides(cfg.Overrides, nil, prometheus.DefaultRegisterer)
+	if err != nil {
+		return fmt.Errorf("failed to load overrides module: %w", err)
+	}
+
+	if err := services.StartAndAwaitRunning(context.Background(), o); err != nil {
+		return fmt.Errorf("failed to start overrides module: %w", err)
+	}
+
+	buffer := bytes.NewBuffer(make([]byte, 0))
+	if err := o.WriteStatusRuntimeConfig(buffer, &http.Request{URL: &url.URL{}}); err != nil {
+		return fmt.Errorf("failed to output runtime config: %w", err)
+	}
+
+	var runtimeConfig struct {
+		Defaults           overrides.Overrides         `yaml:"defaults"`
+		PerTenantOverrides map[string]overrides.Config `yaml:"overrides"`
+	}
+	if err := yaml.UnmarshalStrict(buffer.Bytes(), &runtimeConfig); err != nil {
+		return fmt.Errorf("failed parsing overrides config: %w", err)
+	}
+
+	cfg.Overrides.Defaults = runtimeConfig.Defaults
+	configBytes, err := yaml.Marshal(cfg)
+	if err != nil {
+		return fmt.Errorf("failed to marshal config: %w", err)
+	}
+
+	if cmd.ConfigDest != "" {
+		if err := os.WriteFile(cmd.ConfigDest, configBytes, 0o644); err != nil {
+			return fmt.Errorf("failed to write config file: %w", err)
+		}
+	} else {
+		fmt.Println(cmd.ConfigFile)
+		// Only print the overrides block
+		partialCfg := struct {
+			Overrides overrides.Config `yaml:"overrides"`
+		}{Overrides: cfg.Overrides}
+		overridesBytes, err := yaml.Marshal(partialCfg)
+		if err != nil {
+			return fmt.Errorf("failed to marshal overrides: %w", err)
+		}
+		fmt.Println(string(overridesBytes))
+	}
+
+	overridesBytes, err := yaml.Marshal(runtimeConfig.PerTenantOverrides)
+	if err != nil {
+		return fmt.Errorf("failed to marshal overrides: %w", err)
+	}
+
+	if cmd.OverridesDest != "" {
+		if err := os.WriteFile(cmd.OverridesDest, overridesBytes, 0o644); err != nil {
+			return fmt.Errorf("failed to write overrides file: %w", err)
+		}
+	} else {
+		fmt.Println(cfg.Overrides.PerTenantOverrideConfig)
+		fmt.Println(string(overridesBytes))
+	}
+
+	return nil
+}
@@ -5,7 +5,6 @@ import (
 	"fmt"

 	"github.com/dustin/go-humanize"
-	"github.com/pkg/errors"

 	"github.com/grafana/tempo/tempodb/backend"
 	"github.com/grafana/tempo/tempodb/encoding"
@@ -23,7 +22,7 @@ func (cmd *migrateTenantCmd) Run(opts *globalOptions) error {

 	readerSource, readerDest, writerDest, err := cmd.setupBackends(opts)
 	if err != nil {
-		return errors.Wrap(err, "setting up backends")
+		return fmt.Errorf("setting up backends: %w", err)
 	}
 	defer func() {
 		readerSource.Shutdown()
@@ -32,13 +31,13 @@ func (cmd *migrateTenantCmd) Run(opts *globalOptions) error {

 	sourceTenantIndex, err := readerSource.TenantIndex(ctx, cmd.SourceTenantID)
 	if err != nil {
-		return errors.Wrap(err, "reading source tenant index")
+		return fmt.Errorf("reading source tenant index: %w", err)
 	}
 	fmt.Printf("Blocks in source: %d, compacted: %d\n", len(sourceTenantIndex.Meta), len(sourceTenantIndex.CompactedMeta))

 	// TODO create dest directory if it doesn't exist yet?

-	blocksDest, err := readerDest.Blocks(ctx, cmd.DestTenantID)
+	blocksDest, _, err := readerDest.Blocks(ctx, cmd.DestTenantID)
 	if err != nil {
 		return err
 	}
@@ -62,12 +61,12 @@ blocks:

 		encoder, err := encoding.FromVersion(sourceBlockMeta.Version)
 		if err != nil {
-			return errors.Wrap(err, "creating encoder from version")
+			return fmt.Errorf("creating encoder from version: %w", err)
 		}

 		err = encoder.MigrateBlock(ctx, sourceBlockMeta, &destBlockMeta, readerSource, writerDest)
 		if err != nil {
-			return errors.Wrap(err, "copying block")
+			return fmt.Errorf("copying block: %w", err)
 		}

 		copiedBlocks++
@@ -3,11 +3,14 @@ package main
 import (
 	"bytes"
 	"context"
+	"errors"
 	"fmt"
+	"math/rand"
 	"strconv"

 	"github.com/gogo/protobuf/jsonpb"
 	"github.com/google/uuid"
+
 	"github.com/grafana/tempo/pkg/boundedwaitgroup"
 	"github.com/grafana/tempo/pkg/model/trace"
 	"github.com/grafana/tempo/pkg/tempopb"
@@ -26,8 +29,9 @@ type queryResults struct {
 type queryBlocksCmd struct {
 	backendOptions

-	TraceID  string `arg:"" help:"trace ID to retrieve"`
-	TenantID string `arg:"" help:"tenant ID to search"`
+	TraceID    string  `arg:"" help:"trace ID to retrieve"`
+	TenantID   string  `arg:"" help:"tenant ID to search"`
+	Percentage float32 `help:"percentage of blocks to scan e.g..1 for 10%"`
 }

 func (cmd *queryBlocksCmd) Run(ctx *globalOptions) error {
@@ -41,13 +45,13 @@ func (cmd *queryBlocksCmd) Run(ctx *globalOptions) error {
 		return err
 	}

-	results, err := queryBucket(context.Background(), r, c, cmd.TenantID, id)
+	results, err := queryBucket(context.Background(), cmd.Percentage, r, c, cmd.TenantID, id)
 	if err != nil {
 		return err
 	}

 	var (
-		combiner   = trace.NewCombiner()
+		combiner   = trace.NewCombiner(0)
 		marshaller = new(jsonpb.Marshaler)
 		jsonBytes  = bytes.Buffer{}
 	)
@@ -64,7 +68,10 @@ func (cmd *queryBlocksCmd) Run(ctx *globalOptions) error {

 		fmt.Println(jsonBytes.String())
 		jsonBytes.Reset()
-		combiner.ConsumeWithFinal(result.trace, i == len(results)-1)
+		_, err = combiner.ConsumeWithFinal(result.trace, i == len(results)-1)
+		if err != nil {
+			return fmt.Errorf("error combining trace: %w", err)
+		}
 	}

 	combinedTrace, _ := combiner.Result()
@@ -78,16 +85,27 @@ func (cmd *queryBlocksCmd) Run(ctx *globalOptions) error {
 	return nil
 }

-func queryBucket(ctx context.Context, r backend.Reader, c backend.Compactor, tenantID string, traceID common.ID) ([]queryResults, error) {
-	blockIDs, err := r.Blocks(context.Background(), tenantID)
+func queryBucket(ctx context.Context, percentage float32, r backend.Reader, c backend.Compactor, tenantID string, traceID common.ID) ([]queryResults, error) {
+	blockIDs, compactedBlockIDs, err := r.Blocks(context.Background(), tenantID)
 	if err != nil {
 		return nil, err
 	}

-	fmt.Println("total blocks: ", len(blockIDs))
+	if percentage > 0 {
+		// shuffle
+		rand.Shuffle(len(blockIDs), func(i, j int) { blockIDs[i], blockIDs[j] = blockIDs[j], blockIDs[i] })
+
+		// get the first n%
+		total := len(blockIDs)
+		total = int(float32(total) * percentage)
+		blockIDs = blockIDs[:total]
+	}
+	fmt.Println("total blocks to search: ", len(blockIDs))
+
+	blockIDs = append(blockIDs, compactedBlockIDs...)

 	// Load in parallel
-	wg := boundedwaitgroup.New(20)
+	wg := boundedwaitgroup.New(100)
 	resultsCh := make(chan queryResults, len(blockIDs))

 	for blockNum, id := range blockIDs {
@@ -120,28 +138,21 @@ func queryBucket(ctx context.Context, r backend.Reader, c backend.Compactor, ten
 	return results, nil
 }

-func queryBlock(ctx context.Context, r backend.Reader, c backend.Compactor, blockNum int, id uuid.UUID, tenantID string, traceID common.ID) (*queryResults, error) {
+func queryBlock(ctx context.Context, r backend.Reader, _ backend.Compactor, blockNum int, id uuid.UUID, tenantID string, traceID common.ID) (*queryResults, error) {
 	fmt.Print(".")
 	if blockNum%100 == 0 {
 		fmt.Print(strconv.Itoa(blockNum))
 	}

 	meta, err := r.BlockMeta(context.Background(), id, tenantID)
-	if err != nil && err != backend.ErrDoesNotExist {
+	if err != nil && !errors.Is(err, backend.ErrDoesNotExist) {
 		return nil, err
 	}

-	if err == backend.ErrDoesNotExist {
-		compactedMeta, err := c.CompactedBlockMeta(id, tenantID)
-		if err != nil && err != backend.ErrDoesNotExist {
-			return nil, err
-		}
-
-		if compactedMeta == nil {
-			return nil, fmt.Errorf("compacted meta nil?")
-		}
-
-		meta = &compactedMeta.BlockMeta
+	if errors.Is(err, backend.ErrDoesNotExist) {
+		// tempo proper searches compacted blocks, b/c each querier has a different view of the backend blocks.
+		// however, with a single snaphot of the backend, we can only search the noncompacted blocks.
+		return nil, nil
 	}

 	block, err := encoding.OpenBlock(meta, r)
@@ -0,0 +1,135 @@
+package main
+
+import (
+	"bytes"
+	"context"
+	"errors"
+	"io"
+	"net/http"
+	"path"
+	"time"
+
+	"google.golang.org/grpc"
+	"google.golang.org/grpc/credentials/insecure"
+
+	"github.com/gogo/protobuf/jsonpb"
+	"github.com/grafana/dskit/user"
+	"github.com/grafana/tempo/pkg/api"
+	"github.com/grafana/tempo/pkg/tempopb"
+)
+
+type metricsQueryRangeCmd struct {
+	HostPort string `arg:"" help:"tempo host and port. scheme and path will be provided based on query type. e.g. localhost:3200"`
+	TraceQL  string `arg:"" optional:"" help:"traceql query"`
+	Start    string `arg:"" optional:"" help:"start time in ISO8601 format"`
+	End      string `arg:"" optional:"" help:"end time in ISO8601 format"`
+
+	OrgID      string `help:"optional orgID"`
+	UseGRPC    bool   `help:"stream search results over GRPC"`
+	PathPrefix string `help:"string to prefix all http paths with"`
+}
+
+func (cmd *metricsQueryRangeCmd) Run(_ *globalOptions) error {
+	startDate, err := time.Parse(time.RFC3339, cmd.Start)
+	if err != nil {
+		return err
+	}
+	start := startDate.Unix()
+
+	endDate, err := time.Parse(time.RFC3339, cmd.End)
+	if err != nil {
+		return err
+	}
+	end := endDate.Unix()
+
+	req := &tempopb.QueryRangeRequest{
+		Query: cmd.TraceQL,
+		Start: uint64(start),
+		End:   uint64(end),
+		Step:  uint64(5 * time.Second),
+	}
+
+	if cmd.UseGRPC {
+		return cmd.searchGRPC(req)
+	}
+
+	return cmd.searchHTTP(req)
+}
+
+func (cmd *metricsQueryRangeCmd) searchGRPC(req *tempopb.QueryRangeRequest) error {
+	ctx := user.InjectOrgID(context.Background(), cmd.OrgID)
+	ctx, err := user.InjectIntoGRPCRequest(ctx)
+	if err != nil {
+		return err
+	}
+
+	clientConn, err := grpc.DialContext(ctx, cmd.HostPort, grpc.WithTransportCredentials(insecure.NewCredentials()))
+	if err != nil {
+		return err
+	}
+
+	client := tempopb.NewStreamingQuerierClient(clientConn)
+
+	resp, err := client.MetricsQueryRange(ctx, req)
+	if err != nil {
+		return err
+	}
+
+	for {
+		searchResp, err := resp.Recv()
+		if searchResp != nil {
+			err = printAsJSON(searchResp)
+			if err != nil {
+				return err
+			}
+		}
+		if errors.Is(err, io.EOF) {
+			return nil
+		}
+		if err != nil {
+			return err
+		}
+	}
+}
+
+// nolint: goconst // goconst wants us to make http:// a const
+func (cmd *metricsQueryRangeCmd) searchHTTP(req *tempopb.QueryRangeRequest) error {
+	httpReq, err := http.NewRequest("GET", "http://"+path.Join(cmd.HostPort, cmd.PathPrefix, api.PathMetricsQueryRange), nil)
+	if err != nil {
+		return err
+	}
+
+	httpReq = api.BuildQueryRangeRequest(httpReq, req)
+	httpReq.Header = http.Header{}
+	err = user.InjectOrgIDIntoHTTPRequest(user.InjectOrgID(context.Background(), cmd.OrgID), httpReq)
+	if err != nil {
+		return err
+	}
+
+	httpResp, err := http.DefaultClient.Do(httpReq)
+	if err != nil {
+		return err
+	}
+	defer httpResp.Body.Close()
+
+	body, err := io.ReadAll(httpResp.Body)
+	if err != nil {
+		return err
+	}
+
+	if httpResp.StatusCode != http.StatusOK {
+		return errors.New("failed to query. body: " + string(body) + " status: " + httpResp.Status)
+	}
+
+	resp := &tempopb.QueryRangeResponse{}
+	err = jsonpb.Unmarshal(bytes.NewReader(body), resp)
+	if err != nil {
+		panic("failed to parse resp: " + err.Error())
+	}
+	err = printAsJSON(resp)
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
@@ -1,23 +1,117 @@
 package main

 import (
+	"context"
+	"errors"
+	"io"
+	"path"
+	"time"
+
+	"github.com/grafana/dskit/user"
 	"github.com/grafana/tempo/pkg/httpclient"
+	"github.com/grafana/tempo/pkg/tempopb"
+	"google.golang.org/grpc"
+	"google.golang.org/grpc/credentials/insecure"
 )

 type querySearchTagValuesCmd struct {
-	APIEndpoint string `arg:"" help:"tempo api endpoint"`
-	Tag         string `arg:"" help:"tag name"`
+	HostPort string `arg:"" help:"tempo host and port. scheme and path will be provided based on query type. e.g. localhost:3200"`
+	Tag      string `arg:"" help:"tag name"`
+	Start    string `arg:"" optional:"" help:"start time in ISO8601 format"`
+	End      string `arg:"" optional:"" help:"end time in ISO8601 format"`

-	OrgID string `help:"optional orgID"`
+	Query      string `help:"TraceQL query to filter attribute results by (supported by GRPC only)"`
+	OrgID      string `help:"optional orgID"`
+	UseGRPC    bool   `help:"stream search results over GRPC"`
+	PathPrefix string `help:"string to prefix all http paths with"`
 }

 func (cmd *querySearchTagValuesCmd) Run(_ *globalOptions) error {
-	client := httpclient.New(cmd.APIEndpoint, cmd.OrgID)
+	var start, end int64
+
+	if cmd.Start != "" {
+		startDate, err := time.Parse(time.RFC3339, cmd.Start)
+		if err != nil {
+			return err
+		}
+		start = startDate.Unix()
+	}
+
+	if cmd.End != "" {
+		endDate, err := time.Parse(time.RFC3339, cmd.End)
+		if err != nil {
+			return err
+		}
+		end = endDate.Unix()
+	}
+
+	if cmd.UseGRPC {
+		return cmd.searchGRPC(start, end)
+	}
+	return cmd.searchHTTP(start, end)
+}
+
+// nolint: goconst // goconst wants us to make http:// a const
+func (cmd *querySearchTagValuesCmd) searchHTTP(start, end int64) error {
+	if cmd.PathPrefix != "" {
+		cmd.HostPort = path.Join(cmd.HostPort, cmd.PathPrefix)
+	}
+	client := httpclient.New("http://"+cmd.HostPort, cmd.OrgID)
+
+	var tags *tempopb.SearchTagValuesV2Response
+	var err error
+	if start != 0 || end != 0 {
+		tags, err = client.SearchTagValuesV2WithRange(cmd.Tag, start, end)
+	} else {
+		tags, err = client.SearchTagValuesV2(cmd.Tag, "")
+	}

-	tagValues, err := client.SearchTagValues(cmd.Tag)
 	if err != nil {
 		return err
 	}

-	return printAsJSON(tagValues)
+	return printAsJSON(tags)
+}
+
+func (cmd *querySearchTagValuesCmd) searchGRPC(start, end int64) error {
+	ctx := user.InjectOrgID(context.Background(), cmd.OrgID)
+	ctx, err := user.InjectIntoGRPCRequest(ctx)
+	if err != nil {
+		return err
+	}
+
+	clientConn, err := grpc.DialContext(ctx, cmd.HostPort, grpc.WithTransportCredentials(insecure.NewCredentials()))
+	if err != nil {
+		return err
+	}
+
+	client := tempopb.NewStreamingQuerierClient(clientConn)
+
+	tagsRequest := &tempopb.SearchTagValuesRequest{
+		TagName: cmd.Tag,
+		Start:   uint32(start),
+		End:     uint32(end),
+		Query:   cmd.Query,
+	}
+
+	resp, err := client.SearchTagValuesV2(ctx, tagsRequest)
+	if err != nil {
+		return err
+	}
+
+	for {
+		searchResp, err := resp.Recv()
+		if searchResp != nil {
+			err = printAsJSON(searchResp)
+			if err != nil {
+				return err
+			}
+		}
+		if errors.Is(err, io.EOF) {
+			return nil
+		}
+		if err != nil {
+			return err
+		}
+	}
 }
@@ -1,22 +1,113 @@
 package main

 import (
+	"context"
+	"errors"
+	"io"
+	"path"
+	"time"
+
+	"github.com/grafana/dskit/user"
 	"github.com/grafana/tempo/pkg/httpclient"
+	"github.com/grafana/tempo/pkg/tempopb"
+	"google.golang.org/grpc"
+	"google.golang.org/grpc/credentials/insecure"
 )

 type querySearchTagsCmd struct {
-	APIEndpoint string `arg:"" help:"tempo api endpoint"`
+	HostPort string `arg:"" help:"tempo host and port. scheme and path will be provided based on query type. e.g. localhost:3200"`
+	Start    string `arg:"" optional:"" help:"start time in ISO8601 format"`
+	End      string `arg:"" optional:"" help:"end time in ISO8601 format"`

-	OrgID string `help:"optional orgID"`
+	OrgID      string `help:"optional orgID"`
+	UseGRPC    bool   `help:"stream search results over GRPC"`
+	PathPrefix string `help:"string to prefix all http paths with"`
 }

 func (cmd *querySearchTagsCmd) Run(_ *globalOptions) error {
-	client := httpclient.New(cmd.APIEndpoint, cmd.OrgID)
+	var start, end int64
+
+	if cmd.Start != "" {
+		startDate, err := time.Parse(time.RFC3339, cmd.Start)
+		if err != nil {
+			return err
+		}
+		start = startDate.Unix()
+	}
+
+	if cmd.End != "" {
+		endDate, err := time.Parse(time.RFC3339, cmd.End)
+		if err != nil {
+			return err
+		}
+		end = endDate.Unix()
+	}
+
+	if cmd.UseGRPC {
+		return cmd.searchGRPC(start, end)
+	}
+	return cmd.searchHTTP(start, end)
+}
+
+// nolint: goconst // goconst wants us to make http:// a const
+func (cmd *querySearchTagsCmd) searchHTTP(start, end int64) error {
+	if cmd.PathPrefix != "" {
+		cmd.HostPort = path.Join(cmd.HostPort, cmd.PathPrefix)
+	}
+	client := httpclient.New("http://"+cmd.HostPort, cmd.OrgID)
+
+	var tags *tempopb.SearchTagsV2Response
+	var err error
+	if start != 0 || end != 0 {
+		tags, err = client.SearchTagsV2WithRange(start, end)
+	} else {
+		tags, err = client.SearchTagsV2()
+	}

-	tags, err := client.SearchTags()
 	if err != nil {
 		return err
 	}

 	return printAsJSON(tags)
 }
+
+func (cmd *querySearchTagsCmd) searchGRPC(start, end int64) error {
+	ctx := user.InjectOrgID(context.Background(), cmd.OrgID)
+	ctx, err := user.InjectIntoGRPCRequest(ctx)
+	if err != nil {
+		return err
+	}
+
+	clientConn, err := grpc.DialContext(ctx, cmd.HostPort, grpc.WithTransportCredentials(insecure.NewCredentials()))
+	if err != nil {
+		return err
+	}
+
+	client := tempopb.NewStreamingQuerierClient(clientConn)
+
+	tagsRequest := &tempopb.SearchTagsRequest{
+		Start: uint32(start),
+		End:   uint32(end),
+	}
+
+	resp, err := client.SearchTagsV2(ctx, tagsRequest)
+	if err != nil {
+		return err
+	}
+
+	for {
+		searchResp, err := resp.Recv()
+		if searchResp != nil {
+			err = printAsJSON(searchResp)
+			if err != nil {
+				return err
+			}
+		}
+		if errors.Is(err, io.EOF) {
+			return nil
+		}
+		if err != nil {
+			return err
+		}
+	}
+}
@@ -1,23 +1,34 @@
 package main

 import (
+	"bytes"
 	"context"
+	"errors"
 	"io"
+	"net/http"
+	"path"
 	"time"

-	"github.com/grafana/tempo/pkg/tempopb"
-	"github.com/weaveworks/common/user"
 	"google.golang.org/grpc"
 	"google.golang.org/grpc/credentials/insecure"
+
+	"github.com/gogo/protobuf/jsonpb"
+	"github.com/grafana/dskit/user"
+	"github.com/grafana/tempo/pkg/api"
+	"github.com/grafana/tempo/pkg/tempopb"
 )

 type querySearchCmd struct {
-	APIEndpoint string `arg:"" help:"tempo api endpoint"`
-	TraceQL     string `arg:"" optional:"" help:"traceql query"`
-	Start       string `arg:"" optional:"" help:"start time in ISO8601 format"`
-	End         string `arg:"" optional:"" help:"end time in ISO8601 format"`
+	HostPort string `arg:"" help:"tempo host and port. scheme and path will be provided based on query type. e.g. localhost:3200"`
+	TraceQL  string `arg:"" optional:"" help:"traceql query"`
+	Start    string `arg:"" optional:"" help:"start time in ISO8601 format"`
+	End      string `arg:"" optional:"" help:"end time in ISO8601 format"`

-	OrgID string `help:"optional orgID"`
+	OrgID      string `help:"optional orgID"`
+	UseGRPC    bool   `help:"stream search results over GRPC"`
+	SPSS       int    `help:"spans per spanset" default:"0"`
+	Limit      int    `help:"limit number of results" default:"0"`
+	PathPrefix string `help:"string to prefix all http paths with"`
 }

 func (cmd *querySearchCmd) Run(_ *globalOptions) error {
@@ -33,23 +44,36 @@ func (cmd *querySearchCmd) Run(_ *globalOptions) error {
 	}
 	end := endDate.Unix()

+	req := &tempopb.SearchRequest{
+		Query:           cmd.TraceQL,
+		Start:           uint32(start),
+		End:             uint32(end),
+		SpansPerSpanSet: uint32(cmd.SPSS),
+		Limit:           uint32(cmd.Limit),
+	}
+
+	if cmd.UseGRPC {
+		return cmd.searchGRPC(req)
+	}
+
+	return cmd.searchHTTP(req)
+}
+
+func (cmd *querySearchCmd) searchGRPC(req *tempopb.SearchRequest) error {
 	ctx := user.InjectOrgID(context.Background(), cmd.OrgID)
-	ctx, err = user.InjectIntoGRPCRequest(ctx)
+	ctx, err := user.InjectIntoGRPCRequest(ctx)
 	if err != nil {
 		return err
 	}
-	clientConn, err := grpc.DialContext(ctx, cmd.APIEndpoint, grpc.WithTransportCredentials(insecure.NewCredentials()))
+
+	clientConn, err := grpc.DialContext(ctx, cmd.HostPort, grpc.WithTransportCredentials(insecure.NewCredentials()))
 	if err != nil {
 		return err
 	}

 	client := tempopb.NewStreamingQuerierClient(clientConn)

-	resp, err := client.Search(ctx, &tempopb.SearchRequest{
-		Query: cmd.TraceQL,
-		Start: uint32(start),
-		End:   uint32(end),
-	})
+	resp, err := client.Search(ctx, req)
 	if err != nil {
 		return err
 	}
@@ -62,7 +86,7 @@ func (cmd *querySearchCmd) Run(_ *globalOptions) error {
 				return err
 			}
 		}
-		if err == io.EOF {
+		if errors.Is(err, io.EOF) {
 			return nil
 		}
 		if err != nil {
@@ -70,3 +94,50 @@ func (cmd *querySearchCmd) Run(_ *globalOptions) error {
 		}
 	}
 }
+
+// nolint: goconst // goconst wants us to make http:// a const
+func (cmd *querySearchCmd) searchHTTP(req *tempopb.SearchRequest) error {
+	httpReq, err := http.NewRequest("GET", "http://"+path.Join(cmd.HostPort, cmd.PathPrefix, api.PathSearch), nil)
+	if err != nil {
+		return err
+	}
+
+	httpReq, err = api.BuildSearchRequest(httpReq, req)
+	if err != nil {
+		return err
+	}
+
+	httpReq.Header = http.Header{}
+	err = user.InjectOrgIDIntoHTTPRequest(user.InjectOrgID(context.Background(), cmd.OrgID), httpReq)
+	if err != nil {
+		return err
+	}
+
+	// fmt.Println(httpReq)
+	httpResp, err := http.DefaultClient.Do(httpReq)
+	if err != nil {
+		return err
+	}
+	defer httpResp.Body.Close()
+
+	body, err := io.ReadAll(httpResp.Body)
+	if err != nil {
+		return err
+	}
+
+	if httpResp.StatusCode != http.StatusOK {
+		return errors.New("failed to query. body: " + string(body) + " status: " + httpResp.Status)
+	}
+
+	resp := &tempopb.SearchResponse{}
+	err = jsonpb.Unmarshal(bytes.NewReader(body), resp)
+	if err != nil {
+		panic("failed to parse resp: " + err.Error())
+	}
+	err = printAsJSON(resp)
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
@@ -0,0 +1,225 @@
+package main
+
+import (
+	"context"
+	"fmt"
+	"math"
+	"math/rand"
+	"sort"
+	"time"
+
+	"github.com/davecgh/go-spew/spew"
+	"github.com/google/uuid"
+
+	"github.com/grafana/tempo/pkg/boundedwaitgroup"
+	v1resource "github.com/grafana/tempo/pkg/tempopb/resource/v1"
+	v1 "github.com/grafana/tempo/pkg/tempopb/trace/v1"
+	"github.com/grafana/tempo/pkg/util"
+	"github.com/grafana/tempo/tempodb/backend"
+	"github.com/grafana/tempo/tempodb/encoding/common"
+)
+
+type queryTraceSummaryCmd struct {
+	backendOptions
+
+	TraceID  string `arg:"" help:"trace ID to retrieve"`
+	TenantID string `arg:"" help:"tenant ID to search"`
+
+	Percentage float32 `help:"percentage of blocks to scan e.g..1 for 10%"`
+}
+
+func (cmd *queryTraceSummaryCmd) Run(ctx *globalOptions) error {
+	r, _, c, err := loadBackend(&cmd.backendOptions, ctx)
+	if err != nil {
+		return err
+	}
+
+	id, err := util.HexStringToTraceID(cmd.TraceID)
+	if err != nil {
+		return err
+	}
+
+	traceSummary, err := queryBucketForSummary(context.Background(), cmd.Percentage, r, c, cmd.TenantID, id)
+	if err != nil {
+		return err
+	}
+
+	fmt.Printf("Number of blocks: %d \n", traceSummary.NumBlock)
+	fmt.Printf("Span count: %d \n", traceSummary.SpanCount)
+	fmt.Printf("Trace size: %d B \n", traceSummary.TraceSize)
+	fmt.Printf("Trace duration: %d seconds \n", traceSummary.TraceDuration)
+	fmt.Printf("Root service name: %s \n", traceSummary.RootServiceName)
+
+	fmt.Println("Root span info:")
+	if traceSummary.RootSpan != nil {
+		scs := spew.ConfigState{DisableMethods: true, Indent: " "}
+		scs.Dump(traceSummary.RootSpan)
+	} else {
+		fmt.Println("No root span found")
+	}
+
+	fmt.Println("top frequent service.names: ")
+	fmt.Println(traceSummary.ServiceNames)
+
+	return nil
+}
+
+type TraceSummary struct {
+	NumBlock         int
+	SpanCount        int
+	TraceSize        int
+	TraceDuration    uint64
+	RootServiceName  string
+	RootSpan         *v1.Span
+	RootSpanResource *v1resource.Resource
+	ServiceNames     []string
+}
+
+func sortServiceNames(nameFrequencies map[string]int) PairList {
+	pl := make(PairList, len(nameFrequencies))
+	i := 0
+	for k, v := range nameFrequencies {
+		pl[i] = Pair{k, v}
+		i++
+	}
+	sort.Sort(sort.Reverse(pl))
+	return pl
+}
+
+func queryBucketForSummary(ctx context.Context, percentage float32, r backend.Reader, c backend.Compactor, tenantID string, traceID common.ID) (*TraceSummary, error) {
+	blockIDs, _, err := r.Blocks(context.Background(), tenantID)
+	if err != nil {
+		return nil, err
+	}
+
+	if percentage > 0 {
+		// shuffle
+		rand.Shuffle(len(blockIDs), func(i, j int) { blockIDs[i], blockIDs[j] = blockIDs[j], blockIDs[i] })
+
+		// get the first n%
+		total := len(blockIDs)
+		total = int(float32(total) * percentage)
+		blockIDs = blockIDs[:total]
+	}
+
+	fmt.Println("total blocks to search: ", len(blockIDs))
+
+	// Load in parallel
+	wg := boundedwaitgroup.New(50)
+	resultsCh := make(chan *queryResults)
+
+	go func() {
+		for blockNum, id := range blockIDs {
+			wg.Add(1)
+
+			go func(blockNum2 int, id2 uuid.UUID) {
+				defer wg.Done()
+
+				// search here
+				q, err := queryBlock(ctx, r, c, blockNum2, id2, tenantID, traceID)
+				if err != nil {
+					fmt.Println("Error querying block:", err)
+					return
+				}
+
+				if q != nil {
+					resultsCh <- q
+				}
+			}(blockNum, id)
+		}
+	}()
+
+	// cheap way to let the wait group get at least one .Add()
+	time.Sleep(time.Second)
+
+	go func() {
+		wg.Wait()
+		close(resultsCh)
+	}()
+
+	var rootSpan *v1.Span
+	var rootSpanResource *v1resource.Resource
+
+	numBlock := 0
+	size := 0
+	spanCount := 0
+
+	firstStartTime := uint64(math.MaxUint64)
+	lastEndTime := uint64(0)
+
+	rootServiceName := ""
+	serviceNameMap := make(map[string]int)
+
+	for q := range resultsCh {
+		numBlock++
+		for _, b := range q.trace.Batches {
+			size += b.Size()
+			for _, attr := range b.Resource.Attributes {
+				if "service.name" == attr.Key {
+					serviceNameMap[attr.Value.GetStringValue()]++
+					break
+				}
+			}
+			for _, scope := range b.ScopeSpans {
+				spanCount += len(scope.Spans)
+				for _, span := range scope.Spans {
+					if span.StartTimeUnixNano < firstStartTime {
+						firstStartTime = span.StartTimeUnixNano
+					}
+					if span.EndTimeUnixNano > lastEndTime {
+						lastEndTime = span.EndTimeUnixNano
+					}
+					if len(span.ParentSpanId) == 0 {
+						rootSpan = span
+						rootSpanResource = b.Resource
+					}
+				}
+			}
+		}
+
+		if rootSpanResource != nil {
+			for _, attr := range rootSpanResource.Attributes {
+				if attr.Key == "service.name" {
+					rootServiceName = attr.Value.GetStringValue()
+					break
+				}
+			}
+		}
+	}
+
+	duration := lastEndTime - firstStartTime
+	durationSecond := duration / 1000000000
+
+	// get top 5 most frequent service names
+	topFiveSortedPL := sortServiceNames(serviceNameMap)
+	topFiveServiceName := make([]string, 5)
+	length := len(topFiveSortedPL)
+	if length > 5 {
+		length = 5
+	}
+	for index := 0; index < length; index++ {
+		topFiveServiceName[index] = topFiveSortedPL[index].Key
+	}
+
+	return &TraceSummary{
+		NumBlock:         numBlock,
+		SpanCount:        spanCount,
+		TraceSize:        size,
+		TraceDuration:    durationSecond,
+		RootServiceName:  rootServiceName,
+		RootSpan:         rootSpan,
+		RootSpanResource: rootSpanResource,
+		ServiceNames:     topFiveServiceName,
+	}, nil
+}
+
+type Pair struct {
+	Key   string
+	Value int
+}
+
+type PairList []Pair
+
+func (p PairList) Len() int           { return len(p) }
+func (p PairList) Less(i, j int) bool { return p[i].Value < p[j].Value }
+func (p PairList) Swap(i, j int)      { p[i], p[j] = p[j], p[i] }
@@ -2,10 +2,12 @@ package main

 import (
 	"context"
+	"errors"
 	"fmt"
 	"time"

 	"github.com/google/uuid"
+
 	"github.com/grafana/tempo/pkg/boundedwaitgroup"
 	"github.com/grafana/tempo/pkg/tempopb"
 	"github.com/grafana/tempo/tempodb"
@@ -51,7 +53,7 @@ func (cmd *searchBlocksCmd) Run(opts *globalOptions) error {

 	ctx := context.Background()

-	blockIDs, err := r.Blocks(ctx, cmd.TenantID)
+	blockIDs, _, err := r.Blocks(ctx, cmd.TenantID)
 	if err != nil {
 		return err
 	}
@@ -69,7 +71,7 @@ func (cmd *searchBlocksCmd) Run(opts *globalOptions) error {

 			// search here
 			meta, err := r.BlockMeta(ctx, id2, cmd.TenantID)
-			if err == backend.ErrDoesNotExist {
+			if errors.Is(err, backend.ErrDoesNotExist) {
 				return
 			}
 			if err != nil {
@@ -6,9 +6,8 @@ import (
 	"sort"

 	"github.com/google/uuid"
-	"github.com/segmentio/parquet-go"
-
-	"github.com/grafana/tempo/tempodb/encoding/vparquet"
+	"github.com/grafana/tempo/tempodb/encoding/vparquet3"
+	"github.com/parquet-go/parquet-go"
 )

 type viewSchemaCmd struct {
@@ -37,7 +36,7 @@ func (cmd *viewSchemaCmd) Run(ctx *globalOptions) error {
 	fmt.Printf("\n***************     block meta    *********************\n\n\n")
 	fmt.Printf("%+v\n", meta)

-	rr := vparquet.NewBackendReaderAt(context.Background(), r, vparquet.DataFileName, meta.BlockID, meta.TenantID)
+	rr := vparquet3.NewBackendReaderAt(context.Background(), r, vparquet3.DataFileName, meta)
 	pf, err := parquet.OpenFile(rr, int64(meta.Size))
 	if err != nil {
 		return err
@@ -52,8 +51,12 @@ func (cmd *viewSchemaCmd) Run(ctx *globalOptions) error {
 			path, _ := getNodePathByIndex(pf.Root(), "", cc.Column())

 			var size int64
-			for pg := 0; pg < cc.OffsetIndex().NumPages(); pg++ {
-				size += cc.OffsetIndex().CompressedPageSize(pg)
+			idx, err := cc.OffsetIndex()
+			if err != nil {
+				return err
+			}
+			for pg := 0; pg < idx.NumPages(); pg++ {
+				size += idx.CompressedPageSize(pg)
 			}

 			columnSizes[path] = columnSizes[path] + size
@@ -27,7 +27,7 @@ type globalOptions struct {
 }

 type backendOptions struct {
-	Backend string `help:"backend to connect to (s3/gcs/local/azure), optional, overrides backend in config file" enum:",s3,gcs,local,azure"`
+	Backend string `help:"backend to connect to (s3/gcs/local/azure), optional, overrides backend in config file" enum:",s3,gcs,local,azure" default:""`
 	Bucket  string `help:"bucket (or path on local backend) to scan, optional, overrides bucket in config file"`

 	S3Endpoint string `name:"s3-endpoint" help:"s3 endpoint (s3.dualstack.us-east-2.amazonaws.com), optional, overrides endpoint in config file"`
@@ -47,6 +47,11 @@ var cli struct {
 		Column            listColumnCmd            `cmd:"" help:"List values in a given column"`
 	} `cmd:""`

+	Analyse struct {
+		Block  analyseBlockCmd  `cmd:"" help:"Analyse block in a bucket"`
+		Blocks analyseBlocksCmd `cmd:"" help:"Analyse blocks in a bucket"`
+	} `cmd:""`
+
 	View struct {
 		Index  viewIndexCmd  `cmd:"" help:"View contents of block index"`
 		Schema viewSchemaCmd `cmd:"" help:"View parquet schema"`
@@ -63,8 +68,10 @@ var cli struct {
 			SearchTags      querySearchTagsCmd      `cmd:"" help:"query Tempo search tags"`
 			SearchTagValues querySearchTagValuesCmd `cmd:"" help:"query Tempo search tag values"`
 			Search          querySearchCmd          `cmd:"" help:"query Tempo search"`
+			Metrics         metricsQueryRangeCmd    `cmd:"" help:"query Tempo metrics query range"`
 		} `cmd:""`
-		Blocks queryBlocksCmd `cmd:"" help:"query for a traceid directly from backend blocks"`
+		Blocks       queryBlocksCmd       `cmd:"" help:"query for a traceid directly from backend blocks"`
+		TraceSummary queryTraceSummaryCmd `cmd:"" help:"query summary for a traceid directly from backend blocks"`
 	} `cmd:""`

 	Search struct {
@@ -72,12 +79,13 @@ var cli struct {
 	} `cmd:""`

 	Parquet struct {
-		Convert     convertParquet     `cmd:"" help:"convert from an existing file to tempodb parquet schema"`
-		Convert1to2 convertParquet1to2 `cmd:"" help:"convert an exiting vParquet file to vParquet2 schema"`
+		Convert2to3 convertParquet2to3 `cmd:"" help:"convert an existing vParquet2 file to vParquet3 block"`
+		Convert3to4 convertParquet3to4 `cmd:"" help:"convert an existing vParquet3 file to vParquet4 block"`
 	} `cmd:""`

 	Migrate struct {
-		Tenant migrateTenantCmd `cmd:"" help:"migrate tenant between two backends"`
+		Tenant          migrateTenantCmd          `cmd:"" help:"migrate tenant between two backends"`
+		OverridesConfig migrateOverridesConfigCmd `cmd:"" help:"migrate overrides config"`
 	} `cmd:""`
 }

@@ -3,12 +3,14 @@ package main
 import (
 	"context"
 	"encoding/json"
+	"errors"
 	"fmt"
 	"sort"
 	"strconv"
 	"time"

 	"github.com/google/uuid"
+
 	"github.com/grafana/tempo/pkg/boundedwaitgroup"
 	"github.com/grafana/tempo/tempodb/backend"
 )
@@ -54,11 +56,13 @@ type blockStats struct {
 }

 func loadBucket(r backend.Reader, c backend.Compactor, tenantID string, windowRange time.Duration, includeCompacted bool) ([]blockStats, error) {
-	blockIDs, err := r.Blocks(context.Background(), tenantID)
+	blockIDs, compactedBlockIDs, err := r.Blocks(context.Background(), tenantID)
 	if err != nil {
 		return nil, err
 	}

+	blockIDs = append(blockIDs, compactedBlockIDs...)
+
 	fmt.Println("total blocks: ", len(blockIDs))

 	// Load in parallel
@@ -105,14 +109,14 @@ func loadBlock(r backend.Reader, c backend.Compactor, tenantID string, id uuid.U
 	}

 	meta, err := r.BlockMeta(context.Background(), id, tenantID)
-	if err == backend.ErrDoesNotExist && !includeCompacted {
+	if errors.Is(err, backend.ErrDoesNotExist) && !includeCompacted {
 		return nil, nil
-	} else if err != nil && err != backend.ErrDoesNotExist {
+	} else if err != nil && !errors.Is(err, backend.ErrDoesNotExist) {
 		return nil, err
 	}

 	compactedMeta, err := c.CompactedBlockMeta(id, tenantID)
-	if err != nil && err != backend.ErrDoesNotExist {
+	if err != nil && !errors.Is(err, backend.ErrDoesNotExist) {
 		return nil, err
 	}

@@ -1,4 +1,4 @@
-FROM jaegertracing/jaeger-query:1.46.0
+FROM jaegertracing/jaeger-query:1.50.0

 ENV SPAN_STORAGE_TYPE=grpc-plugin \
    GRPC_STORAGE_PLUGIN_BINARY=/tempo-query
@@ -2,14 +2,17 @@ package tempo

 import (
 	"github.com/grafana/dskit/crypto/tls"
+	"github.com/jaegertracing/jaeger/plugin/storage/grpc/shared"
 	"github.com/spf13/viper"
 )

 // Config holds the configuration for redbull.
 type Config struct {
-	Backend    string           `yaml:"backend"`
-	TLSEnabled bool             `yaml:"tls_enabled" category:"advanced"`
-	TLS        tls.ClientConfig `yaml:",inline"`
+	Backend               string           `yaml:"backend"`
+	TLSEnabled            bool             `yaml:"tls_enabled" category:"advanced"`
+	TLS                   tls.ClientConfig `yaml:",inline"`
+	TenantHeaderKey       string           `yaml:"tenant_header_key"`
+	QueryServicesDuration string           `yaml:"services_query_duration"`
 }

 // InitFromViper initializes the options struct with values from Viper
@@ -23,4 +26,11 @@ func (c *Config) InitFromViper(v *viper.Viper) {
 	c.TLS.InsecureSkipVerify = v.GetBool("tls_insecure_skip_verify")
 	c.TLS.CipherSuites = v.GetString("tls_cipher_suites")
 	c.TLS.MinVersion = v.GetString("tls_min_version")
+	c.QueryServicesDuration = v.GetString("services_query_duration")
+
+	tenantHeader := v.GetString("tenant_header_key")
+	if tenantHeader == "" {
+		tenantHeader = shared.BearerTokenKey
+	}
+	c.TenantHeaderKey = tenantHeader
 }
@@ -16,15 +16,14 @@ import (
 	"github.com/go-logfmt/logfmt"
 	"github.com/gogo/protobuf/jsonpb"
 	tlsCfg "github.com/grafana/dskit/crypto/tls"
+	"github.com/grafana/dskit/user"
 	"github.com/grafana/tempo/pkg/tempopb"
 	"github.com/opentracing/opentracing-go"
 	ot_log "github.com/opentracing/opentracing-go/log"
-	"github.com/weaveworks/common/user"
 	"go.opentelemetry.io/collector/pdata/ptrace"
 	"google.golang.org/grpc/metadata"

 	jaeger "github.com/jaegertracing/jaeger/model"
-	"github.com/jaegertracing/jaeger/plugin/storage/grpc/shared"
 	jaeger_spanstore "github.com/jaegertracing/jaeger/storage/spanstore"

 	ot_jaeger "github.com/open-telemetry/opentelemetry-collector-contrib/pkg/translator/jaeger"
@@ -54,10 +53,12 @@ var tlsVersions = map[string]uint16{
 }

 type Backend struct {
-	tempoBackend string
-	tlsEnabled   bool
-	tls          tlsCfg.ClientConfig
-	httpClient   *http.Client
+	tempoBackend          string
+	tlsEnabled            bool
+	tls                   tlsCfg.ClientConfig
+	httpClient            *http.Client
+	tenantHeaderKey       string
+	QueryServicesDuration *time.Duration
 }

 func New(cfg *Config) (*Backend, error) {
@@ -65,11 +66,25 @@ func New(cfg *Config) (*Backend, error) {
 	if err != nil {
 		return nil, err
 	}
+
+	var queryServiceDuration *time.Duration
+
+	if cfg.QueryServicesDuration != "" {
+		queryDuration, err := time.ParseDuration(cfg.QueryServicesDuration)
+		if err != nil {
+			return nil, err
+		}
+		queryServiceDuration = &queryDuration
+
+	}
+
 	return &Backend{
-		tempoBackend: cfg.Backend,
-		tlsEnabled:   cfg.TLSEnabled,
-		tls:          cfg.TLS,
-		httpClient:   httpClient,
+		tempoBackend:          cfg.Backend,
+		tlsEnabled:            cfg.TLSEnabled,
+		tls:                   cfg.TLS,
+		httpClient:            httpClient,
+		tenantHeaderKey:       cfg.TenantHeaderKey,
+		QueryServicesDuration: queryServiceDuration,
 	}, nil
 }

@@ -179,7 +194,7 @@ func (b *Backend) GetTrace(ctx context.Context, traceID jaeger.TraceID) (*jaeger

 	resp, err := b.httpClient.Do(req)
 	if err != nil {
-		return nil, fmt.Errorf("failed GET to tempo %w", err)
+		return nil, fmt.Errorf("failed GET to tempo: %w", err)
 	}
 	defer resp.Body.Close()

@@ -228,6 +243,12 @@ func (b *Backend) GetTrace(ctx context.Context, traceID jaeger.TraceID) (*jaeger
 	return jaegerTrace, nil
 }

+func (b *Backend) calculateTimeRange() (int64, int64) {
+	now := time.Now()
+	start := now.Add(*b.QueryServicesDuration * -1)
+	return start.Unix(), now.Unix()
+}
+
 func (b *Backend) GetServices(ctx context.Context) ([]string, error) {
 	span, ctx := opentracing.StartSpanFromContext(ctx, "tempo-query.GetOperations")
 	defer span.Finish()
@@ -319,7 +340,7 @@ func (b *Backend) FindTraceIDs(ctx context.Context, query *jaeger_spanstore.Trac

 	resp, err := b.httpClient.Do(req)
 	if err != nil {
-		return nil, fmt.Errorf("failed GET to tempo %w", err)
+		return nil, fmt.Errorf("failed GET to tempo: %w", err)
 	}
 	defer resp.Body.Close()

@@ -347,7 +368,7 @@ func (b *Backend) FindTraceIDs(ctx context.Context, query *jaeger_spanstore.Trac
 	for i, traceMetadata := range searchResponse.Traces {
 		jaegerTraceID, err := jaeger.TraceIDFromString(traceMetadata.TraceID)
 		if err != nil {
-			return nil, fmt.Errorf("could not convert traceID into Jaeger's traceID %w", err)
+			return nil, fmt.Errorf("could not convert traceID into Jaeger's traceID: %w", err)
 		}
 		jaegerTraceIDs[i] = jaegerTraceID
 	}
@@ -378,7 +399,14 @@ func createTagsQueryParam(service string, operation string, tags map[string]stri
 }

 func (b *Backend) lookupTagValues(ctx context.Context, span opentracing.Span, tagName string) ([]string, error) {
-	url := fmt.Sprintf("%s://%s/api/search/tag/%s/values", b.apiSchema(), b.tempoBackend, tagName)
+	var url string
+
+	if b.QueryServicesDuration == nil {
+		url = fmt.Sprintf("%s://%s/api/search/tag/%s/values", b.apiSchema(), b.tempoBackend, tagName)
+	} else {
+		startTime, endTime := b.calculateTimeRange()
+		url = fmt.Sprintf("%s://%s/api/search/tag/%s/values?start=%d&end=%d", b.apiSchema(), b.tempoBackend, tagName, startTime, endTime)
+	}

 	req, err := b.newGetRequest(ctx, url, span)
 	if err != nil {
@@ -387,7 +415,7 @@ func (b *Backend) lookupTagValues(ctx context.Context, span opentracing.Span, ta

 	resp, err := b.httpClient.Do(req)
 	if err != nil {
-		return nil, fmt.Errorf("failed GET to tempo %w", err)
+		return nil, fmt.Errorf("failed GET to tempo: %w", err)
 	}
 	defer resp.Body.Close()

@@ -434,7 +462,7 @@ func (b *Backend) newGetRequest(ctx context.Context, url string, span opentracin

 	// currently Jaeger Query will only propagate bearer token to the grpc backend and no other headers
 	// so we are going to extract the tenant id from the header, if it exists and use it
-	tenantID, found := extractBearerToken(ctx)
+	tenantID, found := extractBearerToken(ctx, b.tenantHeaderKey)
 	if found {
 		req.Header.Set(user.OrgIDHeaderName, tenantID)
 	}
@@ -442,9 +470,9 @@ func (b *Backend) newGetRequest(ctx context.Context, url string, span opentracin
 	return req, nil
 }

-func extractBearerToken(ctx context.Context) (string, bool) {
+func extractBearerToken(ctx context.Context, tenantHeader string) (string, bool) {
 	if md, ok := metadata.FromIncomingContext(ctx); ok {
-		values := md.Get(shared.BearerTokenKey)
+		values := md.Get(tenantHeader)
 		if len(values) > 0 {
 			return values[0], true
 		}
@@ -9,7 +9,7 @@
 # build the serverless container image
 #
 #  todo: FROM scratch saves ~5MB which could be meaningful in a serverless setting, but using scratch gave strange errors on query.
-FROM alpine:3.16 as certs
+FROM alpine:3.20 as certs
 RUN apk --update add ca-certificates
 COPY ./main /main
 ENTRYPOINT ["/main"]
@@ -1,6 +1,8 @@
 module github.com/grafana/tempo/cmd/tempo-serverless/cloud-run

-go 1.20
+go 1.21
+
+toolchain go1.21.0

 require (
 	github.com/gogo/protobuf v1.3.2
@@ -8,139 +10,153 @@ require (
 )

 require (
-	cloud.google.com/go v0.107.0 // indirect
-	cloud.google.com/go/compute v1.18.0 // indirect
-	cloud.google.com/go/compute/metadata v0.2.3 // indirect
-	cloud.google.com/go/iam v0.11.0 // indirect
-	cloud.google.com/go/storage v1.27.0 // indirect
+	cloud.google.com/go v0.112.0 // indirect
+	cloud.google.com/go/compute v1.24.0 // indirect
+	cloud.google.com/go/compute/metadata v0.2.4-0.20230617002413-005d2dfb6b68 // indirect
+	cloud.google.com/go/iam v1.1.6 // indirect
+	cloud.google.com/go/storage v1.36.0 // indirect
 	github.com/Azure/azure-pipeline-go v0.2.3 // indirect
+	github.com/Azure/azure-sdk-for-go/sdk/azcore v1.8.0 // indirect
+	github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.4.0 // indirect
+	github.com/Azure/azure-sdk-for-go/sdk/internal v1.3.0 // indirect
+	github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.2.0 // indirect
 	github.com/Azure/azure-storage-blob-go v0.15.0 // indirect
 	github.com/Azure/go-autorest v14.2.0+incompatible // indirect
-	github.com/Azure/go-autorest/autorest v0.11.28 // indirect
-	github.com/Azure/go-autorest/autorest/adal v0.9.21 // indirect
-	github.com/Azure/go-autorest/autorest/azure/auth v0.5.11 // indirect
+	github.com/Azure/go-autorest/autorest v0.11.29 // indirect
+	github.com/Azure/go-autorest/autorest/adal v0.9.23 // indirect
+	github.com/Azure/go-autorest/autorest/azure/auth v0.5.12 // indirect
 	github.com/Azure/go-autorest/autorest/azure/cli v0.4.5 // indirect
 	github.com/Azure/go-autorest/autorest/date v0.3.0 // indirect
 	github.com/Azure/go-autorest/logger v0.2.1 // indirect
 	github.com/Azure/go-autorest/tracing v0.6.0 // indirect
+	github.com/AzureAD/microsoft-authentication-library-for-go v1.1.1 // indirect
 	github.com/VividCortex/gohistogram v1.0.0 // indirect
 	github.com/alecthomas/units v0.0.0-20211218093645-b94a6e3cc137 // indirect
-	github.com/andybalholm/brotli v1.0.4 // indirect
-	github.com/apache/thrift v0.18.1 // indirect
-	github.com/aws/aws-sdk-go v1.44.220 // indirect
+	github.com/andybalholm/brotli v1.0.6 // indirect
+	github.com/apache/thrift v0.20.0 // indirect
+	github.com/aws/aws-sdk-go v1.51.11 // indirect
 	github.com/beorn7/perks v1.0.1 // indirect
 	github.com/cespare/xxhash v1.1.0 // indirect
 	github.com/cespare/xxhash/v2 v2.2.0 // indirect
-	github.com/coreos/go-systemd/v22 v22.4.0 // indirect
-	github.com/cristalhq/hedgedhttp v0.7.0 // indirect
+	github.com/coreos/go-systemd/v22 v22.5.0 // indirect
+	github.com/cristalhq/hedgedhttp v0.9.1 // indirect
 	github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect
 	github.com/dimchansky/utfbom v1.1.1 // indirect
 	github.com/dustin/go-humanize v1.0.1 // indirect
 	github.com/facette/natsort v0.0.0-20181210072756-2cd4dd1e2dcb // indirect
-	github.com/felixge/httpsnoop v1.0.3 // indirect
-	github.com/fsnotify/fsnotify v1.6.0 // indirect
-	github.com/go-kit/kit v0.12.0 // indirect
+	github.com/felixge/httpsnoop v1.0.4 // indirect
+	github.com/fsnotify/fsnotify v1.7.0 // indirect
+	github.com/go-kit/kit v0.13.0 // indirect
 	github.com/go-kit/log v0.2.1 // indirect
-	github.com/go-logfmt/logfmt v0.5.1 // indirect
+	github.com/go-logfmt/logfmt v0.6.0 // indirect
+	github.com/go-logr/logr v1.4.1 // indirect
+	github.com/go-logr/stdr v1.2.2 // indirect
 	github.com/go-redis/redis/v8 v8.11.5 // indirect
 	github.com/gogo/googleapis v1.4.1 // indirect
 	github.com/gogo/status v1.1.1 // indirect
-	github.com/golang-jwt/jwt/v4 v4.2.0 // indirect
+	github.com/golang-jwt/jwt/v4 v4.5.0 // indirect
+	github.com/golang-jwt/jwt/v5 v5.0.0 // indirect
 	github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
-	github.com/golang/protobuf v1.5.3 // indirect
+	github.com/golang/protobuf v1.5.4 // indirect
 	github.com/golang/snappy v0.0.4 // indirect
-	github.com/google/go-cmp v0.5.9 // indirect
-	github.com/google/uuid v1.3.0 // indirect
-	github.com/googleapis/enterprise-certificate-proxy v0.2.3 // indirect
-	github.com/googleapis/gax-go/v2 v2.7.0 // indirect
+	github.com/google/s2a-go v0.1.7 // indirect
+	github.com/google/uuid v1.6.0 // indirect
+	github.com/googleapis/enterprise-certificate-proxy v0.3.2 // indirect
+	github.com/googleapis/gax-go/v2 v2.12.0 // indirect
 	github.com/gorilla/handlers v1.5.1 // indirect
-	github.com/gorilla/mux v1.8.0 // indirect
-	github.com/grafana/dskit v0.0.0-20230417151531-1725bcc1e9a1 // indirect
-	github.com/grafana/gomemcache v0.0.0-20230316202710-a081dae0aba9 // indirect
-	github.com/grpc-ecosystem/go-grpc-middleware v1.3.0 // indirect
+	github.com/gorilla/mux v1.8.1 // indirect
+	github.com/grafana/dskit v0.0.0-20240311184239-73feada6c0d7 // indirect
+	github.com/grafana/gomemcache v0.0.0-20240229205252-cd6a66d6fb56 // indirect
+	github.com/grafana/pyroscope-go/godeltaprof v0.1.6 // indirect
+	github.com/grafana/regexp v0.0.0-20221123153739-15dc172cd2db // indirect
+	github.com/grpc-ecosystem/go-grpc-middleware v1.4.0 // indirect
 	github.com/hashicorp/hcl v1.0.0 // indirect
-	github.com/inconshreveable/mousetrap v1.0.1 // indirect
-	github.com/jaegertracing/jaeger v1.41.0 // indirect
+	github.com/inconshreveable/mousetrap v1.1.0 // indirect
+	github.com/jaegertracing/jaeger v1.55.0 // indirect
 	github.com/jmespath/go-jmespath v0.4.0 // indirect
 	github.com/jpillora/backoff v1.0.0 // indirect
 	github.com/json-iterator/go v1.1.12 // indirect
-	github.com/klauspost/compress v1.16.5 // indirect
-	github.com/klauspost/cpuid/v2 v2.2.4 // indirect
-	github.com/magiconair/properties v1.8.6 // indirect
-	github.com/mattn/go-ieproxy v0.0.1 // indirect
-	github.com/mattn/go-runewidth v0.0.13 // indirect
-	github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect
-	github.com/miekg/dns v1.1.50 // indirect
+	github.com/klauspost/compress v1.17.7 // indirect
+	github.com/klauspost/cpuid/v2 v2.2.5 // indirect
+	github.com/kylelemons/godebug v1.1.0 // indirect
+	github.com/magiconair/properties v1.8.7 // indirect
+	github.com/mattn/go-ieproxy v0.0.11 // indirect
+	github.com/mattn/go-runewidth v0.0.15 // indirect
+	github.com/miekg/dns v1.1.56 // indirect
 	github.com/minio/md5-simd v1.1.2 // indirect
-	github.com/minio/minio-go/v7 v7.0.52 // indirect
-	github.com/minio/sha256-simd v1.0.0 // indirect
+	github.com/minio/minio-go/v7 v7.0.63 // indirect
+	github.com/minio/sha256-simd v1.0.1 // indirect
 	github.com/mitchellh/go-homedir v1.1.0 // indirect
-	github.com/mitchellh/mapstructure v1.5.0 // indirect
+	github.com/mitchellh/mapstructure v1.5.1-0.20231216201459-8508981c8b6c // indirect
 	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
 	github.com/modern-go/reflect2 v1.0.2 // indirect
 	github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f // indirect
 	github.com/olekukonko/tablewriter v0.0.5 // indirect
-	github.com/open-telemetry/opentelemetry-collector-contrib/internal/coreinternal v0.74.0 // indirect
-	github.com/open-telemetry/opentelemetry-collector-contrib/pkg/translator/jaeger v0.74.0 // indirect
+	github.com/open-telemetry/opentelemetry-collector-contrib/internal/coreinternal v0.97.0 // indirect
+	github.com/open-telemetry/opentelemetry-collector-contrib/pkg/translator/jaeger v0.97.0 // indirect
 	github.com/opentracing-contrib/go-grpc v0.0.0-20210225150812-73cb765af46e // indirect
 	github.com/opentracing-contrib/go-stdlib v1.0.0 // indirect
 	github.com/opentracing/opentracing-go v1.2.0 // indirect
-	github.com/pelletier/go-toml v1.9.5 // indirect
-	github.com/pelletier/go-toml/v2 v2.0.5 // indirect
-	github.com/pierrec/lz4/v4 v4.1.17 // indirect
+	github.com/parquet-go/parquet-go v0.20.2-0.20240416173845-962b3c5827c3 // indirect
+	github.com/pelletier/go-toml/v2 v2.1.0 // indirect
+	github.com/pierrec/lz4/v4 v4.1.21 // indirect
+	github.com/pkg/browser v0.0.0-20210911075715-681adbf594b8 // indirect
 	github.com/pkg/errors v0.9.1 // indirect
-	github.com/prometheus/client_golang v1.14.0 // indirect
-	github.com/prometheus/client_model v0.3.0 // indirect
-	github.com/prometheus/common v0.42.0 // indirect
-	github.com/prometheus/exporter-toolkit v0.8.2 // indirect
-	github.com/prometheus/procfs v0.9.0 // indirect
-	github.com/prometheus/prometheus v1.8.2-0.20221021121301-51a44e6657c3 // indirect
-	github.com/rivo/uniseg v0.2.0 // indirect
+	github.com/prometheus/client_golang v1.19.0 // indirect
+	github.com/prometheus/client_model v0.6.0 // indirect
+	github.com/prometheus/common v0.49.0 // indirect
+	github.com/prometheus/exporter-toolkit v0.10.1-0.20230714054209-2f4150c63f97 // indirect
+	github.com/prometheus/procfs v0.12.0 // indirect
+	github.com/prometheus/prometheus v0.48.1 // indirect
+	github.com/rivo/uniseg v0.4.4 // indirect
 	github.com/rs/xid v1.5.0 // indirect
-	github.com/segmentio/encoding v0.3.5 // indirect
-	github.com/segmentio/parquet-go v0.0.0-20230427215636-d483faba23a5 // indirect
-	github.com/sercand/kuberesolver/v4 v4.0.0 // indirect
-	github.com/sirupsen/logrus v1.9.0 // indirect
-	github.com/soheilhy/cmux v0.1.5 // indirect
+	github.com/sagikazarmark/locafero v0.4.0 // indirect
+	github.com/sagikazarmark/slog-shim v0.1.0 // indirect
+	github.com/segmentio/encoding v0.3.6 // indirect
+	github.com/sercand/kuberesolver/v5 v5.1.1 // indirect
+	github.com/sirupsen/logrus v1.9.3 // indirect
 	github.com/sony/gobreaker v0.4.1 // indirect
+	github.com/sourcegraph/conc v0.3.0 // indirect
 	github.com/spaolacci/murmur3 v1.1.0 // indirect
-	github.com/spf13/afero v1.9.2 // indirect
-	github.com/spf13/cast v1.5.0 // indirect
-	github.com/spf13/cobra v1.6.1 // indirect
-	github.com/spf13/jwalterweatherman v1.1.0 // indirect
+	github.com/spf13/afero v1.11.0 // indirect
+	github.com/spf13/cast v1.6.0 // indirect
+	github.com/spf13/cobra v1.8.0 // indirect
 	github.com/spf13/pflag v1.0.5 // indirect
-	github.com/spf13/viper v1.14.0 // indirect
-	github.com/subosito/gotenv v1.4.1 // indirect
+	github.com/spf13/viper v1.18.2 // indirect
+	github.com/subosito/gotenv v1.6.0 // indirect
 	github.com/uber-go/atomic v1.4.0 // indirect
 	github.com/uber/jaeger-client-go v2.30.0+incompatible // indirect
 	github.com/uber/jaeger-lib v2.4.1+incompatible // indirect
-	github.com/weaveworks/common v0.0.0-20230511094633-334485600903 // indirect
-	github.com/weaveworks/promrus v1.2.0 // indirect
 	github.com/willf/bitset v1.1.11 // indirect
 	github.com/willf/bloom v2.0.3+incompatible // indirect
 	go.opencensus.io v0.24.0 // indirect
-	go.opentelemetry.io/collector/pdata v1.0.0-rc8 // indirect
-	go.opentelemetry.io/collector/semconv v0.75.0 // indirect
-	go.opentelemetry.io/otel v1.14.0 // indirect
-	go.opentelemetry.io/otel/trace v1.14.0 // indirect
-	go.uber.org/atomic v1.10.0 // indirect
-	go.uber.org/multierr v1.10.0 // indirect
-	go.uber.org/zap v1.24.0 // indirect
-	golang.org/x/crypto v0.9.0 // indirect
-	golang.org/x/mod v0.8.0 // indirect
-	golang.org/x/net v0.10.0 // indirect
-	golang.org/x/oauth2 v0.5.0 // indirect
-	golang.org/x/sync v0.1.0 // indirect
-	golang.org/x/sys v0.8.0 // indirect
-	golang.org/x/text v0.9.0 // indirect
-	golang.org/x/time v0.1.0 // indirect
-	golang.org/x/tools v0.6.0 // indirect
-	golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 // indirect
-	google.golang.org/api v0.110.0 // indirect
-	google.golang.org/appengine v1.6.7 // indirect
-	google.golang.org/genproto v0.0.0-20230223222841-637eb2293923 // indirect
-	google.golang.org/grpc v1.54.0 // indirect
-	google.golang.org/protobuf v1.30.0 // indirect
+	go.opentelemetry.io/collector/pdata v1.4.0 // indirect
+	go.opentelemetry.io/collector/semconv v0.97.0 // indirect
+	go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.51.0 // indirect
+	go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.49.0 // indirect
+	go.opentelemetry.io/otel v1.26.0 // indirect
+	go.opentelemetry.io/otel/metric v1.26.0 // indirect
+	go.opentelemetry.io/otel/trace v1.26.0 // indirect
+	go.uber.org/atomic v1.11.0 // indirect
+	go.uber.org/multierr v1.11.0 // indirect
+	go.uber.org/zap v1.27.0 // indirect
+	golang.org/x/crypto v0.22.0 // indirect
+	golang.org/x/exp v0.0.0-20240103183307-be819d1f06fc // indirect
+	golang.org/x/mod v0.16.0 // indirect
+	golang.org/x/net v0.24.0 // indirect
+	golang.org/x/oauth2 v0.17.0 // indirect
+	golang.org/x/sync v0.6.0 // indirect
+	golang.org/x/sys v0.19.0 // indirect
+	golang.org/x/text v0.14.0 // indirect
+	golang.org/x/time v0.5.0 // indirect
+	golang.org/x/tools v0.16.0 // indirect
+	google.golang.org/api v0.162.0 // indirect
+	google.golang.org/appengine v1.6.8 // indirect
+	google.golang.org/genproto v0.0.0-20240227224415-6ceb2ff114de // indirect
+	google.golang.org/genproto/googleapis/api v0.0.0-20240227224415-6ceb2ff114de // indirect
+	google.golang.org/genproto/googleapis/rpc v0.0.0-20240415180920-8c6c420018be // indirect
+	google.golang.org/grpc v1.63.2 // indirect
+	google.golang.org/protobuf v1.33.0 // indirect
 	gopkg.in/ini.v1 v1.67.0 // indirect
 	gopkg.in/yaml.v2 v2.4.0 // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect
@@ -12,10 +12,9 @@ import (

 	"github.com/google/uuid"
 	"github.com/grafana/dskit/flagext"
+	"github.com/grafana/dskit/user"
 	"github.com/mitchellh/mapstructure"
-	"github.com/pkg/errors"
 	"github.com/spf13/viper"
-	"github.com/weaveworks/common/user"
 	"gopkg.in/yaml.v2"

 	"github.com/grafana/tempo/pkg/api"
@@ -24,6 +23,7 @@ import (
 	"github.com/grafana/tempo/tempodb"
 	"github.com/grafana/tempo/tempodb/backend"
 	"github.com/grafana/tempo/tempodb/backend/azure"
+	azureconfig "github.com/grafana/tempo/tempodb/backend/azure/config"
 	"github.com/grafana/tempo/tempodb/backend/gcs"
 	"github.com/grafana/tempo/tempodb/backend/local"
 	"github.com/grafana/tempo/tempodb/backend/s3"
@@ -82,17 +82,23 @@ func Handler(r *http.Request) (*tempopb.SearchResponse, *HTTPError) {
 		return nil, httpError("parsing encoding", err, http.StatusBadRequest)
 	}

+	dc, err := backend.DedicatedColumnsFromTempopb(searchReq.DedicatedColumns)
+	if err != nil {
+		return nil, httpError("parsing dedicated columns", err, http.StatusBadRequest)
+	}
+
 	// /giphy so meta
 	meta := &backend.BlockMeta{
-		Version:       searchReq.Version,
-		TenantID:      tenant,
-		Encoding:      enc,
-		IndexPageSize: searchReq.IndexPageSize,
-		TotalRecords:  searchReq.TotalRecords,
-		BlockID:       blockID,
-		DataEncoding:  searchReq.DataEncoding,
-		Size:          searchReq.Size_,
-		FooterSize:    searchReq.FooterSize,
+		Version:          searchReq.Version,
+		TenantID:         tenant,
+		Encoding:         enc,
+		IndexPageSize:    searchReq.IndexPageSize,
+		TotalRecords:     searchReq.TotalRecords,
+		BlockID:          blockID,
+		DataEncoding:     searchReq.DataEncoding,
+		Size:             searchReq.Size_,
+		FooterSize:       searchReq.FooterSize,
+		DedicatedColumns: dc,
 	}

 	block, err := encoding.OpenBlock(meta, reader)
@@ -178,18 +184,18 @@ func loadConfig() (*tempodb.Config, error) {
 		Local: &local.Config{},
 		GCS:   &gcs.Config{},
 		S3:    &s3.Config{},
-		Azure: &azure.Config{},
+		Azure: &azureconfig.Config{},
 	}

 	// horrible viper dance since it won't unmarshal to a struct from env: https://github.com/spf13/viper/issues/188
 	v := viper.NewWithOptions()
 	b, err := yaml.Marshal(defaultConfig)
 	if err != nil {
-		return nil, errors.Wrap(err, "failed to marshal default config")
+		return nil, fmt.Errorf("failed to marshal default config: %w", err)
 	}
 	v.SetConfigType("yaml")
 	if err = v.MergeConfig(bytes.NewReader(b)); err != nil {
-		return nil, errors.Wrap(err, "failed to merge config")
+		return nil, fmt.Errorf("failed to merge config: %w", err)
 	}

 	v.AutomaticEnv()
@@ -199,7 +205,7 @@ func loadConfig() (*tempodb.Config, error) {
 	cfg := &tempodb.Config{}
 	err = v.Unmarshal(cfg, setTagName, setDecodeHooks)
 	if err != nil {
-		return nil, errors.Wrap(err, "failed to unmarshal config")
+		return nil, fmt.Errorf("failed to unmarshal config: %w", err)
 	}

 	return cfg, nil
@@ -13,7 +13,7 @@
 #
 # build the lambda and retrive the lambda-local-proxy
 #
-FROM golang:1.20.4-buster AS build
+FROM golang:1.21.3 AS build

 # copy in the lambda. todo: build in container
 COPY lambda /
@@ -1,150 +1,166 @@
 module github.com/grafana/tempo/cmd/tempo-serverless/lambda

-go 1.20
+go 1.21
+
+toolchain go1.21.0

 require (
 	github.com/aws/aws-lambda-go v1.28.0
 	github.com/gogo/protobuf v1.3.2
 	github.com/grafana/tempo v0.0.0-00010101000000-000000000000
-	github.com/stretchr/testify v1.8.2
+	github.com/stretchr/testify v1.9.0
 )

 require (
-	cloud.google.com/go v0.107.0 // indirect
-	cloud.google.com/go/compute v1.18.0 // indirect
-	cloud.google.com/go/compute/metadata v0.2.3 // indirect
-	cloud.google.com/go/iam v0.11.0 // indirect
-	cloud.google.com/go/storage v1.27.0 // indirect
+	cloud.google.com/go v0.112.0 // indirect
+	cloud.google.com/go/compute v1.24.0 // indirect
+	cloud.google.com/go/compute/metadata v0.2.4-0.20230617002413-005d2dfb6b68 // indirect
+	cloud.google.com/go/iam v1.1.6 // indirect
+	cloud.google.com/go/storage v1.36.0 // indirect
 	github.com/Azure/azure-pipeline-go v0.2.3 // indirect
+	github.com/Azure/azure-sdk-for-go/sdk/azcore v1.8.0 // indirect
+	github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.4.0 // indirect
+	github.com/Azure/azure-sdk-for-go/sdk/internal v1.3.0 // indirect
+	github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.2.0 // indirect
 	github.com/Azure/azure-storage-blob-go v0.15.0 // indirect
 	github.com/Azure/go-autorest v14.2.0+incompatible // indirect
-	github.com/Azure/go-autorest/autorest v0.11.28 // indirect
-	github.com/Azure/go-autorest/autorest/adal v0.9.21 // indirect
-	github.com/Azure/go-autorest/autorest/azure/auth v0.5.11 // indirect
+	github.com/Azure/go-autorest/autorest v0.11.29 // indirect
+	github.com/Azure/go-autorest/autorest/adal v0.9.23 // indirect
+	github.com/Azure/go-autorest/autorest/azure/auth v0.5.12 // indirect
 	github.com/Azure/go-autorest/autorest/azure/cli v0.4.5 // indirect
 	github.com/Azure/go-autorest/autorest/date v0.3.0 // indirect
 	github.com/Azure/go-autorest/logger v0.2.1 // indirect
 	github.com/Azure/go-autorest/tracing v0.6.0 // indirect
+	github.com/AzureAD/microsoft-authentication-library-for-go v1.1.1 // indirect
 	github.com/VividCortex/gohistogram v1.0.0 // indirect
 	github.com/alecthomas/units v0.0.0-20211218093645-b94a6e3cc137 // indirect
-	github.com/andybalholm/brotli v1.0.4 // indirect
-	github.com/apache/thrift v0.18.1 // indirect
-	github.com/aws/aws-sdk-go v1.44.220 // indirect
+	github.com/andybalholm/brotli v1.0.6 // indirect
+	github.com/apache/thrift v0.20.0 // indirect
+	github.com/aws/aws-sdk-go v1.51.11 // indirect
 	github.com/beorn7/perks v1.0.1 // indirect
 	github.com/cespare/xxhash v1.1.0 // indirect
 	github.com/cespare/xxhash/v2 v2.2.0 // indirect
-	github.com/coreos/go-systemd/v22 v22.4.0 // indirect
-	github.com/cristalhq/hedgedhttp v0.7.0 // indirect
-	github.com/davecgh/go-spew v1.1.1 // indirect
+	github.com/coreos/go-systemd/v22 v22.5.0 // indirect
+	github.com/cristalhq/hedgedhttp v0.9.1 // indirect
+	github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect
 	github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect
 	github.com/dimchansky/utfbom v1.1.1 // indirect
 	github.com/dustin/go-humanize v1.0.1 // indirect
 	github.com/facette/natsort v0.0.0-20181210072756-2cd4dd1e2dcb // indirect
-	github.com/felixge/httpsnoop v1.0.3 // indirect
-	github.com/fsnotify/fsnotify v1.6.0 // indirect
-	github.com/go-kit/kit v0.12.0 // indirect
+	github.com/felixge/httpsnoop v1.0.4 // indirect
+	github.com/fsnotify/fsnotify v1.7.0 // indirect
+	github.com/go-kit/kit v0.13.0 // indirect
 	github.com/go-kit/log v0.2.1 // indirect
-	github.com/go-logfmt/logfmt v0.5.1 // indirect
+	github.com/go-logfmt/logfmt v0.6.0 // indirect
+	github.com/go-logr/logr v1.4.1 // indirect
+	github.com/go-logr/stdr v1.2.2 // indirect
 	github.com/go-redis/redis/v8 v8.11.5 // indirect
 	github.com/gogo/googleapis v1.4.1 // indirect
 	github.com/gogo/status v1.1.1 // indirect
-	github.com/golang-jwt/jwt/v4 v4.2.0 // indirect
+	github.com/golang-jwt/jwt/v4 v4.5.0 // indirect
+	github.com/golang-jwt/jwt/v5 v5.0.0 // indirect
 	github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
-	github.com/golang/protobuf v1.5.3 // indirect
+	github.com/golang/protobuf v1.5.4 // indirect
 	github.com/golang/snappy v0.0.4 // indirect
-	github.com/google/go-cmp v0.5.9 // indirect
-	github.com/google/uuid v1.3.0 // indirect
-	github.com/googleapis/enterprise-certificate-proxy v0.2.3 // indirect
-	github.com/googleapis/gax-go/v2 v2.7.0 // indirect
+	github.com/google/s2a-go v0.1.7 // indirect
+	github.com/google/uuid v1.6.0 // indirect
+	github.com/googleapis/enterprise-certificate-proxy v0.3.2 // indirect
+	github.com/googleapis/gax-go/v2 v2.12.0 // indirect
 	github.com/gorilla/handlers v1.5.1 // indirect
-	github.com/gorilla/mux v1.8.0 // indirect
-	github.com/grafana/dskit v0.0.0-20230417151531-1725bcc1e9a1 // indirect
-	github.com/grafana/gomemcache v0.0.0-20230316202710-a081dae0aba9 // indirect
-	github.com/grpc-ecosystem/go-grpc-middleware v1.3.0 // indirect
+	github.com/gorilla/mux v1.8.1 // indirect
+	github.com/grafana/dskit v0.0.0-20240311184239-73feada6c0d7 // indirect
+	github.com/grafana/gomemcache v0.0.0-20240229205252-cd6a66d6fb56 // indirect
+	github.com/grafana/pyroscope-go/godeltaprof v0.1.6 // indirect
+	github.com/grafana/regexp v0.0.0-20221123153739-15dc172cd2db // indirect
+	github.com/grpc-ecosystem/go-grpc-middleware v1.4.0 // indirect
 	github.com/hashicorp/hcl v1.0.0 // indirect
-	github.com/inconshreveable/mousetrap v1.0.1 // indirect
-	github.com/jaegertracing/jaeger v1.41.0 // indirect
+	github.com/inconshreveable/mousetrap v1.1.0 // indirect
+	github.com/jaegertracing/jaeger v1.55.0 // indirect
 	github.com/jmespath/go-jmespath v0.4.0 // indirect
 	github.com/jpillora/backoff v1.0.0 // indirect
 	github.com/json-iterator/go v1.1.12 // indirect
-	github.com/klauspost/compress v1.16.5 // indirect
-	github.com/klauspost/cpuid/v2 v2.2.4 // indirect
-	github.com/magiconair/properties v1.8.6 // indirect
-	github.com/mattn/go-ieproxy v0.0.1 // indirect
-	github.com/mattn/go-runewidth v0.0.13 // indirect
-	github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect
-	github.com/miekg/dns v1.1.50 // indirect
+	github.com/klauspost/compress v1.17.7 // indirect
+	github.com/klauspost/cpuid/v2 v2.2.5 // indirect
+	github.com/kylelemons/godebug v1.1.0 // indirect
+	github.com/magiconair/properties v1.8.7 // indirect
+	github.com/mattn/go-ieproxy v0.0.11 // indirect
+	github.com/mattn/go-runewidth v0.0.15 // indirect
+	github.com/miekg/dns v1.1.56 // indirect
 	github.com/minio/md5-simd v1.1.2 // indirect
-	github.com/minio/minio-go/v7 v7.0.52 // indirect
-	github.com/minio/sha256-simd v1.0.0 // indirect
+	github.com/minio/minio-go/v7 v7.0.63 // indirect
+	github.com/minio/sha256-simd v1.0.1 // indirect
 	github.com/mitchellh/go-homedir v1.1.0 // indirect
-	github.com/mitchellh/mapstructure v1.5.0 // indirect
+	github.com/mitchellh/mapstructure v1.5.1-0.20231216201459-8508981c8b6c // indirect
 	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
 	github.com/modern-go/reflect2 v1.0.2 // indirect
 	github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f // indirect
 	github.com/olekukonko/tablewriter v0.0.5 // indirect
-	github.com/open-telemetry/opentelemetry-collector-contrib/internal/coreinternal v0.74.0 // indirect
-	github.com/open-telemetry/opentelemetry-collector-contrib/pkg/translator/jaeger v0.74.0 // indirect
+	github.com/open-telemetry/opentelemetry-collector-contrib/internal/coreinternal v0.97.0 // indirect
+	github.com/open-telemetry/opentelemetry-collector-contrib/pkg/translator/jaeger v0.97.0 // indirect
 	github.com/opentracing-contrib/go-grpc v0.0.0-20210225150812-73cb765af46e // indirect
 	github.com/opentracing-contrib/go-stdlib v1.0.0 // indirect
 	github.com/opentracing/opentracing-go v1.2.0 // indirect
-	github.com/pelletier/go-toml v1.9.5 // indirect
-	github.com/pelletier/go-toml/v2 v2.0.5 // indirect
-	github.com/pierrec/lz4/v4 v4.1.17 // indirect
+	github.com/parquet-go/parquet-go v0.20.2-0.20240416173845-962b3c5827c3 // indirect
+	github.com/pelletier/go-toml/v2 v2.1.0 // indirect
+	github.com/pierrec/lz4/v4 v4.1.21 // indirect
+	github.com/pkg/browser v0.0.0-20210911075715-681adbf594b8 // indirect
 	github.com/pkg/errors v0.9.1 // indirect
 	github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect
-	github.com/prometheus/client_golang v1.14.0 // indirect
-	github.com/prometheus/client_model v0.3.0 // indirect
-	github.com/prometheus/common v0.42.0 // indirect
-	github.com/prometheus/exporter-toolkit v0.8.2 // indirect
-	github.com/prometheus/procfs v0.9.0 // indirect
-	github.com/prometheus/prometheus v1.8.2-0.20221021121301-51a44e6657c3 // indirect
-	github.com/rivo/uniseg v0.2.0 // indirect
+	github.com/prometheus/client_golang v1.19.0 // indirect
+	github.com/prometheus/client_model v0.6.0 // indirect
+	github.com/prometheus/common v0.49.0 // indirect
+	github.com/prometheus/exporter-toolkit v0.10.1-0.20230714054209-2f4150c63f97 // indirect
+	github.com/prometheus/procfs v0.12.0 // indirect
+	github.com/prometheus/prometheus v0.48.1 // indirect
+	github.com/rivo/uniseg v0.4.4 // indirect
 	github.com/rs/xid v1.5.0 // indirect
-	github.com/segmentio/encoding v0.3.5 // indirect
-	github.com/segmentio/parquet-go v0.0.0-20230427215636-d483faba23a5 // indirect
-	github.com/sercand/kuberesolver/v4 v4.0.0 // indirect
-	github.com/sirupsen/logrus v1.9.0 // indirect
-	github.com/soheilhy/cmux v0.1.5 // indirect
+	github.com/sagikazarmark/locafero v0.4.0 // indirect
+	github.com/sagikazarmark/slog-shim v0.1.0 // indirect
+	github.com/segmentio/encoding v0.3.6 // indirect
+	github.com/sercand/kuberesolver/v5 v5.1.1 // indirect
+	github.com/sirupsen/logrus v1.9.3 // indirect
 	github.com/sony/gobreaker v0.4.1 // indirect
+	github.com/sourcegraph/conc v0.3.0 // indirect
 	github.com/spaolacci/murmur3 v1.1.0 // indirect
-	github.com/spf13/afero v1.9.2 // indirect
-	github.com/spf13/cast v1.5.0 // indirect
-	github.com/spf13/cobra v1.6.1 // indirect
-	github.com/spf13/jwalterweatherman v1.1.0 // indirect
+	github.com/spf13/afero v1.11.0 // indirect
+	github.com/spf13/cast v1.6.0 // indirect
+	github.com/spf13/cobra v1.8.0 // indirect
 	github.com/spf13/pflag v1.0.5 // indirect
-	github.com/spf13/viper v1.14.0 // indirect
-	github.com/subosito/gotenv v1.4.1 // indirect
+	github.com/spf13/viper v1.18.2 // indirect
+	github.com/subosito/gotenv v1.6.0 // indirect
 	github.com/uber-go/atomic v1.4.0 // indirect
 	github.com/uber/jaeger-client-go v2.30.0+incompatible // indirect
 	github.com/uber/jaeger-lib v2.4.1+incompatible // indirect
-	github.com/weaveworks/common v0.0.0-20230511094633-334485600903 // indirect
-	github.com/weaveworks/promrus v1.2.0 // indirect
 	github.com/willf/bitset v1.1.11 // indirect
 	github.com/willf/bloom v2.0.3+incompatible // indirect
 	go.opencensus.io v0.24.0 // indirect
-	go.opentelemetry.io/collector/pdata v1.0.0-rc8 // indirect
-	go.opentelemetry.io/collector/semconv v0.75.0 // indirect
-	go.opentelemetry.io/otel v1.14.0 // indirect
-	go.opentelemetry.io/otel/trace v1.14.0 // indirect
-	go.uber.org/atomic v1.10.0 // indirect
-	go.uber.org/multierr v1.10.0 // indirect
-	go.uber.org/zap v1.24.0 // indirect
-	golang.org/x/crypto v0.9.0 // indirect
-	golang.org/x/mod v0.8.0 // indirect
-	golang.org/x/net v0.10.0 // indirect
-	golang.org/x/oauth2 v0.5.0 // indirect
-	golang.org/x/sync v0.1.0 // indirect
-	golang.org/x/sys v0.8.0 // indirect
-	golang.org/x/text v0.9.0 // indirect
-	golang.org/x/time v0.1.0 // indirect
-	golang.org/x/tools v0.6.0 // indirect
-	golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 // indirect
-	google.golang.org/api v0.110.0 // indirect
-	google.golang.org/appengine v1.6.7 // indirect
-	google.golang.org/genproto v0.0.0-20230223222841-637eb2293923 // indirect
-	google.golang.org/grpc v1.54.0 // indirect
-	google.golang.org/protobuf v1.30.0 // indirect
+	go.opentelemetry.io/collector/pdata v1.4.0 // indirect
+	go.opentelemetry.io/collector/semconv v0.97.0 // indirect
+	go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.51.0 // indirect
+	go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.49.0 // indirect
+	go.opentelemetry.io/otel v1.26.0 // indirect
+	go.opentelemetry.io/otel/metric v1.26.0 // indirect
+	go.opentelemetry.io/otel/trace v1.26.0 // indirect
+	go.uber.org/atomic v1.11.0 // indirect
+	go.uber.org/multierr v1.11.0 // indirect
+	go.uber.org/zap v1.27.0 // indirect
+	golang.org/x/crypto v0.22.0 // indirect
+	golang.org/x/exp v0.0.0-20240103183307-be819d1f06fc // indirect
+	golang.org/x/mod v0.16.0 // indirect
+	golang.org/x/net v0.24.0 // indirect
+	golang.org/x/oauth2 v0.17.0 // indirect
+	golang.org/x/sync v0.6.0 // indirect
+	golang.org/x/sys v0.19.0 // indirect
+	golang.org/x/text v0.14.0 // indirect
+	golang.org/x/time v0.5.0 // indirect
+	golang.org/x/tools v0.16.0 // indirect
+	google.golang.org/api v0.162.0 // indirect
+	google.golang.org/appengine v1.6.8 // indirect
+	google.golang.org/genproto v0.0.0-20240227224415-6ceb2ff114de // indirect
+	google.golang.org/genproto/googleapis/api v0.0.0-20240227224415-6ceb2ff114de // indirect
+	google.golang.org/genproto/googleapis/rpc v0.0.0-20240415180920-8c6c420018be // indirect
+	google.golang.org/grpc v1.63.2 // indirect
+	google.golang.org/protobuf v1.33.0 // indirect
 	gopkg.in/ini.v1 v1.67.0 // indirect
 	gopkg.in/yaml.v2 v2.4.0 // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect
@@ -1,4 +1,4 @@
-FROM alpine:3.16 as certs
+FROM alpine:3.20 as certs
 RUN apk --update add ca-certificates
 ARG TARGETARCH
 COPY bin/linux/tempo-vulture-${TARGETARCH} /tempo-vulture
@@ -3,6 +3,7 @@ package main
 import (
 	"bytes"
 	"crypto/tls"
+	"errors"
 	"flag"
 	"fmt"
 	"log"
@@ -317,7 +318,7 @@ func newJaegerGRPCClient(endpoint string) (*jaeger_grpc.Reporter, error) {
 	return jaeger_grpc.NewReporter(conn, nil, logger), err
 }

-func generateRandomInt(min int64, max int64, r *rand.Rand) int64 {
+func generateRandomInt(min, max int64, r *rand.Rand) int64 {
 	min++
 	number := min + r.Int63n(max-min)
 	return number
@@ -436,7 +437,7 @@ func searchTraceql(client *httpclient.Client, seed time.Time) (traceMetrics, err

 	start := seed.Add(-30 * time.Minute).Unix()
 	end := seed.Add(30 * time.Minute).Unix()
-	resp, err := client.SearchTraceQLWithRange(fmt.Sprintf(`{span.%s = "%s"}`, attr.Key, util.StringifyAnyValue(attr.Value)), start, end)
+	resp, err := client.SearchTraceQLWithRange(fmt.Sprintf(`{.%s = "%s"}`, attr.Key, util.StringifyAnyValue(attr.Value)), start, end)
 	if err != nil {
 		logger.Error(fmt.Sprintf("failed to search traces with traceql %s: %s", attr.Key, err.Error()))
 		tm.requestFailed++
@@ -467,7 +468,7 @@ func queryTrace(client *httpclient.Client, info *util.TraceInfo) (traceMetrics,

 	trace, err := client.QueryTrace(hexID)
 	if err != nil {
-		if err == util.ErrTraceNotFound {
+		if errors.Is(err, util.ErrTraceNotFound) {
 			tm.notFoundByID++
 		} else {
 			tm.requestFailed++
@@ -513,8 +514,8 @@ func queryTrace(client *httpclient.Client, info *util.TraceInfo) (traceMetrics,
 }

 func equalTraces(a, b *tempopb.Trace) bool {
-	trace.SortTrace(a)
-	trace.SortTrace(b)
+	trace.SortTraceAndAttributes(a)
+	trace.SortTraceAndAttributes(b)

 	return reflect.DeepEqual(a, b)
 }
@@ -1,5 +1,14 @@
-FROM alpine:3.16 as certs
+FROM alpine:3.20 as certs
 RUN apk --update add ca-certificates
 ARG TARGETARCH
 COPY bin/linux/tempo-${TARGETARCH} /tempo
+
+RUN addgroup -g 10001 -S tempo && \
+    adduser -u 10001 -S tempo -G tempo
+
+RUN mkdir -p /var/tempo && \
+    chown -R tempo:tempo /var/tempo
+
+USER 10001:10001
+
 ENTRYPOINT ["/tempo"]
@@ -2,7 +2,7 @@ FROM golang:alpine AS build-dlv

 RUN go install github.com/go-delve/delve/cmd/dlv@latest

-FROM alpine:3.16 as certs
+FROM alpine:3.20 as certs
 RUN apk --update add ca-certificates bash
 ARG TARGETARCH
 COPY bin/linux/tempo-${TARGETARCH} /tempo
@@ -3,29 +3,33 @@ package app
 import (
 	"bytes"
 	"context"
+	"encoding/json"
+	"errors"
 	"fmt"
 	"io"
 	"net/http"
 	"sort"
+	"time"

 	"github.com/go-kit/log/level"
 	"github.com/gorilla/mux"
 	"github.com/grafana/dskit/grpcutil"
 	"github.com/grafana/dskit/kv/memberlist"
+	"github.com/grafana/dskit/middleware"
 	"github.com/grafana/dskit/modules"
 	"github.com/grafana/dskit/ring"
+	"github.com/grafana/dskit/server"
 	"github.com/grafana/dskit/services"
+	"github.com/grafana/dskit/signals"
 	"github.com/jedib0t/go-pretty/v6/table"
-	"github.com/pkg/errors"
 	"github.com/prometheus/client_golang/prometheus"
 	"github.com/prometheus/common/version"
-	"github.com/weaveworks/common/middleware"
-	"github.com/weaveworks/common/server"
-	"github.com/weaveworks/common/signals"
+	"go.uber.org/atomic"
 	"google.golang.org/grpc"
 	"google.golang.org/grpc/health/grpc_health_v1"
 	"gopkg.in/yaml.v3"

+	"github.com/grafana/tempo/cmd/tempo/build"
 	"github.com/grafana/tempo/modules/compactor"
 	"github.com/grafana/tempo/modules/distributor"
 	"github.com/grafana/tempo/modules/distributor/receiver"
@@ -35,9 +39,12 @@ import (
 	"github.com/grafana/tempo/modules/overrides"
 	"github.com/grafana/tempo/modules/querier"
 	"github.com/grafana/tempo/modules/storage"
+	"github.com/grafana/tempo/pkg/api"
+	"github.com/grafana/tempo/pkg/cache"
 	"github.com/grafana/tempo/pkg/usagestats"
 	"github.com/grafana/tempo/pkg/util"
 	"github.com/grafana/tempo/pkg/util/log"
+	util_log "github.com/grafana/tempo/pkg/util/log"
 )

 const (
@@ -62,19 +69,21 @@ var (
 type App struct {
 	cfg Config

-	Server         *server.Server
+	Server         TempoServer
 	InternalServer *server.Server
-	readRings      map[string]*ring.Ring
-	Overrides      overrides.Service
-	distributor    *distributor.Distributor
-	querier        *querier.Querier
-	frontend       *frontend_v1.Frontend
-	compactor      *compactor.Compactor
-	ingester       *ingester.Ingester
-	generator      *generator.Generator
-	store          storage.Store
-	usageReport    *usagestats.Reporter
-	MemberlistKV   *memberlist.KVInitService
+
+	readRings     map[string]*ring.Ring
+	Overrides     overrides.Service
+	distributor   *distributor.Distributor
+	querier       *querier.Querier
+	frontend      *frontend_v1.Frontend
+	compactor     *compactor.Compactor
+	ingester      *ingester.Ingester
+	generator     *generator.Generator
+	store         storage.Store
+	usageReport   *usagestats.Reporter
+	cacheProvider cache.Provider
+	MemberlistKV  *memberlist.KVInitService

 	HTTPAuthMiddleware       middleware.Interface
 	TracesConsumerMiddleware receiver.Middleware
@@ -89,6 +98,7 @@ func New(cfg Config) (*App, error) {
 	app := &App{
 		cfg:       cfg,
 		readRings: map[string]*ring.Ring{},
+		Server:    newTempoServer(),
 	}

 	usagestats.Edition("oss")
@@ -111,7 +121,7 @@ func New(cfg Config) (*App, error) {
 	app.setupAuthMiddleware()

 	if err := app.setupModuleManager(); err != nil {
-		return nil, fmt.Errorf("failed to setup module manager %w", err)
+		return nil, fmt.Errorf("failed to setup module manager: %w", err)
 	}

 	return app, nil
@@ -168,7 +178,7 @@ func (t *App) Run() error {

 	serviceMap, err := t.ModuleManager.InitModuleServices(t.cfg.Target)
 	if err != nil {
-		return fmt.Errorf("failed to init module services %w", err)
+		return fmt.Errorf("failed to init module services: %w", err)
 	}
 	t.serviceMap = serviceMap

@@ -179,18 +189,26 @@ func (t *App) Run() error {

 	sm, err := services.NewManager(servs...)
 	if err != nil {
-		return fmt.Errorf("failed to start service manager %w", err)
+		return fmt.Errorf("failed to start service manager: %w", err)
 	}

+	// Used to delay shutdown but return "not ready" during this delay.
+	shutdownRequested := atomic.NewBool(false)
 	// before starting servers, register /ready handler and gRPC health check service.
 	if t.cfg.InternalServer.Enable {
-		t.InternalServer.HTTP.Path("/ready").Methods("GET").Handler(t.readyHandler(sm))
+		t.InternalServer.HTTP.Path("/ready").Methods("GET").Handler(t.readyHandler(sm, shutdownRequested))
 	}

-	t.Server.HTTP.Path("/ready").Handler(t.readyHandler(sm))
-	t.Server.HTTP.Path("/status").Handler(t.statusHandler()).Methods("GET")
-	t.Server.HTTP.Path("/status/{endpoint}").Handler(t.statusHandler()).Methods("GET")
-	grpc_health_v1.RegisterHealthServer(t.Server.GRPC, grpcutil.NewHealthCheck(sm))
+	t.Server.HTTPRouter().Path(addHTTPAPIPrefix(&t.cfg, api.PathBuildInfo)).Handler(t.buildinfoHandler()).Methods("GET")
+
+	t.Server.HTTPRouter().Path("/ready").Handler(t.readyHandler(sm, shutdownRequested))
+	t.Server.HTTPRouter().Path("/status").Handler(t.statusHandler()).Methods("GET")
+	t.Server.HTTPRouter().Path("/status/{endpoint}").Handler(t.statusHandler()).Methods("GET")
+	grpc_health_v1.RegisterHealthServer(t.Server.GRPC(),
+		grpcutil.NewHealthCheckFrom(
+			grpcutil.WithShutdownRequested(shutdownRequested),
+			grpcutil.WithManager(sm),
+		))

 	// Let's listen for events from this manager, and log them.
 	healthy := func() { level.Info(log.Logger).Log("msg", "Tempo started") }
@@ -202,14 +220,14 @@ func (t *App) Run() error {
 		// let's find out which module failed
 		for m, s := range serviceMap {
 			if s == service {
-				switch service.FailureCase() {
-				case modules.ErrStopProcess:
-					level.Info(log.Logger).Log("msg", "received stop signal via return error", "module", m, "err", service.FailureCase())
-				case context.Canceled:
-				default:
-					level.Error(log.Logger).Log("msg", "module failed", "module", m, "err", service.FailureCase())
+				err = service.FailureCase()
+				if errors.Is(err, modules.ErrStopProcess) {
+					level.Info(log.Logger).Log("msg", "received stop signal via return error", "module", m, "err", err)
+				} else if errors.Is(err, context.Canceled) {
+					return
+				} else if err != nil {
+					level.Error(log.Logger).Log("msg", "module failed", "module", m, "err", err)
 				}
-
 				return
 			}
 		}
@@ -219,9 +237,17 @@ func (t *App) Run() error {
 	sm.AddListener(services.NewManagerListener(healthy, stopped, serviceFailed))

 	// Setup signal handler. If signal arrives, we stop the manager, which stops all the services.
-	handler := signals.NewHandler(t.Server.Log)
+	handler := signals.NewHandler(t.Server.Log())
 	go func() {
 		handler.Loop()
+
+		shutdownRequested.Store(true)
+		t.Server.SetKeepAlivesEnabled(false)
+
+		if t.cfg.ShutdownDelay > 0 {
+			time.Sleep(t.cfg.ShutdownDelay)
+		}
+
 		sm.StopAsync()
 	}()

@@ -229,7 +255,7 @@ func (t *App) Run() error {
 	// in other state than New, which should not be the case.
 	err = sm.StartAsync(context.Background())
 	if err != nil {
-		return fmt.Errorf("failed to start service manager %w", err)
+		return fmt.Errorf("failed to start service manager: %w", err)
 	}

 	return sm.AwaitStopped(context.Background())
@@ -271,7 +297,7 @@ func (t *App) writeStatusConfig(w io.Writer, r *http.Request) error {
 	case "":
 		output = t.cfg
 	default:
-		return errors.Errorf("unknown value for mode query parameter: %v", mode)
+		return fmt.Errorf("unknown value for mode query parameter: %v", mode)
 	}

 	out, err := yaml.Marshal(output)
@@ -292,8 +318,14 @@ func (t *App) writeStatusConfig(w io.Writer, r *http.Request) error {
 	return nil
 }

-func (t *App) readyHandler(sm *services.Manager) http.HandlerFunc {
+func (t *App) readyHandler(sm *services.Manager, shutdownRequested *atomic.Bool) http.HandlerFunc {
 	return func(w http.ResponseWriter, r *http.Request) {
+		if shutdownRequested.Load() {
+			level.Debug(util_log.Logger).Log("msg", "application is stopping")
+			http.Error(w, "Application is stopping", http.StatusServiceUnavailable)
+			return
+		}
+
 		if !sm.IsHealthy() {
 			msg := bytes.Buffer{}
 			msg.WriteString("Some services are not Running:\n")
@@ -405,7 +437,7 @@ func (t *App) statusHandler() http.HandlerFunc {
 					if err == nil {
 						err = e
 					} else {
-						err = errors.Wrap(err, e.Error())
+						err = fmt.Errorf("%s: %w", e.Error(), err)
 					}
 				}
 			}
@@ -465,7 +497,7 @@ func (t *App) writeStatusEndpoints(w io.Writer) error {

 	endpoints := []endpoint{}

-	err := t.Server.HTTP.Walk(func(route *mux.Route, router *mux.Router, ancestors []*mux.Route) error {
+	err := t.Server.HTTPRouter().Walk(func(route *mux.Route, router *mux.Router, ancestors []*mux.Route) error {
 		e := endpoint{}

 		pathTemplate, err := route.GetPathTemplate()
@@ -483,7 +515,7 @@ func (t *App) writeStatusEndpoints(w io.Writer) error {
 		return nil
 	})
 	if err != nil {
-		return errors.Wrap(err, "error walking routes")
+		return fmt.Errorf("error walking routes: %w", err)
 	}

 	sort.Slice(endpoints[:], func(i, j int) bool {
@@ -505,8 +537,22 @@ func (t *App) writeStatusEndpoints(w io.Writer) error {

 	_, err = w.Write([]byte(fmt.Sprintf("\nAPI documentation: %s\n\n", apiDocs)))
 	if err != nil {
-		return errors.Wrap(err, "error writing status endpoints")
+		return fmt.Errorf("error writing status endpoints: %w", err)
 	}

 	return nil
 }
+
+func (t *App) buildinfoHandler() http.HandlerFunc {
+	return func(w http.ResponseWriter, r *http.Request) {
+		w.Header().Set("Content-Type", "application/json")
+		err := json.NewEncoder(w).Encode(build.GetVersion())
+
+		if err != nil {
+			http.Error(w, err.Error(), http.StatusInternalServerError)
+			level.Error(log.Logger).Log("msg", "error writing response", "err", err)
+		} else {
+			w.WriteHeader(http.StatusOK)
+		}
+	}
+}
@@ -7,9 +7,10 @@ import (

 	"github.com/grafana/dskit/flagext"
 	"github.com/grafana/dskit/kv/memberlist"
+	"github.com/grafana/dskit/server"
 	"github.com/prometheus/client_golang/prometheus"
-	"github.com/weaveworks/common/server"

+	"github.com/grafana/tempo/modules/cache"
 	"github.com/grafana/tempo/modules/compactor"
 	"github.com/grafana/tempo/modules/distributor"
 	"github.com/grafana/tempo/modules/frontend"
@@ -30,13 +31,15 @@ import (

 // Config is the root config for App.
 type Config struct {
-	Target                       string `yaml:"target,omitempty"`
-	AuthEnabled                  bool   `yaml:"auth_enabled,omitempty"`
-	MultitenancyEnabled          bool   `yaml:"multitenancy_enabled,omitempty"`
-	HTTPAPIPrefix                string `yaml:"http_api_prefix"`
-	UseOTelTracer                bool   `yaml:"use_otel_tracer,omitempty"`
-	EnableGoRuntimeMetrics       bool   `yaml:"enable_go_runtime_metrics,omitempty"`
-	AutocompleteFilteringEnabled bool   `yaml:"autocomplete_filtering_enabled,omitempty"`
+	Target                       string        `yaml:"target,omitempty"`
+	AuthEnabled                  bool          `yaml:"auth_enabled,omitempty"`
+	MultitenancyEnabled          bool          `yaml:"multitenancy_enabled,omitempty"`
+	ShutdownDelay                time.Duration `yaml:"shutdown_delay,omitempty"`
+	StreamOverHTTPEnabled        bool          `yaml:"stream_over_http_enabled,omitempty"`
+	HTTPAPIPrefix                string        `yaml:"http_api_prefix"`
+	UseOTelTracer                bool          `yaml:"use_otel_tracer,omitempty"`
+	EnableGoRuntimeMetrics       bool          `yaml:"enable_go_runtime_metrics,omitempty"`
+	AutocompleteFilteringEnabled bool          `yaml:"autocomplete_filtering_enabled,omitempty"`

 	Server          server.Config           `yaml:"server,omitempty"`
 	InternalServer  internalserver.Config   `yaml:"internal_server,omitempty"`
@@ -49,14 +52,10 @@ type Config struct {
 	Ingester        ingester.Config         `yaml:"ingester,omitempty"`
 	Generator       generator.Config        `yaml:"metrics_generator,omitempty"`
 	StorageConfig   storage.Config          `yaml:"storage,omitempty"`
-	LimitsConfig    overrides.Limits        `yaml:"overrides,omitempty"`
+	Overrides       overrides.Config        `yaml:"overrides,omitempty"`
 	MemberlistKV    memberlist.KVConfig     `yaml:"memberlist,omitempty"`
 	UsageReport     usagestats.Config       `yaml:"usage_report,omitempty"`
-
-	// This is used by applications hosting Tempo to disable the default behavior
-	// of routing grpc over the main http server. Specifically this is for
-	// Grafana Enterprise Traces gateway module which does its own protocol muxing.
-	DoNotRouteHTTPToGRPC bool `yaml:"-"`
+	CacheProvider   cache.Config            `yaml:"cache,omitempty"`
 }

 func newDefaultConfig() *Config {
@@ -69,6 +68,7 @@ func newDefaultConfig() *Config {
 // RegisterFlagsAndApplyDefaults registers flag.
 func (c *Config) RegisterFlagsAndApplyDefaults(prefix string, f *flag.FlagSet) {
 	c.Target = SingleBinary
+	c.StreamOverHTTPEnabled = false
 	// global settings
 	f.StringVar(&c.Target, "target", SingleBinary, "target module")
 	f.BoolVar(&c.AuthEnabled, "auth.enabled", false, "Set to true to enable auth (deprecated: use multitenancy.enabled)")
@@ -76,7 +76,8 @@ func (c *Config) RegisterFlagsAndApplyDefaults(prefix string, f *flag.FlagSet) {
 	f.StringVar(&c.HTTPAPIPrefix, "http-api-prefix", "", "String prefix for all http api endpoints.")
 	f.BoolVar(&c.UseOTelTracer, "use-otel-tracer", false, "Set to true to replace the OpenTracing tracer with the OpenTelemetry tracer")
 	f.BoolVar(&c.EnableGoRuntimeMetrics, "enable-go-runtime-metrics", false, "Set to true to enable all Go runtime metrics")
-	f.BoolVar(&c.AutocompleteFilteringEnabled, "autocomplete-filtering.enabled", false, "Set to true to enable autocomplete filtering")
+	f.BoolVar(&c.AutocompleteFilteringEnabled, "autocomplete-filtering.enabled", true, "Set to false to disable autocomplete filtering")
+	f.DurationVar(&c.ShutdownDelay, "shutdown-delay", 0, "How long to wait between SIGTERM and shutdown. After receiving SIGTERM, Tempo will report not-ready status via /ready endpoint.")

 	// Server settings
 	flagext.DefaultValues(&c.Server)
@@ -85,8 +86,12 @@ func (c *Config) RegisterFlagsAndApplyDefaults(prefix string, f *flag.FlagSet) {
 	// Internal server settings
 	flagext.DefaultValues(&c.InternalServer)

+	// Set log level for internal server as the same as server
+	c.InternalServer.LogLevel = c.Server.LogLevel
+	c.InternalServer.LogFormat = c.Server.LogFormat
+
 	// Increase max message size to 16MB
-	c.Server.GPRCServerMaxRecvMsgSize = 16 * 1024 * 1024
+	c.Server.GRPCServerMaxRecvMsgSize = 16 * 1024 * 1024
 	c.Server.GRPCServerMaxSendMsgSize = 16 * 1024 * 1024

 	// The following GRPC server settings are added to address this issue - https://github.com/grafana/tempo/issues/493
@@ -117,7 +122,7 @@ func (c *Config) RegisterFlagsAndApplyDefaults(prefix string, f *flag.FlagSet) {
 	c.IngesterClient.GRPCClientConfig.GRPCCompression = "snappy"
 	flagext.DefaultValues(&c.GeneratorClient)
 	c.GeneratorClient.GRPCClientConfig.GRPCCompression = "snappy"
-	c.LimitsConfig.RegisterFlagsAndApplyDefaults(f)
+	c.Overrides.RegisterFlagsAndApplyDefaults(f)

 	c.Distributor.RegisterFlagsAndApplyDefaults(util.PrefixConfig(prefix, "distributor"), f)
 	c.Ingester.RegisterFlagsAndApplyDefaults(util.PrefixConfig(prefix, "ingester"), f)
@@ -127,6 +132,7 @@ func (c *Config) RegisterFlagsAndApplyDefaults(prefix string, f *flag.FlagSet) {
 	c.Compactor.RegisterFlagsAndApplyDefaults(util.PrefixConfig(prefix, "compactor"), f)
 	c.StorageConfig.RegisterFlagsAndApplyDefaults(util.PrefixConfig(prefix, "storage"), f)
 	c.UsageReport.RegisterFlagsAndApplyDefaults(util.PrefixConfig(prefix, "reporting"), f)
+	c.CacheProvider.RegisterFlagsAndApplyDefaults(util.PrefixConfig(prefix, "cache"), f)
 }

 // MultitenancyIsEnabled checks if multitenancy is enabled
@@ -157,7 +163,7 @@ func (c *Config) CheckConfig() []ConfigWarning {
 		warnings = append(warnings, warnBlocklistPollConcurrency)
 	}

-	if c.Distributor.LogReceivedTraces {
+	if c.Distributor.LogReceivedSpans.Enabled {
 		warnings = append(warnings, warnLogReceivedTraces)
 	}

@@ -165,6 +171,16 @@ func (c *Config) CheckConfig() []ConfigWarning {
 		warnings = append(warnings, warnStorageTraceBackendLocal)
 	}

+	for _, dc := range c.StorageConfig.Trace.Block.DedicatedColumns {
+		err := dc.Validate()
+		if err != nil {
+			warnings = append(warnings, ConfigWarning{
+				Message: err.Error(),
+				Explain: "Tempo will not start with an invalid dedicated attribute column configuration",
+			})
+		}
+	}
+
 	// check v2 specific settings
 	if c.StorageConfig.Trace.Block.Version != "v2" && c.StorageConfig.Trace.Block.IndexDownsampleBytes != common.DefaultIndexDownSampleBytes {
 		warnings = append(warnings, newV2Warning("v2_index_downsample_bytes"))
@@ -190,6 +206,18 @@ func (c *Config) CheckConfig() []ConfigWarning {
 		warnings = append(warnings, warnTracesAndUserConfigurableOverridesStorageConflict)
 	}

+	if c.Overrides.ConfigType == overrides.ConfigTypeLegacy {
+		warnings = append(warnings, warnLegacyOverridesConfig)
+	}
+
+	if c.StorageConfig.Trace.Backend == backend.S3 && c.StorageConfig.Trace.S3.NativeAWSAuthEnabled {
+		warnings = append(warnings, warnNativeAWSAuthEnabled)
+	}
+
+	if c.StorageConfig.Trace.Cache != "" {
+		warnings = append(warnings, warnConfiguredLegacyCache)
+	}
+
 	return warnings
 }

@@ -239,14 +267,28 @@ var (
 		Explain: fmt.Sprintf("default=%d", tempodb.DefaultBlocklistPollConcurrency),
 	}
 	warnLogReceivedTraces = ConfigWarning{
-		Message: "c.Distributor.LogReceivedTraces is deprecated. The new flag is c.Distributor.log_received_spans.enabled",
+		Message: "Span logging is enabled. This is for debuging only and not recommended for production deployments.",
 	}
 	warnStorageTraceBackendLocal = ConfigWarning{
 		Message: "Local backend will not correctly retrieve traces with a distributed deployment unless all components have access to the same disk. You should probably be using object storage as a backend.",
 	}
+	warnLegacyOverridesConfig = ConfigWarning{
+		Message: "Inline, unscoped overrides are deprecated. Please use the new overrides config format.",
+	}
+
 	warnTracesAndUserConfigurableOverridesStorageConflict = ConfigWarning{
 		Message: "Trace storage conflicts with user-configurable overrides storage",
 	}
+
+	warnNativeAWSAuthEnabled = ConfigWarning{
+		Message: "c.StorageConfig.Trace.S3.NativeAWSAuthEnabled is deprecated and will be removed in a future release.",
+		Explain: "This setting is no longer necessary and will be ignored.",
+	}
+
+	warnConfiguredLegacyCache = ConfigWarning{
+		Message: "c.StorageConfig.Trace.Cache is deprecated and will be removed in a future release.",
+		Explain: "Please migrate to the top level cache settings config.",
+	}
 )

 func newV2Warning(setting string) ConfigWarning {
@@ -258,7 +300,7 @@ func newV2Warning(setting string) ConfigWarning {

 func (c *Config) tracesAndOverridesStorageConflict() bool {
 	traceStorage := c.StorageConfig.Trace
-	overridesStorage := c.LimitsConfig.UserConfigurableOverridesConfig.ClientConfig
+	overridesStorage := c.Overrides.UserConfigurableOverridesConfig.Client

 	if traceStorage.Backend != overridesStorage.Backend {
 		return false
@@ -4,6 +4,7 @@ import (
 	"testing"
 	"time"

+	"github.com/grafana/tempo/tempodb/backend/s3"
 	"github.com/stretchr/testify/assert"

 	"github.com/grafana/tempo/modules/distributor"
@@ -12,7 +13,7 @@ import (
 	"github.com/grafana/tempo/tempodb/backend"
 	"github.com/grafana/tempo/tempodb/encoding/common"
 	v2 "github.com/grafana/tempo/tempodb/encoding/v2"
-	"github.com/grafana/tempo/tempodb/encoding/vparquet2"
+	"github.com/grafana/tempo/tempodb/encoding/vparquet3"
 )

 func TestConfig_CheckConfig(t *testing.T) {
@@ -32,15 +33,21 @@ func TestConfig_CheckConfig(t *testing.T) {
 				Target: MetricsGenerator,
 				StorageConfig: storage.Config{
 					Trace: tempodb.Config{
+						Cache:         "supercache",
 						Backend:       backend.S3,
 						BlocklistPoll: time.Minute,
 						Block: &common.BlockConfig{
 							Version: "v2",
 						},
+						S3: &s3.Config{
+							NativeAWSAuthEnabled: true,
+						},
 					},
 				},
 				Distributor: distributor.Config{
-					LogReceivedTraces: true,
+					LogReceivedSpans: distributor.LogReceivedSpansConfig{
+						Enabled: true,
+					},
 				},
 			},
 			expect: []ConfigWarning{
@@ -50,6 +57,8 @@ func TestConfig_CheckConfig(t *testing.T) {
 				warnStorageTraceBackendS3,
 				warnBlocklistPollConcurrency,
 				warnLogReceivedTraces,
+				warnNativeAWSAuthEnabled,
+				warnConfiguredLegacyCache,
 			},
 		},
 		{
@@ -72,7 +81,7 @@ func TestConfig_CheckConfig(t *testing.T) {
 			name: "warnings for v2 settings when they drift from default",
 			config: func() *Config {
 				cfg := newDefaultConfig()
-				cfg.StorageConfig.Trace.Block.Version = vparquet2.VersionString
+				cfg.StorageConfig.Trace.Block.Version = vparquet3.VersionString
 				cfg.StorageConfig.Trace.Block.IndexDownsampleBytes = 1
 				cfg.StorageConfig.Trace.Block.IndexPageSizeBytes = 1
 				cfg.Compactor.Compactor.ChunkSizeBytes = 1
@@ -108,8 +117,8 @@ func TestConfig_CheckConfig(t *testing.T) {
 				cfg := newDefaultConfig()
 				cfg.StorageConfig.Trace.Backend = backend.Local
 				cfg.StorageConfig.Trace.Local.Path = "/var/tempo"
-				cfg.LimitsConfig.UserConfigurableOverridesConfig.ClientConfig.Backend = backend.Local
-				cfg.LimitsConfig.UserConfigurableOverridesConfig.ClientConfig.Local.Path = "/var/tempo"
+				cfg.Overrides.UserConfigurableOverridesConfig.Client.Backend = backend.Local
+				cfg.Overrides.UserConfigurableOverridesConfig.Client.Local.Path = "/var/tempo"
 				return cfg
 			}(),
 			expect: []ConfigWarning{warnTracesAndUserConfigurableOverridesStorageConflict},
@@ -121,9 +130,9 @@ func TestConfig_CheckConfig(t *testing.T) {
 				cfg.StorageConfig.Trace.Backend = backend.GCS
 				cfg.StorageConfig.Trace.GCS.BucketName = "bucketname"
 				cfg.StorageConfig.Trace.GCS.Prefix = "tempo"
-				cfg.LimitsConfig.UserConfigurableOverridesConfig.ClientConfig.Backend = backend.GCS
-				cfg.LimitsConfig.UserConfigurableOverridesConfig.ClientConfig.GCS.BucketName = "bucketname"
-				cfg.LimitsConfig.UserConfigurableOverridesConfig.ClientConfig.GCS.Prefix = "tempo"
+				cfg.Overrides.UserConfigurableOverridesConfig.Client.Backend = backend.GCS
+				cfg.Overrides.UserConfigurableOverridesConfig.Client.GCS.BucketName = "bucketname"
+				cfg.Overrides.UserConfigurableOverridesConfig.Client.GCS.Prefix = "tempo"
 				return cfg
 			}(),
 			expect: []ConfigWarning{warnTracesAndUserConfigurableOverridesStorageConflict},
@@ -134,8 +143,8 @@ func TestConfig_CheckConfig(t *testing.T) {
 				cfg := newDefaultConfig()
 				cfg.StorageConfig.Trace.Backend = backend.GCS
 				cfg.StorageConfig.Trace.GCS.BucketName = "my-bucket"
-				cfg.LimitsConfig.UserConfigurableOverridesConfig.ClientConfig.Backend = backend.S3
-				cfg.LimitsConfig.UserConfigurableOverridesConfig.ClientConfig.S3.Bucket = "my-bucket"
+				cfg.Overrides.UserConfigurableOverridesConfig.Client.Backend = backend.S3
+				cfg.Overrides.UserConfigurableOverridesConfig.Client.S3.Bucket = "my-bucket"
 				return cfg
 			}(),
 			expect: nil,
@@ -4,9 +4,9 @@ import (
 	"context"
 	"net/http"

+	"github.com/grafana/dskit/middleware"
+	"github.com/grafana/dskit/user"
 	"github.com/grafana/tempo/pkg/util"
-	"github.com/weaveworks/common/middleware"
-	"github.com/weaveworks/common/user"
 	"google.golang.org/grpc"
 )

@@ -3,8 +3,8 @@ package app
 import (
 	"net/http"

+	"github.com/grafana/dskit/middleware"
 	"github.com/klauspost/compress/gzhttp"
-	"github.com/weaveworks/common/middleware"
 )

 func httpGzipMiddleware() middleware.Interface {
@@ -1,33 +1,37 @@
 package app

 import (
+	"context"
+	"errors"
 	"fmt"
 	"io"
 	"net/http"
 	"path"

+	kitlog "github.com/go-kit/log"
 	"github.com/go-kit/log/level"
 	"github.com/grafana/dskit/dns"
 	"github.com/grafana/dskit/kv/codec"
 	"github.com/grafana/dskit/kv/memberlist"
+	"github.com/grafana/dskit/middleware"
 	"github.com/grafana/dskit/modules"
 	"github.com/grafana/dskit/ring"
+	"github.com/grafana/dskit/server"
 	"github.com/grafana/dskit/services"
 	jsoniter "github.com/json-iterator/go"
-	"github.com/pkg/errors"
 	"github.com/prometheus/client_golang/prometheus"
 	"github.com/prometheus/client_golang/prometheus/collectors"
-	"github.com/weaveworks/common/middleware"
-	"github.com/weaveworks/common/server"

+	"github.com/grafana/tempo/modules/cache"
 	"github.com/grafana/tempo/modules/compactor"
 	"github.com/grafana/tempo/modules/distributor"
 	"github.com/grafana/tempo/modules/frontend"
+	"github.com/grafana/tempo/modules/frontend/interceptor"
 	frontend_v1pb "github.com/grafana/tempo/modules/frontend/v1/frontendv1pb"
 	"github.com/grafana/tempo/modules/generator"
 	"github.com/grafana/tempo/modules/ingester"
 	"github.com/grafana/tempo/modules/overrides"
-	"github.com/grafana/tempo/modules/overrides/userconfigurableapi"
+	userconfigurableoverridesapi "github.com/grafana/tempo/modules/overrides/userconfigurable/api"
 	"github.com/grafana/tempo/modules/querier"
 	tempo_storage "github.com/grafana/tempo/modules/storage"
 	"github.com/grafana/tempo/pkg/api"
@@ -52,6 +56,9 @@ const (
 	MemberlistKV   string = "memberlist-kv"
 	UsageReport    string = "usage-report"
 	Overrides      string = "overrides"
+	OverridesAPI   string = "overrides-api"
+	CacheProvider  string = "cache-provider"
+
 	// rings
 	IngesterRing          string = "ring"
 	SecondaryIngesterRing string = "secondary-ring"
@@ -90,19 +97,6 @@ func (t *App) initServer() (services.Service, error) {
 		))
 	}

-	DisableSignalHandling(&t.cfg.Server)
-
-	if !t.cfg.DoNotRouteHTTPToGRPC {
-		// this allows us to serve http and grpc over the primary http server.
-		//  to use this register services with GRPCOnHTTPServer
-		t.cfg.Server.RouteHTTPToGRPC = true
-	}
-
-	server, err := server.New(t.cfg.Server)
-	if err != nil {
-		return nil, fmt.Errorf("failed to create server %w", err)
-	}
-
 	servicesToWaitFor := func() []services.Service {
 		svs := []services.Service(nil)
 		for m, s := range t.serviceMap {
@@ -114,10 +108,14 @@ func (t *App) initServer() (services.Service, error) {
 		return svs
 	}

-	t.Server = server
-	s := NewServerService(server, servicesToWaitFor)
+	// add unary and stream timeout interceptors for the query-frontend if configured
+	// this same timeout is enforced for http in the initQueryFrontend() function
+	if t.cfg.Frontend.APITimeout > 0 && t.isModuleActive(QueryFrontend) {
+		t.cfg.Server.GRPCMiddleware = append(t.cfg.Server.GRPCMiddleware, interceptor.NewFrontendAPIUnaryTimeout(t.cfg.Frontend.APITimeout))
+		t.cfg.Server.GRPCStreamMiddleware = append(t.cfg.Server.GRPCStreamMiddleware, interceptor.NewFrontendAPIStreamTimeout(t.cfg.Frontend.APITimeout))
+	}

-	return s, nil
+	return t.Server.StartAndReturnService(t.cfg.Server, t.cfg.StreamOverHTTPEnabled, servicesToWaitFor)
 }

 func (t *App) initInternalServer() (services.Service, error) {
@@ -174,48 +172,56 @@ func (t *App) initReadRing(cfg ring.Config, name, key string) (*ring.Ring, error
 		return nil, fmt.Errorf("failed to create ring %s: %w", name, err)
 	}

-	t.Server.HTTP.Handle("/"+name+"/ring", ring)
+	t.Server.HTTPRouter().Handle("/"+name+"/ring", ring)
 	t.readRings[name] = ring

 	return ring, nil
 }

 func (t *App) initOverrides() (services.Service, error) {
-	o, err := overrides.NewOverrides(t.cfg.LimitsConfig)
+	o, err := overrides.NewOverrides(t.cfg.Overrides, newRuntimeConfigValidator(&t.cfg), prometheus.DefaultRegisterer)
 	if err != nil {
-		return nil, fmt.Errorf("failed to create overrides %w", err)
+		return nil, fmt.Errorf("failed to create overrides: %w", err)
 	}
 	t.Overrides = o

-	prometheus.MustRegister(&t.cfg.LimitsConfig)
+	prometheus.MustRegister(&t.cfg.Overrides)

-	if t.cfg.LimitsConfig.PerTenantOverrideConfig != "" {
+	if t.cfg.Overrides.PerTenantOverrideConfig != "" {
 		prometheus.MustRegister(t.Overrides)
 	}

-	// User-configurable overrides API
-	userConfigOverridesCfg := t.cfg.LimitsConfig.UserConfigurableOverridesConfig
-
-	// only run API on query-frontend
-	if userConfigOverridesCfg.Enabled && t.isModuleActive(QueryFrontend) {
-		userConfigOverridesAPI, err := userconfigurableapi.NewUserConfigOverridesAPI(&userConfigOverridesCfg.ClientConfig)
-		if err != nil {
-			return nil, errors.Wrap(err, "failed to create user-configurable overrides API")
-		}
-
-		overridesPath := addHTTPAPIPrefix(&t.cfg, api.PathOverrides)
-		wrapHandler := func(h http.HandlerFunc) http.Handler {
-			return t.HTTPAuthMiddleware.Wrap(h)
-		}
-
-		t.Server.HTTP.Path(overridesPath).Methods(http.MethodGet).Handler(wrapHandler(userConfigOverridesAPI.GetOverridesHandler))
-		t.Server.HTTP.Path(overridesPath).Methods(http.MethodPost).Handler(wrapHandler(userConfigOverridesAPI.PostOverridesHandler))
-		t.Server.HTTP.Path(overridesPath).Methods(http.MethodDelete).Handler(wrapHandler(userConfigOverridesAPI.DeleteOverridesHandler))
-	}
+	t.Server.HTTPRouter().Path("/status/overrides").HandlerFunc(overrides.TenantsHandler(t.Overrides)).Methods("GET")
+	t.Server.HTTPRouter().Path("/status/overrides/{tenant}").HandlerFunc(overrides.TenantStatusHandler(t.Overrides)).Methods("GET")

 	return t.Overrides, nil
 }

+func (t *App) initOverridesAPI() (services.Service, error) {
+	cfg := t.cfg.Overrides.UserConfigurableOverridesConfig
+
+	if !cfg.Enabled {
+		return services.NewIdleService(nil, nil), nil
+	}
+
+	userConfigOverridesAPI, err := userconfigurableoverridesapi.New(&cfg.API, &cfg.Client, t.Overrides, newOverridesValidator(&t.cfg))
+	if err != nil {
+		return nil, fmt.Errorf("failed to create user-configurable overrides API: %w", err)
+	}
+
+	overridesPath := addHTTPAPIPrefix(&t.cfg, api.PathOverrides)
+	wrapHandler := func(h http.HandlerFunc) http.Handler {
+		return t.HTTPAuthMiddleware.Wrap(h)
+	}
+
+	t.Server.HTTPRouter().Path(overridesPath).Methods(http.MethodGet).Handler(wrapHandler(userConfigOverridesAPI.GetHandler))
+	t.Server.HTTPRouter().Path(overridesPath).Methods(http.MethodPost).Handler(wrapHandler(userConfigOverridesAPI.PostHandler))
+	t.Server.HTTPRouter().Path(overridesPath).Methods(http.MethodPatch).Handler(wrapHandler(userConfigOverridesAPI.PatchHandler))
+	t.Server.HTTPRouter().Path(overridesPath).Methods(http.MethodDelete).Handler(wrapHandler(userConfigOverridesAPI.DeleteHandler))
+
+	return userConfigOverridesAPI, nil
+}
+
 func (t *App) initDistributor() (services.Service, error) {
 	// todo: make ingester client a module instead of passing the config everywhere
 	distributor, err := distributor.New(t.cfg.Distributor,
@@ -227,12 +233,12 @@ func (t *App) initDistributor() (services.Service, error) {
 		t.TracesConsumerMiddleware,
 		log.Logger, t.cfg.Server.LogLevel, prometheus.DefaultRegisterer)
 	if err != nil {
-		return nil, fmt.Errorf("failed to create distributor %w", err)
+		return nil, fmt.Errorf("failed to create distributor: %w", err)
 	}
 	t.distributor = distributor

 	if distributor.DistributorRing != nil {
-		t.Server.HTTP.Handle("/distributor/ring", distributor.DistributorRing)
+		t.Server.HTTPRouter().Handle("/distributor/ring", distributor.DistributorRing)
 	}

 	return t.distributor, nil
@@ -241,35 +247,39 @@ func (t *App) initDistributor() (services.Service, error) {
 func (t *App) initIngester() (services.Service, error) {
 	t.cfg.Ingester.LifecyclerConfig.ListenPort = t.cfg.Server.GRPCListenPort
 	t.cfg.Ingester.AutocompleteFilteringEnabled = t.cfg.AutocompleteFilteringEnabled
+	t.cfg.Ingester.DedicatedColumns = t.cfg.StorageConfig.Trace.Block.DedicatedColumns
 	ingester, err := ingester.New(t.cfg.Ingester, t.store, t.Overrides, prometheus.DefaultRegisterer)
 	if err != nil {
 		return nil, fmt.Errorf("failed to create ingester: %w", err)
 	}
 	t.ingester = ingester

-	tempopb.RegisterPusherServer(t.Server.GRPC, t.ingester)
-	tempopb.RegisterQuerierServer(t.Server.GRPC, t.ingester)
-	t.Server.HTTP.Path("/flush").Handler(http.HandlerFunc(t.ingester.FlushHandler))
-	t.Server.HTTP.Path("/shutdown").Handler(http.HandlerFunc(t.ingester.ShutdownHandler))
+	tempopb.RegisterPusherServer(t.Server.GRPC(), t.ingester)
+	tempopb.RegisterQuerierServer(t.Server.GRPC(), t.ingester)
+	t.Server.HTTPRouter().Path("/flush").Handler(http.HandlerFunc(t.ingester.FlushHandler))
+	t.Server.HTTPRouter().Path("/shutdown").Handler(http.HandlerFunc(t.ingester.ShutdownHandler))
 	return t.ingester, nil
 }

 func (t *App) initGenerator() (services.Service, error) {
 	t.cfg.Generator.Ring.ListenPort = t.cfg.Server.GRPCListenPort
 	genSvc, err := generator.New(&t.cfg.Generator, t.Overrides, prometheus.DefaultRegisterer, log.Logger)
-	if err == generator.ErrUnconfigured && t.cfg.Target != MetricsGenerator { // just warn if we're not running the metrics-generator
+	if errors.Is(err, generator.ErrUnconfigured) && t.cfg.Target != MetricsGenerator { // just warn if we're not running the metrics-generator
 		level.Warn(log.Logger).Log("msg", "metrics-generator is not configured.", "err", err)
 		return services.NewIdleService(nil, nil), nil
 	}
 	if err != nil {
-		return nil, fmt.Errorf("failed to create metrics-generator %w", err)
+		return nil, fmt.Errorf("failed to create metrics-generator: %w", err)
 	}
 	t.generator = genSvc

 	spanStatsHandler := t.HTTPAuthMiddleware.Wrap(http.HandlerFunc(t.generator.SpanMetricsHandler))
-	t.Server.HTTP.Handle(path.Join(api.PathPrefixGenerator, addHTTPAPIPrefix(&t.cfg, api.PathSpanMetrics)), spanStatsHandler)
+	t.Server.HTTPRouter().Handle(path.Join(api.PathPrefixGenerator, addHTTPAPIPrefix(&t.cfg, api.PathSpanMetrics)), spanStatsHandler)

-	tempopb.RegisterMetricsGeneratorServer(t.Server.GRPC, t.generator)
+	queryRangeHandler := t.HTTPAuthMiddleware.Wrap(http.HandlerFunc(t.generator.QueryRangeHandler))
+	t.Server.HTTPRouter().Handle(path.Join(api.PathPrefixGenerator, addHTTPAPIPrefix(&t.cfg, api.PathMetricsQueryRange)), queryRangeHandler)
+
+	tempopb.RegisterMetricsGeneratorServer(t.Server.GRPC(), t.generator)

 	return t.generator, nil
 }
@@ -289,7 +299,7 @@ func (t *App) initQuerier() (services.Service, error) {

 	// do not enable polling if this is the single binary. in that case the compactor will take care of polling
 	if t.cfg.Target == Querier {
-		t.store.EnablePolling(nil)
+		t.store.EnablePolling(context.Background(), nil)
 	}

 	ingesterRings := []ring.ReadRing{t.readRings[ringIngester]}
@@ -297,6 +307,8 @@ func (t *App) initQuerier() (services.Service, error) {
 		ingesterRings = append(ingesterRings, ring)
 	}

+	t.cfg.Querier.AutocompleteFilteringEnabled = t.cfg.AutocompleteFilteringEnabled
+
 	querier, err := querier.New(
 		t.cfg.Querier,
 		t.cfg.IngesterClient,
@@ -307,7 +319,7 @@ func (t *App) initQuerier() (services.Service, error) {
 		t.Overrides,
 	)
 	if err != nil {
-		return nil, fmt.Errorf("failed to create querier %w", err)
+		return nil, fmt.Errorf("failed to create querier: %w", err)
 	}
 	t.querier = querier

@@ -316,27 +328,30 @@ func (t *App) initQuerier() (services.Service, error) {
 	)

 	tracesHandler := middleware.Wrap(http.HandlerFunc(t.querier.TraceByIDHandler))
-	t.Server.HTTP.Handle(path.Join(api.PathPrefixQuerier, addHTTPAPIPrefix(&t.cfg, api.PathTraces)), tracesHandler)
+	t.Server.HTTPRouter().Handle(path.Join(api.PathPrefixQuerier, addHTTPAPIPrefix(&t.cfg, api.PathTraces)), tracesHandler)

 	searchHandler := t.HTTPAuthMiddleware.Wrap(http.HandlerFunc(t.querier.SearchHandler))
-	t.Server.HTTP.Handle(path.Join(api.PathPrefixQuerier, addHTTPAPIPrefix(&t.cfg, api.PathSearch)), searchHandler)
+	t.Server.HTTPRouter().Handle(path.Join(api.PathPrefixQuerier, addHTTPAPIPrefix(&t.cfg, api.PathSearch)), searchHandler)

 	searchTagsHandler := t.HTTPAuthMiddleware.Wrap(http.HandlerFunc(t.querier.SearchTagsHandler))
-	t.Server.HTTP.Handle(path.Join(api.PathPrefixQuerier, addHTTPAPIPrefix(&t.cfg, api.PathSearchTags)), searchTagsHandler)
+	t.Server.HTTPRouter().Handle(path.Join(api.PathPrefixQuerier, addHTTPAPIPrefix(&t.cfg, api.PathSearchTags)), searchTagsHandler)

 	searchTagsV2Handler := t.HTTPAuthMiddleware.Wrap(http.HandlerFunc(t.querier.SearchTagsV2Handler))
-	t.Server.HTTP.Handle(path.Join(api.PathPrefixQuerier, addHTTPAPIPrefix(&t.cfg, api.PathSearchTagsV2)), searchTagsV2Handler)
+	t.Server.HTTPRouter().Handle(path.Join(api.PathPrefixQuerier, addHTTPAPIPrefix(&t.cfg, api.PathSearchTagsV2)), searchTagsV2Handler)

 	searchTagValuesHandler := t.HTTPAuthMiddleware.Wrap(http.HandlerFunc(t.querier.SearchTagValuesHandler))
-	t.Server.HTTP.Handle(path.Join(api.PathPrefixQuerier, addHTTPAPIPrefix(&t.cfg, api.PathSearchTagValues)), searchTagValuesHandler)
+	t.Server.HTTPRouter().Handle(path.Join(api.PathPrefixQuerier, addHTTPAPIPrefix(&t.cfg, api.PathSearchTagValues)), searchTagValuesHandler)

 	searchTagValuesV2Handler := t.HTTPAuthMiddleware.Wrap(http.HandlerFunc(t.querier.SearchTagValuesV2Handler))
-	t.Server.HTTP.Handle(path.Join(api.PathPrefixQuerier, addHTTPAPIPrefix(&t.cfg, api.PathSearchTagValuesV2)), searchTagValuesV2Handler)
+	t.Server.HTTPRouter().Handle(path.Join(api.PathPrefixQuerier, addHTTPAPIPrefix(&t.cfg, api.PathSearchTagValuesV2)), searchTagValuesV2Handler)

 	spanMetricsSummaryHandler := t.HTTPAuthMiddleware.Wrap(http.HandlerFunc(t.querier.SpanMetricsSummaryHandler))
-	t.Server.HTTP.Handle(path.Join(api.PathPrefixQuerier, addHTTPAPIPrefix(&t.cfg, api.PathSpanMetricsSummary)), spanMetricsSummaryHandler)
+	t.Server.HTTPRouter().Handle(path.Join(api.PathPrefixQuerier, addHTTPAPIPrefix(&t.cfg, api.PathSpanMetricsSummary)), spanMetricsSummaryHandler)

-	return t.querier, t.querier.CreateAndRegisterWorker(t.Server.HTTPServer.Handler)
+	queryRangeHandler := t.HTTPAuthMiddleware.Wrap(http.HandlerFunc(t.querier.QueryRangeHandler))
+	t.Server.HTTPRouter().Handle(path.Join(api.PathPrefixQuerier, addHTTPAPIPrefix(&t.cfg, api.PathMetricsQueryRange)), queryRangeHandler)
+
+	return t.querier, t.querier.CreateAndRegisterWorker(t.Server.HTTPHandler())
 }

 func (t *App) initQueryFrontend() (services.Service, error) {
@@ -349,50 +364,55 @@ func (t *App) initQueryFrontend() (services.Service, error) {
 	t.frontend = v1

 	// create query frontend
-	queryFrontend, err := frontend.New(t.cfg.Frontend, cortexTripper, t.Overrides, t.store, t.cfg.HTTPAPIPrefix, log.Logger, prometheus.DefaultRegisterer)
+	queryFrontend, err := frontend.New(t.cfg.Frontend, cortexTripper, t.Overrides, t.store, t.cacheProvider, t.cfg.HTTPAPIPrefix, log.Logger, prometheus.DefaultRegisterer)
 	if err != nil {
 		return nil, err
 	}

-	// wrap handlers with auth
-	middleware := middleware.Merge(
-		t.HTTPAuthMiddleware,
-		httpGzipMiddleware(),
-	)
-
-	traceByIDHandler := middleware.Wrap(queryFrontend.TraceByIDHandler)
-	searchHandler := middleware.Wrap(queryFrontend.SearchHandler)
-	spanMetricsSummaryHandler := middleware.Wrap(queryFrontend.SpanMetricsSummaryHandler)
-	searchTagsHandler := middleware.Wrap(queryFrontend.SearchTagsHandler)
-
 	// register grpc server for queriers to connect to
-	frontend_v1pb.RegisterFrontendServer(t.Server.GRPC, t.frontend)
+	frontend_v1pb.RegisterFrontendServer(t.Server.GRPC(), t.frontend)
 	// we register the streaming querier service on both the http and grpc servers. Grafana expects
 	// this GRPC service to be available on the HTTP server.
-	tempopb.RegisterStreamingQuerierServer(t.Server.GRPC, queryFrontend)
-	tempopb.RegisterStreamingQuerierServer(t.Server.GRPCOnHTTPServer, queryFrontend)
+	tempopb.RegisterStreamingQuerierServer(t.Server.GRPC(), queryFrontend)
+
+	httpAPIMiddleware := []middleware.Interface{
+		t.HTTPAuthMiddleware,
+		httpGzipMiddleware(),
+	}
+
+	// use the api timeout for http requests if set. note that this is set in initServer() for
+	// grpc requests
+	if t.cfg.Frontend.APITimeout > 0 {
+		httpAPIMiddleware = append(httpAPIMiddleware, middleware.NewTimeoutMiddleware(t.cfg.Frontend.APITimeout, "unable to process request in the configured timeout", kitlog.NewNopLogger()))
+	}
+
+	// wrap handlers with auth
+	base := middleware.Merge(httpAPIMiddleware...)

 	// http trace by id endpoint
-	t.Server.HTTP.Handle(addHTTPAPIPrefix(&t.cfg, api.PathTraces), traceByIDHandler)
+	t.Server.HTTPRouter().Handle(addHTTPAPIPrefix(&t.cfg, api.PathTraces), base.Wrap(queryFrontend.TraceByIDHandler))

 	// http search endpoints
-	t.Server.HTTP.Handle(addHTTPAPIPrefix(&t.cfg, api.PathSearch), searchHandler)
-	t.Server.HTTP.Handle(addHTTPAPIPrefix(&t.cfg, api.PathSearchTags), searchTagsHandler)
-	t.Server.HTTP.Handle(addHTTPAPIPrefix(&t.cfg, api.PathSearchTagsV2), searchTagsHandler)
-	t.Server.HTTP.Handle(addHTTPAPIPrefix(&t.cfg, api.PathSearchTagValues), searchTagsHandler)
-	t.Server.HTTP.Handle(addHTTPAPIPrefix(&t.cfg, api.PathSearchTagValuesV2), searchTagsHandler)
+	t.Server.HTTPRouter().Handle(addHTTPAPIPrefix(&t.cfg, api.PathSearch), base.Wrap(queryFrontend.SearchHandler))
+	t.Server.HTTPRouter().Handle(addHTTPAPIPrefix(&t.cfg, api.PathSearchTags), base.Wrap(queryFrontend.SearchTagsHandler))
+	t.Server.HTTPRouter().Handle(addHTTPAPIPrefix(&t.cfg, api.PathSearchTagsV2), base.Wrap(queryFrontend.SearchTagsV2Handler))
+	t.Server.HTTPRouter().Handle(addHTTPAPIPrefix(&t.cfg, api.PathSearchTagValues), base.Wrap(queryFrontend.SearchTagsValuesHandler))
+	t.Server.HTTPRouter().Handle(addHTTPAPIPrefix(&t.cfg, api.PathSearchTagValuesV2), base.Wrap(queryFrontend.SearchTagsValuesV2Handler))

 	// http metrics endpoints
-	t.Server.HTTP.Handle(addHTTPAPIPrefix(&t.cfg, api.PathSpanMetricsSummary), spanMetricsSummaryHandler)
+	t.Server.HTTPRouter().Handle(addHTTPAPIPrefix(&t.cfg, api.PathSpanMetricsSummary), base.Wrap(queryFrontend.MetricsSummaryHandler))
+	t.Server.HTTPRouter().Handle(addHTTPAPIPrefix(&t.cfg, api.PathMetricsQueryRange), base.Wrap(queryFrontend.MetricsQueryRangeHandler))

 	// the query frontend needs to have knowledge of the blocks so it can shard search jobs
-	t.store.EnablePolling(nil)
+	if t.cfg.Target == QueryFrontend {
+		t.store.EnablePolling(context.Background(), nil)
+	}

 	// http query echo endpoint
-	t.Server.HTTP.Handle(addHTTPAPIPrefix(&t.cfg, api.PathEcho), echoHandler())
+	t.Server.HTTPRouter().Handle(addHTTPAPIPrefix(&t.cfg, api.PathEcho), echoHandler())

 	// http endpoint to see usage stats data
-	t.Server.HTTP.Handle(addHTTPAPIPrefix(&t.cfg, api.PathUsageStats), usageStatsHandler(t.cfg.UsageReport))
+	t.Server.HTTPRouter().Handle(addHTTPAPIPrefix(&t.cfg, api.PathUsageStats), usageStatsHandler(t.cfg.UsageReport))

 	// todo: queryFrontend should implement service.Service and take the cortex frontend a submodule
 	return t.frontend, nil
@@ -405,21 +425,21 @@ func (t *App) initCompactor() (services.Service, error) {

 	compactor, err := compactor.New(t.cfg.Compactor, t.store, t.Overrides, prometheus.DefaultRegisterer)
 	if err != nil {
-		return nil, fmt.Errorf("failed to create compactor %w", err)
+		return nil, fmt.Errorf("failed to create compactor: %w", err)
 	}
 	t.compactor = compactor

 	if t.compactor.Ring != nil {
-		t.Server.HTTP.Handle("/compactor/ring", t.compactor.Ring)
+		t.Server.HTTPRouter().Handle("/compactor/ring", t.compactor.Ring)
 	}

 	return t.compactor, nil
 }

 func (t *App) initStore() (services.Service, error) {
-	store, err := tempo_storage.NewStore(t.cfg.StorageConfig, log.Logger)
+	store, err := tempo_storage.NewStore(t.cfg.StorageConfig, t.cacheProvider, log.Logger)
 	if err != nil {
-		return nil, fmt.Errorf("failed to create store %w", err)
+		return nil, fmt.Errorf("failed to create store: %w", err)
 	}
 	t.store = store

@@ -428,7 +448,6 @@ func (t *App) initStore() (services.Service, error) {

 func (t *App) initMemberlistKV() (services.Service, error) {
 	reg := prometheus.DefaultRegisterer
-	t.cfg.MemberlistKV.MetricsRegisterer = reg
 	t.cfg.MemberlistKV.MetricsNamespace = metricsNamespace
 	t.cfg.MemberlistKV.Codecs = []codec.Codec{
 		ring.GetCodec(),
@@ -451,7 +470,7 @@ func (t *App) initMemberlistKV() (services.Service, error) {
 	t.cfg.Distributor.DistributorRing.KVStore.MemberlistKV = t.MemberlistKV.GetMemberlistKV
 	t.cfg.Compactor.ShardingRing.KVStore.MemberlistKV = t.MemberlistKV.GetMemberlistKV

-	t.Server.HTTP.Handle("/memberlist", t.MemberlistKV)
+	t.Server.HTTPRouter().Handle("/memberlist", t.MemberlistKV)

 	return t.MemberlistKV, nil
 }
@@ -498,6 +517,16 @@ func (t *App) initUsageReport() (services.Service, error) {
 	return ur, nil
 }

+func (t *App) initCacheProvider() (services.Service, error) {
+	c, err := cache.NewProvider(&t.cfg.CacheProvider, util_log.Logger)
+	if err != nil {
+		return nil, fmt.Errorf("failed to create cache provider: %w", err)
+	}
+
+	t.cacheProvider = c
+	return c, nil
+}
+
 func (t *App) setupModuleManager() error {
 	mm := modules.NewManager(log.Logger)

@@ -509,7 +538,9 @@ func (t *App) setupModuleManager() error {
 	mm.RegisterModule(InternalServer, t.initInternalServer, modules.UserInvisibleModule)
 	mm.RegisterModule(MemberlistKV, t.initMemberlistKV, modules.UserInvisibleModule)
 	mm.RegisterModule(Overrides, t.initOverrides, modules.UserInvisibleModule)
+	mm.RegisterModule(OverridesAPI, t.initOverridesAPI)
 	mm.RegisterModule(UsageReport, t.initUsageReport)
+	mm.RegisterModule(CacheProvider, t.initCacheProvider, modules.UserInvisibleModule)
 	mm.RegisterModule(IngesterRing, t.initIngesterRing, modules.UserInvisibleModule)
 	mm.RegisterModule(MetricsGeneratorRing, t.initGeneratorRing, modules.UserInvisibleModule)
 	mm.RegisterModule(SecondaryIngesterRing, t.initSecondaryIngesterRing, modules.UserInvisibleModule)
@@ -527,10 +558,12 @@ func (t *App) setupModuleManager() error {
 	mm.RegisterModule(ScalableSingleBinary, nil)

 	deps := map[string][]string{
-		// Store:          nil,
 		// InternalServer: nil,
+		// CacheProvider:  nil,
+		Store:                 {CacheProvider},
 		Server:                {InternalServer},
 		Overrides:             {Server},
+		OverridesAPI:          {Server, Overrides},
 		MemberlistKV:          {Server},
 		UsageReport:           {MemberlistKV},
 		IngesterRing:          {Server, MemberlistKV},
@@ -540,7 +573,7 @@ func (t *App) setupModuleManager() error {
 		Common: {UsageReport, Server, Overrides},

 		// individual targets
-		QueryFrontend:    {Common, Store},
+		QueryFrontend:    {Common, Store, OverridesAPI},
 		Distributor:      {Common, IngesterRing, MetricsGeneratorRing},
 		Ingester:         {Common, Store, MemberlistKV},
 		MetricsGenerator: {Common, MemberlistKV},
@@ -0,0 +1,92 @@
+package app
+
+import (
+	"fmt"
+	"time"
+
+	"golang.org/x/exp/slices"
+
+	"github.com/grafana/tempo/modules/generator"
+	"github.com/grafana/tempo/modules/overrides"
+	"github.com/grafana/tempo/modules/overrides/userconfigurable/api"
+	"github.com/grafana/tempo/modules/overrides/userconfigurable/client"
+	filterconfig "github.com/grafana/tempo/pkg/spanfilter/config"
+)
+
+type runtimeConfigValidator struct {
+	cfg *Config
+}
+
+var _ overrides.Validator = (*runtimeConfigValidator)(nil)
+
+func newRuntimeConfigValidator(cfg *Config) overrides.Validator {
+	return &runtimeConfigValidator{
+		cfg: cfg,
+	}
+}
+
+func (r *runtimeConfigValidator) Validate(config *overrides.Overrides) error {
+	if config.Ingestion.TenantShardSize != 0 {
+		ingesterReplicationFactor := r.cfg.Ingester.LifecyclerConfig.RingConfig.ReplicationFactor
+		if config.Ingestion.TenantShardSize < ingesterReplicationFactor {
+			return fmt.Errorf("ingester.tenant.shard_size is lower than replication factor (%d < %d)", config.Ingestion.TenantShardSize, ingesterReplicationFactor)
+		}
+	}
+
+	return nil
+}
+
+type overridesValidator struct {
+	cfg *Config
+
+	validForwarders map[string]struct{}
+}
+
+var _ api.Validator = (*overridesValidator)(nil)
+
+func newOverridesValidator(cfg *Config) api.Validator {
+	validForwarders := map[string]struct{}{}
+	for _, f := range cfg.Distributor.Forwarders {
+		validForwarders[f.Name] = struct{}{}
+	}
+
+	return &overridesValidator{
+		cfg: cfg,
+
+		validForwarders: validForwarders,
+	}
+}
+
+func (v *overridesValidator) Validate(limits *client.Limits) error {
+	if forwarders, ok := limits.GetForwarders(); ok {
+		for _, f := range forwarders {
+			if _, ok := v.validForwarders[f]; !ok {
+				return fmt.Errorf("forwarder \"%s\" is not a known forwarder, contact your system administrator", f)
+			}
+		}
+	}
+
+	if processors, ok := limits.GetMetricsGenerator().GetProcessors(); ok {
+		for p := range processors.GetMap() {
+			if !slices.Contains(generator.SupportedProcessors, p) {
+				return fmt.Errorf("metrics_generator.processor \"%s\" is not a known processor, valid values: %v", p, generator.SupportedProcessors)
+			}
+		}
+	}
+
+	if filterPolicies, ok := limits.GetMetricsGenerator().GetProcessor().GetSpanMetrics().GetFilterPolicies(); ok {
+		for _, fp := range filterPolicies {
+			if err := filterconfig.ValidateFilterPolicy(fp); err != nil {
+				return err
+			}
+		}
+	}
+
+	if collectionInterval, ok := limits.GetMetricsGenerator().GetCollectionInterval(); ok {
+		if collectionInterval < 15*time.Second || collectionInterval > 5*time.Minute {
+			return fmt.Errorf("metrics_generator.collection_interval \"%s\" is outside acceptable range of 15s to 5m", collectionInterval.String())
+		}
+	}
+
+	return nil
+}
@@ -0,0 +1,196 @@
+package app
+
+import (
+	"fmt"
+	"testing"
+	"time"
+
+	"github.com/grafana/dskit/ring"
+	"github.com/stretchr/testify/assert"
+
+	"github.com/grafana/tempo/modules/distributor"
+	"github.com/grafana/tempo/modules/distributor/forwarder"
+	"github.com/grafana/tempo/modules/generator"
+	"github.com/grafana/tempo/modules/ingester"
+	"github.com/grafana/tempo/modules/overrides"
+	"github.com/grafana/tempo/modules/overrides/userconfigurable/client"
+	filterconfig "github.com/grafana/tempo/pkg/spanfilter/config"
+)
+
+func Test_runtimeOverridesValidator(t *testing.T) {
+	testCases := []struct {
+		name      string
+		cfg       Config
+		overrides overrides.Overrides
+		expErr    string
+	}{
+		{
+			name: "ingestion.tenant_shard_size smaller than RF",
+			cfg: Config{
+				Ingester: ingester.Config{
+					LifecyclerConfig: ring.LifecyclerConfig{
+						RingConfig: ring.Config{
+							ReplicationFactor: 3,
+						},
+					},
+				},
+			},
+			overrides: overrides.Overrides{Ingestion: overrides.IngestionOverrides{TenantShardSize: 2}},
+			expErr:    "ingester.tenant.shard_size is lower than replication factor (2 < 3)",
+		},
+		{
+			name: "ingestion.tenant_shard_size equal to RF",
+			cfg: Config{
+				Ingester: ingester.Config{
+					LifecyclerConfig: ring.LifecyclerConfig{
+						RingConfig: ring.Config{
+							ReplicationFactor: 3,
+						},
+					},
+				},
+			},
+			overrides: overrides.Overrides{Ingestion: overrides.IngestionOverrides{TenantShardSize: 3}},
+		},
+	}
+
+	for _, tc := range testCases {
+		t.Run(tc.name, func(t *testing.T) {
+			validator := newRuntimeConfigValidator(&tc.cfg)
+
+			err := validator.Validate(&tc.overrides)
+			if tc.expErr != "" {
+				assert.EqualError(t, err, tc.expErr)
+			} else {
+				assert.NoError(t, err)
+			}
+		})
+	}
+}
+
+func Test_overridesValidator(t *testing.T) {
+	testCases := []struct {
+		name   string
+		cfg    Config
+		limits client.Limits
+		expErr string
+	}{
+		{
+			name: "forwarders - valid",
+			cfg: Config{
+				Distributor: distributor.Config{
+					Forwarders: []forwarder.Config{
+						{Name: "forwarder-1"},
+						{Name: "forwarder-2"},
+						{Name: "forwarder-3"},
+					},
+				},
+			},
+			limits: client.Limits{
+				Forwarders: &[]string{"forwarder-1", "forwarder-3"},
+			},
+		},
+		{
+			name: "forwarders - invalid",
+			cfg: Config{
+				Distributor: distributor.Config{
+					Forwarders: []forwarder.Config{
+						{Name: "forwarder-1"},
+						{Name: "forwarder-2"},
+					},
+				},
+			},
+			limits: client.Limits{
+				Forwarders: &[]string{"forwarder-1", "some-forwarder"},
+			},
+			expErr: "forwarder \"some-forwarder\" is not a known forwarder, contact your system administrator",
+		},
+		{
+			name: "metrics_generator.processor",
+			cfg:  Config{},
+			limits: client.Limits{
+				MetricsGenerator: client.LimitsMetricsGenerator{
+					Processors: map[string]struct{}{
+						"service-graphs": {},
+						"span-span":      {},
+					},
+				},
+			},
+			expErr: fmt.Sprintf("metrics_generator.processor \"span-span\" is not a known processor, valid values: %v", generator.SupportedProcessors),
+		},
+		{
+			name: "filter policies",
+			cfg:  Config{},
+			limits: client.Limits{
+				Forwarders: &[]string{},
+				MetricsGenerator: client.LimitsMetricsGenerator{Processor: client.LimitsMetricsGeneratorProcessor{SpanMetrics: client.LimitsMetricsGeneratorProcessorSpanMetrics{FilterPolicies: &[]filterconfig.FilterPolicy{{
+					Include: &filterconfig.PolicyMatch{
+						MatchType: filterconfig.Strict,
+						Attributes: []filterconfig.MatchPolicyAttribute{
+							{
+								Key:   "span.kind",
+								Value: "SPAN_KIND_SERVER",
+							},
+						},
+					},
+				}}}}},
+			},
+		},
+		{
+			name: "filter policies - invalid",
+			cfg:  Config{},
+			limits: client.Limits{
+				Forwarders: &[]string{},
+				MetricsGenerator: client.LimitsMetricsGenerator{Processor: client.LimitsMetricsGeneratorProcessor{SpanMetrics: client.LimitsMetricsGeneratorProcessorSpanMetrics{FilterPolicies: &[]filterconfig.FilterPolicy{
+					{
+						Include: &filterconfig.PolicyMatch{
+							MatchType: "invalid",
+						},
+					},
+				}}}},
+			},
+			expErr: "invalid include policy: invalid match type: invalid",
+		},
+		{
+			name: "metrics_generator.collection_interval valid",
+			cfg:  Config{},
+			limits: client.Limits{
+				MetricsGenerator: client.LimitsMetricsGenerator{
+					CollectionInterval: &client.Duration{Duration: 60 * time.Second},
+				},
+			},
+		},
+		{
+			name: "metrics_generator.collection_interval minimum",
+			cfg:  Config{},
+			limits: client.Limits{
+				MetricsGenerator: client.LimitsMetricsGenerator{
+					CollectionInterval: &client.Duration{Duration: 1 * time.Second},
+				},
+			},
+			expErr: "metrics_generator.collection_interval \"1s\" is outside acceptable range of 15s to 5m",
+		},
+		{
+			name: "metrics_generator.collection_interval maximum",
+			cfg:  Config{},
+			limits: client.Limits{
+				MetricsGenerator: client.LimitsMetricsGenerator{
+					CollectionInterval: &client.Duration{Duration: 10 * time.Minute},
+				},
+			},
+			expErr: "metrics_generator.collection_interval \"10m0s\" is outside acceptable range of 15s to 5m",
+		},
+	}
+
+	for _, tc := range testCases {
+		t.Run(tc.name, func(t *testing.T) {
+			validator := newOverridesValidator(&tc.cfg)
+
+			err := validator.Validate(&tc.limits)
+			if tc.expErr != "" {
+				assert.EqualError(t, err, tc.expErr)
+			} else {
+				assert.NoError(t, err)
+			}
+		})
+	}
+}
@@ -3,14 +3,128 @@ package app
 import (
 	"context"
 	"fmt"
+	"net/http"
+	"strings"
+	"sync"

+	"github.com/go-kit/log"
 	"github.com/go-kit/log/level"
+	"github.com/gorilla/mux"
+	"github.com/grafana/dskit/middleware"
+	"github.com/grafana/dskit/server"
 	"github.com/grafana/dskit/services"
-	"github.com/weaveworks/common/server"
+	"golang.org/x/net/http2"
+	"golang.org/x/net/http2/h2c"
+	"google.golang.org/grpc"

 	util_log "github.com/grafana/tempo/pkg/util/log"
 )

+type TempoServer interface {
+	HTTPRouter() *mux.Router
+	HTTPHandler() http.Handler
+	GRPC() *grpc.Server
+	Log() log.Logger
+	EnableHTTP2()
+	SetKeepAlivesEnabled(enabled bool)
+
+	StartAndReturnService(cfg server.Config, supportGRPCOnHTTP bool, servicesToWaitFor func() []services.Service) (services.Service, error)
+}
+
+// todo: evaluate whether the internal server should be included as part of this
+type tempoServer struct {
+	mux     *mux.Router  // all tempo http routes are added here
+	handler http.Handler // the final handler which includes the router and any middleware
+
+	externalServer  *server.Server // the standard server that all HTTP/GRPC requests are served on
+	enableHTTP2Once sync.Once
+}
+
+func newTempoServer() *tempoServer {
+	return &tempoServer{
+		mux: mux.NewRouter(),
+		// externalServer will be initialized in StartService
+	}
+}
+
+func (s *tempoServer) HTTPRouter() *mux.Router {
+	return s.mux
+}
+
+func (s *tempoServer) HTTPHandler() http.Handler {
+	return s.handler
+}
+
+func (s *tempoServer) GRPC() *grpc.Server {
+	return s.externalServer.GRPC
+}
+
+func (s *tempoServer) Log() log.Logger {
+	return s.externalServer.Log
+}
+
+func (s *tempoServer) EnableHTTP2() {
+	s.enableHTTP2Once.Do(func() {
+		s.externalServer.HTTPServer.Handler = h2c.NewHandler(s.externalServer.HTTPServer.Handler, &http2.Server{})
+	})
+}
+
+func (s *tempoServer) SetKeepAlivesEnabled(enabled bool) {
+	s.externalServer.HTTPServer.SetKeepAlivesEnabled(enabled)
+}
+
+func (s *tempoServer) StartAndReturnService(cfg server.Config, supportGRPCOnHTTP bool, servicesToWaitFor func() []services.Service) (services.Service, error) {
+	var err error
+
+	metrics := server.NewServerMetrics(cfg)
+	DisableSignalHandling(&cfg)
+
+	if !supportGRPCOnHTTP {
+		// We don't do any GRPC handling, let the library handle all routing for us
+		cfg.Router = s.mux
+
+		s.externalServer, err = server.NewWithMetrics(cfg, metrics)
+		if err != nil {
+			return nil, fmt.Errorf("failed to create server: %w", err)
+		}
+		s.handler = s.externalServer.HTTPServer.Handler
+	} else {
+		// We want to route both GRPC and HTTP requests on the same endpoint
+		cfg.Router = nil
+		cfg.DoNotAddDefaultHTTPMiddleware = true // we don't want instrumentation on the "root" router, we want it on our mux. it will be added below.
+
+		s.externalServer, err = server.NewWithMetrics(cfg, metrics)
+		if err != nil {
+			return nil, fmt.Errorf("failed to create server: %w", err)
+		}
+
+		// now that we have created the server and service let's setup our grpc/http router if necessary
+		// for grpc to work we must enable h2c on the external server
+		s.EnableHTTP2()
+
+		// recreate dskit instrumentation here
+		cfg.DoNotAddDefaultHTTPMiddleware = false
+		httpMiddleware, err := server.BuildHTTPMiddleware(cfg, s.mux, metrics, s.externalServer.Log)
+		if err != nil {
+			return nil, fmt.Errorf("failed to create http middleware: %w", err)
+		}
+
+		s.handler = middleware.Merge(httpMiddleware...).Wrap(s.mux)
+		s.externalServer.HTTP.PathPrefix("/").HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
+			// route to GRPC server if it's a GRPC request
+			if req.ProtoMajor == 2 && strings.Contains(req.Header.Get("Content-Type"), "application/grpc") {
+				s.externalServer.GRPC.ServeHTTP(w, req)
+				return
+			}
+
+			// default to standard http server
+			s.handler.ServeHTTP(w, req)
+		})
+	}
+
+	return NewServerService(s.externalServer, servicesToWaitFor), nil
+}
+
 // NewServerService constructs service from Server component.
 // servicesToWaitFor is called when server is stopping, and should return all
 // services that need to terminate before server actually stops.
@@ -13,13 +13,12 @@ import (
 	"github.com/drone/envsubst"
 	"github.com/go-kit/log/level"
 	"github.com/grafana/dskit/flagext"
+	dslog "github.com/grafana/dskit/log"
+	"github.com/grafana/dskit/spanprofiler"
+	"github.com/grafana/dskit/tracing"
 	ot "github.com/opentracing/opentracing-go"
-	"github.com/pkg/errors"
 	"github.com/prometheus/client_golang/prometheus"
 	"github.com/prometheus/common/version"
-	"github.com/weaveworks/common/logging"
-	"github.com/weaveworks/common/tracing"
-	oc "go.opencensus.io/trace"
 	"go.opentelemetry.io/otel"
 	oc_bridge "go.opentelemetry.io/otel/bridge/opencensus"
 	ot_bridge "go.opentelemetry.io/otel/bridge/opentracing"
@@ -27,11 +26,13 @@ import (
 	"go.opentelemetry.io/otel/propagation"
 	"go.opentelemetry.io/otel/sdk/resource"
 	tracesdk "go.opentelemetry.io/otel/sdk/trace"
-	semconv "go.opentelemetry.io/otel/semconv/v1.18.0"
+	semconv "go.opentelemetry.io/otel/semconv/v1.25.0"
+	"google.golang.org/grpc/encoding"
 	"gopkg.in/yaml.v2"

 	"github.com/grafana/tempo/cmd/tempo/app"
 	"github.com/grafana/tempo/cmd/tempo/build"
+	"github.com/grafana/tempo/pkg/gogocodec"
 	"github.com/grafana/tempo/pkg/util/log"
 )

@@ -49,6 +50,9 @@ func init() {
 	version.Branch = Branch
 	version.Revision = Revision
 	prometheus.MustRegister(version.NewCollector(appName))
+
+	// Register the gogocodec as early as possible.
+	encoding.RegisterCodec(gogocodec.NewCodec())
 }

 func main() {
@@ -56,7 +60,7 @@ func main() {
 	ballastMBs := flag.Int("mem-ballast-size-mbs", 0, "Size of memory ballast to allocate in MBs.")
 	mutexProfileFraction := flag.Int("mutex-profile-fraction", 0, "Enable mutex profiling.")

-	config, warningLogs, err := loadConfig()
+	config, configVerify, err := loadConfig()
 	if err != nil {
 		fmt.Fprintf(os.Stderr, "failed parsing config: %v\n", err)
 		os.Exit(1)
@@ -67,15 +71,21 @@ func main() {
 	}

 	// Init the logger which will honor the log level set in config.Server
-	if reflect.DeepEqual(&config.Server.LogLevel, &logging.Level{}) {
+	if reflect.DeepEqual(&config.Server.LogLevel, &dslog.Level{}) {
 		level.Error(log.Logger).Log("msg", "invalid log level")
 		os.Exit(1)
 	}
 	log.InitLogger(&config.Server)

-	// Log warnings from loading the config now the logger is initialized
-	for _, warning := range warningLogs {
-		level.Warn(log.Logger).Log(warning...)
+	// Verifying the config's validity and log warnings now that the logger is initialized
+	isValid := configIsValid(config)
+
+	// Exit if config.verify flag is true
+	if configVerify {
+		if !isValid {
+			os.Exit(1)
+		}
+		os.Exit(0)
 	}

 	// Init tracer
@@ -114,23 +124,23 @@ func main() {
 	runtime.KeepAlive(ballast)
 }

-func configIsValid(config *app.Config) (warningLogs [][]any, isValid bool) {
+func configIsValid(config *app.Config) bool {
 	// Warn the user for suspect configurations
 	if warnings := config.CheckConfig(); len(warnings) != 0 {
-		warningLogs = append(warningLogs, []any{"msg", "-- CONFIGURATION WARNINGS --"})
+		level.Warn(log.Logger).Log("msg", "-- CONFIGURATION WARNINGS --")
 		for _, w := range warnings {
 			output := []any{"msg", w.Message}
 			if w.Explain != "" {
 				output = append(output, "explain", w.Explain)
 			}
-			warningLogs = append(warningLogs, output)
+			level.Warn(log.Logger).Log(output...)
 		}
-		return warningLogs, false
+		return false
 	}
-	return warningLogs, true
+	return true
 }

-func loadConfig() (*app.Config, [][]any, error) {
+func loadConfig() (*app.Config, bool, error) {
 	const (
 		configFileOption      = "config.file"
 		configExpandEnvOption = "config.expand-env"
@@ -169,24 +179,27 @@ func loadConfig() (*app.Config, [][]any, error) {
 	if configFile != "" {
 		buff, err := os.ReadFile(configFile)
 		if err != nil {
-			return nil, nil, fmt.Errorf("failed to read configFile %s: %w", configFile, err)
+			return nil, false, fmt.Errorf("failed to read configFile %s: %w", configFile, err)
 		}

 		if configExpandEnv {
 			s, err := envsubst.EvalEnv(string(buff))
 			if err != nil {
-				return nil, nil, fmt.Errorf("failed to expand env vars from configFile %s: %w", configFile, err)
+				return nil, false, fmt.Errorf("failed to expand env vars from configFile %s: %w", configFile, err)
 			}
 			buff = []byte(s)
 		}

 		err = yaml.UnmarshalStrict(buff, config)
 		if err != nil {
-			return nil, nil, fmt.Errorf("failed to parse configFile %s: %w", configFile, err)
+			return nil, false, fmt.Errorf("failed to parse configFile %s: %w", configFile, err)
 		}

 	}

+	// Pass --config.expand-env flag to overrides module
+	config.Overrides.ExpandEnv = configExpandEnv
+
 	// overlay with cli
 	flagext.IgnoredFlag(flag.CommandLine, configFileOption, "Configuration file to load")
 	flagext.IgnoredFlag(flag.CommandLine, configExpandEnvOption, "Whether to expand environment variables in config file")
@@ -205,16 +218,7 @@ func loadConfig() (*app.Config, [][]any, error) {
 		config.Generator.Ring.InstanceAddr = "127.0.0.1"
 	}

-	// after finalizing the configuration, verify its validity and exit if config.verify flag is true.
-	warningLogs, isValid := configIsValid(config)
-	if configVerify {
-		if !isValid {
-			os.Exit(1)
-		}
-		os.Exit(0)
-	}
-
-	return config, warningLogs, nil
+	return config, configVerify, nil
 }

 func installOpenTracingTracer(config *app.Config) (func(), error) {
@@ -223,8 +227,10 @@ func installOpenTracingTracer(config *app.Config) (func(), error) {
 	// Setting the environment variable JAEGER_AGENT_HOST enables tracing
 	trace, err := tracing.NewFromEnv(fmt.Sprintf("%s-%s", appName, config.Target))
 	if err != nil {
-		return nil, errors.Wrap(err, "error initialising tracer")
+		return nil, fmt.Errorf("error initialising tracer: %w", err)
 	}
+	ot.SetGlobalTracer(spanprofiler.NewTracer(ot.GlobalTracer()))
+
 	return func() {
 		if err := trace.Close(); err != nil {
 			level.Error(log.Logger).Log("msg", "error closing tracing", "err", err)
@@ -241,7 +247,7 @@ func installOpenTelemetryTracer(config *app.Config) (func(), error) {

 	exp, err := jaeger.New(jaeger.WithCollectorEndpoint())
 	if err != nil {
-		return nil, errors.Wrap(err, "failed to create Jaeger exporter")
+		return nil, fmt.Errorf("failed to create Jaeger exporter: %w", err)
 	}

 	resources, err := resource.New(context.Background(),
@@ -252,7 +258,7 @@ func installOpenTelemetryTracer(config *app.Config) (func(), error) {
 		resource.WithHost(),
 	)
 	if err != nil {
-		return nil, errors.Wrap(err, "failed to initialise trace resuorces")
+		return nil, fmt.Errorf("failed to initialise trace resources: %w", err)
 	}

 	tp := tracesdk.NewTracerProvider(
@@ -283,10 +289,10 @@ func installOpenTelemetryTracer(config *app.Config) (func(), error) {
 	bridgeTracer.SetWarningHandler(func(msg string) {
 		level.Warn(log.Logger).Log("msg", msg, "source", "BridgeTracer.OnWarningHandler")
 	})
-	ot.SetGlobalTracer(bridgeTracer)
+	ot.SetGlobalTracer(spanprofiler.NewTracer(bridgeTracer))

 	// Install the OpenCensus bridge
-	oc.DefaultTracer = oc_bridge.NewTracer(tp.Tracer("OpenCensus"))
+	oc_bridge.InstallTraceBridge(oc_bridge.WithTracerProvider(tp))

 	return shutdown, nil
 }
@@ -0,0 +1,152 @@
+---
+Authors: Martin Disibio (@mdisibio), Joe Elliott (@joe-elliott), Jennie Pham (@ie-pham), Adrian Stoewer (@stoewer)
+Created: 2023 July - 2023 November
+Last updated: 2023-11-28
+---
+
+# TraceQL Extensions
+
+## Summary
+
+This design document describes extensions to the existing TraceQL language that allow for searching new fields and data types.
+
+This document is **not** meant to be a complete language specification. Rather it is meant to invite the community to comment on the basic structure and concepts of the language extensions.
+
+### Table of Contents
+
+- [Escaping Attribute Names](#escaping-attribute-names)
+- [Additional Scopes](#additional-scopes)
+  - [Trace](#trace)
+  - [Instrumentation Scope](#instrumentation-scope)
+  - [Event](#event)
+  - [Link](#link)
+- [Scoped Intrinsics](#scoped-intrinsics)
+- [Additional Data Types](#additional-data-types)
+  - [Arrays](#arrays)
+  - [IDs](#ids)
+- [Future Considerations](#future-considerations)
+
+## Escaping Attribute Names
+
+OpenTelemetry supports [any valid Unicode sequence](https://opentelemetry.io/docs/specs/otel/common/attribute-naming/) for attribute names. This includes characters problematic for a query language such as whitespace, mathematical symbols, various braces, etc. Previously TraceQL did not support [these problematic characters](https://github.com/grafana/tempo/blob/c00e7ef1103a8e7804b6f71fe8e90d0a2d9a9b02/pkg/traceql/lexer.go#L344) in attributes names.
+
+TraceQL will be extended to support these characters by allowing attribute names wrapped in `"` to be unparsed.
+
+`{ span."attribute with spaces" = "foo"}`
+
+TraceQL will also support the escape sequences `\\` and `\"` in an attribute name.
+
+`{ span."this is \"bad\"" = "foo"}`
+
+## Additional Scopes
+
+We have added four additional scopes to the TraceQL language: [trace](#trace), [scope](#instrumentation-scope), [event](#event), and [link](#link). See below for details.
+
+### Trace
+
+The trace scope allows a user to access intrinsic fields that exist at the trace level. There are no attributes at the trace scope. See [intrinsics](#scoped-intrinsics) for a complete list of allowed trace level intrinsics.
+
+```
+{ trace:duration > 100ms }
+```
+
+### Instrumentation Scope
+
+The [Instrumentation Scope](https://github.com/open-telemetry/opentelemetry-proto/blob/ea449ae0e9b282f96ec12a09e796dbb3d390ed4f/opentelemetry/proto/common/v1/common.proto#L71) object contains information about the instrumentation used to generate telemetry data. This scope contains new intrinsics as well as attributes. It will be referenced using the `scope` keyword.
+
+```
+{ scope.foo = "bar" }
+{ scope:name ~= ".*Java.*" }
+```
+
+### Event
+
+Span [events](https://github.com/open-telemetry/opentelemetry-proto/blob/ea449ae0e9b282f96ec12a09e796dbb3d390ed4f/opentelemetry/proto/trace/v1/trace.proto#L213) contain attributes as well as intrinsic properties that describe discrete events that occur during a span's lifetime. Note that both events and [links](#link) are different than existing scopes because there can be many events to one span. Due to this we are being purposefully conservative when defining event scopes and intend to extend functionality as we better understand use cases. See [future considerations](#future-considerations) for additional discussion.
+
+```
+{ event.exception.message =~ ".*Division by zero.*" }
+{ event:name = "exception" }
+```
+
+In the above cases the operators will return true if there are any events on the span that meets the specified condition.
+
+### Link
+
+Span [links](https://github.com/open-telemetry/opentelemetry-proto/blob/ea449ae0e9b282f96ec12a09e796dbb3d390ed4f/opentelemetry/proto/trace/v1/trace.proto#L242) contain attributes as well as intrinsic properties that can create connections to other spans. Note that both [events](#event) and links are different than existing scopes because there can be many events to one span. Due to this we are being purposefully conservative when defining event scopes and intend to extend functionality as we better understand use cases. See [future considerations](#future-considerations) for additional discussion.
+
+```
+{ link.foo = "bar" }
+{ link:traceID = "<hex string>" }
+```
+
+In the above cases the operators will return true if there are any links on the span that meets the specified condition.
+
+## Scoped Intrinsics
+
+TraceQL supports a number of intrinsics that will continue to exist in the language. These will be referred to as "Legacy Intrinsics" and there are currently no plans to remove them. However, due to the increasing number of [scopes](#additional-scopes) we will provide scoped names for all legacy intrinsics and all future intrinsics will be scoped only.
+
+Scoped intrinsics will always use a `:` to separate the scope from the intrinsic to differentiate between an attribute of the same name. i.e. `span.name` accesses an attribute "name" on the span vs `span:name` which accesses the intrinsic span name.
+
+Legacy intrinsics will be kept for the foreseeable future to preserve backwards compatibility, but we will not be adding any new unscoped intrinsics.
+
+| Scope  | Intrinsic      | Legacy | Description |
+| ------ | -------------- | ------ | ----------- |
+| trace  | :duration      | traceDuration   | Max end time of any span minus the min start time of any span in the trace. |
+|        | :id            |                 | Trace id |
+|        | :rootName      | rootName | `span:name` of the root span |
+|        | :rootService   | rootServiceName        | `resource.service.name` of the root span |
+| scope  | :name          |                 | Instrumentation scope name |
+|        | :version       |                 | Instrumentation scope version |
+| span   | :id            |                 | Span id |
+|        | :name          | name            | Span name |
+|        | :duration      | duration        | Span duration |
+|        | :status        | status          | Span status | 
+|        | :statusMessage | statusMessage   | Span status | 
+|        | :kind          | kind            | Span kind |
+| event  | :name          |                 | Event name |
+| link   | :spanID        |                 | Link's span id |
+|        | :traceID       |                 | Link's trace id |
+| parent | :id            |                 | Parent's `span:id` |
+
+## Additional Data Types
+
+### Arrays
+
+TraceQL will support [array](https://github.com/open-telemetry/opentelemetry-proto/blob/ea449ae0e9b282f96ec12a09e796dbb3d390ed4f/opentelemetry/proto/common/v1/common.proto#L44) attribute types with the `[]` syntax.
+
+To access a specific element of an array use a 0-based array syntax.
+
+```
+{ span.http.response.header.content-type[0] = "application/json" }
+```
+
+To test all elements of an array use empty square brackets. This will evaluate to true if any of the elements in the array `http.response.header.content-type` are equal to the value `"application/json"`.
+
+```
+{ span.http.response.header.content-type[] = "application/json" }
+```
+
+Note that currently there is no array literal or operations defined on the array data type itself. For now we will limit ourselves to existing operations on the elements of the array using the above syntax.
+
+### IDs
+
+The new [scoped intrinsics](#scoped-intrinsics) section above defines a few fields with an "id" data type such as `span:id`. The id type is only comparable to hex strings and only supports the operations `=` and `!=`. 
+
+```
+{ span.id = "8bf5306cb6a28" }
+```
+
+Span IDs and trace IDs are always 64 bit or 128 bit. However, for the purposes of comparison, leading 0s will be ignored. The following are equivalent.
+
+```
+{ trace.id = "0007f2b8d1c69375e0d46a9cf8072bc4" }
+{ trace.id = "7f2b8d1c69375e0d46a9cf8072bc4" }
+```
+
+## Future Considerations
+
+[Links](#link) and [events](#event) have been introduced with purposefully limited syntax. We think it wise to not commit to language features that we will regret later and want the community to experiment with basic access to these fields and communicate their needs so we can continue to move the language forward in a way that makes sense for everyone. It would be very easy to unnecessarily over complicate the language and we'd prefer taking a slower approach.
+
+Note that currently you can only assert whether there are any links or events with specific field values on a span. We believe this covers a huge range of use cases, but also leaves some obvious gaps. As you are exploring links and events consider how you would like to express more complex relationships such as asserting two conditions on the same event or comparing fields across links.
+
+Thanks for your patience! Hopefully this functionality will jump start links and events in TraceQL and we can quickly follow up with more advanced functionality.
@@ -0,0 +1,173 @@
+---
+Authors: Martin Disibio (@mdisibio)
+Created: 2023 November
+Last updated: 2023-11-27
+---
+
+# TraceQL Metrics
+
+## Summary
+
+This design document describes TraceQL language additions for extracting time-series metrics like error rates and latencies from traces.   
+
+This document is **not** meant to be a complete language specification. Rather it is meant to invite the community to comment on the direction and capabilities.
+
+### Table of Contents
+
+- [Intro](#intro)
+- [First Stage](#first-stage)
+  - [Interval](#interval)
+  - [Grouping](#grouping)
+  - [Filtering](#filtering)
+- [Additional Stages](#additional-stages)
+- [Arithmetic](#arithmetic)
+- [Future](#future)
+  - [Joins](#joins-between-time-series)
+  - [Batching](#multiple-metrics-per-query)
+- [Notes](#notes)
+  - [Step Interval](#step-interval)
+  - [Departure from PromQL/LogQL](#departure-from-promqllogql)
+
+## Intro
+
+Let's look at an example straight away.  This query plots the error rate by service over time:  
+`{status = error } | rate() by (resource.service.name)`
+
+A normal TraceQL query finds matching spans and returns data points like time, duration, service name, status code, etc.  A metrics query at the core does the same thing: _find matching spans_, but instead of seeing individual results, they are aggregated over time to create time-series.  A metrics query is built by extending a regular query with metrics pipeline stages.  The structure looks like this:
+
+`<span query...> | <metrics query...>`
+
+An important understanding is that _any_ valid TraceQL query can be turned into a metrics query. This finds the call rates to service B but only when initiated upstream by service A:  
+`{resource.service.name="A"} >> {resouce.service.name="B"} | rate()`
+
+
+## First Stage
+
+The first stage in a metrics query is responsible for turning spans into time series. This is a unique intersection between the two data types and has specialized aggregation functions.  We use the `..._over_time()` suffix to distinguish them from their counterparts.
+
+| function                                   | description                |
+---------------------------------------------| ---------------------------- 
+|`rate()`                                    | The rate of spans per second
+|`count_over_time()`                         | The count of spans
+|`avg_over_time(<field>)`                    | The average of a numeric value like `duration` or [http.request.body.size](https://opentelemetry.io/docs/specs/semconv/attributes-registry/http/)
+|`max_over_time(<field>)`                    | The maximum of a numeric value.
+|`min_over_time(<field>)`                    | The minimum of a numeric value.
+|`quantile_over_time(<field>, q1, q2, ...)`  | The quantile of a numeric value (e.g. p95). Multiple quantiles can be requested and a time-series is created for each. |
+
+### Interval
+All aggregations take an optional interval like `30s`, `5m`, `1h`. If not specified then the interval will automatically match the step interval of the query. The interval is the last parameter of each function.
+
+`rate(5m)`  
+`quantile_over_time(duration, 0.95, 5m)`
+
+### Grouping
+All aggregations can group by one or more attributes and generate a time-series for each combination of values.  
+
+Plot the request rate of `/myapi` by user ID and HTTP status code:  
+`{ span.http.path = "/myapi" } | rate() by (span.user_id, span.http.status_code)`
+
+Plot the p95 duration of requests to service by HTTP endpoint:  
+`{ resource.service.name = "myservice" } | quantile_over_time(duration, 0.95) by (span.http.path)`
+
+### Filtering
+Aggregations can be combined with comparison operations to filter the output.
+
+Find the request rate per service but only if more than 1000 req/s:  
+`{ } | rate() by (resource.service.name) > 1000`
+
+
+## Additional Stages
+Additional metrics stages can be added to further refine and transform the time-series. These stages receive time-series and generate new time-series so they are separate from the first stage.  
+
+Most functions support the same ability to regroup using `by()` and comparison operators to refilter.  These can be thought of as instant functions that work across inputs at each point in time. Therefore they do not have an interval and do not alter the frequency of data points. If the original aggregation creates a data point every 60s, these will also output a data point every 60s.
+
+| function                        | description                |
+----------------------------------| ---------------------------- 
+| `... \| max() [by(...)]`        | Maximum value for each point in time
+| `... \| min() [by(...)]`        | Minimum value for each point in time
+| `... \| avg() [by(...)]`        | Average across values for each point in time
+| `... \| stddev() [by(...)]`     | Standard deviation across values for each point in time
+| `... \| quantile(q) [by(...)]`  | Quantile across values for each point in time
+| `... \| topk(N)`                | Return the top series for each point in time (grouping TBD)
+
+Find the highest per-pod failure rate in each cluster:  
+`{ status = error } | rate() by (cluster, pod) | max() by (cluster)`
+
+
+## Arithmetic
+Arithmetic operations between time-series can be used to calculate things like error rates.
+
+Find the 5xx error rate for an endpoint:
+```
+({ span.http.path = "/myapi" && span.http.status_code >= 500 } | rate()) 
+   /
+({ span.http.path = "/myapi" | rate())
+```
+
+Operations:
+* `*`
+* `/`
+* `+`
+* `-`
+
+
+## Future
+These are language features and ideas that will be iterated on in the future.
+
+### Joins between time-series
+Arithmetic operations bring up the need to define the join semantics between time-series.  For context see [PromQL join semantics (vector-matching)](https://prometheus.io/docs/prometheus/latest/querying/operators/#vector-matching)
+
+This area is TBD however the current direction is to provide the same functionality with a useful default behavior and specializations to support one-to-many and many-to-one.
+
+### Multiple metrics per query
+It would be great to extend the query language to support calculating multiple metrics in a single query.  It is common to plot related values like error rate and total rate together. This is more efficient because Tempo could compute all the metrics in a single pass over the data.
+
+`quantile_over_time` is already a step in this direction and supports calculating multiple quantiles (e.g. p50 and p95 in one query).  However the full solution needs a higher-level construct like pipeline-splitting to support embedding the full range of metrics calculations in one query.
+
+## Notes
+
+### Step Interval
+The step interval is the explicit resolution of a query.  It is identical to the [Prometheus step interval](https://prometheus.io/docs/prometheus/latest/querying/api/#range-queries).  A step interval of `1m` means that a data point will be returned once every 60 seconds.  It is separate from the `interval` of an aggregation like `rate(1h)`.  The query `rate(1h)` with `step=1m` will still return a data point every 60 seconds but each data point is the rate smoothed over the previous hour.
+
+### Departure from PromQL/LogQL
+The proposed pipeline-oriented syntax and aggregation functions are a departure from PromQL and LogQL. This is not done lightly but after much consideration deemed a better fit for Tempo architecture and goals.  
+
+#### Labels
+Tempo's fundamental architecture is different than Prometheus/Loki which would require awkward and misleading translations of even the most basic PromQL queries to the tracing data model.
+
+Let's take a look at an example. Suppose we want to plot the rate by service.  The PromQL-like query would look like this:
+
+`sum(rate({}[5m])) by (resource.service.name)`
+
+The outer sum expects the resource.service.name label to be provided by the rate, but it is not. Let's unwind one level.
+
+`rate({}[5m])`
+
+In PromQL/LogQL this returns a time-series per stream and they have inherent labels, defined from a combination of the application instrumentation and the metrics pipeline.  But in Tempo there is no stream definition and we are left with two possibilities. 
+
+1. It returns an unlabeled total rate unless it has knowledge of the grouping labels.  This is the proposed syntax for `rate() by(...)`
+
+2. It simulates streams using the available data such as resource/span attributes.  Unfortunately this is not feasible because there is no rule to choose which attributes should be included, and including everything leads to impossibly high cardinality.  I.e. imagine a standard http request: attributes like path, method, status code are desirable for a stream definition, but others like request/response content length, headers, or full url with query params leads to uncontrollable cardinality.
+
+To summarize:  what does `{} | rate()` return?  Option 1 is a single time-series, Option 2 would be billions.
+
+#### Rate
+PromQL and LogQL rates are calculated using a [range vector selector](https://prometheus.io/docs/prometheus/latest/querying/basics/#range-vector-selectors) like `[5m]` combined with a metric.  Both of these examples are valid constructs:
+
+`metric{}`
+
+`metric{}[5m]`
+
+This does not translate to Tempo because there are no streams, only a sea of spans.
+
+`{}` is valid and finds spans
+
+`{}[5m]` But this doesn't mean anything without aggregation.
+
+From there, the interval can be simplified as an parameter to the aggregation function instead of specialized syntax.
+
+#### Pipeline
+The pipeline syntax was chosen over functional syntax for a few reasons.
+
+* Easier to understand as computation flows in a single direction. 
+* Easier to write as modifications require simply adding or deleting a stage instead of the delicate removal/addition of a function, its parameters and closing parenthesees.  An existing query to find spans is converted to a metrics query by adding a stage in one place.
@@ -1,5 +1,5 @@
 # The source of this file is https://raw.githubusercontent.com/grafana/writers-toolkit/main/docs/docs.mk.
-# 3.0.0 (2023-05-18)
+# A changelog is included in the head of the `make-docs` script.
 include variables.mk
 -include variables.mk.local

@@ -34,11 +34,6 @@ endif
 # First project is considered the primary one used for doc-validator.
 PRIMARY_PROJECT := $(subst /,-,$(firstword $(subst :, ,$(firstword $(PROJECTS)))))

-# Name for the container.
-ifeq ($(origin DOCS_CONTAINER), undefined)
-export DOCS_CONTAINER := $(PRIMARY_PROJECT)-docs
-endif
-
 # Host port to publish container port to.
 ifeq ($(origin DOCS_HOST_PORT), undefined)
 export DOCS_HOST_PORT := 3002
@@ -70,50 +65,55 @@ ifeq ($(origin HUGO_REFLINKSERRORLEVEL), undefined)
 export HUGO_REFLINKSERRORLEVEL := WARNING
 endif

+# Whether to pull the latest container image before running the container.
+ifeq ($(origin PULL), undefined)
+export PULL := true
+endif
+
 .PHONY: docs-rm
 docs-rm: ## Remove the docs container.
 	$(PODMAN) rm -f $(DOCS_CONTAINER)

 .PHONY: docs-pull
 docs-pull: ## Pull documentation base image.
-	$(PODMAN) pull $(DOCS_IMAGE)
+	$(PODMAN) pull -q $(DOCS_IMAGE)

 make-docs: ## Fetch the latest make-docs script.
 make-docs:
-	if [[ ! -f "$(PWD)/make-docs" ]]; then
+	if [[ ! -f "$(CURDIR)/make-docs" ]]; then
 		echo 'WARN: No make-docs script found in the working directory. Run `make update` to download it.' >&2
 		exit 1
 	fi

 .PHONY: docs
-docs: ## Serve documentation locally, which includes pulling the latest `DOCS_IMAGE` (default: `grafana/docs-base:latest`) container image. See also `docs-no-pull`.
+docs: ## Serve documentation locally, which includes pulling the latest `DOCS_IMAGE` (default: `grafana/docs-base:latest`) container image. To not pull the image, set `PULL=false`.
+ifeq ($(PULL), true)
 docs: docs-pull make-docs
-	$(PWD)/make-docs $(PROJECTS)
-
-.PHONY: docs-no-pull
-docs-no-pull: ## Serve documentation locally without pulling the `DOCS_IMAGE` (default: `grafana/docs-base:latest`) container image.
-docs-no-pull: make-docs
-	$(PWD)/make-docs $(PROJECTS)
+else
+docs: make-docs
+endif
+	$(CURDIR)/make-docs $(PROJECTS)

 .PHONY: docs-debug
 docs-debug: ## Run Hugo web server with debugging enabled. TODO: support all SERVER_FLAGS defined in website Makefile.
 docs-debug: make-docs
-	WEBSITE_EXEC='hugo server --bind 0.0.0.0 --port 3002 --debug' $(PWD)/make-docs $(PROJECTS)
+	WEBSITE_EXEC='hugo server --bind 0.0.0.0 --port 3002 --debug' $(CURDIR)/make-docs $(PROJECTS)

 .PHONY: doc-validator
-doc-validator: ## Run doc-validator on the entire docs folder.
+doc-validator: ## Run doc-validator on the entire docs folder which includes pulling the latest `DOC_VALIDATOR_IMAGE` (default: `grafana/doc-validator:latest`) container image. To not pull the image, set `PULL=false`.
 doc-validator: make-docs
-	DOCS_IMAGE=$(DOC_VALIDATOR_IMAGE) $(PWD)/make-docs $(PROJECTS)
-
-.PHONY: doc-validator/%
-doc-validator/%: ## Run doc-validator on a specific path. To lint the path /docs/sources/administration, run 'make doc-validator/administration'.
-doc-validator/%: make-docs
-	DOCS_IMAGE=$(DOC_VALIDATOR_IMAGE) DOC_VALIDATOR_INCLUDE=$(subst doc-validator/,,$@) $(PWD)/make-docs $(PROJECTS)
+ifeq ($(PULL), true)
+	$(PODMAN) pull -q $(DOC_VALIDATOR_IMAGE)
+endif
+	DOCS_IMAGE=$(DOC_VALIDATOR_IMAGE) $(CURDIR)/make-docs $(PROJECTS)

 .PHONY: vale
-vale: ## Run vale on the entire docs folder.
+vale: ## Run vale on the entire docs folder which includes pulling the latest `VALE_IMAGE` (default: `grafana/vale:latest`) container image. To not pull the image, set `PULL=false`.
 vale: make-docs
-	DOCS_IMAGE=$(VALE_IMAGE) $(PWD)/make-docs $(PROJECTS)
+ifeq ($(PULL), true)
+	$(PODMAN) pull -q $(VALE_IMAGE)
+endif
+	DOCS_IMAGE=$(VALE_IMAGE) $(CURDIR)/make-docs $(PROJECTS)

 .PHONY: update
 update: ## Fetch the latest version of this Makefile and the `make-docs` script from Writers' Toolkit.
@@ -1,10 +1,250 @@
 #!/bin/sh
 # The source of this file is https://raw.githubusercontent.com/grafana/writers-toolkit/main/docs/make-docs.
-# 3.0.0 (2023-05-18)
+# # `make-docs` procedure changelog
+#
+# Updates should conform to the guidelines in https://keepachangelog.com/en/1.1.0/.
+# [Semantic versioning](https://semver.org/) is used to help the reader identify the significance of changes.
+# Changes are relevant to this script and the support docs.mk GNU Make interface.
+#
+# ## 7.0.0 (2024-05-03)
+#
+# ### Changed
+#
+# - Pull images for all recipes that use containers by default.
+#
+#   Use the `PULL=false` variable to disable this behavior.
+#
+# ### Removed
+#
+# - The `docs-no-pull` target as it's redundant with the new `PULL=false` variable.
+#
+# ## 6.1.0 (2024-04-22)
+#
+# ### Changed
+#
+# - Mount volumes with SELinux labels.
+#
+#   https://docs.docker.com/storage/bind-mounts/#configure-the-selinux-label
+#
+# ### Added
+#
+# - Pseudo project for including only website resources and no website content.
+#
+#   Facilitates testing shortcodes and layout changes with a small documentation set instead of Grafana Cloud or the entire website.
+#
+# ## 6.0.1 (2024-02-28)
+#
+# ### Added
+#
+# - Suppress new errors relating to absent content introduced in https://github.com/grafana/website/pull/17561.
+#
+# ## 6.0.0 (2024-02-16)
+#
+# ### Changed
+#
+# - Require `jq` for human readable `make doc-validator` output.
+#
+# ## 5.4.0 (2024-02-12)
+#
+# ### Changed
+#
+# - Set `WEBSITE_MOUNTS=true` when a user includes the `website` project.
+#
+#   Ensures consistent behavior across repositories.
+#   To disable website mounts, add `export WEBSITE_MOUNTS := false` to your `variables.mk` or `variables.mk.local` file.
+# - Use website mounts and container volumes also when a user includes the `grafana-cloud` project.
+#
+# ## 5.3.0 (2024-02-08)
+#
+# ### Changed
+#
+# - Updated support for plugins monorepo now that multiple projects have been moved into it.
+# - Use `printf` instead of `echo` for better portability of output.
+#
+#   https://www.in-ulm.de/~mascheck/various/echo+printf/
+#
+# ## 5.2.0 (2024-01-18)
+#
+# ### Changed
+#
+# - Updated `make vale` to use latest Vale style and configuration.
+# - Updated `make vale` to use platform appropriate image.
+#
+# ## 5.1.2 (2023-11-08)
+#
+# ### Added
+#
+# - Hide manual_mount warning messages from non-debug output.
+#   Set the DEBUG environment variable to see all hidden messages.
+#
+# ## 5.1.1 (2023-10-30)
+#
+# ### Added
+#
+# - Support for Datadog and Oracle data source plugins repositories.
+#
+# ## 5.1.0 (2023-10-20)
+#
+# ### Added
+#
+# - Support for the plugins monorepo.
+#
+# ## 5.0.0 (2023-10-18)
+#
+# ### Added
+#
+# - Improved support for website repository.
+#
+#   Mount more content and provide some feedback to users that the build can take time.
+#
+# - Ability to enter the `grafana/docs-base` container with a shell using the `ENTER` environment variable.
+#
+# ### Fixed
+#
+# - Correct key combination for interrupting the process.
+#
+#   Keyboards use capital letters so this more accurately reflects the exact key combination users are expected to press.
+#
+# ### Removed
+#
+# - Imperfect implementation of container name.
+#
+#   Facilitates running `make vale` and `make docs` at once.
+#   Container names are convenient for recognition in `docker ps` but the current implementation has more downsides than upsides.
+#
+# - Forced platform specification now that multiple architecture images exist.
+#
+#  Significantly speeds up build times on larger repositories.
+#
+# ## 4.2.2 (2023-10-05)
+
+# - Added support for Jira data source and MongoDB data source plugins repositories.
+#
+# ## 4.2.1 (2023-09-13)
+
+# ## Fixed
+#
+# - Improved consistency of the webserver request loop by polling the Hugo port rather than the proxy port.
+#
+# ## 4.2.0 (2023-09-01)
+#
+# ### Added
+#
+# - Retry the initial webserver request up to ten times to allow for the process to start.
+#   If it is still failing after ten seconds, an error message is logged.
+#
+# ## 4.1.1 (2023-07-20)
+#
+# ### Fixed
+#
+# - Replaced use of `realpath` with POSIX compatible alternative to determine default value for REPOS_PATH.
+#
+# ## 4.1.0 (2023-06-16)
+#
+# ### Added
+#
+# - Mounts of `layouts` and `config` directories for the `website` project.
+#   Ensures that local changes to mounts or shortcodes are reflected in the development server.
+#
+# ### Fixed
+#
+# - Version inference for versioned docs pages.
+#   Pages in versioned projects now have the `versioned: true` front matter set to ensure that "version" in $.Page.Scratch is set on builds.
+#
+# ## 4.0.0 (2023-06-06)
+#
+# ### Removed
+#
+# - `doc-validator/%` target.
+#   The behavior of the target was not as described.
+#   Instead, to limit `doc-validator` to only specific files, refer to https://grafana.com/docs/writers-toolkit/writing-guide/tooling-and-workflows/validate-technical-documentation/#run-on-specific-files.
+#
+# ## 3.0.0 (2023-05-18)
+#
+# ### Fixed
+#
+# - Compatibility with the updated Make targets in the `website` repository.
+#   `docs` now runs this script itself, `server-docs` builds the site with the `docs` Hugo environment.
+#
+# ## 2.0.0 (2023-05-18)
+#
+# ### Added
+#
+# - Support for the grafana-cloud/frontend-observability/faro-web-sdk project.
+# - Use of `doc-validator` v2.0.x which includes breaking changes to command line options.
+#
+# ### Fixed
+#
+# - Source grafana-cloud project from website repository.
+#
+# ### Added
+#
+# - Support for running the Vale linter with `make vale`.
+#
+# ## 1.2.1 (2023-05-05)
+#
+# ### Fixed
+#
+# - Use `latest` tag of `grafana/vale` image by default instead of hardcoded older version.
+# - Fix mounting multiple projects broken by the changes in 1.0.1
+#
+# ## 1.2.0 (2023-05-05)
+#
+# ### Added
+#
+# - Support for running the Vale linter with `make vale`.
+#
+# ### Fixed
+#
+# ## 1.1.0 (2023-05-05)
+#
+# ### Added
+#
+# - Rewrite error output so it can be followed by text editors.
+#
+# ### Fixed
+#
+# - Fix `docs-debug` container process port.
+#
+# ## 1.0.1 (2023-05-04)
+#
+# ### Fixed
+#
+# - Ensure complete section hierarchy so that all projects have a visible menu.
+#
+# ## 1.0.0 (2023-05-04)
+#
+# ### Added
+#
+# - Build multiple projects simultaneously if all projects are checked out locally.
+# - Run [`doc-validator`](https://github.com/grafana/technical-documentation/tree/main/tools/cmd/doc-validator) over projects.
+# - Redirect project root to mounted version.
+#   For example redirect `/docs/grafana/` to `/docs/grafana/latest/`.
+# - Support for Podman or Docker containers with `PODMAN` environment variable.
+# - Support for projects:
+#   - agent
+#   - enterprise-logs
+#   - enterprise-metrics
+#   - enterprise-traces
+#   - grafana
+#   - grafana-cloud
+#   - grafana-cloud/machine-learning
+#   - helm-charts/mimir-distributed
+#   - helm-charts/tempo-distributed
+#   - incident
+#   - loki
+#   - mimir
+#   - oncall
+#   - opentelemetry
+#   - phlare
+#   - plugins
+#   - slo
+#   - tempo
+#   - writers-toolkit
+

 set -ef

-readonly DOCS_CONTAINER="${DOCS_CONTAINER:-make-docs}"
 readonly DOCS_HOST_PORT="${DOCS_HOST_PORT:-3002}"
 readonly DOCS_IMAGE="${DOCS_IMAGE:-grafana/docs-base:latest}"

@@ -14,11 +254,24 @@ readonly DOC_VALIDATOR_SKIP_CHECKS="${DOC_VALIDATOR_SKIP_CHECKS:-^image-}"
 readonly HUGO_REFLINKSERRORLEVEL="${HUGO_REFLINKSERRORLEVEL:-WARNING}"
 readonly VALE_MINALERTLEVEL="${VALE_MINALERTLEVEL:-error}"
 readonly WEBSITE_EXEC="${WEBSITE_EXEC:-make server-docs}"
-# If set, the docs-base image will run a prebuild script that sets up Hugo mounts.
-readonly WEBSITE_MOUNTS="${WEBSITE_MOUNTS:-}"

 PODMAN="$(if command -v podman >/dev/null 2>&1; then echo podman; else echo docker; fi)"

+if ! command -v curl >/dev/null 2>&1; then
+  if ! command -v wget >/dev/null 2>&1; then
+    errr 'either `curl` or `wget` must be installed for this script to work.'
+
+    exit 1
+  fi
+fi
+
+if ! command -v "${PODMAN}" >/dev/null 2>&1; then
+  errr 'either `podman` or `docker` must be installed for this script to work.'
+
+  exit 1
+fi
+
+
 about() {
  cat <<EOF
 Test documentation locally with multiple source repositories.
@@ -48,7 +301,7 @@ EOF
  exit 1
 fi

-readonly REPOS_PATH="${REPOS_PATH:-$(realpath "$(git rev-parse --show-toplevel)/..")}"
+readonly REPOS_PATH="${REPOS_PATH:-$(cd "$(git rev-parse --show-toplevel)/.." && echo "${PWD}")}"

 if [ -z "${REPOS_PATH}" ]; then
  cat <<EOF >&2
@@ -63,21 +316,29 @@ SOURCES_as_code='as-code-docs'
 SOURCES_enterprise_metrics='backend-enterprise'
 SOURCES_enterprise_metrics_='backend-enterprise'
 SOURCES_grafana_cloud='website'
+SOURCES_grafana_cloud_alerting_and_irm_machine_learning='machine-learning'
+SOURCES_grafana_cloud_alerting_and_irm_slo='slo'
 SOURCES_grafana_cloud_k6='k6-docs'
 SOURCES_grafana_cloud_data_configuration_integrations='cloud-onboarding'
 SOURCES_grafana_cloud_frontend_observability_faro_web_sdk='faro-web-sdk'
-SOURCES_grafana_cloud_machine_learning='machine-learning'
 SOURCES_helm_charts_mimir_distributed='mimir'
 SOURCES_helm_charts_tempo_distributed='tempo'
 SOURCES_opentelemetry='opentelemetry-docs'
+SOURCES_plugins_grafana_datadog_datasource='datadog-datasource'
+SOURCES_plugins_grafana_oracle_datasource='oracle-datasource'
+SOURCES_resources='website'

 VERSIONS_as_code='UNVERSIONED'
 VERSIONS_grafana_cloud='UNVERSIONED'
+VERSIONS_grafana_cloud_alerting_and_irm_machine_learning='UNVERSIONED'
+VERSIONS_grafana_cloud_alerting_and_irm_slo='UNVERSIONED'
 VERSIONS_grafana_cloud_k6='UNVERSIONED'
 VERSIONS_grafana_cloud_data_configuration_integrations='UNVERSIONED'
 VERSIONS_grafana_cloud_frontend_observability_faro_web_sdk='UNVERSIONED'
-VERSIONS_grafana_cloud_machine_learning='UNVERSIONED'
 VERSIONS_opentelemetry='UNVERSIONED'
+VERSIONS_plugins_grafana_datadog_datasource='latest'
+VERSIONS_plugins_grafana_oracle_datasource='latest'
+VERSIONS_resources='UNVERSIONED'
 VERSIONS_technical_documentation='UNVERSIONED'
 VERSIONS_website='UNVERSIONED'
 VERSIONS_writers_toolkit='UNVERSIONED'
@@ -86,8 +347,11 @@ PATHS_grafana_cloud='content/docs/grafana-cloud'
 PATHS_helm_charts_mimir_distributed='docs/sources/helm-charts/mimir-distributed'
 PATHS_helm_charts_tempo_distributed='docs/sources/helm-charts/tempo-distributed'
 PATHS_mimir='docs/sources/mimir'
+PATHS_plugins_grafana_datadog_datasource='docs/sources'
+PATHS_plugins_grafana_oracle_datasource='docs/sources'
+PATHS_resources='content'
 PATHS_tempo='docs/sources/tempo'
-PATHS_website='content/docs'
+PATHS_website='content'

 # identifier STR
 # Replace characters that are not valid in an identifier with underscores.
@@ -102,6 +366,77 @@ aget() {
  eval echo '$'"$(identifier "$1")_$(identifier "$2")"
 }

+# src returns the project source repository name for a project.
+src() {
+  _project="$1"
+
+  case "${_project}" in
+    plugins/*)
+      if [ -z "$(aget SOURCES "${_project}")" ]; then
+        echo plugins-private
+      else
+        aget SOURCES "${_project}"
+      fi
+      ;;
+    *)
+      if [ -z "$(aget SOURCES "${_project}")" ]; then
+        echo "${_project}"
+      else
+        aget SOURCES "${_project}"
+      fi
+      ;;
+  esac
+
+  unset _project
+}
+
+# path returns the relative path within the repository that contain the docs for a project.
+path() {
+  _project="$1"
+
+  case "${_project}" in
+    plugins/*)
+      if [ -z "$(aget PATHS "${_project}")" ]; then
+        echo "${_project}/docs/sources"
+      else
+        aget PATHS "${_project}"
+      fi
+      ;;
+    *)
+      if [ -z "$(aget PATHS "${_project}")" ]; then
+        echo "docs/sources"
+      else
+        aget PATHS "${_project}"
+      fi
+  esac
+
+  unset _project
+}
+
+# version returns the version for a project. Unversioned projects return the special value 'UNVERSIONED'.
+version() {
+  _project="$1"
+
+  case "${_project}" in
+    plugins/*)
+      if [ -z "$(aget VERSIONS "${_project}")" ]; then
+        echo "UNVERSIONED"
+      else
+        aget VERSIONS "${_project}"
+      fi
+      ;;
+    *)
+    if [ -z "$(aget VERSIONS "${_project}")" ]; then
+      echo latest
+    else
+      aget VERSIONS "${_project}"
+    fi
+  esac
+
+  unset _project
+}
+
+
 # new_proj populates a new project structure.
 new_proj() {
  _project="$1"
@@ -112,31 +447,19 @@ new_proj() {
  # If version is not set, use the script mapping of project to default versions if it exists.
  # Fallback to 'latest'.
  if [ -z "${_version}" ]; then
-    if [ -z "$(aget VERSIONS "${_project}")" ]; then
-      _version=latest
-    else
-      _version="$(aget VERSIONS "${_project}")"
-    fi
+    _version="$(version "${_project}")"
  fi

  # If repo is not set, use the script mapping of project to repo name if it exists.
  # Fallback to using the project name.
  if [ -z "${_repo}" ]; then
-    if [ -z "$(aget SOURCES "${_project}")" ]; then
-      _repo="${_project}"
-    else
-      _repo="$(aget SOURCES "${_project}")"
-    fi
+    _repo="$(src "${_project}")"
  fi

  # If path is not set, use the script mapping of project to docs sources path if it exists.
  # Fallback to using 'docs/sources'.
  if [ -z "${_path}" ]; then
-    if [ -z "$(aget PATHS "${_project}")" ]; then
-      _path="docs/sources"
-    else
-      _path="$(aget PATHS "${_project}")"
-    fi
+    _path="$(path "${_project}")"
  fi

  echo "${_project}:${_version}:${_repo}:${_path}"
@@ -150,7 +473,7 @@ proj_url() {
 $1
 POSIX_HERESTRING

-  if [ "${_project}" = 'website' ]; then
+  if [ "${_project}" = website ]; then
    echo "http://localhost:${DOCS_HOST_PORT}/docs/"

    unset _project _version
@@ -184,8 +507,8 @@ proj_dst() {
 $1
 POSIX_HERESTRING

-  if [ "${_project}" = 'website' ]; then
-    echo '/hugo/content/docs'
+  if [ "${_project}" = website ]; then
+    echo '/hugo/content'

    unset _project _version
    return
@@ -200,6 +523,28 @@ POSIX_HERESTRING
  unset _project _version
 }

+# repo_path returns the host path to the project repository.
+# It looks for the provided repository name in each of the paths specified in the REPOS_PATH environment variable.
+repo_path() {
+  _repo="$1"
+  IFS=:
+  for lookup in ${REPOS_PATH}; do
+    if [ -d "${lookup}/${_repo}" ]; then
+      echo "${lookup}/${_repo}"
+      unset _path _repo
+      return
+    fi
+  done
+  unset IFS
+
+  errr "could not find project '${_repo}' in any of the paths in REPOS_PATH '${REPOS_PATH}'."
+  note "you must have a checkout of the project '${_repo}' at '${REPOS_PATH##:*}/${_repo}'."
+  note "if you have cloned the repository into a directory with a different name, consider changing it to ${_repo}."
+
+  unset _repo
+  exit 1
+}
+
 # proj_src returns the host path to content source for a project.
 # It expects a complete project structure as input.
 # It looks for the provided repository name in each of the paths specified in the REPOS_PATH environment variable.
@@ -208,21 +553,10 @@ proj_src() {
 $1
 POSIX_HERESTRING

-  IFS=:
-  for lookup in ${REPOS_PATH}; do
-    if [ -d "${lookup}/${_repo}" ]; then
-      echo "${lookup}/${_repo}/${_path}"
-      unset _path _repo
-      return
-    fi
-  done
-  unset IFS
+  _repo="$(repo_path "${_repo}")"
+  echo "${_repo}/${_path}"

-  echo "ERRR: could not find project '${_repo}' in any of the paths in REPOS_PATH '${REPOS_PATH}'." >&2
-  echo "NOTE: you must have a checkout of the project '${_repo}' at '${REPOS_PATH##:*}/${_repo}'." >&2
-  echo "NOTE: if you have cloned the repository into a directory with a different name, consider changing it to ${_repo}." >&2
  unset _path _repo
-  exit 1
 }

 # proj_canonical returns the canonical absolute path partial URI for a project.
@@ -232,7 +566,7 @@ proj_canonical() {
 $1
 POSIX_HERESTRING

-  if [ "${_project}" = 'website' ]; then
+  if [ "${_project}" = website ]; then
    echo '/docs'

    unset _project _version
@@ -279,6 +613,11 @@ POSIX_HERESTRING
        proj_to_url_src_dst_ver "$(new_proj helm-charts/mimir-distributed "${_version}")"
        proj_to_url_src_dst_ver "$(new_proj enterprise-metrics "${_version}")"
        ;;
+      resources)
+        _repo="$(repo_path website)"
+        echo "arbitrary^${_repo}/config^/hugo/config" "arbitrary^${_repo}/layouts^/hugo/layouts" "arbitrary^${_repo}/scripts^/hugo/scripts"
+        unset _repo
+        ;;
      traces)
        proj_to_url_src_dst_ver "$(new_proj tempo "${_version}")"
        proj_to_url_src_dst_ver "$(new_proj enterprise-traces "${_version}")"
@@ -292,29 +631,105 @@ POSIX_HERESTRING
  unset _project _version _repo _path
 }

+await_build() {
+  url="$1"
+  req="$(if command -v curl >/dev/null 2>&1; then echo 'curl -s -o /dev/null'; else echo 'wget -q'; fi)"
+
+  i=1
+  max=10
+  while [ "${i}" -ne "${max}" ]
+  do
+    sleep 1
+    debg "Retrying request to web server assuming the process is still starting up."
+    i=$((i + 1))
+
+    if ${req} "${url}"; then
+      printf '\r\nView documentation locally:\r\n'
+      for x in ${url_src_dst_vers}; do
+        IFS='^' read -r url _ _ <<POSIX_HERESTRING
+$x
+POSIX_HERESTRING
+
+        if [ -n "${url}" ]; then
+          if [ "${url}" != arbitrary ]; then
+            printf '\r  %s\r\n' "${url}"
+          fi
+        fi
+      done
+      printf '\r\nPress Ctrl+C to stop the server\r\n'
+
+      unset i max req url
+      return
+    fi
+  done
+
+  printf '\r\n'
+  errr 'The build was interrupted or a build error occurred, check the previous logs for possible causes.'
+  note 'You might need to use Ctrl+C to end the process.'
+
+  unset i max req url
+}
+
+debg() {
+  if [ -n "${DEBUG}" ]; then
+    printf 'DEBG: %s\r\n' "$1" >&2
+  fi
+}
+
+errr() {
+  printf 'ERRR: %s\r\n' "$1" >&2
+}
+
+note() {
+  printf 'NOTE: %s\r\n' "$1" >&2
+}
+
 url_src_dst_vers="$(url_src_dst_vers "$@")"

 volumes=""
 redirects=""

+for arg in "$@"; do
+  IFS=: read -r _project _ _repo _ <<POSIX_HERESTRING
+${arg}
+POSIX_HERESTRING
+  if [ "${_project}" = website ] || [ "${_project}" = grafana-cloud ]; then
+    note "Please be patient, building the website can take some time."
+
+      # If set, the docs-base image will run a prebuild script that sets up Hugo mounts.
+    if [ "${WEBSITE_MOUNTS}" = false ]; then
+      unset WEBSITE_MOUNTS
+    else
+      readonly WEBSITE_MOUNTS=true
+    fi
+
+    _repo="$(repo_path website)"
+    volumes="--volume=${_repo}/config:/hugo/config:z"
+    volumes="${volumes} --volume=${_repo}/layouts:/hugo/layouts:z"
+    volumes="${volumes} --volume=${_repo}/scripts:/hugo/scripts:z"
+  fi
+  unset _project _repo
+done
+
 for x in ${url_src_dst_vers}; do
  IFS='^' read -r _url _src _dst _ver <<POSIX_HERESTRING
 $x
 POSIX_HERESTRING

-  if [ "${_url}" != "arbitrary" ]; then
+  if [ "${_url}" != arbitrary ]; then
    if [ ! -f "${_src}/_index.md" ]; then
-      echo "ERRR: Index file '${_src}/_index.md' does not exist." >&2
-      echo "Is '${_src}' the correct source directory?" >&2
+      errr "Index file '${_src}/_index.md' does not exist."
+      note "Is '${_src}' the correct source directory?"
      exit 1
    fi
  fi

-  echo "DEBG: Mounting '${_src}' at container path '${_dst}'" >&2
+  debg "Mounting '${_src}' at container path '${_dst}'"
+
  if [ -z "${volumes}" ]; then
-    volumes="--volume=${_src}:${_dst}"
+    volumes="--volume=${_src}:${_dst}:z"
  else
-    volumes="${volumes} --volume=${_src}:${_dst}"
+    volumes="${volumes} --volume=${_src}:${_dst}:z"
  fi

  if [ -n "${_ver}" ] && [ "${_ver}" != 'UNVERSIONED' ]; then
@@ -333,53 +748,70 @@ POSIX_HERESTRING

 case "${image}" in
  'grafana/doc-validator')
+    if ! command -v jq >/dev/null 2>&1; then
+      errr '`jq` must be installed for the `doc-validator` target to work.'
+      note 'To install `jq`, refer to https://jqlang.github.io/jq/download/,'
+
+      exit 1
+    fi
+
    proj="$(new_proj "$1")"
-    echo
+    printf '\r\n'
    "${PODMAN}" run \
-      --init \
-      --interactive \
-      --name "${DOCS_CONTAINER}" \
-      --platform linux/amd64 \
-      --rm \
-      --tty \
-      ${volumes} \
-      "${DOCS_IMAGE}" \
-      --include="${DOC_VALIDATOR_INCLUDE}" \
-      --skip-checks="${DOC_VALIDATOR_SKIP_CHECKS}" \
-      /hugo/content/docs \
-      "$(proj_canonical "${proj}")" | sed "s#$(proj_dst "${proj}")#sources#"
+                --init \
+                --interactive \
+                --platform linux/amd64 \
+                --rm \
+                --tty \
+                ${volumes} \
+                "${DOCS_IMAGE}" \
+                "--include=${DOC_VALIDATOR_INCLUDE}" \
+                "--skip-checks=${DOC_VALIDATOR_SKIP_CHECKS}" \
+                "/hugo/content$(proj_canonical "${proj}")" \
+                "$(proj_canonical "${proj}")" \
+                | sed "s#$(proj_dst "${proj}")#sources#" \
+                | jq -r '"ERROR: \(.location.path):\(.location.range.start.line // 1):\(.location.range.start.column // 1): \(.message)" + if .suggestions[0].text then "\nSuggestion: \(.suggestions[0].text)" else "" end'
    ;;
  'grafana/vale')
    proj="$(new_proj "$1")"
-    echo
+    printf '\r\n'
    "${PODMAN}" run \
-      --init \
-      --interactive \
-      --name "${DOCS_CONTAINER}" \
-      --platform linux/amd64 \
-      --rm \
-      --tty \
-      ${volumes} \
-      "${DOCS_IMAGE}" \
-      --minAlertLevel="${VALE_MINALERTLEVEL}" \
-      --config=/etc/vale/.vale.ini \
-      --output=line \
-      /hugo/content/docs | sed "s#$(proj_dst "${proj}")#sources#"
+                --init \
+                --interactive \
+                --rm \
+                --workdir /etc/vale \
+                --tty \
+                ${volumes} \
+                "${DOCS_IMAGE}" \
+                "--minAlertLevel=${VALE_MINALERTLEVEL}" \
+                '--glob=*.md' \
+                --output=/etc/vale/rdjsonl.tmpl \
+                /hugo/content/docs | sed "s#$(proj_dst "${proj}")#sources#"
    ;;
  *)
    tempfile="$(mktemp -t make-docs.XXX)"
    cat <<EOF >"${tempfile}"
 #!/usr/bin/env bash
+
+tc() {
+  set \${*,,}
+  echo \${*^}
+}
+
 for redirect in ${redirects}; do
  IFS='^' read -r path ver <<<"\${redirect}"
-  echo -e "---\\nredirectURL: \"\${path/\/hugo\/content/}\"\\ntype: redirect\\n---\\n" > "\${path/\${ver}/_index.md}"
+  echo -e "---\\nredirectURL: \"\${path/\/hugo\/content/}\"\\ntype: redirect\\nversioned: true\\n---\\n" > "\${path/\${ver}/_index.md}"
 done

 for x in "${url_src_dst_vers}"; do
  IFS='^' read -r _ _ dst _ <<<"\${x}"

+  title="\${dst%/*}"
+  title="\$(tc \${title##*/})"
  while [[ -n "\${dst}" ]]; do
-    touch "\${dst}/_index.md"
+    if [[ ! -f "\${dst}/_index.md" ]]; then
+        echo -e "---title: \${title}\\n---\\n\\n# \${title}\\n\\n{{< section >}}" > "\${dst}/_index.md"
+    fi
    dst="\${dst%/*}"
  done
 done
@@ -391,36 +823,47 @@ fi
 ${WEBSITE_EXEC}
 EOF
    chmod +x "${tempfile}"
-    volumes="${volumes} --volume=$(realpath "${tempfile}"):/entrypoint"
+    volumes="${volumes} --volume=${tempfile}:/entrypoint:z"
    readonly volumes

-    echo
-    echo "Documentation will be served at the following URLs:"
-    for x in ${url_src_dst_vers}; do
-      IFS='^' read -r url _ _ <<POSIX_HERESTRING
-$x
-POSIX_HERESTRING
+    IFS='' read -r cmd <<EOF
+${PODMAN} run \
+  --env=HUGO_REFLINKSERRORLEVEL=${HUGO_REFLINKSERRORLEVEL} \
+  --init \
+  --interactive \
+  --publish=${DOCS_HOST_PORT}:3002 \
+  --publish=3003:3003 \
+  --rm \
+  --tty \
+  ${volumes} \
+  ${DOCS_IMAGE}
+EOF

-      if [ -n "${url}" ]; then
-        if [ "${_url}" != "arbitrary" ]; then
-          echo "  ${url}"
-        fi
-      fi
-    done
+    if [ -n "${ENTER}" ]; then
+      ${cmd} /bin/bash
+    elif [ -n "${DEBUG}" ]; then
+      await_build http://localhost:3003 &

-    echo
-    "${PODMAN}" run \
-      --env "HUGO_REFLINKSERRORLEVEL=${HUGO_REFLINKSERRORLEVEL}" \
-      --init \
-      --interactive \
-      --name "${DOCS_CONTAINER}" \
-      --platform linux/amd64 \
-      --publish "${DOCS_HOST_PORT}:3002" \
-      --publish "3003:3003" \
-      --rm \
-      --tty \
-      ${volumes} \
-      "${DOCS_IMAGE}" \
-      /entrypoint
+      debg "${cmd} /entrypoint"
+      ${cmd} /entrypoint
+    else
+      await_build http://localhost:3003 &
+
+      ${cmd} /entrypoint  2>&1\
+        | sed -u \
+              -e '/Web Server is available at http:\/\/localhost:3003\/ (bind address 0.0.0.0)/ d' \
+              -e '/^hugo server/ d' \
+              -e '/fatal: not a git repository (or any parent up to mount point \/)/ d' \
+              -e '/Stopping at filesystem boundary (GIT_DISCOVERY_ACROSS_FILESYSTEM not set)./ d' \
+              -e "/Makefile:[0-9]*: warning: overriding recipe for target 'docs'/ d" \
+              -e "/docs.mk:[0-9]*: warning: ignoring old recipe for target 'docs'/ d" \
+              -e '/\/usr\/bin\/make -j 2 proxy hserver-docs HUGO_PORT=3003/ d' \
+              -e '/website-proxy/ d' \
+              -e '/rm -rf dist*/ d' \
+              -e '/Press Ctrl+C to stop/ d' \
+              -e '/make/ d' \
+              -e '/WARNING: The manual_mount source directory/ d' \
+              -e '/docs\/_index.md .* not found/ d'
+    fi
    ;;
 esac
@@ -258,6 +258,12 @@ The `storage` block defined in the `values.yaml` file is used to configure the s
 The procedure below configures MinIO as the local storage option managed by the Helm chart.
 However, you can use a other storage provides. Refer to the Optional storage section below.

+{{% admonition type="note" %}}
+
+The MinIO installation that is included with this Helm chart is for demonstration purposes only, and is configured for a maximum storage size of 5GiB. This MinIO installation is not suitable for production environments and should only be used for example purposes. Performant, Enterprise-grade object storage should always be used in production
+
+{{% /admonition %}}
+
 The Helm chart values provided include the basic MinIO set up values. If you need to customize them, the steps below walk you through which sections to update.
 If you do not need to change the values, you can skip this section.

@@ -290,7 +296,7 @@ If you do not need to change the values, you can skip this section.
          insecure: true
    ```

-1. Optional: Locate the MinIO section and change the the username and password to something you wish to use.
+1. Optional: Locate the MinIO section and change the username and password to something you wish to use.

   ```yaml
   minio:
@@ -319,11 +325,11 @@ minio:

 Update the `storage` configuration options based upon your requirements:

- [Amazon S3 configuration documentation]({{< relref "/docs/tempo/latest/configuration/s3" >}}). The Amazon S3 example is identical to the MinIO configuration. The two last options, `endpoint` and `insecure`, are dropped.
+- [Amazon S3 configuration documentation]({{< relref "/docs/tempo/latest/configuration/hosted-storage/s3" >}}). The Amazon S3 example is identical to the MinIO configuration. The two last options, `endpoint` and `insecure`, are dropped.

- [Azure Blob Storage configuration documentation]({{< relref "/docs/tempo/latest/configuration/azure" >}})
+- [Azure Blob Storage configuration documentation]({{< relref "/docs/tempo/latest/configuration/hosted-storage/azure" >}})

- [Google Cloud Storage configuration documentation]({{< relref "/docs/tempo/latest/configuration/gcs" >}})
+- [Google Cloud Storage configuration documentation]({{< relref "/docs/tempo/latest/configuration/hosted-storage/gcs" >}})

 ### Set traces receivers

@@ -426,7 +432,7 @@ NOTES:
 ***********************************************************************
 Welcome to Grafana Tempo
 Chart version: 1.0.1
- Tempo version: 2.1.1
+ Tempo version: 2.2.0
 ***********************************************************************

 Installed components:
@@ -1,27 +1,59 @@
 ---
-title: Tempo documentation
+title: Grafana Tempo
 description: Grafana Tempo is an open source distributed tracing backend.
 aliases:
  - /docs/tempo/
+cascade:
+  GRAFANA_VERSION: next
+hero:
+  title: Grafana Tempo
+  level: 1
+  image: /static/assets/img/blog/tempo.png
+  width: 110
+  height: 110
+  description: >-
+    Grafana Tempo is an open-source, easy-to-use, and high-scale distributed tracing backend. Tempo lets you search for traces, generate metrics from spans, and link your tracing data with logs and metrics.
+cards:
+  title_class: pt-0 lh-1
+  items:
+    - title: Learn about tracing
+      href: /docs/tempo/latest/getting-started/
+      description: What is distributed tracing? Learn about traces and how you can use them, how you can instrument your app for tracing, and how you can visualize tracing data in Grafana.
+    - title: Set up Tempo
+      href: /docs/tempo/latest/setup/
+      description: Plan your deployment to meet your needs, deploy Tempo, test your installation, and configure Tempo services.
+    - title: Manage Tempo
+      href: /docs/tempo/latest/operations/
+      description: Learn about Tempo architecture, best practices, Parquet backend, dedicated attribute columns, metrics from traces, and more.
+    - title: Metrics and tracing
+      href: /docs/tempo/latest/metrics-generator/
+      description: Use metrics-generator to derive metrics from ingested traces. The metrics-generator processes spans and writes metrics to a Prometheus data source using the Prometheus remote write protocol.
+    - title: Query with TraceQL
+      href: /docs/tempo/latest/traceql/
+      description: Inspired by PromQL and LogQL, TraceQL is a query language designed for selecting traces in Tempo. This query language lets you precisely and easily select spans and jump directly to the spans fulfilling the specified conditions.
 ---

-# Tempo documentation
+{{< docs/hero-simple key="hero" >}}

-<p align="center"><img src="logo_and_name.png" alt="Tempo Logo"></p>
+---

-Grafana Tempo is an open source, easy-to-use, and high-volume distributed tracing backend. Tempo is cost-efficient, and only requires an object storage to operate. Tempo is deeply integrated with Grafana, Mimir, Prometheus, and Loki. You can use Tempo with open-source tracing protocols, including Jaeger, Zipkin, or OpenTelemetry.
+## Overview

-Tempo integrates well with a number of existing open source tools:
-
- **Grafana** ships with native support for Tempo using the built-in [Tempo data source](/docs/grafana/latest/datasources/tempo/).
- **Grafana Loki**, with its powerful query language [LogQL v2](/blog/2020/10/28/loki-2.0-released-transform-logs-as-youre-querying-them-and-set-up-alerts-within-loki/) allows you to filter requests that you care about, and jump to traces using the [Derived fields support in Grafana](/docs/grafana/latest/datasources/loki/#derived-fields).
- **Prometheus exemplars** let you jump from Prometheus metrics to Tempo traces by clicking on recorded exemplars. Read more about this integration in the blog post [Intro to exemplars, which enable Grafana Tempo’s distributed tracing at massive scale](/blog/2021/03/31/intro-to-exemplars-which-enable-grafana-tempos-distributed-tracing-at-massive-scale/).
+Distributed tracing visualizes the lifecycle of a request as it passes through a set of applications.

+Tempo is cost-efficient and only requires an object storage to operate.
+Tempo is deeply integrated with Grafana, Mimir, Prometheus, and Loki.
+You can use Tempo with open source tracing protocols, including Jaeger, Zipkin, or OpenTelemetry.
 <p align="center"><img src="getting-started/assets/trace_custom_metrics_dash.png" alt="Trace visualization in Grafana "></p>

-Grafana Tempo builds an index from the high-cardinality trace-id field. Because Tempo uses an object store as a backend, Tempo can query many blocks simultaneously, so queries are highly parallelized.
-For more information, see [Architecture]({{< relref "./operations/architecture" >}}).
+Tempo integrates well with a number of open source tools:

-## Learn more about Tempo
+- **Grafana** ships with native support using the built-in [Tempo data source](/docs/grafana/latest/datasources/tempo/).
+- **Grafana Loki**, with its powerful query language LogQL v2 lets you filter requests that you care about, and jump to traces using the [Derived fields support in Grafana](/docs/grafana/latest/datasources/loki/#derived-fields).
+- **Prometheus exemplars** let you jump from Prometheus metrics to Tempo traces by clicking on recorded exemplars.

-{{< section >}}
+## Explore
+
+{{< card-grid key="cards" type="simple" >}}
+
+<p align="center"><img src="/static/img/search/tempo.svg"  alt="Tempo Logo"></p>
@@ -15,7 +15,7 @@ These endpoints are exposed both when running Tempo in microservices and monolit
 - **microservices**: each service exposes its own endpoints
 - **monolithic**: the Tempo process exposes all API endpoints for the services running internally

-For externally support GRPC API [see below](#tempo-grpc-api)
+For externally supported GRPC API, [see below](#tempo-grpc-api).

 ## Endpoints

@@ -32,6 +32,7 @@ For externally support GRPC API [see below](#tempo-grpc-api)
 | [Search tag values](#search-tag-values) | Query-frontend | HTTP | `GET /api/search/tag/<tag>/values` |
 | [Search tag values V2](#search-tag-values-v2) | Query-frontend | HTTP | `GET /api/v2/search/tag/<tag>/values` |
 | [Query Echo Endpoint](#query-echo-endpoint) | Query-frontend |  HTTP | `GET /api/echo` |
+| [Overrides API](#overrides-api) | Query-frontend | HTTP | `GET,POST,PATCH,DELETE /api/overrides` |
 | Memberlist | Distributor, Ingester, Querier, Compactor |  HTTP | `GET /memberlist` |
 | [Flush](#flush) | Ingester |  HTTP | `GET,POST /flush` |
 | [Shutdown](#shutdown) | Ingester |  HTTP | `GET,POST /shutdown` |
@@ -40,8 +41,9 @@ For externally support GRPC API [see below](#tempo-grpc-api)
 | [Metrics-generator ring status](#metrics-generator-ring-status) (*) | Distributor |  HTTP | `GET /metrics-generator/ring` |
 | [Compactor ring status](#compactor-ring-status) | Compactor |  HTTP | `GET /compactor/ring` |
 | [Status](#status) | Status |  HTTP | `GET /status` |
+| [List build information](#list-build-information) | Status |  HTTP | `GET /api/status/buildinfo` |

-_(*) This endpoint is not always available, check the specific section for more details._
+_(*) This endpoint isn't always available, check the specific section for more details._

 ### Readiness probe

@@ -101,52 +103,57 @@ a microservices deployment or the Tempo endpoint in a monolithic mode deployment
 ```
 GET /api/traces/<traceid>?start=<start>&end=<end>
 ```
+
 Parameters:
+
 - `start = (unix epoch seconds)`
-  Optional.  Along with `end` define a time range from which traces should be returned.
+  Optional. Along with `end` define a time range from which traces should be returned.
 - `end = (unix epoch seconds)`
-  Optional.  Along with `start` define a time range from which traces should be returned. Providing both `start` and `end` will include traces for the specified time range only. If the parameters are not provided then Tempo will check for the trace across all blocks in backend. If the parameters are provided, it will only check in the blocks within the specified time range, this can result in trace not being found or partial results if it does not fall in the specified time range.
+  Optional. Along with `start` define a time range from which traces should be returned. Providing both `start` and `end` includes traces for the specified time range only. If the parameters aren't provided then Tempo checks for the trace across all blocks in backend. If the parameters are provided, it only checks in the blocks within the specified time range, this can result in trace not being found or partial results if it doesn't fall in the specified time range.

 The following query API is also provided on the querier service for _debugging_ purposes.

 ```
 GET /querier/api/traces/<traceid>?mode=xxxx&blockStart=0000&blockEnd=FFFF&start=<start>&end=<end>
 ```
+
 Parameters:
+
 - `mode = (blocks|ingesters|all)`
  Specifies whether the querier should look for the trace in blocks, ingesters or both (all).
  Default = `all`
 - `blockStart = (GUID)`
-  Specifies the blockID start boundary. If specified, the querier will only search blocks with IDs > blockStart.
+  Specifies the blockID start boundary. If specified, the querier only searches blocks with IDs > blockStart.
  Default = `00000000-0000-0000-0000-000000000000`
  Example: `blockStart=12345678-0000-0000-1235-000001240000`
 - `blockEnd = (GUID)`
-  Specifies the blockID finish boundary. If specified, the querier will only search blocks with IDs < blockEnd.
+  Specifies the blockID finish boundary. If specified, the querier only searches blocks with IDs < blockEnd.
  Default = `FFFFFFFF-FFFF-FFFF-FFFF-FFFFFFFFFFFF`
  Example: `blockStart=FFFFFFFF-FFFF-FFFF-FFFF-456787652341`
 - `start = (unix epoch seconds)`
-  Optional.  Along with `end` define a time range from which traces should be returned.
+  Optional. Along with `end` define a time range from which traces should be returned.
 - `end = (unix epoch seconds)`
-  Optional.  Along with `start` define a time range from which traces should be returned. Providing both `start` and `end` will include blocks for the specified time range only.
+  Optional. Along with `start` define a time range from which traces should be returned. Providing both `start` and `end` includes blocks for the specified time range only.

-This API is not meant to be used directly unless for debugging the sharding functionality of the query
+This API isn't meant to be used directly unless for debugging the sharding functionality of the query
 frontend.

-Returns:
-By default this endpoint returns [OpenTelemetry](https://github.com/open-telemetry/opentelemetry-proto/tree/main/opentelemetry/proto/trace/v1) JSON,
+**Returns**
+
+By default, this endpoint returns [OpenTelemetry](https://github.com/open-telemetry/opentelemetry-proto/tree/main/opentelemetry/proto/trace/v1) JSON,
 but if it can also send OpenTelemetry proto if `Accept: application/protobuf` is passed.

 ### Search

-Tempo's Search API finds traces based on span and process attributes (tags and values). Note that search functionality is **not** available on
+The Tempo Search API finds traces based on span and process attributes (tags and values). Note that search functionality is **not** available on
 [v2 blocks]({{< relref "../configuration/parquet#choose-a-different-block-format" >}}).

-When performing a search, Tempo does a massively parallel search over the given time range, and takes the first N results. Even identical searches will differ due to things like machine load and network latency. TraceQL follows the same behavior.
+When performing a search, Tempo does a massively parallel search over the given time range, and takes the first N results. Even identical searches differs due to things like machine load and network latency. TraceQL follows the same behavior.

 The API is available in the query frontend service in
 a microservices deployment, or the Tempo endpoint in a monolithic mode deployment.

-The following request is used to find traces containing spans from service `myservice` and the url contains `api/myapi`.
+The following request is used to find traces containing spans from service `myservice` and the URL contains `api/myapi`.

 ```
 GET /api/search?tags=service.name%3Dmyservice%20http.url%3Dapi%2Fmyapi
@@ -155,30 +162,33 @@ GET /api/search?tags=service.name%3Dmyservice%20http.url%3Dapi%2Fmyapi
 The URL query parameters support the following values:

 **Parameters for TraceQL Search**
+
 - `q = (TraceQL query)`: Url encoded [TraceQL query]({{< relref "../traceql" >}}).

 **Parameters for Tag Based Search**
+
 - `tags = (logfmt)`: logfmt encoding of any span-level or process-level attributes to filter on. The value is matched as a case-insensitive substring. Key-value pairs are separated by spaces. If a value contains a space, it should be enclosed within double quotes.
 - `minDuration = (go duration value)`
-  Optional.  Find traces with at least this duration.  Duration values are of the form `10s` for 10 seconds, `100ms`, `30m`, etc.
+  Optional. Find traces with at least this duration. Duration values are of the form `10s` for 10 seconds, `100ms`, `30m`, etc.
 - `maxDuration = (go duration value)`
-  Optional.  Find traces with no greater than this duration.  Uses the same form as `minDuration`.
+  Optional. Find traces with no greater than this duration. Uses the same form as `minDuration`.

 **Parameters supported for all searches**
+
 - `limit = (integer)`
-  Optional.  Limit the number of search results. Default is 20, but this is configurable in the querier. Refer to [Configuration]({{< relref "../configuration#querier" >}}).
+  Optional. Limit the number of search results. Default is 20, but this is configurable in the querier. Refer to [Configuration]({{< relref "../configuration#querier" >}}).
 - `start = (unix epoch seconds)`
-  Optional.  Along with `end` define a time range from which traces should be returned.
+  Optional. Along with `end` define a time range from which traces should be returned.
 - `end = (unix epoch seconds)`
- Optional.  Along with `start`, define a time range from which traces should be returned. Providing both `start` and `end` will change the way that Tempo searches.
- If the parameters are not provided, then Tempo will search the recent trace data stored in the ingesters. If the parameters are provided, it will search the backend as well.
+ Optional. Along with `start`, define a time range from which traces should be returned. Providing both `start` and `end` changes the way that Tempo searches.
+ If the parameters aren't provided, then Tempo searches the recent trace data stored in the ingesters. If the parameters are provided, it searches the backend as well.
 - `spss = (integer)`
  Optional. Limit the number of spans per span-set. Default value is 3.

 #### Example of TraceQL search

 Example of how to query Tempo using curl.
-This query will return all traces that have their status set to error.
+This query returns all traces that have their status set to error.

 ```bash
 $ curl -G -s http://localhost:3200/api/search --data-urlencode 'q={ status=error }' | jq
@@ -217,10 +227,10 @@ $ curl -G -s http://localhost:3200/api/search --data-urlencode 'q={ status=error
 }
 ```

-#### Example of Tags Based Search
+#### Example of tags-based search

 Example of how to query Tempo using curl.
-This query will return all traces that have a tag `service.name` containing `cartservice` and a minimum duration of 600 ms.
+This query returns all traces that have a tag `service.name` containing `cartservice` and a minimum duration of 600 ms.

 ```bash
 $ curl -G -s http://localhost:3200/api/search --data-urlencode 'tags=service.name=cartservice' --data-urlencode minDuration=600ms | jq
@@ -253,9 +263,9 @@ $ curl -G -s http://localhost:3200/api/search --data-urlencode 'tags=service.nam

 Ingester configuration `complete_block_timeout` affects how long tags are available for search.

-This endpoint retrieves all discovered tag names that can be used in search.  The endpoint is available in the query frontend service in
-a microservices deployment, or the Tempo endpoint in a monolithic mode deployment. The tags endpoint takes a scope that controls the kinds 
-of tags or attributes returned. If nothing is provided, the endpoint will return all resource and span tags.
+This endpoint retrieves all discovered tag names that can be used in search. The endpoint is available in the query frontend service in
+a microservices deployment, or the Tempo endpoint in a monolithic mode deployment. The tags endpoint takes a scope that controls the kinds
+of tags or attributes returned. If nothing is provided, the endpoint returns all resource and span tags.

 ```
 GET /api/search/tags?scope=<resource|span|intrinsic>
@@ -264,7 +274,7 @@ GET /api/search/tags?scope=<resource|span|intrinsic>
 #### Example

 Example of how to query Tempo using curl.
-This query will return all discovered tag names.
+This query returns all discovered tag names.

 ```bash
 $ curl -G -s http://localhost:3200/api/search/tags?scope=span  | jq
@@ -288,22 +298,43 @@ $ curl -G -s http://localhost:3200/api/search/tags?scope=span  | jq
 }
 ```

+Parameters:
+
+- `scope = (resource|span|intrinsic)`
+  Optional. Specifies the scope of the tags. If not specified, it means all scopes.
+  Default = `all`
+- `start = (unix epoch seconds)`
+  Optional. Along with `end`, defines a time range from which tags should be returned.
+- `end = (unix epoch seconds)`
+  Optional. Along with `start`, defines a time range from which tags should be returned. Providing both `start` and `end` includes blocks for the specified time range only.
+
+
 ### Search tags V2

-Ingester configuration `complete_block_timeout` affects how long tags are available for search.
+Ingester configuration `complete_block_timeout` affects how long tags are available for search.If start or end aren't specified, it only fetches blocks that wasn't flushed to backend.

-This endpoint retrieves all discovered tag names that can be used in search.  The endpoint is available in the query frontend service in
-a microservices deployment, or the Tempo endpoint in a monolithic mode deployment. The tags endpoint takes a scope that controls the kinds 
-of tags or attributes returned. If nothing is provided, the endpoint will return all resource and span tags.
+This endpoint retrieves all discovered tag names that can be used in search. The endpoint is available in the query frontend service in
+a microservices deployment, or the Tempo endpoint in a monolithic mode deployment. The tags endpoint takes a scope that controls the kinds
+of tags or attributes returned. If nothing is provided, the endpoint returns all resource and span tags.

-```
+```bash
 GET /api/v2/search/tags?scope=<resource|span|intrinsic>
 ```

+Parameters:
+
+- `scope = (resource|span|intrinsic)`
+  Specifies the scope of the tags, this is an optional parameter, if not specified it means all scopes.
+  Default = `all`
+- `start = (unix epoch seconds)`
+  Optional. Along with `end` define a time range from which tags should be returned.
+- `end = (unix epoch seconds)`
+  Optional. Along with `start` define a time range from which tags should be returned. Providing both `start` and `end` includes blocks for the specified time range only.
+
 #### Example

 Example of how to query Tempo using curl.
-This query will return all discovered tag names.
+This query returns all discovered tag names.

 ```bash
 $ curl -G -s http://localhost:3200/api/v2/search/tags  | jq
@@ -340,18 +371,20 @@ $ curl -G -s http://localhost:3200/api/v2/search/tags  | jq
 ### Search tag values

 Ingester configuration `complete_block_timeout` affects how long tags are available for search.
+If start or end aren't specified, it only fetches blocks that wasn't flushed to backend.

-This endpoint retrieves all discovered values for the given tag, which can be used in search.  The endpoint is available in the query frontend service in
-a microservices deployment, or the Tempo endpoint in a monolithic mode deployment.  The following request will return all discovered service names.
+This endpoint retrieves all discovered values for the given tag, which can be used in search.
+The endpoint is available in the query frontend service in a microservices deployment, or the Tempo endpoint in a monolithic mode deployment.
+The following request returns all discovered service names.

-```
+```bash
 GET /api/search/tag/service.name/values
 ```

 #### Example

 Example of how to query Tempo using curl.
-This query will return all discovered values for the tag `service.name`.
+This query returns all discovered values for the tag `service.name`.

 ```bash
 $ curl -G -s http://localhost:3200/api/search/tag/service.name/values  | jq
@@ -367,28 +400,19 @@ $ curl -G -s http://localhost:3200/api/search/tag/service.name/values  | jq
 }
 ```

+Parameters:
+- `start = (unix epoch seconds)`
+  Optional. Along with `end`, defines a time range from which tags should be returned.
+- `end = (unix epoch seconds)`
+  Optional. Along with `start`, defines a time range from which tags should be returned. Providing both `start` and `end` includes blocks for the specified time range only.
+
+
 ### Search tag values V2

 This endpoint retrieves all discovered values and their data types for the given TraceQL identifier.
-The endpoint is available in the query frontend service in a microservices deployment, or the Tempo endpoint in a monolithic mode deployment. This endpoint is similar to `/api/search/tag/<tag>/values` but operates on TraceQL identifiers and types. 
+The endpoint is available in the query frontend service in a microservices deployment, or the Tempo endpoint in a monolithic mode deployment. This endpoint is similar to `/api/search/tag/<tag>/values` but operates on TraceQL identifiers and types.
 See [TraceQL]({{< relref "../traceql" >}}) documentation for more information.

-The URL query parameters support the following values:
-
- `q = (TraceQL query)`: Url encoded [TraceQL query]({{< relref "../traceql" >}}).
-  - If a query is provided, results are filtered down to only those traces that match the query.
-  - Queries can be incomplete (eg. `{ .cluster = }`).
-    Tempo will extract only the valid matchers and build a valid query.
-  - Only queries with a single selector `{}` and AND `&&` operators are supported.
-    - Example supported: `{ .cluster = "us-east-1" && .service = "frontend" }`
-    - Example unsupported: `{ .cluster = "us-east-1" || .service = "frontend" } && { .cluster = "us-east-2" }`
-
-The following request returns all discovered service names.
-
-```
-GET /api/v2/search/tag/.service.name/values?q="{span.http.method='GET'}"
-```
-
 #### Example

 This example queries Tempo using curl and returns all discovered values for the tag `service.name`.
@@ -420,8 +444,29 @@ $ curl http://localhost:3200/api/v2/search/tag/.service.name/values | jq .
  ]
 }
 ```
+This endpoint can also receive `start` and `end` optional parameters. These parameters define the time range from which the tags are fetched

-### Query Echo Endpoint
+#### Filtered tag values
+
+The `autocomplete_filtering_enabled` configuration parameter is set to `true` by default. This provides an optional URL query parameter, `q`, to your request.
+The `q` parameter is a URL-encoded [TraceQL query]({{< relref "../traceql" >}}).
+If provided, the tag values returned by the API are filtered to only return values seen on spans matching your filter parameters.
+
+Queries can be incomplete: for example, `{ .cluster = }`. Tempo extracts only the valid matchers and build a valid query.
+
+Only queries with a single selector `{}` and AND `&&` operators are supported.
+  - Example supported: `{ .cluster = "us-east-1" && .service = "frontend" }`
+  - Example unsupported: `{ .cluster = "us-east-1" || .service = "frontend" } && { .cluster = "us-east-2" }`
+
+The following request returns all discovered service names on spans with `span.http.method=GET`:
+
+```
+GET /api/v2/search/tag/.service.name/values?q="{span.http.method='GET'}"
+```
+
+If a particular service name (for example, `shopping-cart`) is only present on spans with `span.http.method=POST`, it won't be included in the list of values returned.
+
+### Query Echo endpoint

 ```
 GET /api/echo
@@ -429,10 +474,13 @@ GET /api/echo

 Returns status code 200 and body `echo` when the query frontend is up and ready to receive requests.

-{{% admonition type="note" %}}
+{{< admonition type="note" >}}
 Meant to be used in a Query Visualization UI like Grafana to test that the Tempo data source is working.
 {{% /admonition %}}

+### Overrides API
+
+For more information about user-configurable overrides API, refer to the [user-configurable overrides]{{< relref "../operations/user-configurable-overrides#api" >}} documentation.

 ### Flush

@@ -457,13 +505,13 @@ GET,POST /shutdown
 Flushes all in-memory traces and the WAL to the long term backend. Gracefully exits from the ring. Shuts down the
 ingester service.

-{{% admonition type="note" %}}
+{{< admonition type="note" >}}
 This is usually used at the time of scaling down a cluster.
 {{% /admonition %}}

 ### Distributor ring status

-{{% admonition type="note" %}}
+{{< admonition type="note" >}}
 This endpoint is only available when Tempo is configured with [the global override strategy]({{< relref "../configuration#overrides" >}}).
 {{% /admonition %}}

@@ -471,7 +519,7 @@ This endpoint is only available when Tempo is configured with [the global overri
 GET /distributor/ring
 ```

-Displays a web page with the distributor hash ring status, including the state, healthy and last heartbeat time of each
+Displays a web page with the distributor hash ring status, including the state, healthy, and last heartbeat time of each
 distributor.

 _For more information, check the page on [consistent hash ring]({{< relref "../operations/consistent_hash_ring" >}})._
@@ -482,9 +530,9 @@ _For more information, check the page on [consistent hash ring]({{< relref "../o
 GET /ingester/ring
 ```

-Displays a web page with the ingesters hash ring status, including the state, healthy and last heartbeat time of each ingester.
+Displays a web page with the ingesters hash ring status, including the state, healthy, and last heartbeat time of each ingester.

-_For more information, check the page on [consistent hash ring]({{< relref "../operations/consistent_hash_ring" >}})_
+_For more information, check the page on [consistent hash ring]({{< relref "../operations/consistent_hash_ring" >}}).

 ### Metrics-generator ring status

@@ -494,9 +542,9 @@ GET /metrics-generator/ring

 Displays a web page with the metrics-generator hash ring status, including the state, health, and last heartbeat time of each metrics-generator.

-This endpoint is only available when the metrics-generator is enabled. See [metrics-generator]({{< relref "../configuration#metrics-generator" >}}).
+This endpoint is only available when the metrics-generator is enabled. Refer to [metrics-generator]({{< relref "../configuration#metrics-generator" >}}).

-_For more information, check the page on [consistent hash ring]({{< relref "../operations/consistent_hash_ring" >}})_
+For more information, refer to [consistent hash ring]({{< relref "../operations/consistent_hash_ring" >}}).

 ### Compactor ring status

@@ -504,10 +552,9 @@ _For more information, check the page on [consistent hash ring]({{< relref "../o
 GET /compactor/ring
 ```

-Displays a web page with the compactor hash ring status, including the state, healthy and last heartbeat time of each
-compactor.
+Displays a web page with the compactor hash ring status, including the state, healthy and last heartbeat time of each compactor.

-_For more information, check the page on [consistent hash ring]({{< relref "../operations/consistent_hash_ring" >}})_
+For more information, refer to [consistent hash ring]({{< relref "../operations/consistent_hash_ring" >}}).

 ### Status

@@ -526,7 +573,7 @@ Print the version information.
 GET /status/services
 ```

-Displays a list of services and their status. If a service failed it will show the failure case.
+Displays a list of services and their status. If a service failed it shows the failure case.

 ```
 GET /status/endpoints
@@ -544,6 +591,7 @@ Displays the configuration currently applied to Tempo (in YAML format), includin
 Sensitive data is masked. Please be aware that the exported configuration **doesn't include the per-tenant overrides**.

 Optional query parameter:
+
 - `mode = (diff|defaults)`: `diff` shows the difference between the default values and the current configuration. `defaults` shows the default values.

 ```
@@ -553,71 +601,58 @@ GET /status/runtime_config
 Displays the override configuration.

 Query parameter:
+
 - `mode = (diff)`: Show the difference between defaults and overrides.

+```
+GET /status/overrides
+```
+
+Displays all tenants that have non-default overrides configured.
+
+```
+GET /status/overrides/{tenant}
+```
+
+Displays all overrides configured for the specified tenant.
+
 ```
 GET /status/usage-stats
 ```

-Displays anonymous usage stats data that is reported back to Grafana Labs.
+Displays anonymous usage stats data that's reported back to Grafana Labs.
+
+### List build information
+
+```
+GET /api/status/buildinfo
+```
+Exposes the build information in a JSON object. The fields are `version`, `revision`, `branch`, `buildDate`, `buildUser`, and `goVersion`.

 ## Tempo GRPC API

-Tempo uses GRPC to internally communicate with itself, but only has one externally supported client. The query-frontend component implements
-the streaming querier interface defined below. [See here](https://github.com/grafana/tempo/blob/main/pkg/tempopb/) for the complete proto definition and generated code.
+Tempo uses GRPC to internally communicate with itself, but only has one externally supported client.
+The query-frontend component implements the streaming querier interface defined below.
+[See here](https://github.com/grafana/tempo/blob/main/pkg/tempopb/) for the complete proto definition and generated code.

-The below `rpc` call returns only traces that are new or have updated each time `SearchResponse` is returned except for the last response. The
-final response sent is guaranteed to have the entire resultset.
+By default, this service is only offered over the GRPC port.
+You can use streaming service over the HTTP port as well, which Grafana expects.
+
+To enable the streaming service over the HTTP port for use with Grafana, set the following:
+
+```
+stream_over_http_enabled: true
+```
+
+The query frontend supports the following interface. Refer to [`tempo.proto`](https://github.com/grafana/tempo/blob/main/pkg/tempopb/tempo.proto) for complete details of all objects.

 ```protobuf
 service StreamingQuerier {
  rpc Search(SearchRequest) returns (stream SearchResponse);
-}
-
-message SearchRequest {
-  map<string, string> Tags = 1
-  uint32 MinDurationMs = 2;
-  uint32 MaxDurationMs = 3;
-  uint32 Limit = 4;
-  uint32 start = 5;
-  uint32 end = 6;
-  string Query = 8;
-}
-
-message SearchResponse {
-  repeated TraceSearchMetadata traces = 1;
-  SearchMetrics metrics = 2;
-}
-
-message TraceSearchMetadata {
-  string traceID = 1;
-  string rootServiceName = 2;
-  string rootTraceName = 3;
-  uint64 startTimeUnixNano = 4;
-  uint32 durationMs = 5;
-  SpanSet spanSet = 6; // deprecated. use SpanSets field below
-  repeated SpanSet spanSets = 7;
-}
-
-message SpanSet {
-  repeated Span spans = 1;
-  uint32 matched = 2;
-}
-
-message Span {
-  string spanID = 1;
-  string name = 2;
-  uint64 startTimeUnixNano = 3;
-  uint64 durationNanos = 4;
-  repeated tempopb.common.v1.KeyValue attributes = 5;
-}
-
-message SearchMetrics {
-  uint32 inspectedTraces = 1;
-  uint64 inspectedBytes = 2;
-  uint32 totalBlocks = 3;
-  uint32 completedJobs = 4;
-  uint32 totalJobs = 5;
-  uint64 totalBlockBytes = 6;
+  rpc SearchTags(SearchTagsRequest) returns (stream SearchTagsResponse) {}
+  rpc SearchTagsV2(SearchTagsRequest) returns (stream SearchTagsV2Response) {}
+  rpc SearchTagValues(SearchTagValuesRequest) returns (stream SearchTagValuesResponse) {}
+  rpc SearchTagValuesV2(SearchTagValuesRequest) returns (stream SearchTagValuesV2Response) {}
+  rpc MetricsQueryRange(QueryRangeRequest) returns (stream QueryRangeResponse) {} 
 }
 ```
@@ -0,0 +1,139 @@
+---
+description: Learn how to use the metrics summary API in Tempo
+keywords:
+  - Metrics summary
+  - API
+title: Metrics summary API
+weight: 600
+---
+
+# Metrics summary API
+
+{{< admonition type="warning" >}}
+The Metrics summary API is an [experimental feature](/docs/release-life-cycle) that is disabled by default. To enable it, adjust your configuration as suggested below.
+{{% /admonition %}}
+
+This document explains how to use the metrics summary API in Tempo.
+This API returns RED metrics (span count, erroring span count, and latency information) for `kind=server` spans sent to Tempo in the last hour, grouped by a user-specified attribute.
+
+{{< youtube id="g97CjKOZqT4" >}}
+
+## Configuration
+
+To enable the experimental metrics summary API, you must turn on the local blocks processor in the metrics generator.
+Be aware that the generator will use considerably more resources, including disk space, if it is enabled:
+
+```yaml
+overrides:
+  defaults:
+    metrics_generator:
+      processors: [..., 'local-blocks']
+```
+
+## Request
+
+To make a request to this API, use the following endpoint on the query-frontend:
+
+```
+GET http://<tempo>/api/metrics/summary
+```
+
+### Query Parameters
+
+All query parameters must be URL-encoded to preserve non-URL-safe characters in the query such as `&`.
+
+| Name      | Examples                                                                                         | Definition                                                                                                                                | Required? |
+| --------- | ------------------------------------------------------------------------------------------------ | ----------------------------------------------------------------------------------------------------------------------------------------- | --------- |
+| `q`       | `{ resource.service.name = "foo" && span.http.status_code != 200 }`                              | The TraceQL query with full syntax. All spans matching this query are included in the calculations. Any valid TraceQL query is supported. | Yes       |
+| `groupBy` | `name` <br /> `.foo` <br/> `resource.namespace` <br/> `span.http.url,span.http.status_code` <br> | One or more TraceQL values to group by. Any valid intrinsic or attribute with scope. To group by multiple values use a comma-delimited list.   | Yes       |
+| `start `  | 1672549200                                                                                       | Start of time range in Unix seconds. If not specified, then all recent data is queried.                                                  | No        |
+| `end`     | 1672549200                                                                                       | End of the time range in Unix seconds. If not specified, then all recent data is queried.                                                | No        |
+
+Example:
+
+```bash
+curl "$URL/api/metrics/summary" --data-urlencode 'q={resource.service.name="checkout-service"}' --data-urlencode 'groupBy=name'
+```
+
+## Response
+
+The Tempo response is a `SpanMetricsSummary` object defined in [tempo.proto](https://github.com/grafana/tempo/blob/main/pkg/tempopb/tempo.proto#L234), relevant section pasted below:
+
+```
+message SpanMetricsSummaryResponse {
+  repeated SpanMetricsSummary summaries = 1;
+}
+
+message SpanMetricsSummary {
+  uint64 spanCount = 1;
+  uint64 errorSpanCount = 2;
+  TraceQLStatic static = 3;
+  uint64 p99 = 4;
+  uint64 p95 = 5;
+  uint64 p90 = 6;
+  uint64 p50 = 7;
+}
+
+message TraceQLStatic {
+  int32 type = 1;
+  int64 n = 2;
+  double f = 3;
+  string s = 4;
+  bool b = 5;
+  uint64 d = 6;
+  int32 status = 7;
+  int32 kind = 8;
+}
+
+```
+
+The response is returned as JSON following [standard protobuf->JSON mapping rules](https://protobuf.dev/programming-guides/proto3/#json).
+
+{{< admonition type="note" >}}
+The `uint64` fields cannot be fully expressed by JSON numeric values so the fields are serialized as strings.
+{{% /admonition %}}
+
+Example:
+
+```javascript
+{
+   "summaries": [
+       {
+           "spanCount": "20",
+           "series" : [
+               {
+                   "key": ".attr1",
+                   "value": {
+                       "type": 5,
+                       "s": "foo"
+                   },
+               },
+               ...
+           ],
+           "p99": "68719476736",
+           "p95": "1073741824",
+           "p90": "1017990479",
+           "p50": "664499239"
+       },
+```
+
+| Field             | Notes                                                                                                                                                                                                                                                                                               |
+| ----------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
+| `summaries`       | The list of metrics per group.                                                                                                                                                                                                                                                                     |
+| `.spanCount`      | Number of spans in this group.                                                                                                                                                                                                                                                                     |
+| `.errorSpanCount` | Number of spans with `status`=`error`. (This field will not be present if the value is `0`.)                                                                                                                                                                                                        |
+| `.series`         | The unique values for this group. A key/value pair will be returned for each entry in `groupBy`.                                                                                                                                                                                                   |
+| `.key`            | Key name.                                                                                                                                                                                                                                                                                          |
+| `.value`          | Value with TraceQL underlying type.                                                                                                                                                                                                                                                                |
+| `.type`           | Data type `enum`` defined [here](https://github.com/grafana/tempo/blob/main/pkg/traceql/enum_statics.go#L8) (This field will not be present if the value is `0`.) <br/>0 = `nil`<br/>3 = `integer`<br/> 4 = `float` <br/> 5 = `string`<br/> 6 = `bool`<br/> 7 = `duration`<br/> 8 = span status<br/> 9 = span kind |
+| `.n`              | Populated if this is an integer value.                                                                                                                                                                                                                                                             |
+| `.s`              | Populated if this is a string value.                                                                                                                                                                                                                                                               |
+| `.f`              | Populated if this is a float value.                                                                                                                                                                                                                                                                |
+| `.b`              | Populated if this is a boolean value.                                                                                                                                                                                                                                                              |
+| `.d`              | Populated if this is a duration value.                                                                                                                                                                                                                                                             |
+| `.status`         | Populated if this is a span status value.                                                                                                                                                                                                                                                          |
+| `.kind`           | Populated if this is a span kind value.                                                                                                                                                                                                                                                            |
+| `.p99`            | The p99 latency of this group in nanoseconds.                                                                                                                                                                                                                                                      |
+| `.p95`            | The p95 latency of this group in nanoseconds.                                                                                                                                                                                                                                                      |
+| `.p90`            | The p90 latency of this group in nanoseconds.                                                                                                                                                                                                                                                      |
+| `.p50`            | The p50 latency of this group in nanoseconds.                                                                                                                                                                                                                                                      |
@@ -8,102 +8,127 @@ aliases:
 # Push spans with HTTP

 Sometimes using a tracing system is intimidating because it seems like you need complex application instrumentation
-or a span ingestion pipeline in order to push spans.  This guide aims to show an extremely basic technique for
-pushing spans with HTTP/JSON from a Bash script using the [Zipkin](https://zipkin.io/) receiver.
+or a span ingestion pipeline in order to push spans. This guide aims to show an extremely basic technique for
+pushing spans with HTTP/JSON from a Bash script using the [OpenTelemetry](https://opentelemetry.io/docs/specs/otlp/) receiver.

-## Start Tempo
+## Before you begin

-Let's first start Tempo with the Zipkin receiver configured.  In order to do this create a config file like so:
+This procedure uses an example Docker Compose setup to run Tempo, so you do not need an existing installation. The Docker image also includes a Grafana container which will allow us to visualize traces.

-```yaml
-server:
-  http_listen_port: 3200
+To use this procedure, you need to have Docker and `docker compose` installed.

-distributor:
-  receivers:
-    zipkin:

-storage:
-  trace:
-    backend: local
-    local:
-      path: /tmp/tempo/blocks
-```
+## Start Tempo using the quick start

-and run Tempo using it:
+Use the instructions in the [Quick start for Tempo documentation]({{< relref "../getting-started/docker-example" >}}) to start a local instance of Tempo and Grafana.
+
+## Push spans with OTLP
+
+Now that Tempo is running and listening on port 4318 for [OTLP spans](https://opentelemetry.io/docs/specs/otlp/#otlphttp), let’s push a span to it using `curl`.
+
+Before you can use this example, you need to update the start and end time as instructed.

 ```bash
-docker run -p 9411:9411 -p 3200:3200 -v $(pwd)/config.yaml:/config.yaml grafana/tempo:latest -config.file /config.yaml
+curl -X POST -H 'Content-Type: application/json' http://localhost:4318/v1/traces -d '
+{
+	"resourceSpans": [{
+    	"resource": {
+        	"attributes": [{
+            	"key": "service.name",
+            	"value": {
+                	"stringValue": "my.service"
+            	}
+        	}]
+    	},
+    	"scopeSpans": [{
+        	"scope": {
+            	"name": "my.library",
+            	"version": "1.0.0",
+            	"attributes": [{
+                	"key": "my.scope.attribute",
+                	"value": {
+                    	"stringValue": "some scope attribute"
+                	}
+            	}]
+        	},
+        	"spans": [
+        	{
+            	"traceId": "5B8EFFF798038103D269B633813FC700",
+            	"spanId": "EEE19B7EC3C1B100",
+            	"name": "I am a span!",
+            	"startTimeUnixNano": 1689969302000000000,
+            	"endTimeUnixNano": 1689970000000000000,
+            	"kind": 2,
+            	"attributes": [
+            	{
+                	"key": "my.span.attr",
+                	"value": {
+                    	"stringValue": "some value"
+                	}
+            	}]
+        	}]
+    	}]
+	}]
+}'
 ```

-## Push spans
+Note that the `startTimeUnixNano` field is in nanoseconds and can be obtained by any tool that provides the epoch date in nanoseconds (for example, under Linux, `date +%s%8N`). The `endTimeUnixNano` field is also in nanoseconds, where 100000000 nanoseconds is 100 milliseconds.

-Now that Tempo is running and listening on port 9411 for [Zipkin spans](https://zipkin.io/zipkin-api/#/default/post_spans), let's push a span to it using `curl`.
+1. Copy and paste the curl command into a text editor.

-```bash
-curl -X POST http://localhost:9411 -H 'Content-Type: application/json' -d '[{
- "id": "1234",
- "traceId": "0123456789abcdef",
- "timestamp": 1608239395286533,
- "duration": 100000,
- "name": "span from bash!",
- "tags": {
-    "http.method": "GET",
-    "http.path": "/api"
-  },
-  "localEndpoint": {
-    "serviceName": "shell script"
-  }
-}]'
-```
+1. Replace `startTimeUnixNano` and `endTimeUnixNano` with current values for the last 24 hours to allow you to search for them using a 24 hour relative time range. You can get this in seconds and milliseconds from the following link.
+Multiple the milliseconds value by 1,000,000 to turn it into nanoseconds. You can do this from a bash terminal with:

-Note that the `timestamp` field is in microseconds and was obtained by running `date +%s%6N`.  The `duration` field is also in microseconds and so 100000 is 100 milliseconds.
+   ```bash
+   echo $((<epochTimeMilliseconds> * 1000000))
+   ```
+
+1. Copy the updated curl command to a terminal window and run it.
+
+1. View the trace in Grafana:
+   1. Open a browser window to http://localhost:3000.
+   1. Open the **Explorer** page and select the Tempo data source.
+   1. Select the **Search** query type.
+   1. Select **Run query** to list available traces.
+   1. Select the trace ID (yellow box) to view details about the trace and its spans.
+
+![Using the TraceQL query builder on Explore to view pushed trace in Grafana.](/static/img/docs/tempo/push-spans-search-span-grafana.png "View the span in Grafana")

 ## Retrieve traces

-The easiest way to get the trace is to execute a simple curl command to Tempo.  The returned format is [OTLP](https://github.com/open-telemetry/opentelemetry-proto/blob/main/opentelemetry/proto/trace/v1/trace.proto).
+The easiest way to get the trace is to execute a simple curl command to Tempo. The returned format is [OTLP](https://github.com/open-telemetry/opentelemetry-proto/blob/main/opentelemetry/proto/trace/v1/trace.proto).
+
+1. Replace the trace ID in the `curl` command with the trace ID that was generated from the push. This information is is in the data that's sent with the `curl`. You could use Grafana’s Explorer page to find this, as shown in the previous section.
+
+	```bash
+	curl http://localhost:3200/api/traces/5b8efff798038103d269b633813fc700
+
+	{"batches":[{"resource":{"attributes":[{"key":"service.name","value":{"stringValue":"my.service"}}]},"scopeSpans":[{"scope":{"name":"my.library","version":"1.0.0"},"spans":[{"traceId":"W47/95gDgQPSabYzgT/HAA==","spanId":"7uGbfsPBsQA=","name":"I am a span!","kind":"SPAN_KIND_SERVER","startTimeUnixNano":"1689969302000000000","endTimeUnixNano":"1689970000000000000","attributes":[{"key":"my.span.attr","value":{"stringValue":"some value"}}],"status":{}}]}]}]}
+	```
+
+1. Copy and paste the updated `curl` command into a terminal window.
+
+### Use TraceQL to search for a trace
+
+Alternatively, you can also use [TraceQL]({{< relref "../traceql" >}}) to search for the trace that was pushed.
+You can search by using the unique trace attributes that were set:

 ```bash
-curl http://localhost:3200/api/traces/0123456789abcdef
+curl -G -s http://localhost:3200/api/search --data-urlencode 'q={ .service.name = "my.service" }'

-{"batches":[{"resource":{"attributes":[{"key":"service.name","value":{"stringValue":"shell script"}}]},"instrumentationLibrarySpans":[{"spans":[{"traceId":"AAAAAAAAAAABI0VniavN7w==","spanId":"AAAAAAAAEjQ=","name":"span from bash!","startTimeUnixNano":"1608239395286533000","endTimeUnixNano":"1608239395386533000","attributes":[{"key":"http.path","value":{"stringValue":"/api"}},{"key":"http.method","value":{"stringValue":"GET"}}]}]}]}]}
+{"traces":[{"traceID":"5b8efff798038103d269b633813fc700","rootServiceName":"my.service","rootTraceName":"I am a span!","startTimeUnixNano":"1694718625557000000","durationMs":10000,"spanSet":{"spans":[{"spanID":"eee19b7ec3c1b100","startTimeUnixNano":"1694718625557000000","durationNanos":"10000000000","attributes":[{"key":"service.name","value":{"stringValue":"my.service"}}]}],"matched":1},"spanSets":[{"spans":[{"spanID":"eee19b7ec3c1b100","startTimeUnixNano":"1694718625557000000","durationNanos":"10000000000","attributes":[{"key":"service.name","value":{"stringValue":"my.service"}}]}],"matched":1}]}],"metrics":{"inspectedBytes":"292781","completedJobs":1,"totalJobs":1}}
 ```

-However, staring at a json blob in bash is not very fun.  Let's start up Tempo query so we can visualize our trace.  Tempo query is [Jaeger Query](https://hub.docker.com/r/jaegertracing/jaeger-query/) with a [GRPC Plugin](https://github.com/jaegertracing/jaeger/tree/master/plugin/storage/grpc) that allows it to query Tempo.
+To format this in a more human-readable output, consider using a [tool such as `jq`](https://jqlang.github.io/jq/), which lets you to run the same `curl` command and pipe it to `jq` to format the block. For example:

 ```bash
-docker run --env BACKEND=localhost:3200 --net host grafana/tempo-query:latest
+curl -G -s http://localhost:3200/api/search --data-urlencode 'q={ .service.name = "my.service" }' | jq
 ```

-And open `http://localhost:16686/trace/0123456789abcdef` in the browser of your choice to see:
+## Spans from everything

-<p align="center"><img src="../pushing-spans-with-http.png" alt="single span"></p>
-
-## More spans
-
-Now that we have the basics down, it's easy to continue building our trace.  By specifying the same trace ID and a parent span ID, we can start building a trace.
-
-```bash
-curl -X POST http://localhost:9411 -H 'Content-Type: application/json' -d '[{
- "id": "5678",
- "traceId": "0123456789abcdef",
- "parentId": "1234",
- "timestamp": 1608239395316533,
- "duration": 100000,
- "name": "child span from bash!",
-  "localEndpoint": {
-    "serviceName": "shell script"
-  }
-}]'
-```
-
-And now the UI shows:
-<p align="center"><img src="../pushing-spans-with-http2.png" alt="parent and child spans"></p>
-
-## Spans from everything!
-
-Tracing is not limited to enterprise languages with complex frameworks.  As you can see it's easy to store and track events from your js, python or bash scripts.
-You can use Tempo/distributed tracing today to trace CI pipelines, long running bash processes, python data processing flows or anything else
+Tracing is not limited to enterprise languages with complex frameworks. As you can see it's easy to store and track events from your js, python or bash scripts.
+You can use Tempo/distributed tracing today to trace CI pipelines, long running bash processes, python data processing flows, or anything else
 you can think of.

 Happy tracing!
@@ -4,16 +4,22 @@ description: Participate in and contribute to the Tempo open source community.
 weight: 900
 ---

+# Community
+
+If you are interested in distributed tracing, consider participating and contributing to the [open source Tempo project](https://github.com/grafana/tempo).
+
+Want to learn more about open source communities? Read David Allen's blog post, [What makes a good open source community?](/blog/2023/09/01/what-makes-a-good-open-source-community/) on the Grafana website.
+
 ## Communicate

 - [Grafana Slack](https://slack.grafana.com/) #tempo channel
 - [Community Forum](https://community.grafana.com/c/grafana-tempo/40) - for questions/feedback.
 - [Community Call](https://docs.google.com/document/d/1yGsI6ywU-PxZBjmq3p3vAXr9g5yBXSDk4NU8LGo8qeY/edit#) - Monthly
-  on the second thursday at 1630 UTC.
+  on the second Thursday at 1630 UTC.
  Recordings available [online](https://www.youtube.com/playlist?list=PLDGkOdUX1Ujqe8WZ8T1h2pNjpll0t-KLw).
- [Google Groups](https://groups.google.com/forum/#!forum/tempo-users)

 ## Contribute
+
 This page lists resources for developers who want to contribute to the Tempo software ecosystem.

 - [Governance](https://github.com/grafana/tempo/blob/main/GOVERNANCE.md)
@@ -1,109 +0,0 @@
---
-title: Azure blob storage permissions and management
-menuTitle: Azure blob storage
-description: Azure blog storage permissions and configuration options for Tempo.
-weight: 30
---
-
-# Azure blob storage permissions and management
-
-Tempo requires the following configuration to authenticate to and access Azure blob storage:
-
- Storage Account name specified in the configuration file as `storage_account_name` or in the environment variable `AZURE_STORAGE_ACCOUNT`
- Credentials for accessing the Storage Account; can be one of the following
-  - Storage Account access key specified in the configuration file as `storage_account_key` or in the environment variable `AZURE_STORAGE_KEY`
-  - An Azure Managed Identity; either system or user assigned. To use Azure Managed Identities, you'll need to set `use_managed_identity` to `true` in the configuration file or set `user_assigned_id` to the client ID for the managed identity you'd like to use.
-      - For a system-assigned managed identity, no additional configuration is required.
-      - For a user-assigned managed identity, you'll need to set `user_assigned_id` to the client ID for the managed identity in the configuration file.
-  - Via Azure Workload Identity. To use Azure Workload Identity, you'll need to enable Azure Workload Identity on your cluster, add the required label and annotation to the service account and the required pod label.
-
-## Azure blocklist polling
-
-If you are hosting Tempo on Azure, two values may need to be updated to ensure consistent successful blocklist polling. If you are
-experiencing [this issue](https://stackoverflow.com/questions/12917857/the-specified-block-list-is-invalid-while-uploading-blobs-in-parallel/55902744#55902744), we recommend to set `blocklist_poll_tenant_index_builders` to 1.
-
-Additionally, if you are seeing DNS failures like the ones below, try increasing `blocklist_poll_jitter_ms`. Discussion [here](https://github.com/grafana/tempo/issues/1462).
-```
-reading storage container: Head "https://tempoe**************.blob.core.windows.net/tempo/single-tenant/d8aafc48-5796-4221-ac0b-58e001d18515/meta.compacted.json?timeout=61": dial tcp: lookup tempoe**************.blob.core.windows.net on 10.0.0.10:53: dial udp 10.0.0.10:53: operation was canceled
-```
-
-Your final config may look something like:
-```
-  storage:
-    trace:
-      blocklist_poll_tenant_index_builders: 1
-      blocklist_poll_jitter_ms: 500
-```
-
-## (Optional) Storage Account management policy for cleaning up the storage container
-
-The following Storage Account management policy shows an example of cleaning up
-files from the container after they have been deleted for a period of time.
-
-```json
-{
-  "id": "/subscriptions/00000000-0000-0000000000000000000000/resourceGroups/resourceGroupName/providers/Microsoft.Storage/storageAccounts/accountName/managementPolicies/default",
-  "lastModifiedTime": "2021-11-30T19:19:54.855455+00:00",
-  "name": "DefaultManagementPolicy",
-  "policy": {
-    "rules": [
-      {
-        "definition": {
-          "actions": {
-            "baseBlob": {
-              "delete": {
-                "daysAfterLastAccessTimeGreaterThan": null,
-                "daysAfterModificationGreaterThan": 60.0
-              },
-              "enableAutoTierToHotFromCool": null,
-              "tierToArchive": null,
-              "tierToCool": null
-            },
-            "snapshot": null,
-            "version": null
-          },
-          "filters": {
-            "blobIndexMatch": null,
-            "blobTypes": [
-              "blockBlob"
-            ],
-            "prefixMatch": [
-              "tempo-data"
-            ]
-          }
-        },
-        "enabled": true,
-        "name": "TempoBlobRetention",
-        "type": "Lifecycle"
-      },
-      {
-        "definition": {
-          "actions": {
-            "baseBlob": null,
-            "snapshot": null,
-            "version": {
-              "delete": {
-                "daysAfterCreationGreaterThan": 7.0
-              },
-              "tierToArchive": null,
-              "tierToCool": null
-            }
-          },
-          "filters": {
-            "blobIndexMatch": null,
-            "blobTypes": [
-              "blockBlob"
-            ],
-            "prefixMatch": []
-          }
-        },
-        "enabled": true,
-        "name": "VersionRetention",
-        "type": "Lifecycle"
-      }
-    ]
-  },
-  "resourceGroup": "resource-group-name",
-  "type": "Microsoft.Storage/storageAccounts/managementPolicies"
-}
-```
@@ -1,7 +1,7 @@
 ---
 title: Compression and encoding
 description: Learn about compression and encoding options available for Tempo.
-weight: 50
+weight: 200
 ---

 <!-- Page needs to be updated. -->
@@ -11,7 +11,7 @@ weight: 50
 Tempo can compress traces that it pushes to backend storage. This requires extra
 memory and CPU, but it reduces the quantity of stored data.
 Anecdotal tests suggest that `zstd` will cut your storage costs to ~15% of the uncompressed amount.
-It is _highly_ recommended to use the default `zstd`.
+It is _highly_ recommended to use the default `zstd`. (The compression field is used for the old v2 format. the vParquet* formats compress columns individually.)

 Compression is configured under storage like so:

@@ -49,4 +49,4 @@ storage:
  trace:
    wal:
      v2_encoding: snappy
-```
+```
@@ -8,28 +8,43 @@ aliases:

 # Grafana Agent

-The [Grafana Agent](https://github.com/grafana/agent) is a telemetry
+{{< docs/shared source="alloy" lookup="agent-deprecation.md" version="next" >}}
+
+[Grafana Agent](https://github.com/grafana/agent) is a telemetry
 collector for sending metrics, logs, and trace data to the opinionated
 Grafana observability stack.

-It is commonly used as a tracing pipeline, offloading traces from the
+It's commonly used as a tracing pipeline, offloading traces from the
 application and forwarding them to a storage backend.
-The Grafana Agent tracing stack is built using OpenTelemetry.
+Grafana Agent tracing stack is built using OpenTelemetry.

-The Grafana Agent supports receiving traces in multiple formats:
+Grafana Agent supports receiving traces in multiple formats:
 OTLP (OpenTelemetry), Jaeger, Zipkin, and OpenCensus.

-On top of receiving and exporting traces, the Grafana Agent contains many
+On top of receiving and exporting traces, Grafana Agent contains many
 features that make your distributed tracing system more robust, and
-leverages all the data that is processed in the pipeline.
+leverages all the data that's processed in the pipeline.
+
+## Agent modes
+
+Grafana Agent is available in two different variants:
+
+* [Static mode](/docs/agent/latest/static): The original Grafana Agent.
+* [Flow mode](/docs/agent/latest/flow): The new, component-based Grafana Agent.
+
+Grafana Agent Flow configuration files are [written in River](/docs/agent/latest/flow/concepts/config-language/).
+Static configuration files are [written in YAML](/docs/agent/latest/static/configuration/).
+Examples in this document are for Flow mode.
+
+For more information, refer to the [Introduction to Grafana Agent](/docs/agent/latest/about/).

 ## Architecture

 The Grafana Agent can be configured to run a set of tracing pipelines to collect data from your applications and write it to Tempo.
 Pipelines are built using OpenTelemetry,
-and consist of `receivers`, `processors` and `exporters`. The architecture mirrors that of the OTel Collector's [design](https://github.com/open-telemetry/opentelemetry-collector/blob/846b971758c92b833a9efaf742ec5b3e2fbd0c89/docs/design.md).
-See the [configuration reference](/docs/agent/latest/configuration/traces-config) for all available configuration options.
-For a quick start, refer to this [blog post](/blog/2020/11/17/tracing-with-the-grafana-cloud-agent-and-grafana-tempo/).
+and consist of `receivers`, `processors`, and `exporters`.
+The architecture mirrors that of the OTel Collector's [design](https://github.com/open-telemetry/opentelemetry-collector/blob/846b971758c92b833a9efaf742ec5b3e2fbd0c89/docs/design.md).
+See the [configuration reference](/agent/latest/static/configuration/traces-config/) for all available configuration options.

 <p align="center"><img src="https://raw.githubusercontent.com/open-telemetry/opentelemetry-collector/846b971758c92b833a9efaf742ec5b3e2fbd0c89/docs/images/design-pipelines.png" alt="Tracing pipeline architecture"></p>

@@ -38,12 +53,13 @@ pipelines, each of which collects separate spans and sends them to different
 backends.

 ### Receiving traces
-
+<!-- vale Grafana.Parentheses = NO -->
 The Grafana Agent supports multiple ingestion receivers:
-OTLP (OpenTelemetry), Jaeger, Zipkin, OpenCensus and Kafka.
+OTLP (OpenTelemetry), Jaeger, Zipkin, OpenCensus, and Kafka.
+<!-- vale Grafana.Parentheses = YES -->

 Each tracing pipeline can be configured to receive traces in all these formats.
-Traces that arrive to a pipeline will go through the receivers/processors/exporters defined in it.
+Traces that arrive to a pipeline go through the receivers/processors/exporters defined in that pipeline.

 ### Pipeline processing

@@ -64,7 +80,7 @@ To configure it, refer to the `attributes` block in the [configuration reference
 #### Attaching metadata with Prometheus Service Discovery

 Prometheus Service Discovery mechanisms enable you to attach the same metadata to your traces as your metrics.
-For example, for Kubernetes users this means that you can dynamically attach metadata for namespace, pod, and name of the container sending spans.
+For example, for Kubernetes users this means that you can dynamically attach metadata for namespace, Pod, and name of the container sending spans.

 ```yaml
 traces:
@@ -88,10 +104,10 @@ traces:

 This feature isn’t just useful for Kubernetes users, however.
 All of Prometheus' [various service discovery mechanisms](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#configuration-file) are supported here.
-This means you can use the same scrape_configs between your metrics, logs, and traces to get the same set of labels,
+This means you can use the same `scrape_configs` between your metrics, logs, and traces to get the same set of labels,
 and easily transition between your observability data when moving from your metrics, logs, and traces.

-To configure it, refer to the `scrape_configs` block in the [configuration reference](/docs/agent/latest/configuration/traces-config).
+Refer to the `scrape_configs` block in the [configuration reference](/docs/agent/latest/configuration/traces-config).

 #### Trace discovery through automatic logging

@@ -106,6 +122,8 @@ With this feature, sampling decisions can be made based on data from a trace, ra

 For a detailed description, go to [Tail-based sampling]({{< relref "./tail-based-sampling" >}}).

+For additional information, refer to the blog post, [An introduction to trace sampling with Grafana Tempo and Grafana Agent](/blog/2022/05/11/an-introduction-to-trace-sampling-with-grafana-tempo-and-grafana-agent).
+
 #### Generating metrics from spans

 The Agent can take advantage of the span data flowing through the pipeline to generate Prometheus metrics.
@@ -117,7 +135,7 @@ Go to [Span metrics]({{< relref "./span-metrics" >}}) for a more detailed explan
 Service graph metrics represent the relationships between services within a distributed system.

 This service graphs processor builds a map of services by analyzing traces, with the objective to find _edges_.
-Edges are spans with a parent-child relationship, that represent a jump (e.g. a request) between two services.
+Edges are spans with a parent-child relationship, that represent a jump, such as a request, between two services.
 The amount of requests and their duration are recorded as metrics, which are used to represent the graph.

 To read more about this processor, go to its [section]({{< relref "./service-graphs" >}}).
@@ -9,15 +9,27 @@ aliases:

 # Automatic logging: Trace discovery through logs

-Running distributed tracing systems is very powerful, but it brings its own challenges,
-and one of them is trace discovery.
-Tempo supports finding a trace if you know the trace identifier,
-so we leverage other tools like logs and metrics to discover traces.
+Running instrumented distributed systems is a very powerful way to gain
+understanding over a system, but it brings its own challenges. One of them is
+discovering which traces exist.

-Automatic logging provides an easy and fast way of getting trace discovery through logs.
-Automatic logging writes a well formatted log line to a Loki instance or to stdout for each span, root or process that passes through the tracing pipeline.
-This allows for automatically building a mechanism for trace discovery.
-On top of that, we also get metrics from traces using Loki.
+In the beginning of Tempo, querying for a trace was only possible if you knew
+the ID of the trace you were looking for. One solution was automatic logging.
+Automatic logging provides an easy and fast way of discovering trace IDs
+through log messages. Well-formatted log lines are written to a Loki instance
+or to `stdout` for each span, root, or process that passes through the tracing
+pipeline. This allows for automatically building a mechanism for trace
+discovery. On top of that, we also get metrics from traces using Loki, and
+allow quickly jumping from a log message to the trace view in Grafana.
+
+While this approach is useful, it isn't as powerful as [TraceQL]({{< relref
+"../../traceql" >}}). If you are here because you know you want to log the
+trace ID, to enable jumping from logs to traces, then read on!
+
+If you want to query the system directly, read the [TraceQL
+documentation]({{< relref "../../traceql" >}}). 
+
+## Configuration

 For high throughput systems, logging for every span may generate too much volume.
 In such cases, logging per root span or process is recommended.
@@ -4,7 +4,7 @@ menuTitle: Enable service graphs
 description: Service graphs help to understand the structure of a distributed system, and the connections and dependencies between its components.
 weight:
 aliases:
- /docs/tempo/grafana-agent/service-graphs
+  - ../../grafana-agent/service-graphs/ # /docs/tempo/<TEMPO_VERSION>/grafana-agent/service-graphs/
 ---

 # Enable service graphs
@@ -22,13 +22,15 @@ For more information, refer to the [service graph view documentation]({{< relref

 ## Before you begin

-Service graphs are generated in the Grafana Agent and pushed to a Prometheus-compatible backend.
+Service graphs are generated in Grafana Agent and pushed to a Prometheus-compatible backend.
 Once generated, they can be represented in Grafana as a graph.
-You will need these components to fully use service graphs.
+You need these components to fully use service graphs.

 ### Enable service graphs in Grafana Agent

-To start using service graphs, enable the feature in the Grafana Agent config.
+{{< docs/shared source="alloy" lookup="agent-deprecation.md" version="next" >}}
+
+To start using service graphs, enable the feature in Grafana Agent configuration.

 ```yaml
 traces:
@@ -39,10 +41,10 @@ traces:
        enabled: true
 ```

-To see all the available config options, refer to the [configuration reference](/docs/agent/latest/configuration/traces-config).
+To see all the available configuration options, refer to the [configuration reference](/docs/agent/latest/configuration/traces-config).

 Metrics are registered in the Agent's default registerer.
-Therefore, they are exposed at `/metrics` in the Agent's server port (default 12345).
+Therefore, they are exposed at `/metrics` in the Agent's server port (default `12345`).
 One option is to use the Agent self-scrape capabilities to export the metrics to a Prometheus-compatible backend.

 ```yaml
@@ -62,4 +64,4 @@ metrics:
 The same service graph metrics can also be generated by Tempo.
 This is more efficient and recommended for larger installations.

-For additional information about viewing service graph metrics in Grafana and calculating cardinality, refer to the [server side documentation]({{< relref "../../metrics-generator/service_graphs#enable-service-graphs-in-Grafana" >}}).
+For additional information about viewing service graph metrics in Grafana and calculating cardinality, refer to the [server side documentation]({{< relref "../../metrics-generator/service_graphs/enable-service-graphs" >}}).
@@ -10,23 +10,30 @@ aliases:
 # Tail-based sampling

 Tempo aims to provide an inexpensive solution that makes 100% sampling possible.
-However, sometimes constraints will make a lower sampling percentage necessary or desirable,
+However, sometimes constraints make a lower sampling percentage necessary or desirable,
 such as runtime or egress traffic related costs.
 Probabilistic sampling strategies are easy to implement,
 but also run the risk of discarding relevant data that you'll later want.

+Tail-based sampling works with Grafana Agent in Flow or static modes.
+Flow mode configuration files are [written in River](/docs/agent/latest/flow/concepts/config-language).
+Static mode configuration files are [written in YAML](/docs/agent/latest/static/configuration).
+Examples in this document are for Flow mode. You can also use the [Static mode Kubernetes operator](/docs/agent/latest/operator).
+
+## How tail-based sampling works
+
 In tail-based sampling, sampling decisions are made at the end of the workflow allowing for a more accurate sampling decision.
-The Grafana Agent groups span by trace ID and check its data to see
-if it meets one of the defined policies (for example, latency or status_code).
+The Grafana Agent groups spans by trace ID and checks its data to see
+if it meets one of the defined policies (for example, `latency` or `status_code`).
 For instance, a policy can check if a trace contains an error or if it took
 longer than a certain duration.

-A trace will be sampled if it meets at least one policy.
+A trace is sampled if it meets at least one policy.

 To group spans by trace ID, the Agent buffers spans for a configurable amount of time,
-after which it will consider the trace complete.
-Longer running traces will be split into more than one.
-However, waiting longer times will increase the memory overhead of buffering.
+after which it considers the trace complete.
+Longer running traces are split into more than one.
+However, waiting longer times increases the memory overhead of buffering.

 One particular challenge of grouping trace data is for multi-instance Agent deployments,
 where spans that belong to the same trace can arrive to different Agents.
@@ -35,11 +42,11 @@ by exporting spans belonging to the same trace to the same instance.

 This is achieved by redistributing spans by trace ID once they arrive from the application.
 The Agent must be able to discover and connect to other Agent instances where spans for the same trace can arrive.
-For kubernetes users, that can be done with a [headless service](https://kubernetes.io/docs/concepts/services-networking/service/#headless-services).
+Kubernetes users should use a [headless service](https://kubernetes.io/docs/concepts/services-networking/service/#headless-services).

 Redistributing spans by trace ID means that spans are sent and received twice,
 which can cause a significant increase in CPU usage.
-This overhead will increase with the number of Agent instances that share the same traces.
+This overhead increases with the number of Agent instances that share the same traces.

 <p align="center"><img src="../tail-based-sampling.png" alt="Tail-based sampling overview"></p>

@@ -50,194 +57,148 @@ If you're using a multi-instance deployment of the agent,
 add load balancing and specify the resolving mechanism to find other Agents in the setup.
 To see all the available configuration options, refer to the [configuration reference](/docs/agent/latest/configuration/traces-config/).

-```yaml
-traces:
-  configs:
-    - name: default
-      ...
-      tail_sampling:
-        policies:
-          # sample traces that have a total duration longer than 100ms
-          - latency:
-              threshold_ms: 100
-          # sample traces that contain at least one span with status code ERROR
-          - status_code:
-              status_codes:
-                - "ERROR"
-      load_balancing:
-        resolver:
-          dns:
-            hostname: host.namespace.svc.cluster.local
+## Example for Grafana Agent Flow
+
+{{< docs/shared source="alloy" lookup="agent-deprecation.md" version="next" >}}
+
+[Grafana Agent Flow](/docs/agent/latest/flow/) is a component-based revision of Grafana Agent with a focus on ease-of-use, debuggability, and ability to adapt to the needs of power users.
+Flow configuration files are written in River instead of YAML.
+
+Grafana Agent Flow uses the [`otelcol.processor.tail_sampling component`](/docs/agent/latest/flow/reference/components/otelcol.processor.tail_sampling/)` for tail-based sampling.
+
+```river
+otelcol.receiver.otlp "otlp_receiver" {
+    grpc {
+        endpoint = "0.0.0.0:4317"
+    }
+
+    output {
+        traces = [
+            otelcol.processor.tail_sampling.policies.input,
+        ]
+    }
+}
+
+otelcol.exporter.otlp "tempo" {
+    client {
+        endpoint = "tempo:4317"
+    }
+}
+
+// The Tail Sampling processor will use a set of policies to determine which received
+// traces to keep and send to Tempo.
+otelcol.processor.tail_sampling "policies" {
+    // Total wait time from the start of a trace before making a sampling decision.
+    // Note that smaller time periods can potentially cause a decision to be made
+    // before the end of a trace has occurred.
+    decision_wait = "30s"
+
+    // The following policies follow a logical OR pattern, meaning that if any of the
+    // policies match, the trace will be kept. For logical AND, you can use the `and`
+    // policy. Every span of a trace is examined by each policy in turn. A match will
+    // cause a short-circuit.
+
+    // This policy defines that traces that contain errors should be kept.
+    policy {
+        // The name of the policy can be used for logging purposes.
+        name = "sample-erroring-traces"
+        // The type must match the type of policy to be used, in this case examining
+        // the status code of every span in the trace.
+        type = "status_code"
+        // This block determines the error codes that should match in order to keep
+        // the trace, in this case the OpenTelemetry 'ERROR' code.
+        status_code {
+            status_codes = [ "ERROR" ]
+        }
+    }
+
+    // This policy defines that only traces that are longer than 200ms in total
+    // should be kept.
+    policy {
+        // The name of the policy can be used for logging purposes.
+        name = "sample-long-traces"
+        // The type must match the policy to be used, in this case the total latency
+        // of the trace.
+        type = "latency"
+        // This block determines the total length of the trace in milliseconds.
+        latency {
+            threshold_ms = 200
+        }
+    }
+
+    // The output block forwards the kept traces onto the batch processor, which
+    // will marshall them for exporting to Tempo.
+    output {
+        traces = [otelcol.exporter.otlp.tempo.input]
+    }
+}
+
 ```

-## Examples
+## Examples for Grafana Agent static mode

-Sampling logic can be summarized into two rules:
+For additional information, refer to the blog post, [An introduction to trace sampling with Grafana Tempo and Grafana Agent](/blog/2022/05/11/an-introduction-to-trace-sampling-with-grafana-tempo-and-grafana-agent).

-1. If a policy is met, the trace is sampled. Even if other policies are not met.
-2. If any of the spans of the trace meet the policy requirements, the trace is sampled.
+### Status code tail sampling policy

-Next, there are a couple of examples of `tail_sampling` configurations,
-with descriptions of the policies and the expected behavior.
-
-### Sampling by latency and status
-
-Sampling traces that have a total duration longer than 100ms and traces that contain at least one span with status code `ERROR`.
-Total duration is determined by looking at the earliest start time and latest end time.
-
-```yaml
-traces:
-  configs:
-    - name: default
-      ...
-      tail_sampling:
-        policies:
-          - type: latency
-            latency:
-              threshold_ms: 100
-          - type: status_code
-            status_code:
-              status_codes:
-                - "ERROR"
-```
-
-### Sampling by attributes
-
-Sampling traces that do not contain the attribute `http.endpoint` equal to `/status` and `/metrics`.
-
-```yaml
-traces:
-  configs:
-    - name: default
-      ...
-      tail_sampling:
-        policies:
-        - attributes:
-          - name: http.endpoint
-            values:
-              - /status
-              - /metrics
-            invert_match: true
-```
-
-### Sampling with `and` policy
-
-Sampling traces that have an attribute `http.endpoint` that matches `/api/v1/*`
-and that have a total duration longer than 100ms.
-
-```yaml
-traces:
-  configs:
-    - name: default
-      ...
-      tail_sampling:
-      policies:
-      - type: and
-        and:
-          and_sub_policy:
-            - type: string_attribute
-              string_attribute:
-              - name: http.endpoint
-                value: /api/v1/*
-                enabled_regex_matching: true
-                cache_max_size: 10
-            - type: latency
-              latency:
-                threshold_ms: 100
-```
-
-### Multi-requirement sampling
-
-Sampling requirements are the following:
-
- Service A: latency> 3s
- Service B: only error spans or latency> 5s
- Service C: all spans
-
-```yaml
-traces:
-  configs:
-    - name: default
-      ...
-      tail_sampling:
-        policies:
-        # Service A: latency> 3s
-        - type: and
-          and:
-            and_sub_policy:
-              - type: latency
-                name: latency
-                latency:
-                  threshold_ms: 3000
-              - type: string_attribute
-                name: service-name
-                string_attribute:
-                  name: service.name
-                  values:
-                    - serviceA
-        # Service B requires two and policies
-        # 1. spans with status code ERROR
-        - type: and
-          and:
-            and_sub_policy:
-              - type: status_code
-                name: status_code
-                status_code:
-                  status_codes:
-                    - "ERROR"
-              - type: string_attribute
-                name: service-name
-                string_attribute:
-                  name: service.name
-                  values:
-                    - serviceB
-        # 2. latency> 5s
-        - type: and
-          and:
-            and_sub_policy:
-              - type: latency
-                name: latency
-                latency:
-                  threshold_ms: 5000
-              - type: string_attribute
-                name: service name
-                string_attribute:
-                  name: service.name
-                  values:
-                    - serviceB
-        # Service C: all spans
-        - type: string_attribute
-          string_attribute:
-            name: service.name
-            values:
-              - serviceC
-```
-
-## Sampling based on k8s metadata
-
-In this example, the Agent will sample traces that come from pods from the namespace `default`.
-Via `scrape_configs`, spans are relabeled with kubernetes metadata,
-in this case injecting the namespace attribute.
+The following policy only samples traces where at least one span contains an OpenTelemetry Error status code.

 ```yaml
 traces:
  configs:
    - name: default
    ...
-    scrape_configs:
-      - job_name: kubernetes-pods
-        kubernetes_sd_configs:
-          - role: pod
-        relabel_configs:
-          - action: replace
-            source_labels:
-              - __meta_kubernetes_namespace
-            target_label: namespace
    tail_sampling:
      policies:
-      - type: string_attribute
-        string_attribute:
-          name: namespace
-          values:
-          - default
+        - type: status_code
+          status_code:
+            status_codes:
+              - ERROR
+```
+
+### Span attribute tail sampling policy
+
+The following policy only samples traces where the span attribute `http.target` does *not* contain the value `/healthcheck` or is prefixed with `/metrics/`.
+
+```yaml
+traces:
+   configs:
+   - name: default
+    tail_sampling:
+      policies:
+        - type: string_attribute
+          string_attribute:
+            key: http.target
+            values:
+              - ^\/(?:metrics\/.*|healthcheck)$
+            enabled_regex_matching: true
+            invert_match: true
+```
+
+### And compound tail sampling policy
+
+The following policy will only sample traces where all of the conditions for the sub-policies are met.
+In this case, it takes the prior two policies and will only sample traces where the span attribute `http.target` does *not* contain the value `/healthcheck` or is prefixed with `/metrics/` *and* at least one of the spans of the trace contains an OpenTelemetry Error status code.
+
+```yaml
+traces:
+   configs:
+   - name: default
+    tail_sampling:
+      policies:
+       - type: and
+            and_sub_policy:
+            - name: and_tag_policy
+              type: string_attribute
+              string_attribute:
+                key: http.target
+                values:
+                    - ^\/(?:metrics\/.*|healthcheck)$
+                enabled_regex_matching: true
+                invert_match: true
+            - name: and_error_policy
+              type: status_code
+              status_code:
+                status_codes:
+                  - ERROR
 ```
@@ -0,0 +1,16 @@
+---
+title: Hosted storage
+menuTitle: Hosted storage
+description: Learn about available hosted storage options and how to configure them.
+weight: 400
+aliases:
+- /docs/tempo/latest/configuration/
+---
+
+# Hosted storage
+
+Tempo provides additional hosted storage configuration options discussed on the pages below. These options relate to providers such as Google Cloud, AWS S3, and Azure.
+
+For additional details about storage configuration, refer to [Storage configuration]({{< relref "../../configuration#storage" >}}).
+
+{{< section withDescriptions="true">}}
@@ -0,0 +1,219 @@
+---
+title: Azure blob storage permissions and management
+menuTitle: Azure blob storage
+description: Azure blog storage permissions and configuration options for Tempo.
+aliases:
+  - ../../configuration/azure/ # /docs/tempo/<TEMPO_VERSION>/configuration/azure/
+---
+
+# Azure blob storage permissions and management
+
+Tempo requires the following configuration to authenticate to and access Azure blob storage:
+
+- Storage Account name specified in the configuration file as `storage_account_name` or in the environment variable `AZURE_STORAGE_ACCOUNT`
+- Credentials for accessing the Storage Account that are one of the following:
+  - Storage Account access key specified in the configuration file as `storage_account_key` or in the environment variable `AZURE_STORAGE_KEY`
+  - An Azure Managed Identity that is either system or user assigned. To use Azure Managed Identities, you'll need to set `use_managed_identity` to `true` in the configuration file or set `user_assigned_id` to the client ID for the managed identity you'd like to use.
+      - For a system-assigned managed identity, no additional configuration is required.
+      - For a user-assigned managed identity, you'll need to set `user_assigned_id` to the client ID for the managed identity in the configuration file.
+  - Via Azure Workload Identity. To use Azure Workload Identity, you'll need to enable Azure Workload Identity on your cluster, add the required label and annotation to the service account and the required pod label. Additionally, you will need to set `use_federated_token` to `true` to utilize Azure Workload Identity.
+
+## Sample configuration (for Tempo Monolithic Mode)
+
+### Access key
+
+This sample configuration shows how to set up Azure blob storage using Helm charts and an access key from Kubernetes secrets.
+
+```yaml
+tempo:
+  storage:
+    trace:
+      backend: azure
+      azure:
+        container_name: container-name
+        storage_account_name: storage-account-name
+        storage_account_key: ${STORAGE_ACCOUNT_ACCESS_KEY}
+
+  extraArgs:
+    config.expand-env: true
+  extraEnv:
+    - name: STORAGE_ACCOUNT_ACCESS_KEY
+      valueFrom:
+        secretKeyRef:
+          name: secret-name
+          key: STORAGE_ACCOUNT_ACCESS_KEY
+```
+
+### Azure Workload Identity
+Here is an example config for using Azure Workload Identity.
+```yaml
+tempo:
+  storage:
+    trace:
+      backend: azure
+      azure:
+        container_name: container-name
+        storage_account_name: storage-account-name
+        use_federated_token: true
+```
+
+## Sample configuration (for Tempo Distributed Mode)
+
+In Distributed mode the `trace` configuration needs to be applied against the `storage` object, which resides at the root of the Values object. Additionally, the `extraArgs` and `extraEnv` configuration need to be applied to each of the following services:
+ - distributor
+ - compactor
+ - ingester
+ - querier
+ - queryFrontend
+
+```
+storage:
+  trace:
+    backend: azure
+    azure:
+      container_name: tempo-traces
+      storage_account_name: stgappgeneraluks
+      storage_account_key: ${STORAGE_ACCOUNT_ACCESS_KEY}
+
+
+distributor:
+  extraArgs:
+  - "-config.expand-env=true"
+  extraEnv:
+  - name: STORAGE_ACCOUNT_ACCESS_KEY
+    valueFrom:
+      secretKeyRef:
+        name: tempo-traces-stg-key
+        key: tempo-traces-key
+
+compactor:
+  extraArgs:
+  - "-config.expand-env=true"
+  extraEnv:
+  - name: STORAGE_ACCOUNT_ACCESS_KEY
+    valueFrom:
+      secretKeyRef:
+        name: tempo-traces-stg-key
+        key: tempo-traces-key
+
+ingester:
+  extraArgs:
+  - "-config.expand-env=true"
+  extraEnv:
+  - name: STORAGE_ACCOUNT_ACCESS_KEY
+    valueFrom:
+      secretKeyRef:
+        name: tempo-traces-stg-key
+        key: tempo-traces-key
+
+querier:
+  extraArgs:
+  - "-config.expand-env=true"
+  extraEnv:
+  - name: STORAGE_ACCOUNT_ACCESS_KEY
+    valueFrom:
+      secretKeyRef:
+        name: tempo-traces-stg-key
+        key: tempo-traces-key
+
+queryFrontend:
+  extraArgs:
+  - "-config.expand-env=true"
+  extraEnv:
+  - name: STORAGE_ACCOUNT_ACCESS_KEY
+    valueFrom:
+      secretKeyRef:
+        name: tempo-traces-stg-key
+        key: tempo-traces-key
+```
+
+## Azure blocklist polling
+
+If you are hosting Tempo on Azure, two values may need to be updated to ensure consistent successful blocklist polling. If you are
+experiencing [this issue](https://stackoverflow.com/questions/12917857/the-specified-block-list-is-invalid-while-uploading-blobs-in-parallel/55902744#55902744), try setting `blocklist_poll_tenant_index_builders` to 1.
+
+Additionally, if you are seeing DNS failures like the ones below, try increasing `blocklist_poll_jitter_ms`. Discussion [here](https://github.com/grafana/tempo/issues/1462).
+```
+reading storage container: Head "https://tempoe**************.blob.core.windows.net/tempo/single-tenant/d8aafc48-5796-4221-ac0b-58e001d18515/meta.compacted.json?timeout=61": dial tcp: lookup tempoe**************.blob.core.windows.net on 10.0.0.10:53: dial udp 10.0.0.10:53: operation was canceled
+```
+
+Your final configuration may look something like:
+```
+  storage:
+    trace:
+      blocklist_poll_tenant_index_builders: 1
+      blocklist_poll_jitter_ms: 500
+```
+
+## (Optional) Storage Account management policy for cleaning up the storage container
+
+The following Storage Account management policy shows an example of cleaning up
+files from the container after they have been deleted for a period of time.
+
+```json
+{
+  "id": "/subscriptions/00000000-0000-0000000000000000000000/resourceGroups/resourceGroupName/providers/Microsoft.Storage/storageAccounts/accountName/managementPolicies/default",
+  "lastModifiedTime": "2021-11-30T19:19:54.855455+00:00",
+  "name": "DefaultManagementPolicy",
+  "policy": {
+    "rules": [
+      {
+        "definition": {
+          "actions": {
+            "baseBlob": {
+              "delete": {
+                "daysAfterLastAccessTimeGreaterThan": null,
+                "daysAfterModificationGreaterThan": 60.0
+              },
+              "enableAutoTierToHotFromCool": null,
+              "tierToArchive": null,
+              "tierToCool": null
+            },
+            "snapshot": null,
+            "version": null
+          },
+          "filters": {
+            "blobIndexMatch": null,
+            "blobTypes": [
+              "blockBlob"
+            ],
+            "prefixMatch": [
+              "tempo-data"
+            ]
+          }
+        },
+        "enabled": true,
+        "name": "TempoBlobRetention",
+        "type": "Lifecycle"
+      },
+      {
+        "definition": {
+          "actions": {
+            "baseBlob": null,
+            "snapshot": null,
+            "version": {
+              "delete": {
+                "daysAfterCreationGreaterThan": 7.0
+              },
+              "tierToArchive": null,
+              "tierToCool": null
+            }
+          },
+          "filters": {
+            "blobIndexMatch": null,
+            "blobTypes": [
+              "blockBlob"
+            ],
+            "prefixMatch": []
+          }
+        },
+        "enabled": true,
+        "name": "VersionRetention",
+        "type": "Lifecycle"
+      }
+    ]
+  },
+  "resourceGroup": "resource-group-name",
+  "type": "Microsoft.Storage/storageAccounts/managementPolicies"
+}
+```
@@ -1,12 +1,13 @@
 ---
-title: Google Cloud Storage permissions
+title: Google Cloud Storage
 description: Learn about Google Cloud Storage permissions for Tempo.
-weight: 10
+aliases:
+  - ../../configuration/gcs/ # /docs/tempo/<TEMPO_VERSION>/configuration/gcs/
 ---

-# Google Cloud Storage permissions
+# Google Cloud Storage

-For configuration options, check the storage section on the [configuration]({{< relref "../configuration#storage" >}}) page.
+For configuration options, check the storage section on the [configuration]({{< relref "../../configuration#storage" >}}) page.

 ## Permissions

@@ -1,11 +1,14 @@
 ---
 title: Amazon S3 permissions
+menuTitle: Amazon S3
 description: Set Amazon S3 permissions for Tempo.
-weight: 20
+aliases:
+  - ../../configuration/s3/ # /docs/tempo/<TEMPO_VERSION>/configuration/s3/
 ---
+
 # Amazon S3 permissions

-For configuration options, refer to the storage section on the [configuration]({{< relref "../configuration#storage" >}}) page.
+For configuration options, refer to the storage section on the [configuration]({{< relref "../../configuration#storage" >}}) page.

 The following authentication methods are supported:
 - AWS environment variables `AWS_ACCESS_KEY_ID` and `AWS_SECRET_ACCESS_KEY`
@@ -1,7 +1,7 @@
 ---
 title: Manifest
 description: This manifest lists of all Tempo options and their defaults.
-weight: 70
+weight: 110
 ---

 # Manifest
@@ -12,18 +12,19 @@ and other [configuration documentation]({{< relref "../configuration" >}}). Most

 It was generated by running Tempo with a minimal configuration and accessing the `/status/config` endpoint:
 ```
-go run ./cmd/tempo --storage.trace.backend=local --storage.trace.local.path=/tmp/tempo/traces --storage.trace.wal.path=/tmp/tempo/wal
+go run ./cmd/tempo --storage.trace.backend=local --storage.trace.local.path=/var/tempo/traces --storage.trace.wal.path=/var/tempo/wal
 ```

 ## Complete configuration

-{{% admonition type="note" %}}
-This manifest was generated on 2023-06-01.
+{{< admonition type="note" >}}
+This manifest was generated on 2023-11-13.
 {{% /admonition %}}

 ```yaml
 target: all
 http_api_prefix: ""
+autocomplete_filtering_enabled: true
 server:
    http_listen_network: tcp
    http_listen_address: ""
@@ -36,20 +37,29 @@ server:
    tls_cipher_suites: ""
    tls_min_version: ""
    http_tls_config:
+        cert: ""
+        key: null
+        client_ca: ""
        cert_file: ""
        key_file: ""
        client_auth_type: ""
        client_ca_file: ""
    grpc_tls_config:
+        cert: ""
+        key: null
+        client_ca: ""
        cert_file: ""
        key_file: ""
        client_auth_type: ""
        client_ca_file: ""
    register_instrumentation: true
+    report_grpc_codes_in_instrumentation_label_enabled: false
    graceful_shutdown_timeout: 30s
    http_server_read_timeout: 30s
+    http_server_read_header_timeout: 0s
    http_server_write_timeout: 30s
    http_server_idle_timeout: 2m0s
+    http_log_closed_connections_without_response_enabled: false
    grpc_server_max_recv_msg_size: 16777216
    grpc_server_max_send_msg_size: 16777216
    grpc_server_max_concurrent_streams: 100
@@ -60,6 +70,7 @@ server:
    grpc_server_keepalive_timeout: 20s
    grpc_server_min_time_between_pings: 10s
    grpc_server_ping_without_stream_allowed: true
+    grpc_server_num_workers: 0
    log_format: logfmt
    log_level: info
    log_source_ips_enabled: false
@@ -71,7 +82,7 @@ server:
    http_path_prefix: ""
 internal_server:
    http_listen_network: tcp
-    http_listen_address: localhost
+    http_listen_address: ""
    http_listen_port: 3101
    http_listen_conn_limit: 0
    grpc_listen_network: ""
@@ -81,20 +92,29 @@ internal_server:
    tls_cipher_suites: ""
    tls_min_version: ""
    http_tls_config:
+        cert: ""
+        key: null
+        client_ca: ""
        cert_file: ""
        key_file: ""
        client_auth_type: ""
        client_ca_file: ""
    grpc_tls_config:
+        cert: ""
+        key: null
+        client_ca: ""
        cert_file: ""
        key_file: ""
        client_auth_type: ""
        client_ca_file: ""
    register_instrumentation: false
+    report_grpc_codes_in_instrumentation_label_enabled: false
    graceful_shutdown_timeout: 30s
    http_server_read_timeout: 30s
+    http_server_read_header_timeout: 0s
    http_server_write_timeout: 30s
    http_server_idle_timeout: 2m0s
+    http_log_closed_connections_without_response_enabled: false
    grpc_server_max_recv_msg_size: 0
    grpc_server_max_send_msg_size: 0
    grpc_server_max_concurrent_streams: 0
@@ -105,8 +125,9 @@ internal_server:
    grpc_server_keepalive_timeout: 0s
    grpc_server_min_time_between_pings: 0s
    grpc_server_ping_without_stream_allowed: false
-    log_format: ""
-    log_level: ""
+    grpc_server_num_workers: 0
+    log_format: logfmt
+    log_level: info
    log_source_ips_enabled: false
    log_source_ips_header: ""
    log_source_ips_regex: ""
@@ -157,9 +178,9 @@ distributor:
        instance_addr: ""
    receivers: {}
    override_ring_key: distributor
-    log_received_traces: false
    forwarders: []
    extend_writes: true
+    retry_after_on_resource_exhausted: 0s
 ingester_client:
    pool_config:
        checkinterval: 15s
@@ -178,6 +199,8 @@ ingester_client:
            min_period: 100ms
            max_period: 10s
            max_retries: 10
+        initial_stream_window_size: 63KiB1023B
+        initial_connection_window_size: 63KiB1023B
        tls_enabled: false
        tls_cert_path: ""
        tls_key_path: ""
@@ -186,6 +209,9 @@ ingester_client:
        tls_insecure_skip_verify: false
        tls_cipher_suites: ""
        tls_min_version: ""
+        connect_timeout: 5s
+        connect_backoff_base_delay: 1s
+        connect_backoff_max_delay: 5s
 metrics_generator_client:
    pool_config:
        checkinterval: 15s
@@ -204,6 +230,8 @@ metrics_generator_client:
            min_period: 100ms
            max_period: 10s
            max_retries: 10
+        initial_stream_window_size: 63KiB1023B
+        initial_connection_window_size: 63KiB1023B
        tls_enabled: false
        tls_cert_path: ""
        tls_key_path: ""
@@ -212,13 +240,18 @@ metrics_generator_client:
        tls_insecure_skip_verify: false
        tls_cipher_suites: ""
        tls_min_version: ""
+        connect_timeout: 5s
+        connect_backoff_base_delay: 1s
+        connect_backoff_max_delay: 5s
 querier:
    search:
        query_timeout: 30s
        prefer_self: 10
-        external_endpoints: []
        external_hedge_requests_at: 8s
        external_hedge_requests_up_to: 2
+        external_backend: ""
+        google_cloud_run: null
+        external_endpoints: []
    trace_by_id:
        query_timeout: 10s
    max_concurrent_queries: 20
@@ -239,6 +272,8 @@ querier:
                min_period: 100ms
                max_period: 1s
                max_retries: 5
+            initial_stream_window_size: 0B
+            initial_connection_window_size: 0B
            tls_enabled: false
            tls_cert_path: ""
            tls_key_path: ""
@@ -247,10 +282,14 @@ querier:
            tls_insecure_skip_verify: false
            tls_cipher_suites: ""
            tls_min_version: ""
+            connect_timeout: 0s
+            connect_backoff_base_delay: 0s
+            connect_backoff_max_delay: 0s
    query_relevant_ingesters: false
 query_frontend:
    max_outstanding_per_tenant: 2000
    querier_forget_delay: 0s
+    max_batch_size: 5
    max_retries: 2
    search:
        concurrent_jobs: 1000
@@ -262,8 +301,13 @@ query_frontend:
        query_ingesters_until: 30m0s
    trace_by_id:
        query_shards: 50
-        hedge_requests_at: 2s
-        hedge_requests_up_to: 2
+    metrics:
+        concurrent_jobs: 1000
+        target_bytes_per_job: 104857600
+        max_duration: 0s
+        query_backend_after: 1h0m0s
+        interval: 5m0s
+    multi_tenant_queries_enabled: true
 compactor:
    ring:
        kvstore:
@@ -366,6 +410,7 @@ ingester:
        min_ready_duration: 15s
        interface_names:
            - en0
+            - bridge100
        enable_inet6: false
        final_sleep: 0s
        tokens_file_path: ""
@@ -383,6 +428,7 @@ ingester:
    max_block_bytes: 524288000
    complete_block_timeout: 15m0s
    override_ring_key: ring
+    flush_all_on_shutdown: false
 metrics_generator:
    ring:
        kvstore:
@@ -439,6 +485,8 @@ metrics_generator:
                - 6.4
                - 12.8
            dimensions: []
+            enable_client_server_prefix: false
+            enable_messaging_system_latency_histogram: false
            peer_attributes:
                - peer.service
                - db.name
@@ -474,17 +522,19 @@ metrics_generator:
                1: true
                2: true
            filter_policies: []
+            target_info_excluded_dimensions: []
        local_blocks:
            block:
                bloom_filter_false_positive: 0.01
                bloom_filter_shard_size_bytes: 102400
-                version: vParquet2
+                version: vParquet3
                search_encoding: snappy
                search_page_size_bytes: 1048576
                v2_index_downsample_bytes: 1048576
                v2_index_page_size_bytes: 256000
                v2_encoding: zstd
                parquet_row_group_size_bytes: 100000000
+                parquet_dedicated_columns: []
            search:
                chunk_size_bytes: 1000000
                prefetch_trace_count: 1000
@@ -500,6 +550,8 @@ metrics_generator:
            max_block_bytes: 500000000
            complete_block_timeout: 1h0m0s
            max_live_traces: 0
+            concurrent_blocks: 10
+            filter_server_spans: true
    registry:
        collection_interval: 15s
        stale_duration: 15m0s
@@ -509,13 +561,14 @@ metrics_generator:
        path: ""
        wal:
            wal_segment_size: 134217728
-            wal_compression: false
+            wal_compression: none
            stripe_size: 16384
            truncate_frequency: 2h0m0s
            min_wal_time: 300000
            max_wal_time: 14400000
            no_lockfile: false
        remote_write_flush_deadline: 1m0s
+        remote_write_add_org_id_header: true
    traces_storage:
        path: ""
        completedfilepath: ""
@@ -523,32 +576,34 @@ metrics_generator:
        v2_encoding: none
        search_encoding: none
        ingestion_time_range_slack: 0s
-        version: vParquet2
+        version: vParquet3
    metrics_ingestion_time_range_slack: 30s
    query_timeout: 30s
+    override_ring_key: metrics-generator
 storage:
    trace:
        pool:
            max_workers: 400
            queue_depth: 20000
        wal:
-            path: /tmp/tempo/wal
-            completedfilepath: /tmp/tempo/wal/completed
-            blocksfilepath: /tmp/tempo/wal/blocks
+            path: /var/tempo/wal
+            completedfilepath: /var/tempo/wal/completed
+            blocksfilepath: /var/tempo/wal/blocks
            v2_encoding: snappy
            search_encoding: none
            ingestion_time_range_slack: 2m0s
-            version: vParquet2
+            version: vParquet3
        block:
            bloom_filter_false_positive: 0.01
            bloom_filter_shard_size_bytes: 102400
-            version: vParquet2
+            version: vParquet3
            search_encoding: snappy
            search_page_size_bytes: 1048576
            v2_index_downsample_bytes: 1048576
            v2_index_page_size_bytes: 256000
            v2_encoding: zstd
            parquet_row_group_size_bytes: 100000000
+            parquet_dedicated_columns: []
        search:
            chunk_size_bytes: 1000000
            prefetch_trace_count: 1000
@@ -567,7 +622,7 @@ storage:
        blocklist_poll_tolerate_consecutive_errors: 1
        backend: local
        local:
-            path: /tmp/tempo/traces
+            path: /var/tempo/traces
        gcs:
            bucket_name: ""
            prefix: ""
@@ -578,6 +633,7 @@ storage:
            insecure: false
            object_cache_control: ""
            object_metadata: {}
+            list_blocks_concurrency: 3
        s3:
            tls_cert_path: ""
            tls_key_path: ""
@@ -603,6 +659,8 @@ storage:
            tags: {}
            storage_class: ""
            metadata: {}
+            native_aws_auth_enabled: false
+            list_blocks_concurrency: 3
        azure:
            storage_account_name: ""
            storage_account_key: ""
@@ -616,54 +674,94 @@ storage:
            buffer_size: 3145728
            hedge_requests_at: 0s
            hedge_requests_up_to: 2
+            use_v2_sdk: false
        cache: ""
-        cache_min_compaction_level: 0
-        cache_max_block_age: 0s
        background_cache:
            writeback_goroutines: 10
            writeback_buffer: 10000
        memcached: null
        redis: null
+        cache_min_compaction_level: 0
+        cache_max_block_age: 0s
 overrides:
-    ingestion_rate_strategy: local
-    ingestion_rate_limit_bytes: 15000000
-    ingestion_burst_size_bytes: 20000000
-    max_traces_per_user: 10000
-    max_global_traces_per_user: 0
-    forwarders: []
-    metrics_generator_ring_size: 0
-    metrics_generator_processors: null
-    metrics_generator_max_active_series: 0
-    metrics_generator_collection_interval: 0s
-    metrics_generator_disable_collection: false
-    metrics_generator_forwarder_queue_size: 0
-    metrics_generator_forwarder_workers: 0
-    metrics_generator_processor_service_graphs_histogram_buckets: []
-    metrics_generator_processor_service_graphs_dimensions: []
-    metrics_generator_processor_service_graphs_peer_attributes: []
-    metrics_generator_processor_span_metrics_histogram_buckets: []
-    metrics_generator_processor_span_metrics_dimensions: []
-    metrics_generator_processor_span_metrics_intrinsic_dimensions: {}
-    metrics_generator_processor_span_metrics_filter_policies: []
-    metrics_generator_processor_span_metrics_dimension_mappings: []
-    metrics_generator_processor_span_metrics_enable_target_info: false
-    metrics_generator_processor_local_blocks_max_live_traces: 0
-    metrics_generator_processor_local_blocks_max_block_duration: 0s
-    metrics_generator_processor_local_blocks_max_block_bytes: 0
-    metrics_generator_processor_local_blocks_flush_check_period: 0s
-    metrics_generator_processor_local_blocks_trace_idle_period: 0s
-    metrics_generator_processor_local_blocks_complete_block_timeout: 0s
-    block_retention: 0s
-    max_bytes_per_tag_values_query: 5000000
-    max_blocks_per_tag_values_query: 0
-    max_search_duration: 0s
-    max_bytes_per_trace: 5000000
+    defaults:
+        ingestion:
+            rate_strategy: local
+            rate_limit_bytes: 15000000
+            burst_size_bytes: 20000000
+            max_traces_per_user: 10000
+        read:
+            max_bytes_per_tag_values_query: 5000000
+        global:
+            max_bytes_per_trace: 5000000
    per_tenant_override_config: ""
    per_tenant_override_period: 10s
+    user_configurable_overrides:
+        enabled: false
+        poll_interval: 1m0s
+        client:
+            backend: ""
+            confirm_versioning: true
+            local:
+                path: ""
+            gcs:
+                bucket_name: ""
+                prefix: ""
+                chunk_buffer_size: 10485760
+                endpoint: ""
+                hedge_requests_at: 0s
+                hedge_requests_up_to: 2
+                insecure: false
+                object_cache_control: ""
+                object_metadata: {}
+                list_blocks_concurrency: 3
+            s3:
+                tls_cert_path: ""
+                tls_key_path: ""
+                tls_ca_path: ""
+                tls_server_name: ""
+                tls_insecure_skip_verify: false
+                tls_cipher_suites: ""
+                tls_min_version: VersionTLS12
+                bucket: ""
+                prefix: ""
+                endpoint: ""
+                region: ""
+                access_key: ""
+                secret_key: ""
+                session_token: ""
+                insecure: false
+                part_size: 0
+                hedge_requests_at: 0s
+                hedge_requests_up_to: 2
+                signature_v2: false
+                forcepathstyle: false
+                bucket_lookup_type: 0
+                tags: {}
+                storage_class: ""
+                metadata: {}
+                native_aws_auth_enabled: false
+                list_blocks_concurrency: 3
+            azure:
+                storage_account_name: ""
+                storage_account_key: ""
+                use_managed_identity: false
+                use_federated_token: false
+                user_assigned_id: ""
+                container_name: ""
+                prefix: ""
+                endpoint_suffix: blob.core.windows.net
+                max_buffers: 4
+                buffer_size: 3145728
+                hedge_requests_at: 0s
+                hedge_requests_up_to: 2
+                use_v2_sdk: false
+        api:
+            check_for_conflicting_runtime_overrides: false
 memberlist:
    node_name: ""
    randomize_node_name: true
-    stream_timeout: 10s
+    stream_timeout: 2s
    retransmit_factor: 2
    pull_push_interval: 30s
    gossip_interval: 1s
@@ -702,4 +800,9 @@ usage_report:
        min_period: 100ms
        max_period: 10s
        max_retries: 0
+cache:
+    background:
+        writeback_goroutines: 10
+        writeback_buffer: 10000
+    caches: []
 ```
@@ -0,0 +1,12 @@
+---
+title: Network options
+menuTitle: Network options
+description: Learn about available network options and how to configure them.
+weight: 500
+---
+
+# Network options
+
+Tempo provides additional network configuration options discussed on the pages below.
+
+{{< section withDescriptions="true" >}}
@@ -1,14 +1,18 @@
 ---
 title: Configure IPv6
 description: Learn how to configure IPv6 for Tempo.
-weight: 37
+menuTitle: Configure IPv6
+aliases:
+  - ../../configuration/ipv6/ # /docs/tempo/<TEMPO_VERSION>/configuration/ipv6/
 ---

 # Configure IPv6

 Tempo can be configured to communicate between the components using Internet Protocol Version 6, or IPv6.

-> Note: In order to support this support this configuration, the underlying infrastructure must support this address family. This configuration may be used in a single-stack IPv6 environment, or in a dual-stack environment where both IPv6 and IPv4 are present. In a dual-stack scenario, only one address family may be configured at a time, and all components must be configured for that address family.
+{{< admonition type="note" >}}
+The underlying infrastructure must support this address family. This configuration may be used in a single-stack IPv6 environment, or in a dual-stack environment where both IPv6 and IPv4 are present. In a dual-stack scenario, only one address family may be configured at a time, and all components must be configured for that address family.
+{{% /admonition %}}

 ## Protocol configuration

@@ -36,9 +40,9 @@ ingester:
    enable_inet6: true

 server:
-  grpc_listen_address: '[::0]'
+  grpc_listen_address: '::0'
  grpc_listen_port: 9095
-  http_listen_address: '[::0]'
+  http_listen_address: '::0'
  http_listen_port: 3200
 ```

@@ -1,13 +1,15 @@
 ---
-title: Run Tempo Distributed with sidecar proxies
+title: Run Tempo distributed with sidecar proxies
 menuTitle: Configure sidecar proxies
-description: Configure Tempo Distributed to run with sidecar proxies
-weight: 36
+description: Configure Tempo distributed to run with sidecar proxies
+aliases:
+  - ../../configuration/sidecar-proxy/ # /docs/tempo/<TEMPO_VERSION>/configuration/sidecar-proxy/
 ---

-# Run Tempo Distributed with sidecar proxies
+# Run Tempo distributed with sidecar proxies

-You can route inter-pod gRPC traffic run through a sidecar proxy to meet requirements such as custom security, routing, or logging.  Common examples include Envoy, Nginx, Traefik, or service meshes like Istio and Linkerd. 
+You can route inter-pod gRPC traffic run through a sidecar proxy to meet requirements such as custom security, routing, or logging.
+Common examples include Envoy, Nginx, Traefik, or service meshes like Istio and Linkerd.

 ## How Tempo pods communicate

@@ -20,7 +22,7 @@ The overall network looks like this:

 ![Tempo distributed network overview](/static/img/docs/tempo/sidecar-proxy/tempo-network-sidecar-proxy-simple.svg)

-The low-level ring data for ingesters can be viewed by browsing to the `/ingester/ring` URL on a distributor.  It looks like this:
+The low-level ring data for ingesters can be viewed by browsing to the `/ingester/ring` URL on a distributor. It looks like this:

 ![Ring status with default port](/static/img/docs/tempo/sidecar-proxy/screenshot-tempo-sidecar.png)

@@ -2,14 +2,15 @@
 title: Configure TLS communication
 menuTitle: Configure TLS
 description: Configure Tempo components to communicate using TLS.
-weight: 35
+aliases:
+  - ../../configuration/tls/ # /docs/tempo/<TEMPO_VERSION>/configuration/tls/
 ---

 # Configure TLS communication

 Tempo can be configured to communicate between the components using Transport Layer Security, or TLS.

-{{% admonition type="note" %}}
+{{< admonition type="note" >}}
 The ciphers and TLS version here are for example purposes only. We are not recommending which ciphers or TLS versions for use in production environments.
 {{% /admonition %}}

@@ -1,8 +1,8 @@
 ---
 title: Apache Parquet block format
 menuTitle: Apache Parquet
-description: Learn about Tempo's Parquet block format.
-weight: 75
+description: Learn about Parquet block format in Tempo.
+weight: 300
 ---

 # Apache Parquet block format
@@ -10,33 +10,40 @@ weight: 75

 Tempo has a default columnar block format based on Apache Parquet. This format is required for tags-based search as well as [TraceQL]({{< relref "../traceql" >}}), the query language for traces. The columnar block format improves search performance and enables a large ecosystem of tools to access the underlying trace data.

-For more information, refer to the [Parquet schema]({{< relref "../operations/schema" >}}) and the [Parquet design document](https://github.com/mdisibio/tempo/blob/design-proposal-parquet/docs/design-proposals/2022-04%20Parquet.md).
+For more information, refer to the [Parquet design document](https://github.com/grafana/tempo/blob/main/docs/design-proposals/2022-04%20Parquet.md) and [Issue 1480](https://github.com/grafana/tempo/issues/1480).
+Additionally, there is now a [Parquet v3 design document](https://github.com/grafana/tempo/blob/main/docs/design-proposals/2023-05%20vParquet3.md).

 If you install using the new Helm charts, then Parquet is enabled by default.

 ## Considerations

-The Parquet block format is enabled by default in Tempo 2.0. No data conversion or upgrade process is necessary. As soon as the format is enabled, Tempo starts writing data in that format, leaving existing data as-is.
+The Parquet block format is enabled by default since Tempo 2.0. No data conversion or upgrade process is necessary. As soon as the format is enabled, Tempo starts writing data in that format, leaving existing data as-is.

 Block formats based on Parquet require more CPU and memory resources than the previous `v2` format but provide search and TraceQL functionality.

 ## Choose a different block format

-The default block format is `vParquet2` which is the latest iteration of Tempo's Parquet based columnar block format. It is still possible to use the previous format `vParquet`. To enable it, set the block version option to `vParquet` in the Storage section of the configuration file.
+The default block format is `vParquet3`, which is the latest iteration of the Parquet-based columnar block format in Tempo.
+It introduces dedicated attribute columns, which improve query performance by storing attributes in own columns,
+rather than in the generic attribute key-value list.
+For more information, see [Dedicated attribute columns]({{< relref "../operations/tempo_cli" >}}).
+
+You can also use the latest more experimental version `vParquet4`.
+To enable it, set the block version option to `vParquet4` in the Storage section of the configuration file.

 ```yaml
-# block format version. options: v2, vParquet, vParquet2
-[version: vParquet]
+# block format version. options: v2, vParquet2, vParquet3, vParquet4
+[version: vParquet4]
 ```

-It is possible to disable Parquet and use the previous `v2` block format. This disables all forms of search, but also reduces resource consumption, and may be desired for a high-throughput cluster that does not need these capabilities. Set the block version option to `v2` in the Storage section of the configuration file.
+In some cases, you may choose to disable Parquet and use the old `v2` block format. Using the `v2` block format disables all forms of search, but also reduces resource consumption, and may be desired for a high-throughput cluster that does not need these capabilities. To make this change, set the block version option to `v2` in the Storage section of the configuration file.

 ```yaml
-# block format version. options: v2, vParquet, vParquet2
+# block format version. options: v2, vParquet2, vParquet3, vParquet4
 [version: v2]
 ```

-To re-enable the default `vParquet2` format, remove the block version option from the Storage section of the configuration file or set the option to `vParquet2`.
+To re-enable the default `vParquet3` format, remove the block version option from the Storage section of the configuration file or set the option to `vParquet3`.

 ## Parquet configuration parameters

--- a/Show More
+++ b/Show More