ingester: use a 64-bit hash to avoid collisions (#5276) (#5302)

* ingester: use a 64-bit hash to avoid collisions

We had this issue in the distributor, it it can happen in the
ingester too with live traces.

* Update changelog

* Add test for live traces collision

.drone/docker-manifest.tmpl

@@ -1,13 +0,0 @@
-image: grafana/{{config.target}}
-tags:
-  - latest
-  - {{build.tags}}
-manifests:
-  - image: grafana/{{config.target}}:{{build.tags}}-amd64
-    platform:
-      architecture: amd64
-      os: linux
-  - image: grafana/{{config.target}}:{{build.tags}}-arm64
-    platform:
-      architecture: arm64
-      os: linux

.drone/drone.jsonnet

@@ -1,355 +0,0 @@
-local apps = ['tempo', 'tempo-vulture', 'tempo-query', 'tempo-cli'];
-local archs = ['amd64', 'arm64'];
-
-//# Building blocks ##
-
-local pipeline(name, arch='amd64') = {
-  kind: 'pipeline',
-  name: name,
-  platform: {
-    os: 'linux',
-    arch: arch,
-  },
-  steps: [],
-  depends_on: [],
-  trigger: {
-    ref: [
-      'refs/heads/main',
-      'refs/tags/v*',
-      // weekly release branches
-      'refs/heads/r?',
-      'refs/heads/r??',
-      'refs/heads/r???',
-    ],
-  },
-};
-
-local secret(name, vault_path, vault_key) = {
-  kind: 'secret',
-  name: name,
-  get: {
-    path: vault_path,
-    name: vault_key,
-  },
-};
-
-local docker_username_secret = secret('docker_username', 'infra/data/ci/docker_hub', 'username');
-local docker_password_secret = secret('docker_password', 'infra/data/ci/docker_hub', 'password');
-
-// secret needed to access us.gcr.io in deploy_to_dev()
-local docker_config_json_secret = secret('dockerconfigjson', 'secret/data/common/gcr', '.dockerconfigjson');
-
-// secret needed for dep-tools
-local gh_token_secret = secret('gh_token', 'infra/data/ci/github/grafanabot', 'pat');
-local tempo_app_id_secret = secret('tempo_app_id_secret', 'infra/data/ci/tempo/github-app', 'app-id');
-local tempo_app_installation_id_secret = secret('tempo_app_installation_id_secret', 'infra/data/ci/tempo/github-app', 'app-installation-id');
-local tempo_app_private_key_secret = secret('tempo_app_private_key_secret', 'infra/data/ci/tempo/github-app', 'app-private-key');
-
-// secret to sign linux packages
-local gpg_passphrase = secret('gpg_passphrase', 'infra/data/ci/packages-publish/gpg', 'passphrase');
-local gpg_private_key = secret('gpg_private_key', 'infra/data/ci/packages-publish/gpg', 'private-key');
-
-local aws_dev_access_key_id = secret('AWS_ACCESS_KEY_ID-dev', 'infra/data/ci/tempo-dev/aws-credentials-drone', 'access_key_id');
-local aws_dev_secret_access_key = secret('AWS_SECRET_ACCESS_KEY-dev', 'infra/data/ci/tempo-dev/aws-credentials-drone', 'secret_access_key');
-local aws_prod_access_key_id = secret('AWS_ACCESS_KEY_ID-prod', 'infra/data/ci/tempo-prod/aws-credentials-drone', 'access_key_id');
-local aws_prod_secret_access_key = secret('AWS_SECRET_ACCESS_KEY-prod', 'infra/data/ci/tempo-prod/aws-credentials-drone', 'secret_access_key');
-
-//# Steps ##
-
-// the alpine/git image has apk errors when run on aarch64, this is the most recent image that does not have this issue
-// https://github.com/alpine-docker/git/issues/35
-local alpine_git_image = 'alpine/git:v2.30.2';
-
-local image_tag(arch='') = {
-  name: 'image-tag',
-  image: alpine_git_image,
-  commands: [
-    'apk --update --no-cache add bash',
-    'git fetch origin --tags',
-  ] + (
-    if arch == '' then [
-      'echo $(./tools/image-tag) > .tags',
-    ] else [
-      'echo $(./tools/image-tag)-%s > .tags' % arch,
-    ]
-  ),
-};
-
-local image_tag_for_cd() = {
-  name: 'image-tag-for-cd',
-  image: alpine_git_image,
-  commands: [
-    'apk --update --no-cache add bash',
-    'git fetch origin --tags',
-    'echo "grafana/tempo:$(./tools/image-tag)" > .tags-for-cd-tempo',
-    'echo "grafana/tempo-query:$(./tools/image-tag)" > .tags-for-cd-tempo_query',
-    'echo "grafana/tempo-vulture:$(./tools/image-tag)" > .tags-for-cd-tempo_vulture',
-  ],
-};
-
-local build_binaries(arch) = {
-  name: 'build-tempo-binaries',
-  image: 'golang:1.23-alpine',
-  commands: [
-    'apk --update --no-cache add make git bash',
-  ] + [
-    'COMPONENT=%s GOARCH=%s make exe' % [app, arch]
-    for app in apps
-  ],
-};
-
-local docker_build(arch, app, dockerfile='') = {
-  name: 'build-%s-image' % app,
-  image: 'plugins/docker',
-  settings: {
-    dockerfile: if dockerfile != '' then dockerfile else 'cmd/%s/Dockerfile' % app,
-    repo: 'grafana/%s' % app,
-    username: { from_secret: docker_username_secret.name },
-    password: { from_secret: docker_password_secret.name },
-    platform: '%s/%s' % ['linux', arch],
-    build_args: [
-      'TARGETARCH=' + arch,
-    ],
-  },
-};
-
-local docker_manifest(app) = {
-  name: 'manifest-%s' % app,
-  image: 'plugins/manifest:1.4.0',
-  settings: {
-    username: { from_secret: docker_username_secret.name },
-    password: { from_secret: docker_password_secret.name },
-    spec: '.drone/docker-manifest.tmpl',
-    target: app,
-  },
-};
-
-local deploy_to_dev() = {
-  image: 'us.gcr.io/kubernetes-dev/drone/plugins/updater',
-  name: 'update-dev-images',
-  settings: {
-    config_json: std.manifestJsonEx(
-      {
-        destination_branch: 'master',
-        pull_request_branch_prefix: 'auto-merge/cd-tempo-dev',
-        pull_request_enabled: true,
-        pull_request_existing_strategy: "ignore",
-        repo_name: 'deployment_tools',
-        update_jsonnet_attribute_configs: [
-          {
-            file_path: 'ksonnet/environments/tempo/dev-us-central-0.tempo-dev-01/images.libsonnet',
-            jsonnet_key: app,
-            jsonnet_value_file: '.tags-for-cd-' + app,
-          }
-          for app in ['tempo', 'tempo_query', 'tempo_vulture']
-        ],
-      },
-      '  '
-    ),
-    github_app_id: {
-      from_secret: tempo_app_id_secret.name,
-    },
-    github_app_installation_id: {
-      from_secret: tempo_app_installation_id_secret.name,
-    },
-    github_app_private_key: {
-      from_secret: tempo_app_private_key_secret.name,
-    },
-  },
-};
-
-//# Pipelines & resources
-
-[
-  // A pipeline to build Docker images for every app and for every arch
-  (
-    pipeline('docker-' + arch, arch) {
-      steps+: [
-        image_tag(arch),
-        build_binaries(arch),
-      ] + [
-        docker_build(arch, app)
-        for app in apps
-      ],
-    }
-  )
-  for arch in archs
-] + [
-  // Publish Docker manifests
-  pipeline('manifest') {
-    steps+: [
-      image_tag(),
-    ] + [
-      docker_manifest(app)
-      for app in apps
-    ],
-    depends_on+: [
-      'docker-%s' % arch
-      for arch in archs
-    ],
-  },
-] + [
-  // Continuously Deploy to dev env
-  pipeline('cd-to-dev-env') {
-    trigger: {
-      ref: [
-        // always deploy tip of main to dev
-        'refs/heads/main',
-      ],
-    },
-    image_pull_secrets: [
-      docker_config_json_secret.name,
-    ],
-    steps+: [
-      image_tag_for_cd(),
-    ] + [
-      deploy_to_dev(),
-    ],
-    depends_on+: [
-      // wait for images to be published on dockerhub
-      'manifest',
-    ],
-  },
-] + [
-
-  local ghTokenFilename = '/drone/src/gh-token.txt';
-  // Build and release packages
-  // Tested by installing the packages on a systemd container
-  pipeline('release') {
-    trigger: {
-      event: ['tag', 'pull_request'],
-    },
-    image_pull_secrets: [
-      docker_config_json_secret.name,
-    ],
-    volumes+: [
-      {
-        name: 'cgroup',
-        host: {
-          path: '/sys/fs/cgroup',
-        },
-      },
-      {
-        name: 'docker',
-        host: {
-          path: '/var/run/docker.sock',
-        },
-      },
-    ],
-    // Launch systemd containers to test the packages
-    services: [
-      {
-        name: 'systemd-debian',
-        image: 'jrei/systemd-debian:12',
-        volumes: [
-          {
-            name: 'cgroup',
-            path: '/sys/fs/cgroup',
-          },
-        ],
-        privileged: true,
-      },
-      {
-        name: 'systemd-centos',
-        image: 'jrei/systemd-centos:8',
-        volumes: [
-          {
-            name: 'cgroup',
-            path: '/sys/fs/cgroup',
-          },
-        ],
-        privileged: true,
-      },
-    ],
-    steps+: [
-      {
-        name: 'fetch',
-        image: 'docker:git',
-        commands: ['git fetch --tags'],
-      },
-      {
-        name: 'Generate GitHub token',
-        image: 'us.gcr.io/kubernetes-dev/github-app-secret-writer:latest',
-        environment: {
-          GITHUB_APP_ID: { from_secret:  tempo_app_id_secret.name },
-          GITHUB_APP_INSTALLATION_ID: { from_secret:  tempo_app_installation_id_secret.name },
-          GITHUB_APP_PRIVATE_KEY: { from_secret: tempo_app_private_key_secret.name },
-        },
-        commands: [
-          '/usr/bin/github-app-external-token > %s' % ghTokenFilename,
-        ],
-      },
-      {
-        name: 'write-key',
-        image: 'golang:1.23',
-        commands: ['printf "%s" "$NFPM_SIGNING_KEY" > $NFPM_SIGNING_KEY_FILE'],
-        environment: {
-          NFPM_SIGNING_KEY: { from_secret: gpg_private_key.name },
-          NFPM_SIGNING_KEY_FILE: '/drone/src/private-key.key',
-        },
-      },
-      {
-        name: 'test release',
-        image: 'golang:1.23',
-        commands: ['make release-snapshot'],
-        environment: {
-          NFPM_DEFAULT_PASSPHRASE: { from_secret: gpg_passphrase.name },
-          NFPM_SIGNING_KEY_FILE: '/drone/src/private-key.key',
-        },
-      },
-      {
-        name: 'test deb package',
-        image: 'docker',
-        commands: ['./tools/packaging/verify-deb-install.sh'],
-        volumes: [
-          {
-            name: 'docker',
-            path: '/var/run/docker.sock',
-          },
-        ],
-        privileged: true,
-      },
-      {
-        name: 'test rpm package',
-        image: 'docker',
-        commands: ['./tools/packaging/verify-rpm-install.sh'],
-        volumes: [
-          {
-            name: 'docker',
-            path: '/var/run/docker.sock',
-          },
-        ],
-        privileged: true,
-      },
-      {
-        name: 'release',
-        image: 'golang:1.23',
-        commands: [
-          'export GITHUB_TOKEN=$(cat %s)' % ghTokenFilename,
-          'make release'
-        ],
-        environment: {
-          NFPM_DEFAULT_PASSPHRASE: { from_secret: gpg_passphrase.name },
-          NFPM_SIGNING_KEY_FILE: '/drone/src/private-key.key',
-        },
-        when: {
-          event: ['tag'],
-        },
-      },
-    ],
-  },
-] + [
-  docker_username_secret,
-  docker_password_secret,
-  docker_config_json_secret,
-  gh_token_secret,
-  tempo_app_id_secret,
-  tempo_app_installation_id_secret,
-  tempo_app_private_key_secret,
-  aws_dev_access_key_id,
-  aws_dev_secret_access_key,
-  aws_prod_access_key_id,
-  aws_prod_secret_access_key,
-  gpg_private_key,
-  gpg_passphrase,
-]

.drone/drone.yml

@@ -1,445 +0,0 @@
----
-depends_on: []
-kind: pipeline
-name: docker-amd64
-platform:
-  arch: amd64
-  os: linux
-steps:
-- commands:
-  - apk --update --no-cache add bash
-  - git fetch origin --tags
-  - echo $(./tools/image-tag)-amd64 > .tags
-  image: alpine/git:v2.30.2
-  name: image-tag
-- commands:
-  - apk --update --no-cache add make git bash
-  - COMPONENT=tempo GOARCH=amd64 make exe
-  - COMPONENT=tempo-vulture GOARCH=amd64 make exe
-  - COMPONENT=tempo-query GOARCH=amd64 make exe
-  - COMPONENT=tempo-cli GOARCH=amd64 make exe
-  image: golang:1.23-alpine
-  name: build-tempo-binaries
-- image: plugins/docker
-  name: build-tempo-image
-  settings:
-    build_args:
-    - TARGETARCH=amd64
-    dockerfile: cmd/tempo/Dockerfile
-    password:
-      from_secret: docker_password
-    platform: linux/amd64
-    repo: grafana/tempo
-    username:
-      from_secret: docker_username
-- image: plugins/docker
-  name: build-tempo-vulture-image
-  settings:
-    build_args:
-    - TARGETARCH=amd64
-    dockerfile: cmd/tempo-vulture/Dockerfile
-    password:
-      from_secret: docker_password
-    platform: linux/amd64
-    repo: grafana/tempo-vulture
-    username:
-      from_secret: docker_username
-- image: plugins/docker
-  name: build-tempo-query-image
-  settings:
-    build_args:
-    - TARGETARCH=amd64
-    dockerfile: cmd/tempo-query/Dockerfile
-    password:
-      from_secret: docker_password
-    platform: linux/amd64
-    repo: grafana/tempo-query
-    username:
-      from_secret: docker_username
-- image: plugins/docker
-  name: build-tempo-cli-image
-  settings:
-    build_args:
-    - TARGETARCH=amd64
-    dockerfile: cmd/tempo-cli/Dockerfile
-    password:
-      from_secret: docker_password
-    platform: linux/amd64
-    repo: grafana/tempo-cli
-    username:
-      from_secret: docker_username
-trigger:
-  ref:
-  - refs/heads/main
-  - refs/tags/v*
-  - refs/heads/r?
-  - refs/heads/r??
-  - refs/heads/r???
----
-depends_on: []
-kind: pipeline
-name: docker-arm64
-platform:
-  arch: arm64
-  os: linux
-steps:
-- commands:
-  - apk --update --no-cache add bash
-  - git fetch origin --tags
-  - echo $(./tools/image-tag)-arm64 > .tags
-  image: alpine/git:v2.30.2
-  name: image-tag
-- commands:
-  - apk --update --no-cache add make git bash
-  - COMPONENT=tempo GOARCH=arm64 make exe
-  - COMPONENT=tempo-vulture GOARCH=arm64 make exe
-  - COMPONENT=tempo-query GOARCH=arm64 make exe
-  - COMPONENT=tempo-cli GOARCH=arm64 make exe
-  image: golang:1.23-alpine
-  name: build-tempo-binaries
-- image: plugins/docker
-  name: build-tempo-image
-  settings:
-    build_args:
-    - TARGETARCH=arm64
-    dockerfile: cmd/tempo/Dockerfile
-    password:
-      from_secret: docker_password
-    platform: linux/arm64
-    repo: grafana/tempo
-    username:
-      from_secret: docker_username
-- image: plugins/docker
-  name: build-tempo-vulture-image
-  settings:
-    build_args:
-    - TARGETARCH=arm64
-    dockerfile: cmd/tempo-vulture/Dockerfile
-    password:
-      from_secret: docker_password
-    platform: linux/arm64
-    repo: grafana/tempo-vulture
-    username:
-      from_secret: docker_username
-- image: plugins/docker
-  name: build-tempo-query-image
-  settings:
-    build_args:
-    - TARGETARCH=arm64
-    dockerfile: cmd/tempo-query/Dockerfile
-    password:
-      from_secret: docker_password
-    platform: linux/arm64
-    repo: grafana/tempo-query
-    username:
-      from_secret: docker_username
-- image: plugins/docker
-  name: build-tempo-cli-image
-  settings:
-    build_args:
-    - TARGETARCH=arm64
-    dockerfile: cmd/tempo-cli/Dockerfile
-    password:
-      from_secret: docker_password
-    platform: linux/arm64
-    repo: grafana/tempo-cli
-    username:
-      from_secret: docker_username
-trigger:
-  ref:
-  - refs/heads/main
-  - refs/tags/v*
-  - refs/heads/r?
-  - refs/heads/r??
-  - refs/heads/r???
----
-depends_on:
-- docker-amd64
-- docker-arm64
-kind: pipeline
-name: manifest
-platform:
-  arch: amd64
-  os: linux
-steps:
-- commands:
-  - apk --update --no-cache add bash
-  - git fetch origin --tags
-  - echo $(./tools/image-tag) > .tags
-  image: alpine/git:v2.30.2
-  name: image-tag
-- image: plugins/manifest:1.4.0
-  name: manifest-tempo
-  settings:
-    password:
-      from_secret: docker_password
-    spec: .drone/docker-manifest.tmpl
-    target: tempo
-    username:
-      from_secret: docker_username
-- image: plugins/manifest:1.4.0
-  name: manifest-tempo-vulture
-  settings:
-    password:
-      from_secret: docker_password
-    spec: .drone/docker-manifest.tmpl
-    target: tempo-vulture
-    username:
-      from_secret: docker_username
-- image: plugins/manifest:1.4.0
-  name: manifest-tempo-query
-  settings:
-    password:
-      from_secret: docker_password
-    spec: .drone/docker-manifest.tmpl
-    target: tempo-query
-    username:
-      from_secret: docker_username
-- image: plugins/manifest:1.4.0
-  name: manifest-tempo-cli
-  settings:
-    password:
-      from_secret: docker_password
-    spec: .drone/docker-manifest.tmpl
-    target: tempo-cli
-    username:
-      from_secret: docker_username
-trigger:
-  ref:
-  - refs/heads/main
-  - refs/tags/v*
-  - refs/heads/r?
-  - refs/heads/r??
-  - refs/heads/r???
----
-depends_on:
-- manifest
-image_pull_secrets:
-- dockerconfigjson
-kind: pipeline
-name: cd-to-dev-env
-platform:
-  arch: amd64
-  os: linux
-steps:
-- commands:
-  - apk --update --no-cache add bash
-  - git fetch origin --tags
-  - echo "grafana/tempo:$(./tools/image-tag)" > .tags-for-cd-tempo
-  - echo "grafana/tempo-query:$(./tools/image-tag)" > .tags-for-cd-tempo_query
-  - echo "grafana/tempo-vulture:$(./tools/image-tag)" > .tags-for-cd-tempo_vulture
-  image: alpine/git:v2.30.2
-  name: image-tag-for-cd
-- image: us.gcr.io/kubernetes-dev/drone/plugins/updater
-  name: update-dev-images
-  settings:
-    config_json: |-
-      {
-        "destination_branch": "master",
-        "pull_request_branch_prefix": "auto-merge/cd-tempo-dev",
-        "pull_request_enabled": true,
-        "pull_request_existing_strategy": "ignore",
-        "repo_name": "deployment_tools",
-        "update_jsonnet_attribute_configs": [
-          {
-            "file_path": "ksonnet/environments/tempo/dev-us-central-0.tempo-dev-01/images.libsonnet",
-            "jsonnet_key": "tempo",
-            "jsonnet_value_file": ".tags-for-cd-tempo"
-          },
-          {
-            "file_path": "ksonnet/environments/tempo/dev-us-central-0.tempo-dev-01/images.libsonnet",
-            "jsonnet_key": "tempo_query",
-            "jsonnet_value_file": ".tags-for-cd-tempo_query"
-          },
-          {
-            "file_path": "ksonnet/environments/tempo/dev-us-central-0.tempo-dev-01/images.libsonnet",
-            "jsonnet_key": "tempo_vulture",
-            "jsonnet_value_file": ".tags-for-cd-tempo_vulture"
-          }
-        ]
-      }
-    github_app_id:
-      from_secret: tempo_app_id_secret
-    github_app_installation_id:
-      from_secret: tempo_app_installation_id_secret
-    github_app_private_key:
-      from_secret: tempo_app_private_key_secret
-trigger:
-  ref:
-  - refs/heads/main
----
-depends_on: []
-image_pull_secrets:
-- dockerconfigjson
-kind: pipeline
-name: release
-platform:
-  arch: amd64
-  os: linux
-services:
-- image: jrei/systemd-debian:12
-  name: systemd-debian
-  privileged: true
-  volumes:
-  - name: cgroup
-    path: /sys/fs/cgroup
-- image: jrei/systemd-centos:8
-  name: systemd-centos
-  privileged: true
-  volumes:
-  - name: cgroup
-    path: /sys/fs/cgroup
-steps:
-- commands:
-  - git fetch --tags
-  image: docker:git
-  name: fetch
-- commands:
-  - /usr/bin/github-app-external-token > /drone/src/gh-token.txt
-  environment:
-    GITHUB_APP_ID:
-      from_secret: tempo_app_id_secret
-    GITHUB_APP_INSTALLATION_ID:
-      from_secret: tempo_app_installation_id_secret
-    GITHUB_APP_PRIVATE_KEY:
-      from_secret: tempo_app_private_key_secret
-  image: us.gcr.io/kubernetes-dev/github-app-secret-writer:latest
-  name: Generate GitHub token
-- commands:
-  - printf "%s" "$NFPM_SIGNING_KEY" > $NFPM_SIGNING_KEY_FILE
-  environment:
-    NFPM_SIGNING_KEY:
-      from_secret: gpg_private_key
-    NFPM_SIGNING_KEY_FILE: /drone/src/private-key.key
-  image: golang:1.23
-  name: write-key
-- commands:
-  - make release-snapshot
-  environment:
-    NFPM_DEFAULT_PASSPHRASE:
-      from_secret: gpg_passphrase
-    NFPM_SIGNING_KEY_FILE: /drone/src/private-key.key
-  image: golang:1.23
-  name: test release
-- commands:
-  - ./tools/packaging/verify-deb-install.sh
-  image: docker
-  name: test deb package
-  privileged: true
-  volumes:
-  - name: docker
-    path: /var/run/docker.sock
-- commands:
-  - ./tools/packaging/verify-rpm-install.sh
-  image: docker
-  name: test rpm package
-  privileged: true
-  volumes:
-  - name: docker
-    path: /var/run/docker.sock
-- commands:
-  - export GITHUB_TOKEN=$(cat /drone/src/gh-token.txt)
-  - make release
-  environment:
-    NFPM_DEFAULT_PASSPHRASE:
-      from_secret: gpg_passphrase
-    NFPM_SIGNING_KEY_FILE: /drone/src/private-key.key
-  image: golang:1.23
-  name: release
-  when:
-    event:
-    - tag
-trigger:
-  event:
-  - tag
-  - pull_request
-volumes:
-- host:
-    path: /sys/fs/cgroup
-  name: cgroup
-- host:
-    path: /var/run/docker.sock
-  name: docker
----
-get:
-  name: username
-  path: infra/data/ci/docker_hub
-kind: secret
-name: docker_username
----
-get:
-  name: password
-  path: infra/data/ci/docker_hub
-kind: secret
-name: docker_password
----
-get:
-  name: .dockerconfigjson
-  path: secret/data/common/gcr
-kind: secret
-name: dockerconfigjson
----
-get:
-  name: pat
-  path: infra/data/ci/github/grafanabot
-kind: secret
-name: gh_token
----
-get:
-  name: app-id
-  path: infra/data/ci/tempo/github-app
-kind: secret
-name: tempo_app_id_secret
----
-get:
-  name: app-installation-id
-  path: infra/data/ci/tempo/github-app
-kind: secret
-name: tempo_app_installation_id_secret
----
-get:
-  name: app-private-key
-  path: infra/data/ci/tempo/github-app
-kind: secret
-name: tempo_app_private_key_secret
----
-get:
-  name: access_key_id
-  path: infra/data/ci/tempo-dev/aws-credentials-drone
-kind: secret
-name: AWS_ACCESS_KEY_ID-dev
----
-get:
-  name: secret_access_key
-  path: infra/data/ci/tempo-dev/aws-credentials-drone
-kind: secret
-name: AWS_SECRET_ACCESS_KEY-dev
----
-get:
-  name: access_key_id
-  path: infra/data/ci/tempo-prod/aws-credentials-drone
-kind: secret
-name: AWS_ACCESS_KEY_ID-prod
----
-get:
-  name: secret_access_key
-  path: infra/data/ci/tempo-prod/aws-credentials-drone
-kind: secret
-name: AWS_SECRET_ACCESS_KEY-prod
----
-get:
-  name: private-key
-  path: infra/data/ci/packages-publish/gpg
-kind: secret
-name: gpg_private_key
----
-get:
-  name: passphrase
-  path: infra/data/ci/packages-publish/gpg
-kind: secret
-name: gpg_passphrase
----
-kind: signature
-hmac: 549009bef21fd03b82e361c40ad6928b584e18ff762f0890f9302802b124c8b0
-
-...

.drone/readme.md

@@ -1,49 +0,0 @@
-# Drone
-
-Drone is used for building our official dockerhub images. It is broken into 3 
-pipelines. Note that none of the pipelines include testing so it's important that 
-the codebase is otherwise tested when it begins this process. Currently we use GitHub
-Actions for testing every PR and only build the main branch, tags and weekly release
-branches (`r**`).
-
-# Pipelines
-
-The pipelines are `docker-amd64`, `docker-arm64`, and `manifest`. The two docker pipelines
-run concurrently and create images tagged like `tempo:<tag>-<arch>` or `tempo:<branch>-<sha>-<arch>`.
-E.g. `tempo:1.1.0-arm64` or `tempo:main-e2a314-amd64`. The manifest step then creates a manifest 
-that combines the mentioned images into one multiarch image named as you would expect: 
-`tempo:1.1.0` or `tempo:main-e2a314`.  
-
-The documentation on the manifest step is basically non-existent. There's some very
-weak documentation in the Drone docs, but it's not even worth looking at. To understand
-how to use the manifest step I'd recommend looking at the code itself:
-
-https://github.com/drone-plugins/drone-manifest
-
-It is a very simple wrapper that takes the configuration options and runs the following 
-cli tool:
-
-https://github.com/estesp/manifest-tool
-
-[`docker-manifest.tmpl`](./docker-manifest.tmpl) is pushed through the standard go templating library with access
-to these objects: https://github.com/drone-plugins/drone-manifest/blob/master/plugin.go#L23
-
-# Updating drone.yml
-
-`drone.yml` is generated based upon `drone.jsonnet`. To change the Drone pipelines edit
-`drone.jsonnet` and run:
-
-```
-make drone
-```
-
-# Signature
-
-`drone.yml` contains a signature that can only be generated with an access token from the Grafana
-Drone server. If you do not have an access token the last step of `make drone` will fail. Feel free
-to still submit a PR, a Tempo maintainer can update the signature before merging the PR. To regenerate
-the signature run:
-
-```
-make drone-signature
-```

.github/copilot-instructions.md

@@ -0,0 +1,622 @@
+---
+applyTo: "**/*.md"
+---
+
+## Role
+
+Act as an experienced software engineer and technical writer for Grafana Labs.
+
+Write for software developers and engineers.
+
+Assume users know general programming concepts.
+
+## Grafana
+
+Grafana Labs' product suite contains open source projects, enterprise products,
+and the managed Grafana Cloud.
+
+Grafana Labs' open source suite contains:
+
+- Grafana: visualizations, Explore queries, Grafana Drilldown queryless explore
+- Grafana Mimir: scalable and performant metrics backend
+- Grafana Loki: multi-tenant log aggregation system
+- Grafana Tempo: high-scale distributed tracing backend
+- Grafana Pyroscope: scalable continuous profiling backend
+- Grafana Beyla: eBPF auto-instrumentation
+- Grafana Faro: frontend application observability web SDK
+- Grafana Alloy: OpenTelemetry Collector distribution with Prometheus pipelines
+- Grafana OnCall: on-call management
+- Grafana k6: load testing for engineering teams
+
+Grafana Cloud solutions include:
+
+- Grafana: for visualization
+- Metrics: powered by Grafana Mimir and Prometheus
+- Logs: powered by Grafana Loki
+- Traces: powered by Grafana Tempo
+- Profiles: powered by Grafana Pyroscope
+- Frontend Observability: gain real user monitoring insights
+- Application Observability: application performance monitoring
+- Infrastructure observability: ensure infrastructure health and performance
+- Performance & load testing: powered by Grafana k6
+- Synthetic Monitoring: powered by Grafana k6
+- Grafana IRM: observability native incident response
+- Incident: routine task automation for incidents
+- OnCall: flexible on-call management
+
+If a product name starts with "Grafana",
+use the full name on first use and short name after, for example:
+
+- Grafana Alloy (full), Alloy (short)
+- Grafana Beyla (full), Beyla (short)
+
+Refer to the "OpenTelemetry Collector" as "Collector" after the first use.
+Still use "OpenTelemetry Collector" when referring to a distribution,
+and for headings and links.
+
+Always use the full name for "Grafana Cloud".
+
+Never use abbreviations for product names unless specifically asked to,
+for example:
+
+- use "OpenTelemetry" (correct) and not "OTel" (wrong)
+- use "Kubernetes" (correct) and not "K8s" (wrong)
+
+Refer to metrics, logs, traces, and profiles in that order.
+If referring to a subset, still use this ordering, for example:
+
+- metrics, logs, and traces (correct)
+- traces and profiles (correct), profiles and traces (wrong)
+- metrics and logs (correct), logs and metrics (wrong)
+
+You can freely mention open source projects, for example:
+
+- Prometheus
+- OpenTelemetry
+- Linux
+- Docker
+- Kubernetes
+
+Only mention other companies or products for integrations or migrations.
+Focus on Grafana and not the partner product, for example:
+
+- For an integration with Azure don't document Azure set up
+- For a migration from DataDog don't document DataDog set up or usage
+
+## frontmatter
+
+Never remove front matter content at the start of the file.
+This includes all content from the start of the file,
+inbetween a pair of tripple dashes (---).
+
+Never removed YAML front matter meta data unless specifically asked to.
+
+For example, never remove or delete this or other front matter:
+
+```markdown
+---
+title: OpenTelemetry
+cascade:
+  search_section: OpenTelemetry
+  search_type: doc
+---
+
+# OpenTelemetry
+
+Markdown content...
+```
+
+Only edit front matter copy if specifically asked to.
+When performing a copy edit task,
+ask the user if they'd like you to also edit the front matter copy.
+Still never remove front matter meta data properpties.
+
+## Structure
+
+Structure articles into sections with headings.
+
+The frontmatter YAML `title` and the content h1 (#) heading should be the same.
+Never remove the content h1 heading, this redundancy is required.
+
+Always include copy after a heading, for example:
+
+```markdown
+## Heading
+
+Immediately followed by copy and not another heading.
+```
+
+Never nest a heading immediately after another heading, for example:
+
+```markdown
+## Heading
+
+## Sub heading
+```
+
+Add a blank line after headings, for example:
+
+```markdown
+## Heading
+
+Copy after the heading and a blank line.
+```
+
+The immediate copy after a heading should introduce and overview what is
+covered in the section.
+
+Start articles with an introduction that covers the goal of the article,
+example goals:
+
+- Learn concepts
+- Set up or install something
+- Configure something
+- Use a product to solve a business problem
+- Troubleshoot a problem
+- Integrate with other software or systems
+- Migrate from one thing to another
+- Refer to APIs or reference documentation
+
+Follow the goal with a list of prerequisites, for example:
+
+```markdown
+Before you begin ensure you have the following:
+
+- <Prerequisite 1>
+- <Prerequisite 2>
+- ...
+```
+
+Suggest and link to next steps and related resources at the end of the article,
+for example:
+
+- Learn more about A, B, C
+- Configure X
+- Use X to achieve Y
+- Use X to achieve Z
+- Project homepage or documentation
+- Project repository (for example, GitHub, GitLab)
+- Project package (for example, pip or npm)
+
+You don't need to use the "Refer to..." syntax for next steps,
+use the list time text for the link text.
+
+## Style
+
+Write simple copy.
+
+Write short sentences and paragraphs.
+
+Use short words whenever possible, for example:
+
+- use "use" and not "utilize"
+- use "use" and not "make use of"
+
+Always use contractions over multiple words, for example:
+
+- use "it's" and not "it is"
+- use "isn't" and not "is not"
+- use "that's" and not "that is"
+- use "you're" and not "you are"
+- use "Don't" and not "do not"
+
+Don't use filler words or phrases, for example:
+
+- "there is"
+- "there are"
+- "in order to"
+- "it is important to"
+- "keep in mind"
+
+In most cases, use verbs and nouns without adverbs or adjectives.
+You may use minimal adverbs and adjectives,
+when introucing or overviewing a Grafana Labs product.
+
+Don't use figures of speech.
+
+Never use buzzwords, jargon, or cliches.
+
+Never use cultural references or charged language.
+
+## Tense
+
+Write in present simple tense.
+
+Avoid present continous tense.
+
+Only write in future tense to show future actions.
+
+## Voice
+
+Always write in an active voice.
+
+Never write in a passive voice.
+
+## Perspective
+
+Address users as "you".
+
+Don't use first person perspective.
+
+## Wordlist
+
+Use allowlist/blocklist instead of whitelist/blacklist.
+
+Use primary/secondary instead of master/slave.
+
+Use "refer to" instead of "see", "consult", "check out", and other phrases.
+
+## Formatting
+
+Use sentence case for titles and headings, for example:
+
+- This is sentence case (correct)
+- This is Title Case (wrong)
+
+Use the exact page or section title for link text.
+
+Use inline Markdown links, for example, [Link text](https://example.com).
+
+When linking to other sections within the same document,
+use a descriptive phrase that includes the section name,
+and a relative link to its heading anchor.
+For example, "For further details on setup, refer to the [Installation](#installation) section."
+
+Never remove links from copy unless specifically asked to.
+If you edit content that includes a link,
+always include the link in the edited version, for example:
+
+- It is [here](https://grafana.com) go check it out for details (before)
+- Refer to the [Grafana homepage](https://grafana.com) for details (after)
+
+Use two asterisks to bold text, for example `**bold**`.
+
+Use one underscore to emphasize copy, for example `_italics_`.
+
+For UI elements and product features that aren't product names,
+use sentence case as they appear in the UI, for example:
+
+- Click **Submit**. (If "Submit" is the exact text on the button)
+- Navigate to the **User settings** page. (If "User settings" is the title/label)
+- Configure the **alerting rules**. (General concept)
+- Open **Explore** in Grafana. (If "Explore" is the branded name of that section)
+
+Avoid using words created for UI features when possible, for example
+
+- In your Grafana Cloud stack, click **Connections**. (correct)
+- In your Grafana Cloud stack, click the **Connections** button. (wrong)
+
+## Lists
+
+Write complete sentences for lists, for example:
+
+- Works with all languages and frameworks (correct)
+- all languages and frameworks (wrong)
+
+Always use dashes for unordered lists, for example
+
+```markdown
+- List item 1
+- List item 2
+- List item 3
+```
+
+Never use asterisks for unordered lists, for example:
+
+```markdown
+* List item 1 (wrong)
+* List item 2 (wrong)
+* List item 3 (wrong)
+```
+
+Never use full stops at the end of unordered list items, for example:
+
+```markdown
+- Works with all languages and frameworks (correct)
+- Works with all languages and frameworks (wrong)
+```
+
+Always start every ordered list item with 1, for example:
+
+```markdown
+1. Ordered list item 1.
+1. Ordered list item 2.
+1. Ordered list item 3.
+```
+
+Never increment the numbers for ordered list items, for example:
+
+```markdown
+1. Ordered list item 1. (wrong)
+2. Ordered list item 2. (wrong)
+3. Ordered list item 3. (wrong)
+```
+
+Always start the next list item immediately on the next line, for example:
+
+```markdown
+- List item 1
+- List item 2
+- List item 3
+```
+
+Never use new lines between list items, for example:
+
+```markdown
+- List item 1
+
+- List item 2 (wrong)
+
+- List item 3 (wrong)
+```
+
+If a list starts with a keyword, bold the keyword and follow with a colon,
+for example:
+
+```markdown
+- **Keyword 1**: list item 1
+- **Keyword 2**: list item 2
+- **Keyword 3**: list item 3
+```
+
+## Images
+
+Always include descriptive alt text for images.
+The alt text should convey the essential information or purpose of the image.
+Avoid redundant phrases like "Image of..." or "Picture of...".
+
+For example:
+
+- Instead of `![Diagram of system](diagram.png)`
+- Use `![Architecture diagram showing data flow from client applications, through a load balancer, to backend services.](architecture-data-flow.png)`
+
+## Code
+
+Use single code backticks for:
+
+- user input
+- placeholders in markdown, for example _`<PLACEHOLDER_NAME>`_
+- files and directories, for example `/opt/file.md`
+- source code keywords and identifiers,
+  for example variables, function and class names
+- configuration options and values, for example `PORT` and `80`
+- status codes, for example `404`
+
+Use triple code backticks followed by the syntax for code blocks, for example:
+
+```javascript
+console.log("Hello World!");
+```
+
+Always introduce each code blocks with a short description.
+End the introduction with a colon if the code sample follows it, for example:
+
+```markdown
+The code sample outputs "Hello World!" to the browser console:
+
+<CODE_BLOCK>
+```
+
+Use descriptive placeholder names in code samples.
+Use uppercase letters with underscores to separate words in placeholders,
+for example:
+
+```sh
+OTEL_RESOURCE_ATTRIBUTES="service.name=<SERVICE_NAME>
+OTEL_EXPORTER_OTLP_ENDPOINT=<OTLP_ENDPOINT>
+```
+
+The placeholder includes the name and the less than and greater than symbols,
+for example <PLACEHOLDER_NAME>.
+
+If the placeholder is markdown emphasize it with underscores,
+for example _`<PLACEHOLDER_NAME>`_.
+
+In code blocks use the placeholder without additional backticks or emphasis,
+for example <PLACEHOLDER_NAME>.
+
+Always provide an explanation for each placeholder,
+typically in the text following the code block or in a configuration table.
+
+Never add new code block unless specifically asked to, for example,
+when asked to improve an article don't add new code.
+
+Never change existing code blocks unless specifically asked to, for example
+when asked to improve an article don't edit code.
+
+Always follow code samples with an explanation
+and configuration options for placeholders, for example:
+
+```markdown
+<CODE_BLOCK>
+
+This code sets required environment variables
+to send OTLP data to an OTLP endpoint.
+To configure the code for your needs,
+refer to the configuration table
+and select the appropriate configuration options for your use case.
+
+<CONFIGURATION_TABLE>
+```
+
+Never put configuration for a code block before the code block.
+
+## APIs
+
+When documenting API endpoints specify the HTTP method,
+for example `GET`, `POST`, `PUT`, `DELETE`.
+
+Provide the full request path, using backticks.
+
+Use backticks for parameter names and example values.
+
+Use placeholders like `{userId}` for path parameters, for example:
+
+- To retrieve user details, make a `GET` request to `/api/v1/users/{userId}`.
+
+Use a configuration table to describe query and header parameters,
+and request body fields.
+
+## CLI commands
+
+When presenting CLI commands and their output,
+introduce the command with a brief explanation of its purpose.
+Clearly distinguish the command from its output.
+
+For commands, use `sh` to specify the code block language.
+
+For output, use a generic specifier like `text`, `console`,
+or `json`/`yaml` if the output is structured.
+
+For example:
+
+````markdown
+To list all running pods in the `default` namespace, use the following command:
+
+```sh
+kubectl get pods --namespace default
+```
+````
+
+The output will resemble the following:
+
+```text
+NAME                               READY   STATUS    RESTARTS   AGE
+my-app-deployment-7fdb6c5f65-abcde   1/1     Running   0          2d1h
+another-service-pod-xyz123           2/2     Running   0          5h30m
+```
+
+## Configuration table
+
+Use Markdown tables to document configuration, including:
+
+- placeholders
+- parameters
+- YAML/JSON configuration
+- environment variables
+
+Add columns for "Option", "Summary", "Required", "Type", "Values", "Default"
+in that order.
+
+Use the following values for option:
+
+- YAML/JSON, code, or parameter field if one exists for that configuration
+- Environment variable field if one exists for that configuration
+- If there is a YAML and Environment variable option separate them with `<br>`
+
+Use a short sentence for the summary.
+After the summary sentence and in the same cell,
+link to further details using the following format,
+"Refer to [option](#option-anchor) for details."
+
+In the values column, describe acceptable values
+and include value options or ranges if they exist.
+Don't include example values in the table.
+
+An example configuration table:
+
+```markdown
+| Option                         | Summary                                                                                              | Required | Type   | Values                               | Default |
+| ------------------------------ | ---------------------------------------------------------------------------------------------------- | -------- | ------ | ------------------------------------ | ------- |
+| `SERVICE_NAME`                 | The logical name of your application or service. Refer to [service name](#service-name) for details. | Yes      | String | A descriptive name for your service. | N/A     |
+| `OTEL_EXPORTER_OTLP_ENDPOINT`  | The target URL for the OTLP exporter. Refer to [OTLP endpoint](#otlp-endpoint) for details.          | Yes      | String | A valid URL.                         | N/A     |
+| `logging.level`<br>`LOG_LEVEL` | Sets the logging verbosity. Refer to [logging level](#logging-level) for details.                    | No       | String | `debug`, `info`, `warn`, `error`     | `info`  |
+```
+
+In some ocassions, you can drop the Values column, for example,
+when all the configuration have a Boolean type (yes/no) or (1/0).
+
+After the table, for each configuration, add a sub-section.
+
+For each sub-section include a heading and more detailed information including:
+
+- a full description
+- an explanation of concepts
+- any related configuration
+- example values with explanations, don't use code blocks only single backticks
+  for example values
+
+Use sentence case for the heading and only capitalize known keywords.
+
+Example configuration sub-sections:
+
+```markdown
+<CONFIGURATION_TABLE>
+
+### Service name
+
+The `SERVICE_NAME` option specifies the name of your application or service.
+This name identifies your service in Grafana Cloud.
+It helps filter and query telemetry data.
+A name that describes the service helps organize your observability data.
+
+For example, if your microservice handles payments, a service name could be `payment-service`.
+If your application is a frontend, you might use `webapp-frontend`.
+
+### OTLP endpoint
+
+The `OTEL_EXPORTER_OTLP_ENDPOINT` option defines the target URL.
+The OpenTelemetry (OTLP) exporter sends your telemetry data to this URL.
+This endpoint is the ingestion point for your metrics, logs, traces, and profiles.
+For Grafana Cloud, this is your Grafana Cloud OTLP endpoint.
+
+An example OTLP endpoint URL is `https://otlp-gateway-prod-us-central-0.grafana.net/otlp`.
+Your endpoint URL depends on your Grafana Cloud stack and region.
+You find this URL in your Grafana Cloud account details.
+
+### Logging level
+
+The `logging.level` and `LOG_LEVEL` options control the verbosity of logging output.
+Use these options to adjust the level of detail in logs for debugging or monitoring purposes.
+
+The available levels are `debug`, `info`, `warn`, and `error`.
+For example, setting the level to `debug` provides detailed logs useful for troubleshooting,
+while `error` limits logs to critical issues.
+
+The default logging level is `info`, which provides a balance between verbosity and relevance.
+```
+
+## Comparison table
+
+Use a comparison table when users have multiple options to achieve similar
+outcomes and need to understand the trade-offs or key differences between them.
+
+Comparison tables help users make informed decisions by highlighting distinct features,
+requirements, or characteristics of each option side-by-side.
+
+Structure the table with options as rows and differentiating criteria as columns.
+Ensure each cell provides concise, directly comparable information.
+
+An example comparison table:
+
+```markdown
+| Method                      | Code changes             | Language support   | Use case                                                      |
+| --------------------------- | ------------------------ | ------------------ | ------------------------------------------------------------- |
+| Grafana OpenTelemetry Java  | Not required (JVM agent) | Java               | Offers advanced instrumentation features and Grafana support. |
+| Grafana OpenTelemetry .NET  | Required                 | .NET               | Offers advanced instrumentation features and Grafana support. |
+| Upstream OpenTelemetry SDKs | Required                 | Multiple languages | Provides standard instrumentation with community support.     |
+```
+
+## Shortcodes
+
+Don't use blockquotes for notes, cautions, or warnings.
+
+Use the custom admonition Hugo shortcode
+with <TYPE> as "note", "caution", or "warning":
+
+```markdown
+{{< admonition type="<TYPE>" >}}
+...
+{{< /admonition >}}
+```
+
+Use admonitions sparingly.
+Only include exceptional information in admonitions,
+this means you will use it more often for warnings than notes and cautions.
+
+Never delete Hugo shortcodes unless specifically asked to,
+for example don't delete shortcodes:
+
+```markdown
+{{< docs/shared source="tempo" lookup="grafana-cloud.md" version="" >}}
+```

.github/workflows/add-to-docs-project.yml

@@ -1,15 +0,0 @@
-name: Add to docs project
-on:
-  issues:
-    types: [labeled]
-  pull_request:
-    types: [labeled]
-jobs:
-  main:
-    if: ${{ github.event.label.name == 'type/docs' && github.repository == 'grafana/tempo' }}
-    permissions:
-      contents: read
-      id-token: write
-    runs-on: ubuntu-24.04
-    steps:
-      - uses: grafana/writers-toolkit/add-to-docs-project@add-to-docs-project/v1

.github/workflows/backport.yml

@@ -4,16 +4,21 @@
 # will only check a few things and detect that the PR is not yet merged. 
 # At this time only squashed PRs are supported since the cherry-pick 
 # command does not include "-m <n>" arg required for merge commits.
-name: Backport
+# TODO: this workflow is broken, suraj will fix it in a day or two
+name: Backport PR Creator
 on:
-  pull_request_target:
+  pull_request:
     types:
       - closed
       - labeled
 
+permissions: {}
+
 jobs:
   main:
-    runs-on: ubuntu-24.04
+    # skip it in all forks, only run in grafana/tempo.
+    if: github.repository == 'grafana/tempo'
+    runs-on: ubuntu-latest
     steps:
       - name: Checkout Actions
         uses: actions/checkout@v4
@@ -21,13 +26,28 @@ jobs:
           repository: "grafana/grafana-github-actions"
           path: ./actions
           ref: main
-
+          persist-credentials: false
       - name: Install Actions
         run: npm install --production --prefix ./actions
-
+      - name: Get Github App secrets from vault
+        id: get-secrets
+        uses: grafana/shared-workflows/actions/get-vault-secrets@5d7e361bc7e0a183cde8afe9899fb7b596d2659b #get-vault-secrets 1.2.0
+        with:
+          export_env: false
+          repo_secrets: |
+            APP_ID=tempo-ci-app:app-id
+            PRIVATE_KEY=tempo-ci-app:private-key
+      - name: Generate GitHub App token
+        id: app-token
+        uses: actions/create-github-app-token@df432ceedc7162793a195dd1713ff69aefc7379e # v2.0.6
+        with:
+          app-id: ${{ fromJSON(steps.get-secrets.outputs.secrets).APP_ID }}
+          private-key: ${{ fromJSON(steps.get-secrets.outputs.secrets).PRIVATE_KEY }}
+          owner: ${{ github.repository_owner }}
+          repositories: tempo
       - name: Run backport
         uses: ./actions/backport
         with:
-          token: ${{secrets.GITHUB_TOKEN}}
+          token: ${{ steps.app-token.outputs.token }}
           labelsToAdd: "backport"
-          title: "[{{base}}] {{originalTitle}}"
+          title: "[{{base}}] {{originalTitle}}"

.github/workflows/ci.yml

@@ -16,6 +16,8 @@ concurrency:
   group: "${{ github.ref_name }}-${{ github.head_ref }}"
   cancel-in-progress: true
 
+permissions: {}
+
 jobs:
   lint:
     name: Lint
@@ -26,6 +28,7 @@ jobs:
         uses: actions/checkout@v4
         with:
           fetch-depth: "0"
+          persist-credentials: false
       - name: Set up Go
         uses: actions/setup-go@v5
         with:
@@ -48,15 +51,20 @@ jobs:
           key: golangci-lint-${{ runner.os }}-${{ steps.get-year-week-number.outputs.date }}-${{ hashFiles('go.mod', '.golangci.yml') }}
 
       - name: lint
+        # using env instead of template expansions to avoid template injection
+        # https://woodruffw.github.io/zizmor/audits/#template-injection
         run: |
-          make lint base=origin/${{github.base_ref}}
+          make lint base=origin/${BASE_REF}
           sudo chown -R $(id -u):$(id -g) .cache/golangci-lint  # needed to archive cache
+        env:
+          BASE_REF: ${{ github.base_ref }}
 
 
   unit-tests:
     name: Run Unit Tests
     runs-on: ubuntu-24.04
     strategy:
+      fail-fast: false
       matrix:
         test-target:
           [
@@ -69,6 +77,8 @@ jobs:
     steps:
       - name: Check out code
         uses: actions/checkout@v4
+        with:
+          persist-credentials: false
 
       - name: Set up Go
         uses: actions/setup-go@v5
@@ -82,18 +92,21 @@ jobs:
     name: Run integration tests
     runs-on: ubuntu-24.04
     strategy:
+      fail-fast: false
       matrix:
         test-target:
           [
             test-e2e,
+            test-e2e-api,
             test-integration-poller,
             test-e2e-deployments,
-            test-e2e-serverless,
           ]
 
     steps:
       - name: Check out code
         uses: actions/checkout@v4
+        with:
+          persist-credentials: false
       - name: Fetch tags
         run: git fetch --prune --unshallow --tags
       - name: Set up Go
@@ -109,6 +122,8 @@ jobs:
     steps:
       - name: Check out code
         uses: actions/checkout@v4
+        with:
+          persist-credentials: false
 
       - name: Set up Go
         uses: actions/setup-go@v5
@@ -136,6 +151,8 @@ jobs:
     steps:
       - name: Check out code
         uses: actions/checkout@v4
+        with:
+          persist-credentials: false
 
       - name: Set up Go
         uses: actions/setup-go@v5
@@ -151,6 +168,8 @@ jobs:
     steps:
       - name: Check out code
         uses: actions/checkout@v4
+        with:
+          persist-credentials: false
 
       - name: Set up Go
         uses: actions/setup-go@v5
@@ -158,7 +177,7 @@ jobs:
           go-version-file: "go.mod"
 
       - name: Install Protoc
-        uses: arduino/setup-protoc@v3
+        uses: arduino/setup-protoc@c65c819552d16ad3c9b72d9dfd5ba5237b9c906b # v3
 
       - name: Check vendor
         run: make vendor-check
@@ -169,6 +188,8 @@ jobs:
     steps:
       - name: Check out code
         uses: actions/checkout@v4
+        with:
+          persist-credentials: false
 
       - name: Set up Go
         uses: actions/setup-go@v5
@@ -190,5 +211,8 @@ jobs:
     steps:
       - name: Check out code
         uses: actions/checkout@v4
+        with:
+          persist-credentials: false
+
       - name: Build Website
         run: docker run -v ${PWD}/docs/sources:/hugo/content/docs/tempo/latest --rm grafana/docs-base:latest make prod

.github/workflows/dependabot_serverless_gomod.yml

@@ -1,58 +0,0 @@
-name: Dependabot
-on: pull_request
-permissions:
-  contents: write
-
-# NOTE: For this job, we are unable to use the GITHUB_TOKEN from the action
-# itself.  The reason for this is that the tokens handed out for actions dont'
-# contain the necessary permissions to kick off workflows in the case a new
-# commit is added, as is the case here.  For this job, we use the GH_BOT token
-# secret in the repo, which has the necessary permissions.  This allows the
-# final commit+push on this job to execute the rest of the workflow required by
-# the PR build configuration.
-
-jobs:
-  dependabot:
-    name: Serverless gomod update
-    runs-on: ubuntu-24.04
-    if: ${{ github.event.pull_request.user.login == 'dependabot[bot]' }}
-    steps:
-      - name: Set up Go 1.23
-        uses: actions/setup-go@v5
-        with:
-          go-version: 1.23.3
-
-      - name: Get app token
-        uses: actions/create-github-app-token@v1
-        id: get-github-app-token
-        with:
-          app-id: ${{secrets.APP_ID}}
-          private-key: ${{secrets.APP_PRIVATE_KEY}}
-          owner: ${{ github.repository_owner }}
-
-      - name: Check out code
-        uses: actions/checkout@v4
-        with:
-          ref: ${{ github.head_ref }}
-          token: ${{ steps.get-github-app-token.outputs.token }}
-
-      - name: Update serverless gomod
-        run: make -C cmd/tempo-serverless update-mod
-
-      - name: Commit serverless gomod changes
-        env:
-          USER: x-access-token
-          TOKEN: ${{ steps.get-github-app-token.outputs.token }}
-        run: |
-          git config --global user.email "bot@grafana.com"
-          git config --global user.name "tempo-gh-bot[bot]"
-          git config --global url."https://${USER}:${TOKEN}@github.com/grafana/tempo".insteadOf "https://github.com/grafana/tempo"
-          git add cmd/tempo-serverless/lambda/go.mod
-          git add cmd/tempo-serverless/lambda/go.sum
-          git add cmd/tempo-serverless/cloud-run/go.mod
-          git add cmd/tempo-serverless/cloud-run/go.sum
-          git diff --quiet --staged || git commit -m 'Update serverless gomod'
-
-      - name: Push changes
-        run: |
-          git push origin ${{ github.head_ref }}

.github/workflows/docker-ci-tools.yml

@@ -1,6 +1,7 @@
 name: docker-ci-tools
 on:
   push:
+    # If you define both branches/branches-ignore and paths/paths-ignore, the workflow will only run when both filters are satisfied.
     branches:
       - 'main'
     paths:
@@ -12,7 +13,6 @@ env:
 # Needed to login to DockerHub
 permissions:
   contents: read
-  id-token: write
 
 jobs:
 
@@ -24,6 +24,8 @@ jobs:
     steps:
       - name: Checkout
         uses: actions/checkout@v4
+        with:
+          persist-credentials: false
 
       - id: get-tag
         run: |
@@ -36,14 +38,18 @@ jobs:
       matrix:
         runner_arch: [ { runner: ubuntu-24.04, arch: amd64 }, { runner: github-hosted-ubuntu-arm64, arch: arm64 } ]
     runs-on: ${{ matrix.runner_arch.runner }}
+    permissions:
+      id-token: write
     env:
       TAG: ${{ needs.get-image-tag.outputs.tag }}
     steps:
       - name: Checkout
         uses: actions/checkout@v4
+        with:
+          persist-credentials: false
 
       - name: Login to DockerHub
-        uses: grafana/shared-workflows/actions/dockerhub-login@dockerhub-login-v1.0.0
+        uses: grafana/shared-workflows/actions/dockerhub-login@5d7e361bc7e0a183cde8afe9899fb7b596d2659b # dockerhub-login-v1.0.1
 
       - name: docker-build-and-push
         run: |
@@ -55,14 +61,18 @@ jobs:
     if: github.repository == 'grafana/tempo'
     needs: ['get-image-tag', 'docker-ci-tools']
     runs-on: ubuntu-latest
+    permissions:
+      id-token: write
     env:
       TAG: ${{ needs.get-image-tag.outputs.tag }}
     steps:
       - name: Checkout
         uses: actions/checkout@v4
+        with:
+          persist-credentials: false
 
       - name: Login to DockerHub
-        uses: grafana/shared-workflows/actions/dockerhub-login@dockerhub-login-v1.0.0
+        uses: grafana/shared-workflows/actions/dockerhub-login@5d7e361bc7e0a183cde8afe9899fb7b596d2659b # dockerhub-login-v1.0.1
 
       - name: docker-manifest-create-and-push
         run: |

.github/workflows/docker.yml

@@ -0,0 +1,188 @@
+name: docker
+on:
+  push:
+    branches:
+      - 'main'
+      - 'r[0-9]+'
+    tags:
+      - 'v*'
+
+# Needed to login to DockerHub
+permissions:
+  contents: read
+
+jobs:
+
+  get-tag:
+    if: github.repository == 'grafana/tempo'  # skip in forks
+    runs-on: ubuntu-24.04
+    outputs:
+      tag: ${{ steps.get-tag.outputs.tag }}
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+        with:
+          persist-credentials: false
+
+      - name: fetch tags
+        run: git fetch --tags --force
+
+      - id: get-tag
+        run: |
+          echo "tag=$(./tools/image-tag)" >> "$GITHUB_OUTPUT"
+
+  docker:
+    if: github.repository == 'grafana/tempo'
+    needs: get-tag
+    strategy:
+      matrix:
+        component: [ tempo, tempo-vulture, tempo-query, tempo-cli ]
+        runner_arch: [ { runner: ubuntu-24.04, arch: amd64 }, { runner: github-hosted-ubuntu-arm64, arch: arm64 } ]
+    runs-on: ${{ matrix.runner_arch.runner }}
+    permissions:
+      id-token: write
+    env:
+      TAG: ${{ needs.get-tag.outputs.tag }}
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+        with:
+          persist-credentials: false
+
+      - name: fetch tags
+        run: git fetch --tags --force
+
+      - name: build-tempo-binaries
+        run: |
+          COMPONENT=${{ matrix.component }} GOARCH=${{ matrix.runner_arch.arch }} make exe
+
+      - name: docker-build
+        run: |
+          TAG_ARCH="$TAG-${{ matrix.runner_arch.arch }}"
+          docker build -f cmd/${{ matrix.component }}/Dockerfile -t grafana/${{ matrix.component }}:$TAG_ARCH .
+
+      - name: Login to DockerHub
+        uses: grafana/shared-workflows/actions/dockerhub-login@5d7e361bc7e0a183cde8afe9899fb7b596d2659b # dockerhub-login-v1.0.1
+
+      - name: docker-push
+        run: |
+          TAG_ARCH="$TAG-${{ matrix.runner_arch.arch }}"          
+          docker push grafana/${{ matrix.component }}:$TAG_ARCH
+
+  manifest:
+    if: github.repository == 'grafana/tempo'
+    needs: ['get-tag', 'docker']
+    permissions:
+      id-token: write
+    strategy:
+      matrix:
+        component: [ tempo, tempo-vulture, tempo-query, tempo-cli ]
+    runs-on: ubuntu-24.04
+    env:
+      TAG: ${{ needs.get-tag.outputs.tag }}
+      IMAGE_NAME: grafana/${{ matrix.component }}
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+        with:
+          persist-credentials: false
+
+      - name: Login to DockerHub
+        uses: grafana/shared-workflows/actions/dockerhub-login@5d7e361bc7e0a183cde8afe9899fb7b596d2659b # dockerhub-login-v1.0.1
+
+      - name: docker-manifest-create-and-push
+        run: |
+          docker manifest create \
+          	$IMAGE_NAME:$TAG               \
+          	--amend $IMAGE_NAME:$TAG-amd64  \
+          	--amend $IMAGE_NAME:$TAG-arm64
+          docker manifest push $IMAGE_NAME:$TAG
+
+          docker manifest create \
+          	$IMAGE_NAME:latest            \
+          	--amend $IMAGE_NAME:$TAG-amd64   \
+          	--amend $IMAGE_NAME:$TAG-arm64
+          docker manifest push $IMAGE_NAME:latest
+
+  cd-to-dev-env:
+    # This job deploys the latest main commit to the dev environment
+    if: github.repository == 'grafana/tempo' && github.ref == 'refs/heads/main'
+    runs-on: ubuntu-24.04
+    needs: manifest
+    permissions:
+      id-token: write
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+        with:
+          persist-credentials: false
+
+      - name: fetch tags
+        run: git fetch --tags --force
+
+      - name: get-tag
+        run: |
+          echo "grafana/tempo:$(./tools/image-tag)" > tags_for_cd_tempo
+          echo "grafana/tempo-query:$(./tools/image-tag)" > tags_for_cd_tempo_query
+          echo "grafana/tempo-vulture:$(./tools/image-tag)" > tags_for_cd_tempo_vulture
+
+      - name: Authenticate to GAR
+        uses: grafana/shared-workflows/actions/login-to-gar@5d7e361bc7e0a183cde8afe9899fb7b596d2659b # login-to-gar-v0.3.0
+        id: login-to-gar
+        with:
+          registry: us-docker.pkg.dev
+          environment: prod
+
+      - name: Get Vault secrets
+        id: get-secrets
+        uses: grafana/shared-workflows/actions/get-vault-secrets@5d7e361bc7e0a183cde8afe9899fb7b596d2659b # get-vault-secrets-v1.2.0
+        with:
+          export_env: false
+          common_secrets: |
+            GITHUB_APP_ID=updater-app:app-id
+            GITHUB_APP_INSTALLATION_ID=updater-app:app-installation-id
+            GITHUB_APP_PRIVATE_KEY=updater-app:private-key
+
+      - name: Update jsonnet
+        env:
+          GITHUB_APP_ID: ${{ fromJSON(steps.get-secrets.outputs.secrets).GITHUB_APP_ID }}
+          GITHUB_APP_INSTALLATION_ID: ${{ fromJSON(steps.get-secrets.outputs.secrets).GITHUB_APP_INSTALLATION_ID }}
+          GITHUB_APP_PRIVATE_KEY: ${{ fromJSON(steps.get-secrets.outputs.secrets).GITHUB_APP_PRIVATE_KEY }}
+        run: |
+          set -e -o pipefail
+
+          cat << EOF > config.json
+          {
+            "destination_branch": "master",
+            "pull_request_branch_prefix": "auto-merge/cd-tempo-dev",
+            "pull_request_enabled": true,
+            "pull_request_existing_strategy": "ignore",
+            "repo_name": "deployment_tools",
+            "update_jsonnet_attribute_configs": [
+              {
+                "file_path": "ksonnet/environments/tempo/dev-us-central-0.tempo-dev-01/images.libsonnet",
+                "jsonnet_key": "tempo",
+                "jsonnet_value_file": "tags_for_cd_tempo"
+              },
+              {
+                "file_path": "ksonnet/environments/tempo/dev-us-central-0.tempo-dev-01/images.libsonnet",
+                "jsonnet_key": "tempo_query",
+                "jsonnet_value_file": "tags_for_cd_tempo_query"
+              },
+              {
+                "file_path": "ksonnet/environments/tempo/dev-us-central-0.tempo-dev-01/images.libsonnet",
+                "jsonnet_key": "tempo_vulture",
+                "jsonnet_value_file": "tags_for_cd_tempo_vulture"
+              }
+            ]
+          }
+          EOF
+
+          docker run --rm \
+          -e GITHUB_APP_ID \
+          -e GITHUB_APP_INSTALLATION_ID \
+          -e GITHUB_APP_PRIVATE_KEY \
+          -e CONFIG_JSON="$(cat config.json)" \
+          -v ./tags_for_cd_tempo:/app/tags_for_cd_tempo \
+          -v ./tags_for_cd_tempo_query:/app/tags_for_cd_tempo_query \
+          -v ./tags_for_cd_tempo_vulture:/app/tags_for_cd_tempo_vulture us-docker.pkg.dev/grafanalabs-global/docker-deployment-tools-prod/updater

.github/workflows/drone-signature-check.yml

@@ -1,23 +0,0 @@
-name: Check Drone CI Signature
-
-on:
-  push:
-    branches:
-      - "main"
-    paths:
-      - ".drone/drone.yml"
-  pull_request:
-    paths:
-      - ".drone/drone.yml"
-
-permissions:
-  id-token: write
-  contents: read
-
-jobs:
-  drone-signature-check:
-    # only run in grafana/tempo.
-    if: github.repository == 'grafana/tempo'    
-    uses: grafana/shared-workflows/.github/workflows/check-drone-signature.yaml@main
-    with:
-      drone_config_path: .drone/drone.yml

.github/workflows/metrics-collector.yml

@@ -3,6 +3,8 @@ on:
   schedule:
     - cron: "*/30 * * * *"
 
+permissions: {}
+
 jobs:
   main:
     # this action keeps failing in all forks, only run in grafana/tempo.
@@ -10,19 +12,29 @@ jobs:
     if: github.repository == 'grafana/tempo'
     runs-on: ubuntu-24.04
     steps:
+      - name: Get Github App secrets from vault
+        id: get-secrets
+        uses: grafana/shared-workflows/actions/get-vault-secrets@5d7e361bc7e0a183cde8afe9899fb7b596d2659b #get-vault-secrets 1.2.0
+        with:
+          export_env: false
+          repo_secrets: |
+            APP_ID=tempo-ci-app:app-id
+            PRIVATE_KEY=tempo-ci-app:private-key
       - name: Get app token
-        uses: actions/create-github-app-token@v1
+        uses: actions/create-github-app-token@df432ceedc7162793a195dd1713ff69aefc7379e # v2.0.6
         id: get-github-app-token
         with:
-          app-id: ${{secrets.APP_ID}}
-          private-key: ${{secrets.APP_PRIVATE_KEY}}
+          app-id: ${{ fromJSON(steps.get-secrets.outputs.secrets).APP_ID }}
+          private-key: ${{ fromJSON(steps.get-secrets.outputs.secrets).PRIVATE_KEY }}
           owner: ${{ github.repository_owner }}
+          repositories: tempo
       - name: Checkout Actions
         uses: actions/checkout@v4
         with:
           repository: "grafana/grafana-github-actions"
           path: ./actions
           ref: main
+          persist-credentials: false
       - name: Install Actions
         run: npm install --production --prefix ./actions
       - name: Run metrics collector

.github/workflows/milestoned_to_project.yml

@@ -1,25 +0,0 @@
-name: Milestoned issue to Tempo Squad Project.
-
-on:
-  # Triggers the workflow on when an issue is milestoned.
-  issues:
-    types: [milestoned]
-    
-jobs:
-  build:
-    # only run in grafana/tempo.
-    if: github.repository == 'grafana/tempo'
-    runs-on: ubuntu-24.04
-
-    steps:
-      - name: Get app token
-        uses: actions/create-github-app-token@v1
-        id: get-github-app-token
-        with:
-          app-id: ${{secrets.APP_ID}}
-          private-key: ${{secrets.APP_PRIVATE_KEY}}
-          owner: ${{ github.repository_owner }}
-      - uses: actions/add-to-project@v1.0.2
-        with:
-          project-url: https://github.com/orgs/grafana/projects/135
-          github-token: ${{ steps.get-github-app-token.outputs.token }}

.github/workflows/publish-technical-documentation-next.yml

@@ -1,5 +1,7 @@
 name: publish-technical-documentation-next
 
+permissions: {}
+
 on:
   push:
     branches:
@@ -16,11 +18,13 @@ jobs:
     runs-on: ubuntu-24.04
     steps:
       - uses: actions/checkout@v4
-      - uses: grafana/writers-toolkit/publish-technical-documentation@publish-technical-documentation/v1
+        with:
+          persist-credentials: false
+      - uses: grafana/writers-toolkit/publish-technical-documentation@6200164349ac8f24bab065ce3689dd8c9a093061
         with:
           source_directory: docs/sources/tempo
           website_directory: content/docs/tempo/next
-      - uses: grafana/writers-toolkit/publish-technical-documentation@publish-technical-documentation/v1
+      - uses: grafana/writers-toolkit/publish-technical-documentation@6200164349ac8f24bab065ce3689dd8c9a093061
         with:
           source_directory: docs/sources/helm-charts/tempo-distributed
           website_directory: content/docs/helm-charts/tempo-distributed/next

.github/workflows/publish-technical-documentation-release.yml

@@ -1,5 +1,7 @@
 name: publish-technical-documentation-release
 
+permissions: {}
+
 on:
   push:
     branches:
@@ -19,8 +21,9 @@ jobs:
     steps:
       - uses: actions/checkout@v4
         with:
+          persist-credentials: false
           fetch-depth: 0
-      - uses: grafana/writers-toolkit/publish-technical-documentation-release@publish-technical-documentation-release/v1
+      - uses: grafana/writers-toolkit/publish-technical-documentation-release@6200164349ac8f24bab065ce3689dd8c9a093061
         with:
           release_tag_regexp: "^v(\\d+)\\.(\\d+)\\.(\\d+)$"
           release_branch_regexp: "^release-v(\\d+)\\.(\\d+)$"

.github/workflows/release.yml

@@ -0,0 +1,64 @@
+name: release
+on:
+  push:
+    tags:
+      - 'v*'
+
+# Needed to login to DockerHub
+permissions: {}
+jobs:
+
+  release:
+    if: github.repository == 'grafana/tempo'  # skip in forks
+    runs-on: ubuntu-24.04
+    permissions:
+      contents: write
+      id-token: write
+    env:
+      NFPM_SIGNING_KEY_FILE: /tmp/nfpm-private-key.key
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+        with:
+          persist-credentials: false
+
+      - name: fetch tags
+        run: git fetch --tags --force
+
+      - id: "get-secrets"
+        name: "get nfpm signing keys"
+        uses: "grafana/shared-workflows/actions/get-vault-secrets@5d7e361bc7e0a183cde8afe9899fb7b596d2659b" #get-vault-secrets-v1.2.0
+        with:
+          common_secrets: |
+            NFPM_SIGNING_KEY=packages-gpg:private-key
+            NFPM_DEFAULT_PASSPHRASE=packages-gpg:passphrase
+
+      - name: write-key
+        run: printenv NFPM_SIGNING_KEY > $NFPM_SIGNING_KEY_FILE
+
+      - name: test release
+        run: make release-snapshot
+
+      - name: test deb package
+        run: |
+          # podman is simpler to make it work with systemd inside containers
+          podman run -d --name systemd-debian --privileged -v /sys/fs/cgroup:/sys/fs/cgroup:ro jrei/systemd-debian:12
+          podman cp ./dist/tempo_*_linux_amd64.deb systemd-debian:.
+          podman cp ./tools/packaging/verify-deb-install.sh systemd-debian:.
+          podman cp ./tools/packaging/wait-for-ready.sh systemd-debian:. 
+          podman exec systemd-debian ./verify-deb-install.sh
+          podman rm -f systemd-debian
+
+      - name: test rpm package
+        run: |
+          podman run -d --name systemd-centos --privileged -v /sys/fs/cgroup:/sys/fs/cgroup:ro jrei/systemd-centos:8
+          podman cp ./dist/tempo_*_linux_amd64.rpm systemd-centos:.
+          podman cp ./tools/packaging/verify-rpm-install.sh systemd-centos:.
+          podman cp ./tools/packaging/wait-for-ready.sh systemd-centos:.
+          podman exec systemd-centos ./verify-rpm-install.sh
+          podman rm -f systemd-centos
+
+      - name: release
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        run: make release

.github/workflows/sbom-report.yml

@@ -1,5 +1,7 @@
 name: syft-sbom-ci
 
+permissions: {}
+
 on:
   release:
     types: [created]
@@ -12,9 +14,11 @@ jobs:
     steps:
     - name: Checkout
       uses: actions/checkout@v4
+      with:
+        persist-credentials: false
         
     - name: Anchore SBOM Action
-      uses: anchore/sbom-action@v0.17.9
+      uses: anchore/sbom-action@9f7302141466aa6482940f15371237e9d9f4c34a # v0.19.0
       with:
          artifact-name: ${{ github.event.repository.name }}-spdx.json
 

.github/workflows/snyk.yml

@@ -1,17 +0,0 @@
-name: Snyk Monitor Scanning
-on:
- release:
-   types: [published]
- push:
-    branches:
-      - 'main'
-      - 'master'
- workflow_dispatch:
-
-jobs:
-  snyk-scan-ci:
-    # only run in grafana/tempo.
-    if: github.repository == 'grafana/tempo'    
-    uses: 'grafana/security-github-actions/.github/workflows/snyk_monitor.yml@main'
-    secrets:
-      SNYK_TOKEN: ${{ secrets.SNYK_TOKEN }}

.github/workflows/stale.yml

@@ -1,18 +1,20 @@
 name: Stale check
+
 on:
   workflow_dispatch: {}
   schedule:
     - cron: '0 0 * * *'
-permissions:
-  issues: write
-  pull-requests: write
+permissions: {}
 jobs:
   stale:
     # only run in grafana/tempo.
     if: github.repository == 'grafana/tempo'
     runs-on: ubuntu-24.04
+    permissions:
+      issues: write
+      pull-requests: write
     steps:
-      - uses: actions/stale@v9.0.0
+      - uses: actions/stale@v9.1.0
         with:
           close-issue-reason: not_planned
           operations-per-run: 100

.github/workflows/update-make-docs.yml

@@ -1,4 +1,7 @@
 name: Update `make docs` procedure
+
+permissions: {}
+
 on:
   schedule:
     - cron: '0 7 * * 1-5'
@@ -9,4 +12,6 @@ jobs:
     runs-on: ubuntu-24.04
     steps:
       - uses: actions/checkout@v4
-      - uses: grafana/writers-toolkit/update-make-docs@update-make-docs/v1
+        with:
+          persist-credentials: false
+      - uses: grafana/writers-toolkit/update-make-docs@6200164349ac8f24bab065ce3689dd8c9a093061

.github/zizmor.yml

@@ -0,0 +1,11 @@
+# This is also used as the default configuration for the Zizmor reusable workflow.
+# We need this file because we allow PRs from forks, and PRs from forks
+# can't access config files from shared-workflows repo.
+# This file is copied from: https://github.com/grafana/shared-workflows/blob/main/.github/zizmor.yml
+
+rules:
+  unpinned-uses:
+    config:
+      policies:
+        actions/*: any # trust GitHub
+        grafana/*: any # trust Grafana

.gitignore

@@ -8,7 +8,6 @@
 /bin
 /cmd/tempo-cli/tempo-cli
 /cmd/tempo-query/tempo-query
-/cmd/tempo-serverless/vendor/
 /dist
 /example/docker-compose/**/gcs-data/
 /example/docker-compose/**/tempo-data/
@@ -21,6 +20,7 @@
 /tempodb/encoding/benchmark_block
 private-key.key
 integration/e2e/e2e_integration_test[0-9]*
+integration/e2e/api/e2e_integration_test[0-9]*
 integration/e2e/deployments/e2e_integration_test[0-9]*
 .tempo.yaml
 /tmp

.golangci.yml

@@ -1,81 +1,66 @@
-# This file contains all available configuration options
-# with their default values.
-
-# options for analysis running
+version: "2"
 run:
-  # default concurrency is a available CPU number
   concurrency: 16
-
-  # timeout for analysis, e.g. 30s, 5m, default is 1m
-  timeout: 5m
-
-  # exit code when at least one issue was found, default is 1
   issues-exit-code: 1
-
-  # include test files or not, default is true
   tests: true
 
-  # list of build tags, all linters use it. Default is empty list.
-  build-tags:
-
-  # which dirs to skip: they won't be analyzed;
-  # can use regexp here: generated.*, regexp is applied on full path;
-  # default value is empty list, but next dirs are always skipped independently
-  # from this option's value:
-  #     vendor$, third_party$, testdata$, examples$, Godeps$, builtin$
-  skip-dirs:
-  # which files to skip: they will be analyzed, but issues from them
-  # won't be reported. Default value is empty list, but there is
-  # no need to include all autogenerated files, we confidently recognize
-  # autogenerated files. If it's not please let us know.
-  skip-files:
-# output configuration options
 output:
-  # colored-line-number|line-number|json|tab|checkstyle, default is "colored-line-number"
   formats:
-    - format: colored-line-number
-
-  # print lines of code with issue, default is true
-  print-issued-lines: true
-
-  # print linter name in the end of issue text, default is true
-  print-linter-name: true
+    text:
+      path: stdout
+      print-linter-name: true
+      print-issued-lines: true
 
 linters:
   enable:
-    - errcheck
     - errorlint
+    - forbidigo
+    - gocritic
+    - gosec
     - goconst
-    - gofmt
-    - goimports
-    - gosimple
-    - govet
-    - ineffassign
-    - megacheck
     - misspell
     - revive
-    - typecheck
     - unconvert
     - unparam
-    - unused
+  settings:
+    errcheck:
+      exclude-functions:
+        - (github.com/go-kit/kit/log.Logger).Log
+        - (github.com/go-kit/log.Logger).Log
+    errorlint:
+      errorf: false
+  exclusions:
+    generated: lax
+    presets:
+      - comments
+      - common-false-positives
+      - legacy
+      - std-error-handling
+    rules:
+      - linters:
+          - unused
+        path: integration/e2e
+      - path: (.+)\.go$
+        text: Error return value of .*log\.Logger\)\.Log\x60 is not checked
+      - path: (.+)\.go$
+        text: package github.com/golang/protobuf/proto is deprecated
+      - path: (.+)\.go$
+        text: package github.com/golang/protobuf/jsonpb is deprecated
+      - linters:
+          - goconst
+        path: (.+)_test\.go
+    paths:
+      - third_party$
+      - builtin$
+      - examples$
 
-linters-settings:
-  errcheck:
-    exclude-functions:
-      - (github.com/go-kit/kit/log.Logger).Log
-      - (github.com/go-kit/log.Logger).Log
-  errorlint:
-    errorf: false # don't check whether fmt.Errorf uses %w to wrap errors
-  goconst:
-    ignore-tests: true
-
-issues:
-  exclude:
-    - Error return value of .*log\.Logger\)\.Log\x60 is not checked
-    - package github.com/golang/protobuf/proto is deprecated
-    - package github.com/golang/protobuf/jsonpb is deprecated
-  exclude-rules:
-    # Exclude some linters from running on tests files.
-    - path: integration/e2e
-      linters:
-        - unused
+formatters:
+  enable:
+    - gofmt
+    - goimports
+  exclusions:
+    generated: lax
+    paths:
+      - third_party$
+      - builtin$
+      - examples$

.goreleaser.yml

@@ -11,7 +11,7 @@ builds:
     env:
       - CGO_ENABLED=0
     goos:
-      - darwin
+#      - darwin  re-enable when https://github.com/golang/go/issues/73617 is fixed
       - linux
       - windows
     goarch:
@@ -41,7 +41,7 @@ builds:
     env:
       - CGO_ENABLED=0
     goos:
-      - darwin
+#      - darwin  re-enable when https://github.com/golang/go/issues/73617 is fixed
       - linux
       - windows
     goarch:
@@ -71,7 +71,7 @@ builds:
     env:
       - CGO_ENABLED=0
     goos:
-      - darwin
+#      - darwin  re-enable when https://github.com/golang/go/issues/73617 is fixed
       - linux
       - windows
     goarch:

CHANGELOG.md

@@ -1,6 +1,143 @@
 ## main / unreleased
 
-# v2.7.0-rc.0
+* [BUGFIX] Fix ingester issue where a hash collision could lead to spans stored incorrectly [#5276](https://github.com/grafana/tempo/pull/5276) (@carles-grafana)
+
+# v2.8.0
+
+* [CHANGE] **BREAKING CHANGE** Change default http-listen-port from 80 to 3200 [#4960](https://github.com/grafana/tempo/pull/4960) (@martialblog)
+* [CHANGE] **BREAKING CHANGE** Upgrade OTEL Collector to v0.122.1 [#4893](https://github.com/grafana/tempo/pull/4893) (@javiermolinar)
+  The `name` dimension from `tempo_receiver_accepted_span` and `tempo_receiver_refused_spans` changes from `tempo/jaeger_receiver` to `jaeger/jaeger_receiver`
+* [CHANGE] **BREAKING CHANGE** Convert SLO metric `query_frontend_bytes_processed_per_second` from a histogram to a counter as it's more performant. [#4748](https://github.com/grafana/tempo/pull/4748) (@carles-grafana)
+* [CHANGE] **BREAKING CHANGE** Remove tempo serverless [#4599](https://github.com/grafana/tempo/pull/4599/) (@electron0zero)
+  Following config options are no longer valid, please remove them if you are using these in your tempo config:
+  ```
+  querier:
+      search:
+          prefer_self: <int>
+          external_hedge_requests_at: <duration>
+          external_hedge_requests_up_to:  <duration>
+          external_backend: <string>
+          google_cloud_run: <string>
+          external_endpoints: <array>
+  ```
+  Tempo serverless related metric `tempo_querier_external_endpoint_duration_seconds`,  `tempo_querier_external_endpoint_hedged_roundtrips_total` and `tempo_feature_enabled` are being removed.
+* [CHANGE] **BREAKING CHANGE** Removed `internal_error` as a reason from `tempo_discarded_spans_total`. [#4554](https://github.com/grafana/tempo/pull/4554) (@joe-elliott)
+* [CHANGE] **BREAKING CHANGE** Enforce max attribute size at event, link, and instrumentation scope. Make config per-tenant.
+  Renamed max_span_attr_byte to max_attribute_bytes [#4633](https://github.com/grafana/tempo/pull/4633) (@ie-pham)
+* [CHANGE] **BREAKING CHANGE** Removed otel jaeger exporter. [#4926](https://github.com/grafana/tempo/pull/4926) (@javiermolinar)
+* [CHANGE] Update query range error message [#4929](https://github.com/grafana/tempo/pull/4929) (@joey-grafana)
+* [CHANGE] Improve rate limit error message when traces size exceeds rate limit [#4986](https://github.com/grafana/tempo/pull/4986/) (@electron0zero)
+* [CHANGE] Tempo Query: replace opentracing-contrib/go-grpc by otelgrpc. [#4958](https://github.com/grafana/tempo/pull/4958) (@javiermolinar)
+* [CHANGE] Improve parquet readers io.ReaderAt compatibility [#4963](https://github.com/grafana/tempo/pull/4963) (@joe-elliott)
+* [CHANGE] Finish polling current tenants on poller shutdown [#4897](https://github.com/grafana/tempo/pull/4897) (@zalegrala)
+* [CHANGE] Set querier default level to INFO [#4943](https://github.com/grafana/tempo/pull/4943) (@javiermolinar)
+* [CHANGE] Change retention to honor compactor disablement [#5044](https://github.com/grafana/tempo/pull/5044) (@zalegrala)
+* [CHANGE] Continue on error in tempo-cli rewrite-blocks [#5043](https://github.com/grafana/tempo/pull/5043) (@zalegrala)
+* [CHANGE] Update jsonnet memcached and exporter image versions [#5056](https://github.com/grafana/tempo/pull/5056) (@zalegrala)
+* [CHANGE] Enforce max series in response for metrics queries with new query-frontend configuration [#4525](https://github.com/grafana/tempo/pull/4525) (@ie-pham)
+* [CHANGE] Update to go 1.24.3 [#5110](https://github.com/grafana/tempo/pull/5110) (@stoewer)
+* [CHANGE] Update to go 1.24.2 [#5057](https://github.com/grafana/tempo/pull/5057) [#5082](https://github.com/grafana/tempo/pull/5082) (@carsontham)
+* [CHANGE] Update to go 1.24.1 [#4704](https://github.com/grafana/tempo/pull/4704) (@ruslan-mikhailov) [#4793](https://github.com/grafana/tempo/pull/4793) (@javiermolinar)
+* [FEATURE] Added most_recent=true query hint to TraceQL to return most recent results. [#4238](https://github.com/grafana/tempo/pull/4238) (@joe-elliott)
+* [FEATURE] TraceQL metrics: sum_over_time [#4786](https://github.com/grafana/tempo/pull/4786) (@javiermolinar)
+* [FEATURE] Add support for topk and bottomk functions for TraceQL metrics [#4646](https://github.com/grafana/tempo/pull/4646/) (@electron0zero)
+* [FEATURE] TraceQL: add support for querying by parent span id [#4692](https://github.com/grafana/tempo/pull/4692) (@ie-pham)
+* [ENHANCEMENT] Add throughput SLO and metrics for the TraceByID endpoint. [#4668](https://github.com/grafana/tempo/pull/4668) (@carles-grafana)
+configurable via the throughput_bytes_slo field, and it will populate op="traces" label in slo and throughput metrics.
+* [ENHANCEMENT] Add ability to add artificial delay to push requests [#4716](https://github.com/grafana/tempo/pull/4716) [#4899](https://github.com/grafana/tempo/pull/4899) [#5035](https://github.com/grafana/tempo/pull/5035) (@yvrhdn, @mapno)
+* [ENHANCEMENT] tempo-vulture now generates spans with a parent, instead of only root spans [#5154](https://github.com/grafana/tempo/pull/5154) (@carles-grafana)
+* [ENHANCEMENT] Add default mutex and blocking values. [#4979](https://github.com/grafana/tempo/pull/4979) (@mattdurham) 
+* [ENHANCEMENT] Improve Tempo build options [#4755](https://github.com/grafana/tempo/pull/4755) (@stoewer)
+* [ENHANCEMENT] Rewrite traces using rebatching [#4690](https://github.com/grafana/tempo/pull/4690) (@stoewer @joe-elliott)
+* [ENHANCEMENT] Reorder span iterators [#4754](https://github.com/grafana/tempo/pull/4754) (@stoewer)
+* [ENHANCEMENT] Update minio to version [#4341](https://github.com/grafana/tempo/pull/4568) (@javiermolinar)
+* [ENHANCEMENT] Prevent queries in the ingester from blocking flushing traces to disk and memory spikes. [#4483](https://github.com/grafana/tempo/pull/4483) (@joe-elliott)
+* [ENHANCEMENT] Update tempo operational dashboard for new block-builder and v2 traces api [#4559](https://github.com/grafana/tempo/pull/4559) (@mdisibio)
+* [ENHANCEMENT] Improve metrics-generator performance and stability by applying queue back pressure and concurrency [#4721](https://github.com/grafana/tempo/pull/4721) (@mdisibio)
+* [ENHANCEMENT] Improve block-builder performance by flushing blocks concurrently [#4565](https://github.com/grafana/tempo/pull/4565) (@mdisibio)
+* [ENHANCEMENT] Improve block-builder performance [#4596](https://github.com/grafana/tempo/pull/4596) (@mdisibio)
+* [ENHANCEMENT] Improve block-builder performance by not using WAL stage [#4647](https://github.com/grafana/tempo/pull/4647) [#4671](https://github.com/grafana/tempo/pull/4671) (@mdisibio)
+* [ENHANCEMENT] Export new `tempo_ingest_group_partition_lag` metric from block-builders and metrics-generators [#4571](https://github.com/grafana/tempo/pull/4571) (@mdisibio)
+* [ENHANCEMENT] Overall iterator performance improvement by using max definition level to ignore parts of the RowNumber while nexting. [#4753](https://github.com/grafana/tempo/pull/4753) (@joe-elliott)
+* [ENHANCEMENT] Use distroless base container images for improved security [#4556](https://github.com/grafana/tempo/pull/4556) (@carles-grafana)
+* [ENHANCEMENT] Rhythm: add block builder to resources dashboard [#4556](https://github.com/grafana/tempo/pull/4669) (@javiermolinar)
+* [ENHANCEMENT] Upgrade prometheus to version 3.1.0 [#4805](https://github.com/grafana/tempo/pull/4805) (@javiermolinar)
+* [ENHANCEMENT] Update dskit to latest version [#4681](https://github.com/grafana/tempo/pull/4681) (@javiermolinar) [#4865](https://github.com/grafana/tempo/pull/4865) (@javiermolinar)
+* [ENHANCEMENT] Increase query-frontend default batch size [#4844](https://github.com/grafana/tempo/pull/4844) (@javiermolinar) 
+* [ENHANCEMENT] Improve TraceQL perf by reverting EqualRowNumber to an inlineable function.[#4705](https://github.com/grafana/tempo/pull/4705) (@joe-elliott)
+* [ENHANCEMENT] Rhythm: Implement MaxBytesPerCycle [#4835](https://github.com/grafana/tempo/pull/4835) (@javiermolinar)
+* [ENHANCEMENT] Rhythm: fair partition consumption in blockbuilders [#4655](https://github.com/grafana/tempo/pull/4655) (@javiermolinar)
+* [ENHANCEMENT] Rhythm: retry on commit error [#4874](https://github.com/grafana/tempo/pull/4874) (@javiermolinar)
+* [ENHANCEMENT] Skip creating one span-traces for every pushed spans in metrics generator [#4844](https://github.com/grafana/tempo/pull/4844) (@javiermolinar)
+* [ENHANCEMENT] Improve Tempo / Writes dashboard by adding a kafka panel [#4947](https://github.com/grafana/tempo/pull/4947) (@javiermolinar)
+* [ENHANCEMENT] Improve memcached memory usage by pooling buffers [#4970](https://github.com/grafana/tempo/pull/4970) (@joe-elliott)
+* [ENHANCEMENT] metrics-generator: allow skipping localblocks and consuming from a different source of data [#4686](https://github.com/grafana/tempo/pull/4686) (@flxbk)
+* [ENHANCEMENT] compactor: restore dedicated columns logging for completed blocks [#4832](https://github.com/grafana/tempo/pull/4832) (@edgarkz)
+* [ENHANCEMENT] Compactor: pooling changes to reduce memory usage [#4985](https://github.com/grafana/tempo/pull/4985) (@mdisibio)
+* [ENHANCEMENT] distributor: add IPv6 support [#4840](https://github.com/grafana/tempo/pull/4840) (@gjacquet)
+* [ENHANCEMENT] Support TraceQL Metrics checks in Vulture [#4886](https://github.com/grafana/tempo/pull/4886) (@ruslan-mikhailov)
+* [ENHANCEMENT] Add memcached to the resources dashboard [#5049](https://github.com/grafana/tempo/pull/5049) (@javiermolinar)
+* [ENHANCEMENT] Include partition owned metric for blockbuilders [#5042](https://github.com/grafana/tempo/pull/5042) (@javiermolinar)
+* [ENHANCEMENT] Query-frontend: logs add msg to the log line [#4975](https://github.com/grafana/tempo/pull/4975) (@jmichalek132)
+* [ENHANCEMENT] Host Info Processor: track host identifying resource attribute in metric [#5152](https://github.com/grafana/tempo/pull/5152) (@rlankfo)
+* [ENHANCEMENT] Vulture checks recent traces [#5157](https://github.com/grafana/tempo/pull/5157) (@ruslan-mikhailov)
+* [ENHANCEMENT] Add jitter in backendworker to avoid thundering herd from workers [#5150](https://github.com/grafana/tempo/pull/5150) (@electron0zero)
+* [ENHANCEMENT] TraceQL Metrics: distribute exemplars over time [#5158](https://github.com/grafana/tempo/pull/5158) (@ruslan-mikhailov)
+* [ENHANCEMENT] TraceQL Metrics: hard limit number of exemplars [#5158](https://github.com/grafana/tempo/pull/5158) (@ruslan-mikhailov)
+* [BUGFIX] Choose a default step for a gRPC streaming query range request if none is provided. [#4546](https://github.com/grafana/tempo/pull/4576) (@joe-elliott)
+  Correctly copy exemplars for metrics like `| rate()` when gRPC streaming.
+* [BUGFIX] Make comparison to nil symmetric [#4869](https://github.com/grafana/tempo/pull/4869) (@stoewer)
+* [BUGFIX] Fix behavior for queries like {.foo && true} and {.foo || false} [#4855](https://github.com/grafana/tempo/pull/4855) (@stoewer)
+* [BUGFIX] Fix performance bottleneck and file cleanup in block builder [#4550](https://github.com/grafana/tempo/pull/4550) (@mdisibio)
+* [BUGFIX] Add object name to cache key in ReadRange [#4982](https://github.com/grafana/tempo/pull/4982) (@joe-elliott)
+* [BUGFIX] TraceQL incorrect results for additional spanset filters after a select operation [#4600](https://github.com/grafana/tempo/pull/4600) (@mdisibio)
+* [BUGFIX] TraceQL metrics incorrect results for queries with multiple filters that reside in non-dedicated columns that also group by the same variable [#4887](https://github.com/grafana/tempo/pull/4887) (@mdisibio)
+* [BUGFIX] TraceQL results caching bug for floats ending in .0 [#4539](https://github.com/grafana/tempo/pull/4539) (@carles-grafana)
+* [BUGFIX] Rhythm: fix sorting order for partition consumption [#4747](https://github.com/grafana/tempo/pull/4747) (@javiermolinar)
+* [BUGFIX] Rhythm: fix block builder to not reuse a block ID if it was already flushed, to prevent read errors [#4872](https://github.com/grafana/tempo/pull/4872) (@mdisibio)
+* [BUGFIX] Rhythm: fix ingester to still clear old blocks when not flushing to object storage [#5005](https://github.com/grafana/tempo/pull/5005) (@mdisibio)
+* [BUGFIX] Fix rare panic during compaction. [#4915](https://github.com/grafana/tempo/pull/4915) (@joe-elliott)
+* [BUGFIX] Fix metrics streaming for all non-trivial metrics [#4624](https://github.com/grafana/tempo/pull/4624) (@joe-elliott)
+* [BUGFIX] Fix starting consuming log [#4539](https://github.com/grafana/tempo/pull/4539) (@javiermolinar)
+* [BUGFIX] TraceQL: Fix behavior of {} >> { span.attr-that-doesnt-exist != "foo" } [#5007](https://github.com/grafana/tempo/pull/5007) (@joe-elliott)
+* [BUGFIX] Rhythm - fix adjustment of the start and end range for livetraces blocks [#4746](https://github.com/grafana/tempo/pull/4746) (@javiermolinar)
+* [BUGFIX] Return the operand as the only value if the tag is already filtered in the query [#4673](https://github.com/grafana/tempo/pull/4673) (@mapno)
+* [BUGFIX] Fix flaky test [#4787](https://github.com/grafana/tempo/pull/4787) [#4995](https://github.com/grafana/tempo/pull/4995) (@javiermolinar)
+* [BUGFIX] Fix flaky ingester test [#4846](https://github.com/grafana/tempo/pull/4846) (@javiermolinar)
+* [BUGFIX] Include cost attribution when converting from default config to legacy one [#4787](https://github.com/grafana/tempo/pull/4937) (@javiermolinar)
+* [BUGFIX] Fix memcached settings for docker compose example [#4346](https://github.com/grafana/tempo/pull/4695) (@ruslan-mikhailov)
+* [BUGFIX] Update memcached to respect cancelled context to prevent panic [#5041](https://github.com/grafana/tempo/pull/5041) (@joe-elliott)
+* [BUGFIX] Fix frontend cache key generation for TraceQL Metrics queries to prevent collisions. [#5017](https://github.com/grafana/tempo/pull/5017) (@joe-elliott)
+* [BUGFIX] Fix setting processors in user configurations overrides via API [#4741](https://github.com/grafana/tempo/pull/4741) (@ruslan-mikhailov)
+* [BUGFIX] Fix panic on startup [#4744](https://github.com/grafana/tempo/pull/4744) (@ruslan-mikhailov)
+* [BUGFIX] Fix intrinsic tag lookups dropped when max tag lookup response size is exceeded [#4784](https://github.com/grafana/tempo/pull/4784) (@mdisibio)
+* [BUGFIX] Use canonical bytesize when parsing partition.id in the distributor [#5033](https://github.com/grafana/tempo/pull/5033) (@javiermolinar)
+* [BUGFIX] Measure the write latency on append errors [#5034](https://github.com/grafana/tempo/pull/5034) (@javiermolinar)
+* [BUGFIX] Correctly cache frontend jobs for query range (TraceQL Metrics). [#4771](https://github.com/grafana/tempo/pull/4771) (@joe-elliott)
+* [BUGFIX] Fix error propagation in the SyncIterator. [#5045](https://github.com/grafana/tempo/pull/5045) (@joe-elliott)
+* [BUGFIX] Apply the artificial latency even on pushTrace errors  [#5153](https://github.com/grafana/tempo/pull/5153) (@javiermolinar)
+* [BUGFIX] Various edge case fixes for query range (TraceQL Metrics) [#4962](https://github.com/grafana/tempo/pull/4962) (@ruslan-mikhailov)
+* [BUGFIX] Fix mixin to include otlp_v1_traces http write route [#5072](https://github.com/grafana/tempo/pull/5072) (@mdisibio)
+* [BUGFIX] Fix `TempoBlockListRisingQuickly` alert grouping. [#4876](https://github.com/grafana/tempo/pull/4876) (@mapno)
+* [BUGFIX] Reset `SkipMetricsGeneration` before reuse. [#5117](https://github.com/grafana/tempo/pull/5117) (@flxbk)
+* [BUGFIX] Fix metrics generator host info processor overrides config. [#5118](https://github.com/grafana/tempo/pull/5118) (@rlankfo)
+* [BUGFIX] Fix metrics generator target_info to skip attributes with no name to prevent downstream errors [#5148](https://github.com/grafana/tempo/pull/5148) (@mdisibio)
+* [BUGFIX] Fix for queried number of exemplars (TraceQL Metrics) [#5115](https://github.com/grafana/tempo/pull/5115) (@ruslan-mikhailov)
+* [BUGFIX] Excluded nestedSetParent and other values from compare() function [#5196](https://github.com/grafana/tempo/pull/5196) (@mdisibio)
+* [BUGFIX] Fix distributor issue where a hash collision could lead to spans stored incorrectly [#5186](https://github.com/grafana/tempo/pull/5186) (@mdisibio)
+* [BUGFIX] Fix structural metrics rate by aggregation [#5204](https://github.com/grafana/tempo/pull/5204) (@zalegrala)
+* [BUGFIX] TraceQL Metrics: right exemplars for histogram and quantiles [#5145](https://github.com/grafana/tempo/pull/5145) (@ruslan-mikhailov)
+
+# v2.7.2
+
+* [BUGFIX] Fix rare panic that occurred when a querier modified results from ingesters/generators while they were being marshalled to proto.  [#4790](https://github.com/grafana/tempo/pull/4790) (@joe-elliott)
+  This bug also impacted query correctness on recent trace data by returning incomplete results before they were ready.
+
+# v2.7.1
+
+* [CHANGE] Default to snappy compression for all gRPC communications internal to Tempo. We feel this is a nice balance of resource usage and network traffic. For a discussion on alternatives see https://github.com/grafana/tempo/discussions/4683. [#4696](https://github.com/grafana/tempo/pull/4696) (@joe-elliott)
+
+# v2.7.0
 
 * [CHANGE] Disable gRPC compression in the querier and distributor for performance reasons [#4429](https://github.com/grafana/tempo/pull/4429) (@carles-grafana)
   If you would like to re-enable it, we recommend 'snappy'. Use the following settings:
@@ -16,47 +153,54 @@ querier:
         grpc_client_config:
             grpc_compression: "snappy"
 ```
-* [CHANGE] **BREAKING CHANGE** Add maximum spans per span set. Users can set `max_spans_per_span_set` to 0 to obtain the old behavior. [#4275](https://github.com/grafana/tempo/pull/4383) (@carles-grafana)
 * [CHANGE] slo: include request cancellations within SLO [#4355] (https://github.com/grafana/tempo/pull/4355) (@electron0zero)
   request cancellations are exposed under `result` label in `tempo_query_frontend_queries_total` and `tempo_query_frontend_queries_within_slo_total` with `completed` or `canceled` values to differentiate between completed and canceled requests.
 * [CHANGE] update default config values to better align with production workloads [#4340](https://github.com/grafana/tempo/pull/4340) (@electron0zero)
-* [CHANGE] Add query-frontend limit for max length of query expression [##4397](https://github.com/grafana/tempo/pull/4397) (@electron0zero)
 * [CHANGE] fix deprecation warning by switching to DoBatchWithOptions [#4343](https://github.com/grafana/tempo/pull/4343) (@dastrobu)
 * [CHANGE] **BREAKING CHANGE** The Tempo serverless is now deprecated and will be removed in an upcoming release [#4017](https://github.com/grafana/tempo/pull/4017/) @electron0zero
-* [CHANGE] **BREAKING CHANGE** Change the AWS Lambda serverless build tooling output from "main" to "bootstrap". Refer to https://aws.amazon.com/blogs/compute/migrating-aws-lambda-functions-from-the-go1-x-runtime-to-the-custom-runtime-on-amazon-linux-2/ for migration steps [#3852](https://github.com/grafana/tempo/pull/3852) (@zatlodan)
-* [CHANGE] Add throughput and SLO metrics in the tags and tag values endpoints [#4148](https://github.com/grafana/tempo/pull/4148) (@electron0zero)
 * [CHANGE] tempo-cli: add support for /api/v2/traces endpoint [#4127](https://github.com/grafana/tempo/pull/4127) (@electron0zero)
-  **BREAKING CHANGE** The `tempo-cli` now uses the `/api/v2/traces` endpoint by default, 
+  **BREAKING CHANGE** The `tempo-cli` now uses the `/api/v2/traces` endpoint by default,
   please use `--v1` flag to use `/api/traces` endpoint, which was the default in previous versions.
 * [CHANGE] TraceByID: don't allow concurrent_shards greater than query_shards. [#4074](https://github.com/grafana/tempo/pull/4074) (@electron0zero)
 * [CHANGE] **BREAKING CHANGE** The dynamic injection of X-Scope-OrgID header for metrics generator remote-writes is changed. If the header is aleady set in per-tenant overrides or global tempo configuration, then it is honored and not overwritten. [#4021](https://github.com/grafana/tempo/pull/4021) (@mdisibio)
-* [CHANGE] **BREAKING CHANGE** Migrate from OpenTracing to OpenTelemetry instrumentation. Removed the `use_otel_tracer` configuration option. Use the OpenTelemetry environment variables to configure the span exporter [#3646](https://github.com/grafana/tempo/pull/3646) (@andreasgerstmayr)
+* [CHANGE] **BREAKING CHANGE** Migrate from OpenTracing to OpenTelemetry instrumentation. Removed the `use_otel_tracer` configuration option. Use the OpenTelemetry environment variables to configure the span exporter [#4028](https://github.com/grafana/tempo/pull/4028),[#3646](https://github.com/grafana/tempo/pull/3646) (@andreasgerstmayr)
   To continue using the Jaeger exporter, use the following environment variable: `OTEL_TRACES_EXPORTER=jaeger`.
 * [CHANGE] No longer send the final diff in GRPC streaming. Instead we rely on the streamed intermediate results. [#4062](https://github.com/grafana/tempo/pull/4062) (@joe-elliott)
 * [CHANGE] Update Go to 1.23.3 [#4146](https://github.com/grafana/tempo/pull/4146) [#4147](https://github.com/grafana/tempo/pull/4147) [#4380](https://github.com/grafana/tempo/pull/4380) (@javiermolinar @mdisibio)
-* [CHANGE] TraceQL: Add range condition for byte predicates [#4198](https://github.com/grafana/tempo/pull/4198) (@ie-pham)
 * [CHANGE] Return 422 for TRACE_TOO_LARGE queries [#4160](https://github.com/grafana/tempo/pull/4160) (@zalegrala)
 * [CHANGE] Upgrade OTEL sdk to reduce allocs [#4243](https://github.com/grafana/tempo/pull/4243) (@joe-elliott)
 * [CHANGE] Update Alpine image version to 3.21 [#4504](https://github.com/grafana/tempo/pull/4504) (@mdisibio)
 * [CHANGE] Tighten file permissions [#4251](https://github.com/grafana/tempo/pull/4251) (@zalegrala)
 * [CHANGE] Drop max live traces log message and rate limit trace too large. [#4418](https://github.com/grafana/tempo/pull/4418) (@joe-elliott)
 * [CHANGE] Update the Open-Telemetry dependencies to v0.116.0 [#4466](https://github.com/grafana/tempo/pull/4466) (@yvrhdn)
-* [CHANGE] Replace `cespare/xxhash` with `cespare/xxhash/v2` [#4485](https://github.com/grafana/tempo/pull/4485) (@Juneezee)
   **BREAKING CHANGE** After this update the Open-Telemetry Collector receiver will connect to `localhost` instead of all interfaces `0.0.0.0`.
   Due to this, Tempo installations running inside Docker have to update the address they listen.
   For more details on this change, see [#4465](https://github.com/grafana/tempo/issues/4465)
   For more information about the security risk this change addresses, see https://github.com/open-telemetry/opentelemetry-collector/blob/main/docs/security-best-practices.md#safeguards-against-denial-of-service-attacks
+* [CHANGE] Replace `cespare/xxhash` with `cespare/xxhash/v2` [#4485](https://github.com/grafana/tempo/pull/4485) (@Juneezee)
+* [CHANGE] chore: remove gofakeit dependency [#4274](https://github.com/grafana/tempo/pull/4274) (@javiermolinar)
+* [CHANGE] Chore: delete spanlogger. [4312](https://github.com/grafana/tempo/pull/4312) (@javiermolinar)
+* [CHANGE] Update Azurite image. [#4298](https://github.com/grafana/tempo/pull/4464) (@javiermolinar)
+* [CHANGE] Update golang.org/x/crypto [#4474](https://github.com/grafana/tempo/pull/4474) (@javiermolinar)
+* [CHANGE] Distributor shim: add test verifying receiver works (including metrics) [#4477](https://github.com/grafana/tempo/pull/4477) (@yvrhdn)
+* [CHANGE] Replace Grafana Agent example by Grafana Alloy[#4030](https://github.com/grafana/tempo/pull/4030) (@javiermolinar)
+* [CHANGE] Update to the latest dskit [#4341](https://github.com/grafana/tempo/pull/4341) (@dastrobu)
 * [FEATURE] tempo-cli: support dropping multiple traces in a single operation [#4266](https://github.com/grafana/tempo/pull/4266) (@ndk)
 * [FEATURE] Discarded span logging `log_discarded_spans` [#3957](https://github.com/grafana/tempo/issues/3957) (@dastrobu)
 * [FEATURE] TraceQL support for instrumentation scope [#3967](https://github.com/grafana/tempo/pull/3967) (@ie-pham)
 * [FEATURE] Export cost attribution usage metrics from distributor [#4162](https://github.com/grafana/tempo/pull/4162) (@mdisibio)
 * [FEATURE] TraceQL metrics: avg_over_time [#4073](https://github.com/grafana/tempo/pull/4073) (@javiermolinar)
 * [FEATURE] Limit tags and tag values search [#4320](https://github.com/grafana/tempo/pull/4320) (@javiermolinar)
+* [FEATURE] TraceQL metrics queries: add min_over_time  [#3975](https://github.com/grafana/tempo/pull/3975) (@javiermolinar)
+* [FEATURE] TraceQL metrics queries: add max_over_time  [#4065](https://github.com/grafana/tempo/pull/4065) (@javiermolinar)
+* [ENHANCEMENT] TraceQL: Add range condition for byte predicates [#4198](https://github.com/grafana/tempo/pull/4198) (@ie-pham)
+* [ENHANCEMENT] Add throughput and SLO metrics in the tags and tag values endpoints [#4148](https://github.com/grafana/tempo/pull/4148) (@electron0zero)
+* [ENHANCEMENT] **BREAKING CHANGE** Add maximum spans per span set. Users can set `max_spans_per_span_set` to 0 to obtain the old behavior. [#4275](https://github.com/grafana/tempo/pull/4383) (@carles-grafana)
+* [ENHANCEMENT] Add query-frontend limit for max length of query expression [#4397](https://github.com/grafana/tempo/pull/4397) (@electron0zero)
 * [ENHANCEMENT] distributor: return trace id length when it is invalid [#4407](https://github.com/grafana/tempo/pull/4407) (@carles-grafana)
-* [ENHANCEMENT] Update to the latest dskit [#4341](https://github.com/grafana/tempo/pull/4341) (@dastrobu)
 * [ENHANCEMENT] Changed log level from INFO to DEBUG for the TempoDB Find operation using traceId to reduce excessive/unwanted logs in log search. [#4179](https://github.com/grafana/tempo/pull/4179) (@Aki0x137)
 * [ENHANCEMENT] Pushdown collection of results from generators in the querier [#4119](https://github.com/grafana/tempo/pull/4119) (@electron0zero)
-* [ENHANCEMENT] The span multiplier now also sources its value from the resource attributes. [#4210](https://github.com/grafana/tempo/pull/4210)
+* [ENHANCEMENT] The span multiplier now also sources its value from the resource attributes. [#4210](https://github.com/grafana/tempo/pull/4210) (@JunhoeKim)
 * [ENHANCEMENT] TraceQL: Attribute iterators collect matched array values [#3867](https://github.com/grafana/tempo/pull/3867) (@electron0zero, @stoewer)
 * [ENHANCEMENT] Allow returning partial traces that exceed the MaxBytes limit for V2 [#3941](https://github.com/grafana/tempo/pull/3941) (@javiermolinar)
 * [ENHANCEMENT] Added new middleware to validate request query values [#3993](https://github.com/grafana/tempo/pull/3993) (@javiermolinar)
@@ -65,30 +209,21 @@ querier:
 * [ENHANCEMENT] Update metrics-generator config in Tempo distributed docker compose example to serve TraceQL metrics [#4003](https://github.com/grafana/tempo/pull/4003) (@javiermolinar)
 * [ENHANCEMENT] Reduce allocs related to marshalling dedicated columns repeatedly in the query frontend. [#4007](https://github.com/grafana/tempo/pull/4007) (@joe-elliott)
 * [ENHANCEMENT] Improve performance of TraceQL queries [#4114](https://github.com/grafana/tempo/pull/4114) (@mdisibio)
-* [ENHANCEMENT] Improve performance of TraceQL queries [#4163](https://github.com/grafana/tempo/pull/4163) (@mdisibio) 
+* [ENHANCEMENT] Improve performance of TraceQL queries [#4163](https://github.com/grafana/tempo/pull/4163) (@mdisibio)
 * [ENHANCEMENT] Improve performance of some TraceQL queries using select() operation [#4438](https://github.com/grafana/tempo/pull/4438) (@mdisibio)
 * [ENHANCEMENT] Reduce memory usage of classic histograms in the span-metrics and service-graphs processors [#4232](https://github.com/grafana/tempo/pull/4232) (@mdisibio)
 * [ENHANCEMENT] Implement simple Fetch by key for cache items  [#4032](https://github.com/grafana/tempo/pull/4032) (@javiermolinar)
-* [ENHANCEMENT] Replace Grafana Agent example by Grafana Alloy[#4030](https://github.com/grafana/tempo/pull/4030) (@javiermolinar)
-* [ENHANCEMENT] Support exporting internal Tempo traces via OTLP exporter when `use_otel_tracer` is enabled. Use the OpenTelemetry SDK environment variables to configure the span exporter. [#4028](https://github.com/grafana/tempo/pull/4028) (@andreasgerstmayr)
-* [ENHANCEMENT] TraceQL metrics queries: add min_over_time  [#3975](https://github.com/grafana/tempo/pull/3975) (@javiermolinar)
-* [ENHANCEMENT] TraceQL metrics queries: add max_over_time  [#4065](https://github.com/grafana/tempo/pull/4065) (@javiermolinar)
-* [ENHANCEMENT] Write tenantindex as proto and json with a prefernce for proto [#4072](https://github.com/grafana/tempo/pull/4072) (@zalegrala)
-* [ENHANCEMENT] Pool zstd encoding/decoding for tmepodb/backend [#4208](https://github.com/grafana/tempo/pull/4208) (@zalegrala)
-* [ENHANCEMENT] The span multiplier now also sources its value from the resource attributes. [#4210](https://github.com/grafana/tempo/pull/4210)
-* [ENHANCEMENT] Changed log level from INFO to DEBUG for the TempoDB Find operation using traceId to reduce excessive/unwanted logs in log search. [#4179](https://github.com/grafana/tempo/pull/4179) (@Aki0x137)
-* [ENHANCEMENT] Pushdown collection of results from generators in the querier [#4119](https://github.com/grafana/tempo/pull/4119) (@electron0zero)
+* [ENHANCEMENT] Write tenantindex as proto and json with a preference for proto [#4072](https://github.com/grafana/tempo/pull/4072) (@zalegrala)
+* [ENHANCEMENT] Pool zstd encoding/decoding for tempodb/backend [#4208](https://github.com/grafana/tempo/pull/4208) (@zalegrala)
 * [ENHANCEMENT] Send semver version in api/stattus/buildinfo for cloud deployments [#4110](https://github.com/grafana/tempo/pull/4110) [@Aki0x137]
 * [ENHANCEMENT] Add completed block validation on startup.[#4256](https://github.com/grafana/tempo/pull/4256) (@joe-elliott)
 * [ENHANCEMENT] Speedup DistinctString and ScopedDistinctString collectors [#4109](https://github.com/grafana/tempo/pull/4109) (@electron0zero)
 * [ENHANCEMENT] Speedup collection of results from ingesters in the querier [#4100](https://github.com/grafana/tempo/pull/4100) (@electron0zero)
 * [ENHANCEMENT] Speedup DistinctValue collector and exit early for ingesters [#4104](https://github.com/grafana/tempo/pull/4104) (@electron0zero)
 * [ENHANCEMENT] Add disk caching in ingester SearchTagValuesV2 for completed blocks [#4069](https://github.com/grafana/tempo/pull/4069) (@electron0zero)
-* [ENHANCEMENT] chore: remove gofakeit dependency [#4274](https://github.com/grafana/tempo/pull/4274) (@javiermolinar)
 * [ENHANCEMENT] Add a max flush attempts and metric to the metrics generator [#4254](https://github.com/grafana/tempo/pull/4254) (@joe-elliott)
 * [ENHANCEMENT] Collection of query-frontend changes to reduce allocs. [#4242](https://github.com/grafana/tempo/pull/4242) (@joe-elliott)
 * [ENHANCEMENT] Added `insecure-skip-verify` option in tempo-cli to skip SSL certificate validation when connecting to the S3 backend. [#4259](https://github.com/grafana/tempo/pull/4259) (@faridtmammadov)
-* [ENHANCEMENT] Chore: delete spanlogger. [4312](https://github.com/grafana/tempo/pull/4312) (@javiermolinar)
 * [ENHANCEMENT] Add `invalid_utf8` to reasons spanmetrics will discard spans. [#4293](https://github.com/grafana/tempo/pull/4293) (@zalegrala)
 * [ENHANCEMENT] Reduce frontend and querier allocations by dropping HTTP headers early in the pipeline. [#4298](https://github.com/grafana/tempo/pull/4298) (@joe-elliott)
 * [ENHANCEMENT] Reduce ingester working set by improving prelloc behavior. [#4344](https://github.com/grafana/tempo/pull/4344),[#4369](https://github.com/grafana/tempo/pull/4369)  (@joe-elliott)
@@ -98,9 +233,6 @@ querier:
 * [ENHANCEMENT] Reuse generator code to better refuse "too large" traces. [#4365](https://github.com/grafana/tempo/pull/4365) (@joe-elliott)
   This will cause the ingester to more aggressively and correctly refuse traces. Also added two metrics to better track bytes consumed per tenant in the ingester.
   `tempo_metrics_generator_live_trace_bytes` and `tempo_ingester_live_trace_bytes`.
-* [ENHANCEMENT] Update Azurite image. [#4298](https://github.com/grafana/tempo/pull/4464) (@javiermolinar)
-* [ENHANCEMENT] Update golang.org/x/crypto [#4474](https://github.com/grafana/tempo/pull/4474) (@javiermolinar)
-* [ENHANCEMENT] Distributor shim: add test verifying receiver works (including metrics) [#4477](https://github.com/grafana/tempo/pull/4477) (@yvrhdn)
 * [ENHANCEMENT] Reduce goroutines in all non-querier components. [#4484](https://github.com/grafana/tempo/pull/4484) (@joe-elliott)
 * [ENHANCEMENT] Add option to enforce max span attribute size [#4335](https://github.com/grafana/tempo/pull/4335) (@ie-pham)
 * [BUGFIX] Handle invalid TraceQL query filter in tag values v2 disk cache [#4392](https://github.com/grafana/tempo/pull/4392) (@electron0zero)
@@ -119,7 +251,7 @@ querier:
 * [BUGFIX] Utilize S3Pass and S3User parameters in tempo-cli options, which were previously unused in the code. [#4259](https://github.com/grafana/tempo/pull/4259) (@faridtmammadov)
 * [BUGFIX] Fixed an issue in the generator where the first batch was counted 2x against a traces size. [#4365](https://github.com/grafana/tempo/pull/4365) (@joe-elliott)
 * [BUGFIX] Fix compaction bug in SingleBinaryMode that could lead to 2x, 3x, etc TraceQL metrics results [#4446](https://github.com/grafana/tempo/pull/4446) (@mdisibio)
-* [BUGFIX] Unstable compactors can occassionally duplicate data. Check for job ownership during compaction and cancel a job if ownership changes. [#4420](https://github.com/grafana/tempo/pull/4420) (@joe-elliott)
+* [BUGFIX] Unstable compactors can occasionally duplicate data. Check for job ownership during compaction and cancel a job if ownership changes. [#4420](https://github.com/grafana/tempo/pull/4420) (@joe-elliott)
 
 # v2.6.1
 
@@ -175,12 +307,11 @@ querier:
 * [ENHANCEMENT] Add vParquet4 support to the tempo-cli analyse blocks command [#3868](https://github.com/grafana/tempo/pull/3868) (@stoewer)
 * [ENHANCEMENT] Improve trace id lookup from Tempo Vulture by selecting a date range [#3874](https://github.com/grafana/tempo/pull/3874) (@javiermolinar)
 * [ENHANCEMENT] Add native histograms for internal metrics[#3870](https://github.com/grafana/tempo/pull/3870) (@zalegrala)
-* [ENHANCEMENT] Expose availability-zone as a cli flag in ingester [#3881](https://github.com/grafana/tempo/pull/3881)
+* [ENHANCEMENT] Expose availability-zone as a cli flag in ingester [#3881](https://github.com/grafana/tempo/pull/3881) (@KyriosGN0)
 * [ENHANCEMENT] Rename batches property of Trace to ResourceSpans to be OTEL compatible [#3895](https://github.com/grafana/tempo/pull/3895)
 * [ENHANCEMENT] Reduce memory consumption of query-frontend[#3888](https://github.com/grafana/tempo/pull/3888) (@joe-elliott)
 * [ENHANCEMENT] Reduce log level verbosity for e2e tests[#3900](https://github.com/grafana/tempo/pull/3900) (@javiermolinar)
 * [ENHANCEMENT] Added new Traces api V2[#3912](https://github.com/grafana/tempo/pull/3912) (@javiermolinar)
-* [ENHANCEMENT] Update to the latest dskit [#3915](https://github.com/grafana/tempo/pull/3915) (@andreasgerstmayr)
 * [ENHANCEMENT] Reduce allocs building queriers sharded requests [#3932](https://github.com/grafana/tempo/pull/3932) (@javiermolinar)
 * [ENHANCEMENT] Allow compaction disablement per-tenant [#3965](https://github.com/grafana/tempo/pull/3965) (@zalegrala)
 * [ENHANCEMENT] Implement polling tenants concurrently [#3647](https://github.com/grafana/tempo/pull/3647) (@zalegrala)
@@ -217,7 +348,7 @@ querier:
   properly.  A jsonnet example of an init container is included with the PR.
   This impacts all users of the `grafana/tempo` Docker image.
 * [CHANGE] Remove vParquet encoding [#3663](https://github.com/grafana/tempo/pull/3663) (@mdisibio)
-  **BREAKING CHANGE** In the last release vParquet (the first version) was deprecated and blocked from writes. Now, it's 
+  **BREAKING CHANGE** In the last release vParquet (the first version) was deprecated and blocked from writes. Now, it's
   removed entirely.  It will no longer be recognized as a valid encoding and cannot read any remaining vParquet blocks. Installations
   running with historical defaults should not require any changes as the default has been migrated for several releases. Installations
   with storage settings pinned to vParquet must run a previous release configured for vParquet2 or higher until all existing vParquet (1) blocks
@@ -230,7 +361,7 @@ querier:
 * [CHANGE] Add golangci to the tools image and update `lint` make target [#3610](https://github.com/grafana/tempo/pull/3610) (@zalegrala)
 * [CHANGE] Update Alpine image version to 3.20 [#3710](https://github.com/grafana/tempo/pull/3710) (@joe-elliott)
 * [FEATURE] Add TLS support for Memcached Client [#3585](https://github.com/grafana/tempo/pull/3585) (@sonisr)
-* [FEATURE] TraceQL metrics queries: add quantile_over_time [#3605](https://github.com/grafana/tempo/pull/3605) [#3633](https://github.com/grafana/tempo/pull/3633) (@mdisibio) 
+* [FEATURE] TraceQL metrics queries: add quantile_over_time [#3605](https://github.com/grafana/tempo/pull/3605) [#3633](https://github.com/grafana/tempo/pull/3633) (@mdisibio)
 * [FEATURE] TraceQL metrics queries: add histogram_over_time [#3644](https://github.com/grafana/tempo/pull/3644) (@mdisibio)
 * [FEATURE] Added gRPC streaming endpoints for Tempo APIs.
   * Added gRPC streaming endpoints for all tag queries. [#3460](https://github.com/grafana/tempo/pull/3460) (@joe-elliott)
@@ -363,13 +494,13 @@ querier:
 * [CHANGE] Upgrade from deprecated [azure-storage-blob-go](https://github.com/Azure/azure-storage-blob-go) SDK to [azure-sdk-for-go](https://github.com/Azure/azure-sdk-for-go) [#2835](https://github.com/grafana/tempo/issues/2835) (@LasseHels)
 * [CHANGE] Metrics summary API validate the requested time range [#2902](https://github.com/grafana/tempo/pull/2902) (@mdisibio)
 * [CHANGE] Restructure Azure backends into versioned backends.  Introduce `use_v2_sdk` config option for switching. [#2952](https://github.com/grafana/tempo/issues/2952) (@zalegrala)
-    v1: [azure-storage-blob-go](https://github.com/Azure/azure-storage-blob-go) original (now deprecated) SDK
-    v2: [azure-sdk-for-go](https://github.com/Azure/azure-sdk-for-go)
+  v1: [azure-storage-blob-go](https://github.com/Azure/azure-storage-blob-go) original (now deprecated) SDK
+  v2: [azure-sdk-for-go](https://github.com/Azure/azure-sdk-for-go)
 * [CHANGE] Adjust trace size estimation to better honor row group size settings. [#3038](https://github.com/grafana/tempo/pull/3038) (@joe-elliott)
 * [CHANGE] Update alpine image version to 3.18 to patch CVE-2022-48174. [#3046](https://github.com/grafana/tempo/pull/3046) (@joe-elliott)
 * [CHANGE] Overrides module refactor [#2688](https://github.com/grafana/tempo/pull/2688) (@mapno)
-    Added new `defaults` block to the overrides' module. Overrides change to indented syntax.
-    Old config:
+  Added new `defaults` block to the overrides' module. Overrides change to indented syntax.
+  Old config:
 
 ```
 overrides:
@@ -424,7 +555,7 @@ defaults:
 * [ENHANCEMENT] Add histogram buckets to metrics-generator config in user-configurable overrides [#2928](https://github.com/grafana/tempo/pull/2928) (@mar4uk)
 * [ENHANCEMENT] Adds websocket support for search streaming. [#2971](https://github.com/grafana/tempo/pull/2840) (@joe-elliott)
 * [ENHANCEMENT] Add new config block to distributors to produce debug metrics. [#3008](https://github.com/grafana/tempo/pull/3008) (@joe-elliott)
-   **Breaking Change** Removed deprecated config option: distributor.log_received_spans
+  **Breaking Change** Removed deprecated config option: distributor.log_received_traces
 * [ENHANCEMENT] added a metrics generator config option to enable/disable X-Scope-OrgID headers on remote write. [#2974](https://github.com/grafana/tempo/pull/2974) (@vineetjp)
 * [ENHANCEMENT] Correctly return RetryInfo to Otel Collector/Grafana Agent on ResourceExhausted. This allows the agents to honor their own retry
   settings. [#3019](https://github.com/grafana/tempo/pull/3019) (@joe-elliott)
@@ -507,7 +638,7 @@ storage:
 * [FEATURE] New TraceQL structural operators descendant (>>), child (>), and sibling (~) [#2625](https://github.com/grafana/tempo/pull/2625) [#2660](https://github.com/grafana/tempo/pull/2660) (@mdisibio)
 * [FEATURE] Add user-configurable overrides module [#2543](https://github.com/grafana/tempo/pull/2543) [#2682](https://github.com/grafana/tempo/pull/2682) [#2681](https://github.com/grafana/tempo/pull/2681) (@electron0zero @kvrhdn)
 * [FEATURE] Add support for `q` query param in `/api/v2/search/<tag.name>/values` to filter results based on a TraceQL query [#2253](https://github.com/grafana/tempo/pull/2253) (@mapno)
-To make use of filtering, configure `autocomplete_filtering_enabled`.
+  To make use of filtering, configure `autocomplete_filtering_enabled`.
 * [FEATURE] Add support for `by()` and `coalesce()` to TraceQL. [#2490](https://github.com/grafana/tempo/pull/2490)
 * [FEATURE] Add a GRPC streaming endpoint for traceql search [#2366](https://github.com/grafana/tempo/pull/2366) (@joe-elliott)
 * [FEATURE] Add new API to summarize span metrics from generators [#2481](https://github.com/grafana/tempo/pull/2481) (@zalegrala)
@@ -632,7 +763,7 @@ To make use of filtering, configure `autocomplete_filtering_enabled`.
 ## v2.0.0 / 2023-01-31
 
 * [CHANGE] **BREAKING CHANGE** Use snake case on Azure Storage config [#1879](https://github.com/grafana/tempo/issues/1879) (@faustodavid)
-Example of using snake case on Azure Storage config:
+  Example of using snake case on Azure Storage config:
 
 ```
 # config.yaml
@@ -650,7 +781,7 @@ storage:
 * [CHANGE] Update Go to 1.19 [#1665](https://github.com/grafana/tempo/pull/1665) (@ie-pham)
 * [CHANGE] Remove unsued scheduler frontend code [#1734](https://github.com/grafana/tempo/pull/1734) (@mapno)
 * [CHANGE] Deprecated `query-frontend.query_shards` in favor of `query_frontend.trace_by_id.query_shards`.
-Old config will still work but will be removed in a future release. [#1735](https://github.com/grafana/tempo/pull/1735) (@mapno)
+  Old config will still work but will be removed in a future release. [#1735](https://github.com/grafana/tempo/pull/1735) (@mapno)
 * [CHANGE] Update alpine image version to 3.16. [#1784](https://github.com/grafana/tempo/pull/1784) (@zalegrala)
 * [CHANGE] Delete TempoRequestErrors alert from mixin [#1810](https://github.com/grafana/tempo/pull/1810) (@zalegrala)
   * **BREAKING CHANGE** Any jsonnet users relying on this alert should copy this into their own environment.
@@ -720,7 +851,7 @@ Old config will still work but will be removed in a future release. [#1735](http
 * [FEATURE] TraceQL support <https://grafana.com/docs/tempo/latest/traceql/>
 * [FEATURE] Parquet backend is GA and default
 * [FEATURE] Add generic forwarder and implement otlpgrpc forwarder [#1775](https://github.com/grafana/tempo/pull/1775) (@Blinkuu)
-    New config options and example configuration:
+  New config options and example configuration:
 
 ```
 # config.yaml
@@ -784,7 +915,7 @@ overrides:
 * [ENHANCEMENT] Add query parameter to search API for traceQL queries [#1729](https://github.com/grafana/tempo/pull/1729) (@kvrhdn)
 * [ENHANCEMENT] metrics-generator: filter out older spans before metrics are aggregated [#1612](https://github.com/grafana/tempo/pull/1612) (@ie-pham)
 * [ENHANCEMENT] Add hedging to trace by ID lookups created by the frontend. [#1735](https://github.com/grafana/tempo/pull/1735) (@mapno)
-    New config options and defaults:
+  New config options and defaults:
 
 ```
 query_frontend:
@@ -795,7 +926,7 @@ query_frontend:
 
 * [ENHANCEMENT] Vulture now has improved distribution of the random traces it searches. [#1763](https://github.com/grafana/tempo/pull/1763) (@rfratto)
 * [ENHANCEMENT] Upgrade opentelemetry-proto submodule to v0.18.0 Internal types are updated to use `scope` instead of `instrumentation_library`.
-                This is a breaking change in trace by ID queries if JSON is requested. [#1754](https://github.com/grafana/tempo/pull/1754) (@mapno)
+  This is a breaking change in trace by ID queries if JSON is requested. [#1754](https://github.com/grafana/tempo/pull/1754) (@mapno)
 * [ENHANCEMENT] Add TLS support to the vulture [#1874](https://github.com/grafana/tempo/pull/1874) (@zalegrala)
 * [ENHANCEMENT] metrics-generator: extract `status_message` field from spans [#1786](https://github.com/grafana/tempo/pull/1786), [#1794](https://github.com/grafana/tempo/pull/1794) (@stoewer)
 * [ENHANCEMENT] metrics-generator: handle collisions between user defined and default dimensions [#1794](https://github.com/grafana/tempo/pull/1794) (@stoewer)
@@ -829,7 +960,7 @@ query_frontend:
 * [CHANGE] Swapped out Google Cloud Functions serverless docs and build for Google Cloud Run. [#1483](https://github.com/grafana/tempo/pull/1483) (@joe-elliott)
 * [CHANGE] **BREAKING CHANGE** Change spanmetrics metric names and labels to match OTel conventions. [#1478](https://github.com/grafana/tempo/pull/1478) (@mapno)
 * [FEATURE] Add support for time picker in jaeger query plugin. [#1631](https://github.com/grafana/tempo/pull/1631) (@rubenvp8510)
-Old metric names:
+  Old metric names:
 
 ```
 traces_spanmetrics_duration_seconds_{sum,count,bucket}
@@ -856,19 +987,19 @@ Additionally, default label `span_status` is renamed to `status_code`.
   ```
 
 * [CHANGE] **BREAKING CHANGE** Include emptyDir for metrics generator wal storage in jsonnet [#1556](https://github.com/grafana/tempo/pull/1556) (@zalegrala)
-Jsonnet users will now need to specify a storage request and limit for the generator wal.
-    _config+:: {
-      metrics_generator+: {
-        ephemeral_storage_request_size: '10Gi',
-        ephemeral_storage_limit_size: '11Gi',
-      },
-    }
+  Jsonnet users will now need to specify a storage request and limit for the generator wal.
+  _config+:: {
+  metrics_generator+: {
+  ephemeral_storage_request_size: '10Gi',
+  ephemeral_storage_limit_size: '11Gi',
+  },
+  }
 * [CHANGE] Two additional latency buckets added to the default settings for generated spanmetrics. Note that this will increase cardinality when using the defaults. [#1593](https://github.com/grafana/tempo/pull/1593) (@fredr)
 * [CHANGE] Mark `log_received_traces` as deprecated. New flag is `log_received_spans`.
   Extend distributor spans logger with optional features to include span attributes and a filter by error status. [#1465](https://github.com/grafana/tempo/pull/1465) (@faustodavid)
 * [FEATURE] Add parquet block format [#1479](https://github.com/grafana/tempo/pull/1479) [#1531](https://github.com/grafana/tempo/pull/1531) [#1564](https://github.com/grafana/tempo/pull/1564) (@annanay25, @mdisibio)
 * [FEATURE] Add anonymous usage reporting, enabled by default. [#1481](https://github.com/grafana/tempo/pull/1481) (@zalegrala)
-**BREAKING CHANGE** As part of the usage stats inclusion, the distributor will also require access to the store.  This is required so the distirbutor can know which cluster it should be reporting membership of.
+  **BREAKING CHANGE** As part of the usage stats inclusion, the distributor will also require access to the store.  This is required so the distirbutor can know which cluster it should be reporting membership of.
 * [FEATURE] Include messaging systems and databases in service graphs. [#1576](https://github.com/grafana/tempo/pull/1576) (@kvrhdn)
 * [ENHANCEMENT] Added the ability to have a per tenant max search duration. [#1421](https://github.com/grafana/tempo/pull/1421) (@joe-elliott)
 * [ENHANCEMENT] metrics-generator: expose max_active_series as a metric [#1471](https://github.com/grafana/tempo/pull/1471) (@kvrhdn)

CODEOWNERS

@@ -11,15 +11,15 @@
 # In each subsection folders are ordered first by depth, then alphabetically.
 # This should make it easy to add new rules without breaking existing ones.
 
-* @joe-elliott @mdisibio @mapno @yvrhdn @zalegrala @electron0zero @ie-pham @stoewer @javiermolinar
+* @joe-elliott @mdisibio @mapno @yvrhdn @zalegrala @electron0zero @ie-pham @stoewer @javiermolinar @carles-grafana
 
-/docs/ @knylander-grafana @joe-elliott @mdisibio @mapno @yvrhdn @zalegrala @electron0zero @ie-pham @stoewer @javiermolinar
+/docs/ @knylander-grafana @joe-elliott @mdisibio @mapno @yvrhdn @zalegrala @electron0zero @ie-pham @stoewer @javiermolinar @carles-grafana
 
-/.github/backport.yml           @jdbaldry @joe-elliott @mdisibio @mapno @yvrhdn @zalegrala @electron0zero @ie-pham @stoewer @javiermolinar
-/.github/update-make-docs.yml   @jdbaldry @joe-elliott @mdisibio @mapno @yvrhdn @zalegrala @electron0zero @ie-pham @stoewer @javiermolinar
-/.github/website-next.yml       @jdbaldry @joe-elliott @mdisibio @mapno @yvrhdn @zalegrala @electron0zero @ie-pham @stoewer @javiermolinar
-/.github/website-versioned.yml  @jdbaldry @joe-elliott @mdisibio @mapno @yvrhdn @zalegrala @electron0zero @ie-pham @stoewer @javiermolinar
-/docs/docs.mk                   @jdbaldry @knylander-grafana @joe-elliott @mdisibio @mapno @yvrhdn @zalegrala @electron0zero @ie-pham @stoewer @javiermolinar
-/docs/make-docs                 @jdbaldry @knylander-grafana @joe-elliott @mdisibio @mapno @yvrhdn @zalegrala @electron0zero @ie-pham @stoewer @javiermolinar
-/docs/Makefile                  @jdbaldry @knylander-grafana @joe-elliott @mdisibio @mapno @yvrhdn @zalegrala @electron0zero @ie-pham @stoewer @javiermolinar
-/docs/variables.mk              @jdbaldry @knylander-grafana @joe-elliott @mdisibio @mapno @yvrhdn @zalegrala @electron0zero @ie-pham @stoewer @javiermolinar
+/.github/backport.yml           @jdbaldry @joe-elliott @mdisibio @mapno @yvrhdn @zalegrala @electron0zero @ie-pham @stoewer @javiermolinar @carles-grafana
+/.github/update-make-docs.yml   @jdbaldry @joe-elliott @mdisibio @mapno @yvrhdn @zalegrala @electron0zero @ie-pham @stoewer @javiermolinar @carles-grafana
+/.github/website-next.yml       @jdbaldry @joe-elliott @mdisibio @mapno @yvrhdn @zalegrala @electron0zero @ie-pham @stoewer @javiermolinar @carles-grafana
+/.github/website-versioned.yml  @jdbaldry @joe-elliott @mdisibio @mapno @yvrhdn @zalegrala @electron0zero @ie-pham @stoewer @javiermolinar @carles-grafana
+/docs/docs.mk                   @jdbaldry @knylander-grafana @joe-elliott @mdisibio @mapno @yvrhdn @zalegrala @electron0zero @ie-pham @stoewer @javiermolinar @carles-grafana
+/docs/make-docs                 @jdbaldry @knylander-grafana @joe-elliott @mdisibio @mapno @yvrhdn @zalegrala @electron0zero @ie-pham @stoewer @javiermolinar @carles-grafana
+/docs/Makefile                  @jdbaldry @knylander-grafana @joe-elliott @mdisibio @mapno @yvrhdn @zalegrala @electron0zero @ie-pham @stoewer @javiermolinar @carles-grafana
+/docs/variables.mk              @jdbaldry @knylander-grafana @joe-elliott @mdisibio @mapno @yvrhdn @zalegrala @electron0zero @ie-pham @stoewer @javiermolinar @carles-grafana

CONTRIBUTING.md

@@ -36,7 +36,6 @@ cmd/
   tempo-cli/          - cli tool for directly inspecting blocks in the backend
   tempo-vulture/      - bird-themed consistency checker.  optional.
   tempo-query/        - jaeger-query GRPC plugin (Apache2 licensed)
-  tempo-serverless/   - serverless handler for searching backend storage for traces
 docs/
 example/              - great place to get started running Tempo
   docker-compose/
@@ -88,8 +87,7 @@ import (
   services and backends. The relevant dashboards can be found in the [Tempo mixin](operations/tempo-mixin).
 - **Logs**: Tempo uses the [go-kit level logging library](https://pkg.go.dev/github.com/go-kit/kit/log/level) and emits
   logs in the `key=value` (logfmt) format.
-- **Traces**: Tempo uses the [Jaeger Golang SDK](https://github.com/jaegertracing/jaeger-client-go) for tracing instrumentation.
-  As of this writing, only the read path of tempo is instrumented for tracing.
+- **Traces**: Tempo uses the [OpenTelemetry](https://pkg.go.dev/go.opentelemetry.io) for tracing instrumentation.
 
 ### Testing
 

GOVERNANCE.md

@@ -46,6 +46,7 @@ In case a member leaves, the [offboarding](#offboarding) procedure is applied.
 
 The current team members are:
 
+- Carles Garcia - [carles-grafana](https://github.com/carles-grafana) ([Grafana Labs](https://grafana.com/))
 - Suraj Nath - [electron0zero](https://github.com/electron0zero) ([Grafana Labs](https://grafana.com/))
 - Jennie Pham - [ie-pham](https://github.com/ie-pham) ([Grafana Labs](https://grafana.com/))
 - Javi Molina - [javiermolinar](https://github.com/javiermolinar) ([Grafana Labs](https://grafana.com/))

MAINTAINERS.md

@@ -1,3 +1,4 @@
+* @carles-grafana
 * @electron0zero
 * @ie-pham
 * @javiermolinar

Makefile

@@ -18,8 +18,8 @@ GOPATH := $(shell go env GOPATH)
 GORELEASER := $(GOPATH)/bin/goreleaser
 
 # Build Images
-DOCKER_PROTOBUF_IMAGE ?= otel/build-protobuf:0.23.0
-LOKI_BUILD_IMAGE ?= grafana/loki-build-image:0.33.2
+DOCKER_PROTOBUF_IMAGE ?= otel/build-protobuf:0.25.0
+LOKI_BUILD_IMAGE ?= grafana/loki-build-image:0.34.6
 DOCS_IMAGE ?= grafana/docs-base:latest
 
 # More exclusions can be added similar with: -not -path './testbed/*'
@@ -27,7 +27,6 @@ ALL_SRC := $(shell find . -name '*.go' \
 								-not -path './tools*/*' \
 								-not -path './vendor*/*' \
 								-not -path './integration/*' \
-								-not -path './cmd/tempo-serverless/*' \
                                 -type f | sort)
 
 # ALL_SRC but without pkg and tempodb packages
@@ -35,7 +34,6 @@ OTHERS_SRC := $(shell find . -name '*.go' \
 								-not -path './tools*/*' \
 								-not -path './vendor*/*' \
 								-not -path './integration/*' \
-								-not -path './cmd/tempo-serverless/*' \
 								-not -path './pkg*/*' \
 								-not -path './tempodb*/*' \
                                 -type f | sort)
@@ -47,11 +45,24 @@ ALL_DOC := $(shell find . \( -name "*.md" -o -name "*.yaml" \) \
 # ALL_PKGS is used with 'go cover'
 ALL_PKGS := $(shell go list $(sort $(dir $(ALL_SRC))))
 
-GO_OPT= -mod vendor -ldflags "-X main.Branch=$(GIT_BRANCH) -X main.Revision=$(GIT_REVISION) -X main.Version=$(VERSION)"
+LD_FLAGS=-X main.Branch=$(GIT_BRANCH) -X main.Revision=$(GIT_REVISION) -X main.Version=$(VERSION)
+ifeq ($(BUILD_DEBUG),)
+	LD_FLAGS+= -w
+endif
+
+GO_OPT= -mod vendor -ldflags "$(LD_FLAGS)"
 ifeq ($(BUILD_DEBUG), 1)
 	GO_OPT+= -gcflags="all=-N -l"
 endif
 
+GO_ENV=CGO_ENABLED=0
+ifeq ($(GOARCH),amd64)
+	GO_ENV+= GOAMD64=v2
+endif
+ifeq ($(GOARCH),arm64)
+	GO_ENV+= GOARM64=v8.0
+endif
+
 GOTEST_OPT?= -race -timeout 25m -count=1 -v
 GOTEST_OPT_WITH_COVERAGE = $(GOTEST_OPT) -cover
 GOTEST=gotestsum --format=testname --
@@ -68,27 +79,27 @@ FILES_TO_JSONNETFMT=$(shell find ./operations/jsonnet ./operations/tempo-mixin -
 ##@ Building
 .PHONY: tempo 	
 tempo: ## Build tempo
-	GO111MODULE=on CGO_ENABLED=0 go build $(GO_OPT) -o ./bin/$(GOOS)/tempo-$(GOARCH) $(BUILD_INFO) ./cmd/tempo 
+	$(GO_ENV) go build $(GO_OPT) -o ./bin/$(GOOS)/tempo-$(GOARCH) $(BUILD_INFO) ./cmd/tempo
 
 .PHONY: tempo-query
 tempo-query: ## Build tempo-query
-	GO111MODULE=on CGO_ENABLED=0 go build $(GO_OPT) -o ./bin/$(GOOS)/tempo-query-$(GOARCH) $(BUILD_INFO) ./cmd/tempo-query
+	$(GO_ENV) go build $(GO_OPT) -o ./bin/$(GOOS)/tempo-query-$(GOARCH) $(BUILD_INFO) ./cmd/tempo-query
 
 .PHONY: tempo-cli
 tempo-cli: ## Build tempo-cli
-	GO111MODULE=on CGO_ENABLED=0 go build $(GO_OPT) -o ./bin/$(GOOS)/tempo-cli-$(GOARCH) $(BUILD_INFO) ./cmd/tempo-cli
+	$(GO_ENV) go build $(GO_OPT) -o ./bin/$(GOOS)/tempo-cli-$(GOARCH) $(BUILD_INFO) ./cmd/tempo-cli
 
 .PHONY: tempo-vulture  ## Build tempo-vulture
 tempo-vulture:
-	GO111MODULE=on CGO_ENABLED=0 go build $(GO_OPT) -o ./bin/$(GOOS)/tempo-vulture-$(GOARCH) $(BUILD_INFO) ./cmd/tempo-vulture
+	$(GO_ENV) go build $(GO_OPT) -o ./bin/$(GOOS)/tempo-vulture-$(GOARCH) $(BUILD_INFO) ./cmd/tempo-vulture
 
 .PHONY: exe  ## Build exe
 exe:
-	GOOS=linux $(MAKE) $(COMPONENT)
+	GOOS=linux make $(COMPONENT)
 
 .PHONY: exe-debug  ## Build exe-debug
 exe-debug:
-	BUILD_DEBUG=1 GOOS=linux $(MAKE) $(COMPONENT)
+	BUILD_DEBUG=1 GOOS=linux make $(COMPONENT)
 
 ##@  Testin' and Lintin'
 
@@ -102,7 +113,7 @@ benchmark: tools ## Run benchmarks
 
 # Not used in CI, tests are split in pkg, tempodb, tempodb-wal and others in CI jobs
 .PHONY: test-with-cover 
-test-with-cover: tools test-serverless ## Run tests with code coverage
+test-with-cover: tools ## Run tests with code coverage
 	$(GOTEST) $(GOTEST_OPT_WITH_COVERAGE) $(ALL_PKGS)
 
 # tests in pkg
@@ -122,7 +133,7 @@ test-with-cover-tempodb-wal: tools  ## Test tempodb/wal with code coverage
 
 # all other tests (excluding pkg & tempodb)
 .PHONY: test-with-cover-others
-test-with-cover-others: tools test-serverless ## Run other tests with code coverage
+test-with-cover-others: tools ## Run other tests with code coverage
 	$(GOTEST) $(GOTEST_OPT_WITH_COVERAGE) $(shell go list $(sort $(dir $(OTHERS_SRC))))
 
 # runs e2e tests in the top level integration/e2e directory
@@ -130,16 +141,16 @@ test-with-cover-others: tools test-serverless ## Run other tests with code cover
 test-e2e: tools docker-tempo docker-tempo-query  ## Run end to end tests
 	$(GOTEST) -v $(GOTEST_OPT) ./integration/e2e
 
-# runs only serverless e2e tests
-.PHONY: test-e2e-serverless
-test-e2e-serverless: tools docker-tempo docker-serverless ## Run serverless end to end tests
-	$(GOTEST) -v $(GOTEST_OPT) ./integration/e2e/serverless
-
 # runs only deployment modes e2e tests
 .PHONY: test-e2e-deployments
 test-e2e-deployments: tools docker-tempo docker-tempo-query ## Run end to end tests for deployments
 	$(GOTEST) -v $(GOTEST_OPT) ./integration/e2e/deployments
 
+# runs only api e2e tests
+.PHONY: test-e2e-api
+test-e2e-api: tools docker-tempo docker-tempo-query ## Run end to end tests for api
+	$(GOTEST) -v $(GOTEST_OPT) ./integration/e2e/api
+
 # runs only poller integration tests
 .PHONY: test-integration-poller
 test-integration-poller: tools ## Run poller integration tests
@@ -147,7 +158,7 @@ test-integration-poller: tools ## Run poller integration tests
 
 # test-all/bench use a docker image so build it first to make sure we're up to date
 .PHONY: test-all ## Run all tests
-test-all: test-with-cover test-e2e test-e2e-serverless test-e2e-deployments test-integration-poller
+test-all: test-with-cover test-e2e test-e2e-deployments test-e2e-api test-integration-poller
 
 .PHONY: test-bench
 test-bench: tools docker-tempo ## Run all benchmarks
@@ -183,6 +194,12 @@ docker-component: check-component exe # not intended to be used directly
 	docker build -t grafana/$(COMPONENT) --build-arg=TARGETARCH=$(GOARCH) -f ./cmd/$(COMPONENT)/Dockerfile .
 	docker tag grafana/$(COMPONENT) $(COMPONENT)
 
+.PHONY: docker-component-multi
+docker-component-multi: check-component # not intended to be used directly
+	GOOS=linux GOARCH=amd64 make $(COMPONENT)
+	GOOS=linux GOARCH=arm64 make $(COMPONENT)
+	docker buildx build -t grafana/$(COMPONENT) --platform linux/amd64,linux/arm64 --output type=docker -f ./cmd/$(COMPONENT)/Dockerfile .
+
 .PHONY: docker-component-debug
 docker-component-debug: check-component exe-debug 
 	docker build -t grafana/$(COMPONENT)-debug --build-arg=TARGETARCH=$(GOARCH) -f ./cmd/$(COMPONENT)/Dockerfile_debug .
@@ -190,22 +207,26 @@ docker-component-debug: check-component exe-debug
 
 .PHONY: docker-tempo 
 docker-tempo: ## Build tempo docker image
-	COMPONENT=tempo $(MAKE) docker-component
+	COMPONENT=tempo make docker-component
+
+.PHONY: docker-tempo-multi
+docker-tempo-multi: ## Build multiarch image locally, requires containerd image store
+	COMPONENT=tempo make docker-component-multi
 
 docker-tempo-debug: ## Build tempo debug docker image
-	COMPONENT=tempo $(MAKE) docker-component-debug
+	COMPONENT=tempo make docker-component-debug
 
 .PHONY: docker-cli
 docker-tempo-cli: ## Build tempo cli docker image
-	COMPONENT=tempo-cli $(MAKE) docker-component
+	COMPONENT=tempo-cli make docker-component
 
 .PHONY: docker-tempo-query
 docker-tempo-query: ## Build tempo query docker image
-	COMPONENT=tempo-query $(MAKE) docker-component
+	COMPONENT=tempo-query make docker-component
 
 .PHONY: docker-tempo-vulture
 docker-tempo-vulture: ## Build tempo vulture docker image
-	COMPONENT=tempo-vulture $(MAKE) docker-component
+	COMPONENT=tempo-vulture make docker-component
 
 .PHONY: docker-images ## Build all docker images
 docker-images: docker-tempo docker-tempo-query docker-tempo-vulture
@@ -221,7 +242,7 @@ endif
 PROTOC = docker run --rm -u ${shell id -u} -v${PWD}:${PWD} -w${PWD} ${DOCKER_PROTOBUF_IMAGE} --proto_path=${PWD}
 PROTO_INTERMEDIATE_DIR = pkg/.patched-proto
 PROTO_INCLUDES = -I$(PROTO_INTERMEDIATE_DIR)
-PROTO_GEN = $(PROTOC) $(PROTO_INCLUDES) --gogofaster_out=plugins=grpc,paths=source_relative:$(2) $(1)
+PROTO_GEN = $(PROTOC) $(PROTO_INCLUDES) --gogofaster_out=plugins=grpc,paths=source_relative,Mgoogle/protobuf/timestamp.proto=github.com/gogo/protobuf/types:$(2) $(1)
 PROTO_GEN_WITH_VENDOR = $(PROTOC) $(PROTO_INCLUDES) -Ivendor -Ivendor/github.com/gogo/protobuf --gogofaster_out=plugins=grpc,paths=source_relative:$(2) $(1)
 PROTO_GEN_WITHOUT_RELATIVE = $(PROTOC) $(PROTO_INCLUDES) --gogofaster_out=plugins=grpc:$(2) $(1)
 
@@ -234,7 +255,7 @@ gen-proto:  ## Generate proto files
 	rm -rf $(PROTO_INTERMEDIATE_DIR)
 	find pkg/tempopb -name *.pb.go | xargs -L 1 -I rm
 	# Here we avoid removing our tempo.proto and our frontend.proto due to reliance on the gogoproto bits.
-	find pkg/tempopb -name *.proto | grep -v tempo.proto | grep -v frontend.proto | xargs -L 1 -I rm
+	find pkg/tempopb -name *.proto | grep -v tempo.proto | grep -v frontend.proto | grep -v backendwork.proto | xargs -L 1 -I rm
 
 	@echo --
 	@echo -- Copying to $(PROTO_INTERMEDIATE_DIR)
@@ -265,6 +286,7 @@ gen-proto:  ## Generate proto files
 	$(call PROTO_GEN,$(PROTO_INTERMEDIATE_DIR)/resource/v1/resource.proto,./pkg/tempopb/)
 	$(call PROTO_GEN,$(PROTO_INTERMEDIATE_DIR)/trace/v1/trace.proto,./pkg/tempopb/)
 	$(call PROTO_GEN,pkg/tempopb/tempo.proto,./)
+	$(call PROTO_GEN,pkg/tempopb/backendwork.proto,./)
 	$(call PROTO_GEN_WITHOUT_RELATIVE,tempodb/backend/v1/v1.proto,./)
 	$(call PROTO_GEN_WITH_VENDOR,modules/frontend/v1/frontendv1pb/frontend.proto,./)
 
@@ -294,12 +316,11 @@ vendor-check: gen-proto update-mod gen-traceql gen-parquet-query ## Keep up to d
 	git diff --exit-code -- **/go.sum **/go.mod vendor/ pkg/tempopb/ pkg/traceql/
 
 
-### Tidy dependencies for tempo and tempo-serverless modules
+### Tidy dependencies for tempo modules
 .PHONY: update-mod 
 update-mod: tools-update-mod ## Update module
 	go mod vendor
 	go mod tidy -e
-	$(MAKE) -C cmd/tempo-serverless update-mod
 
 
 ### Release (intended to be used in the .github/workflows/release.yml)
@@ -328,49 +349,21 @@ docs-test:
 ##@ jsonnet
 .PHONY: jsonnet jsonnet-check jsonnet-test
 jsonnet: tools-image ## Generate jsonnet
-	$(TOOLS_CMD) $(MAKE) -C operations/jsonnet-compiled/util gen
+	$(TOOLS_CMD) make -C operations/jsonnet-compiled/util gen
 
 jsonnet-check: tools-image ## Check jsonnet
-	$(TOOLS_CMD) $(MAKE) -C operations/jsonnet-compiled/util check
+	$(TOOLS_CMD) make -C operations/jsonnet-compiled/util check
 
 jsonnet-test: tools-image ## Test jsonnet
-	$(TOOLS_CMD) $(MAKE) -C operations/jsonnet/microservices test
-
-##@ serverless
-.PHONY: docker-serverless test-serverless
-docker-serverless: ## Build docker Tempo serverless
-	$(MAKE) -C cmd/tempo-serverless build-docker
- 
-test-serverless: ## Run Tempo serverless tests
-	$(MAKE) -C cmd/tempo-serverless test
+	$(TOOLS_CMD) make -C operations/jsonnet/microservices test
 
 ### tempo-mixin
-.PHONY: tempo-mixin tempo-mixin-check
+.PHONY: jsonnetfmt tempo-mixin tempo-mixin-check
 tempo-mixin: tools-image
-	$(TOOLS_CMD) $(MAKE) -C operations/tempo-mixin all
+	$(TOOLS_CMD) make -C operations/tempo-mixin all
 
 tempo-mixin-check: tools-image
-	$(TOOLS_CMD) $(MAKE) -C operations/tempo-mixin check
-
-##@ drone
-.PHONY: drone drone-jsonnet drone-signature
-# this requires the drone-cli https://docs.drone.io/cli/install/
-drone: ## Run Drone targets
-	# piggyback on Loki's build image, this image contains a newer version of drone-cli than is
-	# released currently (1.4.0). The newer version of drone-clie keeps drone.yml human-readable.
-	# This will run 'make drone-jsonnet' from within the container
-	docker run -e DRONE_SERVER -e DRONE_TOKEN --rm -v $(shell pwd):/src/loki ${LOKI_BUILD_IMAGE} drone-jsonnet drone-signature
-
-	drone lint .drone/drone.yml --trusted
-
-drone-jsonnet:
-	drone jsonnet --stream --format --source .drone/drone.jsonnet --target .drone/drone.yml
-
-drone-signature:
-ifndef DRONE_TOKEN
-	$(error DRONE_TOKEN is not set, visit https://drone.grafana.net/account)
-endif
-	DRONE_SERVER=https://drone.grafana.net drone sign --save grafana/tempo .drone/drone.yml
+	$(TOOLS_CMD) make -C operations/tempo-mixin check
 
 .PHONY: generate-manifest
 generate-manifest:

README.md

@@ -8,17 +8,17 @@
   <a href="https://goreportcard.com/report/github.com/grafana/tempo"><img src="https://goreportcard.com/badge/github.com/grafana/tempo" alt="Go Report Card" /></a>
 </p>
 
-Grafana Tempo is an open source, easy-to-use, and high-scale distributed tracing backend. Tempo is cost-efficient, requiring only object storage to operate, and is deeply integrated with Grafana, Prometheus, and Loki. 
+Grafana Tempo is an open source, easy-to-use, and high-scale distributed tracing backend. Tempo is cost-efficient, requiring only object storage to operate, and is deeply integrated with Grafana, Prometheus, and Loki.
 
 
 ## Business value of distributed tracing
 
-Distributed tracing helps teams quickly pinpoint performance issues and understand the flow of requests across services. The Explore Traces UI simplifies this process by offering a user-friendly interface to view and analyze trace data, making it easier to identify and resolve issues without needing to write complex queries.
+Distributed tracing helps teams quickly pinpoint performance issues and understand the flow of requests across services. The Traces Drilldown UI simplifies this process by offering a user-friendly interface to view and analyze trace data, making it easier to identify and resolve issues without needing to write complex queries.
 
-Refer to [Use traces to find solutions](https://grafana.com/docs/tempo/latest/introduction/solutions-with-traces/) to learn more about how you can use distributed tracing to investigate and solve issues. 
+Refer to [Use traces to find solutions](https://grafana.com/docs/tempo/latest/introduction/solutions-with-traces/) to learn more about how you can use distributed tracing to investigate and solve issues.
 
-## Explore Traces UI: A better way to get value from your tracing data
-We are excited to introduce the [Explore Traces app](https://github.com/grafana/explore-traces) as part of the Grafana Explore suite. This app provides a queryless and intuitive experience for analyzing tracing data, allowing teams to quickly identify performance issues, latency bottlenecks, and errors without needing to write complex queries or use TraceQL.
+## Traces Drilldown UI: A better way to get value from your tracing data
+We are excited to introduce the [Traces Drilldown](https://github.com/grafana/traces-drilldown) (formerly Explore Traces) app as part of the Grafana Explore suite. This app provides a queryless and intuitive experience for analyzing tracing data, allowing teams to quickly identify performance issues, latency bottlenecks, and errors without needing to write complex queries or use TraceQL.
 
 Key Features:
 - **Intuitive Trace Analysis**: Spot slow or error-prone traces with easy, point-and-click interactions.
@@ -29,20 +29,19 @@ Key Features:
 ![image](https://github.com/user-attachments/assets/991205df-1b27-489f-8ef0-1a05ee158996)
 
 To learn more see the following links:
-- [Explore Traces repo](https://github.com/grafana/explore-traces)
-- [Explore Traces documentation](https://grafana.com/docs/grafana/latest/explore/simplified-exploration/traces/)
-- [Demo video](https://github.com/user-attachments/assets/8103e173-6dcf-4659-b938-7614c8a5b52d
-)
+- [Traces Drilldown repo](https://github.com/grafana/traces-drilldown)
+- [Traces Drilldown documentation](https://grafana.com/docs/grafana/latest/explore/simplified-exploration/traces/)
+- [Demo video](https://www.youtube.com/watch?v=a3uB1C2oHA4)
 
 ## TraceQL
 
-Tempo implements [TraceQL](https://grafana.com/docs/tempo/latest/traceql/), a traces-first query language inspired by LogQL and PromQL, which enables targeted queries or rich UI-driven analyses. 
+Tempo implements [TraceQL](https://grafana.com/docs/tempo/latest/traceql/), a traces-first query language inspired by LogQL and PromQL, which enables targeted queries or rich UI-driven analyses.
 
-### TraceQL metrics 
+### TraceQL metrics
 
-[TraceQL metrics](https://grafana.com/docs/tempo/latest/traceql/metrics-queries/) is an experimental feature in Grafana Tempo that creates metrics from traces. Metric queries extend trace queries by applying a function to trace query results. This powerful feature allows for ad hoc aggregation of any existing TraceQL query by any dimension available in your traces, much in the same way that LogQL metric queries create metrics from logs. 
+[TraceQL metrics](https://grafana.com/docs/tempo/latest/traceql/metrics-queries/) is an experimental feature in Grafana Tempo that creates metrics from traces. Metric queries extend trace queries by applying a function to trace query results. This powerful feature allows for ad hoc aggregation of any existing TraceQL query by any dimension available in your traces, much in the same way that LogQL metric queries create metrics from logs.
 
-Tempo is Jaeger, Zipkin, Kafka, OpenCensus, and OpenTelemetry compatible. It ingests batches in any of the mentioned formats, buffers them, and then writes them to Azure, GCS, S3, or local disk. As such, it is robust, cheap, and easy to operate!
+Tempo is Jaeger, Zipkin, Kafka, OpenCensus, and OpenTelemetry compatible. It ingests batches in any of the mentioned formats, buffers them, and then writes them to Azure, GCS, S3, or local disk. As such, it's robust, cheap, and easy to operate.
 
 ## Getting started with Tempo
 
@@ -56,6 +55,8 @@ Tempo is Jaeger, Zipkin, Kafka, OpenCensus, and OpenTelemetry compatible. It ing
 
 To learn more about Tempo, consult the following documents & talks:
 
+- [How to get started with Tempo with Joe Elliott (video)](https://www.youtube.com/watch?v=zDrA7Ly3ovU)
+- [Grafana blog posts about Tempo](https://grafana.com/tags/tempo/)
 - [New in Grafana Tempo 2.0: Apache Parquet as the default storage format, support for TraceQL][tempo_20_announce]
 - [Get to know TraceQL: A powerful new query language for distributed tracing][traceql-post]
 

RELEASES.MD

@@ -5,7 +5,7 @@
 - Push a semver tag to main on the merge commit above.  Something like:
   - `git tag -a v1.2.0-rc.0`
   - `git push origin v1.2.0-rc.0`
-- This will initiate the build process in Github Actions and Drone.  The tagged docker image should
+- This will initiate the build process in Github Actions.  The tagged docker image should
   be available here shortly: https://hub.docker.com/r/grafana/tempo/tags?page=1&ordering=last_updated
 - A Github Release Draft should also be available here:  https://github.com/grafana/tempo/releases
   - Copy over the CHANGELOG entries for the release

build/tools.mk

@@ -12,7 +12,7 @@ TOOL_DIR     ?= tools
 TOOL_CONFIG  ?= $(TOOL_DIR)/tools.go
 
 TOOLS_IMAGE ?= grafana/tempo-ci-tools
-TOOLS_IMAGE_TAG ?= main-d02611a
+TOOLS_IMAGE_TAG ?= main-8cdef76
 
 GOTOOLS ?= $(shell cd $(TOOL_DIR) && go list -e -f '{{ .Imports }}' -tags tools |tr -d '[]')
 

cmd/tempo-cli/Dockerfile

@@ -1,5 +1,12 @@
-FROM alpine:3.21 as certs
-RUN apk --update add ca-certificates
+FROM alpine:latest AS ca-certificates
+RUN apk add --update --no-cache ca-certificates
+
+FROM gcr.io/distroless/static-debian12:debug
+
+SHELL ["/busybox/sh", "-c"]
+
 ARG TARGETARCH
 COPY bin/linux/tempo-cli-${TARGETARCH} /tempo-cli
+COPY --from=ca-certificates /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/
+
 ENTRYPOINT ["/tempo-cli"]

cmd/tempo-cli/cmd-analyse-block.go

@@ -240,10 +240,10 @@ type attribute struct {
 }
 
 func aggregateAttributes(pf *parquet.File, keyPath string, valuePaths []string) (genericAttrSummary, error) {
-	keyIdx, _ := pq.GetColumnIndexByPath(pf, keyPath)
+	keyIdx, _, _ := pq.GetColumnIndexByPath(pf, keyPath)
 	valueIdxs := make([]int, 0, len(valuePaths))
 	for _, v := range valuePaths {
-		idx, _ := pq.GetColumnIndexByPath(pf, v)
+		idx, _, _ := pq.GetColumnIndexByPath(pf, v)
 		valueIdxs = append(valueIdxs, idx)
 	}
 
@@ -311,7 +311,7 @@ func aggregateDedicatedColumns(pf *parquet.File, scope backend.DedicatedColumnSc
 }
 
 func aggregateColumn(pf *parquet.File, colName string) (uint64, error) {
-	idx, _ := pq.GetColumnIndexByPath(pf, colName)
+	idx, _, _ := pq.GetColumnIndexByPath(pf, colName)
 	calc, err := inspect.NewRowStatCalculator(pf, inspect.RowStatOptions{
 		Columns: []int{idx},
 	})

cmd/tempo-cli/cmd-gen-attrindex.go

@@ -0,0 +1,901 @@
+package main
+
+import (
+	"cmp"
+	"encoding/binary"
+	"errors"
+	"fmt"
+	"hash/fnv"
+	"io"
+	"math"
+	"os"
+	"path/filepath"
+	"runtime"
+	"sort"
+	"strings"
+	"text/tabwriter"
+	"unsafe"
+
+	"github.com/parquet-go/parquet-go"
+
+	pq "github.com/grafana/tempo/pkg/parquetquery"
+	"github.com/grafana/tempo/tempodb/backend"
+	vp4 "github.com/grafana/tempo/tempodb/encoding/vparquet4"
+)
+
+// attrIndexCmd represents a command to generate attribute indices from a parquet block.
+//
+// This command is highly experimental and meant to facilitate experimentation with different
+// kinds of indexes.
+type attrIndexCmd struct {
+	In            string   `arg:"" help:"The input parquet block to read from."`
+	AddIntrinsics bool     `help:"Add some intrinsic attributes to the index like name, kind, status, etc."`
+	IndexTypes    []string `enum:"rows,codes" help:"The type of index to generate (rows | codes | rows,codes)" default:"rows,codes"`
+	dedicatedRes  []string `kong:"-"`
+	dedicatedSpan []string `kong:"-"`
+}
+
+func (cmd *attrIndexCmd) Run(_ *globalOptions) error {
+	cmd.In = getPathToBlockDir(cmd.In)
+	fmt.Printf("Analyzing parquet block from %s\n", cmd.In)
+
+	meta, err := readBlockMeta(cmd.In)
+	if err != nil {
+		return err
+	}
+	if meta.Version != vp4.VersionString {
+		return fmt.Errorf("unsupported parquet version %s", meta.Version)
+	}
+
+	cmd.readDedicatedAttributes(meta)
+
+	stats, err := cmd.collectAttributeStats()
+	if err != nil {
+		return err
+	}
+	stats.printStats()
+
+	if len(cmd.IndexTypes) == 0 || len(cmd.IndexTypes) == 2 {
+		fmt.Println("Generating combined index with inverted index and key/value codes")
+
+		index := generateCombinedIndex(stats)
+		err = writeAttributeIndex(cmd.In, index)
+	} else if len(cmd.IndexTypes) == 1 {
+		if cmd.IndexTypes[0] == "rows" {
+			fmt.Println("Generating inverted index with rows")
+
+			index := generateRowsIndex(stats)
+			err = writeAttributeIndex(cmd.In, index)
+		} else if cmd.IndexTypes[0] == "codes" {
+			fmt.Println("Generating index with key/value codes")
+
+			index := generateCodesIndex(stats)
+			err = writeAttributeIndex(cmd.In, index)
+		}
+	}
+	if err != nil {
+		return err
+	}
+
+	fmt.Printf("\nSuccessfully generated attribute index in %s/index.parquet\n", cmd.In)
+	return nil
+}
+
+func (cmd *attrIndexCmd) readDedicatedAttributes(meta *backend.BlockMeta) {
+	for _, ded := range meta.DedicatedColumns {
+		switch ded.Scope {
+		case backend.DedicatedColumnScopeResource:
+			cmd.dedicatedRes = append(cmd.dedicatedRes, ded.Name)
+		case backend.DedicatedColumnScopeSpan:
+			cmd.dedicatedSpan = append(cmd.dedicatedSpan, ded.Name)
+		}
+	}
+}
+
+func (cmd *attrIndexCmd) collectAttributeStats() (*fileStats, error) {
+	stats := fileStats{
+		Attributes: make(map[string]attributeInfo, 200),
+	}
+
+	in, pf, err := openParquetFile(cmd.In)
+	if err != nil {
+		return nil, err
+	}
+	defer in.Close()
+
+	reader := parquet.NewGenericReader[vp4.Trace](pf)
+	defer reader.Close()
+
+	var (
+		traceBuffer = make([]vp4.Trace, 1024)
+		readCount   int
+	)
+
+	for {
+		readCount, err = reader.Read(traceBuffer)
+		if err != nil {
+			if !errors.Is(err, io.EOF) {
+				return nil, err
+			}
+			break
+		}
+		runtime.GC() // after reading the new traces to the buffer GC can free the old ones
+
+		if readCount > 0 {
+			cmd.collectAttributeStatsForTraces(&stats, traceBuffer[:readCount])
+		}
+	}
+	if readCount > 0 {
+		cmd.collectAttributeStatsForTraces(&stats, traceBuffer[:readCount])
+	}
+
+	return &stats, nil
+}
+
+func (cmd *attrIndexCmd) collectAttributeStatsForTraces(stats *fileStats, traces []vp4.Trace) {
+	row := pq.EmptyRowNumber()
+	stats.Traces += len(traces)
+	for _, tr := range traces {
+		stats.Resources += len(tr.ResourceSpans)
+		row.Next(0, 0, 3)
+
+		for _, rs := range tr.ResourceSpans {
+			row.Next(1, 1, 3)
+
+			res := rs.Resource
+			stats.addAttributes(row, scopeResource, res.Attrs)
+			stats.addDedicatedAttributes(row, scopeResource, cmd.dedicatedRes, &res.DedicatedAttributes)
+
+			stats.addAttribute(row, scopeResource, "service.name", res.ServiceName)
+			stats.addAttribute(row, scopeResource, "cluster", res.Cluster)
+			stats.addAttribute(row, scopeResource, "namespace", res.Namespace)
+			stats.addAttribute(row, scopeResource, "pod", res.Pod)
+			stats.addAttribute(row, scopeResource, "container", res.Container)
+			stats.addAttribute(row, scopeResource, "k8s.cluster.name", res.K8sClusterName)
+			stats.addAttribute(row, scopeResource, "k8s.namespace.name", res.K8sNamespaceName)
+			stats.addAttribute(row, scopeResource, "k8s.pod.name", res.K8sPodName)
+			stats.addAttribute(row, scopeResource, "k8s.container.name", res.K8sContainerName)
+			for _, ss := range rs.ScopeSpans {
+				row.Next(2, 2, 3)
+
+				scope := ss.Scope
+				stats.Spans += len(ss.Spans)
+
+				stats.addAttributes(row, scopeScope, scope.Attrs)
+				if cmd.AddIntrinsics {
+					// adding scope to distinguish from span.name
+					stats.addAttribute(row, scopeScope, "scope.name", scope.Name)
+					stats.addAttribute(row, scopeScope, "version", scope.Version)
+				}
+				for _, sp := range ss.Spans {
+					row.Next(3, 3, 3)
+
+					stats.Events += len(sp.Events)
+					stats.Links += len(sp.Links)
+
+					stats.addAttributes(row, scopeSpan, sp.Attrs)
+					stats.addDedicatedAttributes(row, scopeSpan, cmd.dedicatedSpan, &sp.DedicatedAttributes)
+					stats.addAttribute(row, scopeSpan, "http.method", sp.HttpMethod)
+					stats.addAttribute(row, scopeSpan, "http.url", sp.HttpUrl)
+					stats.addAttribute(row, scopeSpan, "http.status_code", sp.HttpStatusCode)
+					if cmd.AddIntrinsics {
+						stats.addAttribute(row, scopeSpan, "name", sp.Name)
+						stats.addAttribute(row, scopeSpan, "kind", sp.Kind)
+						stats.addAttribute(row, scopeSpan, "status.code", sp.StatusCode)
+						stats.addAttribute(row, scopeSpan, "status.message", sp.StatusMessage)
+					}
+					for _, ev := range sp.Events {
+						stats.addAttributes(row, scopeEvent, ev.Attrs)
+						if cmd.AddIntrinsics {
+							// adding scope to distinguish from span.name
+							stats.addAttribute(row, scopeEvent, "event.name", ev.Name)
+						}
+					}
+					for _, ln := range sp.Links {
+						stats.addAttributes(row, scopeLink, ln.Attrs)
+					}
+				}
+			}
+		}
+	}
+}
+
+func (fs *fileStats) printStats() {
+	fmt.Println("File stats:")
+
+	w := tabwriter.NewWriter(os.Stdout, 0, 0, 1, ' ', tabwriter.DiscardEmptyColumns)
+	tmpl := "%s\t%d\n"
+	_, _ = fmt.Fprintf(w, tmpl, "Traces", fs.Traces)
+	_, _ = fmt.Fprintf(w, tmpl, "Resources", fs.Resources)
+	_, _ = fmt.Fprintf(w, tmpl, "Spans", fs.Spans)
+	_, _ = fmt.Fprintf(w, tmpl, "Events", fs.Events)
+	_, _ = fmt.Fprintf(w, tmpl, "Links", fs.Links)
+	_, _ = fmt.Fprintf(w, tmpl, "Arrays", fs.Arrays)
+	_ = w.Flush()
+
+	// sort attributes by scope and count
+	attrs := make([]attributeInfo, 0, len(fs.Attributes))
+	for _, attr := range fs.Attributes {
+		attrs = append(attrs, attr)
+	}
+	sort.Slice(attrs, func(i, j int) bool {
+		if n := cmp.Compare(attrs[i].ScopeMask, attrs[j].ScopeMask); n != 0 {
+			return n < 0
+		}
+		return attrs[i].Count > attrs[j].Count
+	})
+
+	fmt.Println("\nAttribute stats:")
+	w = tabwriter.NewWriter(os.Stdout, 0, 0, 1, ' ', tabwriter.DiscardEmptyColumns)
+	_, _ = fmt.Fprintf(w, "%s\t%s\t%s\t%s\n", "Name", "Scopes", "Count", "Cardinality")
+	tmpl = "%s\t%s\t%d\t%d\n"
+	for _, attr := range attrs {
+		_, _ = fmt.Fprintf(w, tmpl, attr.Key, attr.ScopeMask.String(), attr.Count, len(attr.ValuesString)+len(attr.ValuesInt)+len(attr.ValuesFloat)+len(attr.ValuesBool))
+	}
+	_ = w.Flush()
+
+	fmt.Printf("\n\n")
+}
+
+func generateCombinedIndex(stats *fileStats) []indexedAttrCombined {
+	var (
+		index   = make([]indexedAttrCombined, 0, len(stats.Attributes))
+		keyCode int64
+	)
+
+	for _, attr := range stats.Attributes {
+		keyCode++
+
+		a := indexedAttrCombined{
+			Key:       attr.Key,
+			KeyCode:   keyCode,
+			ScopeMask: attr.ScopeMask,
+		}
+
+		if len(attr.ValuesString) > 0 {
+			a.ValuesString = make([]indexedValCombined[string], 0, len(attr.ValuesString))
+
+			for _, v := range attr.ValuesString {
+				a.ValuesString = append(a.ValuesString, indexedValCombined[string]{
+					Value:      v.Value,
+					RowNumbers: v.RowNumbers,
+				})
+			}
+
+			sort.Slice(a.ValuesString, func(i, j int) bool {
+				return cmpSlice(a.ValuesString[i].Value, a.ValuesString[j].Value) < 0
+			})
+
+			var valueCode int64
+			for i := range a.ValuesString {
+				valueCode++
+				a.ValuesString[i].ValueCode = valueCode
+			}
+		}
+
+		if len(attr.ValuesInt) > 0 {
+			a.ValuesInt = make([]indexedValCombined[int64], 0, len(attr.ValuesInt))
+
+			for _, v := range attr.ValuesInt {
+				a.ValuesInt = append(a.ValuesInt, indexedValCombined[int64]{
+					Value:      v.Value,
+					RowNumbers: v.RowNumbers,
+				})
+			}
+
+			sort.Slice(a.ValuesInt, func(i, j int) bool {
+				return cmpSlice(a.ValuesInt[i].Value, a.ValuesInt[j].Value) < 0
+			})
+
+			var valueCode int64
+			for i := range a.ValuesInt {
+				valueCode++
+				a.ValuesInt[i].ValueCode = valueCode
+			}
+		}
+
+		if len(attr.ValuesFloat) > 0 {
+			a.ValuesFloat = make([]indexedValCombined[float64], 0, len(attr.ValuesFloat))
+
+			for _, v := range attr.ValuesFloat {
+				a.ValuesFloat = append(a.ValuesFloat, indexedValCombined[float64]{
+					Value:      v.Value,
+					RowNumbers: v.RowNumbers,
+				})
+			}
+
+			sort.Slice(a.ValuesFloat, func(i, j int) bool {
+				return cmpSlice(a.ValuesFloat[i].Value, a.ValuesFloat[j].Value) < 0
+			})
+
+			var valueCode int64
+			for i := range a.ValuesFloat {
+				valueCode++
+				a.ValuesFloat[i].ValueCode = valueCode
+			}
+		}
+
+		if len(attr.ValuesBool) > 0 {
+			a.ValuesBool = make([]indexedValCombined[bool], 0, len(attr.ValuesBool))
+
+			for _, v := range attr.ValuesBool {
+				a.ValuesBool = append(a.ValuesBool, indexedValCombined[bool]{
+					Value:      v.Value,
+					RowNumbers: v.RowNumbers,
+				})
+			}
+
+			sort.Slice(a.ValuesBool, func(i, j int) bool {
+				return cmpSliceBool(a.ValuesBool[i].Value, a.ValuesBool[j].Value) < 0
+			})
+
+			var valueCode int64
+			for i := range a.ValuesBool {
+				valueCode++
+				a.ValuesBool[i].ValueCode = valueCode
+			}
+		}
+
+		index = append(index, a)
+	}
+
+	return index
+}
+
+func generateRowsIndex(stats *fileStats) []indexedAttrRows {
+	var (
+		index   = make([]indexedAttrRows, 0, len(stats.Attributes))
+		keyCode int64
+	)
+
+	for _, attr := range stats.Attributes {
+		keyCode++
+
+		a := indexedAttrRows{
+			Key:       attr.Key,
+			ScopeMask: attr.ScopeMask,
+		}
+
+		if len(attr.ValuesString) > 0 {
+			a.ValuesString = make([]indexedValRows[string], 0, len(attr.ValuesString))
+
+			for _, v := range attr.ValuesString {
+				a.ValuesString = append(a.ValuesString, indexedValRows[string](v))
+			}
+
+			sort.Slice(a.ValuesString, func(i, j int) bool {
+				return cmpSlice(a.ValuesString[i].Value, a.ValuesString[j].Value) < 0
+			})
+		}
+
+		if len(attr.ValuesInt) > 0 {
+			a.ValuesInt = make([]indexedValRows[int64], 0, len(attr.ValuesInt))
+
+			for _, v := range attr.ValuesInt {
+				a.ValuesInt = append(a.ValuesInt, indexedValRows[int64](v))
+			}
+
+			sort.Slice(a.ValuesInt, func(i, j int) bool {
+				return cmpSlice(a.ValuesInt[i].Value, a.ValuesInt[j].Value) < 0
+			})
+		}
+
+		if len(attr.ValuesFloat) > 0 {
+			a.ValuesFloat = make([]indexedValRows[float64], 0, len(attr.ValuesFloat))
+
+			for _, v := range attr.ValuesFloat {
+				a.ValuesFloat = append(a.ValuesFloat, indexedValRows[float64](v))
+			}
+
+			sort.Slice(a.ValuesFloat, func(i, j int) bool {
+				return cmpSlice(a.ValuesFloat[i].Value, a.ValuesFloat[j].Value) < 0
+			})
+		}
+
+		if len(attr.ValuesBool) > 0 {
+			a.ValuesBool = make([]indexedValRows[bool], 0, len(attr.ValuesBool))
+
+			for _, v := range attr.ValuesBool {
+				a.ValuesBool = append(a.ValuesBool, indexedValRows[bool](v))
+			}
+
+			sort.Slice(a.ValuesBool, func(i, j int) bool {
+				return cmpSliceBool(a.ValuesBool[i].Value, a.ValuesBool[j].Value) < 0
+			})
+		}
+
+		index = append(index, a)
+	}
+
+	return index
+}
+
+func generateCodesIndex(stats *fileStats) []indexedAttrCodes {
+	var (
+		index   = make([]indexedAttrCodes, 0, len(stats.Attributes))
+		keyCode int64
+	)
+
+	for _, attr := range stats.Attributes {
+		keyCode++
+
+		a := indexedAttrCodes{
+			Key:       attr.Key,
+			KeyCode:   keyCode,
+			ScopeMask: attr.ScopeMask,
+		}
+
+		if len(attr.ValuesString) > 0 {
+			a.ValuesString = make([]indexedValCodes[string], 0, len(attr.ValuesString))
+
+			for _, v := range attr.ValuesString {
+				a.ValuesString = append(a.ValuesString, indexedValCodes[string]{
+					Value: v.Value,
+				})
+			}
+
+			sort.Slice(a.ValuesString, func(i, j int) bool {
+				return cmpSlice(a.ValuesString[i].Value, a.ValuesString[j].Value) < 0
+			})
+
+			var valueCode int64
+			for i := range a.ValuesString {
+				valueCode++
+				a.ValuesString[i].ValueCode = valueCode
+			}
+		}
+
+		if len(attr.ValuesInt) > 0 {
+			a.ValuesInt = make([]indexedValCodes[int64], 0, len(attr.ValuesInt))
+
+			for _, v := range attr.ValuesInt {
+				a.ValuesInt = append(a.ValuesInt, indexedValCodes[int64]{
+					Value: v.Value,
+				})
+			}
+
+			sort.Slice(a.ValuesInt, func(i, j int) bool {
+				return cmpSlice(a.ValuesInt[i].Value, a.ValuesInt[j].Value) < 0
+			})
+
+			var valueCode int64
+			for i := range a.ValuesInt {
+				valueCode++
+				a.ValuesInt[i].ValueCode = valueCode
+			}
+		}
+
+		if len(attr.ValuesFloat) > 0 {
+			a.ValuesFloat = make([]indexedValCodes[float64], 0, len(attr.ValuesFloat))
+
+			for _, v := range attr.ValuesFloat {
+				a.ValuesFloat = append(a.ValuesFloat, indexedValCodes[float64]{
+					Value: v.Value,
+				})
+			}
+
+			sort.Slice(a.ValuesFloat, func(i, j int) bool {
+				return cmpSlice(a.ValuesFloat[i].Value, a.ValuesFloat[j].Value) < 0
+			})
+
+			var valueCode int64
+			for i := range a.ValuesFloat {
+				valueCode++
+				a.ValuesFloat[i].ValueCode = valueCode
+			}
+		}
+
+		if len(attr.ValuesBool) > 0 {
+			a.ValuesBool = make([]indexedValCodes[bool], 0, len(attr.ValuesBool))
+
+			for _, v := range attr.ValuesBool {
+				a.ValuesBool = append(a.ValuesBool, indexedValCodes[bool]{
+					Value: v.Value,
+				})
+			}
+
+			sort.Slice(a.ValuesBool, func(i, j int) bool {
+				return cmpSliceBool(a.ValuesBool[i].Value, a.ValuesBool[j].Value) < 0
+			})
+
+			var valueCode int64
+			for i := range a.ValuesBool {
+				valueCode++
+				a.ValuesBool[i].ValueCode = valueCode
+			}
+		}
+
+		index = append(index, a)
+	}
+
+	return index
+}
+
+type indexedAttrCombined struct {
+	Key          string                        `parquet:",snappy"`
+	KeyCode      int64                         `parquet:",snappy,delta"`
+	ScopeMask    scopeMask                     `parquet:",snappy,delta"`
+	ValuesString []indexedValCombined[string]  `parquet:",list"`
+	ValuesInt    []indexedValCombined[int64]   `parquet:",list"`
+	ValuesFloat  []indexedValCombined[float64] `parquet:",list"`
+	ValuesBool   []indexedValCombined[bool]    `parquet:",list"`
+}
+
+type indexedValCombined[T comparable] struct {
+	Value      []T   `parquet:",snappy"`
+	ValueCode  int64 `parquet:",snappy,delta"`
+	RowNumbers []rowNumberCols
+}
+
+type indexedAttrRows struct {
+	Key          string                    `parquet:",snappy"`
+	ScopeMask    scopeMask                 `parquet:",snappy,delta"`
+	ValuesString []indexedValRows[string]  `parquet:",list"`
+	ValuesInt    []indexedValRows[int64]   `parquet:",list"`
+	ValuesFloat  []indexedValRows[float64] `parquet:",list"`
+	ValuesBool   []indexedValRows[bool]    `parquet:",list"`
+}
+
+type indexedValRows[T comparable] struct {
+	Value      []T `parquet:",snappy"`
+	RowNumbers []rowNumberCols
+}
+
+type indexedAttrCodes struct {
+	Key          string                     `parquet:",snappy"`
+	KeyCode      int64                      `parquet:",snappy,delta"`
+	ScopeMask    scopeMask                  `parquet:",snappy,delta"`
+	ValuesString []indexedValCodes[string]  `parquet:",list"`
+	ValuesInt    []indexedValCodes[int64]   `parquet:",list"`
+	ValuesFloat  []indexedValCodes[float64] `parquet:",list"`
+	ValuesBool   []indexedValCodes[bool]    `parquet:",list"`
+}
+
+type indexedValCodes[T comparable] struct {
+	Value     []T   `parquet:",snappy"`
+	ValueCode int64 `parquet:",snappy,delta"`
+}
+
+type rowNumberCols struct {
+	Lvl01 int64 `parquet:",snappy,delta"`
+	Lvl02 int64 `parquet:",snappy,delta"`
+	Lvl03 int64 `parquet:",snappy,delta"`
+	Lvl04 int64 `parquet:",snappy,delta"`
+}
+
+func writeAttributeIndex[T any](in string, index []T) error {
+	stat, err := os.Stat(filepath.Join(in, "data.parquet"))
+	if err != nil {
+		return err
+	}
+
+	out, err := os.OpenFile(filepath.Join(in, "index.parquet"), os.O_WRONLY|os.O_CREATE|os.O_TRUNC, stat.Mode())
+	if err != nil {
+		return err
+	}
+	defer out.Close()
+
+	writer := parquet.NewGenericWriter[T](out)
+	defer writer.Close()
+
+	writeCount := 0
+	for writeCount < len(index) {
+		n, err := writer.Write(index[writeCount:])
+		if err != nil {
+			return err
+		}
+		writeCount += n
+	}
+
+	err = writer.Flush()
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
+type fileStats struct {
+	Traces     int
+	Resources  int
+	Spans      int
+	Events     int
+	Links      int
+	Arrays     int
+	Attributes map[string]attributeInfo
+}
+
+type attributeInfo struct {
+	Key          string
+	ScopeMask    scopeMask
+	Count        int64
+	ValuesString map[uint64]valueInfo[string]
+	ValuesInt    map[uint64]valueInfo[int64]
+	ValuesFloat  map[uint64]valueInfo[float64]
+	ValuesBool   map[uint64]valueInfo[bool]
+}
+
+type valueInfo[T comparable] struct {
+	Value      []T
+	RowNumbers []rowNumberCols
+}
+
+func (fs *fileStats) addAttributes(row pq.RowNumber, scope scopeMask, attrs []vp4.Attribute) {
+	for _, attr := range attrs {
+		if attr.IsArray {
+			fs.addAttribute(row, scope, attr.Key, attr.Value)
+			fs.Arrays++
+		} else if len(attr.Value) > 0 {
+			fs.addAttribute(row, scope, attr.Key, attr.Value[0])
+		} else if len(attr.ValueInt) > 0 {
+			fs.addAttribute(row, scope, attr.Key, attr.ValueInt[0])
+		} else if len(attr.ValueDouble) > 0 {
+			fs.addAttribute(row, scope, attr.Key, attr.ValueDouble[0])
+		} else if len(attr.ValueBool) > 0 {
+			fs.addAttribute(row, scope, attr.Key, attr.ValueBool[0])
+		}
+	}
+}
+
+func (fs *fileStats) addDedicatedAttributes(row pq.RowNumber, scope scopeMask, columns []string, attrs *vp4.DedicatedAttributes) {
+	if attrs == nil {
+		return
+	}
+
+	if attrs.String01 != nil && len(columns) > 0 {
+		fs.addAttribute(row, scope, columns[0], attrs.String01)
+	}
+	if attrs.String02 != nil && len(columns) > 1 {
+		fs.addAttribute(row, scope, columns[1], attrs.String02)
+	}
+	if attrs.String03 != nil && len(columns) > 2 {
+		fs.addAttribute(row, scope, columns[2], attrs.String03)
+	}
+	if attrs.String04 != nil && len(columns) > 3 {
+		fs.addAttribute(row, scope, columns[3], attrs.String04)
+	}
+	if attrs.String05 != nil && len(columns) > 4 {
+		fs.addAttribute(row, scope, columns[4], attrs.String05)
+	}
+	if attrs.String06 != nil && len(columns) > 5 {
+		fs.addAttribute(row, scope, columns[5], attrs.String06)
+	}
+	if attrs.String07 != nil && len(columns) > 6 {
+		fs.addAttribute(row, scope, columns[6], attrs.String07)
+	}
+	if attrs.String08 != nil && len(columns) > 7 {
+		fs.addAttribute(row, scope, columns[7], attrs.String08)
+	}
+	if attrs.String09 != nil && len(columns) > 8 {
+		fs.addAttribute(row, scope, columns[8], attrs.String09)
+	}
+	if attrs.String10 != nil && len(columns) > 9 {
+		fs.addAttribute(row, scope, columns[9], attrs.String10)
+	}
+}
+
+func (fs *fileStats) addAttribute(row pq.RowNumber, scope scopeMask, key string, value any) {
+	attr, ok := fs.Attributes[key]
+	if !ok {
+		attr = attributeInfo{
+			Key:       key,
+			ScopeMask: scope,
+		}
+	}
+
+	attr.Count++
+	attr.ScopeMask.Add(scope)
+
+	switch value := value.(type) {
+	case string, *string, []string:
+		s := toSlice[string](value)
+		if len(s) == 0 {
+			return
+		}
+		if attr.ValuesString == nil {
+			attr.ValuesString = make(map[uint64]valueInfo[string], 10)
+		}
+
+		sum := fnvStrings(s)
+		info, ok := attr.ValuesString[sum]
+		if !ok {
+			info = valueInfo[string]{
+				Value:      s,
+				RowNumbers: make([]rowNumberCols, 0, 1),
+			}
+		}
+
+		info.RowNumbers = append(info.RowNumbers, toRowNumberCols(row))
+		attr.ValuesString[sum] = info
+	case int64, *int64, []int64:
+		s := toSlice[int64](value)
+		if len(s) == 0 {
+			return
+		}
+		if attr.ValuesInt == nil {
+			attr.ValuesInt = make(map[uint64]valueInfo[int64], 10)
+		}
+
+		sum := fnvInts(s)
+		v, ok := attr.ValuesInt[sum]
+		if !ok {
+			v = valueInfo[int64]{
+				Value:      s,
+				RowNumbers: make([]rowNumberCols, 0, 1),
+			}
+		}
+		v.RowNumbers = append(v.RowNumbers, toRowNumberCols(row))
+		attr.ValuesInt[sum] = v
+	case float64, *float64, []float64:
+		s := toSlice[float64](value)
+		if len(s) == 0 {
+			return
+		}
+		if attr.ValuesFloat == nil {
+			attr.ValuesFloat = make(map[uint64]valueInfo[float64])
+		}
+
+		sum := fnvFloats(s)
+		v, ok := attr.ValuesFloat[sum]
+		if !ok {
+			v = valueInfo[float64]{
+				Value:      s,
+				RowNumbers: make([]rowNumberCols, 0, 1),
+			}
+		}
+		v.RowNumbers = append(v.RowNumbers, toRowNumberCols(row))
+		attr.ValuesFloat[sum] = v
+	case bool, *bool, []bool:
+		s := toSlice[bool](value)
+		if len(s) == 0 {
+			return
+		}
+		if attr.ValuesBool == nil {
+			attr.ValuesBool = make(map[uint64]valueInfo[bool])
+		}
+
+		sum := fnvBools(s)
+		v, ok := attr.ValuesBool[sum]
+		if !ok {
+			v = valueInfo[bool]{
+				Value:      s,
+				RowNumbers: make([]rowNumberCols, 0, 1),
+			}
+		}
+		v.RowNumbers = append(v.RowNumbers, toRowNumberCols(row))
+		attr.ValuesBool[sum] = v
+	}
+
+	fs.Attributes[key] = attr
+}
+
+type scopeMask int64
+
+const (
+	scopeResource = 1 << iota
+	scopeScope
+	scopeSpan
+	scopeEvent
+	scopeLink
+)
+
+var scopeMap = [5]struct {
+	mask scopeMask
+	name string
+}{
+	{scopeResource, "resource"},
+	{scopeScope, "scope"},
+	{scopeSpan, "span"},
+	{scopeEvent, "event"},
+	{scopeLink, "link"},
+}
+
+func (s *scopeMask) Add(o scopeMask) {
+	*s |= o
+}
+
+func (s *scopeMask) Has(o scopeMask) bool {
+	return *s&o != 0
+}
+
+func (s *scopeMask) Scopes() []string {
+	scopes := make([]string, 0, 5)
+	for _, scope := range scopeMap {
+		if s.Has(scope.mask) {
+			scopes = append(scopes, scope.name)
+		}
+	}
+	return scopes
+}
+
+func (s *scopeMask) String() string {
+	return strings.Join(s.Scopes(), " ")
+}
+
+func toRowNumberCols(row pq.RowNumber) rowNumberCols {
+	return rowNumberCols{
+		Lvl01: int64(row[0]),
+		Lvl02: int64(row[1]),
+		Lvl03: int64(row[2]),
+		Lvl04: int64(row[3]),
+	}
+}
+
+func toSlice[T any](val any) []T {
+	switch v := val.(type) {
+	case []T:
+		return v
+	case T:
+		return []T{v}
+	case *T:
+		if v == nil {
+			return nil
+		}
+		return []T{*v}
+	default:
+		panic(fmt.Sprintf("unexpected type %T", v))
+	}
+}
+
+func fnvStrings(values []string) uint64 {
+	h := fnv.New64a()
+	for _, v := range values {
+		_, _ = h.Write(unsafe.Slice(unsafe.StringData(v), len(v)))
+	}
+	return h.Sum64()
+}
+
+func fnvInts(values []int64) uint64 {
+	h := fnv.New64a()
+	var buf [8]byte
+	for _, v := range values {
+		binary.LittleEndian.PutUint64(buf[:], uint64(v))
+		_, _ = h.Write(buf[:])
+	}
+	return h.Sum64()
+}
+
+func fnvFloats(values []float64) uint64 {
+	h := fnv.New64a()
+	var buf [8]byte
+	for _, v := range values {
+		binary.LittleEndian.PutUint64(buf[:], math.Float64bits(v))
+		_, _ = h.Write(buf[:])
+	}
+	return h.Sum64()
+}
+
+func fnvBools(values []bool) uint64 {
+	h := fnv.New64a()
+	var buf [1]byte
+	for _, v := range values {
+		if v {
+			buf[0] = 1
+		} else {
+			buf[0] = 0
+		}
+		_, _ = h.Write(buf[:])
+	}
+	return h.Sum64()
+}
+
+func cmpSlice[T cmp.Ordered](a, b []T) int {
+	for i := range min(len(a), len(b)) {
+		if n := cmp.Compare(a[i], b[i]); n != 0 {
+			return n
+		}
+	}
+	return cmp.Compare(len(a), len(b))
+}
+
+func cmpSliceBool(a, b []bool) int {
+	for i := range min(len(a), len(b)) {
+		if !a[i] && b[i] {
+			return -1
+		}
+		if a[i] && !b[i] {
+			return 1
+		}
+	}
+	return cmp.Compare(len(a), len(b))
+}

cmd/tempo-cli/cmd-list-column.go

@@ -38,7 +38,7 @@ func (cmd *listColumnCmd) Run(ctx *globalOptions) error {
 		return err
 	}
 
-	colIndex, _ := pq.GetColumnIndexByPath(pf, cmd.Column)
+	colIndex, _, _ := pq.GetColumnIndexByPath(pf, cmd.Column)
 
 	for i, rg := range pf.RowGroups() {
 

cmd/tempo-cli/cmd-query-blocks.go

@@ -163,17 +163,16 @@ func queryBlock(ctx context.Context, r backend.Reader, _ backend.Compactor, bloc
 	searchOpts := common.SearchOptions{}
 	tempodb.SearchConfig{}.ApplyToOptions(&searchOpts)
 
-	trace, err := block.FindTraceByID(ctx, traceID, searchOpts)
+	res, err := block.FindTraceByID(ctx, traceID, searchOpts)
 	if err != nil {
 		return nil, err
 	}
-
-	if trace == nil {
+	if res == nil || res.Trace == nil {
 		return nil, nil
 	}
 
 	return &queryResults{
 		blockID: id,
-		trace:   trace,
+		trace:   res.Trace,
 	}, nil
 }

cmd/tempo-cli/cmd-query-metrics-query-range.go

@@ -61,7 +61,6 @@ func (cmd *metricsQueryCmd) Run(_ *globalOptions) error {
 		Query: cmd.TraceQL,
 		Start: uint64(start),
 		End:   uint64(end),
-		Step:  uint64(5 * time.Second),
 	}
 
 	if cmd.UseGRPC {

cmd/tempo-cli/cmd-rewrite-blocks.go

@@ -10,6 +10,7 @@ import (
 	"strings"
 
 	"github.com/go-kit/log"
+	"github.com/go-kit/log/level"
 	"github.com/google/uuid"
 	"github.com/grafana/tempo/pkg/boundedwaitgroup"
 	"github.com/grafana/tempo/pkg/model"
@@ -23,102 +24,84 @@ import (
 type dropTracesCmd struct {
 	backendOptions
 
-	TenantID  string `arg:"" help:"tenant ID to search"`
-	TraceIDs  string `arg:"" help:"Trace IDs to drop"`
-	DropTrace bool   `name:"drop-trace" help:"actually attempt to drop the trace" default:"false"`
+	TenantID   string `arg:"" help:"tenant ID to search"`
+	TraceIDs   string `arg:"" help:"Trace IDs to drop"`
+	DropTrace  bool   `name:"drop-trace" help:"actually attempt to drop the trace" default:"false"`
+	Background bool   `name:"background" help:"run in background mode" default:"false"`
 }
 
 func (cmd *dropTracesCmd) Run(opts *globalOptions) error {
-	fmt.Printf("beginning process to drop traces %v from tenant %v\n", cmd.TraceIDs, cmd.TenantID)
-	fmt.Println("**warning**: compaction must be disabled or a compactor may duplicate a block as this process is rewriting it")
-	fmt.Println("")
-	if cmd.DropTrace {
-		fmt.Println("************************************************************************")
-		fmt.Println("**this is not a dry run. blocks will be rewritten and marked compacted**")
-		fmt.Println("************************************************************************")
-		fmt.Println("")
-	}
+	var (
+		logger = log.NewLogfmtLogger(os.Stdout)
+		ctx    = context.Background()
+	)
 
-	ctx := context.Background()
+	level.Info(logger).Log("msg", "beginning process to drop traces", "traces", cmd.TraceIDs, "tenant", cmd.TenantID)
+	level.Warn(logger).Log("msg", "compaction must be disabled or a compactor may duplicate a block as this process is rewriting it")
+	if cmd.DropTrace {
+		level.Warn(logger).Log("msg", "this is not a dry run. blocks will be rewritten and marked compacted")
+	}
 
 	r, w, c, err := loadBackend(&cmd.backendOptions, opts)
 	if err != nil {
 		return err
 	}
 
-	type pair struct {
-		traceIDs  []common.ID
-		blockMeta *backend.BlockMeta
-	}
-	tracesByBlock := map[backend.UUID]pair{}
-
 	// Group trace IDs by blocks
 	ids := strings.Split(cmd.TraceIDs, ",")
+	traceIDs := make([]common.ID, len(ids))
 	for _, id := range ids {
 		traceID, err := util.HexStringToTraceID(id)
 		if err != nil {
 			return err
 		}
 
-		// It might be significantly improved if common.BackendBlock supported bulk searches.
-		blocks, err := blocksWithTraceID(ctx, r, cmd.TenantID, traceID)
-		if err != nil {
-			return err
-		}
+		traceIDs = append(traceIDs, traceID)
+	}
 
-		if len(blocks) == 0 {
-			fmt.Printf("\ntrace %s not found in any block. skipping\n", util.TraceIDToHexString(traceID))
-		}
-		for _, block := range blocks {
-			p, ok := tracesByBlock[block.BlockID]
-			if !ok {
-				p = pair{blockMeta: block}
-			}
-			p.traceIDs = append(p.traceIDs, traceID)
-			tracesByBlock[block.BlockID] = p
-		}
+	// It might be significantly improved if common.BackendBlock supported bulk searches.
+	blocks, err := cmd.blocksWithAnyTraceID(ctx, r, logger, cmd.TenantID, traceIDs...)
+	if err != nil {
+		return err
+	}
+
+	if len(blocks) == 0 {
+		level.Info(logger).Log("msg", "traces not found in any block", "traces", cmd.TraceIDs)
 	}
 
 	// Remove traces from blocks
-	for _, p := range tracesByBlock {
-		// print out trace IDs to be removed in the block
-		strTraceIDs := make([]string, len(p.traceIDs))
-		for i, tid := range p.traceIDs {
-			strTraceIDs[i] = util.TraceIDToHexString(tid)
-		}
-		fmt.Printf("\nFound %d traces: %v in block: %v\n", len(strTraceIDs), strTraceIDs, p.blockMeta.BlockID)
-		fmt.Printf("blockInfo: ID: %v, Size: %d Total Traces: %d\n", p.blockMeta.BlockID, p.blockMeta.Size_, p.blockMeta.TotalObjects)
-
+	for _, block := range blocks {
 		if !cmd.DropTrace {
-			fmt.Println("**not dropping trace, use --drop-trace to actually drop**")
+			level.Warn(logger).Log("msg", "not dropping trace, use --drop-trace to actually drop")
 			continue
 		}
 
-		fmt.Printf("  rewriting %v\n", p.blockMeta.BlockID)
-		newMeta, err := rewriteBlock(ctx, r, w, p.blockMeta, p.traceIDs)
+		level.Info(logger).Log("msg", "rewriting block", "block", block.BlockID, "size", block.Size_, "totalTraces", block.TotalObjects)
+		newMeta, err := rewriteBlock(ctx, r, w, block, traceIDs, logger)
 		if err != nil {
-			return err
+			level.Error(logger).Log("msg", "error rewriting block", "block", block.BlockID, "err", err)
+			continue
 		}
 		if newMeta == nil {
-			fmt.Printf("  block %v was removed\n", p.blockMeta.BlockID)
+			level.Info(logger).Log("msg", "block removed", "block", block.BlockID)
 		} else {
-			fmt.Printf("  rewrote to new block: %v\n", newMeta.BlockID)
+			level.Info(logger).Log("msg", "rewrote block", "block", block.BlockID, "newBlock", newMeta.BlockID)
 		}
 
-		fmt.Printf("  marking %v compacted\n", p.blockMeta.BlockID)
-		err = c.MarkBlockCompacted((uuid.UUID)(p.blockMeta.BlockID), p.blockMeta.TenantID)
+		level.Info(logger).Log("msg", "marking block compacted", "block", block.BlockID)
+		err = c.MarkBlockCompacted((uuid.UUID)(block.BlockID), block.TenantID)
 		if err != nil {
-			return err
+			level.Error(logger).Log("msg", "error marking block compacted", "block", block.BlockID, "err", err)
 		}
 	}
 	if cmd.DropTrace {
-		fmt.Printf("successfully rewrote blocks dropping requested traces: %v from tenant: %v\n", cmd.TraceIDs, cmd.TenantID)
+		level.Info(logger).Log("msg", "successfully rewrote blocks dropping requested traces", "traces", cmd.TraceIDs, "tenant", cmd.TenantID)
 	}
 
 	return nil
 }
 
-func rewriteBlock(ctx context.Context, r backend.Reader, w backend.Writer, meta *backend.BlockMeta, traceIDs []common.ID) (*backend.BlockMeta, error) {
+func rewriteBlock(ctx context.Context, r backend.Reader, w backend.Writer, meta *backend.BlockMeta, traceIDs []common.ID, logger log.Logger) (*backend.BlockMeta, error) {
 	enc, err := encoding.FromVersion(meta.Version)
 	if err != nil {
 		return nil, fmt.Errorf("error getting encoder: %w", err)
@@ -158,6 +141,7 @@ func rewriteBlock(ctx context.Context, r backend.Reader, w backend.Writer, meta
 		DropObject: func(id common.ID) bool {
 			for _, tid := range traceIDs {
 				if bytes.Equal(id, tid) {
+					level.Info(logger).Log("msg", "dropping trace", "traceID", util.TraceIDToHexString(id))
 					return true
 				}
 			}
@@ -175,13 +159,18 @@ func rewriteBlock(ctx context.Context, r backend.Reader, w backend.Writer, meta
 	}
 
 	compactor := enc.NewCompactor(opts)
-	fmt.Println("--beginning compaction logs--")
-	out, err := compactor.Compact(ctx, log.NewLogfmtLogger(os.Stdout), r, w, []*backend.BlockMeta{meta})
-	fmt.Println("--ending compaction logs--")
+
+	level.Info(logger).Log("msg", "beginning compaction logs")
+	out, err := compactor.Compact(ctx, logger, r, w, []*backend.BlockMeta{meta})
+	level.Info(logger).Log("msg", "ending compaction logs")
 	if err != nil {
 		return nil, err
 	}
 
+	if len(out) == 0 {
+		return nil, nil
+	}
+
 	if len(out) != 1 {
 		if meta.TotalObjects == int64(len(traceIDs)) {
 			// we removed all traces from the block
@@ -193,13 +182,17 @@ func rewriteBlock(ctx context.Context, r backend.Reader, w backend.Writer, meta
 	newMeta := out[0]
 
 	if newMeta.TotalObjects != meta.TotalObjects-int64(len(traceIDs)) {
-		return nil, fmt.Errorf("expected output to have one less object then in. out: %d in: %d", newMeta.TotalObjects, meta.TotalObjects)
+		level.Warn(logger).Log("msg", "expected output to have one less object then in", "out", newMeta.TotalObjects, "in", meta.TotalObjects)
 	}
 
 	return newMeta, nil
 }
 
-func blocksWithTraceID(ctx context.Context, r backend.Reader, tenantID string, traceID common.ID) ([]*backend.BlockMeta, error) {
+// blocksWithAnyTraceID returns all blocks that contain any of the trace IDs.
+// It is enough to know if a block contains one of the trace IDs since we will
+// open each block and skip any of the trace IDs which are passed into the
+// command.
+func (cmd *dropTracesCmd) blocksWithAnyTraceID(ctx context.Context, r backend.Reader, logger log.Logger, tenantID string, traceIDs ...common.ID) ([]*backend.BlockMeta, error) {
 	blockIDs, _, err := r.Blocks(ctx, tenantID)
 	if err != nil {
 		return nil, err
@@ -216,9 +209,9 @@ func blocksWithTraceID(ctx context.Context, r backend.Reader, tenantID string, t
 			defer wg.Done()
 
 			// search here
-			meta, err := isInBlock(ctx, r, blockNum2, id2, tenantID, traceID)
+			meta, err := isInBlock(ctx, r, cmd.Background, blockNum2, id2, tenantID, traceIDs...)
 			if err != nil {
-				fmt.Println("\nError querying block:", err)
+				level.Error(logger).Log("msg", "error querying block", "block", id2, "err", err)
 				return
 			}
 
@@ -239,10 +232,12 @@ func blocksWithTraceID(ctx context.Context, r backend.Reader, tenantID string, t
 	return results, nil
 }
 
-func isInBlock(ctx context.Context, r backend.Reader, blockNum int, id uuid.UUID, tenantID string, traceID common.ID) (*backend.BlockMeta, error) {
-	fmt.Print(".")
-	if blockNum%100 == 0 {
-		fmt.Print(strconv.Itoa(blockNum))
+func isInBlock(ctx context.Context, r backend.Reader, background bool, blockNum int, id uuid.UUID, tenantID string, traceIDs ...common.ID) (*backend.BlockMeta, error) {
+	if !background {
+		fmt.Print(".")
+		if blockNum%100 == 0 {
+			fmt.Print(strconv.Itoa(blockNum))
+		}
 	}
 
 	meta, err := r.BlockMeta(ctx, id, tenantID)
@@ -264,16 +259,20 @@ func isInBlock(ctx context.Context, r backend.Reader, blockNum int, id uuid.UUID
 	searchOpts := common.SearchOptions{}
 	tempodb.SearchConfig{}.ApplyToOptions(&searchOpts)
 
-	// technically we could do something even more efficient here by just testing to see if the trace id is in the block w/o
-	// marshalling the whole thing. todo: do that.
-	trace, err := block.FindTraceByID(ctx, traceID, searchOpts)
-	if err != nil {
-		return nil, err
+	for _, traceID := range traceIDs {
+		// technically we could do something even more efficient here by just testing to see if the trace id is in the block w/o
+		// marshalling the whole thing. todo: do that.
+		trace, err := block.FindTraceByID(ctx, traceID, searchOpts)
+		if err != nil {
+			return nil, err
+		}
+
+		if trace == nil {
+			continue
+		}
+
+		return meta, nil
 	}
 
-	if trace == nil {
-		return nil, nil
-	}
-
-	return meta, nil
+	return nil, nil
 }

cmd/tempo-cli/cmd-rewrite-blocks_test.go

@@ -3,6 +3,7 @@ package main
 import (
 	"bytes"
 	"context"
+	"errors"
 	"io"
 	"sort"
 	"strings"
@@ -125,7 +126,7 @@ func getAllTraceIDs(t *testing.T, dir string, tenant string) []string {
 			err := r.Close()
 			require.NoError(t, err)
 		}()
-		traceIDIndex, _ := parquetquery.GetColumnIndexByPath(pf, vparquet4.TraceIDColumnName)
+		traceIDIndex, _, _ := parquetquery.GetColumnIndexByPath(pf, vparquet4.TraceIDColumnName)
 		require.GreaterOrEqual(t, traceIDIndex, 0)
 		defer func() {
 			err := r.Close()
@@ -135,7 +136,9 @@ func getAllTraceIDs(t *testing.T, dir string, tenant string) []string {
 		for read := int64(0); read < r.NumRows(); {
 			rows := make([]parquet.Row, r.NumRows())
 			n, err := r.ReadRows(rows)
-			require.NoError(t, err)
+			if !errors.Is(err, io.EOF) {
+				require.NoError(t, err)
+			}
 			require.Greater(t, n, 0)
 			rows = rows[:n]
 			read += int64(n)

cmd/tempo-cli/main.go

@@ -61,8 +61,9 @@ var cli struct {
 	} `cmd:""`
 
 	Gen struct {
-		Index indexCmd `cmd:"" help:"Generate index for a block"`
-		Bloom bloomCmd `cmd:"" help:"Generate bloom for a block"`
+		Index     indexCmd     `cmd:"" help:"Generate index for a block"`
+		Bloom     bloomCmd     `cmd:"" help:"Generate bloom for a block"`
+		AttrIndex attrIndexCmd `cmd:"" help:"Generate an attribute index for a parquet block (EXPERIMENTAL)"`
 	} `cmd:""`
 
 	Query struct {

cmd/tempo-cli/shared.go

@@ -2,13 +2,14 @@ package main
 
 import (
 	"context"
-	"encoding/json"
 	"errors"
 	"fmt"
 	"sort"
 	"strconv"
 	"time"
 
+	"github.com/gogo/protobuf/jsonpb"
+	"github.com/gogo/protobuf/proto"
 	"github.com/google/uuid"
 	"github.com/grafana/tempo/pkg/boundedwaitgroup"
 	"github.com/grafana/tempo/tempodb/backend"
@@ -124,8 +125,9 @@ func loadBlock(r backend.Reader, c backend.Compactor, tenantID string, id backen
 	}, nil
 }
 
-func printAsJSON[T any](value T) error {
-	traceJSON, err := json.Marshal(value)
+func printAsJSON(pb proto.Message) error {
+	m := jsonpb.Marshaler{}
+	traceJSON, err := m.MarshalToString(pb)
 	if err != nil {
 		return err
 	}

cmd/tempo-query/Dockerfile

@@ -1,13 +1,17 @@
-FROM alpine:3.21 as certs
-RUN apk --update add ca-certificates
+FROM alpine:latest AS ca-certificates
+RUN apk add --update --no-cache ca-certificates
+
+FROM gcr.io/distroless/static-debian12:debug
+
+SHELL ["/busybox/sh", "-c"]
+
+RUN ["/busybox/addgroup", "-g", "10001", "-S", "tempo"]
+RUN ["/busybox/adduser", "-u", "10001", "-S", "tempo", "-G", "tempo"]
+
 ARG TARGETARCH
 COPY bin/linux/tempo-query-${TARGETARCH} /tempo-query
-
-RUN addgroup -g 10001 -S tempo && \
-    adduser -u 10001 -S tempo -G tempo
+COPY --from=ca-certificates /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/
 
 USER 10001:10001
 
-COPY bin/linux/tempo-query-${TARGETARCH} /tempo-query
-
 ENTRYPOINT ["/tempo-query"]

cmd/tempo-query/main.go

@@ -8,9 +8,8 @@ import (
 
 	"github.com/jaegertracing/jaeger/proto-gen/storage_v1"
 	zaplogfmt "github.com/jsternberg/zap-logfmt"
-	otgrpc "github.com/opentracing-contrib/go-grpc"
-	"github.com/opentracing/opentracing-go"
 	"github.com/spf13/viper"
+	"go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc"
 	"go.uber.org/zap"
 	"go.uber.org/zap/zapcore"
 	google_grpc "google.golang.org/grpc"
@@ -55,8 +54,7 @@ func main() {
 	}
 
 	grpcOpts := []google_grpc.ServerOption{
-		google_grpc.UnaryInterceptor(otgrpc.OpenTracingServerInterceptor(opentracing.GlobalTracer())),
-		google_grpc.StreamInterceptor(otgrpc.OpenTracingStreamServerInterceptor(opentracing.GlobalTracer())),
+		google_grpc.StatsHandler(otelgrpc.NewServerHandler()),
 	}
 
 	if cfg.TLSServerEnabeld {

cmd/tempo-query/tempo/config.go

@@ -2,10 +2,12 @@ package tempo
 
 import (
 	"github.com/grafana/dskit/crypto/tls"
-	"github.com/jaegertracing/jaeger/plugin/storage/grpc/shared"
 	"github.com/spf13/viper"
 )
 
+// BearerTokenKey is the key name for the bearer token context value.
+const bearerTokenKey = "bearer.token"
+
 // Config holds the configuration for redbull.
 type Config struct {
 	Address string `yaml:"address"`
@@ -46,7 +48,7 @@ func (c *Config) InitFromViper(v *viper.Viper) {
 	}
 	tenantHeader := v.GetString("tenant_header_key")
 	if tenantHeader == "" {
-		tenantHeader = shared.BearerTokenKey
+		tenantHeader = bearerTokenKey
 	}
 	c.TenantHeaderKey = tenantHeader
 }

cmd/tempo-query/tempo/plugin.go

@@ -4,6 +4,7 @@ import (
 	"context"
 	"crypto/tls"
 	"crypto/x509"
+	"errors"
 	"fmt"
 	"io"
 	"net/http"
@@ -20,7 +21,7 @@ import (
 	"github.com/grafana/dskit/user"
 	jaeger "github.com/jaegertracing/jaeger/model"
 	"github.com/jaegertracing/jaeger/proto-gen/storage_v1"
-	jaeger_spanstore "github.com/jaegertracing/jaeger/storage/spanstore"
+
 	ot_jaeger "github.com/open-telemetry/opentelemetry-collector-contrib/pkg/translator/jaeger"
 	"go.opentelemetry.io/collector/pdata/ptrace"
 	"go.opentelemetry.io/otel"
@@ -62,6 +63,9 @@ var (
 
 var tracer = otel.Tracer("cmd/tempo-query/tempo")
 
+// ErrTraceNotFound is returned by Reader's GetTrace if no data is found for given trace ID.
+var ErrTraceNotFound = errors.New("trace not found")
+
 type Backend struct {
 	logger                       *zap.Logger
 	tempoBackend                 string
@@ -227,7 +231,7 @@ func (b *Backend) getTrace(ctx context.Context, traceID jaeger.TraceID) (*jaeger
 	defer resp.Body.Close()
 
 	if resp.StatusCode == http.StatusNotFound {
-		return nil, jaeger_spanstore.ErrTraceNotFound
+		return nil, ErrTraceNotFound
 	}
 
 	body, err := io.ReadAll(resp.Body)

cmd/tempo-serverless/Makefile

@@ -1,53 +0,0 @@
-PACK=pack
-VERSION=$(shell ../../tools/image-tag | cut -d, -f 1)
-
-IN_CLOUD_RUN=cd cloud-run &&
-IN_LAMBDA=cd lambda &&
-
-LOWER_VERSION = `echo $(VERSION) | tr A-Z a-z`
-
-#
-# build docker images for local testing and code zip files for google cloud run
-#
-.PHONY: build-docker
-build-docker: build-docker-lambda-test build-docker-gcr
-
-#
-# google cloud run
-#
-.PHONY: build-docker-gcr-binary
-build-docker-gcr-binary:
-	$(IN_CLOUD_RUN) CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -o main
-
-.PHONY: build-docker-gcr
-build-docker-gcr: build-docker-gcr-binary
-	$(IN_CLOUD_RUN) docker build -f ./Dockerfile -t tempo-serverless:$(LOWER_VERSION) .
-	$(IN_CLOUD_RUN) rm main
-	docker tag tempo-serverless:$(LOWER_VERSION) tempo-serverless:latest
-
-#
-# aws lambda
-#
-.PHONY: build-docker-lambda-test
-build-docker-lambda-test:
-	$(IN_LAMBDA) CGO_ENABLED=0 go build -o ./lambda
-	$(IN_LAMBDA) docker build -f ./Dockerfile -t tempo-serverless-lambda .
-
-# Lambda zips expect a compiled executable in the root. The filename "bootstrap" is important here.
-# The new AWS Lambda runtime expects an executable with the name "bootstrap" to be provided, the "handler" configuration is ignored when using the new runtime.
-# See https://aws.amazon.com/blogs/compute/migrating-aws-lambda-functions-from-the-go1-x-runtime-to-the-custom-runtime-on-amazon-linux-2/ for more info.
-.PHONY: build-lambda-zip
-build-lambda-zip:
-	$(IN_LAMBDA) CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -o bootstrap
-	$(IN_LAMBDA) zip tempo-serverless-$(VERSION).zip bootstrap
-	$(IN_LAMBDA) rm bootstrap
-
-.PHONY: test
-test:
-	go test -v .
-
-### Tidy dependencies for tempo-serverless module
-.PHONY: update-mod
-update-mod:
-	$(IN_LAMBDA) go mod tidy -e
-	$(IN_CLOUD_RUN) go mod tidy -e

cmd/tempo-serverless/cloud-run/Dockerfile

@@ -1,15 +0,0 @@
-# 
-#  docker run -p 8080:8080 tempo-serverless
-#
-#  to exercise the function
-#    curl http://localhost:8080/?start=1000&end=1001&...
-#
-
-#
-# build the serverless container image
-#
-#  todo: FROM scratch saves ~5MB which could be meaningful in a serverless setting, but using scratch gave strange errors on query.
-FROM alpine:3.21 as certs
-RUN apk --update add ca-certificates
-COPY ./main /main
-ENTRYPOINT ["/main"]

cmd/tempo-serverless/cloud-run/go.mod

@@ -1,150 +0,0 @@
-module github.com/grafana/tempo/cmd/tempo-serverless/cloud-run
-
-go 1.23.3
-
-require (
-	github.com/gogo/protobuf v1.3.2
-	github.com/grafana/tempo v1.2.0-rc.0.0.20211029120833-dee59ebe564c
-)
-
-require (
-	cloud.google.com/go v0.115.0 // indirect
-	cloud.google.com/go/auth v0.12.1 // indirect
-	cloud.google.com/go/auth/oauth2adapt v0.2.6 // indirect
-	cloud.google.com/go/compute/metadata v0.5.2 // indirect
-	cloud.google.com/go/iam v1.1.12 // indirect
-	cloud.google.com/go/storage v1.41.0 // indirect
-	github.com/Azure/azure-sdk-for-go/sdk/azcore v1.16.0 // indirect
-	github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.8.0 // indirect
-	github.com/Azure/azure-sdk-for-go/sdk/internal v1.10.0 // indirect
-	github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.5.0 // indirect
-	github.com/AzureAD/microsoft-authentication-library-for-go v1.2.3 // indirect
-	github.com/alecthomas/units v0.0.0-20240927000941-0f3dac36c52b // indirect
-	github.com/andybalholm/brotli v1.1.1 // indirect
-	github.com/apache/thrift v0.21.0 // indirect
-	github.com/aws/aws-sdk-go v1.55.5 // indirect
-	github.com/beorn7/perks v1.0.1 // indirect
-	github.com/cespare/xxhash/v2 v2.3.0 // indirect
-	github.com/coreos/go-systemd/v22 v22.5.0 // indirect
-	github.com/cristalhq/hedgedhttp v0.9.1 // indirect
-	github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect
-	github.com/dustin/go-humanize v1.0.1 // indirect
-	github.com/facette/natsort v0.0.0-20181210072756-2cd4dd1e2dcb // indirect
-	github.com/felixge/httpsnoop v1.0.4 // indirect
-	github.com/fsnotify/fsnotify v1.8.0 // indirect
-	github.com/go-ini/ini v1.67.0 // indirect
-	github.com/go-kit/log v0.2.1 // indirect
-	github.com/go-logfmt/logfmt v0.6.0 // indirect
-	github.com/go-logr/logr v1.4.2 // indirect
-	github.com/go-logr/stdr v1.2.2 // indirect
-	github.com/go-redis/redis/v8 v8.11.5 // indirect
-	github.com/goccy/go-json v0.10.4 // indirect
-	github.com/gogo/googleapis v1.4.1 // indirect
-	github.com/gogo/status v1.1.1 // indirect
-	github.com/golang-jwt/jwt/v5 v5.2.1 // indirect
-	github.com/golang/groupcache v0.0.0-20241129210726-2c02b8208cf8 // indirect
-	github.com/golang/protobuf v1.5.4 // indirect
-	github.com/golang/snappy v0.0.4 // indirect
-	github.com/google/s2a-go v0.1.8 // indirect
-	github.com/google/uuid v1.6.0 // indirect
-	github.com/googleapis/enterprise-certificate-proxy v0.3.4 // indirect
-	github.com/googleapis/gax-go/v2 v2.14.1 // indirect
-	github.com/gorilla/mux v1.8.1 // indirect
-	github.com/grafana/dskit v0.0.0-20241115082728-f2a7eb3aa0e9 // indirect
-	github.com/grafana/gomemcache v0.0.0-20241016125027-0a5bcc5aef40 // indirect
-	github.com/grafana/pyroscope-go/godeltaprof v0.1.8 // indirect
-	github.com/grafana/regexp v0.0.0-20240518133315-a468a5bfb3bc // indirect
-	github.com/hashicorp/hcl v1.0.0 // indirect
-	github.com/jaegertracing/jaeger v1.63.0 // indirect
-	github.com/jmespath/go-jmespath v0.4.0 // indirect
-	github.com/jpillora/backoff v1.0.0 // indirect
-	github.com/json-iterator/go v1.1.12 // indirect
-	github.com/klauspost/compress v1.17.11 // indirect
-	github.com/klauspost/cpuid/v2 v2.2.9 // indirect
-	github.com/kylelemons/godebug v1.1.0 // indirect
-	github.com/magiconair/properties v1.8.7 // indirect
-	github.com/mattn/go-runewidth v0.0.16 // indirect
-	github.com/miekg/dns v1.1.61 // indirect
-	github.com/minio/md5-simd v1.1.2 // indirect
-	github.com/minio/minio-go/v7 v7.0.82 // indirect
-	github.com/mitchellh/mapstructure v1.5.1-0.20231216201459-8508981c8b6c // indirect
-	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
-	github.com/modern-go/reflect2 v1.0.2 // indirect
-	github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
-	github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f // indirect
-	github.com/olekukonko/tablewriter v0.0.5 // indirect
-	github.com/open-telemetry/opentelemetry-collector-contrib/internal/coreinternal v0.116.0 // indirect
-	github.com/open-telemetry/opentelemetry-collector-contrib/pkg/translator/jaeger v0.116.0 // indirect
-	github.com/opentracing-contrib/go-grpc v0.0.0-20210225150812-73cb765af46e // indirect
-	github.com/opentracing-contrib/go-stdlib v1.0.0 // indirect
-	github.com/opentracing/opentracing-go v1.2.0 // indirect
-	github.com/parquet-go/parquet-go v0.23.1-0.20241011155651-6446d1d0d2fe // indirect
-	github.com/pelletier/go-toml/v2 v2.2.3 // indirect
-	github.com/pierrec/lz4/v4 v4.1.21 // indirect
-	github.com/pires/go-proxyproto v0.7.0 // indirect
-	github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c // indirect
-	github.com/pkg/errors v0.9.1 // indirect
-	github.com/prometheus/client_golang v1.20.5 // indirect
-	github.com/prometheus/client_model v0.6.1 // indirect
-	github.com/prometheus/common v0.61.0 // indirect
-	github.com/prometheus/exporter-toolkit v0.11.0 // indirect
-	github.com/prometheus/procfs v0.15.1 // indirect
-	github.com/prometheus/prometheus v0.54.1 // indirect
-	github.com/rivo/uniseg v0.4.7 // indirect
-	github.com/rs/xid v1.6.0 // indirect
-	github.com/sagikazarmark/locafero v0.4.0 // indirect
-	github.com/sagikazarmark/slog-shim v0.1.0 // indirect
-	github.com/sercand/kuberesolver/v5 v5.1.1 // indirect
-	github.com/sony/gobreaker v0.4.1 // indirect
-	github.com/sourcegraph/conc v0.3.0 // indirect
-	github.com/spaolacci/murmur3 v1.1.0 // indirect
-	github.com/spf13/afero v1.11.0 // indirect
-	github.com/spf13/cast v1.6.0 // indirect
-	github.com/spf13/pflag v1.0.5 // indirect
-	github.com/spf13/viper v1.19.0 // indirect
-	github.com/subosito/gotenv v1.6.0 // indirect
-	github.com/uber-go/atomic v1.4.0 // indirect
-	github.com/uber/jaeger-client-go v2.30.0+incompatible // indirect
-	github.com/uber/jaeger-lib v2.4.1+incompatible // indirect
-	github.com/willf/bitset v1.1.11 // indirect
-	github.com/willf/bloom v2.0.3+incompatible // indirect
-	go.opencensus.io v0.24.0 // indirect
-	go.opentelemetry.io/auto/sdk v1.1.0 // indirect
-	go.opentelemetry.io/collector/pdata v1.22.0 // indirect
-	go.opentelemetry.io/collector/semconv v0.116.0 // indirect
-	go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.58.0 // indirect
-	go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.58.0 // indirect
-	go.opentelemetry.io/otel v1.33.0 // indirect
-	go.opentelemetry.io/otel/metric v1.33.0 // indirect
-	go.opentelemetry.io/otel/trace v1.33.0 // indirect
-	go.uber.org/atomic v1.11.0 // indirect
-	go.uber.org/multierr v1.11.0 // indirect
-	go.uber.org/zap v1.27.0 // indirect
-	golang.org/x/crypto v0.31.0 // indirect
-	golang.org/x/exp v0.0.0-20241108190413-2d47ceb2692f // indirect
-	golang.org/x/mod v0.22.0 // indirect
-	golang.org/x/net v0.33.0 // indirect
-	golang.org/x/oauth2 v0.24.0 // indirect
-	golang.org/x/sync v0.10.0 // indirect
-	golang.org/x/sys v0.28.0 // indirect
-	golang.org/x/text v0.21.0 // indirect
-	golang.org/x/time v0.8.0 // indirect
-	golang.org/x/tools v0.27.0 // indirect
-	google.golang.org/api v0.211.0 // indirect
-	google.golang.org/genproto v0.0.0-20240730163845-b1a4ccb954bf // indirect
-	google.golang.org/genproto/googleapis/api v0.0.0-20241209162323-e6fa225c2576 // indirect
-	google.golang.org/genproto/googleapis/rpc v0.0.0-20241209162323-e6fa225c2576 // indirect
-	google.golang.org/grpc v1.68.1 // indirect
-	google.golang.org/protobuf v1.35.2 // indirect
-	gopkg.in/ini.v1 v1.67.0 // indirect
-	gopkg.in/yaml.v2 v2.4.0 // indirect
-	gopkg.in/yaml.v3 v3.0.1 // indirect
-)
-
-replace github.com/grafana/tempo => ../../../
-
-replace (
-	github.com/bradfitz/gomemcache => github.com/themihai/gomemcache v0.0.0-20180902122335-24332e2d58ab
-	k8s.io/api => k8s.io/api v0.20.4
-	k8s.io/client-go => k8s.io/client-go v0.20.4
-)

cmd/tempo-serverless/cloud-run/go.sum

@@ -1,458 +0,0 @@
-cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
-cloud.google.com/go v0.115.0 h1:CnFSK6Xo3lDYRoBKEcAtia6VSC837/ZkJuRduSFnr14=
-cloud.google.com/go v0.115.0/go.mod h1:8jIM5vVgoAEoiVxQ/O4BFTfHqulPZgs/ufEzMcFMdWU=
-cloud.google.com/go/auth v0.12.1 h1:n2Bj25BUMM0nvE9D2XLTiImanwZhO3DkfWSYS/SAJP4=
-cloud.google.com/go/auth v0.12.1/go.mod h1:BFMu+TNpF3DmvfBO9ClqTR/SiqVIm7LukKF9mbendF4=
-cloud.google.com/go/auth/oauth2adapt v0.2.6 h1:V6a6XDu2lTwPZWOawrAa9HUK+DB2zfJyTuciBG5hFkU=
-cloud.google.com/go/auth/oauth2adapt v0.2.6/go.mod h1:AlmsELtlEBnaNTL7jCj8VQFLy6mbZv0s4Q7NGBeQ5E8=
-cloud.google.com/go/compute/metadata v0.5.2 h1:UxK4uu/Tn+I3p2dYWTfiX4wva7aYlKixAHn3fyqngqo=
-cloud.google.com/go/compute/metadata v0.5.2/go.mod h1:C66sj2AluDcIqakBq/M8lw8/ybHgOZqin2obFxa/E5k=
-cloud.google.com/go/iam v1.1.12 h1:JixGLimRrNGcxvJEQ8+clfLxPlbeZA6MuRJ+qJNQ5Xw=
-cloud.google.com/go/iam v1.1.12/go.mod h1:9LDX8J7dN5YRyzVHxwQzrQs9opFFqn0Mxs9nAeB+Hhg=
-cloud.google.com/go/storage v1.41.0 h1:RusiwatSu6lHeEXe3kglxakAmAbfV+rhtPqA6i8RBx0=
-cloud.google.com/go/storage v1.41.0/go.mod h1:J1WCa/Z2FcgdEDuPUY8DxT5I+d9mFKsCepp5vR6Sq80=
-github.com/Azure/azure-sdk-for-go/sdk/azcore v1.16.0 h1:JZg6HRh6W6U4OLl6lk7BZ7BLisIzM9dG1R50zUk9C/M=
-github.com/Azure/azure-sdk-for-go/sdk/azcore v1.16.0/go.mod h1:YL1xnZ6QejvQHWJrX/AvhFl4WW4rqHVoKspWNVwFk0M=
-github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.8.0 h1:B/dfvscEQtew9dVuoxqxrUKKv8Ih2f55PydknDamU+g=
-github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.8.0/go.mod h1:fiPSssYvltE08HJchL04dOy+RD4hgrjph0cwGGMntdI=
-github.com/Azure/azure-sdk-for-go/sdk/azidentity/cache v0.3.0 h1:+m0M/LFxN43KvULkDNfdXOgrjtg6UYJPFBJyuEcRCAw=
-github.com/Azure/azure-sdk-for-go/sdk/azidentity/cache v0.3.0/go.mod h1:PwOyop78lveYMRs6oCxjiVyBdyCgIYH6XHIVZO9/SFQ=
-github.com/Azure/azure-sdk-for-go/sdk/internal v1.10.0 h1:ywEEhmNahHBihViHepv3xPBn1663uRv2t2q/ESv9seY=
-github.com/Azure/azure-sdk-for-go/sdk/internal v1.10.0/go.mod h1:iZDifYGJTIgIIkYRNWPENUnqx6bJ2xnSDFI2tjwZNuY=
-github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/storage/armstorage v1.6.0 h1:PiSrjRPpkQNjrM8H0WwKMnZUdu1RGMtd/LdGKUrOo+c=
-github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/storage/armstorage v1.6.0/go.mod h1:oDrbWx4ewMylP7xHivfgixbfGBT6APAwsSoHRKotnIc=
-github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.5.0 h1:mlmW46Q0B79I+Aj4azKC6xDMFN9a9SyZWESlGWYXbFs=
-github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.5.0/go.mod h1:PXe2h+LKcWTX9afWdZoHyODqR4fBa5boUM/8uJfZ0Jo=
-github.com/AzureAD/microsoft-authentication-extensions-for-go/cache v0.1.1 h1:WJTmL004Abzc5wDB5VtZG2PJk5ndYDgVacGqfirKxjM=
-github.com/AzureAD/microsoft-authentication-extensions-for-go/cache v0.1.1/go.mod h1:tCcJZ0uHAmvjsVYzEFivsRTN00oz5BEsRgQHu5JZ9WE=
-github.com/AzureAD/microsoft-authentication-library-for-go v1.2.3 h1:6LyjnnaLpcOKK0fbYisI+mb8CE7iNe7i89nMNQxFxs8=
-github.com/AzureAD/microsoft-authentication-library-for-go v1.2.3/go.mod h1:wP83P5OoQ5p6ip3ScPr0BAq0BvuPAvacpEuSzyouqAI=
-github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
-github.com/HdrHistogram/hdrhistogram-go v1.1.2 h1:5IcZpTvzydCQeHzK4Ef/D5rrSqwxob0t8PQPMybUNFM=
-github.com/HdrHistogram/hdrhistogram-go v1.1.2/go.mod h1:yDgFjdqOqDEKOvasDdhWNXYg9BVp4O+o5f6V/ehm6Oo=
-github.com/alecthomas/units v0.0.0-20240927000941-0f3dac36c52b h1:mimo19zliBX/vSQ6PWWSL9lK8qwHozUj03+zLoEB8O0=
-github.com/alecthomas/units v0.0.0-20240927000941-0f3dac36c52b/go.mod h1:fvzegU4vN3H1qMT+8wDmzjAcDONcgo2/SZ/TyfdUOFs=
-github.com/alicebob/gopher-json v0.0.0-20230218143504-906a9b012302 h1:uvdUDbHQHO85qeSydJtItA4T55Pw6BtAejd0APRJOCE=
-github.com/alicebob/gopher-json v0.0.0-20230218143504-906a9b012302/go.mod h1:SGnFV6hVsYE877CKEZ6tDNTjaSXYUk6QqoIK6PrAtcc=
-github.com/alicebob/miniredis v2.5.0+incompatible h1:yBHoLpsyjupjz3NL3MhKMVkR41j82Yjf3KFv7ApYzUI=
-github.com/alicebob/miniredis/v2 v2.21.0 h1:CdmwIlKUWFBDS+4464GtQiQ0R1vpzOgu4Vnd74rBL7M=
-github.com/alicebob/miniredis/v2 v2.21.0/go.mod h1:XNqvJdQJv5mSuVMc0ynneafpnL/zv52acZ6kqeS0t88=
-github.com/andybalholm/brotli v1.1.1 h1:PR2pgnyFznKEugtsUo0xLdDop5SKXd5Qf5ysW+7XdTA=
-github.com/andybalholm/brotli v1.1.1/go.mod h1:05ib4cKhjx3OQYUY22hTVd34Bc8upXjOLL2rKwwZBoA=
-github.com/apache/thrift v0.21.0 h1:tdPmh/ptjE1IJnhbhrcl2++TauVjy242rkV/UzJChnE=
-github.com/apache/thrift v0.21.0/go.mod h1:W1H8aR/QRtYNvrPeFXBtobyRkd0/YVhTc6i07XIAgDw=
-github.com/aws/aws-sdk-go v1.55.5 h1:KKUZBfBoyqy5d3swXyiC7Q76ic40rYcbqH7qjh59kzU=
-github.com/aws/aws-sdk-go v1.55.5/go.mod h1:eRwEWoyTWFMVYVQzKMNHWP5/RV4xIUGMQfXQHfHkpNU=
-github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
-github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw=
-github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
-github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs=
-github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
-github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
-github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc=
-github.com/coreos/go-systemd/v22 v22.5.0 h1:RrqgGjYQKalulkV8NGVIfkXQf6YYmOyiJKk8iXXhfZs=
-github.com/coreos/go-systemd/v22 v22.5.0/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc=
-github.com/cristalhq/hedgedhttp v0.9.1 h1:g68L9cf8uUyQKQJwciD0A1Vgbsz+QgCjuB1I8FAsCDs=
-github.com/cristalhq/hedgedhttp v0.9.1/go.mod h1:XkqWU6qVMutbhW68NnzjWrGtH8NUx1UfYqGYtHVKIsI=
-github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
-github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
-github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM=
-github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
-github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f h1:lO4WD4F/rVNCu3HqELle0jiPLLBs70cWOduZpkS1E78=
-github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f/go.mod h1:cuUVRXasLTGF7a8hSLbxyZXjz+1KgoB3wDUb6vlszIc=
-github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY=
-github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto=
-github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
-github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
-github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98=
-github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c=
-github.com/facette/natsort v0.0.0-20181210072756-2cd4dd1e2dcb h1:IT4JYU7k4ikYg1SCxNI1/Tieq/NFvh6dzLdgi7eu0tM=
-github.com/facette/natsort v0.0.0-20181210072756-2cd4dd1e2dcb/go.mod h1:bH6Xx7IW64qjjJq8M2u4dxNaBiDfKK+z/3eGDpXEQhc=
-github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg=
-github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U=
-github.com/frankban/quicktest v1.14.6 h1:7Xjx+VpznH+oBnejlPUj8oUpdxnVs4f8XU8WnHkI4W8=
-github.com/frankban/quicktest v1.14.6/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7zb5vbUoiM6w0=
-github.com/fsnotify/fsnotify v1.8.0 h1:dAwr6QBTBZIkG8roQaJjGof0pp0EeF+tNV7YBP3F/8M=
-github.com/fsnotify/fsnotify v1.8.0/go.mod h1:8jBTzvmWwFyi3Pb8djgCCO5IBqzKJ/Jwo8TRcHyHii0=
-github.com/go-ini/ini v1.67.0 h1:z6ZrTEZqSWOTyH2FlglNbNgARyHG8oLW9gMELqKr06A=
-github.com/go-ini/ini v1.67.0/go.mod h1:ByCAeIL28uOIIG0E3PJtZPDL8WnHpFKFOtgjp+3Ies8=
-github.com/go-kit/log v0.2.1 h1:MRVx0/zhvdseW+Gza6N9rVzU/IVzaeE1SFI4raAhmBU=
-github.com/go-kit/log v0.2.1/go.mod h1:NwTd00d/i8cPZ3xOwwiv2PO5MOcx78fFErGNcVmBjv0=
-github.com/go-logfmt/logfmt v0.6.0 h1:wGYYu3uicYdqXVgoYbvnkrPVXkuLM1p1ifugDMEdRi4=
-github.com/go-logfmt/logfmt v0.6.0/go.mod h1:WYhtIu8zTZfxdn5+rREduYbwxfcBr/Vr6KEVveWlfTs=
-github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A=
-github.com/go-logr/logr v1.4.2 h1:6pFjapn8bFcIbiKo3XT4j/BhANplGihG6tvd+8rYgrY=
-github.com/go-logr/logr v1.4.2/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY=
-github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag=
-github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE=
-github.com/go-redis/redis/v8 v8.11.5 h1:AcZZR7igkdvfVmQTPnu9WE37LRrO/YrBH5zWyjDC0oI=
-github.com/go-redis/redis/v8 v8.11.5/go.mod h1:gREzHqY1hg6oD9ngVRbLStwAWKhA0FEgq8Jd4h5lpwo=
-github.com/goccy/go-json v0.10.4 h1:JSwxQzIqKfmFX1swYPpUThQZp/Ka4wzJdK0LWVytLPM=
-github.com/goccy/go-json v0.10.4/go.mod h1:oq7eo15ShAhp70Anwd5lgX2pLfOS3QCiwU/PULtXL6M=
-github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA=
-github.com/gogo/googleapis v0.0.0-20180223154316-0cd9801be74a/go.mod h1:gf4bu3Q80BeJ6H1S1vYPm8/ELATdvryBaNFGgqEef3s=
-github.com/gogo/googleapis v1.4.1 h1:1Yx4Myt7BxzvUr5ldGSbwYiZG6t9wGBZ+8/fX3Wvtq0=
-github.com/gogo/googleapis v1.4.1/go.mod h1:2lpHqI5OcWCtVElxXnPt+s8oJvMpySlOyM6xDCrzib4=
-github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q=
-github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q=
-github.com/gogo/status v1.1.1 h1:DuHXlSFHNKqTQ+/ACf5Vs6r4X/dH2EgIzR9Vr+H65kg=
-github.com/gogo/status v1.1.1/go.mod h1:jpG3dM5QPcqu19Hg8lkUhBFBa3TcLs1DG7+2Jqci7oU=
-github.com/golang-jwt/jwt/v5 v5.2.1 h1:OuVbFODueb089Lh128TAcimifWaLhJwVflnrgM17wHk=
-github.com/golang-jwt/jwt/v5 v5.2.1/go.mod h1:pqrtFR0X4osieyHYxtmOUWsAWrfe1Q5UVIyoH402zdk=
-github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
-github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
-github.com/golang/groupcache v0.0.0-20241129210726-2c02b8208cf8 h1:f+oWsMOmNPc8JmEHVZIycC7hBoQxHH9pNKQORJNozsQ=
-github.com/golang/groupcache v0.0.0-20241129210726-2c02b8208cf8/go.mod h1:wcDNUvekVysuuOpQKo3191zZyTpiI6se1N1ULghS0sw=
-github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
-github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
-github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
-github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8=
-github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA=
-github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs=
-github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w=
-github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0=
-github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8=
-github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI=
-github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek=
-github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps=
-github.com/golang/snappy v0.0.4 h1:yAGX7huGHXlcLOEtBnF4w7FQwA26wojNCwOYAEhLjQM=
-github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
-github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=
-github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
-github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
-github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
-github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
-github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
-github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI=
-github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
-github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
-github.com/google/martian/v3 v3.3.3 h1:DIhPTQrbPkgs2yJYdXU/eNACCG5DVQjySNRNlflZ9Fc=
-github.com/google/martian/v3 v3.3.3/go.mod h1:iEPrYcgCF7jA9OtScMFQyAlZZ4YXTKEtJ1E6RWzmBA0=
-github.com/google/s2a-go v0.1.8 h1:zZDs9gcbt9ZPLV0ndSyQk6Kacx2g/X+SKYovpnz3SMM=
-github.com/google/s2a-go v0.1.8/go.mod h1:6iNWHTpQ+nfNRN5E00MSdfDwVesa8hhS32PhPO8deJA=
-github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
-github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
-github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
-github.com/googleapis/enterprise-certificate-proxy v0.3.4 h1:XYIDZApgAnrN1c855gTgghdIA6Stxb52D5RnLI1SLyw=
-github.com/googleapis/enterprise-certificate-proxy v0.3.4/go.mod h1:YKe7cfqYXjKGpGvmSg28/fFvhNzinZQm8DGnaburhGA=
-github.com/googleapis/gax-go/v2 v2.14.1 h1:hb0FFeiPaQskmvakKu5EbCbpntQn48jyHuvrkurSS/Q=
-github.com/googleapis/gax-go/v2 v2.14.1/go.mod h1:Hb/NubMaVM88SrNkvl8X/o8XWwDJEPqouaLeN2IUxoA=
-github.com/gorilla/mux v1.8.1 h1:TuBL49tXwgrFYWhqrNgrUNEY92u81SPhu7sTdzQEiWY=
-github.com/gorilla/mux v1.8.1/go.mod h1:AKf9I4AEqPTmMytcMc0KkNouC66V3BtZ4qD5fmWSiMQ=
-github.com/grafana/dskit v0.0.0-20241115082728-f2a7eb3aa0e9 h1:Dx7+6aU/fhwD2vkMr0PUcyxGat1sjUssHAKQKaS7sDM=
-github.com/grafana/dskit v0.0.0-20241115082728-f2a7eb3aa0e9/go.mod h1:SPLNCARd4xdjCkue0O6hvuoveuS1dGJjDnfxYe405YQ=
-github.com/grafana/gomemcache v0.0.0-20241016125027-0a5bcc5aef40 h1:1TeKhyS+pvzOeyLV1XPZsiqebnKky/AKS3pJNNbHVPo=
-github.com/grafana/gomemcache v0.0.0-20241016125027-0a5bcc5aef40/go.mod h1:IGRj8oOoxwJbHBYl1+OhS9UjQR0dv6SQOep7HqmtyFU=
-github.com/grafana/pyroscope-go/godeltaprof v0.1.8 h1:iwOtYXeeVSAeYefJNaxDytgjKtUuKQbJqgAIjlnicKg=
-github.com/grafana/pyroscope-go/godeltaprof v0.1.8/go.mod h1:2+l7K7twW49Ct4wFluZD3tZ6e0SjanjcUUBPVD/UuGU=
-github.com/grafana/regexp v0.0.0-20240518133315-a468a5bfb3bc h1:GN2Lv3MGO7AS6PrRoT6yV5+wkrOpcszoIsO4+4ds248=
-github.com/grafana/regexp v0.0.0-20240518133315-a468a5bfb3bc/go.mod h1:+JKpmjMGhpgPL+rXZ5nsZieVzvarn86asRlBg4uNGnk=
-github.com/grpc-ecosystem/grpc-opentracing v0.0.0-20180507213350-8e809c8a8645/go.mod h1:6iZfnjpejD4L/4DwD7NryNaJyCQdzwWwH2MWhCA90Kw=
-github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4=
-github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ=
-github.com/hexops/gotextdiff v1.0.3 h1:gitA9+qJrrTCsiCl7+kh75nPqQt1cx4ZkudSTLoUqJM=
-github.com/hexops/gotextdiff v1.0.3/go.mod h1:pSWU5MAI3yDq+fZBTazCSJysOMbxWL1BSow5/V2vxeg=
-github.com/jaegertracing/jaeger v1.63.0 h1:4p1lg4DyZ1rZl5gCt6nnQ0nllcXVLJ96sTSbd+rnOEo=
-github.com/jaegertracing/jaeger v1.63.0/go.mod h1:ZCqGiUdcAPC8HwqQGOjl89HDNEqseArdWcH+yoZxRtk=
-github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg=
-github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo=
-github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8=
-github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U=
-github.com/jpillora/backoff v1.0.0 h1:uvFg412JmmHBHw7iwprIxkPMI+sGQ4kzOWsMeHnm2EA=
-github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4=
-github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM=
-github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo=
-github.com/keybase/go-keychain v0.0.0-20231219164618-57a3676c3af6 h1:IsMZxCuZqKuao2vNdfD82fjjgPLfyHLpR41Z88viRWs=
-github.com/keybase/go-keychain v0.0.0-20231219164618-57a3676c3af6/go.mod h1:3VeWNIJaW+O5xpRQbPp0Ybqu1vJd/pm7s2F473HRrkw=
-github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8=
-github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
-github.com/klauspost/compress v1.17.11 h1:In6xLpyWOi1+C7tXUUWv2ot1QvBjxevKAaI6IXrJmUc=
-github.com/klauspost/compress v1.17.11/go.mod h1:pMDklpSncoRMuLFrf1W9Ss9KT+0rH90U12bZKk7uwG0=
-github.com/klauspost/cpuid/v2 v2.0.1/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg=
-github.com/klauspost/cpuid/v2 v2.2.9 h1:66ze0taIn2H33fBvCkXuv9BmCwDfafmiIVpKV9kKGuY=
-github.com/klauspost/cpuid/v2 v2.2.9/go.mod h1:rqkxqrZ1EhYM9G+hXH7YdowN5R5RGN6NK4QwQ3WMXF8=
-github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE=
-github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk=
-github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
-github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
-github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc=
-github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw=
-github.com/magiconair/properties v1.8.7 h1:IeQXZAiQcpL9mgcAe1Nu6cX9LLw6ExEHKjN0VQdvPDY=
-github.com/magiconair/properties v1.8.7/go.mod h1:Dhd985XPs7jluiymwWYZ0G4Z61jb3vdS329zhj2hYo0=
-github.com/mattn/go-runewidth v0.0.9/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI=
-github.com/mattn/go-runewidth v0.0.16 h1:E5ScNMtiwvlvB5paMFdw9p4kSQzbXFikJ5SQO6TULQc=
-github.com/mattn/go-runewidth v0.0.16/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w=
-github.com/miekg/dns v1.1.61 h1:nLxbwF3XxhwVSm8g9Dghm9MHPaUZuqhPiGL+675ZmEs=
-github.com/miekg/dns v1.1.61/go.mod h1:mnAarhS3nWaW+NVP2wTkYVIZyHNJ098SJZUki3eykwQ=
-github.com/minio/md5-simd v1.1.2 h1:Gdi1DZK69+ZVMoNHRXJyNcxrMA4dSxoYHZSQbirFg34=
-github.com/minio/md5-simd v1.1.2/go.mod h1:MzdKDxYpY2BT9XQFocsiZf/NKVtR7nkE4RoEpN+20RM=
-github.com/minio/minio-go/v7 v7.0.82 h1:tWfICLhmp2aFPXL8Tli0XDTHj2VB/fNf0PC1f/i1gRo=
-github.com/minio/minio-go/v7 v7.0.82/go.mod h1:84gmIilaX4zcvAWWzJ5Z1WI5axN+hAbM5w25xf8xvC0=
-github.com/mitchellh/mapstructure v1.5.1-0.20231216201459-8508981c8b6c h1:cqn374mizHuIWj+OSJCajGr/phAmuMug9qIX3l9CflE=
-github.com/mitchellh/mapstructure v1.5.1-0.20231216201459-8508981c8b6c/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo=
-github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
-github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg=
-github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
-github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M=
-github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk=
-github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA=
-github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ=
-github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f h1:KUppIJq7/+SVif2QVs3tOP0zanoHgBEVAwHxUSIzRqU=
-github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=
-github.com/nxadm/tail v1.4.8 h1:nPr65rt6Y5JFSKQO7qToXr7pePgD6Gwiw05lkbyAQTE=
-github.com/nxadm/tail v1.4.8/go.mod h1:+ncqLTQzXmGhMZNUePPaPqPvBxHAIsmXswZKocGu+AU=
-github.com/olekukonko/tablewriter v0.0.5 h1:P2Ga83D34wi1o9J6Wh1mRuqd4mF/x/lgBS7N7AbDhec=
-github.com/olekukonko/tablewriter v0.0.5/go.mod h1:hPp6KlRPjbx+hW8ykQs1w3UBbZlj6HuIJcUGPhkA7kY=
-github.com/onsi/ginkgo v1.16.5 h1:8xi0RTUf59SOSfEtZMvwTvXYMzG4gV23XVHOZiXNtnE=
-github.com/onsi/ginkgo v1.16.5/go.mod h1:+E8gABHa3K6zRBolWtd+ROzc/U5bkGt0FwiG042wbpU=
-github.com/onsi/gomega v1.24.0 h1:+0glovB9Jd6z3VR+ScSwQqXVTIfJcGA9UBM8yzQxhqg=
-github.com/onsi/gomega v1.24.0/go.mod h1:Z/NWtiqwBrwUt4/2loMmHL63EDLnYHmVbuBpDr2vQAg=
-github.com/open-telemetry/opentelemetry-collector-contrib/internal/coreinternal v0.116.0 h1:xDbf946Zm0rTzWcYEyUfU0Ft2KthhaH4xrNm303vpbI=
-github.com/open-telemetry/opentelemetry-collector-contrib/internal/coreinternal v0.116.0/go.mod h1:yuIyOGmQJOn37u6NVfG8yOCzVvwboqnt+pjOSTvDeLo=
-github.com/open-telemetry/opentelemetry-collector-contrib/pkg/translator/jaeger v0.116.0 h1:wNQ51LBQjPIW7ptgZOMaJT60CnMoP1t3kxgldR7uv9Q=
-github.com/open-telemetry/opentelemetry-collector-contrib/pkg/translator/jaeger v0.116.0/go.mod h1:lx+O1oAnKYPg5KaFW5Ss1Xh+zvCDKZPm73MDAFjPgak=
-github.com/opentracing-contrib/go-grpc v0.0.0-20210225150812-73cb765af46e h1:4cPxUYdgaGzZIT5/j0IfqOrrXmq6bG8AwvwisMXpdrg=
-github.com/opentracing-contrib/go-grpc v0.0.0-20210225150812-73cb765af46e/go.mod h1:DYR5Eij8rJl8h7gblRrOZ8g0kW1umSpKqYIBTgeDtLo=
-github.com/opentracing-contrib/go-stdlib v1.0.0 h1:TBS7YuVotp8myLon4Pv7BtCBzOTo1DeZCld0Z63mW2w=
-github.com/opentracing-contrib/go-stdlib v1.0.0/go.mod h1:qtI1ogk+2JhVPIXVc6q+NHziSmy2W5GbdQZFUHADCBU=
-github.com/opentracing/opentracing-go v1.1.0/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o=
-github.com/opentracing/opentracing-go v1.2.0 h1:uEJPy/1a5RIPAJ0Ov+OIO8OxWu77jEv+1B0VhjKrZUs=
-github.com/opentracing/opentracing-go v1.2.0/go.mod h1:GxEUsuufX4nBwe+T+Wl9TAgYrxe9dPLANfrWvHYVTgc=
-github.com/parquet-go/parquet-go v0.23.1-0.20241011155651-6446d1d0d2fe h1:oUJ5TPnrEK/z+/PeoLL+jCgfngAZIDMyhZASetRcYYg=
-github.com/parquet-go/parquet-go v0.23.1-0.20241011155651-6446d1d0d2fe/go.mod h1:OqBBRGBl7+llplCvDMql8dEKaDqjaFA/VAPw+OJiNiw=
-github.com/pelletier/go-toml/v2 v2.2.3 h1:YmeHyLY8mFWbdkNWwpr+qIL2bEqT0o95WSdkNHvL12M=
-github.com/pelletier/go-toml/v2 v2.2.3/go.mod h1:MfCQTFTvCcUyyvvwm1+G6H/jORL20Xlb6rzQu9GuUkc=
-github.com/pierrec/lz4/v4 v4.1.21 h1:yOVMLb6qSIDP67pl/5F7RepeKYu/VmTyEXvuMI5d9mQ=
-github.com/pierrec/lz4/v4 v4.1.21/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4=
-github.com/pires/go-proxyproto v0.7.0 h1:IukmRewDQFWC7kfnb66CSomk2q/seBuilHBYFwyq0Hs=
-github.com/pires/go-proxyproto v0.7.0/go.mod h1:Vz/1JPY/OACxWGQNIRY2BeyDmpoaWmEP40O9LbuiFR4=
-github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c h1:+mdjkGKdHQG3305AYmdv1U2eRNDiU2ErMBj1gwrq8eQ=
-github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c/go.mod h1:7rwL4CYBLnjLxUqIJNnCWiEdr3bn6IUYi15bNlnbCCU=
-github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
-github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
-github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
-github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U=
-github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
-github.com/prometheus/client_golang v1.20.5 h1:cxppBPuYhUnsO6yo/aoRol4L7q7UFfdm+bR9r+8l63Y=
-github.com/prometheus/client_golang v1.20.5/go.mod h1:PIEt8X02hGcP8JWbeHyeZ53Y/jReSnHgO035n//V5WE=
-github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
-github.com/prometheus/client_model v0.6.1 h1:ZKSh/rekM+n3CeS952MLRAdFwIKqeY8b62p8ais2e9E=
-github.com/prometheus/client_model v0.6.1/go.mod h1:OrxVMOVHjw3lKMa8+x6HeMGkHMQyHDk9E3jmP2AmGiY=
-github.com/prometheus/common v0.61.0 h1:3gv/GThfX0cV2lpO7gkTUwZru38mxevy90Bj8YFSRQQ=
-github.com/prometheus/common v0.61.0/go.mod h1:zr29OCN/2BsJRaFwG8QOBr41D6kkchKbpeNH7pAjb/s=
-github.com/prometheus/exporter-toolkit v0.11.0 h1:yNTsuZ0aNCNFQ3aFTD2uhPOvr4iD7fdBvKPAEGkNf+g=
-github.com/prometheus/exporter-toolkit v0.11.0/go.mod h1:BVnENhnNecpwoTLiABx7mrPB/OLRIgN74qlQbV+FK1Q=
-github.com/prometheus/procfs v0.15.1 h1:YagwOFzUgYfKKHX6Dr+sHT7km/hxC76UB0learggepc=
-github.com/prometheus/procfs v0.15.1/go.mod h1:fB45yRUv8NstnjriLhBQLuOUt+WW4BsoGhij/e3PBqk=
-github.com/prometheus/prometheus v0.54.1 h1:vKuwQNjnYN2/mDoWfHXDhAsz/68q/dQDb+YbcEqU7MQ=
-github.com/prometheus/prometheus v0.54.1/go.mod h1:xlLByHhk2g3ycakQGrMaU8K7OySZx98BzeCR99991NY=
-github.com/redis/go-redis/v9 v9.6.1 h1:HHDteefn6ZkTtY5fGUE8tj8uy85AHk6zP7CpzIAM0y4=
-github.com/redis/go-redis/v9 v9.6.1/go.mod h1:0C0c6ycQsdpVNQpxb1njEQIqkx5UcsM8FJCQLgE9+RA=
-github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc=
-github.com/rivo/uniseg v0.4.7 h1:WUdvkW8uEhrYfLC4ZzdpI2ztxP1I582+49Oc5Mq64VQ=
-github.com/rivo/uniseg v0.4.7/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88=
-github.com/rogpeppe/go-internal v1.13.1 h1:KvO1DLK/DRN07sQ1LQKScxyZJuNnedQ5/wKSR38lUII=
-github.com/rogpeppe/go-internal v1.13.1/go.mod h1:uMEvuHeurkdAXX61udpOXGD/AzZDWNMNyH2VO9fmH0o=
-github.com/rs/xid v1.6.0 h1:fV591PaemRlL6JfRxGDEPl69wICngIQ3shQtzfy2gxU=
-github.com/rs/xid v1.6.0/go.mod h1:7XoLgs4eV+QndskICGsho+ADou8ySMSjJKDIan90Nz0=
-github.com/sagikazarmark/locafero v0.4.0 h1:HApY1R9zGo4DBgr7dqsTH/JJxLTTsOt7u6keLGt6kNQ=
-github.com/sagikazarmark/locafero v0.4.0/go.mod h1:Pe1W6UlPYUk/+wc/6KFhbORCfqzgYEpgQ3O5fPuL3H4=
-github.com/sagikazarmark/slog-shim v0.1.0 h1:diDBnUNK9N/354PgrxMywXnAwEr1QZcOr6gto+ugjYE=
-github.com/sagikazarmark/slog-shim v0.1.0/go.mod h1:SrcSrq8aKtyuqEI1uvTDTK1arOWRIczQRv+GVI1AkeQ=
-github.com/sercand/kuberesolver/v5 v5.1.1 h1:CYH+d67G0sGBj7q5wLK61yzqJJ8gLLC8aeprPTHb6yY=
-github.com/sercand/kuberesolver/v5 v5.1.1/go.mod h1:Fs1KbKhVRnB2aDWN12NjKCB+RgYMWZJ294T3BtmVCpQ=
-github.com/sony/gobreaker v0.4.1 h1:oMnRNZXX5j85zso6xCPRNPtmAycat+WcoKbklScLDgQ=
-github.com/sony/gobreaker v0.4.1/go.mod h1:ZKptC7FHNvhBz7dN2LGjPVBz2sZJmc0/PkyDJOjmxWY=
-github.com/sourcegraph/conc v0.3.0 h1:OQTbbt6P72L20UqAkXXuLOj79LfEanQ+YQFNpLA9ySo=
-github.com/sourcegraph/conc v0.3.0/go.mod h1:Sdozi7LEKbFPqYX2/J+iBAM6HpqSLTASQIKqDmF7Mt0=
-github.com/spaolacci/murmur3 v1.1.0 h1:7c1g84S4BPRrfL5Xrdp6fOJ206sU9y293DDHaoy0bLI=
-github.com/spaolacci/murmur3 v1.1.0/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA=
-github.com/spf13/afero v1.11.0 h1:WJQKhtpdm3v2IzqG8VMqrr6Rf3UYpEF239Jy9wNepM8=
-github.com/spf13/afero v1.11.0/go.mod h1:GH9Y3pIexgf1MTIWtNGyogA5MwRIDXGUr+hbWNoBjkY=
-github.com/spf13/cast v1.6.0 h1:GEiTHELF+vaR5dhz3VqZfFSzZjYbgeKDpBxQVS4GYJ0=
-github.com/spf13/cast v1.6.0/go.mod h1:ancEpBxwJDODSW/UG4rDrAqiKolqNNh2DX3mk86cAdo=
-github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA=
-github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
-github.com/spf13/viper v1.19.0 h1:RWq5SEjt8o25SROyN3z2OrDB9l7RPd3lwTWU8EcEdcI=
-github.com/spf13/viper v1.19.0/go.mod h1:GQUN9bilAbhU/jgc1bKs99f/suXKeUMct8Adx5+Ntkg=
-github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
-github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw=
-github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo=
-github.com/stretchr/objx v0.5.2 h1:xuMeJ0Sdp5ZMRXx/aWO6RZxdr3beISkG5/G/aIRr3pY=
-github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA=
-github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
-github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
-github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
-github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
-github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
-github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo=
-github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
-github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA=
-github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
-github.com/subosito/gotenv v1.6.0 h1:9NlTDc1FTs4qu0DDq7AEtTPNw6SVm7uBMsUCUjABIf8=
-github.com/subosito/gotenv v1.6.0/go.mod h1:Dk4QP5c2W3ibzajGcXpNraDfq2IrhjMIvMSWPKKo0FU=
-github.com/uber-go/atomic v1.4.0 h1:yOuPqEq4ovnhEjpHmfFwsqBXDYbQeT6Nb0bwD6XnD5o=
-github.com/uber-go/atomic v1.4.0/go.mod h1:/Ct5t2lcmbJ4OSe/waGBoaVvVqtO0bmtfVNex1PFV8g=
-github.com/uber/jaeger-client-go v2.30.0+incompatible h1:D6wyKGCecFaSRUpo8lCVbaOOb6ThwMmTEbhRwtKR97o=
-github.com/uber/jaeger-client-go v2.30.0+incompatible/go.mod h1:WVhlPFC8FDjOFMMWRy2pZqQJSXxYSwNYOkTr/Z6d3Kk=
-github.com/uber/jaeger-lib v2.4.1+incompatible h1:td4jdvLcExb4cBISKIpHuGoVXh+dVKhn2Um6rjCsSsg=
-github.com/uber/jaeger-lib v2.4.1+incompatible/go.mod h1:ComeNDZlWwrWnDv8aPp0Ba6+uUTzImX/AauajbLI56U=
-github.com/willf/bitset v1.1.11 h1:N7Z7E9UvjW+sGsEl7k/SJrvY2reP1A07MrGuCjIOjRE=
-github.com/willf/bitset v1.1.11/go.mod h1:83CECat5yLh5zVOf4P1ErAgKA5UDvKtgyUABdr3+MjI=
-github.com/willf/bloom v2.0.3+incompatible h1:QDacWdqcAUI1MPOwIQZRy9kOR7yxfyEmxX8Wdm2/JPA=
-github.com/willf/bloom v2.0.3+incompatible/go.mod h1:MmAltL9pDMNTrvUkxdg0k0q5I0suxmuwp3KbyrZLOZ8=
-github.com/xyproto/randomstring v1.0.5 h1:YtlWPoRdgMu3NZtP45drfy1GKoojuR7hmRcnhZqKjWU=
-github.com/xyproto/randomstring v1.0.5/go.mod h1:rgmS5DeNXLivK7YprL0pY+lTuhNQW3iGxZ18UQApw/E=
-github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
-github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
-github.com/yuin/gopher-lua v0.0.0-20220504180219-658193537a64 h1:5mLPGnFdSsevFRFc9q3yYbBkB6tsm4aCwwQV/j1JQAQ=
-github.com/yuin/gopher-lua v0.0.0-20220504180219-658193537a64/go.mod h1:GBR0iDaNXjAgGg9zfCvksxSRnQx76gclCIb7kdAd1Pw=
-go.opencensus.io v0.24.0 h1:y73uSU6J157QMP2kn2r30vwW1A2W2WFwSCGnAVxeaD0=
-go.opencensus.io v0.24.0/go.mod h1:vNK8G9p7aAivkbmorf4v+7Hgx+Zs0yY+0fOtgBfjQKo=
-go.opentelemetry.io/auto/sdk v1.1.0 h1:cH53jehLUN6UFLY71z+NDOiNJqDdPRaXzTel0sJySYA=
-go.opentelemetry.io/auto/sdk v1.1.0/go.mod h1:3wSPjt5PWp2RhlCcmmOial7AvC4DQqZb7a7wCow3W8A=
-go.opentelemetry.io/collector v0.116.0 h1:Dscd6Nsnc7hjFQosO0SofcPQsXRfcj5N5PjQAslnmj4=
-go.opentelemetry.io/collector/pdata v1.22.0 h1:3yhjL46NLdTMoP8rkkcE9B0pzjf2973crn0KKhX5UrI=
-go.opentelemetry.io/collector/pdata v1.22.0/go.mod h1:nLLf6uDg8Kn5g3WNZwGyu8+kf77SwOqQvMTb5AXEbEY=
-go.opentelemetry.io/collector/pdata/pprofile v0.116.0 h1:iE6lqkO7Hi6lTIIml1RI7yQ55CKqW12R2qHinwF5Zuk=
-go.opentelemetry.io/collector/pdata/pprofile v0.116.0/go.mod h1:xQiPpjzIiXRFb+1fPxUy/3ygEZgo0Bu/xmLKOWu8vMQ=
-go.opentelemetry.io/collector/semconv v0.116.0 h1:63xCZomsKJAWmKGWD3lnORiE3WKW6AO4LjnzcHzGx3Y=
-go.opentelemetry.io/collector/semconv v0.116.0/go.mod h1:N6XE8Q0JKgBN2fAhkUQtqK9LT7rEGR6+Wu/Rtbal1iI=
-go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.58.0 h1:PS8wXpbyaDJQ2VDHHncMe9Vct0Zn1fEjpsjrLxGJoSc=
-go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.58.0/go.mod h1:HDBUsEjOuRC0EzKZ1bSaRGZWUBAzo+MhAcUUORSr4D0=
-go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.58.0 h1:yd02MEjBdJkG3uabWP9apV+OuWRIXGDuJEUJbOHmCFU=
-go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.58.0/go.mod h1:umTcuxiv1n/s/S6/c2AT/g2CQ7u5C59sHDNmfSwgz7Q=
-go.opentelemetry.io/otel v1.33.0 h1:/FerN9bax5LoK51X/sI0SVYrjSE0/yUL7DpxW4K3FWw=
-go.opentelemetry.io/otel v1.33.0/go.mod h1:SUUkR6csvUQl+yjReHu5uM3EtVV7MBm5FHKRlNx4I8I=
-go.opentelemetry.io/otel/metric v1.33.0 h1:r+JOocAyeRVXD8lZpjdQjzMadVZp2M4WmQ+5WtEnklQ=
-go.opentelemetry.io/otel/metric v1.33.0/go.mod h1:L9+Fyctbp6HFTddIxClbQkjtubW6O9QS3Ann/M82u6M=
-go.opentelemetry.io/otel/sdk v1.33.0 h1:iax7M131HuAm9QkZotNHEfstof92xM+N8sr3uHXc2IM=
-go.opentelemetry.io/otel/sdk v1.33.0/go.mod h1:A1Q5oi7/9XaMlIWzPSxLRWOI8nG3FnzHJNbiENQuihM=
-go.opentelemetry.io/otel/trace v1.33.0 h1:cCJuF7LRjUFso9LPnEAHJDB2pqzp+hbO8eu1qqW2d/s=
-go.opentelemetry.io/otel/trace v1.33.0/go.mod h1:uIcdVUZMpTAmz0tI1z04GoVSezK37CbGV4fr1f2nBck=
-go.opentelemetry.io/proto/otlp v1.4.0 h1:TA9WRvW6zMwP+Ssb6fLoUIuirti1gGbP28GcKG1jgeg=
-go.opentelemetry.io/proto/otlp v1.4.0/go.mod h1:PPBWZIP98o2ElSqI35IHfu7hIhSwvc5N38Jw8pXuGFY=
-go.uber.org/atomic v1.11.0 h1:ZvwS0R+56ePWxUNi+Atn9dWONBPp/AUETXlHW0DxSjE=
-go.uber.org/atomic v1.11.0/go.mod h1:LUxbIzbOniOlMKjJjyPfpl4v+PKK2cNJn91OQbhoJI0=
-go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto=
-go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE=
-go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0=
-go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y=
-go.uber.org/zap v1.27.0 h1:aJMhYGrd5QSmlpLMr2MftRKl7t8J8PTZPA732ud/XR8=
-go.uber.org/zap v1.27.0/go.mod h1:GB2qFLM7cTU87MWRP2mPIjqfIDnGu+VIO4V/SdhGo2E=
-golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
-golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
-golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
-golang.org/x/crypto v0.31.0 h1:ihbySMvVjLAeSH1IbfcRTkD/iNscyz8rGzjF/E5hV6U=
-golang.org/x/crypto v0.31.0/go.mod h1:kDsLvtWBEx7MV9tJOj9bnXsPbxwJQ6csT/x4KIN4Ssk=
-golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
-golang.org/x/exp v0.0.0-20241108190413-2d47ceb2692f h1:XdNn9LlyWAhLVp6P/i8QYBW+hlyhrhei9uErw2B5GJo=
-golang.org/x/exp v0.0.0-20241108190413-2d47ceb2692f/go.mod h1:D5SMRVC3C2/4+F/DB1wZsLRnSNimn2Sp/NPsCrsv8ak=
-golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
-golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU=
-golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
-golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
-golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
-golang.org/x/mod v0.22.0 h1:D4nJWe9zXqHOmWqj4VMOJhvzj7bEZg4wEYa759z1pH4=
-golang.org/x/mod v0.22.0/go.mod h1:6SkKJ3Xj0I0BrPOZoBy3bdMptDDU9oJrpohJ3eWZ1fY=
-golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
-golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
-golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
-golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
-golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
-golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
-golang.org/x/net v0.0.0-20190921015927-1a5e07d1ff72/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
-golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
-golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
-golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
-golang.org/x/net v0.33.0 h1:74SYHlV8BIgHIFC/LrYkOGIwL19eTYXQ5wc6TBuO36I=
-golang.org/x/net v0.33.0/go.mod h1:HXLR5J+9DxmrqMwG9qjGCxZ+zKXxBru04zlTvWlWuN4=
-golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
-golang.org/x/oauth2 v0.24.0 h1:KTBBxWqUa0ykRPLtV69rRto9TLXcqYkeswu48x/gvNE=
-golang.org/x/oauth2 v0.24.0/go.mod h1:XYTD2NtWslqkgxebSiOHnXEap4TF09sJSc7H1sXbhtI=
-golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
-golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
-golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
-golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
-golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
-golang.org/x/sync v0.10.0 h1:3NQrjDixjgGwUOCaF8w2+VYHv0Ve/vGYSbdkTa98gmQ=
-golang.org/x/sync v0.10.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
-golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
-golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
-golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.28.0 h1:Fksou7UEQUWlKvIdsqzJmUmCX3cZuD2+P3XyyzwMhlA=
-golang.org/x/sys v0.28.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
-golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
-golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
-golang.org/x/text v0.21.0 h1:zyQAAkrwaneQ066sspRyJaG9VNi/YJ1NfzcGB3hZ/qo=
-golang.org/x/text v0.21.0/go.mod h1:4IBbMaMmOPCJ8SecivzSH54+73PCFmPWxNTLm+vZkEQ=
-golang.org/x/time v0.8.0 h1:9i3RxcPv3PZnitoVGMPDKZSq1xW1gK1Xy3ArNOGZfEg=
-golang.org/x/time v0.8.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM=
-golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
-golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
-golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY=
-golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
-golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
-golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
-golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
-golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
-golang.org/x/tools v0.27.0 h1:qEKojBykQkQ4EynWy4S8Weg69NumxKdn40Fce3uc/8o=
-golang.org/x/tools v0.27.0/go.mod h1:sUi0ZgbwW9ZPAq26Ekut+weQPR5eIM6GQLQ1Yjm1H0Q=
-golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
-golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
-golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
-golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
-golang.org/x/xerrors v0.0.0-20231012003039-104605ab7028 h1:+cNy6SZtPcJQH3LJVLOSmiC7MMxXNOb3PU/VUEz+EhU=
-golang.org/x/xerrors v0.0.0-20231012003039-104605ab7028/go.mod h1:NDW/Ps6MPRej6fsCIbMTohpP40sJ/P/vI1MoTEGwX90=
-google.golang.org/api v0.211.0 h1:IUpLjq09jxBSV1lACO33CGY3jsRcbctfGzhj+ZSE/Bg=
-google.golang.org/api v0.211.0/go.mod h1:XOloB4MXFH4UTlQSGuNUxw0UT74qdENK8d6JNsXKLi0=
-google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
-google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
-google.golang.org/genproto v0.0.0-20180518175338-11a468237815/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=
-google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=
-google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc=
-google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo=
-google.golang.org/genproto v0.0.0-20240730163845-b1a4ccb954bf h1:OqdXDEakZCVtDiZTjcxfwbHPCT11ycCEsTKesBVKvyY=
-google.golang.org/genproto v0.0.0-20240730163845-b1a4ccb954bf/go.mod h1:mCr1K1c8kX+1iSBREvU3Juo11CB+QOEWxbRS01wWl5M=
-google.golang.org/genproto/googleapis/api v0.0.0-20241209162323-e6fa225c2576 h1:CkkIfIt50+lT6NHAVoRYEyAvQGFM7xEwXUUywFvEb3Q=
-google.golang.org/genproto/googleapis/api v0.0.0-20241209162323-e6fa225c2576/go.mod h1:1R3kvZ1dtP3+4p4d3G8uJ8rFk/fWlScl38vanWACI08=
-google.golang.org/genproto/googleapis/rpc v0.0.0-20241209162323-e6fa225c2576 h1:8ZmaLZE4XWrtU3MyClkYqqtl6Oegr3235h7jxsDyqCY=
-google.golang.org/genproto/googleapis/rpc v0.0.0-20241209162323-e6fa225c2576/go.mod h1:5uTbfoYQed2U9p3KIj2/Zzm02PYhndfdmML0qC3q3FU=
-google.golang.org/grpc v1.12.0/go.mod h1:yo6s7OP7yaDglbqo1J04qKzAhqBH6lvTonzMVmEdcZw=
-google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c=
-google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg=
-google.golang.org/grpc v1.23.1/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg=
-google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY=
-google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk=
-google.golang.org/grpc v1.33.2/go.mod h1:JMHMWHQWaTccqQQlmk3MJZS+GWXOdAesneDmEnv2fbc=
-google.golang.org/grpc v1.68.1 h1:oI5oTa11+ng8r8XMMN7jAOmWfPZWbYpCFaMUTACxkM0=
-google.golang.org/grpc v1.68.1/go.mod h1:+q1XYFJjShcqn0QZHvCyeR4CXPA+llXIeUIfIe00waw=
-google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8=
-google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0=
-google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM=
-google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE=
-google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo=
-google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
-google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
-google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
-google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c=
-google.golang.org/protobuf v1.35.2 h1:8Ar7bF+apOIoThw1EdZl0p1oWvMqTHmpA2fRTyZO8io=
-google.golang.org/protobuf v1.35.2/go.mod h1:9fA7Ob0pmnwhb644+1+CVWFRbNajQ6iRojtC/QF5bRE=
-gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
-gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk=
-gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q=
-gopkg.in/ini.v1 v1.67.0 h1:Dgnx+6+nfE+IfzjUEISNeydPJh9AXNNsWbGP9KzCsOA=
-gopkg.in/ini.v1 v1.67.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k=
-gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ=
-gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw=
-gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
-gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
-gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY=
-gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=
-gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
-gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
-gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
-honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
-honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=

cmd/tempo-serverless/cloud-run/main.go

@@ -1,42 +0,0 @@
-package main
-
-import (
-	"log"
-	"net/http"
-	"os"
-
-	"github.com/gogo/protobuf/jsonpb"
-	serverless "github.com/grafana/tempo/cmd/tempo-serverless"
-)
-
-func main() {
-	log.Print("starting server...")
-	http.HandleFunc("/", handler)
-
-	// Determine port for HTTP service.
-	port := os.Getenv("PORT")
-	if port == "" {
-		port = "8080"
-		log.Printf("defaulting to port %s", port)
-	}
-
-	// Start HTTP server.
-	log.Printf("listening on port %s", port)
-	if err := http.ListenAndServe(":"+port, nil); err != nil {
-		log.Fatal(err)
-	}
-}
-
-func handler(w http.ResponseWriter, r *http.Request) {
-	resp, httpErr := serverless.Handler(r)
-	if httpErr != nil {
-		http.Error(w, httpErr.Err.Error(), httpErr.Status)
-	}
-
-	marshaller := &jsonpb.Marshaler{}
-	err := marshaller.Marshal(w, resp)
-	if httpErr != nil {
-		http.Error(w, err.Error(), http.StatusInternalServerError)
-		return
-	}
-}

cmd/tempo-serverless/handler.go

@@ -1,250 +0,0 @@
-package serverless
-
-import (
-	"bytes"
-	"context"
-	"fmt"
-	"net/http"
-	"reflect"
-	"runtime"
-	"strings"
-	"sync"
-
-	"github.com/grafana/dskit/flagext"
-	"github.com/grafana/dskit/user"
-	"github.com/mitchellh/mapstructure"
-	"github.com/spf13/viper"
-	"gopkg.in/yaml.v2"
-
-	"github.com/grafana/tempo/pkg/api"
-	"github.com/grafana/tempo/pkg/tempopb"
-	"github.com/grafana/tempo/pkg/traceql"
-	"github.com/grafana/tempo/tempodb"
-	"github.com/grafana/tempo/tempodb/backend"
-	"github.com/grafana/tempo/tempodb/backend/azure"
-	"github.com/grafana/tempo/tempodb/backend/gcs"
-	"github.com/grafana/tempo/tempodb/backend/local"
-	"github.com/grafana/tempo/tempodb/backend/s3"
-	"github.com/grafana/tempo/tempodb/encoding"
-	"github.com/grafana/tempo/tempodb/encoding/common"
-)
-
-const (
-	envConfigPrefix = "TEMPO"
-)
-
-// used to initialize a reader one time
-var (
-	reader       backend.Reader
-	readerErr    error
-	readerConfig *tempodb.Config
-	readerOnce   sync.Once
-)
-
-type HTTPError struct {
-	Err    error
-	Status int
-}
-
-// Handler is the main entrypoint for the serverless handler. it expects a tempopb.SearchBlockRequest
-// encoded in its parameters
-func Handler(r *http.Request) (*tempopb.SearchResponse, *HTTPError) {
-	searchReq, err := api.ParseSearchBlockRequest(r)
-	if err != nil {
-		return nil, httpError("parsing search request", err, http.StatusBadRequest)
-	}
-
-	maxBytes, err := api.ExtractServerlessParams(r)
-	if err != nil {
-		return nil, httpError("extracting serverless params", err, http.StatusBadRequest)
-	}
-
-	// load config, fields are set through env vars TEMPO_
-	reader, cfg, err := loadBackend()
-	if err != nil {
-		return nil, httpError("loading backend", err, http.StatusInternalServerError)
-	}
-
-	tenant, _, err := user.ExtractOrgIDFromHTTPRequest(r)
-	if err != nil {
-		return nil, httpError("extracting org id", err, http.StatusBadRequest)
-	}
-
-	blockID, err := backend.ParseUUID(searchReq.BlockID)
-	if err != nil {
-		return nil, httpError("parsing uuid", err, http.StatusBadRequest)
-	}
-
-	enc, err := backend.ParseEncoding(searchReq.Encoding)
-	if err != nil {
-		return nil, httpError("parsing encoding", err, http.StatusBadRequest)
-	}
-
-	dc, err := backend.DedicatedColumnsFromTempopb(searchReq.DedicatedColumns)
-	if err != nil {
-		return nil, httpError("parsing dedicated columns", err, http.StatusBadRequest)
-	}
-
-	// /giphy so meta
-	meta := &backend.BlockMeta{
-		Version:          searchReq.Version,
-		TenantID:         tenant,
-		Encoding:         enc,
-		IndexPageSize:    searchReq.IndexPageSize,
-		TotalRecords:     searchReq.TotalRecords,
-		BlockID:          blockID,
-		DataEncoding:     searchReq.DataEncoding,
-		Size_:            searchReq.Size_,
-		FooterSize:       searchReq.FooterSize,
-		DedicatedColumns: dc,
-	}
-
-	block, err := encoding.OpenBlock(meta, reader)
-	if err != nil {
-		return nil, httpError("creating backend block", err, http.StatusInternalServerError)
-	}
-
-	opts := common.SearchOptions{
-		StartPage:  int(searchReq.StartPage),
-		TotalPages: int(searchReq.PagesToSearch),
-		MaxBytes:   maxBytes,
-	}
-	cfg.Search.ApplyToOptions(&opts)
-
-	var resp *tempopb.SearchResponse
-
-	if api.IsTraceQLQuery(searchReq.SearchReq) {
-		engine := traceql.NewEngine()
-
-		spansetFetcher := traceql.NewSpansetFetcherWrapper(func(ctx context.Context, req traceql.FetchSpansRequest) (traceql.FetchSpansResponse, error) {
-			return block.Fetch(ctx, req, opts)
-		})
-		resp, err = engine.ExecuteSearch(r.Context(), searchReq.SearchReq, spansetFetcher)
-		if err != nil {
-			return nil, httpError("searching block", err, http.StatusInternalServerError)
-		}
-	} else {
-		resp, err = block.Search(r.Context(), searchReq.SearchReq, opts)
-		if err != nil {
-			return nil, httpError("searching block", err, http.StatusInternalServerError)
-		}
-	}
-
-	runtime.GC()
-
-	return resp, nil
-}
-
-func loadBackend() (backend.Reader, *tempodb.Config, error) {
-	readerOnce.Do(func() {
-		cfg, err := loadConfig()
-		if err != nil {
-			readerErr = err
-			return
-		}
-
-		var r backend.RawReader
-
-		// Create the backend with NewNoConfirm() to prevent an extra call to the various backends on
-		// startup. This extra call exists just to confirm the bucket is accessible and force the
-		// standard Tempo components to fail during startup. If permissions are not correct this Lambda
-		// will fail instantly anyway and in a heavy query environment the extra calls will start to add up.
-		switch cfg.Backend {
-		case backend.Local:
-			err = fmt.Errorf("local backend not supported for serverless functions")
-		case backend.GCS:
-			r, _, _, err = gcs.NewNoConfirm(cfg.GCS)
-		case backend.S3:
-			r, _, _, err = s3.NewNoConfirm(cfg.S3)
-		case backend.Azure:
-			r, _, _, err = azure.NewNoConfirm(cfg.Azure)
-		default:
-			err = fmt.Errorf("unknown backend %s", cfg.Backend)
-		}
-		if err != nil {
-			readerErr = err
-			return
-		}
-
-		readerConfig = cfg
-		reader = backend.NewReader(r)
-	})
-
-	return reader, readerConfig, readerErr
-}
-
-func loadConfig() (*tempodb.Config, error) {
-	defaultConfig := &tempodb.Config{
-		Search: &tempodb.SearchConfig{
-			ChunkSizeBytes:     tempodb.DefaultSearchChunkSizeBytes,
-			PrefetchTraceCount: tempodb.DefaultPrefetchTraceCount,
-		},
-		Local: &local.Config{},
-		GCS:   &gcs.Config{},
-		S3:    &s3.Config{},
-		Azure: &azure.Config{},
-	}
-
-	// horrible viper dance since it won't unmarshal to a struct from env: https://github.com/spf13/viper/issues/188
-	v := viper.NewWithOptions()
-	b, err := yaml.Marshal(defaultConfig)
-	if err != nil {
-		return nil, fmt.Errorf("failed to marshal default config: %w", err)
-	}
-	v.SetConfigType("yaml")
-	if err = v.MergeConfig(bytes.NewReader(b)); err != nil {
-		return nil, fmt.Errorf("failed to merge config: %w", err)
-	}
-
-	v.AutomaticEnv()
-	v.SetEnvPrefix(envConfigPrefix)
-	v.SetEnvKeyReplacer(strings.NewReplacer(".", "_", "-", "_"))
-
-	cfg := &tempodb.Config{}
-	err = v.Unmarshal(cfg, setTagName, setDecodeHooks)
-	if err != nil {
-		return nil, fmt.Errorf("failed to unmarshal config: %w", err)
-	}
-
-	return cfg, nil
-}
-
-// this forces mapstructure to use the yaml tag that is already defined
-// for all config struct fields
-func setTagName(d *mapstructure.DecoderConfig) {
-	d.TagName = "yaml"
-}
-
-// install all required decodeHooks so that viper will parse the yaml properly
-func setDecodeHooks(c *mapstructure.DecoderConfig) {
-	c.DecodeHook = mapstructure.ComposeDecodeHookFunc(
-		mapstructure.StringToTimeDurationHookFunc(),
-		mapstructure.StringToSliceHookFunc(","),
-		stringToFlagExt(),
-	)
-}
-
-// stringToFlagExt returns a DecodeHookFunc that converts
-// strings to dskit.FlagExt values.
-func stringToFlagExt() mapstructure.DecodeHookFunc {
-	return func(
-		f reflect.Type,
-		t reflect.Type,
-		data interface{},
-	) (interface{}, error) {
-		if f.Kind() != reflect.String {
-			return data, nil
-		}
-		if t != reflect.TypeOf(flagext.Secret{}) {
-			return data, nil
-		}
-		return flagext.SecretWithValue(data.(string)), nil
-	}
-}
-
-func httpError(action string, err error, status int) *HTTPError {
-	return &HTTPError{
-		Err:    fmt.Errorf("serverless [%s]: %w", action, err),
-		Status: status,
-	}
-}

cmd/tempo-serverless/handler_test.go

@@ -1,81 +0,0 @@
-package serverless
-
-import (
-	"os"
-	"testing"
-	"time"
-
-	"github.com/grafana/tempo/tempodb/backend"
-)
-
-func TestLoadConfig(t *testing.T) {
-	os.Setenv("TEMPO_GCS_BUCKET_NAME", "some-random-gcs-bucket")
-	os.Setenv("TEMPO_GCS_CHUNK_BUFFER_SIZE", "10")
-	os.Setenv("TEMPO_GCS_HEDGE_REQUESTS_AT", "400ms")
-	os.Setenv("TEMPO_BACKEND", backend.GCS)
-
-	// purposefully not using testfiy to reduce dependencies and keep the serverless packages small
-	cfg, err := loadConfig()
-	if err != nil {
-		t.Error("failed to load config:", err)
-		return
-	}
-	if cfg.Backend != backend.GCS {
-		t.Error("backend should be gcs", cfg.Backend)
-	}
-	if cfg.GCS.BucketName != "some-random-gcs-bucket" {
-		t.Error("gcs bucket name should be some-random-gcs-bucket", cfg.GCS.BucketName)
-	}
-	if cfg.GCS.ChunkBufferSize != 10 {
-		t.Error("gcs chunk buffer size should be 10", cfg.GCS.ChunkBufferSize)
-	}
-	if cfg.GCS.HedgeRequestsAt != 400*time.Millisecond {
-		t.Error("gcs hedge requests at should be 400ms", cfg.GCS.HedgeRequestsAt)
-	}
-}
-
-func TestLoadConfigS3(t *testing.T) {
-	os.Setenv("TEMPO_S3_BUCKET", "tempo")
-	os.Setenv("TEMPO_S3_ENDPOINT", "glerg")
-	os.Setenv("TEMPO_BACKEND", backend.S3)
-	os.Setenv("TEMPO_S3_ACCESS_KEY", "access")
-	os.Setenv("TEMPO_S3_SECRET_KEY", "secret")
-
-	cfg, err := loadConfig()
-	if err != nil {
-		t.Error("failed to load config", err)
-		return
-	}
-	if cfg.Backend != backend.S3 {
-		t.Error("backend should be s3", cfg.Backend)
-	}
-	if cfg.S3.Bucket != "tempo" {
-		t.Error("s3 bucket name should be tempo", cfg.S3.Bucket)
-	}
-	if cfg.S3.Endpoint != "glerg" {
-		t.Error("s3 endpoint should be glerg", cfg.S3.Endpoint)
-	}
-	if cfg.S3.AccessKey != "access" {
-		t.Error("s3 access key should be access", cfg.S3.AccessKey)
-	}
-	if cfg.S3.SecretKey.String() != "secret" {
-		t.Error("s3 secret key should be secret", cfg.S3.SecretKey)
-	}
-}
-
-func TestLoadConfigAzure(t *testing.T) {
-	os.Setenv("TEMPO_BACKEND", backend.Azure)
-	os.Setenv("TEMPO_AZURE_MAX_BUFFERS", "3")
-
-	cfg, err := loadConfig()
-	if err != nil {
-		t.Error("failed to load config", err)
-		return
-	}
-	if cfg.Backend != backend.Azure {
-		t.Error("backend should be azure", cfg.Backend)
-	}
-	if cfg.Azure.MaxBuffers != 3 {
-		t.Error("azure max buffers should be 3", cfg.Azure.MaxBuffers)
-	}
-}

cmd/tempo-serverless/lambda/Dockerfile

@@ -1,44 +0,0 @@
-#
-#  docker run -p 8080:8080 -p 9000:9000 tempo-serverless
-#
-#  to exercise the function after building using the raw lambda interface
-#    curl -d '{}' http://localhost:8080/2015-03-31/functions/function/invocations
-#    passed body should be serializable to: events.ALBTargetGroupRequest
-#    response body will be: events.ALBTargetGroupResponse
-#
-#  to exercise the function as if it were an http endpoint through an alb
-#    curl http://localhost:9000/?start=1000&end=1001&...
-#
-
-#
-# build the lambda and retrive the lambda-local-proxy
-#
-FROM golang:1.21.4 AS build
-
-# copy in the lambda. todo: build in container
-COPY lambda /
-
-# copy in https://github.com/treasure-data/lambda-local-proxy to include in the docker image.
-#   this project acts as an "alb" and translates normal http requests into Lambda invokes
-RUN    curl -L https://github.com/treasure-data/lambda-local-proxy/releases/download/v0.0.5/lambda-local-proxy_0.0.5_Linux_x86_64.tar.gz | tar xvz -C / \
-  && chmod +x /lambda-local-proxy
-
-#
-# https://docs.aws.amazon.com/lambda/latest/dg/images-create.html
-#
-FROM public.ecr.aws/lambda/go:1.2022.01.25.13
-
-COPY --from=build /lambda ${LAMBDA_TASK_ROOT}
-COPY --from=build /lambda-local-proxy /lambda-local-proxy
-
-RUN chmod +x /lambda-local-proxy
-
-RUN echo -e '\
-  #!/bin/bash \n\
-  AWS_REGION=dummy AWS_ACCESS_KEY_ID=dummy AWS_SECRET_ACCESS_KEY=dummy /lambda-local-proxy -e http://localhost:8080 -p 9000 -f function & \n\
-  exec /lambda-entrypoint.sh lambda \n\
-  ' > /run.sh
-RUN chmod +x /run.sh
-
-# run lambda-local-proxy and the actual lambda runtime itself.
-ENTRYPOINT /run.sh

cmd/tempo-serverless/lambda/go.mod

@@ -1,154 +0,0 @@
-module github.com/grafana/tempo/cmd/tempo-serverless/lambda
-
-go 1.23.3
-
-require (
-	github.com/aws/aws-lambda-go v1.28.0
-	github.com/gogo/protobuf v1.3.2
-	github.com/grafana/tempo v0.0.0-00010101000000-000000000000
-	github.com/stretchr/testify v1.10.0
-)
-
-require (
-	cloud.google.com/go v0.115.0 // indirect
-	cloud.google.com/go/auth v0.12.1 // indirect
-	cloud.google.com/go/auth/oauth2adapt v0.2.6 // indirect
-	cloud.google.com/go/compute/metadata v0.5.2 // indirect
-	cloud.google.com/go/iam v1.1.12 // indirect
-	cloud.google.com/go/storage v1.41.0 // indirect
-	github.com/Azure/azure-sdk-for-go/sdk/azcore v1.16.0 // indirect
-	github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.8.0 // indirect
-	github.com/Azure/azure-sdk-for-go/sdk/internal v1.10.0 // indirect
-	github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.5.0 // indirect
-	github.com/AzureAD/microsoft-authentication-library-for-go v1.2.3 // indirect
-	github.com/alecthomas/units v0.0.0-20240927000941-0f3dac36c52b // indirect
-	github.com/andybalholm/brotli v1.1.1 // indirect
-	github.com/apache/thrift v0.21.0 // indirect
-	github.com/aws/aws-sdk-go v1.55.5 // indirect
-	github.com/beorn7/perks v1.0.1 // indirect
-	github.com/cespare/xxhash/v2 v2.3.0 // indirect
-	github.com/coreos/go-systemd/v22 v22.5.0 // indirect
-	github.com/cristalhq/hedgedhttp v0.9.1 // indirect
-	github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect
-	github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect
-	github.com/dustin/go-humanize v1.0.1 // indirect
-	github.com/facette/natsort v0.0.0-20181210072756-2cd4dd1e2dcb // indirect
-	github.com/felixge/httpsnoop v1.0.4 // indirect
-	github.com/fsnotify/fsnotify v1.8.0 // indirect
-	github.com/go-ini/ini v1.67.0 // indirect
-	github.com/go-kit/log v0.2.1 // indirect
-	github.com/go-logfmt/logfmt v0.6.0 // indirect
-	github.com/go-logr/logr v1.4.2 // indirect
-	github.com/go-logr/stdr v1.2.2 // indirect
-	github.com/go-redis/redis/v8 v8.11.5 // indirect
-	github.com/goccy/go-json v0.10.4 // indirect
-	github.com/gogo/googleapis v1.4.1 // indirect
-	github.com/gogo/status v1.1.1 // indirect
-	github.com/golang-jwt/jwt/v5 v5.2.1 // indirect
-	github.com/golang/groupcache v0.0.0-20241129210726-2c02b8208cf8 // indirect
-	github.com/golang/protobuf v1.5.4 // indirect
-	github.com/golang/snappy v0.0.4 // indirect
-	github.com/google/s2a-go v0.1.8 // indirect
-	github.com/google/uuid v1.6.0 // indirect
-	github.com/googleapis/enterprise-certificate-proxy v0.3.4 // indirect
-	github.com/googleapis/gax-go/v2 v2.14.1 // indirect
-	github.com/gorilla/mux v1.8.1 // indirect
-	github.com/grafana/dskit v0.0.0-20241115082728-f2a7eb3aa0e9 // indirect
-	github.com/grafana/gomemcache v0.0.0-20241016125027-0a5bcc5aef40 // indirect
-	github.com/grafana/pyroscope-go/godeltaprof v0.1.8 // indirect
-	github.com/grafana/regexp v0.0.0-20240518133315-a468a5bfb3bc // indirect
-	github.com/hashicorp/hcl v1.0.0 // indirect
-	github.com/jaegertracing/jaeger v1.63.0 // indirect
-	github.com/jmespath/go-jmespath v0.4.0 // indirect
-	github.com/jpillora/backoff v1.0.0 // indirect
-	github.com/json-iterator/go v1.1.12 // indirect
-	github.com/klauspost/compress v1.17.11 // indirect
-	github.com/klauspost/cpuid/v2 v2.2.9 // indirect
-	github.com/kylelemons/godebug v1.1.0 // indirect
-	github.com/magiconair/properties v1.8.7 // indirect
-	github.com/mattn/go-runewidth v0.0.16 // indirect
-	github.com/miekg/dns v1.1.61 // indirect
-	github.com/minio/md5-simd v1.1.2 // indirect
-	github.com/minio/minio-go/v7 v7.0.82 // indirect
-	github.com/mitchellh/mapstructure v1.5.1-0.20231216201459-8508981c8b6c // indirect
-	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
-	github.com/modern-go/reflect2 v1.0.2 // indirect
-	github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
-	github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f // indirect
-	github.com/olekukonko/tablewriter v0.0.5 // indirect
-	github.com/open-telemetry/opentelemetry-collector-contrib/internal/coreinternal v0.116.0 // indirect
-	github.com/open-telemetry/opentelemetry-collector-contrib/pkg/translator/jaeger v0.116.0 // indirect
-	github.com/opentracing-contrib/go-grpc v0.0.0-20210225150812-73cb765af46e // indirect
-	github.com/opentracing-contrib/go-stdlib v1.0.0 // indirect
-	github.com/opentracing/opentracing-go v1.2.0 // indirect
-	github.com/parquet-go/parquet-go v0.23.1-0.20241011155651-6446d1d0d2fe // indirect
-	github.com/pelletier/go-toml/v2 v2.2.3 // indirect
-	github.com/pierrec/lz4/v4 v4.1.21 // indirect
-	github.com/pires/go-proxyproto v0.7.0 // indirect
-	github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c // indirect
-	github.com/pkg/errors v0.9.1 // indirect
-	github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect
-	github.com/prometheus/client_golang v1.20.5 // indirect
-	github.com/prometheus/client_model v0.6.1 // indirect
-	github.com/prometheus/common v0.61.0 // indirect
-	github.com/prometheus/exporter-toolkit v0.11.0 // indirect
-	github.com/prometheus/procfs v0.15.1 // indirect
-	github.com/prometheus/prometheus v0.54.1 // indirect
-	github.com/rivo/uniseg v0.4.7 // indirect
-	github.com/rs/xid v1.6.0 // indirect
-	github.com/sagikazarmark/locafero v0.4.0 // indirect
-	github.com/sagikazarmark/slog-shim v0.1.0 // indirect
-	github.com/sercand/kuberesolver/v5 v5.1.1 // indirect
-	github.com/sony/gobreaker v0.4.1 // indirect
-	github.com/sourcegraph/conc v0.3.0 // indirect
-	github.com/spaolacci/murmur3 v1.1.0 // indirect
-	github.com/spf13/afero v1.11.0 // indirect
-	github.com/spf13/cast v1.6.0 // indirect
-	github.com/spf13/pflag v1.0.5 // indirect
-	github.com/spf13/viper v1.19.0 // indirect
-	github.com/subosito/gotenv v1.6.0 // indirect
-	github.com/uber-go/atomic v1.4.0 // indirect
-	github.com/uber/jaeger-client-go v2.30.0+incompatible // indirect
-	github.com/uber/jaeger-lib v2.4.1+incompatible // indirect
-	github.com/willf/bitset v1.1.11 // indirect
-	github.com/willf/bloom v2.0.3+incompatible // indirect
-	go.opencensus.io v0.24.0 // indirect
-	go.opentelemetry.io/auto/sdk v1.1.0 // indirect
-	go.opentelemetry.io/collector/pdata v1.22.0 // indirect
-	go.opentelemetry.io/collector/semconv v0.116.0 // indirect
-	go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.58.0 // indirect
-	go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.58.0 // indirect
-	go.opentelemetry.io/otel v1.33.0 // indirect
-	go.opentelemetry.io/otel/metric v1.33.0 // indirect
-	go.opentelemetry.io/otel/trace v1.33.0 // indirect
-	go.uber.org/atomic v1.11.0 // indirect
-	go.uber.org/multierr v1.11.0 // indirect
-	go.uber.org/zap v1.27.0 // indirect
-	golang.org/x/crypto v0.31.0 // indirect
-	golang.org/x/exp v0.0.0-20241108190413-2d47ceb2692f // indirect
-	golang.org/x/mod v0.22.0 // indirect
-	golang.org/x/net v0.33.0 // indirect
-	golang.org/x/oauth2 v0.24.0 // indirect
-	golang.org/x/sync v0.10.0 // indirect
-	golang.org/x/sys v0.28.0 // indirect
-	golang.org/x/text v0.21.0 // indirect
-	golang.org/x/time v0.8.0 // indirect
-	golang.org/x/tools v0.27.0 // indirect
-	google.golang.org/api v0.211.0 // indirect
-	google.golang.org/genproto v0.0.0-20240730163845-b1a4ccb954bf // indirect
-	google.golang.org/genproto/googleapis/api v0.0.0-20241209162323-e6fa225c2576 // indirect
-	google.golang.org/genproto/googleapis/rpc v0.0.0-20241209162323-e6fa225c2576 // indirect
-	google.golang.org/grpc v1.68.1 // indirect
-	google.golang.org/protobuf v1.35.2 // indirect
-	gopkg.in/ini.v1 v1.67.0 // indirect
-	gopkg.in/yaml.v2 v2.4.0 // indirect
-	gopkg.in/yaml.v3 v3.0.1 // indirect
-)
-
-replace github.com/grafana/tempo => ../../../
-
-replace (
-	github.com/bradfitz/gomemcache => github.com/themihai/gomemcache v0.0.0-20180902122335-24332e2d58ab
-	k8s.io/api => k8s.io/api v0.20.4
-	k8s.io/client-go => k8s.io/client-go v0.20.4
-)

cmd/tempo-serverless/lambda/go.sum

@@ -1,467 +0,0 @@
-cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
-cloud.google.com/go v0.115.0 h1:CnFSK6Xo3lDYRoBKEcAtia6VSC837/ZkJuRduSFnr14=
-cloud.google.com/go v0.115.0/go.mod h1:8jIM5vVgoAEoiVxQ/O4BFTfHqulPZgs/ufEzMcFMdWU=
-cloud.google.com/go/auth v0.12.1 h1:n2Bj25BUMM0nvE9D2XLTiImanwZhO3DkfWSYS/SAJP4=
-cloud.google.com/go/auth v0.12.1/go.mod h1:BFMu+TNpF3DmvfBO9ClqTR/SiqVIm7LukKF9mbendF4=
-cloud.google.com/go/auth/oauth2adapt v0.2.6 h1:V6a6XDu2lTwPZWOawrAa9HUK+DB2zfJyTuciBG5hFkU=
-cloud.google.com/go/auth/oauth2adapt v0.2.6/go.mod h1:AlmsELtlEBnaNTL7jCj8VQFLy6mbZv0s4Q7NGBeQ5E8=
-cloud.google.com/go/compute/metadata v0.5.2 h1:UxK4uu/Tn+I3p2dYWTfiX4wva7aYlKixAHn3fyqngqo=
-cloud.google.com/go/compute/metadata v0.5.2/go.mod h1:C66sj2AluDcIqakBq/M8lw8/ybHgOZqin2obFxa/E5k=
-cloud.google.com/go/iam v1.1.12 h1:JixGLimRrNGcxvJEQ8+clfLxPlbeZA6MuRJ+qJNQ5Xw=
-cloud.google.com/go/iam v1.1.12/go.mod h1:9LDX8J7dN5YRyzVHxwQzrQs9opFFqn0Mxs9nAeB+Hhg=
-cloud.google.com/go/storage v1.41.0 h1:RusiwatSu6lHeEXe3kglxakAmAbfV+rhtPqA6i8RBx0=
-cloud.google.com/go/storage v1.41.0/go.mod h1:J1WCa/Z2FcgdEDuPUY8DxT5I+d9mFKsCepp5vR6Sq80=
-github.com/Azure/azure-sdk-for-go/sdk/azcore v1.16.0 h1:JZg6HRh6W6U4OLl6lk7BZ7BLisIzM9dG1R50zUk9C/M=
-github.com/Azure/azure-sdk-for-go/sdk/azcore v1.16.0/go.mod h1:YL1xnZ6QejvQHWJrX/AvhFl4WW4rqHVoKspWNVwFk0M=
-github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.8.0 h1:B/dfvscEQtew9dVuoxqxrUKKv8Ih2f55PydknDamU+g=
-github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.8.0/go.mod h1:fiPSssYvltE08HJchL04dOy+RD4hgrjph0cwGGMntdI=
-github.com/Azure/azure-sdk-for-go/sdk/azidentity/cache v0.3.0 h1:+m0M/LFxN43KvULkDNfdXOgrjtg6UYJPFBJyuEcRCAw=
-github.com/Azure/azure-sdk-for-go/sdk/azidentity/cache v0.3.0/go.mod h1:PwOyop78lveYMRs6oCxjiVyBdyCgIYH6XHIVZO9/SFQ=
-github.com/Azure/azure-sdk-for-go/sdk/internal v1.10.0 h1:ywEEhmNahHBihViHepv3xPBn1663uRv2t2q/ESv9seY=
-github.com/Azure/azure-sdk-for-go/sdk/internal v1.10.0/go.mod h1:iZDifYGJTIgIIkYRNWPENUnqx6bJ2xnSDFI2tjwZNuY=
-github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/storage/armstorage v1.6.0 h1:PiSrjRPpkQNjrM8H0WwKMnZUdu1RGMtd/LdGKUrOo+c=
-github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/storage/armstorage v1.6.0/go.mod h1:oDrbWx4ewMylP7xHivfgixbfGBT6APAwsSoHRKotnIc=
-github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.5.0 h1:mlmW46Q0B79I+Aj4azKC6xDMFN9a9SyZWESlGWYXbFs=
-github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.5.0/go.mod h1:PXe2h+LKcWTX9afWdZoHyODqR4fBa5boUM/8uJfZ0Jo=
-github.com/AzureAD/microsoft-authentication-extensions-for-go/cache v0.1.1 h1:WJTmL004Abzc5wDB5VtZG2PJk5ndYDgVacGqfirKxjM=
-github.com/AzureAD/microsoft-authentication-extensions-for-go/cache v0.1.1/go.mod h1:tCcJZ0uHAmvjsVYzEFivsRTN00oz5BEsRgQHu5JZ9WE=
-github.com/AzureAD/microsoft-authentication-library-for-go v1.2.3 h1:6LyjnnaLpcOKK0fbYisI+mb8CE7iNe7i89nMNQxFxs8=
-github.com/AzureAD/microsoft-authentication-library-for-go v1.2.3/go.mod h1:wP83P5OoQ5p6ip3ScPr0BAq0BvuPAvacpEuSzyouqAI=
-github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
-github.com/HdrHistogram/hdrhistogram-go v1.1.2 h1:5IcZpTvzydCQeHzK4Ef/D5rrSqwxob0t8PQPMybUNFM=
-github.com/HdrHistogram/hdrhistogram-go v1.1.2/go.mod h1:yDgFjdqOqDEKOvasDdhWNXYg9BVp4O+o5f6V/ehm6Oo=
-github.com/alecthomas/units v0.0.0-20240927000941-0f3dac36c52b h1:mimo19zliBX/vSQ6PWWSL9lK8qwHozUj03+zLoEB8O0=
-github.com/alecthomas/units v0.0.0-20240927000941-0f3dac36c52b/go.mod h1:fvzegU4vN3H1qMT+8wDmzjAcDONcgo2/SZ/TyfdUOFs=
-github.com/alicebob/gopher-json v0.0.0-20230218143504-906a9b012302 h1:uvdUDbHQHO85qeSydJtItA4T55Pw6BtAejd0APRJOCE=
-github.com/alicebob/gopher-json v0.0.0-20230218143504-906a9b012302/go.mod h1:SGnFV6hVsYE877CKEZ6tDNTjaSXYUk6QqoIK6PrAtcc=
-github.com/alicebob/miniredis v2.5.0+incompatible h1:yBHoLpsyjupjz3NL3MhKMVkR41j82Yjf3KFv7ApYzUI=
-github.com/alicebob/miniredis/v2 v2.21.0 h1:CdmwIlKUWFBDS+4464GtQiQ0R1vpzOgu4Vnd74rBL7M=
-github.com/alicebob/miniredis/v2 v2.21.0/go.mod h1:XNqvJdQJv5mSuVMc0ynneafpnL/zv52acZ6kqeS0t88=
-github.com/andybalholm/brotli v1.1.1 h1:PR2pgnyFznKEugtsUo0xLdDop5SKXd5Qf5ysW+7XdTA=
-github.com/andybalholm/brotli v1.1.1/go.mod h1:05ib4cKhjx3OQYUY22hTVd34Bc8upXjOLL2rKwwZBoA=
-github.com/apache/thrift v0.21.0 h1:tdPmh/ptjE1IJnhbhrcl2++TauVjy242rkV/UzJChnE=
-github.com/apache/thrift v0.21.0/go.mod h1:W1H8aR/QRtYNvrPeFXBtobyRkd0/YVhTc6i07XIAgDw=
-github.com/aws/aws-lambda-go v1.28.0 h1:fZiik1PZqW2IyAN4rj+Y0UBaO1IDFlsNo9Zz/XnArK4=
-github.com/aws/aws-lambda-go v1.28.0/go.mod h1:jJmlefzPfGnckuHdXX7/80O3BvUUi12XOkbv4w9SGLU=
-github.com/aws/aws-sdk-go v1.55.5 h1:KKUZBfBoyqy5d3swXyiC7Q76ic40rYcbqH7qjh59kzU=
-github.com/aws/aws-sdk-go v1.55.5/go.mod h1:eRwEWoyTWFMVYVQzKMNHWP5/RV4xIUGMQfXQHfHkpNU=
-github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
-github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw=
-github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
-github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs=
-github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
-github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
-github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc=
-github.com/coreos/go-systemd/v22 v22.5.0 h1:RrqgGjYQKalulkV8NGVIfkXQf6YYmOyiJKk8iXXhfZs=
-github.com/coreos/go-systemd/v22 v22.5.0/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc=
-github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU=
-github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU=
-github.com/cristalhq/hedgedhttp v0.9.1 h1:g68L9cf8uUyQKQJwciD0A1Vgbsz+QgCjuB1I8FAsCDs=
-github.com/cristalhq/hedgedhttp v0.9.1/go.mod h1:XkqWU6qVMutbhW68NnzjWrGtH8NUx1UfYqGYtHVKIsI=
-github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
-github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
-github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM=
-github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
-github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f h1:lO4WD4F/rVNCu3HqELle0jiPLLBs70cWOduZpkS1E78=
-github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f/go.mod h1:cuUVRXasLTGF7a8hSLbxyZXjz+1KgoB3wDUb6vlszIc=
-github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY=
-github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto=
-github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
-github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
-github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98=
-github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c=
-github.com/facette/natsort v0.0.0-20181210072756-2cd4dd1e2dcb h1:IT4JYU7k4ikYg1SCxNI1/Tieq/NFvh6dzLdgi7eu0tM=
-github.com/facette/natsort v0.0.0-20181210072756-2cd4dd1e2dcb/go.mod h1:bH6Xx7IW64qjjJq8M2u4dxNaBiDfKK+z/3eGDpXEQhc=
-github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg=
-github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U=
-github.com/frankban/quicktest v1.14.6 h1:7Xjx+VpznH+oBnejlPUj8oUpdxnVs4f8XU8WnHkI4W8=
-github.com/frankban/quicktest v1.14.6/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7zb5vbUoiM6w0=
-github.com/fsnotify/fsnotify v1.8.0 h1:dAwr6QBTBZIkG8roQaJjGof0pp0EeF+tNV7YBP3F/8M=
-github.com/fsnotify/fsnotify v1.8.0/go.mod h1:8jBTzvmWwFyi3Pb8djgCCO5IBqzKJ/Jwo8TRcHyHii0=
-github.com/go-ini/ini v1.67.0 h1:z6ZrTEZqSWOTyH2FlglNbNgARyHG8oLW9gMELqKr06A=
-github.com/go-ini/ini v1.67.0/go.mod h1:ByCAeIL28uOIIG0E3PJtZPDL8WnHpFKFOtgjp+3Ies8=
-github.com/go-kit/log v0.2.1 h1:MRVx0/zhvdseW+Gza6N9rVzU/IVzaeE1SFI4raAhmBU=
-github.com/go-kit/log v0.2.1/go.mod h1:NwTd00d/i8cPZ3xOwwiv2PO5MOcx78fFErGNcVmBjv0=
-github.com/go-logfmt/logfmt v0.6.0 h1:wGYYu3uicYdqXVgoYbvnkrPVXkuLM1p1ifugDMEdRi4=
-github.com/go-logfmt/logfmt v0.6.0/go.mod h1:WYhtIu8zTZfxdn5+rREduYbwxfcBr/Vr6KEVveWlfTs=
-github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A=
-github.com/go-logr/logr v1.4.2 h1:6pFjapn8bFcIbiKo3XT4j/BhANplGihG6tvd+8rYgrY=
-github.com/go-logr/logr v1.4.2/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY=
-github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag=
-github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE=
-github.com/go-redis/redis/v8 v8.11.5 h1:AcZZR7igkdvfVmQTPnu9WE37LRrO/YrBH5zWyjDC0oI=
-github.com/go-redis/redis/v8 v8.11.5/go.mod h1:gREzHqY1hg6oD9ngVRbLStwAWKhA0FEgq8Jd4h5lpwo=
-github.com/goccy/go-json v0.10.4 h1:JSwxQzIqKfmFX1swYPpUThQZp/Ka4wzJdK0LWVytLPM=
-github.com/goccy/go-json v0.10.4/go.mod h1:oq7eo15ShAhp70Anwd5lgX2pLfOS3QCiwU/PULtXL6M=
-github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA=
-github.com/gogo/googleapis v0.0.0-20180223154316-0cd9801be74a/go.mod h1:gf4bu3Q80BeJ6H1S1vYPm8/ELATdvryBaNFGgqEef3s=
-github.com/gogo/googleapis v1.4.1 h1:1Yx4Myt7BxzvUr5ldGSbwYiZG6t9wGBZ+8/fX3Wvtq0=
-github.com/gogo/googleapis v1.4.1/go.mod h1:2lpHqI5OcWCtVElxXnPt+s8oJvMpySlOyM6xDCrzib4=
-github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q=
-github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q=
-github.com/gogo/status v1.1.1 h1:DuHXlSFHNKqTQ+/ACf5Vs6r4X/dH2EgIzR9Vr+H65kg=
-github.com/gogo/status v1.1.1/go.mod h1:jpG3dM5QPcqu19Hg8lkUhBFBa3TcLs1DG7+2Jqci7oU=
-github.com/golang-jwt/jwt/v5 v5.2.1 h1:OuVbFODueb089Lh128TAcimifWaLhJwVflnrgM17wHk=
-github.com/golang-jwt/jwt/v5 v5.2.1/go.mod h1:pqrtFR0X4osieyHYxtmOUWsAWrfe1Q5UVIyoH402zdk=
-github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
-github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
-github.com/golang/groupcache v0.0.0-20241129210726-2c02b8208cf8 h1:f+oWsMOmNPc8JmEHVZIycC7hBoQxHH9pNKQORJNozsQ=
-github.com/golang/groupcache v0.0.0-20241129210726-2c02b8208cf8/go.mod h1:wcDNUvekVysuuOpQKo3191zZyTpiI6se1N1ULghS0sw=
-github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
-github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
-github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
-github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8=
-github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA=
-github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs=
-github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w=
-github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0=
-github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8=
-github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI=
-github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek=
-github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps=
-github.com/golang/snappy v0.0.4 h1:yAGX7huGHXlcLOEtBnF4w7FQwA26wojNCwOYAEhLjQM=
-github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
-github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=
-github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
-github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
-github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
-github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
-github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
-github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI=
-github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
-github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
-github.com/google/martian/v3 v3.3.3 h1:DIhPTQrbPkgs2yJYdXU/eNACCG5DVQjySNRNlflZ9Fc=
-github.com/google/martian/v3 v3.3.3/go.mod h1:iEPrYcgCF7jA9OtScMFQyAlZZ4YXTKEtJ1E6RWzmBA0=
-github.com/google/s2a-go v0.1.8 h1:zZDs9gcbt9ZPLV0ndSyQk6Kacx2g/X+SKYovpnz3SMM=
-github.com/google/s2a-go v0.1.8/go.mod h1:6iNWHTpQ+nfNRN5E00MSdfDwVesa8hhS32PhPO8deJA=
-github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
-github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
-github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
-github.com/googleapis/enterprise-certificate-proxy v0.3.4 h1:XYIDZApgAnrN1c855gTgghdIA6Stxb52D5RnLI1SLyw=
-github.com/googleapis/enterprise-certificate-proxy v0.3.4/go.mod h1:YKe7cfqYXjKGpGvmSg28/fFvhNzinZQm8DGnaburhGA=
-github.com/googleapis/gax-go/v2 v2.14.1 h1:hb0FFeiPaQskmvakKu5EbCbpntQn48jyHuvrkurSS/Q=
-github.com/googleapis/gax-go/v2 v2.14.1/go.mod h1:Hb/NubMaVM88SrNkvl8X/o8XWwDJEPqouaLeN2IUxoA=
-github.com/gorilla/mux v1.8.1 h1:TuBL49tXwgrFYWhqrNgrUNEY92u81SPhu7sTdzQEiWY=
-github.com/gorilla/mux v1.8.1/go.mod h1:AKf9I4AEqPTmMytcMc0KkNouC66V3BtZ4qD5fmWSiMQ=
-github.com/grafana/dskit v0.0.0-20241115082728-f2a7eb3aa0e9 h1:Dx7+6aU/fhwD2vkMr0PUcyxGat1sjUssHAKQKaS7sDM=
-github.com/grafana/dskit v0.0.0-20241115082728-f2a7eb3aa0e9/go.mod h1:SPLNCARd4xdjCkue0O6hvuoveuS1dGJjDnfxYe405YQ=
-github.com/grafana/gomemcache v0.0.0-20241016125027-0a5bcc5aef40 h1:1TeKhyS+pvzOeyLV1XPZsiqebnKky/AKS3pJNNbHVPo=
-github.com/grafana/gomemcache v0.0.0-20241016125027-0a5bcc5aef40/go.mod h1:IGRj8oOoxwJbHBYl1+OhS9UjQR0dv6SQOep7HqmtyFU=
-github.com/grafana/pyroscope-go/godeltaprof v0.1.8 h1:iwOtYXeeVSAeYefJNaxDytgjKtUuKQbJqgAIjlnicKg=
-github.com/grafana/pyroscope-go/godeltaprof v0.1.8/go.mod h1:2+l7K7twW49Ct4wFluZD3tZ6e0SjanjcUUBPVD/UuGU=
-github.com/grafana/regexp v0.0.0-20240518133315-a468a5bfb3bc h1:GN2Lv3MGO7AS6PrRoT6yV5+wkrOpcszoIsO4+4ds248=
-github.com/grafana/regexp v0.0.0-20240518133315-a468a5bfb3bc/go.mod h1:+JKpmjMGhpgPL+rXZ5nsZieVzvarn86asRlBg4uNGnk=
-github.com/grpc-ecosystem/grpc-opentracing v0.0.0-20180507213350-8e809c8a8645/go.mod h1:6iZfnjpejD4L/4DwD7NryNaJyCQdzwWwH2MWhCA90Kw=
-github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4=
-github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ=
-github.com/hexops/gotextdiff v1.0.3 h1:gitA9+qJrrTCsiCl7+kh75nPqQt1cx4ZkudSTLoUqJM=
-github.com/hexops/gotextdiff v1.0.3/go.mod h1:pSWU5MAI3yDq+fZBTazCSJysOMbxWL1BSow5/V2vxeg=
-github.com/jaegertracing/jaeger v1.63.0 h1:4p1lg4DyZ1rZl5gCt6nnQ0nllcXVLJ96sTSbd+rnOEo=
-github.com/jaegertracing/jaeger v1.63.0/go.mod h1:ZCqGiUdcAPC8HwqQGOjl89HDNEqseArdWcH+yoZxRtk=
-github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg=
-github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo=
-github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8=
-github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U=
-github.com/jpillora/backoff v1.0.0 h1:uvFg412JmmHBHw7iwprIxkPMI+sGQ4kzOWsMeHnm2EA=
-github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4=
-github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM=
-github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo=
-github.com/keybase/go-keychain v0.0.0-20231219164618-57a3676c3af6 h1:IsMZxCuZqKuao2vNdfD82fjjgPLfyHLpR41Z88viRWs=
-github.com/keybase/go-keychain v0.0.0-20231219164618-57a3676c3af6/go.mod h1:3VeWNIJaW+O5xpRQbPp0Ybqu1vJd/pm7s2F473HRrkw=
-github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8=
-github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
-github.com/klauspost/compress v1.17.11 h1:In6xLpyWOi1+C7tXUUWv2ot1QvBjxevKAaI6IXrJmUc=
-github.com/klauspost/compress v1.17.11/go.mod h1:pMDklpSncoRMuLFrf1W9Ss9KT+0rH90U12bZKk7uwG0=
-github.com/klauspost/cpuid/v2 v2.0.1/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg=
-github.com/klauspost/cpuid/v2 v2.2.9 h1:66ze0taIn2H33fBvCkXuv9BmCwDfafmiIVpKV9kKGuY=
-github.com/klauspost/cpuid/v2 v2.2.9/go.mod h1:rqkxqrZ1EhYM9G+hXH7YdowN5R5RGN6NK4QwQ3WMXF8=
-github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE=
-github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk=
-github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
-github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
-github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc=
-github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw=
-github.com/magiconair/properties v1.8.7 h1:IeQXZAiQcpL9mgcAe1Nu6cX9LLw6ExEHKjN0VQdvPDY=
-github.com/magiconair/properties v1.8.7/go.mod h1:Dhd985XPs7jluiymwWYZ0G4Z61jb3vdS329zhj2hYo0=
-github.com/mattn/go-runewidth v0.0.9/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI=
-github.com/mattn/go-runewidth v0.0.16 h1:E5ScNMtiwvlvB5paMFdw9p4kSQzbXFikJ5SQO6TULQc=
-github.com/mattn/go-runewidth v0.0.16/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w=
-github.com/miekg/dns v1.1.61 h1:nLxbwF3XxhwVSm8g9Dghm9MHPaUZuqhPiGL+675ZmEs=
-github.com/miekg/dns v1.1.61/go.mod h1:mnAarhS3nWaW+NVP2wTkYVIZyHNJ098SJZUki3eykwQ=
-github.com/minio/md5-simd v1.1.2 h1:Gdi1DZK69+ZVMoNHRXJyNcxrMA4dSxoYHZSQbirFg34=
-github.com/minio/md5-simd v1.1.2/go.mod h1:MzdKDxYpY2BT9XQFocsiZf/NKVtR7nkE4RoEpN+20RM=
-github.com/minio/minio-go/v7 v7.0.82 h1:tWfICLhmp2aFPXL8Tli0XDTHj2VB/fNf0PC1f/i1gRo=
-github.com/minio/minio-go/v7 v7.0.82/go.mod h1:84gmIilaX4zcvAWWzJ5Z1WI5axN+hAbM5w25xf8xvC0=
-github.com/mitchellh/mapstructure v1.5.1-0.20231216201459-8508981c8b6c h1:cqn374mizHuIWj+OSJCajGr/phAmuMug9qIX3l9CflE=
-github.com/mitchellh/mapstructure v1.5.1-0.20231216201459-8508981c8b6c/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo=
-github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
-github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg=
-github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
-github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M=
-github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk=
-github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA=
-github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ=
-github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f h1:KUppIJq7/+SVif2QVs3tOP0zanoHgBEVAwHxUSIzRqU=
-github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=
-github.com/nxadm/tail v1.4.8 h1:nPr65rt6Y5JFSKQO7qToXr7pePgD6Gwiw05lkbyAQTE=
-github.com/nxadm/tail v1.4.8/go.mod h1:+ncqLTQzXmGhMZNUePPaPqPvBxHAIsmXswZKocGu+AU=
-github.com/olekukonko/tablewriter v0.0.5 h1:P2Ga83D34wi1o9J6Wh1mRuqd4mF/x/lgBS7N7AbDhec=
-github.com/olekukonko/tablewriter v0.0.5/go.mod h1:hPp6KlRPjbx+hW8ykQs1w3UBbZlj6HuIJcUGPhkA7kY=
-github.com/onsi/ginkgo v1.16.5 h1:8xi0RTUf59SOSfEtZMvwTvXYMzG4gV23XVHOZiXNtnE=
-github.com/onsi/ginkgo v1.16.5/go.mod h1:+E8gABHa3K6zRBolWtd+ROzc/U5bkGt0FwiG042wbpU=
-github.com/onsi/gomega v1.24.0 h1:+0glovB9Jd6z3VR+ScSwQqXVTIfJcGA9UBM8yzQxhqg=
-github.com/onsi/gomega v1.24.0/go.mod h1:Z/NWtiqwBrwUt4/2loMmHL63EDLnYHmVbuBpDr2vQAg=
-github.com/open-telemetry/opentelemetry-collector-contrib/internal/coreinternal v0.116.0 h1:xDbf946Zm0rTzWcYEyUfU0Ft2KthhaH4xrNm303vpbI=
-github.com/open-telemetry/opentelemetry-collector-contrib/internal/coreinternal v0.116.0/go.mod h1:yuIyOGmQJOn37u6NVfG8yOCzVvwboqnt+pjOSTvDeLo=
-github.com/open-telemetry/opentelemetry-collector-contrib/pkg/translator/jaeger v0.116.0 h1:wNQ51LBQjPIW7ptgZOMaJT60CnMoP1t3kxgldR7uv9Q=
-github.com/open-telemetry/opentelemetry-collector-contrib/pkg/translator/jaeger v0.116.0/go.mod h1:lx+O1oAnKYPg5KaFW5Ss1Xh+zvCDKZPm73MDAFjPgak=
-github.com/opentracing-contrib/go-grpc v0.0.0-20210225150812-73cb765af46e h1:4cPxUYdgaGzZIT5/j0IfqOrrXmq6bG8AwvwisMXpdrg=
-github.com/opentracing-contrib/go-grpc v0.0.0-20210225150812-73cb765af46e/go.mod h1:DYR5Eij8rJl8h7gblRrOZ8g0kW1umSpKqYIBTgeDtLo=
-github.com/opentracing-contrib/go-stdlib v1.0.0 h1:TBS7YuVotp8myLon4Pv7BtCBzOTo1DeZCld0Z63mW2w=
-github.com/opentracing-contrib/go-stdlib v1.0.0/go.mod h1:qtI1ogk+2JhVPIXVc6q+NHziSmy2W5GbdQZFUHADCBU=
-github.com/opentracing/opentracing-go v1.1.0/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o=
-github.com/opentracing/opentracing-go v1.2.0 h1:uEJPy/1a5RIPAJ0Ov+OIO8OxWu77jEv+1B0VhjKrZUs=
-github.com/opentracing/opentracing-go v1.2.0/go.mod h1:GxEUsuufX4nBwe+T+Wl9TAgYrxe9dPLANfrWvHYVTgc=
-github.com/parquet-go/parquet-go v0.23.1-0.20241011155651-6446d1d0d2fe h1:oUJ5TPnrEK/z+/PeoLL+jCgfngAZIDMyhZASetRcYYg=
-github.com/parquet-go/parquet-go v0.23.1-0.20241011155651-6446d1d0d2fe/go.mod h1:OqBBRGBl7+llplCvDMql8dEKaDqjaFA/VAPw+OJiNiw=
-github.com/pelletier/go-toml/v2 v2.2.3 h1:YmeHyLY8mFWbdkNWwpr+qIL2bEqT0o95WSdkNHvL12M=
-github.com/pelletier/go-toml/v2 v2.2.3/go.mod h1:MfCQTFTvCcUyyvvwm1+G6H/jORL20Xlb6rzQu9GuUkc=
-github.com/pierrec/lz4/v4 v4.1.21 h1:yOVMLb6qSIDP67pl/5F7RepeKYu/VmTyEXvuMI5d9mQ=
-github.com/pierrec/lz4/v4 v4.1.21/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4=
-github.com/pires/go-proxyproto v0.7.0 h1:IukmRewDQFWC7kfnb66CSomk2q/seBuilHBYFwyq0Hs=
-github.com/pires/go-proxyproto v0.7.0/go.mod h1:Vz/1JPY/OACxWGQNIRY2BeyDmpoaWmEP40O9LbuiFR4=
-github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c h1:+mdjkGKdHQG3305AYmdv1U2eRNDiU2ErMBj1gwrq8eQ=
-github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c/go.mod h1:7rwL4CYBLnjLxUqIJNnCWiEdr3bn6IUYi15bNlnbCCU=
-github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
-github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
-github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
-github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U=
-github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
-github.com/prometheus/client_golang v1.20.5 h1:cxppBPuYhUnsO6yo/aoRol4L7q7UFfdm+bR9r+8l63Y=
-github.com/prometheus/client_golang v1.20.5/go.mod h1:PIEt8X02hGcP8JWbeHyeZ53Y/jReSnHgO035n//V5WE=
-github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
-github.com/prometheus/client_model v0.6.1 h1:ZKSh/rekM+n3CeS952MLRAdFwIKqeY8b62p8ais2e9E=
-github.com/prometheus/client_model v0.6.1/go.mod h1:OrxVMOVHjw3lKMa8+x6HeMGkHMQyHDk9E3jmP2AmGiY=
-github.com/prometheus/common v0.61.0 h1:3gv/GThfX0cV2lpO7gkTUwZru38mxevy90Bj8YFSRQQ=
-github.com/prometheus/common v0.61.0/go.mod h1:zr29OCN/2BsJRaFwG8QOBr41D6kkchKbpeNH7pAjb/s=
-github.com/prometheus/exporter-toolkit v0.11.0 h1:yNTsuZ0aNCNFQ3aFTD2uhPOvr4iD7fdBvKPAEGkNf+g=
-github.com/prometheus/exporter-toolkit v0.11.0/go.mod h1:BVnENhnNecpwoTLiABx7mrPB/OLRIgN74qlQbV+FK1Q=
-github.com/prometheus/procfs v0.15.1 h1:YagwOFzUgYfKKHX6Dr+sHT7km/hxC76UB0learggepc=
-github.com/prometheus/procfs v0.15.1/go.mod h1:fB45yRUv8NstnjriLhBQLuOUt+WW4BsoGhij/e3PBqk=
-github.com/prometheus/prometheus v0.54.1 h1:vKuwQNjnYN2/mDoWfHXDhAsz/68q/dQDb+YbcEqU7MQ=
-github.com/prometheus/prometheus v0.54.1/go.mod h1:xlLByHhk2g3ycakQGrMaU8K7OySZx98BzeCR99991NY=
-github.com/redis/go-redis/v9 v9.6.1 h1:HHDteefn6ZkTtY5fGUE8tj8uy85AHk6zP7CpzIAM0y4=
-github.com/redis/go-redis/v9 v9.6.1/go.mod h1:0C0c6ycQsdpVNQpxb1njEQIqkx5UcsM8FJCQLgE9+RA=
-github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc=
-github.com/rivo/uniseg v0.4.7 h1:WUdvkW8uEhrYfLC4ZzdpI2ztxP1I582+49Oc5Mq64VQ=
-github.com/rivo/uniseg v0.4.7/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88=
-github.com/rogpeppe/go-internal v1.13.1 h1:KvO1DLK/DRN07sQ1LQKScxyZJuNnedQ5/wKSR38lUII=
-github.com/rogpeppe/go-internal v1.13.1/go.mod h1:uMEvuHeurkdAXX61udpOXGD/AzZDWNMNyH2VO9fmH0o=
-github.com/rs/xid v1.6.0 h1:fV591PaemRlL6JfRxGDEPl69wICngIQ3shQtzfy2gxU=
-github.com/rs/xid v1.6.0/go.mod h1:7XoLgs4eV+QndskICGsho+ADou8ySMSjJKDIan90Nz0=
-github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
-github.com/sagikazarmark/locafero v0.4.0 h1:HApY1R9zGo4DBgr7dqsTH/JJxLTTsOt7u6keLGt6kNQ=
-github.com/sagikazarmark/locafero v0.4.0/go.mod h1:Pe1W6UlPYUk/+wc/6KFhbORCfqzgYEpgQ3O5fPuL3H4=
-github.com/sagikazarmark/slog-shim v0.1.0 h1:diDBnUNK9N/354PgrxMywXnAwEr1QZcOr6gto+ugjYE=
-github.com/sagikazarmark/slog-shim v0.1.0/go.mod h1:SrcSrq8aKtyuqEI1uvTDTK1arOWRIczQRv+GVI1AkeQ=
-github.com/sercand/kuberesolver/v5 v5.1.1 h1:CYH+d67G0sGBj7q5wLK61yzqJJ8gLLC8aeprPTHb6yY=
-github.com/sercand/kuberesolver/v5 v5.1.1/go.mod h1:Fs1KbKhVRnB2aDWN12NjKCB+RgYMWZJ294T3BtmVCpQ=
-github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc=
-github.com/sony/gobreaker v0.4.1 h1:oMnRNZXX5j85zso6xCPRNPtmAycat+WcoKbklScLDgQ=
-github.com/sony/gobreaker v0.4.1/go.mod h1:ZKptC7FHNvhBz7dN2LGjPVBz2sZJmc0/PkyDJOjmxWY=
-github.com/sourcegraph/conc v0.3.0 h1:OQTbbt6P72L20UqAkXXuLOj79LfEanQ+YQFNpLA9ySo=
-github.com/sourcegraph/conc v0.3.0/go.mod h1:Sdozi7LEKbFPqYX2/J+iBAM6HpqSLTASQIKqDmF7Mt0=
-github.com/spaolacci/murmur3 v1.1.0 h1:7c1g84S4BPRrfL5Xrdp6fOJ206sU9y293DDHaoy0bLI=
-github.com/spaolacci/murmur3 v1.1.0/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA=
-github.com/spf13/afero v1.11.0 h1:WJQKhtpdm3v2IzqG8VMqrr6Rf3UYpEF239Jy9wNepM8=
-github.com/spf13/afero v1.11.0/go.mod h1:GH9Y3pIexgf1MTIWtNGyogA5MwRIDXGUr+hbWNoBjkY=
-github.com/spf13/cast v1.6.0 h1:GEiTHELF+vaR5dhz3VqZfFSzZjYbgeKDpBxQVS4GYJ0=
-github.com/spf13/cast v1.6.0/go.mod h1:ancEpBxwJDODSW/UG4rDrAqiKolqNNh2DX3mk86cAdo=
-github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA=
-github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
-github.com/spf13/viper v1.19.0 h1:RWq5SEjt8o25SROyN3z2OrDB9l7RPd3lwTWU8EcEdcI=
-github.com/spf13/viper v1.19.0/go.mod h1:GQUN9bilAbhU/jgc1bKs99f/suXKeUMct8Adx5+Ntkg=
-github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
-github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw=
-github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo=
-github.com/stretchr/objx v0.5.2 h1:xuMeJ0Sdp5ZMRXx/aWO6RZxdr3beISkG5/G/aIRr3pY=
-github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA=
-github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
-github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
-github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
-github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
-github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
-github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
-github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo=
-github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
-github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA=
-github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
-github.com/subosito/gotenv v1.6.0 h1:9NlTDc1FTs4qu0DDq7AEtTPNw6SVm7uBMsUCUjABIf8=
-github.com/subosito/gotenv v1.6.0/go.mod h1:Dk4QP5c2W3ibzajGcXpNraDfq2IrhjMIvMSWPKKo0FU=
-github.com/uber-go/atomic v1.4.0 h1:yOuPqEq4ovnhEjpHmfFwsqBXDYbQeT6Nb0bwD6XnD5o=
-github.com/uber-go/atomic v1.4.0/go.mod h1:/Ct5t2lcmbJ4OSe/waGBoaVvVqtO0bmtfVNex1PFV8g=
-github.com/uber/jaeger-client-go v2.30.0+incompatible h1:D6wyKGCecFaSRUpo8lCVbaOOb6ThwMmTEbhRwtKR97o=
-github.com/uber/jaeger-client-go v2.30.0+incompatible/go.mod h1:WVhlPFC8FDjOFMMWRy2pZqQJSXxYSwNYOkTr/Z6d3Kk=
-github.com/uber/jaeger-lib v2.4.1+incompatible h1:td4jdvLcExb4cBISKIpHuGoVXh+dVKhn2Um6rjCsSsg=
-github.com/uber/jaeger-lib v2.4.1+incompatible/go.mod h1:ComeNDZlWwrWnDv8aPp0Ba6+uUTzImX/AauajbLI56U=
-github.com/urfave/cli/v2 v2.2.0/go.mod h1:SE9GqnLQmjVa0iPEY0f1w3ygNIYcIJ0OKPMoW2caLfQ=
-github.com/willf/bitset v1.1.11 h1:N7Z7E9UvjW+sGsEl7k/SJrvY2reP1A07MrGuCjIOjRE=
-github.com/willf/bitset v1.1.11/go.mod h1:83CECat5yLh5zVOf4P1ErAgKA5UDvKtgyUABdr3+MjI=
-github.com/willf/bloom v2.0.3+incompatible h1:QDacWdqcAUI1MPOwIQZRy9kOR7yxfyEmxX8Wdm2/JPA=
-github.com/willf/bloom v2.0.3+incompatible/go.mod h1:MmAltL9pDMNTrvUkxdg0k0q5I0suxmuwp3KbyrZLOZ8=
-github.com/xyproto/randomstring v1.0.5 h1:YtlWPoRdgMu3NZtP45drfy1GKoojuR7hmRcnhZqKjWU=
-github.com/xyproto/randomstring v1.0.5/go.mod h1:rgmS5DeNXLivK7YprL0pY+lTuhNQW3iGxZ18UQApw/E=
-github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
-github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
-github.com/yuin/gopher-lua v0.0.0-20220504180219-658193537a64 h1:5mLPGnFdSsevFRFc9q3yYbBkB6tsm4aCwwQV/j1JQAQ=
-github.com/yuin/gopher-lua v0.0.0-20220504180219-658193537a64/go.mod h1:GBR0iDaNXjAgGg9zfCvksxSRnQx76gclCIb7kdAd1Pw=
-go.opencensus.io v0.24.0 h1:y73uSU6J157QMP2kn2r30vwW1A2W2WFwSCGnAVxeaD0=
-go.opencensus.io v0.24.0/go.mod h1:vNK8G9p7aAivkbmorf4v+7Hgx+Zs0yY+0fOtgBfjQKo=
-go.opentelemetry.io/auto/sdk v1.1.0 h1:cH53jehLUN6UFLY71z+NDOiNJqDdPRaXzTel0sJySYA=
-go.opentelemetry.io/auto/sdk v1.1.0/go.mod h1:3wSPjt5PWp2RhlCcmmOial7AvC4DQqZb7a7wCow3W8A=
-go.opentelemetry.io/collector v0.116.0 h1:Dscd6Nsnc7hjFQosO0SofcPQsXRfcj5N5PjQAslnmj4=
-go.opentelemetry.io/collector/pdata v1.22.0 h1:3yhjL46NLdTMoP8rkkcE9B0pzjf2973crn0KKhX5UrI=
-go.opentelemetry.io/collector/pdata v1.22.0/go.mod h1:nLLf6uDg8Kn5g3WNZwGyu8+kf77SwOqQvMTb5AXEbEY=
-go.opentelemetry.io/collector/pdata/pprofile v0.116.0 h1:iE6lqkO7Hi6lTIIml1RI7yQ55CKqW12R2qHinwF5Zuk=
-go.opentelemetry.io/collector/pdata/pprofile v0.116.0/go.mod h1:xQiPpjzIiXRFb+1fPxUy/3ygEZgo0Bu/xmLKOWu8vMQ=
-go.opentelemetry.io/collector/semconv v0.116.0 h1:63xCZomsKJAWmKGWD3lnORiE3WKW6AO4LjnzcHzGx3Y=
-go.opentelemetry.io/collector/semconv v0.116.0/go.mod h1:N6XE8Q0JKgBN2fAhkUQtqK9LT7rEGR6+Wu/Rtbal1iI=
-go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.58.0 h1:PS8wXpbyaDJQ2VDHHncMe9Vct0Zn1fEjpsjrLxGJoSc=
-go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.58.0/go.mod h1:HDBUsEjOuRC0EzKZ1bSaRGZWUBAzo+MhAcUUORSr4D0=
-go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.58.0 h1:yd02MEjBdJkG3uabWP9apV+OuWRIXGDuJEUJbOHmCFU=
-go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.58.0/go.mod h1:umTcuxiv1n/s/S6/c2AT/g2CQ7u5C59sHDNmfSwgz7Q=
-go.opentelemetry.io/otel v1.33.0 h1:/FerN9bax5LoK51X/sI0SVYrjSE0/yUL7DpxW4K3FWw=
-go.opentelemetry.io/otel v1.33.0/go.mod h1:SUUkR6csvUQl+yjReHu5uM3EtVV7MBm5FHKRlNx4I8I=
-go.opentelemetry.io/otel/metric v1.33.0 h1:r+JOocAyeRVXD8lZpjdQjzMadVZp2M4WmQ+5WtEnklQ=
-go.opentelemetry.io/otel/metric v1.33.0/go.mod h1:L9+Fyctbp6HFTddIxClbQkjtubW6O9QS3Ann/M82u6M=
-go.opentelemetry.io/otel/sdk v1.33.0 h1:iax7M131HuAm9QkZotNHEfstof92xM+N8sr3uHXc2IM=
-go.opentelemetry.io/otel/sdk v1.33.0/go.mod h1:A1Q5oi7/9XaMlIWzPSxLRWOI8nG3FnzHJNbiENQuihM=
-go.opentelemetry.io/otel/trace v1.33.0 h1:cCJuF7LRjUFso9LPnEAHJDB2pqzp+hbO8eu1qqW2d/s=
-go.opentelemetry.io/otel/trace v1.33.0/go.mod h1:uIcdVUZMpTAmz0tI1z04GoVSezK37CbGV4fr1f2nBck=
-go.opentelemetry.io/proto/otlp v1.4.0 h1:TA9WRvW6zMwP+Ssb6fLoUIuirti1gGbP28GcKG1jgeg=
-go.opentelemetry.io/proto/otlp v1.4.0/go.mod h1:PPBWZIP98o2ElSqI35IHfu7hIhSwvc5N38Jw8pXuGFY=
-go.uber.org/atomic v1.11.0 h1:ZvwS0R+56ePWxUNi+Atn9dWONBPp/AUETXlHW0DxSjE=
-go.uber.org/atomic v1.11.0/go.mod h1:LUxbIzbOniOlMKjJjyPfpl4v+PKK2cNJn91OQbhoJI0=
-go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto=
-go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE=
-go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0=
-go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y=
-go.uber.org/zap v1.27.0 h1:aJMhYGrd5QSmlpLMr2MftRKl7t8J8PTZPA732ud/XR8=
-go.uber.org/zap v1.27.0/go.mod h1:GB2qFLM7cTU87MWRP2mPIjqfIDnGu+VIO4V/SdhGo2E=
-golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
-golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
-golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
-golang.org/x/crypto v0.31.0 h1:ihbySMvVjLAeSH1IbfcRTkD/iNscyz8rGzjF/E5hV6U=
-golang.org/x/crypto v0.31.0/go.mod h1:kDsLvtWBEx7MV9tJOj9bnXsPbxwJQ6csT/x4KIN4Ssk=
-golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
-golang.org/x/exp v0.0.0-20241108190413-2d47ceb2692f h1:XdNn9LlyWAhLVp6P/i8QYBW+hlyhrhei9uErw2B5GJo=
-golang.org/x/exp v0.0.0-20241108190413-2d47ceb2692f/go.mod h1:D5SMRVC3C2/4+F/DB1wZsLRnSNimn2Sp/NPsCrsv8ak=
-golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
-golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU=
-golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
-golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
-golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
-golang.org/x/mod v0.22.0 h1:D4nJWe9zXqHOmWqj4VMOJhvzj7bEZg4wEYa759z1pH4=
-golang.org/x/mod v0.22.0/go.mod h1:6SkKJ3Xj0I0BrPOZoBy3bdMptDDU9oJrpohJ3eWZ1fY=
-golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
-golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
-golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
-golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
-golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
-golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
-golang.org/x/net v0.0.0-20190921015927-1a5e07d1ff72/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
-golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
-golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
-golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
-golang.org/x/net v0.33.0 h1:74SYHlV8BIgHIFC/LrYkOGIwL19eTYXQ5wc6TBuO36I=
-golang.org/x/net v0.33.0/go.mod h1:HXLR5J+9DxmrqMwG9qjGCxZ+zKXxBru04zlTvWlWuN4=
-golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
-golang.org/x/oauth2 v0.24.0 h1:KTBBxWqUa0ykRPLtV69rRto9TLXcqYkeswu48x/gvNE=
-golang.org/x/oauth2 v0.24.0/go.mod h1:XYTD2NtWslqkgxebSiOHnXEap4TF09sJSc7H1sXbhtI=
-golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
-golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
-golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
-golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
-golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
-golang.org/x/sync v0.10.0 h1:3NQrjDixjgGwUOCaF8w2+VYHv0Ve/vGYSbdkTa98gmQ=
-golang.org/x/sync v0.10.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
-golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
-golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
-golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.28.0 h1:Fksou7UEQUWlKvIdsqzJmUmCX3cZuD2+P3XyyzwMhlA=
-golang.org/x/sys v0.28.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
-golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
-golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
-golang.org/x/text v0.21.0 h1:zyQAAkrwaneQ066sspRyJaG9VNi/YJ1NfzcGB3hZ/qo=
-golang.org/x/text v0.21.0/go.mod h1:4IBbMaMmOPCJ8SecivzSH54+73PCFmPWxNTLm+vZkEQ=
-golang.org/x/time v0.8.0 h1:9i3RxcPv3PZnitoVGMPDKZSq1xW1gK1Xy3ArNOGZfEg=
-golang.org/x/time v0.8.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM=
-golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
-golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
-golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY=
-golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
-golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
-golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
-golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
-golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
-golang.org/x/tools v0.27.0 h1:qEKojBykQkQ4EynWy4S8Weg69NumxKdn40Fce3uc/8o=
-golang.org/x/tools v0.27.0/go.mod h1:sUi0ZgbwW9ZPAq26Ekut+weQPR5eIM6GQLQ1Yjm1H0Q=
-golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
-golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
-golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
-golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
-golang.org/x/xerrors v0.0.0-20231012003039-104605ab7028 h1:+cNy6SZtPcJQH3LJVLOSmiC7MMxXNOb3PU/VUEz+EhU=
-golang.org/x/xerrors v0.0.0-20231012003039-104605ab7028/go.mod h1:NDW/Ps6MPRej6fsCIbMTohpP40sJ/P/vI1MoTEGwX90=
-google.golang.org/api v0.211.0 h1:IUpLjq09jxBSV1lACO33CGY3jsRcbctfGzhj+ZSE/Bg=
-google.golang.org/api v0.211.0/go.mod h1:XOloB4MXFH4UTlQSGuNUxw0UT74qdENK8d6JNsXKLi0=
-google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
-google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
-google.golang.org/genproto v0.0.0-20180518175338-11a468237815/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=
-google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=
-google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc=
-google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo=
-google.golang.org/genproto v0.0.0-20240730163845-b1a4ccb954bf h1:OqdXDEakZCVtDiZTjcxfwbHPCT11ycCEsTKesBVKvyY=
-google.golang.org/genproto v0.0.0-20240730163845-b1a4ccb954bf/go.mod h1:mCr1K1c8kX+1iSBREvU3Juo11CB+QOEWxbRS01wWl5M=
-google.golang.org/genproto/googleapis/api v0.0.0-20241209162323-e6fa225c2576 h1:CkkIfIt50+lT6NHAVoRYEyAvQGFM7xEwXUUywFvEb3Q=
-google.golang.org/genproto/googleapis/api v0.0.0-20241209162323-e6fa225c2576/go.mod h1:1R3kvZ1dtP3+4p4d3G8uJ8rFk/fWlScl38vanWACI08=
-google.golang.org/genproto/googleapis/rpc v0.0.0-20241209162323-e6fa225c2576 h1:8ZmaLZE4XWrtU3MyClkYqqtl6Oegr3235h7jxsDyqCY=
-google.golang.org/genproto/googleapis/rpc v0.0.0-20241209162323-e6fa225c2576/go.mod h1:5uTbfoYQed2U9p3KIj2/Zzm02PYhndfdmML0qC3q3FU=
-google.golang.org/grpc v1.12.0/go.mod h1:yo6s7OP7yaDglbqo1J04qKzAhqBH6lvTonzMVmEdcZw=
-google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c=
-google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg=
-google.golang.org/grpc v1.23.1/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg=
-google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY=
-google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk=
-google.golang.org/grpc v1.33.2/go.mod h1:JMHMWHQWaTccqQQlmk3MJZS+GWXOdAesneDmEnv2fbc=
-google.golang.org/grpc v1.68.1 h1:oI5oTa11+ng8r8XMMN7jAOmWfPZWbYpCFaMUTACxkM0=
-google.golang.org/grpc v1.68.1/go.mod h1:+q1XYFJjShcqn0QZHvCyeR4CXPA+llXIeUIfIe00waw=
-google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8=
-google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0=
-google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM=
-google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE=
-google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo=
-google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
-google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
-google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
-google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c=
-google.golang.org/protobuf v1.35.2 h1:8Ar7bF+apOIoThw1EdZl0p1oWvMqTHmpA2fRTyZO8io=
-google.golang.org/protobuf v1.35.2/go.mod h1:9fA7Ob0pmnwhb644+1+CVWFRbNajQ6iRojtC/QF5bRE=
-gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
-gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk=
-gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q=
-gopkg.in/ini.v1 v1.67.0 h1:Dgnx+6+nfE+IfzjUEISNeydPJh9AXNNsWbGP9KzCsOA=
-gopkg.in/ini.v1 v1.67.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k=
-gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ=
-gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw=
-gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
-gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
-gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY=
-gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=
-gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
-gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
-gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
-gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
-honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
-honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=

cmd/tempo-serverless/lambda/main.go

@@ -1,116 +0,0 @@
-package main
-
-import (
-	"context"
-	"fmt"
-	"net/http"
-	"net/url"
-
-	"github.com/aws/aws-lambda-go/events"
-	"github.com/aws/aws-lambda-go/lambda"
-	"github.com/gogo/protobuf/jsonpb"
-
-	serverless "github.com/grafana/tempo/cmd/tempo-serverless"
-)
-
-func main() {
-	lambda.Start(HandleLambdaEvent)
-}
-
-func HandleLambdaEvent(ctx context.Context, event events.ALBTargetGroupRequest) (events.ALBTargetGroupResponse, error) {
-	req, err := httpRequest(event)
-	if err != nil {
-		return events.ALBTargetGroupResponse{
-			Body:       err.Error(),
-			StatusCode: http.StatusInternalServerError,
-			Headers:    map[string]string{}, // alb will 502 if we don't set this: https://github.com/awslabs/aws-lambda-go-api-proxy/issues/79
-		}, nil
-	}
-
-	resp, httpErr := serverless.Handler(req.WithContext(ctx))
-	if httpErr != nil {
-		return events.ALBTargetGroupResponse{
-			Body:       httpErr.Err.Error(),
-			StatusCode: httpErr.Status,
-			Headers:    map[string]string{},
-		}, nil
-	}
-
-	marshaller := &jsonpb.Marshaler{}
-	body, err := marshaller.MarshalToString(resp)
-	if err != nil {
-		return events.ALBTargetGroupResponse{
-			Body:       err.Error(),
-			StatusCode: http.StatusInternalServerError,
-			Headers:    map[string]string{},
-		}, nil
-	}
-
-	return events.ALBTargetGroupResponse{
-		Body:       body,
-		StatusCode: http.StatusOK,
-		Headers:    map[string]string{},
-	}, nil
-}
-
-// adapted with love from: https://github.com/akrylysov/algnhsa/blob/4c6f78589c506c0f060512adf96b97e8285ac80c/request.go#L65
-func httpRequest(event events.ALBTargetGroupRequest) (*http.Request, error) {
-	// params
-	params := url.Values{}
-	for k, v := range event.QueryStringParameters {
-		unescaped, err := url.QueryUnescape(v)
-		if err != nil {
-			return nil, fmt.Errorf("failed to unescape query string parameter %s: %s: %w", k, v, err)
-		}
-		params.Set(k, unescaped)
-	}
-	for k, vals := range event.MultiValueQueryStringParameters {
-		for i, v := range vals {
-			var err error
-			vals[i], err = url.QueryUnescape(v)
-			if err != nil {
-				return nil, fmt.Errorf("failed to unescape multi val query string parameter %s: %s: %w", k, v, err)
-			}
-		}
-		params[k] = vals
-	}
-
-	// headers
-	headers := make(http.Header)
-	for k, v := range event.Headers {
-		headers.Set(k, v)
-	}
-	for k, vals := range event.MultiValueHeaders {
-		headers[http.CanonicalHeaderKey(k)] = vals
-	}
-
-	// url
-	u := url.URL{
-		Host:     headers.Get("host"),
-		RawPath:  event.Path,
-		RawQuery: params.Encode(),
-	}
-
-	// Unescape request path
-	p, err := url.PathUnescape(u.RawPath)
-	if err != nil {
-		return nil, err
-	}
-	u.Path = p
-
-	if u.Path == u.RawPath {
-		u.RawPath = ""
-	}
-
-	// we don't use the body. ignore
-
-	req, err := http.NewRequest(event.HTTPMethod, u.String(), nil)
-	if err != nil {
-		return nil, err
-	}
-
-	req.RequestURI = u.RequestURI()
-	req.Header = headers
-
-	return req, nil
-}

cmd/tempo-serverless/lambda/main_test.go

@@ -1,111 +0,0 @@
-package main
-
-import (
-	"net/http"
-	"net/url"
-	"testing"
-
-	"github.com/aws/aws-lambda-go/events"
-	"github.com/stretchr/testify/require"
-)
-
-// Test that httpRequest() works as expected
-func Test_httpRequest(t *testing.T) {
-	tests := []struct {
-		event   events.ALBTargetGroupRequest
-		want    *http.Request
-		wantErr string
-	}{
-		// empty
-		{
-			event: events.ALBTargetGroupRequest{},
-			want: &http.Request{
-				Method:     "GET",
-				RequestURI: "/",
-				URL:        &url.URL{},
-				Header:     map[string][]string{},
-			},
-		},
-		// rando
-		{
-			event: events.ALBTargetGroupRequest{
-				HTTPMethod: "GET",
-				Path:       "/",
-				Headers: map[string]string{
-					"Content-Type": "application/json",
-				},
-				QueryStringParameters: map[string]string{
-					"query": "test",
-				},
-				Body: "test",
-			},
-			want: &http.Request{
-				Method:     "GET",
-				RequestURI: "/?query=test",
-				URL: &url.URL{
-					Path:     "/",
-					RawQuery: "query=test",
-				},
-				Header: map[string][]string{
-					"Content-Type": {"application/json"},
-				},
-			},
-		},
-		// unescape params
-		{
-			event: events.ALBTargetGroupRequest{
-				QueryStringParameters: map[string]string{
-					"q": "%7B%20.foo%20%3D%20bar%20%7D",
-				},
-			},
-			want: &http.Request{
-				Method:     "GET",
-				RequestURI: "/?q=%7B+.foo+%3D+bar+%7D",
-				URL: &url.URL{
-					RawQuery: "q=%7B+.foo+%3D+bar+%7D",
-				},
-				Header: map[string][]string{},
-			},
-		},
-		// unescape multivalue params
-		{
-			event: events.ALBTargetGroupRequest{
-				MultiValueQueryStringParameters: map[string][]string{
-					"q2": {"%7B%20.foo%20%3D%20bar%20%7D", "%7B%20.foo%20%3D%20bar%20%7D"},
-				},
-			},
-			want: &http.Request{
-				Method:     "GET",
-				RequestURI: "/?q2=%7B+.foo+%3D+bar+%7D&q2=%7B+.foo+%3D+bar+%7D",
-				URL: &url.URL{
-					RawQuery: "q2=%7B+.foo+%3D+bar+%7D&q2=%7B+.foo+%3D+bar+%7D",
-				},
-				Header: map[string][]string{},
-			},
-		},
-		// unescape error
-		{
-			event: events.ALBTargetGroupRequest{
-				QueryStringParameters: map[string]string{
-					"q": "%7B%20.foo%20%3D%20bar%20%7D%ZZ",
-				},
-			},
-			wantErr: "failed to unescape query string parameter q: %7B%20.foo%20%3D%20bar%20%7D%ZZ: invalid URL escape \"%ZZ\"",
-		},
-	}
-
-	for _, tt := range tests {
-		got, err := httpRequest(tt.event)
-		if len(tt.wantErr) > 0 {
-			require.EqualError(t, err, tt.wantErr)
-			continue
-		} else {
-			require.NoError(t, err)
-		}
-
-		require.Equal(t, tt.want.Method, got.Method)
-		require.Equal(t, tt.want.RequestURI, got.RequestURI)
-		require.Equal(t, tt.want.URL, got.URL)
-		require.Equal(t, tt.want.Header, got.Header)
-	}
-}

cmd/tempo-serverless/readme.md

@@ -1,45 +0,0 @@
-# tempo-serverless
-
-## NOTE: The Tempo serverless is now deprecated and will be removed in an upcoming release.
-
-This folder is intended to wrap a simple handler for searching backend storage for traces. Subfolders 
-`./cloud-run` and `./lambda` contain specific code necessary to run this handler in the respective
-environments.
-
-## Configuration
-The serverless handler is setup to be configured via environment variables starting with the `TEMPO_` prefix.
-```
-TEMPO_GCS_BUCKET_NAME=some-random-gcs-bucket
-TEMPO_GCS_CHUNK_BUFFER_SIZE=10485760
-TEMPO_GCS_HEDGE_REQUESTS_AT=400ms
-TEMPO_BACKEND=gcs
-```
-All fields in tempodb.Config are accessible using their all caps yaml names. Also config objects can be descended
-using the `_` character. Note that in the above example `TEMPO_BCS_BUCKET_NAME` refers to tempodb.Config.GCS.BucketName.
-
-## Make
-
-### make build-docker
-
-Builds two docker images named `tempo-serverlesss` and `tempo-serverless-lambda` that are used for e2e testing.
-
-```
-docker run --rm -p 8080:8080 tempo-serverless
-curl http://localhost:8080
-```
-
-### make build-docker-gcr-binary
-### make build-docker-gcr
-
-Building the GCR docker image is split into two steps to make it easier to build as part of CI. (See /.drone/drone.jsonnet).
-Together these two steps build a binary out of container and then package that up using a Dockerfile. The docker image
-created in `build-docker-gcr` is the code artifact that is shipped to Google Cloud Run.
-
-### make build-docker-lambda-test
-
-Builds the docker image for lambda e2e testing. This image is not shipped anywhere.
-
-### make build-lambda-zip
-
-Builds a lambda zip file. This builds the code artifact that is shipped to AWS Lambda.
-

cmd/tempo-vulture/Dockerfile

@@ -1,10 +1,16 @@
-FROM alpine:3.21 as certs
-RUN apk --update add ca-certificates
+FROM alpine:latest AS ca-certificates
+RUN apk add --update --no-cache ca-certificates
+
+FROM gcr.io/distroless/static-debian12:debug
+
+SHELL ["/busybox/sh", "-c"]
+
+RUN ["/busybox/addgroup", "-g", "10001", "-S", "tempo"]
+RUN ["/busybox/adduser", "-u", "10001", "-S", "tempo", "-G", "tempo"]
+
 ARG TARGETARCH
 COPY bin/linux/tempo-vulture-${TARGETARCH} /tempo-vulture
-
-RUN addgroup -g 10001 -S tempo && \
-    adduser -u 10001 -S tempo -G tempo
+COPY --from=ca-certificates /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/
 
 USER 10001:10001
 

cmd/tempo-vulture/main.go

@@ -7,6 +7,7 @@ import (
 	"flag"
 	"fmt"
 	"log"
+	"math"
 	"math/rand"
 	"net/http"
 	"net/url"
@@ -15,6 +16,7 @@ import (
 	"time"
 
 	"github.com/go-test/deep"
+	"github.com/grafana/tempo/pkg/api"
 	jaeger_grpc "github.com/jaegertracing/jaeger/cmd/agent/app/reporter/grpc"
 	zaplogfmt "github.com/jsternberg/zap-logfmt"
 	"github.com/prometheus/client_golang/prometheus/promhttp"
@@ -34,25 +36,32 @@ var (
 	prometheusListenAddress string
 	prometheusPath          string
 
-	tempoQueryURL                 string
-	tempoPushURL                  string
-	tempoOrgID                    string
-	tempoWriteBackoffDuration     time.Duration
-	tempoLongWriteBackoffDuration time.Duration
-	tempoReadBackoffDuration      time.Duration
-	tempoSearchBackoffDuration    time.Duration
-	tempoRetentionDuration        time.Duration
-	tempoPushTLS                  bool
+	tempoQueryURL                   string
+	tempoPushURL                    string
+	tempoOrgID                      string
+	tempoWriteBackoffDuration       time.Duration
+	tempoLongWriteBackoffDuration   time.Duration
+	tempoReadBackoffDuration        time.Duration
+	tempoSearchBackoffDuration      time.Duration
+	tempoMetricsBackoffDuration     time.Duration
+	tempoRetentionDuration          time.Duration
+	tempoRecentTracesCutoffDuration time.Duration
+	tempoPushTLS                    bool
+
+	rf1After time.Time
 
 	logger *zap.Logger
 )
 
 type traceMetrics struct {
 	incorrectResult         int
+	incorrectMetricsResult  int
 	missingSpans            int
 	notFoundByID            int
 	notFoundSearch          int
 	notFoundTraceQL         int
+	notFoundByMetrics       int
+	inaccurateMetrics       int
 	requested               int
 	requestFailed           int
 	notFoundSearchAttribute int
@@ -63,15 +72,43 @@ const (
 )
 
 type vultureConfiguration struct {
-	tempoQueryURL                 string
-	tempoPushURL                  string
-	tempoOrgID                    string
-	tempoWriteBackoffDuration     time.Duration
-	tempoLongWriteBackoffDuration time.Duration
-	tempoReadBackoffDuration      time.Duration
-	tempoSearchBackoffDuration    time.Duration
-	tempoRetentionDuration        time.Duration
-	tempoPushTLS                  bool
+	tempoQueryURL                   string
+	tempoPushURL                    string
+	tempoOrgID                      string
+	tempoWriteBackoffDuration       time.Duration
+	tempoLongWriteBackoffDuration   time.Duration
+	tempoReadBackoffDuration        time.Duration
+	tempoSearchBackoffDuration      time.Duration
+	tempoMetricsBackoffDuration     time.Duration
+	tempoRetentionDuration          time.Duration
+	tempoRecentTracesCutoffDuration time.Duration
+	tempoPushTLS                    bool
+}
+
+var _ flag.Value = (*timeVar)(nil)
+
+type timeVar struct {
+	t *time.Time
+}
+
+func newTimeVar(t *time.Time) *timeVar { return &timeVar{t: t} }
+
+func (v timeVar) String() string {
+	return (*v.t).Format(time.RFC3339)
+}
+
+func (v timeVar) Set(s string) error {
+	if s == "" {
+		return nil
+	}
+
+	t, err := time.Parse(time.RFC3339, s)
+	if err != nil {
+		return err
+	}
+	*v.t = t
+
+	return nil
 }
 
 func init() {
@@ -86,7 +123,11 @@ func init() {
 	flag.DurationVar(&tempoLongWriteBackoffDuration, "tempo-long-write-backoff-duration", 1*time.Minute, "The amount of time to pause between long write Tempo calls")
 	flag.DurationVar(&tempoReadBackoffDuration, "tempo-read-backoff-duration", 30*time.Second, "The amount of time to pause between read Tempo calls")
 	flag.DurationVar(&tempoSearchBackoffDuration, "tempo-search-backoff-duration", 60*time.Second, "The amount of time to pause between search Tempo calls.  Set to 0s to disable search.")
+	flag.DurationVar(&tempoMetricsBackoffDuration, "tempo-metrics-backoff-duration", 0, "The amount of time to pause between TraceQL Metrics Tempo calls.  Set to 0s to disable.")
 	flag.DurationVar(&tempoRetentionDuration, "tempo-retention-duration", 336*time.Hour, "The block retention that Tempo is using")
+	flag.DurationVar(&tempoRecentTracesCutoffDuration, "tempo-recent-traces-backoff-duration", 14*time.Minute, "Cutoff between recent and old traces query checks")
+
+	flag.Var(newTimeVar(&rf1After), "rhythm-rf1-after", "Timestamp (RFC3339) after which only blocks with RF==1 are included in search and ID lookups")
 }
 
 func main() {
@@ -102,15 +143,17 @@ func main() {
 	logger.Info("Tempo Vulture starting")
 
 	vultureConfig := vultureConfiguration{
-		tempoQueryURL:                 tempoQueryURL,
-		tempoPushURL:                  tempoPushURL,
-		tempoOrgID:                    tempoOrgID,
-		tempoWriteBackoffDuration:     tempoWriteBackoffDuration,
-		tempoLongWriteBackoffDuration: tempoLongWriteBackoffDuration,
-		tempoReadBackoffDuration:      tempoReadBackoffDuration,
-		tempoSearchBackoffDuration:    tempoSearchBackoffDuration,
-		tempoRetentionDuration:        tempoRetentionDuration,
-		tempoPushTLS:                  tempoPushTLS,
+		tempoQueryURL:                   tempoQueryURL,
+		tempoPushURL:                    tempoPushURL,
+		tempoOrgID:                      tempoOrgID,
+		tempoWriteBackoffDuration:       tempoWriteBackoffDuration,
+		tempoLongWriteBackoffDuration:   tempoLongWriteBackoffDuration,
+		tempoReadBackoffDuration:        tempoReadBackoffDuration,
+		tempoSearchBackoffDuration:      tempoSearchBackoffDuration,
+		tempoMetricsBackoffDuration:     tempoMetricsBackoffDuration,
+		tempoRetentionDuration:          tempoRetentionDuration,
+		tempoRecentTracesCutoffDuration: tempoRecentTracesCutoffDuration,
+		tempoPushTLS:                    tempoPushTLS,
 	}
 
 	jaegerClient, err := newJaegerGRPCClient(vultureConfig, logger)
@@ -119,7 +162,16 @@ func main() {
 	}
 	httpClient := httpclient.New(vultureConfig.tempoQueryURL, vultureConfig.tempoOrgID)
 
-	tickerWrite, tickerRead, tickerSearch, err := initTickers(vultureConfig.tempoWriteBackoffDuration, vultureConfig.tempoReadBackoffDuration, vultureConfig.tempoSearchBackoffDuration)
+	if !rf1After.IsZero() {
+		httpClient.SetQueryParam(api.URLParamRF1After, rf1After.Format(time.RFC3339))
+	}
+
+	tickerWrite, tickerRead, tickerSearch, tickerMetrics, err := initTickers(
+		vultureConfig.tempoWriteBackoffDuration,
+		vultureConfig.tempoReadBackoffDuration,
+		vultureConfig.tempoSearchBackoffDuration,
+		vultureConfig.tempoMetricsBackoffDuration,
+	)
 	if err != nil {
 		panic(err)
 	}
@@ -127,9 +179,48 @@ func main() {
 	r := rand.New(rand.NewSource(startTime.Unix()))
 	interval := vultureConfig.tempoWriteBackoffDuration
 
+	selectRecentTimestamp := func(now time.Time) (newStart, ts time.Time, skip bool) {
+		oldest := now.Add(-vultureConfig.tempoRecentTracesCutoffDuration)
+		if oldest.Before(startTime) { // if vulture's just started
+			oldest = startTime
+		}
+		newStart, ts = selectPastTimestamp(oldest, now, interval, vultureConfig.tempoRetentionDuration, r)
+		return
+	}
+
+	selectOldTimestamp := func(now time.Time) (newStart, ts time.Time, skip bool) {
+		newest := now.Add(-vultureConfig.tempoRecentTracesCutoffDuration)
+		if newest.Before(startTime) { // if vulture's just started and no traces to query
+			skip = true
+			return
+		}
+		newStart, ts = selectPastTimestamp(startTime, newest, interval, vultureConfig.tempoRetentionDuration, r)
+		return
+	}
+
 	doWrite(jaegerClient, tickerWrite, interval, vultureConfig, logger)
-	doRead(httpClient, tickerRead, startTime, interval, r, vultureConfig, logger)
-	doSearch(httpClient, tickerSearch, startTime, interval, r, vultureConfig, logger)
+
+	// Recent traces
+	runChecker(tickerRead, vultureConfig, selectRecentTimestamp, func(info *util.TraceInfo, l *zap.Logger) {
+		doRead(httpClient, vultureConfig, info, l)
+	}, logger)
+	runChecker(tickerSearch, vultureConfig, selectRecentTimestamp, func(info *util.TraceInfo, l *zap.Logger) {
+		doSearch(httpClient, vultureConfig, info, l)
+	}, logger)
+	runChecker(tickerMetrics, vultureConfig, selectRecentTimestamp, func(info *util.TraceInfo, l *zap.Logger) {
+		doMetrics(httpClient, vultureConfig, info, l)
+	}, logger)
+
+	// Old traces
+	runChecker(tickerRead, vultureConfig, selectOldTimestamp, func(info *util.TraceInfo, l *zap.Logger) {
+		doRead(httpClient, vultureConfig, info, l)
+	}, logger)
+	runChecker(tickerSearch, vultureConfig, selectOldTimestamp, func(info *util.TraceInfo, l *zap.Logger) {
+		doSearch(httpClient, vultureConfig, info, l)
+	}, logger)
+	runChecker(tickerMetrics, vultureConfig, selectOldTimestamp, func(info *util.TraceInfo, l *zap.Logger) {
+		doMetrics(httpClient, vultureConfig, info, l)
+	}, logger)
 
 	http.Handle(prometheusPath, promhttp.Handler())
 	log.Fatal(http.ListenAndServe(prometheusListenAddress, nil))
@@ -148,9 +239,20 @@ func getGRPCEndpoint(endpoint string) (string, error) {
 	return dialAddress, nil
 }
 
-func initTickers(tempoWriteBackoffDuration time.Duration, tempoReadBackoffDuration time.Duration, tempoSearchBackoffDuration time.Duration) (tickerWrite *time.Ticker, tickerRead *time.Ticker, tickerSearch *time.Ticker, err error) {
+func initTickers(
+	tempoWriteBackoffDuration time.Duration,
+	tempoReadBackoffDuration time.Duration,
+	tempoSearchBackoffDuration time.Duration,
+	tempoMetricsBackoffDuration time.Duration,
+) (
+	tickerWrite *time.Ticker,
+	tickerRead *time.Ticker,
+	tickerSearch *time.Ticker,
+	tickerMetrics *time.Ticker,
+	err error,
+) {
 	if tempoWriteBackoffDuration <= 0 {
-		return nil, nil, nil, errors.New("tempo-write-backoff-duration must be greater than 0")
+		return nil, nil, nil, nil, errors.New("tempo-write-backoff-duration must be greater than 0")
 	}
 	tickerWrite = time.NewTicker(tempoWriteBackoffDuration)
 	if tempoReadBackoffDuration > 0 {
@@ -159,10 +261,13 @@ func initTickers(tempoWriteBackoffDuration time.Duration, tempoReadBackoffDurati
 	if tempoSearchBackoffDuration > 0 {
 		tickerSearch = time.NewTicker(tempoSearchBackoffDuration)
 	}
-	if tickerRead == nil && tickerSearch == nil {
-		return nil, nil, nil, errors.New("at least one of tempo-search-backoff-duration or tempo-read-backoff-duration must be set")
+	if tempoMetricsBackoffDuration > 0 {
+		tickerMetrics = time.NewTicker(tempoMetricsBackoffDuration)
 	}
-	return tickerWrite, tickerRead, tickerSearch, nil
+	if tickerRead == nil && tickerSearch == nil && tickerMetrics == nil {
+		return nil, nil, nil, nil, errors.New("at least one of tempo-search-backoff-duration, tempo-read-backoff-duration or tempo-metrics-backoff-duration must be set")
+	}
+	return tickerWrite, tickerRead, tickerSearch, tickerMetrics, nil
 }
 
 // Don't attempt to read on the first iteration if we can't reasonably
@@ -235,88 +340,93 @@ func queueFutureBatches(client util.JaegerClient, info *util.TraceInfo, config v
 	}()
 }
 
-func doRead(httpClient httpclient.TempoHTTPClient, tickerRead *time.Ticker, startTime time.Time, interval time.Duration, r *rand.Rand, config vultureConfiguration, l *zap.Logger) {
-	if tickerRead != nil {
-		go func() {
-			for now := range tickerRead.C {
-				var seed time.Time
-				startTime, seed = selectPastTimestamp(startTime, now, interval, tempoRetentionDuration, r)
-
-				logger := l.With(
-					zap.String("org_id", config.tempoOrgID),
-					zap.Int64("seed", seed.Unix()),
-				)
-
-				info := util.NewTraceInfo(seed, config.tempoOrgID)
-
-				// Don't query for a trace we don't expect to be complete
-				if !traceIsReady(info, now, startTime,
-					config.tempoWriteBackoffDuration, config.tempoLongWriteBackoffDuration) {
-					continue
-				}
-
-				// query the trace
-				queryMetrics, err := queryTrace(httpClient, info, l)
-				if err != nil {
-					metricErrorTotal.Inc()
-					logger.Error("query for metrics failed",
-						zap.Error(err),
-					)
-				}
-				pushMetrics(queryMetrics)
-			}
-		}()
+func runChecker(
+	ticker *time.Ticker,
+	config vultureConfiguration,
+	selectPastTimestamp func(now time.Time) (newStart, ts time.Time, skip bool),
+	checker func(*util.TraceInfo, *zap.Logger),
+	l *zap.Logger,
+) {
+	if ticker == nil {
+		return
 	}
+	go func() {
+		for now := range ticker.C {
+			startTime, seed, skip := selectPastTimestamp(now)
+			if skip {
+				continue
+			}
+
+			logger := l.With(
+				zap.String("org_id", config.tempoOrgID),
+				zap.Int64("seed", seed.Unix()),
+			)
+
+			info := util.NewTraceInfo(seed, config.tempoOrgID)
+
+			// Don't query for a trace we don't expect to be complete
+			if !traceIsReady(info, now, startTime,
+				config.tempoWriteBackoffDuration, config.tempoLongWriteBackoffDuration) {
+				continue
+			}
+
+			checker(info, logger)
+		}
+	}()
 }
 
-func doSearch(httpClient httpclient.TempoHTTPClient, tickerSearch *time.Ticker, startTime time.Time, interval time.Duration, r *rand.Rand, config vultureConfiguration, l *zap.Logger) {
-	if tickerSearch != nil {
-		go func() {
-			for now := range tickerSearch.C {
-				_, seed := selectPastTimestamp(startTime, now, interval, config.tempoRetentionDuration, r)
-				logger := l.With(
-					zap.String("org_id", config.tempoOrgID),
-					zap.Int64("seed", seed.Unix()),
-				)
-
-				info := util.NewTraceInfo(seed, config.tempoOrgID)
-
-				if !traceIsReady(info, now, startTime,
-					config.tempoWriteBackoffDuration, config.tempoLongWriteBackoffDuration) {
-					continue
-				}
-
-				// query a tag we expect the trace to be found within
-				searchMetrics, err := searchTag(httpClient, seed, config, l)
-				if err != nil {
-					metricErrorTotal.Inc()
-					logger.Error("search tag for metrics failed",
-						zap.Error(err),
-					)
-				}
-				pushMetrics(searchMetrics)
-
-				// traceql query
-				traceqlSearchMetrics, err := searchTraceql(httpClient, seed, config, l)
-				if err != nil {
-					metricErrorTotal.Inc()
-					logger.Error("traceql query for metrics failed",
-						zap.Error(err),
-					)
-				}
-				pushMetrics(traceqlSearchMetrics)
-			}
-		}()
+func doRead(httpClient httpclient.TempoHTTPClient, _ vultureConfiguration, info *util.TraceInfo, l *zap.Logger) {
+	// query the trace
+	queryMetrics, err := queryTrace(httpClient, info, l)
+	if err != nil {
+		metricErrorTotal.Inc()
+		logger.Error("query for metrics failed",
+			zap.Error(err),
+		)
 	}
+	pushVultureMetrics(queryMetrics)
 }
 
-func pushMetrics(metrics traceMetrics) {
+func doSearch(httpClient httpclient.TempoHTTPClient, config vultureConfiguration, info *util.TraceInfo, l *zap.Logger) {
+	// query a tag we expect the trace to be found within
+	searchMetrics, err := searchTag(httpClient, info.Timestamp(), config, l)
+	if err != nil {
+		metricErrorTotal.Inc()
+		logger.Error("search tag for metrics failed",
+			zap.Error(err),
+		)
+	}
+	pushVultureMetrics(searchMetrics)
+
+	// traceql query
+	traceqlSearchMetrics, err := searchTraceql(httpClient, info.Timestamp(), config, l)
+	if err != nil {
+		metricErrorTotal.Inc()
+		logger.Error("traceql query for metrics failed",
+			zap.Error(err),
+		)
+	}
+	pushVultureMetrics(traceqlSearchMetrics)
+}
+
+func doMetrics(httpClient httpclient.TempoHTTPClient, config vultureConfiguration, info *util.TraceInfo, l *zap.Logger) {
+	m, err := queryMetrics(httpClient, info.Timestamp(), config, l)
+	if err != nil {
+		logger.Error("query metrics failed", zap.Error(err))
+	}
+	pushVultureMetrics(m)
+}
+
+func pushVultureMetrics(metrics traceMetrics) {
 	metricTracesInspected.Add(float64(metrics.requested))
 	metricTracesErrors.WithLabelValues("incorrectresult").Add(float64(metrics.incorrectResult))
+	metricTracesErrors.WithLabelValues("incorrect_metrics_result").Add(float64(metrics.incorrectMetricsResult))
 	metricTracesErrors.WithLabelValues("missingspans").Add(float64(metrics.missingSpans))
 	metricTracesErrors.WithLabelValues("notfound_search").Add(float64(metrics.notFoundSearch))
 	metricTracesErrors.WithLabelValues("notfound_traceql").Add(float64(metrics.notFoundTraceQL))
 	metricTracesErrors.WithLabelValues("notfound_byid").Add(float64(metrics.notFoundByID))
+	metricTracesErrors.WithLabelValues("notfound_metrics").Add(float64(metrics.notFoundByMetrics))
+	metricTracesErrors.WithLabelValues("inaccurate_metrics").Add(float64(metrics.inaccurateMetrics))
 	metricTracesErrors.WithLabelValues("requestfailed").Add(float64(metrics.requestFailed))
 	metricTracesErrors.WithLabelValues("notfound_search_attribute").Add(float64(metrics.notFoundSearchAttribute))
 }
@@ -405,7 +515,7 @@ func searchTag(client httpclient.TempoHTTPClient, seed time.Time, config vulture
 	// Get the expected
 	expected, err := info.ConstructTraceFromEpoch()
 	if err != nil {
-		logger.Error("unable to construct trace from epoch", zap.Error(err))
+		l.Error("unable to construct trace from epoch", zap.Error(err))
 		return traceMetrics{}, err
 	}
 
@@ -524,7 +634,6 @@ func queryTrace(client httpclient.TempoHTTPClient, info *util.TraceInfo, l *zap.
 		return tm, nil
 	}
 
-	// iterate through
 	if hasMissingSpans(trace) {
 		logger.Error("trace has missing spans")
 		tm.missingSpans++
@@ -554,6 +663,124 @@ func queryTrace(client httpclient.TempoHTTPClient, info *util.TraceInfo, l *zap.
 	return tm, nil
 }
 
+// queryMetrics performs a TraceQL metrics query and verifies the results.
+// It is a basic smoke test to ensure that the traceql query is working.
+// It randomly selects an attribute from an expected trace and queries for its presence
+// in the metrics API within a time window around the seed time.
+func queryMetrics(client httpclient.TempoHTTPClient, seed time.Time, config vultureConfiguration, l *zap.Logger) (traceMetrics, error) {
+	tm := traceMetrics{
+		requested: 1,
+	}
+
+	info := util.NewTraceInfo(seed, config.tempoOrgID)
+	hexID := info.HexID()
+
+	expected, err := info.ConstructTraceFromEpoch()
+	if err != nil {
+		err = fmt.Errorf("unable to construct trace from epoch: %w", err)
+		return traceMetrics{}, err
+	}
+
+	attr := util.RandomAttrFromTrace(expected)
+	if attr == nil {
+		tm.notFoundSearchAttribute++
+		return tm, fmt.Errorf("no search attr selected from trace")
+	}
+
+	logger := l.With(
+		zap.Int64("seed", seed.Unix()),
+		zap.String("hexID", hexID),
+		zap.Duration("ago", time.Since(seed)),
+		zap.String("key", attr.Key),
+		zap.String("value", util.StringifyAnyValue(attr.Value)),
+	)
+	logger.Info("searching Tempo via metrics")
+
+	// Use the API to find details about the expected trace. give an hour range around the seed.
+	start := seed.Add(-30 * time.Minute).Unix()
+	end := seed.Add(30 * time.Minute).Unix()
+
+	resp, err := client.MetricsQueryRange(
+		fmt.Sprintf(`{.%s = "%s"} | count_over_time()`, attr.Key, util.StringifyAnyValue(attr.Value)),
+		int(start), int(end), "1m", 0,
+	)
+	if err != nil {
+		logger.Error("failed to query metrics", zap.Error(err))
+		tm.requestFailed++
+		return tm, err
+	}
+
+	if len(resp.Series) == 0 {
+		tm.notFoundByMetrics++
+		logger.Error("failed to find trace by metrics", zap.Error(err))
+		return tm, fmt.Errorf("expected trace %s not found in metrics", hexID)
+	}
+
+	if len(resp.Series) > 1 {
+		tm.incorrectMetricsResult++
+		return tm, fmt.Errorf("expected exactly 1 series, got %d", len(resp.Series))
+	}
+	timeSeries := resp.Series[0]
+	if timeSeries == nil {
+		tm.incorrectMetricsResult++
+		return tm, errors.New("expected time series, got nil")
+	}
+
+	var sum float64
+	for _, sample := range timeSeries.Samples {
+		sum += sample.Value
+	}
+
+	if sum < 1 {
+		tm.notFoundByMetrics++
+		logger.Error("failed to find trace by metrics", zap.Error(err))
+		return tm, fmt.Errorf("expected trace %s not found in metrics", hexID)
+	}
+
+	// Advanced check: ensure metric results are accurate
+	// by checking actual number of spans
+	// skip if search check is disabled
+	if config.tempoSearchBackoffDuration == 0 {
+		return tm, nil
+	}
+	searchResp, err := client.SearchTraceQLWithRange(fmt.Sprintf(`{.%s = "%s"}`, attr.Key, util.StringifyAnyValue(attr.Value)), start, end)
+	if err != nil {
+		logger.Error(fmt.Sprintf("failed to search traces with traceql %s: %s", attr.Key, err.Error()))
+		tm.requestFailed++
+		return tm, err
+	}
+
+	var spansCount int
+	var traces []*tempopb.TraceSearchMetadata
+	if searchResp != nil {
+		traces = searchResp.Traces
+	}
+	for _, trace := range traces {
+		if trace == nil {
+			continue
+		}
+		for _, spanSet := range trace.SpanSets {
+			if spanSet == nil {
+				continue
+			}
+			spansCount += int(spanSet.Matched)
+		}
+	}
+
+	const delta = 1e-6
+	// if number of traces is not equal to the sum of metric values
+	if math.Abs(float64(spansCount)-sum) > delta {
+		tm.inaccurateMetrics++
+		err = fmt.Errorf(
+			"TraceQL Metrics results are inaccurate: metric count sum=%f, actual span count=%d",
+			sum, spansCount,
+		)
+		return tm, err
+	}
+
+	return tm, nil
+}
+
 func equalTraces(a, b *tempopb.Trace) bool {
 	trace.SortTraceAndAttributes(a)
 	trace.SortTraceAndAttributes(b)
@@ -562,20 +789,20 @@ func equalTraces(a, b *tempopb.Trace) bool {
 }
 
 func hasMissingSpans(t *tempopb.Trace) bool {
-	// collect all parent span IDs
-	linkedSpanIDs := make([][]byte, 0)
+	// check that all parent spans exist in the trace
+	parentSpanIDs := make([][]byte, 0)
 
 	for _, b := range t.ResourceSpans {
 		for _, ss := range b.ScopeSpans {
 			for _, s := range ss.Spans {
 				if len(s.ParentSpanId) > 0 {
-					linkedSpanIDs = append(linkedSpanIDs, s.ParentSpanId)
+					parentSpanIDs = append(parentSpanIDs, s.ParentSpanId)
 				}
 			}
 		}
 	}
 
-	for _, id := range linkedSpanIDs {
+	for _, id := range parentSpanIDs {
 		found := false
 
 	B:

cmd/tempo-vulture/main_test.go

@@ -3,8 +3,9 @@ package main
 import (
 	"bytes"
 	"errors"
-	"math/rand"
+	"fmt"
 	"os"
+	"sync"
 	"testing"
 	"time"
 
@@ -112,66 +113,90 @@ func TestEqualTraces(t *testing.T) {
 	require.NoError(t, err)
 
 	require.True(t, equalTraces(a, b))
+
+	// Subsequent calls also reconstruct identical traces
+	c, err := info1.ConstructTraceFromEpoch()
+	require.NoError(t, err)
+	require.True(t, equalTraces(b, c))
 }
 
 func TestInitTickers(t *testing.T) {
 	tests := []struct {
-		name                                        string
-		writeDuration, readDuration, searchDuration time.Duration
-		expectedWriteTicker                         bool
-		expectedReadTicker                          bool
-		expectedSearchTicker                        bool
-		expectedError                               string
+		name                            string
+		writeDuration, readDuration     time.Duration
+		searchDuration, metricsDuration time.Duration
+		expectedWriteTicker             bool
+		expectedReadTicker              bool
+		expectedSearchTicker            bool
+		expectedMetricsTicker           bool
+		expectedError                   string
 	}{
 		{
-			name:                 "Valid write and read durations",
-			writeDuration:        1 * time.Second,
-			readDuration:         2 * time.Second,
-			searchDuration:       0,
-			expectedWriteTicker:  true,
-			expectedReadTicker:   true,
-			expectedSearchTicker: false,
-			expectedError:        "",
+			name:                  "Valid write and read durations",
+			writeDuration:         1 * time.Second,
+			readDuration:          2 * time.Second,
+			searchDuration:        0,
+			expectedWriteTicker:   true,
+			expectedReadTicker:    true,
+			expectedSearchTicker:  false,
+			expectedMetricsTicker: false,
+			expectedError:         "",
 		},
 		{
-			name:                 "Invalid write duration (zero)",
-			writeDuration:        0,
-			readDuration:         0,
-			searchDuration:       0,
-			expectedWriteTicker:  false,
-			expectedReadTicker:   false,
-			expectedSearchTicker: false,
-			expectedError:        "tempo-write-backoff-duration must be greater than 0",
+			name:                  "Invalid write duration (zero)",
+			writeDuration:         0,
+			readDuration:          0,
+			searchDuration:        0,
+			expectedWriteTicker:   false,
+			expectedReadTicker:    false,
+			expectedSearchTicker:  false,
+			expectedMetricsTicker: false,
+			expectedError:         "tempo-write-backoff-duration must be greater than 0",
 		},
 		{
-			name:                 "No read durations set",
-			writeDuration:        1 * time.Second,
-			readDuration:         0,
-			searchDuration:       1 * time.Second,
-			expectedWriteTicker:  true,
-			expectedReadTicker:   false,
-			expectedSearchTicker: true,
-			expectedError:        "",
+			name:                  "No read durations set",
+			writeDuration:         1 * time.Second,
+			readDuration:          0,
+			searchDuration:        1 * time.Second,
+			expectedWriteTicker:   true,
+			expectedReadTicker:    false,
+			expectedSearchTicker:  true,
+			expectedMetricsTicker: false,
+			expectedError:         "",
 		},
 		{
-			name:                 "No read or search durations set",
-			writeDuration:        1 * time.Second,
-			readDuration:         0,
-			searchDuration:       0,
-			expectedWriteTicker:  false,
-			expectedReadTicker:   false,
-			expectedSearchTicker: false,
-			expectedError:        "at least one of tempo-search-backoff-duration or tempo-read-backoff-duration must be set",
+			name:                  "Valid metrics duration",
+			writeDuration:         1 * time.Second,
+			readDuration:          0,
+			searchDuration:        0,
+			metricsDuration:       1 * time.Second,
+			expectedWriteTicker:   true,
+			expectedReadTicker:    false,
+			expectedSearchTicker:  false,
+			expectedMetricsTicker: true,
+			expectedError:         "",
+		},
+		{
+			name:                  "No read or search durations set",
+			writeDuration:         1 * time.Second,
+			readDuration:          0,
+			searchDuration:        0,
+			expectedWriteTicker:   false,
+			expectedReadTicker:    false,
+			expectedSearchTicker:  false,
+			expectedMetricsTicker: false,
+			expectedError:         "at least one of tempo-search-backoff-duration, tempo-read-backoff-duration or tempo-metrics-backoff-duration must be set",
 		},
 	}
 
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
-			tickerWrite, tickerRead, tickerSearch, err := initTickers(tt.writeDuration, tt.readDuration, tt.searchDuration)
+			tickerWrite, tickerRead, tickerSearch, tickerMetrics, err := initTickers(tt.writeDuration, tt.readDuration, tt.searchDuration, tt.metricsDuration)
 
 			assert.Equal(t, tt.expectedWriteTicker, tickerWrite != nil, "TickerWrite")
 			assert.Equal(t, tt.expectedReadTicker, tickerRead != nil, "TickerRead")
 			assert.Equal(t, tt.expectedSearchTicker, tickerSearch != nil, "TickerSearch")
+			assert.Equal(t, tt.expectedMetricsTicker, tickerMetrics != nil, "TickerMetrics")
 
 			if tt.expectedError != "" {
 				assert.NotNil(t, err, "Expected error but got nil")
@@ -359,33 +384,11 @@ func doQueryTrace(f traceOps, err error) (traceMetrics, error) {
 	return queryTrace(&mockHTTPClient, traceInfo, logger)
 }
 
-func TestDoReadWhenTickerIsNil(t *testing.T) {
-	seed := time.Date(2008, 1, 1, 12, 0, 0, 0, time.UTC)
-	traceInfo := util.NewTraceInfo(seed, "test")
-
-	trace, _ := traceInfo.ConstructTraceFromEpoch()
-	startTime := time.Date(2007, 1, 1, 12, 0, 0, 0, time.UTC)
-	mockHTTPClient := MockHTTPClient{err: nil, traceResp: trace}
-	// Define the configuration
-	config := vultureConfiguration{
-		tempoOrgID:                "orgID",
-		tempoWriteBackoffDuration: time.Second,
-	}
-
-	logger = zap.NewNop()
-	r := rand.New(rand.NewSource(startTime.Unix()))
-
-	// Assert ticker is nil
-	doRead(&mockHTTPClient, nil, startTime, config.tempoWriteBackoffDuration, r, config, logger)
-	assert.Equal(t, 0, mockHTTPClient.GetRequestsCount())
-}
-
 func TestDoReadForAnOkRead(t *testing.T) {
 	seed := time.Date(2008, 1, 1, 12, 0, 0, 0, time.UTC)
 	traceInfo := util.NewTraceInfo(seed, "test")
 
 	trace, _ := traceInfo.ConstructTraceFromEpoch()
-	startTime := time.Date(2007, 1, 1, 12, 0, 0, 0, time.UTC)
 	mockHTTPClient := MockHTTPClient{err: nil, traceResp: trace}
 	// Define the configuration
 	config := vultureConfiguration{
@@ -393,15 +396,10 @@ func TestDoReadForAnOkRead(t *testing.T) {
 		tempoWriteBackoffDuration: time.Second,
 	}
 
-	ticker := time.NewTicker(10 * time.Millisecond)
 	logger = zap.NewNop()
-	r := rand.New(rand.NewSource(startTime.Unix()))
 
-	// Assert an ok read
-	doRead(&mockHTTPClient, ticker, startTime, config.tempoWriteBackoffDuration, r, config, logger)
-	time.Sleep(time.Second)
-	ticker.Stop()
-	assert.Greater(t, mockHTTPClient.GetRequestsCount(), 0)
+	doRead(&mockHTTPClient, config, traceInfo, logger)
+	assert.Equal(t, 1, mockHTTPClient.GetRequestsCount())
 }
 
 func TestDoReadForAnErroredRead(t *testing.T) {
@@ -409,7 +407,6 @@ func TestDoReadForAnErroredRead(t *testing.T) {
 	traceInfo := util.NewTraceInfo(seed, "test")
 
 	trace, _ := traceInfo.ConstructTraceFromEpoch()
-	startTime := time.Date(2007, 1, 1, 12, 0, 0, 0, time.UTC)
 
 	// Define the configuration
 	config := vultureConfiguration{
@@ -417,14 +414,11 @@ func TestDoReadForAnErroredRead(t *testing.T) {
 		tempoWriteBackoffDuration: time.Second,
 	}
 
-	ticker := time.NewTicker(10 * time.Millisecond)
 	logger = zap.NewNop()
-	r := rand.New(rand.NewSource(startTime.Unix()))
 
 	// Assert a read with errors
 	mockHTTPClient := MockHTTPClient{err: errors.New("an error"), traceResp: trace}
-	doRead(&mockHTTPClient, ticker, startTime, config.tempoWriteBackoffDuration, r, config, logger)
-	time.Sleep(time.Second)
+	doRead(&mockHTTPClient, config, traceInfo, logger)
 	assert.Equal(t, 0, mockHTTPClient.GetRequestsCount())
 }
 
@@ -539,7 +533,6 @@ func TestSearchTag(t *testing.T) {
 func TestDoSearch(t *testing.T) {
 	seed := time.Date(2008, 1, 1, 12, 0, 0, 0, time.UTC)
 	traceInfo := util.NewTraceInfo(seed, "test")
-	startTime := time.Date(2000, 1, 1, 12, 0, 0, 0, time.UTC)
 
 	// Define the configuration
 	config := vultureConfiguration{
@@ -549,9 +542,7 @@ func TestDoSearch(t *testing.T) {
 		tempoRetentionDuration:    time.Second * 10,
 	}
 
-	ticker := time.NewTicker(10 * time.Millisecond)
 	logger = zap.NewNop()
-	r := rand.New(rand.NewSource(startTime.Unix()))
 
 	searchResponse := []*tempopb.TraceSearchMetadata{
 		{
@@ -572,61 +563,30 @@ func TestDoSearch(t *testing.T) {
 			},
 		},
 	}
-	logger = zap.NewNop()
+
 	mockHTTPClient := MockHTTPClient{err: nil, searchResponse: searchResponse}
 
-	doSearch(&mockHTTPClient, ticker, startTime, config.tempoWriteBackoffDuration, r, config, logger)
-
-	time.Sleep(time.Second)
-	ticker.Stop()
+	doSearch(&mockHTTPClient, config, traceInfo, logger)
 	assert.Greater(t, mockHTTPClient.GetSearchesCount(), 0)
 }
 
-func TestDoSearchWhenTickerIsNil(t *testing.T) {
-	startTime := time.Date(2000, 1, 1, 12, 0, 0, 0, time.UTC)
+func TestDoSearchError(t *testing.T) {
+	seed := time.Date(2008, 1, 1, 12, 0, 0, 0, time.UTC)
+	traceInfo := util.NewTraceInfo(seed, "test")
 
 	// Define the configuration
 	config := vultureConfiguration{
-		tempoOrgID: "orgID",
-		// This is a hack to ensure the trace is "ready"
-		tempoWriteBackoffDuration: -time.Hour * 10000,
-		tempoRetentionDuration:    time.Second * 10,
+		tempoOrgID:                "orgID",
+		tempoWriteBackoffDuration: time.Second,
 	}
 
-	logger = zap.NewNop()
-	r := rand.New(rand.NewSource(startTime.Unix()))
-
-	logger = zap.NewNop()
-
-	mockHTTPClient := MockHTTPClient{err: nil}
-	doSearch(&mockHTTPClient, nil, startTime, config.tempoWriteBackoffDuration, r, config, logger)
-	assert.Equal(t, mockHTTPClient.GetSearchesCount(), 0)
-}
-
-func TestDoSearchOnSearchError(t *testing.T) {
-	startTime := time.Date(2000, 1, 1, 12, 0, 0, 0, time.UTC)
-
-	// Define the configuration
-	config := vultureConfiguration{
-		tempoOrgID: "orgID",
-		// This is a hack to ensure the trace is "ready"
-		tempoWriteBackoffDuration: -time.Hour * 10000,
-		tempoRetentionDuration:    time.Second * 10,
-	}
-
-	ticker := time.NewTicker(10 * time.Millisecond)
-	logger = zap.NewNop()
-	r := rand.New(rand.NewSource(startTime.Unix()))
-
 	logger = zap.NewNop()
 
 	// Assert an errored search
 	mockHTTPClient := MockHTTPClient{err: errors.New("an error")}
 
-	doSearch(&mockHTTPClient, ticker, startTime, config.tempoWriteBackoffDuration, r, config, logger)
-	ticker.Stop()
-
-	assert.Equal(t, mockHTTPClient.searchesCount, 0)
+	doSearch(&mockHTTPClient, config, traceInfo, logger)
+	assert.Equal(t, 0, mockHTTPClient.GetSearchesCount())
 }
 
 func TestGetGrpcEndpoint(t *testing.T) {
@@ -674,3 +634,333 @@ func TestNewJaegerGRPCClient(t *testing.T) {
 	assert.Error(t, err)
 	assert.Nil(t, client)
 }
+
+func TestQueryMetrics(t *testing.T) {
+	seed := time.Date(2008, 1, 1, 12, 0, 0, 0, time.UTC)
+
+	config := vultureConfiguration{
+		tempoOrgID:                 "orgID",
+		tempoWriteBackoffDuration:  time.Second,
+		tempoSearchBackoffDuration: time.Second,
+	}
+
+	info := util.NewTraceInfo(seed, config.tempoOrgID)
+	hexID := info.HexID()
+
+	successMetricsResponse := &tempopb.QueryRangeResponse{
+		Series: []*tempopb.TimeSeries{
+			{
+				Samples: []tempopb.Sample{
+					{
+						TimestampMs: seed.UnixMilli(),
+						Value:       2.0,
+					},
+				},
+			},
+		},
+	}
+
+	tests := []struct {
+		name            string
+		response        *tempopb.QueryRangeResponse
+		searchResponse  []*tempopb.TraceSearchMetadata
+		err             error
+		expectedMetrics traceMetrics
+		expectedError   string
+	}{
+		{
+			name:     "successful metrics query: 1 trace, 2 spans",
+			response: successMetricsResponse,
+			searchResponse: []*tempopb.TraceSearchMetadata{
+				{
+					SpanSets: []*tempopb.SpanSet{
+						{
+							Matched: 2,
+						},
+					},
+				},
+			},
+			expectedMetrics: traceMetrics{
+				requested: 1,
+			},
+		},
+		{
+			name:     "successful metrics query: 2 traces, 1 span each",
+			response: successMetricsResponse,
+			searchResponse: []*tempopb.TraceSearchMetadata{
+				{
+					SpanSets: []*tempopb.SpanSet{
+						{
+							Matched: 1,
+						},
+					},
+				},
+				{
+					SpanSets: []*tempopb.SpanSet{
+						{
+							Matched: 1,
+						},
+					},
+				},
+			},
+			expectedMetrics: traceMetrics{
+				requested: 1,
+			},
+		},
+		{
+			name:     "Less than expected",
+			response: successMetricsResponse,
+			searchResponse: []*tempopb.TraceSearchMetadata{
+				{
+					SpanSets: []*tempopb.SpanSet{
+						{
+							Matched: 4,
+						},
+					},
+				},
+			},
+			expectedMetrics: traceMetrics{
+				inaccurateMetrics: 1,
+				requested:         1,
+			},
+			expectedError: "TraceQL Metrics results are inaccurate: metric count sum=2.000000, actual span count=4",
+		},
+		{
+			name:           "No traces",
+			response:       successMetricsResponse,
+			searchResponse: make([]*tempopb.TraceSearchMetadata, 0),
+			expectedMetrics: traceMetrics{
+				inaccurateMetrics: 1,
+				requested:         1,
+			},
+			expectedError: "TraceQL Metrics results are inaccurate: metric count sum=2.000000, actual span count=0",
+		},
+		{
+			name: "no series in response",
+			response: &tempopb.QueryRangeResponse{
+				Series: []*tempopb.TimeSeries{},
+			},
+			expectedMetrics: traceMetrics{
+				requested:         1,
+				notFoundByMetrics: 1,
+			},
+			expectedError: fmt.Sprintf("expected trace %s not found in metrics", hexID),
+		},
+		{
+			name: "no series in response (nil)",
+			response: &tempopb.QueryRangeResponse{
+				Series: nil,
+			},
+			expectedMetrics: traceMetrics{
+				requested:         1,
+				notFoundByMetrics: 1,
+			},
+			expectedError: fmt.Sprintf("expected trace %s not found in metrics", hexID),
+		},
+		{
+			name: "invalid series data",
+			response: &tempopb.QueryRangeResponse{
+				Series: make([]*tempopb.TimeSeries, 1),
+			},
+			expectedMetrics: traceMetrics{
+				requested:              1,
+				incorrectMetricsResult: 1,
+			},
+			expectedError: "expected time series, got nil",
+		},
+		{
+			name: "too many series",
+			response: &tempopb.QueryRangeResponse{
+				Series: make([]*tempopb.TimeSeries, 2),
+			},
+			expectedMetrics: traceMetrics{
+				requested:              1,
+				incorrectMetricsResult: 1,
+			},
+			expectedError: "expected exactly 1 series, got 2",
+		},
+		{
+			name:     "query error",
+			response: nil,
+			err:      errors.New("metrics query failed"),
+			expectedMetrics: traceMetrics{
+				requested:     1,
+				requestFailed: 1,
+			},
+			expectedError: "metrics query failed",
+		},
+	}
+
+	logger = zap.NewNop()
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			mockHTTPClient := &MockHTTPClient{
+				err:            tt.err,
+				metricsResp:    tt.response,
+				searchResponse: tt.searchResponse,
+			}
+
+			metrics, err := queryMetrics(mockHTTPClient, seed, config, logger)
+			assert.Equal(t, tt.expectedMetrics, metrics)
+
+			if tt.expectedError != "" {
+				require.Error(t, err)
+				assert.Contains(t, err.Error(), tt.expectedError)
+			} else {
+				require.NoError(t, err)
+			}
+		})
+	}
+}
+
+func TestDoMetrics(t *testing.T) {
+	seed := time.Date(2008, 1, 1, 12, 0, 0, 0, time.UTC)
+	traceInfo := util.NewTraceInfo(seed, "test")
+
+	// Define the configuration
+	config := vultureConfiguration{
+		tempoOrgID:                "orgID",
+		tempoWriteBackoffDuration: time.Second,
+	}
+
+	logger = zap.NewNop()
+
+	mockHTTPClient := &MockHTTPClient{
+		err: nil,
+		metricsResp: &tempopb.QueryRangeResponse{
+			Series: []*tempopb.TimeSeries{
+				{
+					Samples: []tempopb.Sample{
+						{
+							TimestampMs: seed.UnixMilli(),
+							Value:       1.0,
+						},
+					},
+				},
+			},
+		},
+	}
+
+	doMetrics(mockHTTPClient, config, traceInfo, logger)
+	assert.Equal(t, 1, mockHTTPClient.GetMetricsCount())
+}
+
+func TestRunCheckerWithNilTicker(t *testing.T) {
+	config := vultureConfiguration{
+		tempoOrgID:                "orgID",
+		tempoWriteBackoffDuration: time.Second,
+	}
+
+	logger = zap.NewNop()
+
+	checkerCalled := false
+	selectPastTimestamp := func(_ time.Time) (newStart, ts time.Time, skip bool) {
+		return time.Now(), time.Now(), false
+	}
+	checker := func(_ *util.TraceInfo, _ *zap.Logger) {
+		checkerCalled = true
+	}
+
+	runChecker(nil, config, selectPastTimestamp, checker, logger)
+	assert.False(t, checkerCalled)
+}
+
+func TestRunCheckerWithSkip(t *testing.T) {
+	config := vultureConfiguration{
+		tempoOrgID:                "orgID",
+		tempoWriteBackoffDuration: time.Second,
+	}
+
+	logger = zap.NewNop()
+	ticker := time.NewTicker(time.Millisecond) // fires immediately
+	defer ticker.Stop()
+
+	// Checker function that signals completion
+	var checkerCalled bool
+	checker := func(_ *util.TraceInfo, _ *zap.Logger) {
+		checkerCalled = true
+	}
+
+	alwaysSkip := func(_ time.Time) (newStart, ts time.Time, skip bool) {
+		return time.Now(), time.Now(), true
+	}
+
+	runChecker(ticker, config, alwaysSkip, checker, logger)
+	time.Sleep(5 * time.Millisecond)
+	// Ensure the checker was not called due to skip=true
+	assert.False(t, checkerCalled)
+}
+
+func TestRunCheckerTraceNotReady(t *testing.T) {
+	seed := time.Date(2008, 1, 1, 12, 0, 0, 0, time.UTC)
+
+	config := vultureConfiguration{
+		tempoOrgID:                    "orgID",
+		tempoWriteBackoffDuration:     10 * time.Hour, // Very long to ensure trace not ready
+		tempoLongWriteBackoffDuration: 20 * time.Hour,
+	}
+
+	logger = zap.NewNop()
+
+	ticker := time.NewTicker(time.Millisecond) // fires immediately
+
+	// Checker function that signals completion
+	var checkerCalled bool
+	checker := func(_ *util.TraceInfo, _ *zap.Logger) {
+		checkerCalled = true
+	}
+
+	selectPastTimestamp := func(_ time.Time) (newStart, ts time.Time, skip bool) {
+		return seed, seed, false
+	}
+	runChecker(ticker, config, selectPastTimestamp, checker, logger)
+	time.Sleep(5 * time.Millisecond)
+	// Ensure the checker was not called because trace is not ready
+	assert.False(t, checkerCalled)
+}
+
+func TestRunCheckerSuccess(t *testing.T) {
+	seed := time.Date(2008, 1, 1, 12, 0, 0, 0, time.UTC)
+	startTime := time.Date(2007, 1, 1, 12, 0, 0, 0, time.UTC)
+	now := time.Date(2009, 1, 1, 12, 0, 0, 0, time.UTC) // Far in the future to ensure trace is ready
+
+	config := vultureConfiguration{
+		tempoOrgID:                    "orgID",
+		tempoWriteBackoffDuration:     time.Second, // Small value to ensure trace is ready
+		tempoLongWriteBackoffDuration: time.Second,
+	}
+
+	logger = zap.NewNop()
+
+	tickChan := make(chan time.Time, 1)
+	mockTicker := &time.Ticker{C: tickChan}
+
+	// Send the time on the channel to trigger the ticker
+	go func() {
+		tickChan <- now
+	}()
+
+	// Checker function that signals completion
+	mx := sync.Mutex{}
+	checkerCalled := false
+	var checkedInfo *util.TraceInfo
+	checker := func(info *util.TraceInfo, _ *zap.Logger) {
+		mx.Lock()
+		defer mx.Unlock()
+		checkerCalled = true
+		checkedInfo = info
+	}
+
+	selectPastTimestamp := func(_ time.Time) (newStart, ts time.Time, skip bool) {
+		return startTime, seed, false
+	}
+	runChecker(mockTicker, config, selectPastTimestamp, checker, logger)
+	time.Sleep(10 * time.Millisecond)
+
+	mx.Lock()
+	defer mx.Unlock()
+	assert.True(t, checkerCalled)
+	require.NotNil(t, checkedInfo)
+	assert.Equal(t, seed.Unix(), checkedInfo.Timestamp().Unix())
+}

cmd/tempo-vulture/mocks.go

@@ -46,6 +46,8 @@ type MockHTTPClient struct {
 	requestsCount  int
 	searchResponse []*tempopb.TraceSearchMetadata
 	searchesCount  int
+	metricsResp    *tempopb.QueryRangeResponse
+	metricsCount   int
 	// We need the lock to control concurrent accesses to shared variables in the tests
 	m sync.Mutex
 }
@@ -70,6 +72,17 @@ func (m *MockHTTPClient) MetricsSummary(query string, groupBy string, start int6
 	panic("unimplemented")
 }
 
+//nolint:all
+func (m *MockHTTPClient) MetricsQueryRange(query string, start, end int, step string, exemplars int) (*tempopb.QueryRangeResponse, error) {
+	if m.err != nil {
+		return nil, m.err
+	}
+	m.m.Lock()
+	defer m.m.Unlock()
+	m.metricsCount++
+	return m.metricsResp, nil
+}
+
 //nolint:all
 func (m *MockHTTPClient) PatchOverrides(limits *userconfigurableoverrides.Limits) (*userconfigurableoverrides.Limits, string, error) {
 	panic("unimplemented")
@@ -198,3 +211,9 @@ func (m *MockHTTPClient) SetOverrides(limits *userconfigurableoverrides.Limits,
 func (m *MockHTTPClient) WithTransport(t http.RoundTripper) {
 	panic("unimplemented")
 }
+
+func (m *MockHTTPClient) GetMetricsCount() int {
+	m.m.Lock()
+	defer m.m.Unlock()
+	return m.metricsCount
+}

cmd/tempo-vulture/testdata/trace.json

@@ -112,6 +112,7 @@
             {
               "traceId": "CSCPj2v0UkJjHavLBtfJxg==",
               "spanId": "MaSlu9+OmtM=",
+              "parentSpanId": "JOzOehYQAj4=",
               "name": "vulture-47",
               "startTimeUnixNano": "1636729665000000000",
               "endTimeUnixNano": "1636729665000082000",
@@ -375,6 +376,7 @@
             {
               "traceId": "CSCPj2v0UkJjHavLBtfJxg==",
               "spanId": "U0zLC99qJRg=",
+              "parentSpanId": "YoO3VnVCu5A=",
               "name": "vulture-10",
               "startTimeUnixNano": "1636729665000000000",
               "endTimeUnixNano": "1636729665000094000",
@@ -558,6 +560,7 @@
             {
               "traceId": "CSCPj2v0UkJjHavLBtfJxg==",
               "spanId": "RKpoZaS43G8=",
+              "parentSpanId": "FJqfJhH5hHM=",
               "name": "vulture-38",
               "startTimeUnixNano": "1636729665000000000",
               "endTimeUnixNano": "1636729665000016000",
@@ -698,6 +701,7 @@
             {
               "traceId": "CSCPj2v0UkJjHavLBtfJxg==",
               "spanId": "VwCj9LoPYtw=",
+              "parentSpanId": "RKpoZaS43G8=",
               "name": "vulture-62",
               "startTimeUnixNano": "1636729665000000000",
               "endTimeUnixNano": "1636729665000003000",
@@ -957,6 +961,7 @@
             {
               "traceId": "CSCPj2v0UkJjHavLBtfJxg==",
               "spanId": "MpZcQbqgWkE=",
+              "parentSpanId": "b1kdTyceTsU=",
               "name": "vulture-36",
               "startTimeUnixNano": "1636729665000000000",
               "endTimeUnixNano": "1636729665000030000",
@@ -1039,6 +1044,7 @@
             {
               "traceId": "CSCPj2v0UkJjHavLBtfJxg==",
               "spanId": "Qnv5gLdo3mA=",
+              "parentSpanId": "MpZcQbqgWkE=",
               "name": "vulture-67",
               "startTimeUnixNano": "1636729665000000000",
               "endTimeUnixNano": "1636729665000082000",
@@ -1109,6 +1115,7 @@
             {
               "traceId": "CSCPj2v0UkJjHavLBtfJxg==",
               "spanId": "HKgTGlqMnxg=",
+              "parentSpanId": "Qnv5gLdo3mA=",
               "name": "vulture-81",
               "startTimeUnixNano": "1636729665000000000",
               "endTimeUnixNano": "1636729665000045000",
@@ -1298,6 +1305,7 @@
             {
               "traceId": "CSCPj2v0UkJjHavLBtfJxg==",
               "spanId": "LHSmWp3GTZc=",
+              "parentSpanId": "V44AV0XxAeI=",
               "name": "vulture-64",
               "startTimeUnixNano": "1636729665000000000",
               "endTimeUnixNano": "1636729665000018000",
@@ -1486,6 +1494,7 @@
             {
               "traceId": "CSCPj2v0UkJjHavLBtfJxg==",
               "spanId": "OAVNaHlwL5k=",
+              "parentSpanId": "cOmTcl2W+TU=",
               "name": "vulture-73",
               "startTimeUnixNano": "1636729665000000000",
               "endTimeUnixNano": "1636729665000062000",
@@ -1556,6 +1565,7 @@
             {
               "traceId": "CSCPj2v0UkJjHavLBtfJxg==",
               "spanId": "Z2q4YtdoMOc=",
+              "parentSpanId": "OAVNaHlwL5k=",
               "name": "vulture-94",
               "startTimeUnixNano": "1636729665000000000",
               "endTimeUnixNano": "1636729665000061000",
@@ -1803,6 +1813,7 @@
             {
               "traceId": "CSCPj2v0UkJjHavLBtfJxg==",
               "spanId": "Zuy0zydsCKc=",
+              "parentSpanId": "dEvAKsJfeN8=",
               "name": "vulture-24",
               "startTimeUnixNano": "1636729665000000000",
               "endTimeUnixNano": "1636729665000097000",
@@ -1925,6 +1936,7 @@
             {
               "traceId": "CSCPj2v0UkJjHavLBtfJxg==",
               "spanId": "LhkOF9Qir1o=",
+              "parentSpanId": "Zuy0zydsCKc=",
               "name": "vulture-8",
               "startTimeUnixNano": "1636729665000000000",
               "endTimeUnixNano": "1636729665000073000",
@@ -1983,6 +1995,7 @@
             {
               "traceId": "CSCPj2v0UkJjHavLBtfJxg==",
               "spanId": "QoiSbdw1jWA=",
+              "parentSpanId": "LhkOF9Qir1o=",
               "name": "vulture-76",
               "startTimeUnixNano": "1636729665000000000",
               "endTimeUnixNano": "1636729665000033000",
@@ -2172,6 +2185,7 @@
             {
               "traceId": "CSCPj2v0UkJjHavLBtfJxg==",
               "spanId": "GQZfyLGl9DU=",
+              "parentSpanId": "eX11PkWR7/U=",
               "name": "vulture-22",
               "startTimeUnixNano": "1636729665000000000",
               "endTimeUnixNano": "1636729665000097000",
@@ -2242,6 +2256,7 @@
             {
               "traceId": "CSCPj2v0UkJjHavLBtfJxg==",
               "spanId": "amlTKv36azs=",
+              "parentSpanId": "GQZfyLGl9DU=",
               "name": "vulture-83",
               "startTimeUnixNano": "1636729665000000000",
               "endTimeUnixNano": "1636729665000036000",
@@ -2523,6 +2538,7 @@
             {
               "traceId": "CSCPj2v0UkJjHavLBtfJxg==",
               "spanId": "KoCq8Fwo+O4=",
+              "parentSpanId": "HoKLYl94RRg=",
               "name": "vulture-61",
               "startTimeUnixNano": "1636729665000000000",
               "endTimeUnixNano": "1636729665000021000",
@@ -2593,6 +2609,7 @@
             {
               "traceId": "CSCPj2v0UkJjHavLBtfJxg==",
               "spanId": "VvqffnRCAVw=",
+              "parentSpanId": "KoCq8Fwo+O4=",
               "name": "vulture-48",
               "startTimeUnixNano": "1636729665000000000",
               "endTimeUnixNano": "1636729665000070000",
@@ -2816,6 +2833,7 @@
             {
               "traceId": "CSCPj2v0UkJjHavLBtfJxg==",
               "spanId": "JRfsQbbqoTA=",
+              "parentSpanId": "FiHBKfLuSo8=",
               "name": "vulture-96",
               "startTimeUnixNano": "1636729665000000000",
               "endTimeUnixNano": "1636729665000055000",
@@ -2944,6 +2962,7 @@
             {
               "traceId": "CSCPj2v0UkJjHavLBtfJxg==",
               "spanId": "JehDhsbZwwg=",
+              "parentSpanId": "JRfsQbbqoTA=",
               "name": "vulture-45",
               "startTimeUnixNano": "1636729665000000000",
               "endTimeUnixNano": "1636729665000033000",
@@ -3139,6 +3158,7 @@
             {
               "traceId": "CSCPj2v0UkJjHavLBtfJxg==",
               "spanId": "A6W2oLkGL1U=",
+              "parentSpanId": "CLF/KWb5tZE=",
               "name": "vulture-42",
               "startTimeUnixNano": "1636729665000000000",
               "endTimeUnixNano": "1636729665000044000",
@@ -3250,6 +3270,7 @@
             {
               "traceId": "CSCPj2v0UkJjHavLBtfJxg==",
               "spanId": "NS8nJt7eKtM=",
+              "parentSpanId": "A6W2oLkGL1U=",
               "name": "vulture-37",
               "startTimeUnixNano": "1636729665000000000",
               "endTimeUnixNano": "1636729665000073000",
@@ -3456,6 +3477,7 @@
             {
               "traceId": "CSCPj2v0UkJjHavLBtfJxg==",
               "spanId": "D0OHr0cI5lo=",
+              "parentSpanId": "bBfIWctLWVo=",
               "name": "vulture-92",
               "startTimeUnixNano": "1636729665000000000",
               "endTimeUnixNano": "1636729665000054000",
@@ -3561,6 +3583,7 @@
             {
               "traceId": "CSCPj2v0UkJjHavLBtfJxg==",
               "spanId": "fDQCFQXBISk=",
+              "parentSpanId": "D0OHr0cI5lo=",
               "name": "vulture-94",
               "startTimeUnixNano": "1636729665000000000",
               "endTimeUnixNano": "1636729665000083000",
@@ -3666,6 +3689,7 @@
             {
               "traceId": "CSCPj2v0UkJjHavLBtfJxg==",
               "spanId": "QQM1rdF2I/E=",
+              "parentSpanId": "fDQCFQXBISk=",
               "name": "vulture-53",
               "startTimeUnixNano": "1636729665000000000",
               "endTimeUnixNano": "1636729665000075000",
@@ -3877,6 +3901,7 @@
             {
               "traceId": "CSCPj2v0UkJjHavLBtfJxg==",
               "spanId": "H8bLLHJKAW4=",
+              "parentSpanId": "Du5ajMQehCA=",
               "name": "vulture-44",
               "startTimeUnixNano": "1636729665000000000",
               "endTimeUnixNano": "1636729665000065000",
@@ -3988,6 +4013,7 @@
             {
               "traceId": "CSCPj2v0UkJjHavLBtfJxg==",
               "spanId": "VZ9YK67tu40=",
+              "parentSpanId": "H8bLLHJKAW4=",
               "name": "vulture-3",
               "startTimeUnixNano": "1636729665000000000",
               "endTimeUnixNano": "1636729665000097000",
@@ -4069,6 +4095,7 @@
             {
               "traceId": "CSCPj2v0UkJjHavLBtfJxg==",
               "spanId": "WcSlH3kOa6A=",
+              "parentSpanId": "VZ9YK67tu40=",
               "name": "vulture-21",
               "startTimeUnixNano": "1636729665000000000",
               "endTimeUnixNano": "1636729665000044000",
@@ -4286,6 +4313,7 @@
             {
               "traceId": "CSCPj2v0UkJjHavLBtfJxg==",
               "spanId": "UeGD9oKF+sA=",
+              "parentSpanId": "b/phdO27Mbk=",
               "name": "vulture-64",
               "startTimeUnixNano": "1636729665000000000",
               "endTimeUnixNano": "1636729665000020000",

cmd/tempo/Dockerfile

@@ -1,13 +1,19 @@
-FROM alpine:3.21 AS certs
-RUN apk --update add ca-certificates
+FROM alpine:latest AS ca-certificates
+RUN apk add --update --no-cache ca-certificates
+
+FROM gcr.io/distroless/static-debian12:debug
+
+# we need this because some docker-compose files call chown assuming there's a shell
+SHELL ["/busybox/sh", "-c"]
+
+RUN ["/busybox/addgroup", "-g", "10001", "-S", "tempo"]
+RUN ["/busybox/adduser", "-u", "10001", "-S", "tempo", "-G", "tempo"]
+RUN ["/busybox/mkdir", "-p", "/var/tempo", "-m", "0700"]
+RUN ["/busybox/chown", "-R", "tempo:tempo", "/var/tempo"]
+
 ARG TARGETARCH
 COPY bin/linux/tempo-${TARGETARCH} /tempo
-
-RUN addgroup -g 10001 -S tempo && \
-    adduser -u 10001 -S tempo -G tempo
-
-RUN mkdir -p /var/tempo -m 0700 && \
-    chown -R tempo:tempo /var/tempo
+COPY --from=ca-certificates /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/
 
 USER 10001:10001
 

cmd/tempo/app/app.go

@@ -21,8 +21,9 @@ import (
 	"github.com/grafana/dskit/server"
 	"github.com/grafana/dskit/services"
 	"github.com/grafana/dskit/signals"
+	"github.com/grafana/tempo/modules/backendworker"
+	"github.com/grafana/tempo/modules/blockbuilder"
 	"github.com/jedib0t/go-pretty/v6/table"
-	"github.com/prometheus/client_golang/prometheus"
 	"github.com/prometheus/common/version"
 	"go.uber.org/atomic"
 	"google.golang.org/grpc"
@@ -30,6 +31,7 @@ import (
 	"gopkg.in/yaml.v3"
 
 	"github.com/grafana/tempo/cmd/tempo/build"
+	"github.com/grafana/tempo/modules/backendscheduler"
 	"github.com/grafana/tempo/modules/compactor"
 	"github.com/grafana/tempo/modules/distributor"
 	"github.com/grafana/tempo/modules/distributor/receiver"
@@ -53,13 +55,6 @@ const (
 )
 
 var (
-	metricConfigFeatDesc = prometheus.NewDesc(
-		"tempo_feature_enabled",
-		"Boolean for configuration variables",
-		[]string{"feature"},
-		nil,
-	)
-
 	statFeatureEnabledAuth         = usagestats.NewInt("feature_enabled_auth_stats")
 	statFeatureEnabledMultitenancy = usagestats.NewInt("feature_enabled_multitenancy")
 )
@@ -71,18 +66,24 @@ type App struct {
 	Server         TempoServer
 	InternalServer *server.Server
 
-	readRings     map[string]*ring.Ring
-	Overrides     overrides.Service
-	distributor   *distributor.Distributor
-	querier       *querier.Querier
-	frontend      *frontend_v1.Frontend
-	compactor     *compactor.Compactor
-	ingester      *ingester.Ingester
-	generator     *generator.Generator
-	store         storage.Store
-	usageReport   *usagestats.Reporter
-	cacheProvider cache.Provider
-	MemberlistKV  *memberlist.KVInitService
+	readRings            map[string]*ring.Ring
+	partitionRing        *ring.PartitionInstanceRing
+	partitionRingWatcher *ring.PartitionRingWatcher
+	generatorRingWatcher *ring.PartitionRingWatcher
+	Overrides            overrides.Service
+	distributor          *distributor.Distributor
+	querier              *querier.Querier
+	frontend             *frontend_v1.Frontend
+	compactor            *compactor.Compactor
+	ingester             *ingester.Ingester
+	generator            *generator.Generator
+	blockBuilder         *blockbuilder.BlockBuilder
+	store                storage.Store
+	usageReport          *usagestats.Reporter
+	cacheProvider        cache.Provider
+	MemberlistKV         *memberlist.KVInitService
+	backendScheduler     *backendscheduler.BackendScheduler
+	backendWorker        *backendworker.BackendWorker
 
 	HTTPAuthMiddleware       middleware.Interface
 	TracesConsumerMiddleware receiver.Middleware
@@ -128,6 +129,8 @@ func (t *App) setupAuthMiddleware() {
 		noGRPCAuthOn := []string{
 			"/frontend.Frontend/Process",
 			"/frontend.Frontend/NotifyClientShutdown",
+			"/tempopb.BackendScheduler/Next",
+			"/tempopb.BackendScheduler/UpdateJob",
 		}
 		ignoredMethods := map[string]bool{}
 		for _, m := range noGRPCAuthOn {

cmd/tempo/app/config.go

@@ -8,8 +8,10 @@ import (
 	"github.com/grafana/dskit/flagext"
 	"github.com/grafana/dskit/kv/memberlist"
 	"github.com/grafana/dskit/server"
-	"github.com/prometheus/client_golang/prometheus"
-
+	"github.com/grafana/tempo/modules/backendscheduler"
+	backendscheduler_client "github.com/grafana/tempo/modules/backendscheduler/client"
+	"github.com/grafana/tempo/modules/backendworker"
+	"github.com/grafana/tempo/modules/blockbuilder"
 	"github.com/grafana/tempo/modules/cache"
 	"github.com/grafana/tempo/modules/compactor"
 	"github.com/grafana/tempo/modules/distributor"
@@ -21,6 +23,7 @@ import (
 	"github.com/grafana/tempo/modules/overrides"
 	"github.com/grafana/tempo/modules/querier"
 	"github.com/grafana/tempo/modules/storage"
+	"github.com/grafana/tempo/pkg/ingest"
 	internalserver "github.com/grafana/tempo/pkg/server"
 	"github.com/grafana/tempo/pkg/usagestats"
 	"github.com/grafana/tempo/pkg/util"
@@ -39,21 +42,26 @@ type Config struct {
 	HTTPAPIPrefix          string        `yaml:"http_api_prefix"`
 	EnableGoRuntimeMetrics bool          `yaml:"enable_go_runtime_metrics,omitempty"`
 
-	Server          server.Config           `yaml:"server,omitempty"`
-	InternalServer  internalserver.Config   `yaml:"internal_server,omitempty"`
-	Distributor     distributor.Config      `yaml:"distributor,omitempty"`
-	IngesterClient  ingester_client.Config  `yaml:"ingester_client,omitempty"`
-	GeneratorClient generator_client.Config `yaml:"metrics_generator_client,omitempty"`
-	Querier         querier.Config          `yaml:"querier,omitempty"`
-	Frontend        frontend.Config         `yaml:"query_frontend,omitempty"`
-	Compactor       compactor.Config        `yaml:"compactor,omitempty"`
-	Ingester        ingester.Config         `yaml:"ingester,omitempty"`
-	Generator       generator.Config        `yaml:"metrics_generator,omitempty"`
-	StorageConfig   storage.Config          `yaml:"storage,omitempty"`
-	Overrides       overrides.Config        `yaml:"overrides,omitempty"`
-	MemberlistKV    memberlist.KVConfig     `yaml:"memberlist,omitempty"`
-	UsageReport     usagestats.Config       `yaml:"usage_report,omitempty"`
-	CacheProvider   cache.Config            `yaml:"cache,omitempty"`
+	Server                server.Config                  `yaml:"server,omitempty"`
+	InternalServer        internalserver.Config          `yaml:"internal_server,omitempty"`
+	Distributor           distributor.Config             `yaml:"distributor,omitempty"`
+	IngesterClient        ingester_client.Config         `yaml:"ingester_client,omitempty"`
+	GeneratorClient       generator_client.Config        `yaml:"metrics_generator_client,omitempty"`
+	Querier               querier.Config                 `yaml:"querier,omitempty"`
+	Frontend              frontend.Config                `yaml:"query_frontend,omitempty"`
+	Compactor             compactor.Config               `yaml:"compactor,omitempty"`
+	Ingester              ingester.Config                `yaml:"ingester,omitempty"`
+	Generator             generator.Config               `yaml:"metrics_generator,omitempty"`
+	Ingest                ingest.Config                  `yaml:"ingest,omitempty"`
+	BlockBuilder          blockbuilder.Config            `yaml:"block_builder,omitempty"`
+	StorageConfig         storage.Config                 `yaml:"storage,omitempty"`
+	Overrides             overrides.Config               `yaml:"overrides,omitempty"`
+	MemberlistKV          memberlist.KVConfig            `yaml:"memberlist,omitempty"`
+	UsageReport           usagestats.Config              `yaml:"usage_report,omitempty"`
+	CacheProvider         cache.Config                   `yaml:"cache,omitempty"`
+	BackendScheduler      backendscheduler.Config        `yaml:"backend_scheduler,omitempty"`
+	BackenSchedulerClient backendscheduler_client.Config `yaml:"backend_scheduler_client,omitempty"`
+	BackendWorker         backendworker.Config           `yaml:"backend_worker,omitempty"`
 }
 
 func NewDefaultConfig() *Config {
@@ -96,7 +104,7 @@ func (c *Config) RegisterFlagsAndApplyDefaults(prefix string, f *flag.FlagSet) {
 	c.Server.GRPCServerMinTimeBetweenPings = 10 * time.Second
 	c.Server.GRPCServerPingWithoutStreamAllowed = true
 
-	f.IntVar(&c.Server.HTTPListenPort, "server.http-listen-port", 80, "HTTP server listen port.")
+	f.IntVar(&c.Server.HTTPListenPort, "server.http-listen-port", 3200, "HTTP server listen port.")
 	f.IntVar(&c.Server.GRPCListenPort, "server.grpc-listen-port", 9095, "gRPC server listen port.")
 
 	// Memberlist settings
@@ -116,20 +124,26 @@ func (c *Config) RegisterFlagsAndApplyDefaults(prefix string, f *flag.FlagSet) {
 
 	// Everything else
 	flagext.DefaultValues(&c.IngesterClient)
-	c.IngesterClient.GRPCClientConfig.GRPCCompression = ""
+	c.IngesterClient.GRPCClientConfig.GRPCCompression = "snappy"
 	flagext.DefaultValues(&c.GeneratorClient)
-	c.GeneratorClient.GRPCClientConfig.GRPCCompression = ""
+	c.GeneratorClient.GRPCClientConfig.GRPCCompression = "snappy"
+	flagext.DefaultValues(&c.BackenSchedulerClient)
+	c.BackenSchedulerClient.GRPCClientConfig.GRPCCompression = "snappy"
 	c.Overrides.RegisterFlagsAndApplyDefaults(f)
 
 	c.Distributor.RegisterFlagsAndApplyDefaults(util.PrefixConfig(prefix, "distributor"), f)
 	c.Ingester.RegisterFlagsAndApplyDefaults(util.PrefixConfig(prefix, "ingester"), f)
 	c.Generator.RegisterFlagsAndApplyDefaults(util.PrefixConfig(prefix, "generator"), f)
+	c.Ingest.RegisterFlagsAndApplyDefaults(util.PrefixConfig(prefix, "ingest"), f)
+	c.BlockBuilder.RegisterFlagsAndApplyDefaults(util.PrefixConfig(prefix, "block-builder"), f)
 	c.Querier.RegisterFlagsAndApplyDefaults(util.PrefixConfig(prefix, "querier"), f)
 	c.Frontend.RegisterFlagsAndApplyDefaults(util.PrefixConfig(prefix, "frontend"), f)
 	c.Compactor.RegisterFlagsAndApplyDefaults(util.PrefixConfig(prefix, "compactor"), f)
 	c.StorageConfig.RegisterFlagsAndApplyDefaults(util.PrefixConfig(prefix, "storage"), f)
 	c.UsageReport.RegisterFlagsAndApplyDefaults(util.PrefixConfig(prefix, "reporting"), f)
 	c.CacheProvider.RegisterFlagsAndApplyDefaults(util.PrefixConfig(prefix, "cache"), f)
+	c.BackendScheduler.RegisterFlagsAndApplyDefaults(util.PrefixConfig(prefix, "backend-scheduler"), f)
+	c.BackendWorker.RegisterFlagsAndApplyDefaults(util.PrefixConfig(prefix, "backend-worker"), f)
 }
 
 // MultitenancyIsEnabled checks if multitenancy is enabled
@@ -223,27 +237,13 @@ func (c *Config) CheckConfig() []ConfigWarning {
 		warnings = append(warnings, warnTraceByIDConcurrentShards)
 	}
 
+	if c.BlockBuilder.BlockConfig.BlockCfg.Version != c.BlockBuilder.WAL.Version {
+		warnings = append(warnings, warnBlockAndWALVersionMismatch)
+	}
+
 	return warnings
 }
 
-func (c *Config) Describe(ch chan<- *prometheus.Desc) {
-	ch <- metricConfigFeatDesc
-}
-
-func (c *Config) Collect(ch chan<- prometheus.Metric) {
-	features := map[string]int{
-		"search_external_endpoints": 0,
-	}
-
-	if len(c.Querier.Search.ExternalEndpoints) > 0 {
-		features["search_external_endpoints"] = 1
-	}
-
-	for label, value := range features {
-		ch <- prometheus.MustNewConstMetric(metricConfigFeatDesc, prometheus.GaugeValue, float64(value), label)
-	}
-}
-
 // ConfigWarning bundles message and explanation strings in one structure.
 type ConfigWarning struct {
 	Message string
@@ -302,6 +302,11 @@ var (
 		Message: "c.Frontend.TraceByID.ConcurrentShards greater than query_shards is invalid. concurrent_shards will be set to query_shards",
 		Explain: "Please remove ConcurrentShards or set it to a value less than or equal to QueryShards",
 	}
+
+	warnBlockAndWALVersionMismatch = ConfigWarning{
+		Message: "c.BlockConfig.BlockCfg.Version != c.WAL.Version",
+		Explain: "Block version and WAL version must match. WAL version will be set to block version",
+	}
 )
 
 func newV2Warning(setting string) ConfigWarning {

cmd/tempo/app/modules.go

@@ -12,6 +12,7 @@ import (
 	kitlog "github.com/go-kit/log"
 	"github.com/go-kit/log/level"
 	"github.com/grafana/dskit/dns"
+	"github.com/grafana/dskit/kv"
 	"github.com/grafana/dskit/kv/memberlist"
 	"github.com/grafana/dskit/middleware"
 	"github.com/grafana/dskit/modules"
@@ -22,6 +23,9 @@ import (
 	"github.com/prometheus/client_golang/prometheus"
 	"github.com/prometheus/client_golang/prometheus/collectors"
 
+	"github.com/grafana/tempo/modules/backendscheduler"
+	"github.com/grafana/tempo/modules/backendworker"
+	"github.com/grafana/tempo/modules/blockbuilder"
 	"github.com/grafana/tempo/modules/cache"
 	"github.com/grafana/tempo/modules/compactor"
 	"github.com/grafana/tempo/modules/distributor"
@@ -64,14 +68,20 @@ const (
 	IngesterRing          string = "ring"
 	SecondaryIngesterRing string = "secondary-ring"
 	MetricsGeneratorRing  string = "metrics-generator-ring"
+	PartitionRing         string = "partition-ring"
+	GeneratorRingWatcher  string = "generator-ring-watcher"
 
 	// individual targets
-	Distributor      string = "distributor"
-	Ingester         string = "ingester"
-	MetricsGenerator string = "metrics-generator"
-	Querier          string = "querier"
-	QueryFrontend    string = "query-frontend"
-	Compactor        string = "compactor"
+	Distributor                   string = "distributor"
+	Ingester                      string = "ingester"
+	MetricsGenerator              string = "metrics-generator"
+	MetricsGeneratorNoLocalBlocks string = "metrics-generator-no-local-blocks"
+	Querier                       string = "querier"
+	QueryFrontend                 string = "query-frontend"
+	Compactor                     string = "compactor"
+	BlockBuilder                  string = "block-builder"
+	BackendScheduler              string = "backend-scheduler"
+	BackendWorker                 string = "backend-worker"
 
 	// composite targets
 	SingleBinary         string = "all"
@@ -87,8 +97,6 @@ func (t *App) initServer() (services.Service, error) {
 	t.cfg.Server.MetricsNamespace = metricsNamespace
 	t.cfg.Server.ExcludeRequestInLog = true
 
-	prometheus.MustRegister(&t.cfg)
-
 	if t.cfg.EnableGoRuntimeMetrics {
 		// unregister default Go collector
 		prometheus.Unregister(collectors.NewGoCollector())
@@ -179,6 +187,25 @@ func (t *App) initReadRing(cfg ring.Config, name, key string) (*ring.Ring, error
 	return ring, nil
 }
 
+func (t *App) initPartitionRing() (services.Service, error) {
+	if !t.cfg.Ingest.Enabled {
+		return nil, nil
+	}
+
+	kvClient, err := kv.NewClient(t.cfg.Ingester.IngesterPartitionRing.KVStore, ring.GetPartitionRingCodec(), kv.RegistererWithKVName(prometheus.DefaultRegisterer, ingester.PartitionRingName+"-watcher"), util_log.Logger)
+	if err != nil {
+		return nil, fmt.Errorf("creating KV store for ingester partitions ring watcher: %w", err)
+	}
+
+	t.partitionRingWatcher = ring.NewPartitionRingWatcher(ingester.PartitionRingName, ingester.PartitionRingKey, kvClient, util_log.Logger, prometheus.WrapRegistererWithPrefix("tempo_", prometheus.DefaultRegisterer))
+	t.partitionRing = ring.NewPartitionInstanceRing(t.partitionRingWatcher, t.readRings[ringIngester], t.cfg.Ingester.LifecyclerConfig.RingConfig.HeartbeatTimeout)
+
+	// Expose a web page to view the partitions ring state.
+	t.Server.HTTPRouter().Path("/partition-ring").Methods("GET", "POST").Handler(ring.NewPartitionRingPageHandler(t.partitionRingWatcher, ring.NewPartitionRingEditor(ingester.PartitionRingKey, kvClient)))
+
+	return t.partitionRingWatcher, nil
+}
+
 func (t *App) initOverrides() (services.Service, error) {
 	o, err := overrides.NewOverrides(t.cfg.Overrides, newRuntimeConfigValidator(&t.cfg), prometheus.DefaultRegisterer)
 	if err != nil {
@@ -224,12 +251,16 @@ func (t *App) initOverridesAPI() (services.Service, error) {
 }
 
 func (t *App) initDistributor() (services.Service, error) {
+	t.cfg.Distributor.KafkaConfig = t.cfg.Ingest.Kafka
+	t.cfg.Distributor.KafkaWritePathEnabled = t.cfg.Ingest.Enabled // TODO: Don't mix config params
+
 	// todo: make ingester client a module instead of passing the config everywhere
 	distributor, err := distributor.New(t.cfg.Distributor,
 		t.cfg.IngesterClient,
 		t.readRings[ringIngester],
 		t.cfg.GeneratorClient,
 		t.readRings[ringMetricsGenerator],
+		t.partitionRing,
 		t.Overrides,
 		t.TracesConsumerMiddleware,
 		log.Logger, t.cfg.Server.LogLevel, prometheus.DefaultRegisterer)
@@ -252,7 +283,13 @@ func (t *App) initDistributor() (services.Service, error) {
 func (t *App) initIngester() (services.Service, error) {
 	t.cfg.Ingester.LifecyclerConfig.ListenPort = t.cfg.Server.GRPCListenPort
 	t.cfg.Ingester.DedicatedColumns = t.cfg.StorageConfig.Trace.Block.DedicatedColumns
-	ingester, err := ingester.New(t.cfg.Ingester, t.store, t.Overrides, prometheus.DefaultRegisterer)
+	t.cfg.Ingester.IngestStorageConfig = t.cfg.Ingest
+
+	// In SingleBinary mode don't try to discover parition from host name. Always use
+	// partition 0. This is for small installs or local/debugging setups.
+	singlePartition := t.cfg.Target == SingleBinary
+
+	ingester, err := ingester.New(t.cfg.Ingester, t.store, t.Overrides, prometheus.DefaultRegisterer, singlePartition)
 	if err != nil {
 		return nil, fmt.Errorf("failed to create ingester: %w", err)
 	}
@@ -272,7 +309,11 @@ func (t *App) initGenerator() (services.Service, error) {
 	}
 
 	t.cfg.Generator.Ring.ListenPort = t.cfg.Server.GRPCListenPort
-	genSvc, err := generator.New(&t.cfg.Generator, t.Overrides, prometheus.DefaultRegisterer, t.store, log.Logger)
+
+	t.cfg.Generator.Ingest = t.cfg.Ingest
+	t.cfg.Generator.Ingest.Kafka.ConsumerGroup = generator.ConsumerGroup
+
+	genSvc, err := generator.New(&t.cfg.Generator, t.Overrides, prometheus.DefaultRegisterer, t.partitionRing, t.store, log.Logger)
 	if errors.Is(err, generator.ErrUnconfigured) && t.cfg.Target != MetricsGenerator { // just warn if we're not running the metrics-generator
 		level.Warn(log.Logger).Log("msg", "metrics-generator is not configured.", "err", err)
 		return services.NewIdleService(nil, nil), nil
@@ -293,6 +334,81 @@ func (t *App) initGenerator() (services.Service, error) {
 	return t.generator, nil
 }
 
+func (t *App) initGeneratorNoLocalBlocks() (services.Service, error) {
+	reg := prometheus.DefaultRegisterer
+
+	t.cfg.Generator.Ingest = t.cfg.Ingest
+
+	// In this mode, the generator runs as a stateless queue consumer that reads from
+	// Kafka and remote writes to a Prometheus-compatible metrics store.
+	if !t.cfg.Ingest.Enabled {
+		return nil, errors.New("ingest storage must be enabled to run metrics generator in this mode")
+	}
+	// The localblocks processor is disabled in this mode.
+	t.cfg.Generator.DisableLocalBlocks = true
+	// The store is used only by the localblocks processor. We don't need it when
+	// running with that processor disabled so we keep the default zero value.
+	var store tempo_storage.Store
+	// In this mode, the generator does not need to become available to serve
+	// queries, so we can skip setting up a gRPC server.
+	t.cfg.Generator.DisableGRPC = true
+
+	var err error
+	t.generator, err = generator.New(&t.cfg.Generator, t.Overrides, reg, t.generatorRingWatcher, store, log.Logger)
+	if err != nil {
+		return nil, fmt.Errorf("failed to create metrics-generator: %w", err)
+	}
+
+	return t.generator, nil
+}
+
+func (t *App) initGeneratorRingWatcher() (services.Service, error) {
+	reg := prometheus.DefaultRegisterer
+
+	kvRegisterer := kv.RegistererWithKVName(reg, t.cfg.Generator.OverrideRingKey+"-watcher")
+	kvClient, err := kv.NewClient(t.cfg.Generator.Ring.KVStore, ring.GetPartitionRingCodec(), kvRegisterer, util_log.Logger)
+	if err != nil {
+		return nil, fmt.Errorf("creating KV store for generator partition ring watcher: %w", err)
+	}
+
+	t.generatorRingWatcher = ring.NewPartitionRingWatcher(
+		t.cfg.Generator.OverrideRingKey,
+		t.cfg.Generator.OverrideRingKey,
+		kvClient,
+		util_log.Logger,
+		prometheus.WrapRegistererWithPrefix("tempo_", reg),
+	)
+
+	// Expose a web page to view the partition ring state.
+	editor := ring.NewPartitionRingEditor(t.cfg.Generator.OverrideRingKey, kvClient)
+	t.Server.HTTPRouter().Path("/partition/ring").Methods("GET", "POST").
+		Handler(ring.NewPartitionRingPageHandler(t.generatorRingWatcher, editor))
+
+	return t.generatorRingWatcher, nil
+}
+
+func (t *App) initBlockBuilder() (services.Service, error) {
+	if !t.cfg.Ingest.Enabled {
+		return services.NewIdleService(nil, nil), nil
+	}
+
+	t.cfg.BlockBuilder.IngestStorageConfig = t.cfg.Ingest
+	t.cfg.BlockBuilder.IngestStorageConfig.Kafka.ConsumerGroup = blockbuilder.ConsumerGroup
+
+	if t.cfg.Target == SingleBinary && len(t.cfg.BlockBuilder.AssignedPartitions) == 0 {
+		// In SingleBinary mode always use partition 0. This is for small installs or local/debugging setups.
+		t.cfg.BlockBuilder.AssignedPartitions = map[string][]int32{t.cfg.BlockBuilder.InstanceID: {0}}
+	}
+
+	bb, err := blockbuilder.New(t.cfg.BlockBuilder, log.Logger, t.partitionRing, t.Overrides, t.store)
+	if err != nil {
+		return nil, fmt.Errorf("failed to create block-builder: %w", err)
+	}
+	t.blockBuilder = bb
+
+	return t.blockBuilder, nil
+}
+
 func (t *App) initQuerier() (services.Service, error) {
 	// validate worker config
 	// if we're not in single binary mode and worker address is not specified - bail
@@ -483,6 +599,7 @@ func (t *App) initMemberlistKV() (services.Service, error) {
 	t.cfg.MemberlistKV.MetricsNamespace = metricsNamespace
 	t.cfg.MemberlistKV.Codecs = append(t.cfg.MemberlistKV.Codecs,
 		ring.GetCodec(),
+		ring.GetPartitionRingCodec(),
 		usagestats.JSONCodec,
 	)
 
@@ -498,9 +615,11 @@ func (t *App) initMemberlistKV() (services.Service, error) {
 	t.MemberlistKV = memberlist.NewKVInitService(&t.cfg.MemberlistKV, log.Logger, dnsProvider, reg)
 
 	t.cfg.Ingester.LifecyclerConfig.RingConfig.KVStore.MemberlistKV = t.MemberlistKV.GetMemberlistKV
+	t.cfg.Ingester.IngesterPartitionRing.KVStore.MemberlistKV = t.MemberlistKV.GetMemberlistKV
 	t.cfg.Generator.Ring.KVStore.MemberlistKV = t.MemberlistKV.GetMemberlistKV
 	t.cfg.Distributor.DistributorRing.KVStore.MemberlistKV = t.MemberlistKV.GetMemberlistKV
 	t.cfg.Compactor.ShardingRing.KVStore.MemberlistKV = t.MemberlistKV.GetMemberlistKV
+	t.cfg.BackendWorker.Ring.KVStore.MemberlistKV = t.MemberlistKV.GetMemberlistKV
 
 	// Only the memberlist endpoint uses static files currently
 	t.Server.HTTPRouter().PathPrefix("/static/").HandlerFunc(http.FileServer(http.FS(staticFiles)).ServeHTTP).Methods("GET")
@@ -562,6 +681,67 @@ func (t *App) initCacheProvider() (services.Service, error) {
 	return c, nil
 }
 
+func (t *App) initBackendScheduler() (services.Service, error) {
+	if t.cfg.Target == BackendScheduler {
+		t.cfg.BackendScheduler.Poll = true
+	}
+
+	var (
+		err    error
+		reader backend.RawReader
+		writer backend.RawWriter
+	)
+
+	switch t.cfg.StorageConfig.Trace.Backend {
+	case backend.Local:
+		reader, writer, _, err = local.New(t.cfg.StorageConfig.Trace.Local)
+	case backend.GCS:
+		reader, writer, _, err = gcs.New(t.cfg.StorageConfig.Trace.GCS)
+	case backend.S3:
+		reader, writer, _, err = s3.New(t.cfg.StorageConfig.Trace.S3)
+	case backend.Azure:
+		reader, writer, _, err = azure.New(t.cfg.StorageConfig.Trace.Azure)
+	default:
+		err = fmt.Errorf("unknown backend %s", t.cfg.StorageConfig.Trace.Backend)
+	}
+
+	if err != nil {
+		return nil, fmt.Errorf("failed to initialize backendscheduler reader/writer: %w", err)
+	}
+
+	scheduler, err := backendscheduler.New(t.cfg.BackendScheduler, t.store, t.Overrides, reader, writer)
+	if err != nil {
+		return nil, fmt.Errorf("failed to create backend scheduler: %w", err)
+	}
+
+	// Register the GRPC service
+	tempopb.RegisterBackendSchedulerServer(t.Server.GRPC(), scheduler)
+
+	t.Server.HTTPRouter().Path("/status/backendscheduler").HandlerFunc(scheduler.StatusHandler)
+
+	t.backendScheduler = scheduler
+
+	return scheduler, nil
+}
+
+func (t *App) initBackendWorker() (services.Service, error) {
+	if t.cfg.Target == BackendWorker {
+		t.cfg.BackendWorker.Poll = true
+	}
+
+	worker, err := backendworker.New(t.cfg.BackendWorker, t.cfg.BackenSchedulerClient, t.store, t.Overrides, prometheus.DefaultRegisterer)
+	if err != nil {
+		return nil, fmt.Errorf("failed to create backend scheduler: %w", err)
+	}
+	t.backendWorker = worker
+
+	if t.backendWorker.Ring != nil {
+		t.Server.HTTPRouter().Handle("/backend-worker/ring", t.backendWorker.Ring)
+	}
+
+	return worker, nil
+}
+
 func (t *App) setupModuleManager() error {
 	mm := modules.NewManager(log.Logger)
 
@@ -579,7 +759,9 @@ func (t *App) setupModuleManager() error {
 	mm.RegisterModule(CacheProvider, t.initCacheProvider, modules.UserInvisibleModule)
 	mm.RegisterModule(IngesterRing, t.initIngesterRing, modules.UserInvisibleModule)
 	mm.RegisterModule(MetricsGeneratorRing, t.initGeneratorRing, modules.UserInvisibleModule)
+	mm.RegisterModule(GeneratorRingWatcher, t.initGeneratorRingWatcher, modules.UserInvisibleModule)
 	mm.RegisterModule(SecondaryIngesterRing, t.initSecondaryIngesterRing, modules.UserInvisibleModule)
+	mm.RegisterModule(PartitionRing, t.initPartitionRing, modules.UserInvisibleModule)
 
 	mm.RegisterModule(Common, nil, modules.UserInvisibleModule)
 
@@ -589,6 +771,10 @@ func (t *App) setupModuleManager() error {
 	mm.RegisterModule(QueryFrontend, t.initQueryFrontend)
 	mm.RegisterModule(Compactor, t.initCompactor)
 	mm.RegisterModule(MetricsGenerator, t.initGenerator)
+	mm.RegisterModule(MetricsGeneratorNoLocalBlocks, t.initGeneratorNoLocalBlocks)
+	mm.RegisterModule(BlockBuilder, t.initBlockBuilder)
+	mm.RegisterModule(BackendScheduler, t.initBackendScheduler)
+	mm.RegisterModule(BackendWorker, t.initBackendWorker)
 
 	mm.RegisterModule(SingleBinary, nil)
 	mm.RegisterModule(ScalableSingleBinary, nil)
@@ -605,18 +791,25 @@ func (t *App) setupModuleManager() error {
 		IngesterRing:          {Server, MemberlistKV},
 		SecondaryIngesterRing: {Server, MemberlistKV},
 		MetricsGeneratorRing:  {Server, MemberlistKV},
+		PartitionRing:         {MemberlistKV, Server, IngesterRing},
+		GeneratorRingWatcher:  {MemberlistKV},
 
 		Common: {UsageReport, Server, Overrides},
 
 		// individual targets
-		QueryFrontend:    {Common, Store, OverridesAPI},
-		Distributor:      {Common, IngesterRing, MetricsGeneratorRing},
-		Ingester:         {Common, Store, MemberlistKV},
-		MetricsGenerator: {Common, OptionalStore, MemberlistKV},
-		Querier:          {Common, Store, IngesterRing, MetricsGeneratorRing, SecondaryIngesterRing},
-		Compactor:        {Common, Store, MemberlistKV},
+		QueryFrontend:                 {Common, Store, OverridesAPI},
+		Distributor:                   {Common, IngesterRing, MetricsGeneratorRing, PartitionRing},
+		Ingester:                      {Common, Store, MemberlistKV, PartitionRing},
+		MetricsGenerator:              {Common, OptionalStore, MemberlistKV, PartitionRing},
+		MetricsGeneratorNoLocalBlocks: {Common, GeneratorRingWatcher},
+		Querier:                       {Common, Store, IngesterRing, MetricsGeneratorRing, SecondaryIngesterRing},
+		Compactor:                     {Common, Store, MemberlistKV},
+		BlockBuilder:                  {Common, Store, MemberlistKV, PartitionRing},
+		BackendScheduler:              {Common, Store},
+		BackendWorker:                 {Common, Store, MemberlistKV},
+
 		// composite targets
-		SingleBinary:         {Compactor, QueryFrontend, Querier, Ingester, Distributor, MetricsGenerator},
+		SingleBinary:         {Compactor, QueryFrontend, Querier, Ingester, Distributor, MetricsGenerator, BlockBuilder},
 		ScalableSingleBinary: {SingleBinary},
 	}
 

cmd/tempo/main.go

@@ -23,7 +23,6 @@ import (
 	"go.opentelemetry.io/otel"
 	oc_bridge "go.opentelemetry.io/otel/bridge/opencensus"
 	ot_bridge "go.opentelemetry.io/otel/bridge/opentracing"
-	"go.opentelemetry.io/otel/exporters/jaeger"
 	"go.opentelemetry.io/otel/propagation"
 	"go.opentelemetry.io/otel/sdk/resource"
 	tracesdk "go.opentelemetry.io/otel/sdk/trace"
@@ -54,17 +53,13 @@ func init() {
 
 	// Register the gogocodec as early as possible.
 	encoding.RegisterCodec(gogocodec.NewCodec())
-
-	// Register jaeger exporter
-	autoexport.RegisterSpanExporter("jaeger", func(_ context.Context) (tracesdk.SpanExporter, error) {
-		return jaeger.New(jaeger.WithAgentEndpoint())
-	})
 }
 
 func main() {
 	printVersion := flag.Bool("version", false, "Print this builds version information")
 	ballastMBs := flag.Int("mem-ballast-size-mbs", 0, "Size of memory ballast to allocate in MBs.")
-	mutexProfileFraction := flag.Int("mutex-profile-fraction", 0, "Enable mutex profiling.")
+	mutexProfileFraction := flag.Int("mutex-profile-fraction", 0, "Override default mutex profiling fraction.")
+	blockProfileThreshold := flag.Int("block-profile-threshold", 0, "Override default block profiling threshold.")
 
 	config, configVerify, err := loadConfig()
 	if err != nil {
@@ -104,9 +99,7 @@ func main() {
 		defer shutdownTracer()
 	}
 
-	if *mutexProfileFraction > 0 {
-		runtime.SetMutexProfileFraction(*mutexProfileFraction)
-	}
+	setMutexBlockProfiling(*mutexProfileFraction, *blockProfileThreshold)
 
 	// Allocate a block of memory to alter GC behaviour. See https://github.com/golang/go/issues/23044
 	ballast := make([]byte, *ballastMBs*1024*1024)
@@ -118,7 +111,11 @@ func main() {
 		os.Exit(1)
 	}
 
-	level.Info(log.Logger).Log("msg", "Starting Tempo", "version", version.Info())
+	level.Info(log.Logger).Log(
+		"msg", "Starting Tempo",
+		"version", version.Info(),
+		"target", config.Target,
+	)
 
 	if err := t.Run(); err != nil {
 		level.Error(log.Logger).Log("msg", "error running Tempo", "err", err)
@@ -285,3 +282,19 @@ type otelErrorHandlerFunc func(error)
 func (f otelErrorHandlerFunc) Handle(err error) {
 	f(err)
 }
+
+func setMutexBlockProfiling(mutexFraction int, blockThreshold int) {
+	if mutexFraction > 0 {
+		// The this is evaluated as 1/mutexFraction sampling, so 1 is 100%.
+		runtime.SetMutexProfileFraction(mutexFraction)
+	} else {
+		// Why 1000 because that is what istio defaults to and that seemed reasonable to start with.
+		runtime.SetMutexProfileFraction(1000)
+	}
+	if blockThreshold > 0 {
+		runtime.SetBlockProfileRate(blockThreshold)
+	} else {
+		// This should have a negligible impact. This will track anything over 10_000ns, and will randomly sample shorter durations.
+		runtime.SetBlockProfileRate(10_000)
+	}
+}

docs/docs.mk

@@ -31,9 +31,6 @@ ifeq ($(PROJECTS),)
 $(error "PROJECTS variable must be defined in variables.mk")
 endif
 
-# First project is considered the primary one used for doc-validator.
-PRIMARY_PROJECT := $(subst /,-,$(firstword $(subst :, ,$(firstword $(PROJECTS)))))
-
 # Host port to publish container port to.
 ifeq ($(origin DOCS_HOST_PORT), undefined)
 export DOCS_HOST_PORT := 3002
@@ -44,12 +41,7 @@ ifeq ($(origin DOCS_IMAGE), undefined)
 export DOCS_IMAGE := grafana/docs-base:latest
 endif
 
-# Container image used for doc-validator linting.
-ifeq ($(origin DOC_VALIDATOR_IMAGE), undefined)
-export DOC_VALIDATOR_IMAGE := grafana/doc-validator:latest
-endif
-
-# Container image used for vale linting.
+# Container image used for Vale linting.
 ifeq ($(origin VALE_IMAGE), undefined)
 export VALE_IMAGE := grafana/vale:latest
 endif
@@ -99,14 +91,6 @@ docs-debug: ## Run Hugo web server with debugging enabled. TODO: support all SER
 docs-debug: make-docs
 	WEBSITE_EXEC='hugo server --bind 0.0.0.0 --port 3002 --debug' $(CURDIR)/make-docs $(PROJECTS)
 
-.PHONY: doc-validator
-doc-validator: ## Run doc-validator on the entire docs folder which includes pulling the latest `DOC_VALIDATOR_IMAGE` (default: `grafana/doc-validator:latest`) container image. To not pull the image, set `PULL=false`.
-doc-validator: make-docs
-ifeq ($(PULL), true)
-	$(PODMAN) pull -q $(DOC_VALIDATOR_IMAGE)
-endif
-	DOCS_IMAGE=$(DOC_VALIDATOR_IMAGE) $(CURDIR)/make-docs $(PROJECTS)
-
 .PHONY: vale
 vale: ## Run vale on the entire docs folder which includes pulling the latest `VALE_IMAGE` (default: `grafana/vale:latest`) container image. To not pull the image, set `PULL=false`.
 vale: make-docs
@@ -120,3 +104,12 @@ update: ## Fetch the latest version of this Makefile and the `make-docs` script
 	curl -s -LO https://raw.githubusercontent.com/grafana/writers-toolkit/main/docs/docs.mk
 	curl -s -LO https://raw.githubusercontent.com/grafana/writers-toolkit/main/docs/make-docs
 	chmod +x make-docs
+
+# ls static/templates/ | sed 's/-template\.md//' | xargs
+TOPIC_TYPES := concept multiple-tasks reference section task tutorial visualization
+.PHONY: $(patsubst %,topic/%,$(TOPIC_TYPES))
+topic/%: ## Create a topic from the Writers' Toolkit template. Specify the topic type as the target, for example, `make topic/task TOPIC_PATH=sources/my-new-topic.md`.
+$(patsubst %,topic/%,$(TOPIC_TYPES)):
+	$(if $(TOPIC_PATH),,$(error "You must set the TOPIC_PATH variable to the path where the $(@F) topic will be created. For example: make $(@) TOPIC_PATH=sources/my-new-topic.md"))
+	mkdir -p $(dir $(TOPIC_PATH))
+	curl -s -o $(TOPIC_PATH) https://raw.githubusercontent.com/grafana/writers-toolkit/refs/heads/main/docs/static/templates/$(@F)-template.md

docs/make-docs

@@ -1,4 +1,6 @@
 #!/bin/sh
+# shellcheck disable=SC2034
+#
 # The source of this file is https://raw.githubusercontent.com/grafana/writers-toolkit/main/docs/make-docs.
 # # `make-docs` procedure changelog
 #
@@ -6,6 +8,38 @@
 # [Semantic versioning](https://semver.org/) is used to help the reader identify the significance of changes.
 # Changes are relevant to this script and the support docs.mk GNU Make interface.
 #
+# ## 9.0.0 (2025-04-05)
+#
+# ### Removed
+#
+# - doc-validator target and associated scripts.
+#
+#   Most useful rules have been migrated to Vale and the others are often false positives.
+#
+# ## 8.5.2 (2025-02-28)
+#
+# ### Fixed
+#
+# - topic/<KIND> targets are no longer no-ops as a result of 8.5.1.
+#
+# ## 8.5.1 (2025-02-18)
+#
+# ### Fixed
+#
+# - PHONY declaration for topic/<KIND> targets.
+#
+# ## 8.5.0 (2025-02-13)
+#
+# ### Added
+#
+# - make topic/<KIND> TOPIC_PATH=<PATH> target to create a new topic from the Writers' Toolkit templates.
+#
+# ## 8.4.0 (2025-01-27)
+#
+# ### Fixed
+#
+# - Correct mount for the /docs/grafana-cloud/send-data/fleet-management/ project.
+#
 # ## 8.3.0 (2024-12-27)
 #
 # ### Added
@@ -304,6 +338,7 @@ PODMAN="$(if command -v podman >/dev/null 2>&1; then echo podman; else echo dock
 
 if ! command -v curl >/dev/null 2>&1; then
   if ! command -v wget >/dev/null 2>&1; then
+    # shellcheck disable=SC2016
     errr 'either `curl` or `wget` must be installed for this script to work.'
 
     exit 1
@@ -311,6 +346,7 @@ if ! command -v curl >/dev/null 2>&1; then
 fi
 
 if ! command -v "${PODMAN}" >/dev/null 2>&1; then
+  # shellcheck disable=SC2016
   errr 'either `podman` or `docker` must be installed for this script to work.'
 
   exit 1
@@ -357,6 +393,10 @@ EOF
   exit 1
 fi
 
+# The following variables comprise a pseudo associative array of project names to source repositories.
+# You only need to set a SOURCES variable if the project name does not match the source repository name.
+# You can get a key identifier using the `identifier` function.
+# To look up the value of any pseudo associative array, use the `aget` function.
 SOURCES_as_code='as-code-docs'
 SOURCES_enterprise_metrics='backend-enterprise'
 SOURCES_enterprise_metrics_='backend-enterprise'
@@ -366,11 +406,16 @@ SOURCES_grafana_cloud_alerting_and_irm_slo='slo'
 SOURCES_grafana_cloud_k6='k6-docs'
 SOURCES_grafana_cloud_data_configuration_integrations='cloud-onboarding'
 SOURCES_grafana_cloud_frontend_observability_faro_web_sdk='faro-web-sdk'
+SOURCES_grafana_cloud_send_data_fleet_management='fleet-management'
 SOURCES_helm_charts_mimir_distributed='mimir'
 SOURCES_helm_charts_tempo_distributed='tempo'
 SOURCES_opentelemetry='opentelemetry-docs'
 SOURCES_resources='website'
 
+# The following variables comprise a pseudo associative array of project names to versions.
+# You only need to set a VERSIONS variable if it is not the default of 'latest'.
+# You can get a key identifier using the `identifier` function.
+# To look up the value of any pseudo associative array, use the `aget` function.
 VERSIONS_as_code='UNVERSIONED'
 VERSIONS_grafana_cloud='UNVERSIONED'
 VERSIONS_grafana_cloud_alerting_and_irm_machine_learning='UNVERSIONED'
@@ -378,12 +423,17 @@ VERSIONS_grafana_cloud_alerting_and_irm_slo='UNVERSIONED'
 VERSIONS_grafana_cloud_k6='UNVERSIONED'
 VERSIONS_grafana_cloud_data_configuration_integrations='UNVERSIONED'
 VERSIONS_grafana_cloud_frontend_observability_faro_web_sdk='UNVERSIONED'
+VERSIONS_grafana_cloud_send_data_fleet_management='UNVERSIONED'
 VERSIONS_opentelemetry='UNVERSIONED'
 VERSIONS_resources='UNVERSIONED'
 VERSIONS_technical_documentation='UNVERSIONED'
 VERSIONS_website='UNVERSIONED'
 VERSIONS_writers_toolkit='UNVERSIONED'
 
+# The following variables comprise a pseudo associative array of project names to source repository paths.
+# You only need to set a PATHS variable if it is not the default of 'docs/sources'.
+# You can get a key identifier using the `identifier` function.
+# To look up the value of any pseudo associative array, use the `aget` function.
 PATHS_grafana_cloud='content/docs/grafana-cloud'
 PATHS_helm_charts_mimir_distributed='docs/sources/helm-charts/mimir-distributed'
 PATHS_helm_charts_tempo_distributed='docs/sources/helm-charts/tempo-distributed'
@@ -789,49 +839,6 @@ ${DOCS_IMAGE}
 POSIX_HERESTRING
 
 case "${image}" in
-  'grafana/doc-validator')
-    proj="$(new_proj "$1")"
-    printf '\r\n'
-
-    IFS='' read -r cmd <<EOF
-    ${PODMAN} run \
-                --init \
-                --interactive \
-                --platform linux/amd64 \
-                --rm \
-                --tty \
-                ${volumes} \
-                ${DOCS_IMAGE} \
-                --include=${DOC_VALIDATOR_INCLUDE} \
-                --skip-checks=${DOC_VALIDATOR_SKIP_CHECKS} \
-                /hugo/content$(proj_canonical "${proj}") \
-                "$(proj_canonical "${proj}") \
-                | sed "s#$(proj_dst "${proj}")#sources#"
-EOF
-
-    if [ -n "${DEBUG}" ]; then
-      debg "${cmd}"
-    fi
-
-    case "${OUTPUT_FORMAT}" in
-      human)
-        if ! command -v jq >/dev/null 2>&1; then
-          errr '`jq` must be installed for the `doc-validator` target to work.'
-          note 'To install `jq`, refer to https://jqlang.github.io/jq/download/,'
-
-          exit 1
-        fi
-
-        ${cmd} \
-        | jq -r '"ERROR: \(.location.path):\(.location.range.start.line // 1):\(.location.range.start.column // 1): \(.message)" + if .suggestions[0].text then "\nSuggestion: \(.suggestions[0].text)" else "" end'
-      ;;
-      json)
-        ${cmd}
-      ;;
-      *)  # default
-        errr "Invalid output format '${OUTPUT_FORMAT}'"
-    esac
-    ;;
   'grafana/vale')
     proj="$(new_proj "$1")"
     printf '\r\n'

docs/sources/helm-charts/tempo-distributed/get-started-helm-charts/_index.md

@@ -468,8 +468,8 @@ For instructions, refer to [Configure TLS with Helm](https://grafana.com/docs/te
 
 The `tempo-distributed` Helm chart provides a module for users to set global or per-tenant override settings:
 
-* Global overrides come under the `global_overrides` property, which pertain to the standard overrides
-* Per-tenant overrides come under the `overrides` property, and allow specific tenants to alter configuration associated with them as per tenant-specific runtime overrides. The Helm chart generates a `/runtime/overrides.yaml` configuration file for all per-tenant configuration.
+* Global overrides come under the `overrides` property, which pertain to the standard overrides
+* Per-tenant overrides come under the `per_tenant_overrides` property, and allow specific tenants to alter configuration associated with them as per tenant-specific runtime overrides. The Helm chart generates a `/runtime/overrides.yaml` configuration file for all per-tenant configuration.
 
 These overrides correlate to the standard (global) and tenant-specific (`per_tenant_overide_config`)overrides in Tempo and GET configuration.
 For more information about overrides, refer to the [Overrides configuration](https://grafana.com/docs/tempo/<TEMPO_VERSION>/configuration/#overrides) documentation.
@@ -479,8 +479,8 @@ Overrides can be used with both GET and Tempo.
 The following example configuration sets some global configuration options, as well as a set of options for a specific tenant:
 
 ```yaml
-global_overrides:
-    default:
+overrides:
+    defaults:
         ingestion:
           rate_limit_bytes: 5 * 1000 * 1000
           burst_size_bytes: 5 * 1000 * 1000
@@ -491,7 +491,7 @@ global_overrides:
         metrics_generator:
           processors: ['service-graphs', 'span-metrics']
 
-overrides:
+per_tenant_overrides:
     '1234':
         ingestion:
           rate_limit_bytes: 2 * 1000 * 1000

docs/sources/tempo/api_docs/_index.md

@@ -7,6 +7,11 @@ weight: 800
 
 # Tempo HTTP API
 
+<!-- The API docs are mounted into GET in the website repository. -->
+
+<!-- vale Grafana.GooglePassive = NO -->
+<!-- vale Grafana.Parentheses = NO -->
+
 Tempo exposes an API for pushing and querying traces, and operating the cluster itself.
 
 For the sake of clarity, API endpoints are grouped by service.
@@ -15,10 +20,11 @@ These endpoints are exposed both when running Tempo in microservices and monolit
 - **microservices**: each service exposes its own endpoints
 - **monolithic**: the Tempo process exposes all API endpoints for the services running internally
 
-For externally supported GRPC API, [see below](#tempo-grpc-api).
+For externally supported gRPC API, [refer to Tempo gRPC API](#tempo-grpc-api).
 
 ## Endpoints
 
+<!-- vale Grafana.Spelling = NO -->
 | API | Service | Type | Endpoint |
 | --- | ------- | ---- | -------- |
 | [Readiness probe](#readiness-probe) | _All services_ |  HTTP | `GET /ready` |
@@ -26,6 +32,7 @@ For externally supported GRPC API, [see below](#tempo-grpc-api).
 | [Pprof](#pprof) | _All services_ |  HTTP | `GET /debug/pprof` |
 | [Ingest traces](#ingest) | Distributor |  - | See section for details |
 | [Querying traces by id](#query) | Query-frontend |  HTTP | `GET /api/traces/<traceID>` |
+| [Querying traces by id V2](#query-v2) | Query-frontend |  HTTP | `GET /api/v2/traces/<traceID>` |
 | [Searching traces](#search) | Query-frontend | HTTP | `GET /api/search?<params>` |
 | [Search tag names](#search-tags) | Query-frontend | HTTP | `GET /api/search/tags` |
 | [Search tag names V2](#search-tags-v2) | Query-frontend | HTTP | `GET /api/v2/search/tags` |
@@ -48,6 +55,8 @@ For externally supported GRPC API, [see below](#tempo-grpc-api).
 
 _(*) This endpoint isn't always available, check the specific section for more details._
 
+<!-- vale Grafana.Spelling = YES -->
+
 ### Readiness probe
 
 ```
@@ -63,9 +72,10 @@ GET /metrics
 ```
 
 Returns the metrics for the running Tempo service in the Prometheus exposition format.
-
+<!-- vale Grafana.Spelling = NO -->
 ### Pprof
 
+
 ```
 GET /debug/pprof/heap
 GET /debug/pprof/block
@@ -78,7 +88,9 @@ GET /debug/pprof/mutex
 Returns the runtime profiling data in the format expected by the pprof visualization tool.
 There are many things which can be profiled using this including heap, trace, goroutine, etc.
 
-_For more information, please check out the official documentation of [pprof](https://golang.org/pkg/net/http/pprof/)._
+For more information, refer to the official documentation of [pprof](https://golang.org/pkg/net/http/pprof/).
+
+<!-- vale Grafana.Spelling = YES -->
 
 ### Ingest
 
@@ -88,15 +100,22 @@ Agent, OpenTelemetry Collector, or Jaeger Agent.
 
 |  Protocol | Type | Docs |
 |  -------- | ---- | ---- |
-|  OpenTelemetry | GRPC | [Link](https://github.com/open-telemetry/opentelemetry-specification/blob/main/specification/protocol/otlp.md) |
-|  OpenTelemetry | HTTP | [Link](https://github.com/open-telemetry/opentelemetry-specification/blob/main/specification/protocol/otlp.md) |
+|  OpenTelemetry | gRPC | [Link](https://opentelemetry.io/docs/specs/otlp/) |
+|  OpenTelemetry | HTTP | [Link](https://opentelemetry.io/docs/specs/otlp/) |
 |  Jaeger | Thrift Compact | [Link](https://www.jaegertracing.io/docs/latest/apis/#span-reporting-apis) |
 |  Jaeger | Thrift Binary | [Link](https://www.jaegertracing.io/docs/latest/apis/#span-reporting-apis) |
 |  Jaeger | Thrift HTTP |  [Link](https://www.jaegertracing.io/docs/latest/apis/#span-reporting-apis) |
-|  Jaeger | GRPC | [Link](https://www.jaegertracing.io/docs/latest/apis/#span-reporting-apis) |
+|  Jaeger | gRPC | [Link](https://www.jaegertracing.io/docs/latest/apis/#span-reporting-apis) |
 |  Zipkin | HTTP | [Link](https://zipkin.io/zipkin-api/) |
 
-For information on how to use the Zipkin endpoint with curl (for debugging purposes), refer to [Pushing spans with HTTP]({{< relref "./pushing-spans-with-http" >}}).
+For information on how to use the OTLP endpoint with curl (for debugging purposes), refer to [Pushing spans with HTTP](https://grafana.com/docs/tempo/<TEMPO_VERSION>/api_docs/pushing-spans-with-http/).
+
+If you are using Grafana Enterprise Traces (GET), then it only supports OpenTelemetry (OTLP):
+
+|  Protocol | Type | Docs |
+|  -------- | ---- | ---- |
+|  OpenTelemetry | gRPC | [Link](https://opentelemetry.io/docs/specs/otlp/) |
+|  OpenTelemetry | HTTP | [Link](https://opentelemetry.io/docs/specs/otlp/) |
 
 ### Query
 
@@ -195,7 +214,7 @@ but if it can also send OpenTelemetry proto if `Accept: application/protobuf` is
 ### Search
 
 The Tempo Search API finds traces based on span and process attributes (tags and values). Note that search functionality is **not** available on
-[v2 blocks]({{< relref "../configuration/parquet#choose-a-different-block-format" >}}).
+[v2 blocks](https://grafana.com/docs/tempo/<TEMPO_VERSION>/configuration/parquet/#choose-a-different-block-format).
 
 When performing a search, Tempo does a massively parallel search over the given time range, and takes the first N results. Even identical searches differs due to things like machine load and network latency. TraceQL follows the same behavior.
 
@@ -212,11 +231,11 @@ The URL query parameters support the following values:
 
 **Parameters for TraceQL Search**
 
-- `q = (TraceQL query)`: Url encoded [TraceQL query]({{< relref "../traceql" >}}).
+- `q = (TraceQL query)`: URL encoded [TraceQL query](../traceql).
 
 **Parameters for Tag Based Search**
 
-- `tags = (logfmt)`: logfmt encoding of any span-level or process-level attributes to filter on. The value is matched as a case-insensitive substring. Key-value pairs are separated by spaces. If a value contains a space, it should be enclosed within double quotes.
+- `tags = (logfmt)`: `logfmt` encoding of any span-level or process-level attributes to filter on. The value is matched as a case-insensitive substring. Key-value pairs are separated by spaces. If a value contains a space, it should be enclosed within double quotes.
 - `minDuration = (go duration value)`
   Optional. Find traces with at least this duration. Duration values are of the form `10s` for 10 seconds, `100ms`, `30m`, etc.
 - `maxDuration = (go duration value)`
@@ -225,7 +244,7 @@ The URL query parameters support the following values:
 **Parameters supported for all searches**
 
 - `limit = (integer)`
-  Optional. Limit the number of search results. Default is 20, but this is configurable in the querier. Refer to [Configuration]({{< relref "../configuration#querier" >}}).
+  Optional. Limit the number of search results. Default is 20, but this is configurable in the querier. Refer to [Configuration](https://grafana.com/docs/tempo/<TEMPO_VERSION>/configuration/#querier).
 - `start = (unix epoch seconds)`
   Optional. Along with `end` define a time range from which traces should be returned.
 - `end = (unix epoch seconds)`
@@ -240,7 +259,7 @@ Example of how to query Tempo using curl.
 This query returns all traces that have their status set to error.
 
 ```bash
-$ curl -G -s http://localhost:3200/api/search --data-urlencode 'q={ status=error }' | jq
+curl -G -s http://localhost:3200/api/search --data-urlencode 'q={ status=error }' | jq
 {
   "traces": [
     {
@@ -282,7 +301,7 @@ Example of how to query Tempo using curl.
 This query returns all traces that have a tag `service.name` containing `cartservice` and a minimum duration of 600 ms.
 
 ```bash
-$ curl -G -s http://localhost:3200/api/search --data-urlencode 'tags=service.name=cartservice' --data-urlencode minDuration=600ms | jq
+curl -G -s http://localhost:3200/api/search --data-urlencode 'tags=service.name=cartservice' --data-urlencode minDuration=600ms | jq
 {
   "traces": [
     {
@@ -327,7 +346,7 @@ Example of how to query Tempo using curl.
 This query returns all discovered tag names.
 
 ```bash
-$ curl -G -s http://localhost:3200/api/search/tags?scope=span  | jq
+curl -G -s http://localhost:3200/api/search/tags?scope=span  | jq
 {
   "tagNames": [
     "host.name",
@@ -363,7 +382,7 @@ Parameters:
 - `limit = (integer)`
   Optional. Limits the maximum number of tags values.
 - `maxStaleValues = (integer)`
-  Optional. Limits the search for tags names. If the number of stale (already known) values reaches or exceeds this limit, the search stops. i.e. If Tempo processes `maxStaleValues` matches without finding a new tag name, the search is returned early.
+  Optional. Limits the search for tags names. If the number of stale (already known) values reaches or exceeds this limit, the search stops. If Tempo processes `maxStaleValues` matches without finding a new tag name, the search is returned early.
 
 ### Search tags V2
 
@@ -402,7 +421,7 @@ Example of how to query Tempo using curl.
 This query returns all discovered tag names.
 
 ```bash
-$ curl -G -s http://localhost:3200/api/v2/search/tags  | jq
+curl -G -s http://localhost:3200/api/v2/search/tags  | jq
 {
   "scopes": [
     {
@@ -504,7 +523,7 @@ Example of how to query Tempo using curl.
 This query returns all discovered values for the tag `service.name`.
 
 ```bash
-$ curl -G -s http://localhost:3200/api/search/tag/service.name/values  | jq
+curl -G -s http://localhost:3200/api/search/tag/service.name/values  | jq
 {
   "tagValues": [
     "article-service",
@@ -528,21 +547,21 @@ Parameters:
 - `limit = (integer)`
   Optional. Limits the maximum number of tags values.
 - `maxStaleValues = (integer)`
-  Optional. Limits the search for tags values. If the number of stale (already known) values reaches or exceeds this limit, the search stops. i.e. If Tempo processes `maxStaleValues` matches without finding a new tag name, the search is returned early.
+  Optional. Limits the search for tags values. If the number of stale (already known) values reaches or exceeds this limit, the search stops. If Tempo processes `maxStaleValues` matches without finding a new tag name, the search is returned early.
 
 
 ### Search tag values V2
 
 This endpoint retrieves all discovered values and their data types for the given TraceQL identifier.
 The endpoint is available in the query frontend service in a microservices deployment, or the Tempo endpoint in a monolithic mode deployment. This endpoint is similar to `/api/search/tag/<tag>/values` but operates on TraceQL identifiers and types.
-See [TraceQL]({{< relref "../traceql" >}}) documentation for more information.
+Refer to [TraceQL](../traceql) documentation for more information.
 
 #### Example
 
 This example queries Tempo using curl and returns all discovered values for the tag `service.name`.
 
 ```bash
-$ curl -G -s http://localhost:3200/api/v2/search/tag/.service.name/values | jq
+curl -G -s http://localhost:3200/api/v2/search/tag/.service.name/values | jq
 {
   "tagValues": [
     {
@@ -575,28 +594,29 @@ $ curl -G -s http://localhost:3200/api/v2/search/tag/.service.name/values | jq
   }
 }
 ```
+
 Parameters:
 - `start = (unix epoch seconds)`
   Optional. Along with `end`, defines a time range from which tags values should be returned.
 - `end = (unix epoch seconds)`
   Optional. Along with `start`, defines a time range from which tags values should be returned. Providing both `start` and `end` includes blocks for the specified time range only.
 - `q = (traceql query)`
-  Optional. A TraceQL query to filter tag values by. Currently only works for a single spanset of `&&`ed conditions. For example: `{ span.foo = "bar" && resource.baz = "bat" ...}`. See also [Filtered tag values](#filtered-tag-values).
+  Optional. A TraceQL query to filter tag values by. Currently only works for a single spanset of `&&`ed conditions. For example: `{ span.foo = "bar" && resource.baz = "bat" ...}`. Refer to [Filtered tag values](#filtered-tag-values).
 - `limit = (integer)`
   Optional. Limits the maximum number of tags values
 - `maxStaleValues = (integer)`
-  Optional. Limits the search for tags values. If the number of stale (already known) values reaches or exceeds this limit, the search stops. i.e. If Tempo processes `maxStaleValues` matches without finding a new tag name, the search is returned early.
+  Optional. Limits the search for tags values. If the number of stale (already known) values reaches or exceeds this limit, the search stops. If Tempo processes `maxStaleValues` matches without finding a new tag name, the search is returned early.
 
 #### Filtered tag values
 
 You can pass an optional URL query parameter, `q`, to your request.
-The `q` parameter is a URL-encoded [TraceQL query]({{< relref "../traceql" >}}).
+The `q` parameter is a URL-encoded [TraceQL query](../traceql).
 If provided, the tag values returned by the API are filtered to only return values seen on spans matching your filter parameters.
 
 Queries can be incomplete: for example, `{ resource.cluster = }`.
 Tempo extracts only the valid matchers and builds a valid query.
-If an input is invalid, Tempo doesn't provide an error. Instead, 
-you'll see the whole list when a failure of parsing input. This behavior helps with backwards compatibility. 
+If an input is invalid, Tempo doesn't provide an error. Instead,
+you'll see the whole list when a failure of parsing input. This behavior helps with backwards compatibility.
 
 Only queries with a single selector `{}` and AND `&&` operators are supported.
   - Example supported: `{ resource.cluster = "us-east-1" && resource.service = "frontend" }`
@@ -631,7 +651,7 @@ Parameters:
 - `step = (duration string)`
   Optional. Defines the granularity of the returned time-series. For example, `step=15s` returns a data point every 15s within the time range. If not specified, then the default behavior chooses a dynamic step based on the time range.
 - `exemplars = (integer)`
-  Optional. Defines the maximun number of exemplars for the query. It will be trimmed to max_exemplars if exceed it.
+  Optional. Defines the maximum number of exemplars for the query. It's trimmed to `max_exemplars` if it exceeds it.
 
 The API is available in the query frontend service in
 a microservices deployment, or the Tempo endpoint in a monolithic mode deployment.
@@ -639,7 +659,7 @@ a microservices deployment, or the Tempo endpoint in a monolithic mode deploymen
 For example, the following request computes the rate of spans received for `myservice` over the last three hours, at 1 minute intervals.
 
 {{< admonition type="note" >}}
-Actual API parameters must be url-encoded. This example is left unencoded for readability.
+Actual API parameters must be URL-encoded. This example is left unencoded for readability.
 {{< /admonition >}}
 
 ```
@@ -648,7 +668,8 @@ GET /api/metrics/query_range?q={resource.service.name="myservice"} | min_over_ti
 
 #### Instant
 
-The instant version of the metrics API is similar to the range version, but instead returns a single value for the query. This version is useful when you don't need the granularity of a full time-series, but instead want a total sum, or single value computed across the whole time range.
+The instant version of the metrics API is similar to the range version, but instead returns a single value for the query.
+This version is useful when you don't need the granularity of a full time-series, but instead want a total sum, or single value computed across the whole time range.
 
 The parameters are identical to the range version except there is no `step`.
 
@@ -661,7 +682,7 @@ Parameters:
 - `end = (unix epoch seconds | unix epoch nanoseconds | RFC3339 string)`
   Optional. Along with `start` define the time range. Providing both `start` and `end` includes blocks for the specified time range only.
 - `since = (duration string)`
-  Optional. Can be used instead of `start` and `end` to define the time range in relative values. For example `since=15m` will query the last 15 minutes.  Default is last 1 hour.
+  Optional. Can be used instead of `start` and `end` to define the time range in relative values. For example, `since=15m` queries the last 15 minutes. Default is last 1 hour.
 
 The API is available in the query frontend service in
 a microservices deployment, or the Tempo endpoint in a monolithic mode deployment.
@@ -669,8 +690,8 @@ a microservices deployment, or the Tempo endpoint in a monolithic mode deploymen
 For example the following request computes the total number of failed spans over the last hour per service.
 
 {{< admonition type="note" >}}
-Actual API parameters must be url-encoded. This example is left unencoded for readability.
-{{% /admonition %}}
+Actual API parameters must be URL-encoded. This example is left unencoded for readability.
+{{< /admonition >}}
 
 ```
 GET /api/metrics/query?q={status=error}|count_over_time()by(resource.service.name)
@@ -686,11 +707,11 @@ Returns status code 200 and body `echo` when the query frontend is up and ready
 
 {{< admonition type="note" >}}
 Meant to be used in a Query Visualization UI like Grafana to test that the Tempo data source is working.
-{{% /admonition %}}
+{{< /admonition >}}
 
 ### Overrides API
 
-For more information about user-configurable overrides API, refer to the [user-configurable overrides]({{< relref "../operations/user-configurable-overrides#api" >}}) documentation.
+For more information about user-configurable overrides API, refer to the [user-configurable overrides](https://grafana.com/docs/tempo/<TEMPO_VERSION>/operations/manage-advanced-systems/user-configurable-overrides/#api) documentation.
 
 ### Flush
 
@@ -717,19 +738,19 @@ ingester service.
 
 {{< admonition type="note" >}}
 This is usually used at the time of scaling down a cluster.
-{{% /admonition %}}
+{{< /admonition >}}
 
 ### Usage metrics
 
 {{< admonition type="note" >}}
-This endpoint is only available when one or more usage trackers are enabled in [the distributor]({{< relref "../configuration#distributor" >}}).
-{{% /admonition %}}
+This endpoint is only available when one or more usage trackers are enabled in [the distributor](https://grafana.com/docs/tempo/<TEMPO_VERSION>/configuration/#distributor).
+{{< /admonition >}}
 
 ```
 GET /usage_metrics
 ```
 
-Special metrics scrape endpoint that provides per-tenant metrics on ingested data. Per-tenant grouping rules are configured in [the per-tenant overrides]({{< relref "../configuration#overrides" >}})
+Special metrics scrape endpoint that provides per-tenant metrics on ingested data. Per-tenant grouping rules are configured in [the per-tenant overrides](https://grafana.com/docs/tempo/<TEMPO_VERSION>/configuration/#overrides)
 
 Example:
 ```
@@ -746,8 +767,8 @@ tempo_usage_tracker_bytes_received_total{service="service-A",tenant="single-tena
 ### Distributor ring status
 
 {{< admonition type="note" >}}
-This endpoint is only available when Tempo is configured with [the global override strategy]({{< relref "../configuration#overrides" >}}).
-{{% /admonition %}}
+This endpoint is only available when Tempo is configured with [the global override strategy](https://grafana.com/docs/tempo/<TEMPO_VERSION>/configuration/#overrides).
+{{< /admonition >}}
 
 ```
 GET /distributor/ring
@@ -756,7 +777,7 @@ GET /distributor/ring
 Displays a web page with the distributor hash ring status, including the state, healthy, and last heartbeat time of each
 distributor.
 
-_For more information, check the page on [consistent hash ring]({{< relref "../operations/consistent_hash_ring" >}})._
+For more information, refer to [consistent hash ring](https://grafana.com/docs/tempo/<TEMPO_VERSION>/operations/manage-advanced-systems/consistent_hash_ring/).
 
 ### Ingesters ring status
 
@@ -766,7 +787,7 @@ GET /ingester/ring
 
 Displays a web page with the ingesters hash ring status, including the state, healthy, and last heartbeat time of each ingester.
 
-_For more information, check the page on [consistent hash ring]({{< relref "../operations/consistent_hash_ring" >}}).
+For more information, refer to [consistent hash ring](http://grafana.com/docs/tempo/<TEMPO_VERSION>/operations/manage-advanced-systems/consistent_hash_ring/).
 
 ### Metrics-generator ring status
 
@@ -776,9 +797,9 @@ GET /metrics-generator/ring
 
 Displays a web page with the metrics-generator hash ring status, including the state, health, and last heartbeat time of each metrics-generator.
 
-This endpoint is only available when the metrics-generator is enabled. Refer to [metrics-generator]({{< relref "../configuration#metrics-generator" >}}).
+This endpoint is only available when the metrics-generator is enabled. Refer to [metrics-generator](https://grafana.com/docs/tempo/<TEMPO_VERSION>/configuration/#metrics-generator).
 
-For more information, refer to [consistent hash ring]({{< relref "../operations/consistent_hash_ring" >}}).
+For more information, refer to [consistent hash ring](http://grafana.com/docs/tempo/<TEMPO_VERSION>/operations/manage-advanced-systems/consistent_hash_ring/).
 
 ### Compactor ring status
 
@@ -786,9 +807,9 @@ For more information, refer to [consistent hash ring]({{< relref "../operations/
 GET /compactor/ring
 ```
 
-Displays a web page with the compactor hash ring status, including the state, healthy and last heartbeat time of each compactor.
+Displays a web page with the compactor hash ring status, including the state, healthy, and last heartbeat time of each compactor.
 
-For more information, refer to [consistent hash ring]({{< relref "../operations/consistent_hash_ring" >}}).
+For more information, refer to [consistent hash ring](http://grafana.com/docs/tempo/<TEMPO_VERSION>/operations/manage-advanced-systems/consistent_hash_ring/).
 
 ### Status
 
@@ -822,7 +843,7 @@ GET /status/config
 Displays the configuration.
 
 Displays the configuration currently applied to Tempo (in YAML format), including default values and settings via CLI flags.
-Sensitive data is masked. Please be aware that the exported configuration **doesn't include the per-tenant overrides**.
+Sensitive data is masked. Be aware that the exported configuration **doesn't include the per-tenant overrides**.
 
 Optional query parameter:
 
@@ -863,18 +884,18 @@ GET /api/status/buildinfo
 ```
 Exposes the build information in a JSON object. The fields are `version`, `revision`, `branch`, `buildDate`, `buildUser`, and `goVersion`.
 
-## Tempo GRPC API
+## Tempo gRPC API
 
-Tempo uses GRPC to internally communicate with itself, but only has one externally supported client.
+Tempo uses [gRPC](https://grpc.io) to internally communicate with itself, but only has one externally supported client.
 The query-frontend component implements the streaming querier interface defined below.
-[See here](https://github.com/grafana/tempo/blob/main/pkg/tempopb/) for the complete proto definition and generated code.
+[Refer here](https://github.com/grafana/tempo/blob/main/pkg/tempopb/) for the complete proto definition and generated code.
 
-By default, this service is only offered over the GRPC port.
+By default, this service is only offered over the gRPC port.
 You can use streaming service over the HTTP port as well, which Grafana expects.
 
 To enable the streaming service over the HTTP port for use with Grafana, set the following:
 
-```
+```yaml
 stream_over_http_enabled: true
 ```
 
@@ -890,3 +911,9 @@ service StreamingQuerier {
   rpc MetricsQueryRange(QueryRangeRequest) returns (stream QueryRangeResponse) {}
 }
 ```
+
+{{< admonition type="note" >}}
+gRPC compression is disabled by default.
+Refer to [gRPC compression configuration](https://grafana.com/docs/tempo/<TEMPO_VERSION>/configuration/#grpc-compression) for more information.
+{{< /admonition >}}
+<!-- vale Grafana.GooglePassive = YES -->

docs/sources/tempo/api_docs/metrics-summary.md

@@ -12,7 +12,7 @@ weight: 600
 {{< admonition type="warning" >}}
 The metrics summary API is deprecated as of Tempo 2.7. Features powered by the metrics summary API, like the [Aggregate by table](https://grafana.com/docs/grafana/<GRAFANA_VERSION>/datasources/tempo/query-editor/traceql-search/#optional-use-aggregate-by), are also deprecated in Grafana Cloud and Grafana 11.3 and later.
 It will be removed in a future release.
-{{% /admonition %}}
+{{< /admonition >}}
 
 This document explains how to use the metrics summary API in Tempo.
 This API returns RED metrics (span count, erroring span count, and latency information) for `kind=server` spans sent to Tempo in the last hour, grouped by a user-specified attribute.
@@ -44,11 +44,11 @@ The 50th, 90th, and 99th percentile latencies (for example, p99, p90, and p50) b
 { } | quantile_over_time(duration, .99, .9, .5) by (resource.cloud.region)
 ```
 
-If you want something faster than typing these queries out in Explore's code mode, use [Explore Traces](https://grafana.com/docs/grafana/<GRAFANA_VERSION>/explore/simplified-exploration/traces/), a queryless experience for navigating your trace data stored in Tempo and powered by TraceQL metrics queries under the hood.
+If you want something faster than typing these queries out in Explore's code mode, use [Grafana Traces Drilldown](https://grafana.com/docs/grafana/<GRAFANA_VERSION>/explore/simplified-exploration/traces/), a queryless experience for navigating your trace data stored in Tempo and powered by TraceQL metrics queries under the hood.
 
 ## Activate metrics summary
 
-To enable the (deprecated) metrics summary API, you must turn on the local blocks processor in the metrics generator.
+To enable the deprecated) metrics summary API, you must turn on the local blocks processor in the metrics generator.
 Be aware that the generator uses considerably more resources, including disk space, if it's enabled:
 
 ```yaml
@@ -121,8 +121,8 @@ message TraceQLStatic {
 The response is returned as JSON following [standard protobuf->JSON mapping rules](https://protobuf.dev/programming-guides/proto3/#json).
 
 {{< admonition type="note" >}}
-The `uint64` fields cannot be fully expressed by JSON numeric values so the fields are serialized as strings.
-{{% /admonition %}}
+The `uint64` fields can't be fully expressed by JSON numeric values so the fields are serialized as strings.
+{{< /admonition >}}
 
 Example:
 
@@ -152,8 +152,8 @@ Example:
 | ----------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
 | `summaries`       | The list of metrics per group.                                                                                                                                                                                                                                                                     |
 | `.spanCount`      | Number of spans in this group.                                                                                                                                                                                                                                                                     |
-| `.errorSpanCount` | Number of spans with `status`=`error`. (This field will not be present if the value is `0`.)                                                                                                                                                                                                        |
-| `.series`         | The unique values for this group. A key/value pair will be returned for each entry in `groupBy`.                                                                                                                                                                                                   |
+| `.errorSpanCount` | Number of spans with `status`=`error`. (This field isn't present if the value is `0`.)                                                                                                                                                                                                        |
+| `.series`         | The unique values for this group. A key/value pair is returned for each entry in `groupBy`.                                                                                                                                                                                                   |
 | `.key`            | Key name.                                                                                                                                                                                                                                                                                          |
 | `.value`          | Value with TraceQL underlying type.                                                                                                                                                                                                                                                                |
 | `.type`           | Data type `enum`` defined [here](https://github.com/grafana/tempo/blob/main/pkg/traceql/enum_statics.go#L8) (This field will not be present if the value is `0`.) <br/>0 = `nil`<br/>3 = `integer`<br/> 4 = `float` <br/> 5 = `string`<br/> 6 = `bool`<br/> 7 = `duration`<br/> 8 = span status<br/> 9 = span kind |

docs/sources/tempo/api_docs/pushing-spans-with-http.md

@@ -13,14 +13,13 @@ pushing spans with HTTP/JSON from a Bash script using the [OpenTelemetry](https:
 
 ## Before you begin
 
-This procedure uses an example Docker Compose setup to run Tempo, so you do not need an existing installation. The Docker image also includes a Grafana container which will allow us to visualize traces.
+This procedure uses an example Docker Compose setup to run Tempo, so you don't need an existing installation. The Docker image also includes a Grafana container which lets you visualize traces.
 
 To use this procedure, you need to have Docker and `docker compose` installed.
 
-
 ## Start Tempo using the quick start
 
-Use the instructions in the [Quick start for Tempo documentation]({{< relref "../getting-started/docker-example" >}}) to start a local instance of Tempo and Grafana.
+Use the instructions in the [Quick start for Tempo documentation](https://grafana.com/docs/tempo/<TEMPO_VERSION>/getting-started/docker-example/) to start a local instance of Tempo and Grafana.
 
 ## Push spans with OTLP
 
@@ -98,7 +97,7 @@ Multiple the milliseconds value by 1,000,000 to turn it into nanoseconds. You ca
 
 The easiest way to get the trace is to execute a simple curl command to Tempo. The returned format is [OTLP](https://github.com/open-telemetry/opentelemetry-proto/blob/main/opentelemetry/proto/trace/v1/trace.proto).
 
-1. Replace the trace ID in the `curl` command with the trace ID that was generated from the push. This information is is in the data that's sent with the `curl`. You could use Grafana’s Explorer page to find this, as shown in the previous section.
+1. Replace the trace ID in the `curl` command with the trace ID that was generated from the push. This information is in the data that's sent with the `curl`. You could use Grafana’s Explorer page to find this, as shown in the previous section.
 
 	```bash
 	curl http://localhost:3200/api/v2/traces/5b8efff798038103d269b633813fc700
@@ -110,7 +109,7 @@ The easiest way to get the trace is to execute a simple curl command to Tempo. T
 
 ### Use TraceQL to search for a trace
 
-Alternatively, you can also use [TraceQL]({{< relref "../traceql" >}}) to search for the trace that was pushed.
+Alternatively, you can also use [TraceQL](https://grafana.com/docs/tempo/<TEMPO_VERSION>/traceql/) to search for the trace that was pushed.
 You can search by using the unique trace attributes that were set:
 
 ```bash
@@ -127,8 +126,9 @@ curl -G -s http://localhost:3200/api/search --data-urlencode 'q={ .service.name
 
 ## Spans from everything
 
-Tracing is not limited to enterprise languages with complex frameworks. As you can see it's easy to store and track events from your js, python or bash scripts.
-You can use Tempo/distributed tracing today to trace CI pipelines, long running bash processes, python data processing flows, or anything else
+Tracing isn't limited to enterprise languages with complex frameworks.
+As you can see, it's easy to store and track events from your js, python or bash scripts.
+You can use Tempo and distributed tracing today to trace CI pipelines, long running bash processes, python data processing flows, or anything else
 you can think of.
 
 Happy tracing!

docs/sources/tempo/configuration/_index.md

@@ -11,7 +11,7 @@ This document explains the configuration options for Tempo as well as the detail
 
 {{< admonition type="tip" >}}
 Instructions for configuring Tempo data sources are available in the [Grafana Cloud](/docs/grafana-cloud/send-data/traces/) and [Grafana](/docs/grafana/latest/datasources/tempo/) documentation.
-{{% /admonition %}}
+{{< /admonition >}}
 
 The Tempo configuration options include:
 
@@ -19,13 +19,16 @@ The Tempo configuration options include:
   - [Use environment variables in the configuration](#use-environment-variables-in-the-configuration)
   - [Server](#server)
   - [Distributor](#distributor)
+    - [Set max attribute size to help control out of memory errors](#set-max-attribute-size-to-help-control-out-of-memory-errors)
+    - [gRPC compression](#grpc-compression)
   - [Ingester](#ingester)
+    - [Ingester configuration block](#ingester-configuration-block)
   - [Metrics-generator](#metrics-generator)
   - [Query-frontend](#query-frontend)
     - [Limit query size to improve performance and stability](#limit-query-size-to-improve-performance-and-stability)
       - [Limit the spans per spanset](#limit-the-spans-per-spanset)
-    - [Cap the maximum query length](#cap-the-maximum-query-length)
-    - [Querier](#querier)
+      - [Cap the maximum query length](#cap-the-maximum-query-length)
+  - [Querier](#querier)
   - [Compactor](#compactor)
   - [Storage](#storage)
     - [Local storage recommendations](#local-storage-recommendations)
@@ -50,7 +53,7 @@ The Tempo configuration options include:
   - [Usage-report](#usage-report)
   - [Cache](#cache)
 
-Additionally, you can review [TLS]({{< relref "./network/tls" >}}) to configure the cluster components to communicate over TLS, or receive traces over TLS.
+Additionally, you can review [TLS](network/tls/) to configure the cluster components to communicate over TLS, or receive traces over TLS.
 
 ## Use environment variables in the configuration
 
@@ -92,7 +95,7 @@ server:
     [http_listen_address: <string>]
 
     # HTTP server listen port
-    [http_listen_port: <int> | default = 80]
+    [http_listen_port: <int> | default = 3200]
 
     # gRPC server listen host
     [grpc_listen_address: <string>]
@@ -235,7 +238,7 @@ distributor:
     # Optional
     # Configures the max size an attribute can be. Any key or value that exceeds this limit will be truncated before storing
     # Setting this parameter to '0' would disable this check against attribute size
-    [max_span_attr_byte: <int> | default = '2048']
+    [max_attribute_bytes: <int> | default = '2048']
 
     # Optional.
     # Configures usage trackers in the distributor which expose metrics of ingested traffic grouped by configurable
@@ -251,6 +254,56 @@ distributor:
             [stale_duration: <duration> | default = 15m0s]
 ```
 
+### Set max attribute size to help control out of memory errors
+
+Tempo queriers can run out of memory when fetching traces that have spans with very large attributes.
+This issue has been observed when trying to fetch a single trace using the [`tracebyID` endpoint](https://grafana.com/docs/tempo/<TEMPO_VERSION>/api_docs/#query).
+While a trace might not have a lot of spans (roughly 500), it can have a larger size (approximately 250KB).
+Some of the spans in that trace had attributes whose values were very large in size.
+
+To avoid these out-of-memory crashes, use `max_attribute_bytes` to limit the maximum allowable size of any individual attribute.
+Any key or values that exceed the configured limit are truncated before storing.
+The default value is `2048`.
+
+Use the `tempo_distributor_attributes_truncated_total` metric to track how many attributes are truncated.
+
+For additional information, refer to [Troubleshoot out-of-memory errors](https://grafana.com/docs/tempo/<TEMPO_VERSION>/troubleshooting/out-of-memory-errors/).
+
+### gRPC compression
+
+Starting with Tempo 2.7.1, gRPC compression between all components defaults to `snappy`.
+Using `snappy` provides a balanced approach to compression between components that will work for most installations.
+
+If you prefer a different balance of CPU/Memory and bandwidth, consider disabling compression or using `zstd`.
+
+For a discussion on alternatives, refer to [this discussion thread](https://github.com/grafana/tempo/discussions/4683). ([#4696](https://github.com/grafana/tempo/pull/4696)).
+
+
+Disabling comrpession may provide some performance boosts.
+Benchmark testing suggested that without compression, queriers and distributors used less CPU and memory.
+
+However, you may notice an increase in ingester data and network traffic especially for larger clusters.
+This increased data can impact billing for Grafana Cloud.
+
+You can configure the gRPC compression in the `querier`, `ingester`, and `metrics_generator` clients of the distributor.
+
+To disable compression, remove `snappy` from the `grpc_compression` lines.
+
+To re-enable the compression, use `snappy` with the following settings:
+
+  ```yaml
+  ingester_client:
+      grpc_client_config:
+          grpc_compression: "snappy"
+  metrics_generator_client:
+      grpc_client_config:
+          grpc_compression: "snappy"
+  querier:
+      frontend_worker:
+          grpc_client_config:
+              grpc_compression: "snappy"
+```
+
 ## Ingester
 
 For more information on configuration options, refer to [this file](https://github.com/grafana/tempo/blob/main/modules/ingester/config.go).
@@ -261,6 +314,10 @@ A live, or active, trace is a trace that has received a new batch of spans in mo
 After 10 seconds (or the configured amount of time), the trace is flushed to disk and appended to the WAL.
 When Tempo receives a new batch, a new live trace is created in memory.
 
+Refer to [Data quality metrics](../troubleshooting/querying/long-running-traces) for information about `trace_idle_period` and warning metrics like `disconnected_trace_flushed_to_wal`.
+
+### Ingester configuration block
+
 ```yaml
 # Ingester configuration block
 ingester:
@@ -304,7 +361,7 @@ ingester:
 For more information on configuration options, refer to [this file](https://github.com/grafana/tempo/blob/main/modules/generator/config.go).
 
 The metrics-generator processes spans and write metrics using the Prometheus remote write protocol.
-For more information on the metrics-generator, refer to the [Metrics-generator documentation]({{< relref "../metrics-generator" >}}).
+For more information on the metrics-generator, refer to the [Metrics-generator documentation](../metrics-generator/).
 
 Metrics-generator processors are disabled by default. To enable it for a specific tenant, set `metrics_generator.processors` in the [overrides](#overrides) section.
 
@@ -315,7 +372,7 @@ If you want to enable metrics-generator for your Grafana Cloud account, refer to
 You can limit spans with end times that occur within a configured duration to be considered in metrics generation using `metrics_ingestion_time_range_slack`.
 In Grafana Cloud, this value defaults to 30 seconds so all spans sent to the metrics-generation more than 30 seconds in the past are discarded or rejected.
 
-For more information about the `local-blocks` configuration option, refer to [TraceQL metrics](https://grafana.com/docs/tempo/latest/operations/traceql-metrics/#configure-the-local-blocks-processor).
+For more information about the `local-blocks` configuration option, refer to [TraceQL metrics](https://grafana.com/docs/tempo/<TEMPO_VERSION>/operations/traceql-metrics/#activate-and-configure-the-local-blocks-processor).
 
 ```yaml
 # Metrics-generator configuration block
@@ -478,7 +535,11 @@ metrics_generator:
             # If enabled, only parent spans or spans with the SpanKind of `server` will be retained
             [filter_server_spans: <bool> | default = true]
 
-            # Number of blocks that are allowed to be processed concurently
+            # Whether server spans should be flushed to storage.
+            # Setting `flush_to_storage` to `true` ensures that metrics blocks are flushed to storage so TraceQL metrics queries against historical data.
+            [flush_to_storage: <bool> | default = false]
+
+            # Number of blocks that are allowed to be processed concurrently.
             [concurrent_blocks: <uint> | default = 10]
 
             # A tuning factor that controls whether the trace-level timestamp columns are used in a metrics query.
@@ -574,7 +635,7 @@ query_frontend:
 
     # The number of jobs to batch together in one http request to the querier. Set to 1 to
     # disable.
-    # (default: 5)
+    # (default: 7)
     [max_batch_size: <int>]
 
     # Enable multi-tenant queries.
@@ -650,11 +711,15 @@ query_frontend:
         # Query is within SLO if it returned 200 within duration_slo seconds OR processed throughput_slo bytes/s data.
         [throughput_bytes_slo: <float> | default = 0 ]
 
+        # The number of time windows to break a search up into when doing a most recent TraceQL search. This only impacts TraceQL
+        # searches with (most_recent=true).
+        [most_recent_shards: <int> | default = 200]
+
         # The number of shards to break ingester queries into.
-        [ingester_shards]: <int> | default = 3]
+        [ingester_shards: <int> | default = 3]
 
         # The maximum allowed value of spans per span set. 0 disables this limit.
-        [max_spans_per_span_set]: <int> | default = 100]
+        [max_spans_per_span_set: <int> | default = 100]
 
         # SLO configuration for Metadata (tags and tag values) endpoints.
         metadata_slo:
@@ -663,6 +728,7 @@ query_frontend:
             # NOTE: Requires `duration_slo` AND `throughput_bytes_slo` to be configured.
             [duration_slo: <duration> | default = 0s ]
 
+
             # If set to a non-zero value, it's value will be used to decide if metadata query is within SLO or not.
             # Query is within SLO if it returned 200 within duration_slo seconds OR processed throughput_slo bytes/s data.
             [throughput_bytes_slo: <float> | default = 0 ]
@@ -677,10 +743,15 @@ query_frontend:
         # (default: 0)
         [concurrent_shards: <int>]
 
-        # If set to a non-zero value, it's value will be used to decide if query is within SLO or not.
-        # Query is within SLO if it returned 200 within duration_slo seconds.
+        # If set to a non-zero value, it's value will be used to decide if metadata query is within SLO or not.
+        # Query is within SLO if it returned 200 within duration_slo seconds OR processed throughput_slo bytes/s data.
+        # NOTE: Requires `duration_slo` AND `throughput_bytes_slo` to be configured.
         [duration_slo: <duration> | default = 0s ]
 
+        # If set to a non-zero value, it's value will be used to decide if metadata query is within SLO or not.
+        # Query is within SLO if it returned 200 within duration_slo seconds OR processed throughput_slo bytes/s data.
+        [throughput_bytes_slo: <float> | default = 0 ]
+
     # Metrics query configuration
     metrics:
         # The number of concurrent jobs to execute when querying the backend.
@@ -696,6 +767,10 @@ query_frontend:
         # Maximun number of exemplars per range query. Limited to 100.
         [max_exemplars: <int> | default = 100 ]
 
+        # Maximum number of time series returned for a metrics query.
+        # Default is 0, which means there is no limit
+        [max_response_series: <int> | default = 0]
+
         # query_backend_after controls where the query-frontend searches for traces.
         # Time ranges older than query_backend_after will be searched in the backend/object storage only.
         # Time ranges between query_backend_after and now will be queried from the metrics-generators.
@@ -724,14 +799,14 @@ In a similar manner, excessive queries result size can also negatively impact qu
 #### Limit the spans per spanset
 
 You can set the maximum spans per spanset by setting `max_spans_per_span_set` for the query-frontend.
-The default value is 100. 
+The default value is 100.
 
 In Grafana or Grafana Cloud, you can use the **Span Limit** field in the [TraceQL query editor](https://grafana.com/docs/grafana-cloud/connect-externally-hosted/data-sources/tempo/query-editor/) in Grafana Explore.
 This field sets the maximum number of spans to return for each span set.
 The maximum value that you can set for the **Span Limit** value (or the spss query) is controlled by `max_spans_per_span_set`.
 To disable the maximum spans per span set limit, set `max_spans_per_span_set` to `0`.
-When set to `0`, there is no maximum and users can put any value in **Span Limit**. 
-However, this can only be set by a Tempo administrator, not by the user. 
+When set to `0`, there is no maximum and users can put any value in **Span Limit**.
+However, this can only be set by a Tempo administrator, not by the user.
 
 #### Cap the maximum query length
 
@@ -785,42 +860,6 @@ querier:
         # Timeout for search requests
         [query_timeout: <duration> | default = 30s]
 
-        # NOTE: The Tempo serverless feature is now deprecated and will be removed in an upcoming release.
-        # A list of external endpoints that the querier will use to offload backend search requests. They must
-        # take and return the same value as /api/search endpoint on the querier. This is intended to be
-        # used with serverless technologies for massive parallelization of the search path.
-        # The default value of "" disables this feature.
-        [external_endpoints: <list of strings> | default = <empty list>]
-
-        # If search_external_endpoints is set then the querier will primarily act as a proxy for whatever serverless backend
-        # you have configured. This setting allows the operator to have the querier prefer itself for a configurable
-        # number of subqueries. In the default case of 2 the querier will process up to 2 search requests subqueries before starting
-        # to reach out to search_external_endpoints.
-        # Setting this to 0 will disable this feature and the querier will proxy all search subqueries to search_external_endpoints.
-        [prefer_self: <int> | default = 10 ]
-
-        # If set to a non-zero value a second request will be issued at the provided duration. Recommended to
-        # be set to p99 of external search requests to reduce long tail latency.
-        # (default: 8s)
-        [external_hedge_requests_at: <duration>]
-
-        # The maximum number of requests to execute when hedging. Requires hedge_requests_at to be set.
-        # (default: 2)
-        [external_hedge_requests_up_to: <int>]
-
-        # The serverless backend to use. If external_backend is set, then authorization credentials will be provided
-        # when querying the external endpoints. "google_cloud_run" is the only value supported at this time.
-        # The default value of "" omits credentials when querying the external backend.
-        [external_backend: <string> | default = ""]
-
-        # Google Cloud Run configuration. Will be used only if the value of external_backend is "google_cloud_run".
-        google_cloud_run:
-            # A list of external endpoints that the querier will use to offload backend search requests. They must
-            # take and return the same value as /api/search endpoint on the querier. This is intended to be
-            # used with serverless technologies for massive parallelization of the search path.
-            # The default value of "" disables this feature.
-            [external_endpoints: <list of strings> | default = <empty list>]
-
     # config of the worker that connects to the query frontend
     frontend_worker:
 
@@ -914,10 +953,10 @@ The storage block configures TempoDB.
 The following example shows common options.
 For further platform-specific information, refer to the following:
 
-* [GCS]({{< relref "./hosted-storage/gcs" >}})
-* [S3]({{< relref "./hosted-storage/s3" >}})
-* [Azure]({{< relref "./hosted-storage/azure" >}})
-* [Parquet]({{< relref "./parquet" >}})
+* [GCS](hosted-storage/gcs/)
+* [S3](hosted-storage/s3/)
+* [Azure](hosted-storage/azure/)
+* [Parquet](parquet/)
 
 ```yaml
 # Storage configuration for traces
@@ -1095,6 +1134,22 @@ storage:
             # See the [S3 documentation on object tagging](https://docs.aws.amazon.com/AmazonS3/latest/userguide/object-tagging.html) for more detail.
             [tags: <map[string]string>]
 
+
+            [sse: <map[string]string>]:
+              # Optional
+              # Example: type: SSE-S3
+              # Type of encryption to use with s3 bucket, either SSE-KMS or SSE-S3
+              [type: string]:
+
+              # Optional
+              # Example: kms_key_id: "1234abcd-12ab-34cd-56ef-1234567890ab"
+              # the kms key id is the identification of the key in an account or region
+              kms_key_id:
+              # Optional
+              # Example: kms_encryption_context: "encryptionContext": {"department": "10103.0"}
+              # KMS Encryption Context used for object encryption. It expects JSON formatted string
+              kms_encryption_context:
+
         # azure configuration. Will be used only if value of backend is "azure"
         # EXPERIMENTAL
         azure:
@@ -1610,6 +1665,13 @@ overrides:
       # Should not be lower than RF.
       [tenant_shard_size: <int> | default = 0]
 
+      # Maximum bytes any attribute can be for both keys and values.
+      [max_attribute_bytes: <int> | default = 0]
+
+      # Pad push requests with an artificial delay, if set push requests will be delayed to ensure
+      # an average latency of at least artificial_delay.
+      [artificial_delay: <duration> | default = 0ms]
+
     # Read related overrides
     read:
       # Maximum size in bytes of a tag-values query. Tag-values query is used mainly
@@ -1642,7 +1704,7 @@ overrides:
       # Per-user compaction window. If this value is set to 0 (default),
       # then block_retention in the compactor configuration is used.
       [compaction_window: <duration> | default = 0s]
-      # Allow compaction to be deactivated on a per-tenant basis. Default value
+      # Allow compaction and retention to be deactivated on a per-tenant basis. Default value
       # is false (compaction active). Useful to perform operations on the backend
       # that require compaction to be disabled for a period of time.
       [compaction_disabled: <bool> | default = false]
@@ -1874,7 +1936,7 @@ overrides:
 
 These tenant-specific overrides are stored in an object store and can be modified using API requests.
 User-configurable overrides have priority over runtime overrides.
-Refer to [user-configurable overrides]{{< relref "../operations/user-configurable-overrides" >}} for more details.
+Refer to [user-configurable overrides](https://grafana.com/docs/tempo/<TEMPO_VERSION>/operations/manage-advanced-systems/user-configurable-overrides/) for more details.
 
 #### Override strategies
 
@@ -1999,7 +2061,7 @@ cache:
             # Optional
             # Comma separated addresses list in DNS Service Discovery format. Refer - https://cortexmetrics.io/docs/configuration/arguments/#dns-service-discovery.
             # (default: "")
-            # Example: "addresses: memcached"
+            # Example: "addresses: dns+memcached:11211"
             [addresses: <comma separated strings>]
 
             # Optional

docs/sources/tempo/configuration/grafana-agent/_index.md

@@ -3,7 +3,8 @@ title: Grafana Agent
 description: Configure the Grafana Agent to work with Tempo
 weight: 600
 aliases:
-- /docs/tempo/grafana-agent
+  - /docs/tempo/grafana-agent
+  - ../../grafana-agent # /docs/tempo/latest/grafana-agent
 ---
 
 # Grafana Agent
@@ -35,14 +36,14 @@ leverages all the data that's processed in the pipeline.
 
 Grafana Agent is available in two different variants:
 
-* [Static mode](/docs/agent/latest/static): The original Grafana Agent.
-* [Flow mode](/docs/agent/latest/flow): The new, component-based Grafana Agent.
+* [Static mode](/docs/agent/<AGENT_VERSION>/static): The original Grafana Agent.
+* [Flow mode](/docs/agent/<AGENT_VERSION>/flow): The new, component-based Grafana Agent.
 
-Grafana Agent Flow configuration files are [written in River](/docs/agent/latest/flow/concepts/config-language/).
-Static configuration files are [written in YAML](/docs/agent/latest/static/configuration/).
+Grafana Agent Flow configuration files are [written in River](/docs/agent/<AGENT_VERSION>/flow/concepts/config-language/).
+Static configuration files are [written in YAML](/docs/agent/<AGENT_VERSION>/static/configuration/).
 Examples in this document are for Flow mode.
 
-For more information, refer to the [Introduction to Grafana Agent](/docs/agent/latest/about/).
+For more information, refer to the [Introduction to Grafana Agent](/docs/agent/<AGENT_VERSION>/about/).
 
 ## Architecture
 
@@ -50,7 +51,7 @@ The Grafana Agent can be configured to run a set of tracing pipelines to collect
 Pipelines are built using OpenTelemetry,
 and consist of `receivers`, `processors`, and `exporters`.
 The architecture mirrors that of the OTel Collector's [design](https://github.com/open-telemetry/opentelemetry-collector/blob/846b971758c92b833a9efaf742ec5b3e2fbd0c89/docs/design.md).
-See the [configuration reference](/agent/latest/static/configuration/traces-config/) for all available configuration options.
+See the [configuration reference](/agent/<AGENT_VERSION>/static/configuration/traces-config/) for all available configuration options.
 
 <p align="center"><img src="https://raw.githubusercontent.com/open-telemetry/opentelemetry-collector/846b971758c92b833a9efaf742ec5b3e2fbd0c89/docs/images/design-pipelines.png" alt="Tracing pipeline architecture"></p>
 
@@ -75,13 +76,13 @@ The Grafana Agent processes tracing data as it flows through the pipeline to mak
 
 The Agent supports batching of traces.
 Batching helps better compress the data, reduces the number of outgoing connections, and is a recommended best practice.
-To configure it, refer to the `batch` block in the [configuration reference](/docs/agent/latest/configuration/traces-config).
+To configure it, refer to the `batch` block in the [configuration reference](/docs/agent/<AGENT_VERSION>/configuration/traces-config).
 
 #### Attributes manipulation
 
 The Grafana Agent allows for general manipulation of attributes on spans that pass through this agent.
 A common use may be to add an environment or cluster variable.
-To configure it, refer to the `attributes` block in the [configuration reference](/docs/agent/latest/configuration/traces-config).
+To configure it, refer to the `attributes` block in the [configuration reference](/docs/agent/<AGENT_VERSION>/configuration/traces-config).
 
 #### Attaching metadata with Prometheus Service Discovery
 
@@ -113,20 +114,20 @@ All of Prometheus' [various service discovery mechanisms](https://prometheus.io/
 This means you can use the same `scrape_configs` between your metrics, logs, and traces to get the same set of labels,
 and easily transition between your observability data when moving from your metrics, logs, and traces.
 
-Refer to the `scrape_configs` block in the [configuration reference](/docs/agent/latest/configuration/traces-config).
+Refer to the `scrape_configs` block in the [configuration reference](/docs/agent/<AGENT_VERSION>/configuration/traces-config).
 
 #### Trace discovery through automatic logging
 
 Automatic logging writes well formatted log lines to help with trace discovery.
 
-For a closer look into the feature, visit [Automatic logging]({{< relref "./automatic-logging" >}}).
+For a closer look into the feature, visit [Automatic logging](automatic-logging/).
 
 #### Tail-based sampling
 
 The Agent implements tail-based sampling for distributed tracing systems and multi-instance Agent deployments.
 With this feature, sampling decisions can be made based on data from a trace, rather than exclusively with probabilistic methods.
 
-For a detailed description, go to [Tail-based sampling]({{< relref "./tail-based-sampling" >}}).
+For a detailed description, go to [Tail-based sampling](tail-based-sampling/).
 
 For additional information, refer to the blog post, [An introduction to trace sampling with Grafana Tempo and Grafana Agent](/blog/2022/05/11/an-introduction-to-trace-sampling-with-grafana-tempo-and-grafana-agent).
 
@@ -134,7 +135,7 @@ For additional information, refer to the blog post, [An introduction to trace sa
 
 The Agent can take advantage of the span data flowing through the pipeline to generate Prometheus metrics.
 
-Go to [Span metrics]({{< relref "./span-metrics" >}}) for a more detailed explanation of the feature.
+Go to [Span metrics](span-metrics/) for a more detailed explanation of the feature.
 
 #### Service graph metrics
 
@@ -144,7 +145,7 @@ This service graphs processor builds a map of services by analyzing traces, with
 Edges are spans with a parent-child relationship, that represent a jump, such as a request, between two services.
 The amount of requests and their duration are recorded as metrics, which are used to represent the graph.
 
-To read more about this processor, go to its [section]({{< relref "./service-graphs" >}}).
+To read more about this processor, go to its [section](service-graphs/).
 
 ### Exporting spans
 
@@ -156,4 +157,4 @@ Aside from endpoint and authentication, the exporter also provides mechanisms fo
 and implements a queue buffering mechanism for transient failures, such as networking issues.
 
 To see all available options,
-refer to the `remote_write` block in the [Agent configuration reference](/docs/agent/latest/configuration/traces-config).
+refer to the `remote_write` block in the [Agent configuration reference](/docs/agent/<AGENT_VERSION>/configuration/traces-config).

docs/sources/tempo/configuration/grafana-agent/automatic-logging.md

@@ -51,7 +51,7 @@ For more information, refer to [Migrate to Alloy](https://grafana.com/docs/tempo
 
 To configure automatic logging, you need to select your preferred backend and the trace data to log.
 
-To see all the available configuration options, refer to the [configuration reference](https://grafana.com/docs/agent/latest/configuration/traces-config).
+To see all the available configuration options, refer to the [configuration reference](https://grafana.com/docs/agent/<AGENT_VERSION>/static/configuration/traces-config//).
 
 This simple example logs trace roots to `stdout` and is a good way to get started using automatic logging:
 ```yaml

docs/sources/tempo/configuration/grafana-agent/service-graphs.md

@@ -17,10 +17,10 @@ and the connections and dependencies between its components.
 
 The same service graph metrics can also be generated by Tempo.
 This is more efficient and recommended for larger installations.
-For a deep look into service graphs, visit [this section]({{< relref "../../metrics-generator/service_graphs" >}}).
+For a deep look into service graphs, visit [this section](../../../metrics-generator/service_graphs/).
 
 Service graphs are also used in the application performance management dashboard.
-For more information, refer to the [service graph view documentation]({{< relref "../../metrics-generator/service-graph-view" >}}).
+For more information, refer to the [service graph view documentation](../../../metrics-generator/service-graph-view/).
 
 ## Before you begin
 
@@ -47,7 +47,7 @@ traces:
         enabled: true
 ```
 
-To see all the available configuration options, refer to the [configuration reference](/docs/agent/latest/configuration/traces-config).
+To see all the available configuration options, refer to the [configuration reference](/docs/agent/<AGENT_VERSION>/configuration/traces-config).
 
 Metrics are registered in the Agent's default registerer.
 Therefore, they are exposed at `/metrics` in the Agent's server port (default `12345`).
@@ -70,4 +70,4 @@ metrics:
 The same service graph metrics can also be generated by Tempo.
 This is more efficient and recommended for larger installations.
 
-For additional information about viewing service graph metrics in Grafana and calculating cardinality, refer to the [server side documentation]({{< relref "../../metrics-generator/service_graphs/enable-service-graphs" >}}).
+For additional information about viewing service graph metrics in Grafana and calculating cardinality, refer to the [server side documentation](../../../metrics-generator/service_graphs/enable-service-graphs/).

docs/sources/tempo/configuration/grafana-agent/span-metrics.md

@@ -40,7 +40,7 @@ For more information, refer to [Migrate to Alloy](https://grafana.com/docs/tempo
 
 The same span metrics can also be generated by Tempo.
 This is more efficient and recommended for larger installations.
-For a deep look into span metrics, visit [this section]({{< relref "../../metrics-generator/span_metrics" >}}).
+For a deep look into span metrics, visit [this section](../../../metrics-generator/span_metrics/).
 
 ## Example
 

docs/sources/tempo/configuration/grafana-agent/tail-based-sampling.md

@@ -18,9 +18,9 @@ Probabilistic sampling strategies are easy to implement,
 but also run the risk of discarding relevant data that you'll later want.
 
 Tail-based sampling works with Grafana Agent in Flow or static modes.
-Flow mode configuration files are [written in River](/docs/agent/latest/flow/concepts/config-language).
-Static mode configuration files are [written in YAML](/docs/agent/latest/static/configuration).
-Examples in this document are for Flow mode. You can also use the [Static mode Kubernetes operator](/docs/agent/latest/operator).
+Flow mode configuration files are [written in River](/docs/agent/<AGENT_VERSION>/flow/concepts/config-language).
+Static mode configuration files are [written in YAML](/docs/agent/<AGENT_VERSION>/static/configuration).
+Examples in this document are for Flow mode. You can also use the [Static mode Kubernetes operator](/docs/agent/<AGENT_VERSION>/operator).
 
 ## How tail-based sampling works
 
@@ -57,7 +57,7 @@ This overhead increases with the number of Agent instances that share the same t
 To start using tail-based sampling, define a sampling policy.
 If you're using a multi-instance deployment of the agent,
 add load balancing and specify the resolving mechanism to find other Agents in the setup.
-To see all the available configuration options, refer to the [configuration reference](/docs/agent/latest/configuration/traces-config/).
+To see all the available configuration options, refer to the [configuration reference](/docs/agent/<AGENT_VERSION>/configuration/traces-config/).
 
 {{< admonition type="note">}}
 Grafana Alloy provides tooling to convert your Agent Static or Flow configuration files into a format that can be used by Alloy.
@@ -67,10 +67,10 @@ For more information, refer to [Migrate to Alloy](https://grafana.com/docs/tempo
 
 ### Example for Grafana Agent Flow
 
-[Grafana Agent Flow](/docs/agent/latest/flow/) is a component-based revision of Grafana Agent with a focus on ease-of-use, debuggability, and ability to adapt to the needs of power users.
+[Grafana Agent Flow](/docs/agent/<AGENT_VERSION>/flow/) is a component-based revision of Grafana Agent with a focus on ease-of-use, debuggability, and ability to adapt to the needs of power users.
 Flow configuration files are written in River instead of YAML.
 
-Grafana Agent Flow uses the [`otelcol.processor.tail_sampling component`](/docs/agent/latest/flow/reference/components/otelcol.processor.tail_sampling/)` for tail-based sampling.
+Grafana Agent Flow uses the [`otelcol.processor.tail_sampling component`](https://grafana.com/docs/agent/<AGENT_VERSION>/flow/reference/components/otelcol.processor.tail_sampling/) for tail-based sampling.
 
 ```river
 otelcol.receiver.otlp "otlp_receiver" {

docs/sources/tempo/configuration/grafana-alloy/_index.md

@@ -18,7 +18,7 @@ Alloy is flexible, and you can easily configure it to fit your needs in on-prem,
 It's commonly used as a tracing pipeline, offloading traces from the
 application and forwarding them to a storage backend.
 
-Grafana Alloy configuration files are written in the [Alloy configuration syntax](https://grafana.com/docs/alloy/latest/concepts/configuration-syntax/).
+Grafana Alloy configuration files are written in the [Alloy configuration syntax](https://grafana.com/docs/alloy/<ALLOY_VERSION>/get-started/configuration-syntax/).
 
 For more information, refer to the [Introduction to Grafana Alloy](https://grafana.com/docs/alloy/latest/introduction).
 
@@ -52,13 +52,13 @@ Grafana Alloy processes tracing data as it flows through the pipeline to make th
 
 Alloy supports batching of traces.
 Batching helps better compress the data, reduces the number of outgoing connections, and is a recommended best practice.
-To configure it, refer to the `otelcol.processor.batch` block in the [components reference](https://grafana.com/docs/alloy/latest/reference/components/otelcol.processor.batch/).
+To configure it, refer to the `otelcol.processor.batch` block in the [components reference](https://grafana.com/docs/alloy/<ALLOY_VERSION>/reference/components/otelcol/otelcol.processor.batch/).
 
 #### Attributes manipulation
 
 Grafana Alloy allows for general manipulation of attributes on spans that pass through it.
 A common use may be to add an environment or cluster variable.
-There are several processors that can manipulate attributes, some examples include: the `otelcol.processor.attributes` block in the [component reference](https://grafana.com/docs/alloy/latest/reference/components/otelcol.processor.attributes/) and the `otelcol.processor.transform` block [component reference](https://grafana.com/docs/alloy/latest/reference/components/otelcol.processor.transform/)
+There are several processors that can manipulate attributes, some examples include: the `otelcol.processor.attributes` block in the [component reference](https://grafana.com/docs/alloy/<ALLOY_VERSION>/reference/components/otelcol/otelcol.processor.attributes/) and the `otelcol.processor.transform` block [component reference](https://grafana.com/docs/alloy/<ALLOY_VERSION>/reference/components/otelcol/otelcol.processor.transform/)
 
 #### Attaching metadata with Prometheus Service Discovery
 
@@ -97,7 +97,7 @@ otelcol.exporter.otlp "default" {
 }
 ```
 
-Refer to the `otelcol.processor.k8sattributes` block in the [components reference](https://grafana.com/docs/alloy/latest/reference/components/otelcol.processor.k8sattributes/).
+Refer to the `otelcol.processor.k8sattributes` block in the [components reference](https://grafana.com/docs/alloy/<ALLOY_VERSION>/reference/components/otelcol/otelcol.processor.k8sattributes/).
 
 #### Trace discovery through automatic logging
 
@@ -138,4 +138,4 @@ Aside from endpoint and authentication, the exporter also provides mechanisms fo
 and implements a queue buffering mechanism for transient failures, such as networking issues.
 
 To see all available options,
-refer to the `otelcol.exporter.otlp` block in the [Alloy configuration reference](https://grafana.com/docs/alloy/latest/reference/components/otelcol.exporter.otlp/) and the `otelcol.exporter.otlphttp` block in the [Alloy configuration reference](https://grafana.com/docs/alloy/latest/reference/components/otelcol.exporter.otlphttp/).
+refer to the `otelcol.exporter.otlp` block in the [Alloy configuration reference](https://grafana.com/docs/alloy/<ALLOY_VERSION>/reference/components/otelcol/otelcol.exporter.otlp/) and the `otelcol.exporter.otlphttp` block in the [Alloy configuration reference](https://grafana.com/docs/alloy/<ALLOY_VERSION>/reference/components/otelcol/otelcol.exporter.otlphttp/).

docs/sources/tempo/configuration/grafana-alloy/automatic-logging.md

@@ -25,7 +25,7 @@ pipeline. This allows for automatically building a mechanism for trace discovery
 On top of that, you can also get metrics from traces using a logs source, and
 allow quickly jumping from a log message to the trace view in Grafana.
 
-While this approach is useful, it isn't as powerful as TraceQL. 
+While this approach is useful, it isn't as powerful as TraceQL.
 If you are here because you know you want to log the
 trace ID, to enable jumping from logs to traces, then read on.
 
@@ -47,7 +47,7 @@ This allows searching by those key-value pairs in Loki.
 To configure automatic logging, you need to configure the `otelcol.connector.spanlogs` connector with
 appropriate options.
 
-To see all the available configuration options, refer to the `otelcol.connector.spanlogs` [components reference](https://grafana.com/docs/alloy/latest/reference/components/otelcol.connector.spanlogs/).
+To see all the available configuration options, refer to the `otelcol.connector.spanlogs` [components reference](https://grafana.com/docs/alloy/<ALLOY_VERSION>/reference/components/otelcol/otelcol.connector.spanlogs/).
 
 This simple example logs trace roots before exporting them to the Grafana OTLP gateway,
 and is a good way to get started using automatic logging:

docs/sources/tempo/configuration/grafana-alloy/service-graphs.md

@@ -44,7 +44,7 @@ otelcol.receiver.otlp "default" {
   output {
     traces = [
       otelcol.connector.servicegraph.default.input,
-      otelcol.exporter.otlp.default.input
+      otelcol.exporter.otlp.default.input,
     ]
   }
 }
@@ -63,7 +63,7 @@ otelcol.exporter.otlp "default" {
 }
 ```
 
-To see all the available configuration options, refer to the [component reference](https://grafana.com/docs/alloy/latest/reference/components/otelcol.connector.servicegraph/).
+To see all the available configuration options, refer to the [component reference](https://grafana.com/docs/alloy/<ALLOY_VERSION>/reference/components/otelcol/otelcol.connector.servicegraph/).
 
 ### Grafana
 

docs/sources/tempo/configuration/grafana-alloy/span-metrics.md

@@ -68,7 +68,7 @@ otelcol.connector.spanmetrics "default" {
 
   metrics_flush_interval = "15s"
 
-  namespace = "traces_spanmetrics"
+  namespace = "traces.spanmetrics"
 
   output {
     metrics = [otelcol.exporter.otlp.default.input]

docs/sources/tempo/configuration/grafana-alloy/tail-based-sampling.md

@@ -1,144 +0,0 @@
----
-title: Enable tail-based sampling
-menuTItle: Enable tail-based sampling
-description: Use tail-based sampling to optimize sampling decisions
-weight:
-aliases:
-- /docs/tempo/grafana-alloy/tail-based-sampling
----
-
-# Enable tail-based sampling
-
-Tempo provides an inexpensive solution that aims to reduce the amount of tail-based sampling required.
-However, sometimes constraints make a lower sampling percentage necessary or desirable,
-such as runtime or egress traffic related costs.
-Probabilistic sampling strategies are easy to implement,
-but also run the risk of discarding relevant data that you'll later want.
-
-Tail-based sampling works with Grafana Alloy.
-Alloy configuration files are written in [Alloy configuration syntax](https://grafana.com/docs/alloy/latest/concepts/configuration-syntax/).
-
-## How tail-based sampling works
-
-In tail-based sampling, sampling decisions are made at the end of the workflow allowing for a more accurate sampling decision.
-Alloy groups spans by trace ID and checks its data to see
-if it meets one of the defined policies (for example, `latency` or `status_code`).
-For instance, a policy can check if a trace contains an error or if it took
-longer than a certain duration.
-
-A trace is sampled if it meets at least one policy.
-
-To group spans by trace ID, Alloy buffers spans for a configurable amount of time,
-after which it considers the trace complete.
-Longer running traces are split into more than one.
-However, waiting longer times increases the memory overhead of buffering.
-
-One particular challenge of grouping trace data is for multi-instance Alloy deployments,
-where spans that belong to the same trace can arrive to different Alloys.
-To solve that, you can configure Alloy to load balance traces across Alloy instances
-by exporting spans belonging to the same trace to the same instance.
-
-This is achieved by redistributing spans by trace ID once they arrive from the application.
-Alloy must be able to discover and connect to other Alloy instances where spans for the same trace can arrive.
-Kubernetes users should use a [headless service](https://kubernetes.io/docs/concepts/services-networking/service/#headless-services).
-
-Redistributing spans by trace ID means that spans are sent and received twice,
-which can cause a significant increase in CPU usage.
-This overhead increases with the number of Alloy instances that share the same traces.
-
-<p align="center"><img src="../tempo-tail-based-sampling.svg" alt="Tail-based sampling overview"></p>
-
-### Sampling load balancing
-
-Tail sampling load balancing is usually carried out by running two layers of collectors, the first layer receiving the telemetry data (in this case trace spans), and then distributing these to the second layer that carry out the sampling policies.
-
-Alloy includes a [load balancing export](https://grafana.com/docs/alloy/latest/reference/components/otelcol/otelcol.exporter.loadbalancing/) that can carry out routing to further collector targets based on a set number of keys (in the case of trace sampling, the `traceID` key).
-Alloy uses the OpenTelemetry load balancing exporter.
-
-The routing key ensures that a specific collector in the second layer always deals with spans from the same trace ID, ensuring that sampling decisions are made correctly.
-There is functionality to configure the exporter with targets via a few different methods.
-This includes static IPs, multi-IP DNS A record entries, and a Kubernetes headless service resolver.
-This has the advantage of allowing you to scale up/down the number of layer two collectors.
-
-There are some important points to note with the load balancer exporter around scaling and resilience, mostly around its eventual consistency model. For more infortmation, refer to [Resiliance and scaling considerations](https://github.com/open-telemetry/opentelemetry-collector-contrib/blob/main/exporter/loadbalancingexporter/README.md#resilience-and-scaling-considerations).
-The most important in terms of tail sampling is that routing occurs based on an algorithm taking into account the number of backends available to the load balancer, and this can affect the target for trace ID spans before eventual consistency occurs.
-
-For an example manifest for a two layer OTel Collector deployment based around Kubernetes services, refer to the [K8s resolver README](https://github.com/open-telemetry/opentelemetry-collector-contrib/blob/main/exporter/loadbalancingexporter/example/k8s-resolver/README.md).
-
-## Configure tail-based sampling
-
-To start using tail-based sampling, define a sampling policy in your configuration file.
-
-If you're using a multi-instance deployment of Alloy,
-add load balancing and specify the resolving mechanism to find other Alloy instances in the setup.
-
-To see all the available configuration options for load balancing, refer to the [Alloy component reference](https://grafana.com/docs/alloy/latest/reference/components/otelcol.exporter.loadbalancing/).
-
-### Example for Alloy
-
-Alloy uses the [`otelcol.processor.tail_sampling component`](https://grafana.com/docs/alloy/latest/reference/components/otelcol.processor.tail_sampling/) for tail-based sampling.
-
-```alloy
-otelcol.receiver.otlp "default" {
-  http {}
-  grpc {}
-
-  output {
-    traces  = [otelcol.processor.tail_sampling.policies.input]
-  }
-}
-
-// The Tail Sampling processor will use a set of policies to determine which received
-// traces to keep and send to Tempo.
-otelcol.processor.tail_sampling "policies" {
-    // Total wait time from the start of a trace before making a sampling decision.
-    // Note that smaller time periods can potentially cause a decision to be made
-    // before the end of a trace has occurred.
-    decision_wait = "30s"
-
-    // The following policies follow a logical OR pattern, meaning that if any of the
-    // policies match, the trace will be kept. For logical AND, you can use the `and`
-    // policy. Every span of a trace is examined by each policy in turn. A match will
-    // cause a short-circuit.
-
-    // This policy defines that traces that contain errors should be kept.
-    policy {
-        // The name of the policy can be used for logging purposes.
-        name = "sample-erroring-traces"
-        // The type must match the type of policy to be used, in this case examining
-        // the status code of every span in the trace.
-        type = "status_code"
-        // This block determines the error codes that should match in order to keep
-        // the trace, in this case the OpenTelemetry 'ERROR' code.
-        status_code {
-            status_codes = [ "ERROR" ]
-        }
-    }
-
-    // This policy defines that only traces that are longer than 200ms in total
-    // should be kept.
-    policy {
-        // The name of the policy can be used for logging purposes.
-        name = "sample-long-traces"
-        // The type must match the policy to be used, in this case the total latency
-        // of the trace.
-        type = "latency"
-        // This block determines the total length of the trace in milliseconds.
-        latency {
-            threshold_ms = 200
-        }
-    }
-
-    // The output block forwards the kept traces onto the batch processor, which
-    // will marshall them for exporting to the Grafana OTLP gateway.
-    output {
-        traces = [otelcol.exporter.otlp.default.input]
-    }
-}
-
-otelcol.exporter.otlp "default" {
-  client {
-    endpoint = env("OTLP_ENDPOINT")
-  }
-}
-```

docs/sources/tempo/configuration/grafana-alloy/tail-sampling/_index.md

@@ -0,0 +1,147 @@
+---
+title: Sampling
+menuTitle: Sampling
+description: Learn how to streamline tracing data by using sampling to determine which data to keep and which to drop.
+aliases:
+- ./tail-based-sampling/ # /docs/tempo/latest/configuration/grafana-alloy/tail-based-sampling/
+---
+
+# Sampling
+
+Grafana Tempo is a cost-effective solution that ingests and stores traces that provide maximum observability across your application estate.
+However, sometimes constraints mean that storing all of your traces is not desirable, for example runtime or egress traffic related costs.
+There are a number of ways to lower trace volume, including varying sampling strategies.
+
+Sampling is the process of determining which traces to store (in Tempo or Grafana Cloud Traces) and which to discard. Sampling comes in two different strategy types: head and tail sampling.
+
+Sampling functionality exists in both [Grafana Alloy](https://grafana.com/docs/alloy/) and the OpenTelemetry Collector. Alloy can collect, process, and export telemetry signals, with configuration files written in [Alloy configuration syntax](https://grafana.com/docs/alloy/<ALLOY_VERSION>/get-started/configuration-syntax/).
+
+Refer to [Enable tail sampling](https://grafana.com/docs/tempo/<TEMPO_VERSION>/configuration/grafana-alloy/tail-sampling/enable-tail-sampling/) for instructions.
+
+## Head and tail sampling
+
+When sampling, you can use a head or tail sampling strategy.
+
+With a head sampling strategy, the decision to sample the trace is usually made as early as possible and doesn’t need to take into account the whole trace.
+It’s a simple but effective sampling strategy.
+Refer to the [Head sampling documentation](https://grafana.com/docs/grafana-cloud/monitor-applications/application-observability/collector/sampling/head/#head-sampling) for [Application Observability](https://grafana.com/docs/grafana-cloud/monitor-applications/application-observability/) for more information.
+
+With a tail sampling strategy, the decision to sample a trace is made after considering all or most of the spans. For example, tail sampling is a good option to sample only traces that have errors or traces with long request duration.
+Tail sampling is more complex to configure, implement, and maintain but is the recommended sampling strategy for large systems with a high telemetry volume.
+
+For more information about sampling, refer to the [OpenTelemetry Sampling](https://opentelemetry.io/docs/concepts/sampling/) documentation.
+
+![Tail sampling overview and components with Tempo, Alloy, and Grafana](/media/docs/tempo/sampling/tempo-tail-based-sampling.svg)
+
+## Sampling and telemetry correlation
+
+Sampling is a decision on whether or not to keep (and then store) a trace, or whether to discard it.
+These decisions have implications when it comes to correlating trace data with other signals.
+
+For example, many services that are instrumented also produce logs, metrics, or profiles.
+These signals can reference each other.
+In the case of a trace, this reference can be via a trace ID embedded into a [log line](https://grafana.com/docs/grafana/<GRAFANA_VERSION>/datasources/tempo/traces-in-grafana/link-trace-id/), an [exemplar](https://grafana.com/docs/grafana/<GRAFANA_VERSION>/fundamentals/exemplars/) embedded into a metric value, or a profile ID [embedded into a trace](https://grafana.com/docs/grafana-cloud/monitor-applications/profiles/traces-to-profiles/).
+
+By choosing to not sample a trace, a particular signal references the dropped trace's ID in some cases.
+This drop can lead to a situation in Grafana where following a link to a trace ID from a log line or an exemplar from a metric value, results in a query for that trace ID failing because the trace has not been sampled.
+Profiles may not show up without specifically querying for them, because a trace that would have included the profile's flame graph hasn't been stored.
+
+This isn't usually a huge issue, because sampling policies tend to be chosen that show non-normative behavior, for example, errors being thrown or long latencies on requests.
+An observer is more likely to be choosing traces that show these issues rather than the required behavior.
+Understand how signals correlate between each other helps determine how to choose these policies.
+
+## How tail sampling works in the OpenTelemetry Tail Sampling Processor
+
+In tail sampling, sampling decisions are made at the end of the workflow allowing for a more accurate sampling decision.
+Alloy uses the [OpenTelemetry Tail Sampling Processor](https://github.com/open-telemetry/opentelemetry-collector-contrib/blob/main/processor/tailsamplingprocessor/README.md).
+
+Alloy organizes spans by trace ID and evaluates its data to see if it meets one of the defined policy types (for example, `latency` or `status_code`).
+For instance, a policy can check if a trace contains an error or the trace duration was longer than a specified threshold.
+
+A trace is sampled if it meets the conditions of at least one policy.
+
+### Decision periods
+
+To group spans by trace ID, Alloy buffers spans for a configurable amount of time, after which it considers the trace complete.
+This configurable amount of time is known as the decision period. Longer running traces are split into more than one.
+
+In situations where a specific trace is longer in duration than the decision period, multiple decisions might be made for any future spans that fall outside of the decision period window.
+This can result in some spans for a trace being sampled, while others are not.
+
+For example, consider a situation where the tail sampler decision period is 10 seconds, and a single policy exists to sample traces where an error is set on at least one span.
+One of the traces is 20 seconds in duration and a single span at time offset 15 seconds exhibits an error status.
+
+![Trace Policy: Error when status exists](/media/docs/tempo/sampling/tempo-decision-point-sampling.svg)
+
+When the first span for the trace is observed, the decision period time of 10 seconds is initiated.
+After the decision period has expired, the tail sampler won't have observed any spans with an error status, and will therefore discard the trace spans.
+
+When the next span for the trace arrives, a new decision period of 10 seconds begins.
+In this period, one of the observed spans has an error set on it. When the decision period expires, all of the spans for the trace in that period will be sampled.
+
+This leads to a fragmented trace being stored in Tempo, where only the spans for the last 10 seconds of the trace will be available to query.
+While this is still a potentially useful trace, careful determination of how to set the decision period is key to ensuring that trace spans are sampled correctly.
+
+However, using longer decision periods increases the memory overhead of buffering the spans required to make a decision for each trace.
+
+For this reason, enabling a decision cache can ensure that previous sampling decisions for a specific trace ID are honored even after the expiration of the decision period.
+For more details, refer to the Caches section.
+
+### Caches
+
+The OpenTelemetry tail sampling processor includes two separate caches, the sampled and non-sampled caches.
+The sampled cache keeps a list of all trace IDs where a prior decision to keep spans has been made.
+The non-sampled cache keeps a list of all trace IDs where a prior decision to drop spans has been made.
+Both caches are configured by the maximum number of traces that should be stored in the cache, and can be enabled either independently or jointly.
+
+![Decision points and caches workflow](/media/docs/tempo/sampling/tempo-alloy-sampling-policies.svg)
+
+In the above diagram, should both caches be enabled, then a decision to drop samples for the trace is made after 10 seconds and the trace ID stored in the non-sampled cache.
+This means that even spans that have an error status for that trace are dropped after the initial decision period, as the non-sampled cache matches the trace ID and pre-emptively drops the span.
+However, the same is true should a sampled decision have been made, where any future spans do not match any policies but whose trace ID is found in the sampled cache.
+
+Understanding how these caches work ensures that you still keep decisions that have previously been made.
+For example, you could use the sampled cache to short-circuit future decisions for a trace, immediately sampling the incoming span.
+This allows a decision to be made without having to buffer any other spans.
+
+Here are some general guidelines for using caches.
+Every installation is different.
+Using the caches can impact the amount of data generated.
+
+| Cache type    | Use case                                                                                               | Benefits/Considerations                                                                                                                                                                       |
+| ------------- | ------------------------------------------------------------------------------------------------------ | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
+| Sample caches | Keep any future spans from traces that have been sampled.                                              | <ul><li>Cuts down span storage per trace to only those matching policies.</li><li> Can cause fragmented tracing.</li></ul>                                                                    |
+| Non-sampled   | Drop any future spans from traces where a decision to not sample those traces has explicitly occurred. | <ul><li>Lowers chance of storing traces after the initial decision period. </li><li>Misses any trace whose spans exhibit future policy criteria matching.</li></ul>                           |
+| Both          | Use an initial decision period that makes a decision once and uses that decision going forward.        | <ul><li>Guarantees capture of full traces.</li><li>Lower chance of capturing useful traces with a long duration.</li><li>Can lose spans if they are longer than the decision period.</li><ul> |
+
+{{< admonition type="note" >}}
+Enabling both sampled and non-sampled caches exhibits functionality similar to that of not enabling caches.
+However, it short-circuits any future decision making once an initial decision period has expired. Enabling both caches lowers memory requirements for buffering spans.
+{{< /admonition >}}
+
+### Tail sampling load balancing
+
+Situations may arise for multi-instance Alloy deployments where emitted spans belonging to the same trace may arrive at different instances.
+For most cases, sampling decisions rely on all the spans for a specific trace ID being received by a single instance.
+
+You can configure Alloy to load balance traces across instances by exporting spans belonging to a specific trace ID to the same instance.
+For example, if 10 traces are coming in and there are four Alloy instances, then each instance will receive three traces and one instance will receive four traces.
+The load balancing maintains consistent hashing across all instances.
+
+Tail sampling load balancing is usually carried out by running two layers of collectors.
+The first layer receives the telemetry data (in this case trace spans), and then distributes these to the second layer that carries out the sampling policies.
+
+![Load balancing incoming traces using Alloy](/media/docs/tempo/sampling/tempo-alloy-sampling-loadbalancing.svg)
+
+Alloy includes a [load-balancing exporter](https://grafana.com/docs/alloy/<ALLOY_VERSION>/reference/components/otelcol/otelcol.exporter.loadbalancing/) that can carry out routing to further collector targets based on a set number of keys (in the case of trace sampling, usually the `traceID` key).
+Alloy uses the [OpenTelemetry load balancing exporter](https://github.com/open-telemetry/opentelemetry-collector-contrib/blob/main/exporter/loadbalancingexporter/README.md).
+
+The routing key ensures that a specific collector in the second layer always handles spans from the same trace ID, guaranteeing that sampling decisions are made correctly.
+You can configure the exporter with targets using static IP addresses, multi-IP DNS A record entries, and a Kubernetes headless service resolver.
+Using this configuration lets you scale up or down the number of layer two collectors.
+
+There are some important points to note with the load balancer exporter around scaling and resilience, mostly around its eventual consistency model. For more information, refer to [Resilience and scaling considerations](https://github.com/open-telemetry/opentelemetry-collector-contrib/blob/main/exporter/loadbalancingexporter/README.md#resilience-and-scaling-considerations).
+The most important in terms of tail sampling is that routing occurs based on an algorithm taking into account the number of backends available to the load balancer.
+This can affect the target for trace ID spans before eventual consistency occurs.
+
+For an example manifest for a two layer OpenTelemetry Collector deployment based around Kubernetes services, refer to the [Kubernetes service resolver README](https://github.com/open-telemetry/opentelemetry-collector-contrib/blob/main/exporter/loadbalancingexporter/example/k8s-resolver/README.md).

docs/sources/tempo/configuration/grafana-alloy/tail-sampling/enable-tail-sampling.md

@@ -0,0 +1,93 @@
+---
+title: Enable tail-based sampling
+menuTitle: Enable tail sampling
+description: Configure tail sampling with Tempo and Grafana Alloy to optimize sampling decisions.
+---
+
+# Enable tail sampling
+
+You can use tail sampling to use a lower sampling percentage when necessary or desirable,
+such as runtime or egress traffic related costs.
+Probabilistic sampling strategies are easy to implement,
+but also run the risk of discarding relevant data that you'll later want.
+
+Tail sampling works with Grafana Alloy.
+Alloy configuration files are written in [Alloy configuration syntax](https://grafana.com/docs/alloy/<ALLOY_VERSION>/get-started/configuration-syntax/).
+
+## Configure tail sampling
+
+To start using tail sampling, define a sampling policy in your configuration file.
+
+If you're using a multi-instance deployment of Alloy,
+add load balancing and specify the resolving mechanism to find other Alloy instances in the setup.
+
+To see all the available configuration options for load balancing, refer to the [Alloy component reference](https://grafana.com/docs/alloy/<ALLOY_VERSION>/reference/components/otelcol.exporter.loadbalancing/).
+
+### Example for Alloy
+
+Alloy uses the [`otelcol.processor.tail_sampling component`](https://grafana.com/docs/alloy/<ALLOY_VERSION>/reference/components/otelcol/otelcol.processor.tail_sampling/) for tail sampling.
+
+```alloy
+otelcol.receiver.otlp "default" {
+  http {}
+  grpc {}
+
+  output {
+    traces  = [otelcol.processor.tail_sampling.policies.input]
+  }
+}
+
+// The Tail Sampling processor will use a set of policies to determine which received
+// traces to keep and send to Tempo.
+otelcol.processor.tail_sampling "policies" {
+    // Total wait time from the start of a trace before making a sampling decision.
+    // Note that smaller time periods can potentially cause a decision to be made
+    // before the end of a trace has occurred.
+    decision_wait = "30s"
+
+    // The following policies follow a logical OR pattern, meaning that if any of the
+    // policies match, the trace will be kept. For logical AND, you can use the `and`
+    // policy. Every span of a trace is examined by each policy in turn. A match will
+    // cause a short-circuit.
+
+    // This policy defines that traces that contain errors should be kept.
+    policy {
+        // The name of the policy can be used for logging purposes.
+        name = "sample-erroring-traces"
+        // The type must match the type of policy to be used, in this case examining
+        // the status code of every span in the trace.
+        type = "status_code"
+        // This block determines the error codes that should match in order to keep
+        // the trace, in this case the OpenTelemetry 'ERROR' code.
+        status_code {
+            status_codes = [ "ERROR" ]
+        }
+    }
+
+    // This policy defines that only traces that are longer than 200ms in total
+    // should be kept.
+    policy {
+        // The name of the policy can be used for logging purposes.
+        name = "sample-long-traces"
+        // The type must match the policy to be used, in this case the total latency
+        // of the trace.
+        type = "latency"
+        // This block determines the total length of the trace in milliseconds.
+        latency {
+            threshold_ms = 200
+        }
+    }
+
+    // The output block forwards the kept traces onto the batch processor, which
+    // will marshall them for exporting to the Grafana OTLP gateway.
+    output {
+        traces = [otelcol.exporter.otlp.default.input]
+    }
+}
+
+otelcol.exporter.otlp "default" {
+  client {
+    endpoint = env("OTLP_ENDPOINT")
+  }
+}
+```

docs/sources/tempo/configuration/hosted-storage/_index.md

@@ -11,6 +11,6 @@ aliases:
 
 Tempo provides additional hosted storage configuration options discussed on the pages below. These options relate to providers such as Google Cloud, AWS S3, and Azure.
 
-For additional details about storage configuration, refer to [Storage configuration]({{< relref "../../configuration#storage" >}}).
+For additional details about storage configuration, refer to [Storage configuration](../#storage).
 
 {{< section withDescriptions="true">}}

docs/sources/tempo/configuration/hosted-storage/gcs.md

@@ -7,7 +7,7 @@ aliases:
 
 # Google Cloud Storage
 
-For configuration options, check the storage section on the [configuration]({{< relref "../../configuration#storage" >}}) page.
+For configuration options, check the storage section on the [configuration](../../#storage) page.
 
 ## Permissions
 

docs/sources/tempo/configuration/hosted-storage/s3.md

@@ -8,7 +8,7 @@ aliases:
 
 # Amazon S3 permissions
 
-For configuration options, refer to the storage section on the [configuration]({{< relref "../../configuration#storage" >}}) page.
+For configuration options, refer to the storage section on the [configuration](../../#storage) page.
 
 The following authentication methods are supported:
 - AWS environment variables `AWS_ACCESS_KEY_ID` and `AWS_SECRET_ACCESS_KEY`

docs/sources/tempo/configuration/manifest.md

@@ -10,7 +10,7 @@ weight: 110
 
 This document is a reference for all Tempo options and their defaults. If you are just getting
 started with Tempo, refer to [Tempo examples](https://github.com/grafana/tempo/tree/main/example/docker-compose)
-and other [configuration documentation]({{< relref "../configuration" >}}). Most installations will require only setting 10 to 20 of these options.
+and other [configuration documentation](../). Most installations will require only setting 10 to 20 of these options.
 
 ## Complete configuration
 
@@ -20,7 +20,7 @@ http_api_prefix: ""
 server:
     http_listen_network: tcp
     http_listen_address: ""
-    http_listen_port: 80
+    http_listen_port: 3200
     http_listen_conn_limit: 0
     grpc_listen_network: tcp
     grpc_listen_address: ""
@@ -76,6 +76,11 @@ server:
     log_request_at_info_level_enabled: false
     log_request_exclude_headers_list: ""
     http_path_prefix: ""
+    cluster_validation:
+        label: ""
+        grpc:
+            enabled: false
+            soft_validation: false
 internal_server:
     http_listen_network: tcp
     http_listen_address: ""
@@ -135,6 +140,11 @@ internal_server:
     log_request_at_info_level_enabled: false
     log_request_exclude_headers_list: ""
     http_path_prefix: ""
+    cluster_validation:
+        label: ""
+        grpc:
+            enabled: false
+            soft_validation: false
     enable: false
 distributor:
     ring:
@@ -176,6 +186,7 @@ distributor:
             - en0
         instance_port: 0
         instance_addr: ""
+        enable_inet6: false
     receivers: {}
     override_ring_key: distributor
     forwarders: []
@@ -183,9 +194,27 @@ distributor:
         cost_attribution:
             max_cardinality: 10000
             stale_duration: 15m0s
+    kafka_write_path_enabled: false
+    kafka_config:
+        address: ""
+        topic: ""
+        client_id: ""
+        dial_timeout: 0s
+        write_timeout: 0s
+        sasl_username: ""
+        sasl_password: ""
+        consumer_group: ""
+        consumer_group_offset_commit_interval: 0s
+        last_produced_offset_retry_timeout: 0s
+        auto_create_topic_enabled: false
+        auto_create_topic_default_partitions: 0
+        producer_max_record_size_bytes: 0
+        producer_max_buffered_bytes: 0
+        target_consumer_lag_at_startup: 0s
+        max_consumer_lag_at_startup: 0s
     extend_writes: true
     retry_after_on_resource_exhausted: 0s
-    max_span_attr_byte: 2048
+    max_attribute_bytes: 2048
 ingester_client:
     pool_config:
         checkinterval: 15s
@@ -196,7 +225,7 @@ ingester_client:
     grpc_client_config:
         max_recv_msg_size: 104857600
         max_send_msg_size: 104857600
-        grpc_compression: ""
+        grpc_compression: snappy
         rate_limit: 0
         rate_limit_burst: 0
         backoff_on_ratelimits: false
@@ -217,6 +246,8 @@ ingester_client:
         connect_timeout: 5s
         connect_backoff_base_delay: 1s
         connect_backoff_max_delay: 5s
+        cluster_validation:
+            label: ""
 metrics_generator_client:
     pool_config:
         checkinterval: 15s
@@ -227,7 +258,7 @@ metrics_generator_client:
     grpc_client_config:
         max_recv_msg_size: 104857600
         max_send_msg_size: 104857600
-        grpc_compression: ""
+        grpc_compression: snappy
         rate_limit: 0
         rate_limit_burst: 0
         backoff_on_ratelimits: false
@@ -248,15 +279,11 @@ metrics_generator_client:
         connect_timeout: 5s
         connect_backoff_base_delay: 1s
         connect_backoff_max_delay: 5s
+        cluster_validation:
+            label: ""
 querier:
     search:
         query_timeout: 30s
-        prefer_self: 10
-        external_hedge_requests_at: 8s
-        external_hedge_requests_up_to: 2
-        external_backend: ""
-        google_cloud_run: null
-        external_endpoints: []
     trace_by_id:
         query_timeout: 10s
     metrics:
@@ -272,7 +299,7 @@ querier:
         grpc_client_config:
             max_recv_msg_size: 104857600
             max_send_msg_size: 16777216
-            grpc_compression: ""
+            grpc_compression: snappy
             rate_limit: 0
             rate_limit_burst: 0
             backoff_on_ratelimits: false
@@ -293,12 +320,14 @@ querier:
             connect_timeout: 0s
             connect_backoff_base_delay: 0s
             connect_backoff_max_delay: 0s
+            cluster_validation:
+                label: ""
     shuffle_sharding_ingesters_enabled: false
     shuffle_sharding_ingesters_lookback_period: 1h0m0s
     query_relevant_ingesters: false
 query_frontend:
     max_outstanding_per_tenant: 2000
-    max_batch_size: 5
+    max_batch_size: 7
     log_query_request_headers: ""
     max_retries: 2
     search:
@@ -310,6 +339,7 @@ query_frontend:
         query_backend_after: 15m0s
         query_ingesters_until: 30m0s
         ingester_shards: 3
+        most_recent_shards: 200
         max_spans_per_span_set: 100
     trace_by_id:
         query_shards: 50
@@ -328,6 +358,7 @@ query_frontend:
         max_traceql_conditions: 4
         max_regex_conditions: 1
     max_query_expression_size_bytes: 131072
+    rf1_after: 0001-01-01T00:00:00Z
 compactor:
     ring:
         kvstore:
@@ -439,6 +470,40 @@ ingester:
         address: ""
         port: 0
         id: hostname
+    partition_ring:
+        kvstore:
+            store: memberlist
+            prefix: collectors/
+            consul:
+                host: localhost:8500
+                acl_token: ""
+                http_client_timeout: 20s
+                consistent_reads: false
+                watch_rate_limit: 1
+                watch_burst_size: 1
+                cas_retry_delay: 1s
+            etcd:
+                endpoints: []
+                dial_timeout: 10s
+                max_retries: 10
+                tls_enabled: false
+                tls_cert_path: ""
+                tls_key_path: ""
+                tls_ca_path: ""
+                tls_server_name: ""
+                tls_insecure_skip_verify: false
+                tls_cipher_suites: ""
+                tls_min_version: ""
+                username: ""
+                password: ""
+            multi:
+                primary: ""
+                secondary: ""
+                mirror_enabled: false
+                mirror_timeout: 2s
+        min_partition_owners_count: 1
+        min_partition_owners_duration: 10s
+        delete_inactive_partition_after: 13h0m0s
     concurrent_flushes: 4
     flush_check_period: 10s
     flush_op_timeout: 5m0s
@@ -448,6 +513,7 @@ ingester:
     complete_block_timeout: 15m0s
     override_ring_key: ring
     flush_all_on_shutdown: false
+    flush_object_storage: true
 metrics_generator:
     ring:
         kvstore:
@@ -568,12 +634,19 @@ metrics_generator:
             trace_idle_period: 10s
             max_block_duration: 1m0s
             max_block_bytes: 500000000
+            concurrency: 4
             complete_block_timeout: 1h0m0s
             max_live_traces: 0
+            max_live_traces_bytes: 250000000
             filter_server_spans: true
             flush_to_storage: false
             concurrent_blocks: 10
             time_overlap_cutoff: 0.2
+        host_info:
+            host_identifiers:
+                - k8s.node.name
+                - host.id
+            metric_name: traces_host_info
     registry:
         collection_interval: 15s
         stale_duration: 15m0s
@@ -597,9 +670,62 @@ metrics_generator:
         search_encoding: none
         ingestion_time_range_slack: 2m0s
         version: vParquet4
+    traces_query_storage:
+        path: ""
+        v2_encoding: none
+        search_encoding: none
+        ingestion_time_range_slack: 2m0s
+        version: vParquet4
     metrics_ingestion_time_range_slack: 30s
     query_timeout: 30s
     override_ring_key: metrics-generator
+    codec: push-bytes
+    disable_local_blocks: false
+    disable_grpc: false
+    ingest_concurrency: 16
+    instance_id: hostname
+ingest:
+    enabled: false
+    kafka:
+        address: localhost:9092
+        topic: ""
+        client_id: ""
+        dial_timeout: 2s
+        write_timeout: 10s
+        sasl_username: ""
+        sasl_password: ""
+        consumer_group: ""
+        consumer_group_offset_commit_interval: 1s
+        last_produced_offset_retry_timeout: 10s
+        auto_create_topic_enabled: true
+        auto_create_topic_default_partitions: 1000
+        producer_max_record_size_bytes: 15983616
+        producer_max_buffered_bytes: 1073741824
+        target_consumer_lag_at_startup: 2s
+        max_consumer_lag_at_startup: 15s
+block_builder:
+    instance_id: hostname
+    assigned_partitions: {}
+    consume_cycle_duration: 5m0s
+    max_consuming_bytes: 5000000000
+    block:
+        max_block_bytes: 20971520
+        bloom_filter_false_positive: 0.01
+        bloom_filter_shard_size_bytes: 102400
+        version: vParquet4
+        search_encoding: snappy
+        search_page_size_bytes: 1048576
+        v2_index_downsample_bytes: 1048576
+        v2_index_page_size_bytes: 256000
+        v2_encoding: zstd
+        parquet_row_group_size_bytes: 100000000
+        parquet_dedicated_columns: []
+    wal:
+        path: /var/tempo/block-builder/traces
+        v2_encoding: none
+        search_encoding: none
+        ingestion_time_range_slack: 2m0s
+        version: vParquet4
 storage:
     trace:
         pool:
@@ -683,6 +809,10 @@ storage:
             metadata: {}
             native_aws_auth_enabled: false
             list_blocks_concurrency: 3
+            sse:
+                type: ""
+                kms_key_id: ""
+                kms_encryption_context: ""
         azure:
             storage_account_name: ""
             storage_account_key: ""
@@ -767,6 +897,10 @@ overrides:
                 metadata: {}
                 native_aws_auth_enabled: false
                 list_blocks_concurrency: 3
+                sse:
+                    type: ""
+                    kms_key_id: ""
+                    kms_encryption_context: ""
             azure:
                 storage_account_name: ""
                 storage_account_key: ""
@@ -805,6 +939,7 @@ memberlist:
     abort_if_cluster_join_fails: false
     rejoin_interval: 0s
     left_ingesters_timeout: 5m0s
+    obsolete_entries_timeout: 30s
     leave_timeout: 20s
     broadcast_timeout_for_local_updates_on_shutdown: 10s
     message_history_buffer_bytes: 0
@@ -833,4 +968,122 @@ cache:
         writeback_goroutines: 10
         writeback_buffer: 10000
     caches: []
+backend_scheduler:
+    tenant_measurement_interval: 1m0s
+    work:
+        prune_age: 1h0m0s
+        dead_job_timeout: 24h0m0s
+    maintenance_interval: 1m0s
+    provider:
+        retention:
+            interval: 1h0m0s
+        compaction:
+            measure_interval: 1m0s
+            compaction:
+                v2_in_buffer_bytes: 5242880
+                v2_out_buffer_bytes: 20971520
+                v2_prefetch_traces_count: 1000
+                compaction_window: 1h0m0s
+                max_compaction_objects: 6000000
+                max_block_bytes: 107374182400
+                block_retention: 336h0m0s
+                compacted_block_retention: 1h0m0s
+                retention_concurrency: 10
+                max_time_per_tenant: 5m0s
+                compaction_cycle: 30s
+            max_jobs_per_tenant: 1000
+            backoff:
+                min_period: 100ms
+                max_period: 10s
+                max_retries: 0
+    job_timeout: 15s
+backend_scheduler_client:
+    grpc_client_config:
+        max_recv_msg_size: 104857600
+        max_send_msg_size: 104857600
+        grpc_compression: snappy
+        rate_limit: 0
+        rate_limit_burst: 0
+        backoff_on_ratelimits: false
+        backoff_config:
+            min_period: 100ms
+            max_period: 10s
+            max_retries: 10
+        initial_stream_window_size: 63KiB1023B
+        initial_connection_window_size: 63KiB1023B
+        tls_enabled: false
+        tls_cert_path: ""
+        tls_key_path: ""
+        tls_ca_path: ""
+        tls_server_name: ""
+        tls_insecure_skip_verify: false
+        tls_cipher_suites: ""
+        tls_min_version: ""
+        connect_timeout: 5s
+        connect_backoff_base_delay: 1s
+        connect_backoff_max_delay: 5s
+        cluster_validation:
+            label: ""
+backend_worker:
+    backend_scheduler_addr: ""
+    backoff:
+        min_period: 100ms
+        max_period: 1m0s
+        max_retries: 0
+    compaction:
+        v2_in_buffer_bytes: 5242880
+        v2_out_buffer_bytes: 20971520
+        v2_prefetch_traces_count: 1000
+        compaction_window: 1h0m0s
+        max_compaction_objects: 6000000
+        max_block_bytes: 107374182400
+        block_retention: 336h0m0s
+        compacted_block_retention: 1h0m0s
+        retention_concurrency: 10
+        max_time_per_tenant: 5m0s
+        compaction_cycle: 30s
+    override_ring_key: backend-worker
+    ring:
+        kvstore:
+            store: ""
+            prefix: collectors/
+            consul:
+                host: localhost:8500
+                acl_token: ""
+                http_client_timeout: 20s
+                consistent_reads: false
+                watch_rate_limit: 1
+                watch_burst_size: 1
+                cas_retry_delay: 1s
+            etcd:
+                endpoints: []
+                dial_timeout: 10s
+                max_retries: 10
+                tls_enabled: false
+                tls_cert_path: ""
+                tls_key_path: ""
+                tls_ca_path: ""
+                tls_server_name: ""
+                tls_insecure_skip_verify: false
+                tls_cipher_suites: ""
+                tls_min_version: ""
+                username: ""
+                password: ""
+            multi:
+                primary: ""
+                secondary: ""
+                mirror_enabled: false
+                mirror_timeout: 2s
+        heartbeat_period: 5s
+        heartbeat_timeout: 1m0s
+        wait_stability_min_duration: 1m0s
+        wait_stability_max_duration: 5m0s
+        instance_id: hostname
+        instance_interface_names:
+            - eth0
+            - en0
+        instance_port: 0
+        instance_addr: ""
+        enable_inet6: false
+        wait_active_instance_timeout: 10m0s
 ```

docs/sources/tempo/configuration/network/ipv6.md

@@ -12,7 +12,7 @@ Tempo can be configured to communicate between the components using Internet Pro
 
 {{< admonition type="note" >}}
 The underlying infrastructure must support this address family. This configuration may be used in a single-stack IPv6 environment, or in a dual-stack environment where both IPv6 and IPv4 are present. In a dual-stack scenario, only one address family may be configured at a time, and all components must be configured for that address family.
-{{% /admonition %}}
+{{< /admonition >}}
 
 ## Protocol configuration
 

docs/sources/tempo/configuration/network/tls.md

@@ -12,7 +12,7 @@ Tempo can be configured to communicate between the components using Transport La
 
 {{< admonition type="note" >}}
 The ciphers and TLS version here are for example purposes only. We are not recommending which ciphers or TLS versions for use in production environments.
-{{% /admonition %}}
+{{< /admonition >}}
 
 ## Server configuration
 
@@ -77,7 +77,8 @@ memberlist:
 
 ### Receiver TLS
 
-Additional receiver configuration can be added to support TLS communication for traces being sent to Tempo. The receiver configuration is pulled in from the Open Telemetry collector, and is [documented upstream here](https://github.com/open-telemetry/opentelemetry-collector/blob/main/receiver/otlpreceiver/config.md#configtls-tlsserversetting). Addition TLS configuration of OTEL components can be found [here](https://github.com/open-telemetry/opentelemetry-collector/tree/main/config/configtls).
+Additional receiver configuration can be added to support TLS communication for traces being sent to Tempo. The receiver configuration is pulled in from the Open Telemetry collector, and is [documented upstream here](https://github.com/open-telemetry/opentelemetry-collector/blob/main/receiver/otlpreceiver/config.md#configtls-tlsserversetting).
+Addition TLS configuration of OTEL components can be found [here](https://github.com/open-telemetry/opentelemetry-collector/tree/main/config/configtls).
 
 An example `tls` block might look like the following:
 

docs/sources/tempo/configuration/parquet.md

@@ -8,7 +8,7 @@ weight: 300
 # Apache Parquet block format
 
 Tempo has a default columnar block format based on Apache Parquet.
-This format is required for tags-based search as well as [TraceQL]({{< relref "../traceql" >}}), the query language for traces.
+This format is required for tags-based search as well as [TraceQL](../../traceql/), the query language for traces.
 The columnar block format improves search performance and enables an ecosystem of tools, including [Tempo CLI](https://grafana.com/docs/tempo/<TEMPO_VERSION>/operations/tempo_cli/#analyse-blocks), to access the underlying trace data.
 
 For more information, refer to the [Parquet design document](https://github.com/grafana/tempo/blob/main/docs/design-proposals/2022-04%20Parquet.md) and [Issue 1480](https://github.com/grafana/tempo/issues/1480).
@@ -53,7 +53,7 @@ To re-enable the default `vParquet4` format, remove the block version option fro
 ## Parquet configuration parameters
 
 Some parameters in the Tempo configuration are specific to Parquet.
-For more information, refer to the [storage configuration documentation]({{< relref "../configuration#storage" >}}).
+For more information, refer to the [storage configuration documentation](../#storage).
 
 ### Trace search parameters
 

docs/sources/tempo/configuration/polling.md

@@ -35,7 +35,7 @@ storage:
         [blocklist_poll_stale_tenant_index: <duration>]
 ```
 
-Due to the mechanics of the [tenant index]({{< relref "../operations/monitor/polling" >}}), the blocklist will be stale by
+Due to the mechanics of the [tenant index](../../operations/monitor/polling/), the blocklist will be stale by
 at most 2 times the configured `blocklist_poll` duration. There are two configuration options that need to be balanced
 against the `blockist_poll` to handle this:
 

docs/sources/tempo/configuration/usage-tracker.md

@@ -0,0 +1,80 @@
+---
+title: Usage tracker
+description: Learn how to configure the usage tracker for cost attribution.
+weight: 700
+aliases:
+- /docs/tempo/configuration/usage-tracker   
+---
+
+# Usage tracker
+
+The usage tracker accurately tracks the amount of ingested traffic using a set of custom labels on a per-tenant basis, providing fine-grained control over your data.
+
+Use the `cost_attributes` option to configure usage trackers in the distributor, which expose metrics of ingested traffic grouped by configurable attributes exposed on `/usage_metrics`.
+
+
+## Enable the usage tracker
+
+To use this feature, you need to enable it in the [distributor](https://grafana.com/docs/tempo/<TEMPO_VERSION>/configuration/#distributor) and configure the overrides to handle the data. 
+
+1. Open your configuration file, for example, `tempo.yaml`. 
+1. In the `#distributor` section, locate the `cost_attribution` line. If it is not present, then copy and paste the example below into your distributor section and update the options. 
+
+```
+ usage:
+        cost_attribution:
+            # Enables the "cost-attribution" usage tracker. Per-tenant attributes are configured in overrides.
+            [enabled: <boolean> | default = false]
+            # Maximum number of series per tenant.
+            [max_cardinality: <int> | default = 10000]
+            # Interval after which a series is considered stale and will be deleted from the registry.
+            # Once a metrics series is deleted, it won't be emitted anymore, keeping active series low.
+            [stale_duration: <duration> | default = 15m0s]
+```
+
+
+### Configure overrides 
+
+You also need to configure the dimensions to break down your the usage data in the standard overrides. 
+In the overrides section, you can define attributes to group ingested data by and you can rename and combine attributes. 
+
+For more information, refer to the [the standard overrides](https://grafana.com/docs/tempo/<TEMPO_VERSION>/configuration/#standard-overrides) section of the Configuration documentation. 
+
+In this example, usage data is grouped by `service.name`. 
+
+```
+# Overrides configuration block
+overrides:
+  # Global ingestion limits configurations
+  defaults:
+    # Cost attribution usage tracker configuration
+    cost_attribution:
+      dimensions: 
+        - service.name
+```
+
+You can also configure per tenant in the [runtime-overrides](https://grafana.com/docs/tempo/<TEMPO_VERSION>/configuration/#runtime-overrides) or in the [user-configurable-overrides](https://grafana.com/docs/tempo/<TEMPO_VERSION>/configuration/#user-configurable-overrides).
+
+
+
+## Review usage metrics
+
+Once everything is configured, the usage metrics are exposed in the distributor:
+
+
+```
+GET /usage_metrics
+```
+
+Example:
+```
+curl http://localhost:3200/usage_metrics
+# HELP tempo_usage_tracker_bytes_received_total bytes total received with these attributes
+# TYPE tempo_usage_tracker_bytes_received_total counter
+tempo_usage_tracker_bytes_received_total{service_name="article-service",tenant="single-tenant",tracker="cost-attribution"} 7327
+tempo_usage_tracker_bytes_received_total{service_name="auth-service",tenant="single-tenant",tracker="cost-attribution"} 8938
+tempo_usage_tracker_bytes_received_total{service_name="billing-service",tenant="single-tenant",tracker="cost-attribution"} 2401
+tempo_usage_tracker_bytes_received_total{service_name="cart-service",tenant="single-tenant",tracker="cost-attribution"} 4116
+tempo_usage_tracker_bytes_received_total{service_name="postgres",tenant="single-tenant",tracker="cost-attribution"} 3571
+tempo_usage_tracker_bytes_received_total{service_name="shop-backend",tenant="single-tenant",tracker="cost-attribution"} 17619
+```

docs/sources/tempo/configuration/use-trace-data.md

@@ -15,8 +15,8 @@ Grafana Cloud comes pre-configured with a Tempo data source.
 If you are using Grafana on-prem, you need to [set up the Tempo data source](/docs/grafana/<GRAFANA_VERSION>/datasources/tempo).
 
 {{< admonition type="tip" >}}
-If you want to explore tracing data in Grafana, try the [Intro to Metrics, Logs, Traces, and Profiling example]({{< relref "../getting-started/docker-example" >}}).
-{{% /admonition %}}
+If you want to explore tracing data in Grafana, try the [Intro to Metrics, Logs, Traces, and Profiling example](../../getting-started/docker-example/).
+{{< /admonition >}}
 
 This video explains how to add data sources, including Loki, Tempo, and Mimir, to Grafana and Grafana Cloud. Tempo data source set up starts at 4:58 in the video.
 
@@ -31,10 +31,10 @@ To configure Tempo with Grafana:
 1. Point the Grafana data source at your Tempo query frontend (or monolithic mode Tempo).
 1. Enter the URL: `http://<tempo hostname>:<http port number>`. For most of [the Tempo examples](https://github.com/grafana/tempo/tree/main/example/docker-compose) the following works.
 
-The port of 3200 is a common port used in our examples. Tempo default HTTP port is 80.
+The port of 3200 is a common port used in our examples. Tempo default HTTP port is 3200.
 
 ## Query the data source
 
 Refer to [Tempo in Grafana](https://grafana.com/docs/tempo/<TEMPO_VERSION>/getting-started/tempo-in-grafana) for an overview about how tracing data can be viewed and queried in Grafana.
 
-For information on querying the Tempo data source, refer to [Tempo query editor](https://grafana.com/docs/grafana/<GRAFANA_VERSION>/datasources/tempo/query-editor/).
+For information on querying the Tempo data source, refer to [Tempo query editor](https://grafana.com/docs/grafana/<GRAFANA_VERSION>/datasources/tempo/query-editor/).