Tempo 3.0: Integration Tests Rewrite (#6088)

* First pass test harness

Signed-off-by: Joe Elliott <number101010@gmail.com>

* limits - passed

Signed-off-by: Joe Elliott <number101010@gmail.com>

* metrics generator - passed

Signed-off-by: Joe Elliott <number101010@gmail.com>

* harness cleanup/fix target info test

Signed-off-by: Joe Elliott <number101010@gmail.com>

* deployments - passed

Signed-off-by: Joe Elliott <number101010@gmail.com>

* receivers - passed

Signed-off-by: Joe Elliott <number101010@gmail.com>

* harness cleanup

Signed-off-by: Joe Elliott <number101010@gmail.com>

* storage - passed

Signed-off-by: Joe Elliott <number101010@gmail.com>

* api - passing

Signed-off-by: Joe Elliott <number101010@gmail.com>

* cleanup and CI

Signed-off-by: Joe Elliott <number101010@gmail.com>

* Additional tests

Signed-off-by: Joe Elliott <number101010@gmail.com>

* it's me and you linter. how many times are you going to do this to me?

Signed-off-by: Joe Elliott <number101010@gmail.com>

* lint and tests

Signed-off-by: Joe Elliott <number101010@gmail.com>

* restored live store partition tests

Signed-off-by: Joe Elliott <number101010@gmail.com>

* massive lint

Signed-off-by: Joe Elliott <number101010@gmail.com>

* cleanup

Signed-off-by: Joe Elliott <number101010@gmail.com>

* lint >.<

Signed-off-by: Joe Elliott <number101010@gmail.com>

* wups

Signed-off-by: Joe Elliott <number101010@gmail.com>

* cleanup and stabilizing

Signed-off-by: Joe Elliott <number101010@gmail.com>

* cleanup

Signed-off-by: Joe Elliott <number101010@gmail.com>

* omg

Signed-off-by: Joe Elliott <number101010@gmail.com>

* added config file. removed empty file. deleted some debug code

Signed-off-by: Joe Elliott <number101010@gmail.com>

* even more fixes

Signed-off-by: Joe Elliott <number101010@gmail.com>

* maybe this works? maybe it was all local networking issues

Signed-off-by: Joe Elliott <number101010@gmail.com>

* review

Signed-off-by: Joe Elliott <number101010@gmail.com>

* backed off huge write timeout. added missing config file

Signed-off-by: Joe Elliott <number101010@gmail.com>

* restore disable parallelism setting. restore beefy write timeout

Signed-off-by: Joe Elliott <number101010@gmail.com>

* apparently another test relied on the http receiver

Signed-off-by: Joe Elliott <number101010@gmail.com>

* livestore avail zone!

Signed-off-by: Joe Elliott <number101010@gmail.com>

* add support for v1 traces and tags endpoints

Signed-off-by: Joe Elliott <number101010@gmail.com>

* review

Signed-off-by: Joe Elliott <number101010@gmail.com>

* changed the location of the changelog entry to avoid conflicts

Signed-off-by: Joe Elliott <number101010@gmail.com>

---------

Signed-off-by: Joe Elliott <number101010@gmail.com>

.github/workflows/ci.yml

@@ -101,20 +101,59 @@ jobs:
           include-hidden-files: true
           if-no-files-found: error
 
+  integration-tests-validation:
+    name: Validate integration test coverage
+    runs-on: ubuntu-24.04
+    steps:
+      - name: Check out code
+        uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6
+        with:
+          persist-credentials: false
+      - name: Validate all integration folders are covered by CI
+        run: |
+          # Define expected folders that have corresponding test-e2e-* targets
+          EXPECTED_FOLDERS=("api" "limits" "metrics-generator" "operations" "storage" "util")
+
+          # Get actual folders in integration/ (excluding hidden files)
+          ACTUAL_FOLDERS=($(ls -d integration/*/ 2>/dev/null | xargs -n1 basename | sort))
+
+          # Convert arrays to strings for comparison
+          EXPECTED_SORTED=$(printf '%s\n' "${EXPECTED_FOLDERS[@]}" | sort | tr '\n' ' ')
+          ACTUAL_SORTED=$(printf '%s\n' "${ACTUAL_FOLDERS[@]}" | sort | tr '\n' ' ')
+
+          if [ "$EXPECTED_SORTED" != "$ACTUAL_SORTED" ]; then
+            echo "❌ ERROR: Integration folder mismatch detected!"
+            echo ""
+            echo "Expected folders (covered by CI):"
+            printf '  - %s\n' "${EXPECTED_FOLDERS[@]}" | sort
+            echo ""
+            echo "Actual folders in ./integration:"
+            printf '  - %s\n' "${ACTUAL_FOLDERS[@]}" | sort
+            echo ""
+            echo "If you added a new integration test folder, you must:"
+            echo "  1. Add a corresponding 'test-e2e-<folder-name>' target to the Makefile"
+            echo "  2. Add the target to the matrix in .github/workflows/ci.yml (integration-tests job)"
+            echo "  3. Update the EXPECTED_FOLDERS list in this validation step"
+            exit 1
+          fi
+
+          echo "✅ All integration folders are covered by CI"
+
   integration-tests:
     name: Run integration tests
     runs-on: ubuntu-24.04
+    needs: integration-tests-validation
     strategy:
       fail-fast: false
       matrix:
         test-target:
           [
-            test-e2e,
+            test-e2e-operations,
             test-e2e-api,
-            test-integration-poller,
-            test-integration-backendscheduler,
-            test-e2e-deployments,
-            test-e2e-ingest,
+            test-e2e-limits,
+            test-e2e-metrics-generator,
+            test-e2e-storage,
+            test-e2e-util,
           ]
 
     steps:

.golangci.yml

@@ -57,6 +57,9 @@ linters:
           - gosec
           - forbidigo
         path: (.+)_test\.go
+      - linters:
+          - revive
+        text: 'var-naming: avoid meaningless package names'
     paths:
       - third_party$
       - builtin$

CHANGELOG.md

@@ -18,6 +18,7 @@
   have expired and been deleted from the backend, or else it will encounter read errors after upgrading to this release.
 * [CHANGE] Remove busybox from Tempo image to make it more minimal and prevent future vulnerabilities [#5717](https://github.com/grafana/tempo/pull/5717) (@carles-grafana)
 * [CHANGE] Allow RetryInfo to be disabled in per tenant overrides [#5741](https://github.com/grafana/tempo/pull/5741) (@electron0zero)
+* [CHANGE] **BREAKING CHANGE** Added `scope` and `query` params while rewriting integration tests to the Tempo HTTP client `SearchTagsV2WithRange` function.
 * [CHANGE] Added a single binary 3.0 mode `--target=all-3.0` to begin testing single binary 3.0 and updating integration tests. [#6021](https://github.com/grafana/tempo/pull/6021) (@joe-elliott)
   This will be removed in 3.0 and become the standard single binary mode.
 * [CHANGE] Upgrade Tempo to Go 1.25.4 [#5939](https://github.com/grafana/tempo/pull/5939) [#6001](https://github.com/grafana/tempo/pull/6001) (@ruslan-mikhailov)

Makefile

@@ -1,7 +1,7 @@
 # Adapted from https://www.thapaliya.com/en/writings/well-documented-makefiles/
 .PHONY: help
 help:  ## Display this help
-	@awk 'BEGIN {FS = ":.*##"; printf "\nUsage:\n  make \033[36m<target>\033[0m\n"} /^[a-zA-Z_-]+:.*?##/ { printf "  \033[36m%-15s\033[0m %s\n", $$1, $$2 } /^##@/ { printf "\n\033[1m%s\033[0m\n", substr($$0, 5) } ' $(MAKEFILE_LIST)
+	@awk 'BEGIN {FS = ":.*##"; printf "\nUsage:\n  make \033[36m<target>\033[0m\n"} /^[[:alnum:]_-]+:.*##/ { printf "  \033[36m%-15s\033[0m %s\n", $$1, $$2 } /^##@/ { printf "\n\033[1m%s\033[0m\n", substr($$0, 5) } ' $(MAKEFILE_LIST)
 
 .DEFAULT_GOAL:=help
 
@@ -102,7 +102,7 @@ exe:
 exe-debug:
 	BUILD_DEBUG=1 GOOS=linux make $(COMPONENT)
 
-##@  Testin' and Lintin'
+##@ Unit Tests
 
 .PHONY: test
 test: ## Run tests
@@ -120,7 +120,7 @@ test-with-cover: tools ## Run tests with code coverage
 
 # tests in pkg
 .PHONY: test-with-cover-pkg
-test-with-cover-pkg: tools  ##  Run Tempo packages' tests with code coverage
+test-with-cover-pkg: tools  ## Run Tempo packages' tests with code coverage
 	mkdir -p $(COVERAGE_DIR)
 	$(GOTEST) $(GOTEST_OPT) -coverprofile=$(COVERAGE_DIR)/pkg.out $(shell go list $(sort $(dir $(shell find . -name '*.go' -path './pkg*/*' -type f | sort))))
 
@@ -142,42 +142,51 @@ test-with-cover-others: tools ## Run other tests with code coverage
 	mkdir -p $(COVERAGE_DIR)
 	$(GOTEST) $(GOTEST_OPT) -coverprofile=$(COVERAGE_DIR)/others.out $(shell go list $(sort $(dir $(OTHERS_SRC))))
 
+##@ End to End Tests
+
 # runs e2e tests in the top level integration/e2e directory
 .PHONY: test-e2e
-test-e2e: tools docker-tempo docker-tempo-query  ## Run end to end tests
-	$(GOTEST) -v $(GOTEST_OPT) ./integration/e2e
+test-e2e: tools docker-tempo docker-tempo-query test-e2e-operations test-e2e-api test-e2e-limits test-e2e-metrics-generator test-e2e-storage test-e2e-util ## Run all e2e tests
+	@echo "All e2e tests completed"
 
-# runs only deployment modes e2e tests
-.PHONY: test-e2e-deployments
-test-e2e-deployments: tools docker-tempo docker-tempo-query ## Run end to end tests for deployments
-	$(GOTEST) -v $(GOTEST_OPT) ./integration/e2e/deployments
+# runs only operations e2e tests
+.PHONY: test-e2e-operations
+test-e2e-operations: tools docker-tempo docker-tempo-query ## Run operations e2e tests
+	$(GOTEST) -v $(GOTEST_OPT) ./integration/operations
 
 # runs only api e2e tests
 .PHONY: test-e2e-api
-test-e2e-api: tools docker-tempo docker-tempo-query ## Run end to end tests for api
-	$(GOTEST) -v $(GOTEST_OPT) ./integration/e2e/api
+test-e2e-api: tools docker-tempo docker-tempo-query ## Run api e2e tests
+	$(GOTEST) -v $(GOTEST_OPT) ./integration/api
 
-# runs only poller integration tests
-.PHONY: test-integration-poller
-test-integration-poller: tools ## Run poller integration tests
-	$(GOTEST) -v $(GOTEST_OPT) ./integration/poller
+## runs only poller integration tests
+.PHONY: test-e2e-limits
+test-e2e-limits: tools tools docker-tempo ## Run limits e2e tests
+	$(GOTEST) -v $(GOTEST_OPT) ./integration/limits
 
-# runs only backendscheduler integration tests
-.PHONY: test-integration-backendscheduler
-test-integration-backendscheduler: tools docker-tempo ## Run backend-scheduler integration tests
-	$(GOTEST) -v $(GOTEST_OPT) ./integration/backendscheduler
+# runs only metrics-generator integration tests
+.PHONY: test-e2e-metrics-generator
+test-e2e-metrics-generator: tools docker-tempo ## Run metrics-generator e2e tests
+	$(GOTEST) -v $(GOTEST_OPT) ./integration/metrics-generator
 
 # runs only ingest integration tests
-.PHONY: test-e2e-ingest
-test-e2e-ingest: tools docker-tempo ## Run end to end tests for ingest
-	$(GOTEST) -v $(GOTEST_OPT) ./integration/e2e/ingest
+.PHONY: test-e2e-storage
+test-e2e-storage: tools docker-tempo ## Run storage e2e tests
+	$(GOTEST) -v $(GOTEST_OPT) ./integration/storage
+
+# runs only ingest integration tests
+.PHONY: test-e2e-util
+test-e2e-util: tools docker-tempo ## Run unit tests on the e2e test harness
+	$(GOTEST) -v $(GOTEST_OPT) ./integration/util
 
 # test-all use a docker image so build it first to make sure we're up to date
-.PHONY: test-all ## Run all tests
-test-all: test-with-cover test-e2e test-e2e-deployments test-e2e-api test-integration-poller test-integration-backendscheduler test-e2e-ingest
+.PHONY: test-all
+test-all: test-with-cover test-e2e ## Run all tests
+
+##@ Linters/Formatters
 
 .PHONY: fmt check-fmt
-fmt: tools-image ## Check fmt
+fmt: tools-image ## Format codebase with gofumpt and goimports
 	@$(TOOLS_CMD) gofumpt -w $(FILES_TO_FMT)
 	@$(TOOLS_CMD) goimports -w $(FILES_TO_FMT)
 
@@ -185,14 +194,14 @@ check-fmt: fmt
 	@git diff --exit-code -- $(FILES_TO_FMT)
 
 .PHONY: jsonnetfmt check-jsonnetfmt ## Check jsonnetfmt
-jsonnetfmt: tools-image
+jsonnetfmt: tools-image ## Format jsonnet codebase with jsonnetfmt
 	@$(TOOLS_CMD) jsonnetfmt -i $(FILES_TO_JSONNETFMT)
 
 check-jsonnetfmt: jsonnetfmt
 	@git diff --exit-code -- $(FILES_TO_JSONNETFMT)
 
 .PHONY: lint
-lint: # linting
+lint: ## Lint codebase with golangci-lint
 ifneq ($(base),)
 	$(LINT_CMD) $(LINT) run --config .golangci.yml --new-from-rev=$(base)
 else
@@ -255,7 +264,7 @@ ifndef COMPONENT
 	$(error COMPONENT variable was not defined)
 endif
 
-##@ Gen Proto
+##@ Code Generation
 
 PROTOC = docker run --rm -u ${shell id -u} -v${PWD}:${PWD} -w${PWD} ${DOCKER_PROTOBUF_IMAGE} --proto_path=${PWD}
 PROTO_INTERMEDIATE_DIR = pkg/.patched-proto
@@ -310,8 +319,6 @@ gen-proto:  ## Generate proto files
 
 	rm -rf $(PROTO_INTERMEDIATE_DIR)
 
-##@ Gen Traceql
-
 .PHONY: gen-traceql 
 gen-traceql: ## Generate traceql 
 	docker run --rm -v${PWD}:/src/loki ${LOKI_BUILD_IMAGE} gen-traceql-local
@@ -320,9 +327,6 @@ gen-traceql: ## Generate traceql
 gen-traceql-local: ## Generate traceq local
 	goyacc -l -o pkg/traceql/expr.y.go pkg/traceql/expr.y && rm -f y.output
 
-
-##@ Gen Parquet-Query
-
 .PHONY: gen-parquet-query
 gen-parquet-query:  ## Generate Parquet query 
 	go run ./pkg/parquetquerygen/predicates.go > ./pkg/parquetquery/predicates.gen.go

cmd/tempo-cli/cmd-query-search-tag-values.go

@@ -20,7 +20,7 @@ type querySearchTagValuesCmd struct {
 	Start    string `arg:"" optional:"" help:"start time in ISO8601 format"`
 	End      string `arg:"" optional:"" help:"end time in ISO8601 format"`
 
-	Query      string `help:"TraceQL query to filter attribute results by (supported by GRPC only)"`
+	Query      string `help:"TraceQL query to filter attribute results by"`
 	OrgID      string `help:"optional orgID"`
 	UseGRPC    bool   `help:"stream search results over GRPC"`
 	PathPrefix string `help:"string to prefix all http paths with"`
@@ -62,9 +62,9 @@ func (cmd *querySearchTagValuesCmd) searchHTTP(start, end int64) error {
 	var tags *tempopb.SearchTagValuesV2Response
 	var err error
 	if start != 0 || end != 0 {
-		tags, err = client.SearchTagValuesV2WithRange(cmd.Tag, start, end)
+		tags, err = client.SearchTagValuesV2WithRange(cmd.Tag, cmd.Query, start, end)
 	} else {
-		tags, err = client.SearchTagValuesV2(cmd.Tag, "")
+		tags, err = client.SearchTagValuesV2(cmd.Tag, cmd.Query)
 	}
 
 	if err != nil {

cmd/tempo-cli/cmd-query-search-tags.go

@@ -60,7 +60,7 @@ func (cmd *querySearchTagsCmd) searchHTTP(start, end int64) error {
 	var tags *tempopb.SearchTagsV2Response
 	var err error
 	if start != 0 || end != 0 {
-		tags, err = client.SearchTagsV2WithRange(start, end)
+		tags, err = client.SearchTagsV2WithRange("", "", start, end)
 	} else {
 		tags, err = client.SearchTagsV2()
 	}

cmd/tempo-vulture/main.go

@@ -701,7 +701,7 @@ func queryMetrics(client httpclient.TempoHTTPClient, seed time.Time, config vult
 
 	resp, err := client.MetricsQueryRange(
 		fmt.Sprintf(`{.%s = "%s"} | count_over_time()`, attr.Key, util.StringifyAnyValue(attr.Value)),
-		int(start), int(end), "1m", 0,
+		start, end, "1m", 0,
 	)
 	if err != nil {
 		logger.Error("failed to query metrics", zap.Error(err))

cmd/tempo-vulture/main_test.go

@@ -221,7 +221,7 @@ func TestTraceIsReady(t *testing.T) {
 	assert.False(t, ready, "trace should not be ready yet")
 
 	startTime = time.Date(2007, 1, 1, 12, 0, 0, 0, time.UTC)
-	ready = traceIsReady(ti, seed.Add(2*longWriteBackoff), startTime, writeBackoff, longWriteBackoff)
+	ready = traceIsReady(ti, seed.Add(5*longWriteBackoff), startTime, writeBackoff, longWriteBackoff)
 	assert.True(t, ready, "trace should be ready now")
 }
 

cmd/tempo-vulture/mocks.go

@@ -8,7 +8,6 @@ import (
 
 	userconfigurableoverrides "github.com/grafana/tempo/modules/overrides/userconfigurable/client"
 	thrift "github.com/jaegertracing/jaeger-idl/thrift-gen/jaeger"
-	"github.com/jaegertracing/jaeger-idl/thrift-gen/zipkincore"
 
 	"github.com/grafana/tempo/pkg/tempopb"
 )
@@ -20,10 +19,6 @@ type MockReporter struct {
 	m sync.Mutex
 }
 
-func (r *MockReporter) EmitZipkinBatch(_ context.Context, _ []*zipkincore.Span) error {
-	return r.err
-}
-
 func (r *MockReporter) EmitBatch(_ context.Context, b *thrift.Batch) error {
 	if r.err == nil {
 		r.m.Lock()
@@ -41,20 +36,22 @@ func (r *MockReporter) GetEmittedBatches() []*thrift.Batch {
 }
 
 type MockHTTPClient struct {
-	err            error
-	resp           http.Response
-	traceResp      *tempopb.Trace
-	requestsCount  int
-	searchResponse []*tempopb.TraceSearchMetadata
-	searchesCount  int
-	metricsResp    *tempopb.QueryRangeResponse
-	metricsCount   int
+	err                 error
+	resp                http.Response
+	traceResp           *tempopb.Trace
+	requestsCount       int
+	searchResponse      []*tempopb.TraceSearchMetadata
+	searchesCount       int
+	metricsResp         *tempopb.QueryRangeResponse
+	metricsCount        int
+	metricsInstantResp  *tempopb.QueryInstantResponse
+	metricsInstantCount int
 	// We need the lock to control concurrent accesses to shared variables in the tests
 	m sync.Mutex
 }
 
 //nolint:all
-func (m *MockHTTPClient) DeleteOverrides(version string) error {
+func (m *MockHTTPClient) DeleteOverrides(_ string) error {
 	panic("unimplemented")
 }
 
@@ -69,12 +66,12 @@ func (m *MockHTTPClient) GetOverrides() (*userconfigurableoverrides.Limits, stri
 }
 
 //nolint:all
-func (m *MockHTTPClient) MetricsSummary(query string, groupBy string, start int64, end int64) (*tempopb.SpanMetricsSummaryResponse, error) {
+func (m *MockHTTPClient) MetricsSummary(_ string, _ string, _ int64, _ int64) (*tempopb.SpanMetricsSummaryResponse, error) {
 	panic("unimplemented")
 }
 
 //nolint:all
-func (m *MockHTTPClient) MetricsQueryRange(query string, start, end int, step string, exemplars int) (*tempopb.QueryRangeResponse, error) {
+func (m *MockHTTPClient) MetricsQueryRange(_ string, _, _ int64, _ string, _ int) (*tempopb.QueryRangeResponse, error) {
 	if m.err != nil {
 		return nil, m.err
 	}
@@ -85,12 +82,23 @@ func (m *MockHTTPClient) MetricsQueryRange(query string, start, end int, step st
 }
 
 //nolint:all
-func (m *MockHTTPClient) PatchOverrides(limits *userconfigurableoverrides.Limits) (*userconfigurableoverrides.Limits, string, error) {
+func (m *MockHTTPClient) MetricsQueryInstant(_ string, _, _ int64, _ int) (*tempopb.QueryInstantResponse, error) {
+	if m.err != nil {
+		return nil, m.err
+	}
+	m.m.Lock()
+	defer m.m.Unlock()
+	m.metricsInstantCount++
+	return m.metricsInstantResp, nil
+}
+
+//nolint:all
+func (m *MockHTTPClient) PatchOverrides(_ *userconfigurableoverrides.Limits) (*userconfigurableoverrides.Limits, string, error) {
 	panic("unimplemented")
 }
 
 //nolint:all
-func (m *MockHTTPClient) QueryTrace(id string) (*tempopb.Trace, error) {
+func (m *MockHTTPClient) QueryTrace(_ string) (*tempopb.Trace, error) {
 	if m.err != nil {
 		return nil, m.err
 	}
@@ -133,7 +141,7 @@ func (m *MockHTTPClient) SearchTagValuesV2(key string, query string) (*tempopb.S
 }
 
 //nolint:all
-func (m *MockHTTPClient) SearchTagValuesV2WithRange(tag string, start int64, end int64) (*tempopb.SearchTagValuesV2Response, error) {
+func (m *MockHTTPClient) SearchTagValuesV2WithRange(tag, query string, start int64, end int64) (*tempopb.SearchTagValuesV2Response, error) {
 	panic("unimplemented")
 }
 
@@ -148,7 +156,7 @@ func (m *MockHTTPClient) SearchTagsV2() (*tempopb.SearchTagsV2Response, error) {
 }
 
 //nolint:all
-func (m *MockHTTPClient) SearchTagsV2WithRange(start int64, end int64) (*tempopb.SearchTagsV2Response, error) {
+func (m *MockHTTPClient) SearchTagsV2WithRange(scope, query string, start int64, end int64) (*tempopb.SearchTagsV2Response, error) {
 	panic("unimplemented")
 }
 

integration/api/api_test.go

@@ -0,0 +1,766 @@
+package api
+
+import (
+	"bytes"
+	"errors"
+	"fmt"
+	"io"
+	"net/http"
+	"slices"
+	"sort"
+	"testing"
+	"time"
+
+	"github.com/gogo/protobuf/jsonpb"
+	"github.com/grafana/tempo/integration/util"
+	"github.com/grafana/tempo/pkg/collector"
+	"github.com/grafana/tempo/pkg/search"
+	"github.com/grafana/tempo/pkg/tempopb"
+	tempoUtil "github.com/grafana/tempo/pkg/util"
+	"github.com/stretchr/testify/require"
+	"google.golang.org/grpc/codes"
+	"google.golang.org/grpc/status"
+)
+
+type batchTmpl struct {
+	spanCount                  int
+	name                       string
+	resourceAttVal, spanAttVal string
+	resourceAttr, SpanAttr     string
+}
+
+func TestTagEndpoints(t *testing.T) {
+	util.RunIntegrationTests(t, util.TestHarnessConfig{
+		Components: util.ComponentsRecentDataQuerying | util.ComponentsBackendQuerying,
+	}, func(h *util.TempoHarness) {
+		h.WaitTracesWritable(t)
+
+		batches := []batchTmpl{
+			{spanCount: 2, name: "foo", resourceAttr: "firstRes", resourceAttVal: "bar", SpanAttr: "firstSpan", spanAttVal: "bar"},
+			{spanCount: 2, name: "baz", resourceAttr: "secondRes", resourceAttVal: "qux", SpanAttr: "secondSpan", spanAttVal: "qux"},
+			{spanCount: 2, name: "foo", resourceAttr: "twoRes", resourceAttVal: "bar", SpanAttr: "twoSpan", spanAttVal: "bar"},
+			{spanCount: 2, name: "baz", resourceAttr: "twoRes", resourceAttVal: "qux", SpanAttr: "twoSpan", spanAttVal: "qux"},
+		}
+
+		for _, b := range batches {
+			batch := util.MakeThriftBatchWithSpanCountAttributeAndName(b.spanCount, b.name, b.resourceAttVal, b.spanAttVal, b.resourceAttr, b.SpanAttr)
+			require.NoError(t, h.WriteJaegerBatch(batch, ""))
+		}
+
+		// wait for the 2 traces to be written to the WAL
+		h.WaitTracesQueryable(t, 4)
+
+		tagsTestCases := buildSearchTagsV2TestCases(batches)
+		tagValuesTestCases := buildSearchTagValuesV2TestCases(batches)
+
+		for _, tc := range tagsTestCases {
+			t.Run("tags_wal_"+tc.name, func(t *testing.T) {
+				callSearchTagsV2AndAssert(t, h, tc.scope, tc.query, tc.expected, 0, 0)
+			})
+		}
+
+		for _, tc := range tagValuesTestCases {
+			t.Run("values_wal_"+tc.name, func(t *testing.T) {
+				callSearchTagValuesV2AndAssert(t, h, tc.tagName, tc.query, tc.expected, 0, 0)
+			})
+		}
+
+		// Test V1 API endpoints (backwards compatibility)
+		t.Run("tags_v1_wal", func(t *testing.T) {
+			expectedTags := []string{"firstRes", "firstSpan", "secondRes", "secondSpan", "service.name", "twoRes", "twoSpan"}
+			callSearchTagsAndAssert(t, h, expectedTags, 0, 0)
+		})
+
+		t.Run("tag_values_v1_wal", func(t *testing.T) {
+			expectedValues := []string{"my-service"}
+			callSearchTagValuesAndAssert(t, h, "service.name", expectedValues, 0, 0)
+		})
+
+		// wait for 4 objects to be written to the backend
+		h.WaitTracesWrittenToBackend(t, 4)
+		h.ForceBackendQuerying(t)
+
+		// Assert tags on storage backend
+		now := time.Now()
+		start := now.Add(-2 * time.Hour)
+		end := now.Add(2 * time.Hour)
+
+		for _, tc := range tagsTestCases {
+			t.Run("tags_backend_"+tc.name, func(t *testing.T) {
+				callSearchTagsV2AndAssert(t, h, tc.scope, tc.query, tc.expected, start.Unix(), end.Unix())
+			})
+		}
+
+		for _, tc := range tagValuesTestCases {
+			t.Run("values_backend_"+tc.name, func(t *testing.T) {
+				callSearchTagValuesV2AndAssert(t, h, tc.tagName, tc.query, tc.expected, start.Unix(), end.Unix())
+			})
+		}
+
+		// Test V1 API endpoints on backend (backwards compatibility)
+		t.Run("tags_v1_backend", func(t *testing.T) {
+			expectedTags := []string{"firstRes", "firstSpan", "secondRes", "secondSpan", "service.name", "twoRes", "twoSpan"}
+			callSearchTagsAndAssert(t, h, expectedTags, start.Unix(), end.Unix())
+		})
+
+		t.Run("tag_values_v1_backend", func(t *testing.T) {
+			expectedValues := []string{"my-service"}
+			callSearchTagValuesAndAssert(t, h, "service.name", expectedValues, start.Unix(), end.Unix())
+		})
+	})
+}
+
+func buildSearchTagsV2TestCases(batches []batchTmpl) []struct {
+	name     string
+	query    string
+	scope    string
+	expected *tempopb.SearchTagsV2Response
+} {
+	tcs := []struct {
+		name     string
+		query    string
+		scope    string
+		expected *tempopb.SearchTagsV2Response
+	}{
+		{
+			name:  "no filtering",
+			query: "",
+			scope: "none",
+			expected: &tempopb.SearchTagsV2Response{
+				Scopes: []*tempopb.SearchTagsV2Scope{
+					{
+						Name: "span",
+						Tags: []string{batches[0].SpanAttr, batches[1].SpanAttr, batches[2].SpanAttr},
+					},
+					{
+						Name: "resource",
+						Tags: []string{batches[0].resourceAttr, batches[1].resourceAttr, batches[2].resourceAttr, "service.name"},
+					},
+				},
+			},
+		},
+		{
+			name:  "invalid query",
+			query: ` { a="test" } `,
+			scope: "none",
+			// same results as no filtering
+			expected: &tempopb.SearchTagsV2Response{
+				Scopes: []*tempopb.SearchTagsV2Scope{
+					{
+						Name: "span",
+						Tags: []string{batches[0].SpanAttr, batches[1].SpanAttr, batches[2].SpanAttr},
+					},
+					{
+						Name: "resource",
+						Tags: []string{batches[0].resourceAttr, batches[1].resourceAttr, batches[2].resourceAttr, "service.name"},
+					},
+				},
+			},
+		},
+		{
+			name:  "first batch - resource",
+			query: fmt.Sprintf(`{ name="%s" }`, batches[0].name),
+			scope: "resource",
+			expected: &tempopb.SearchTagsV2Response{
+				Scopes: []*tempopb.SearchTagsV2Scope{
+					{
+						Name: "resource",
+						Tags: []string{batches[0].resourceAttr, batches[2].resourceAttr, "service.name"},
+					},
+				},
+			},
+		},
+		{
+			name:  "second batch with incomplete query - span",
+			query: fmt.Sprintf(`{ name="%s" && span.twoSpan = }`, batches[1].name),
+			scope: "span",
+			expected: &tempopb.SearchTagsV2Response{
+				Scopes: []*tempopb.SearchTagsV2Scope{
+					{
+						Name: "span",
+						Tags: []string{batches[1].SpanAttr, batches[3].SpanAttr},
+					},
+				},
+			},
+		},
+		{
+			name:  "first batch - resource att - span",
+			query: fmt.Sprintf(`{ resource.%s="%s" }`, batches[0].resourceAttr, batches[0].resourceAttVal),
+			scope: "span",
+			expected: &tempopb.SearchTagsV2Response{
+				Scopes: []*tempopb.SearchTagsV2Scope{
+					{
+						Name: "span",
+						Tags: []string{batches[0].SpanAttr},
+					},
+				},
+			},
+		},
+		{
+			name:  "first batch - resource att - resource",
+			query: fmt.Sprintf(`{ resource.%s="%s" }`, batches[0].resourceAttr, batches[0].resourceAttVal),
+			scope: "resource",
+			expected: &tempopb.SearchTagsV2Response{
+				Scopes: []*tempopb.SearchTagsV2Scope{
+					{
+						Name: "resource",
+						Tags: []string{batches[0].resourceAttr, "service.name"},
+					},
+				},
+			},
+		},
+		{
+			name:  "second batch - resource attribute - span",
+			query: fmt.Sprintf(`{ resource.%s="%s" }`, batches[1].resourceAttr, batches[1].resourceAttVal),
+			scope: "span",
+			expected: &tempopb.SearchTagsV2Response{
+				Scopes: []*tempopb.SearchTagsV2Scope{
+					{
+						Name: "span",
+						Tags: []string{batches[1].SpanAttr},
+					},
+				},
+			},
+		},
+		{
+			name:  "too restrictive query",
+			query: fmt.Sprintf(`{ resource.%s="%s" && resource.y="%s" }`, batches[0].resourceAttr, batches[0].resourceAttVal, batches[1].resourceAttVal),
+			scope: "none",
+			expected: &tempopb.SearchTagsV2Response{
+				Scopes: []*tempopb.SearchTagsV2Scope{
+					{
+						Name: "resource",
+						Tags: []string{"service.name"}, // well known column so included
+					},
+				},
+			},
+		},
+		// Unscoped not supported, unfiltered results.
+		{
+			name:  "unscoped span attribute",
+			query: fmt.Sprintf(`{ .twoSpan="%s" }`, batches[0].spanAttVal),
+			scope: "none",
+			expected: &tempopb.SearchTagsV2Response{
+				Scopes: []*tempopb.SearchTagsV2Scope{
+					{
+						Name: "span",
+						Tags: []string{batches[0].SpanAttr, batches[1].SpanAttr, batches[2].SpanAttr},
+					},
+					{
+						Name: "resource",
+						Tags: []string{batches[0].resourceAttr, batches[1].resourceAttr, batches[2].resourceAttr, "service.name"},
+					},
+				},
+			},
+		},
+		{
+			name:  "unscoped res attribute",
+			query: fmt.Sprintf(`{ .twoRes="%s" }`, batches[0].resourceAttVal),
+			scope: "none",
+			expected: &tempopb.SearchTagsV2Response{
+				Scopes: []*tempopb.SearchTagsV2Scope{
+					{
+						Name: "span",
+						Tags: []string{batches[0].SpanAttr, batches[1].SpanAttr, batches[2].SpanAttr},
+					},
+					{
+						Name: "resource",
+						Tags: []string{batches[0].resourceAttr, batches[1].resourceAttr, batches[2].resourceAttr, "service.name"},
+					},
+				},
+			},
+		},
+		{
+			name:  "both batches - name and resource attribute",
+			query: `{ resource.service.name="my-service"}`,
+			scope: "none",
+			expected: &tempopb.SearchTagsV2Response{
+				Scopes: []*tempopb.SearchTagsV2Scope{
+					{
+						Name: "span",
+						Tags: []string{batches[0].SpanAttr, batches[1].SpanAttr, batches[2].SpanAttr},
+					},
+					{
+						Name: "resource",
+						Tags: []string{batches[0].resourceAttr, batches[1].resourceAttr, batches[2].resourceAttr, "service.name"},
+					},
+				},
+			},
+		},
+		{
+			name:  "bad query - unfiltered results",
+			query: fmt.Sprintf("%s = bar", "span.twoSpan"), // bad query, missing quotes
+			scope: "none",
+			expected: &tempopb.SearchTagsV2Response{
+				Scopes: []*tempopb.SearchTagsV2Scope{
+					{
+						Name: "span",
+						Tags: []string{batches[0].SpanAttr, batches[1].SpanAttr, batches[2].SpanAttr},
+					},
+					{
+						Name: "resource",
+						Tags: []string{batches[0].resourceAttr, batches[1].resourceAttr, batches[2].resourceAttr, "service.name"},
+					},
+				},
+			},
+		},
+	}
+
+	for _, tc := range tcs {
+		// Expected will not have the intrinsic results to make the tests simpler,
+		// they are added here based on the scope.
+		if tc.scope == "" || tc.scope == "none" || tc.scope == "intrinsic" {
+			tc.expected.Scopes = append(tc.expected.Scopes, &tempopb.SearchTagsV2Scope{
+				Name: "intrinsic",
+				Tags: search.GetVirtualIntrinsicValues(),
+			})
+		}
+		sort.Slice(tc.expected.Scopes, func(i, j int) bool { return tc.expected.Scopes[i].Name < tc.expected.Scopes[j].Name })
+		for _, scope := range tc.expected.Scopes {
+			slices.Sort(scope.Tags)
+		}
+	}
+
+	return tcs
+}
+
+func buildSearchTagValuesV2TestCases(batches []batchTmpl) []struct {
+	name     string
+	query    string
+	tagName  string
+	expected *tempopb.SearchTagValuesV2Response
+} {
+	return []struct {
+		name     string
+		query    string
+		tagName  string
+		expected *tempopb.SearchTagValuesV2Response
+	}{
+		{
+			name:    "no filtering",
+			query:   "",
+			tagName: "span.twoSpan",
+			expected: &tempopb.SearchTagValuesV2Response{
+				TagValues: []*tempopb.TagValue{{Type: "string", Value: batches[2].spanAttVal}, {Type: "string", Value: batches[3].spanAttVal}},
+			},
+		},
+		{
+			name:    "first batch - name",
+			query:   fmt.Sprintf(`{ name="%s" }`, batches[2].name),
+			tagName: "span.twoSpan",
+			expected: &tempopb.SearchTagValuesV2Response{
+				TagValues: []*tempopb.TagValue{{Type: "string", Value: batches[2].spanAttVal}},
+			},
+		},
+		{
+			name:    "second batch with incomplete query - name",
+			query:   fmt.Sprintf(`{ name="%s" && span.twoSpan = }`, batches[3].name),
+			tagName: "span.twoSpan",
+			expected: &tempopb.SearchTagValuesV2Response{
+				TagValues: []*tempopb.TagValue{{Type: "string", Value: batches[3].spanAttVal}},
+			},
+		},
+		{
+			name:    "first batch only - resource attribute",
+			query:   fmt.Sprintf(`{ %s="%s" }`, "resource.twoRes", batches[2].resourceAttVal),
+			tagName: "span.twoSpan",
+			expected: &tempopb.SearchTagValuesV2Response{
+				TagValues: []*tempopb.TagValue{{Type: "string", Value: batches[2].spanAttVal}},
+			},
+		},
+		{
+			name:    "second batch only - resource attribute",
+			query:   fmt.Sprintf(`{ %s="%s" }`, "resource.twoRes", batches[3].resourceAttVal),
+			tagName: "span.twoSpan",
+			expected: &tempopb.SearchTagValuesV2Response{
+				TagValues: []*tempopb.TagValue{{Type: "string", Value: batches[3].spanAttVal}},
+			},
+		},
+		{
+			name:     "too restrictive query",
+			query:    fmt.Sprintf(`{ %s="%s" && resource.y="%s" }`, "resource.twoRes", batches[2].resourceAttVal, batches[3].resourceAttVal),
+			tagName:  "span.twoSpan",
+			expected: &tempopb.SearchTagValuesV2Response{},
+		},
+		// Unscoped not supported, unfiltered results.
+		{
+			name:    "unscoped span attribute",
+			query:   fmt.Sprintf(`{ .twoSpan="%s" }`, batches[2].spanAttVal),
+			tagName: "span.twoSpan",
+			expected: &tempopb.SearchTagValuesV2Response{
+				TagValues: []*tempopb.TagValue{{Type: "string", Value: batches[2].spanAttVal}, {Type: "string", Value: batches[3].spanAttVal}},
+			},
+		},
+		{
+			name:    "unscoped resource attribute",
+			query:   fmt.Sprintf(`{ .twoRes="%s" }`, batches[2].spanAttVal),
+			tagName: "resource.twoRes",
+			expected: &tempopb.SearchTagValuesV2Response{
+				TagValues: []*tempopb.TagValue{{Type: "string", Value: batches[2].resourceAttVal}, {Type: "string", Value: batches[3].resourceAttVal}},
+			},
+		},
+		{
+			name:    "first batch - name and resource attribute",
+			query:   fmt.Sprintf(`{ name="%s" }`, batches[2].name),
+			tagName: "resource.service.name",
+			expected: &tempopb.SearchTagValuesV2Response{
+				TagValues: []*tempopb.TagValue{{Type: "string", Value: "my-service"}},
+			},
+		},
+		{
+			name:    "both batches - name and resource attribute",
+			query:   `{ resource.service.name="my-service"}`,
+			tagName: "name",
+			expected: &tempopb.SearchTagValuesV2Response{
+				TagValues: []*tempopb.TagValue{{Type: "string", Value: batches[3].name}, {Type: "string", Value: batches[2].name}},
+			},
+		},
+		{
+			name:    "only resource attributes",
+			query:   fmt.Sprintf(`{ %s="%s" }`, "resource.twoRes", batches[2].resourceAttVal),
+			tagName: "resource.service.name",
+			expected: &tempopb.SearchTagValuesV2Response{
+				TagValues: []*tempopb.TagValue{{Type: "string", Value: "my-service"}},
+			},
+		},
+		{
+			name:    "bad query - unfiltered results",
+			query:   fmt.Sprintf("%s = bar", "span.twoSpan"), // bad query, missing quotes
+			tagName: "span.twoSpan",
+			expected: &tempopb.SearchTagValuesV2Response{
+				TagValues: []*tempopb.TagValue{{Type: "string", Value: batches[2].spanAttVal}, {Type: "string", Value: batches[3].spanAttVal}},
+			},
+		},
+	}
+}
+
+func TestTraceByIDandTraceQL(t *testing.T) {
+	util.RunIntegrationTests(t, util.TestHarnessConfig{
+		Components: util.ComponentsRecentDataQuerying | util.ComponentsBackendQuerying,
+		Backends:   util.BackendObjectStorageAll, // runs basic querying against all 3 object storage backends. no need to replicate for every test.
+	}, func(h *util.TempoHarness) {
+		h.WaitTracesWritable(t)
+
+		countTraces := 10
+		infos := tempoUtil.NewTraceInfos(time.Now(), countTraces, "")
+		for _, info := range infos {
+			require.NoError(t, h.WriteTraceInfo(info, ""))
+		}
+
+		h.WaitTracesQueryable(t, countTraces)
+
+		grpcClient, ctx, err := h.APIClientGRPC("")
+		require.NoError(t, err)
+		apiClient := h.APIClientHTTP("")
+
+		now := time.Now()
+		for _, i := range infos {
+			util.QueryAndAssertTrace(t, apiClient, i)
+			util.SearchTraceQLAndAssertTraceWithRange(t, apiClient, i, now.Add(-time.Hour).Unix(), now.Add(time.Hour).Unix())
+			util.SearchStreamAndAssertTrace(t, ctx, grpcClient, i, now.Add(-time.Hour).Unix(), now.Add(time.Hour).Unix())
+		}
+
+		h.WaitTracesWrittenToBackend(t, countTraces)
+		h.ForceBackendQuerying(t)
+
+		grpcClient, ctx, err = h.APIClientGRPC("")
+		require.NoError(t, err)
+		apiClient = h.APIClientHTTP("")
+
+		// Assert tags on storage backend
+		for _, i := range infos {
+			util.QueryAndAssertTrace(t, apiClient, i)
+			util.SearchTraceQLAndAssertTraceWithRange(t, apiClient, i, now.Add(-time.Hour).Unix(), now.Add(time.Hour).Unix())
+			util.SearchStreamAndAssertTrace(t, ctx, grpcClient, i, now.Add(-time.Hour).Unix(), now.Add(time.Hour).Unix())
+		}
+	})
+}
+
+func TestStreamingSearch_badRequest(t *testing.T) {
+	util.RunIntegrationTests(t, util.TestHarnessConfig{}, func(h *util.TempoHarness) {
+		h.WaitTracesWritable(t)
+
+		// Send a batch of traces
+		batch := util.MakeThriftBatch()
+		require.NoError(t, h.WriteJaegerBatch(batch, ""))
+
+		// Wait for the traces to be written to the WAL
+		h.WaitTracesQueryable(t, 1)
+
+		// Create gRPC client
+		grpcClient, ctx, err := h.APIClientGRPC("")
+		require.NoError(t, err)
+
+		// Send invalid search query (missing operator)
+		res, err := grpcClient.Search(ctx, &tempopb.SearchRequest{
+			Query: "{resource.service.name=article}",
+		})
+		require.NoError(t, err)
+
+		// Expect error on receive
+		_, err = res.Recv()
+		require.Error(t, err)
+
+		st, ok := status.FromError(err)
+		require.True(t, ok)
+		require.Equal(t, codes.InvalidArgument, st.Code())
+	})
+}
+
+func TestSearchTagValuesV2_badRequest(t *testing.T) {
+	util.RunIntegrationTests(t, util.TestHarnessConfig{}, func(h *util.TempoHarness) {
+		// Test HTTP endpoint returns 400 for invalid tagName
+		invalidTagName := "app.user.id" // not a valid scoped attribute
+		req, err := http.NewRequest(http.MethodGet,
+			fmt.Sprintf("%s/api/v2/search/tag/%s/values", h.BaseURL(), invalidTagName), nil)
+		require.NoError(t, err)
+
+		res, err := http.DefaultClient.Do(req)
+		require.NoError(t, err)
+		require.Equal(t, http.StatusBadRequest, res.StatusCode)
+
+		body, err := io.ReadAll(res.Body)
+		require.NoError(t, err)
+		defer res.Body.Close()
+
+		require.Contains(t, string(body), "tag name is not valid intrinsic or scoped attribute")
+
+		// Test gRPC endpoint returns InvalidArgument for invalid tagName
+		grpcClient, ctx, err := h.APIClientGRPC("")
+		require.NoError(t, err)
+
+		stream, err := grpcClient.SearchTagValuesV2(ctx, &tempopb.SearchTagValuesRequest{
+			TagName: invalidTagName,
+		})
+		require.NoError(t, err)
+
+		_, err = stream.Recv()
+		require.Error(t, err)
+
+		st, ok := status.FromError(err)
+		require.True(t, ok)
+		require.Equal(t, codes.InvalidArgument, st.Code())
+		require.Contains(t, st.Message(), "tag name is not valid intrinsic or scoped attribute")
+	})
+}
+
+func callSearchTagValuesV2AndAssert(t *testing.T, h *util.TempoHarness, tagName, query string, expected *tempopb.SearchTagValuesV2Response, start, end int64) {
+	apiClient := h.APIClientHTTP("")
+	response, err := apiClient.SearchTagValuesV2WithRange(tagName, query, start, end)
+	require.NoError(t, err)
+
+	sort.Slice(response.TagValues, func(i, j int) bool { return response.TagValues[i].Value < response.TagValues[j].Value })
+	require.Equal(t, expected.TagValues, response.TagValues)
+	assertMetrics(t, response.Metrics, len(expected.TagValues))
+
+	// streaming
+	grpcReq := &tempopb.SearchTagValuesRequest{
+		TagName: tagName,
+		Query:   query,
+		Start:   uint32(start),
+		End:     uint32(end),
+	}
+
+	grpcClient, ctx, err := h.APIClientGRPC("")
+	require.NoError(t, err)
+
+	respTagsValuesV2, err := grpcClient.SearchTagValuesV2(ctx, grpcReq)
+	require.NoError(t, err)
+	finalResponse := &tempopb.SearchTagValuesV2Response{
+		Metrics: &tempopb.MetadataMetrics{},
+	}
+	for {
+		resp, err := respTagsValuesV2.Recv()
+		if resp != nil {
+			naiveTagValuesV2Combine(resp, finalResponse)
+		}
+		if errors.Is(err, io.EOF) {
+			break
+		}
+		require.NoError(t, err)
+	}
+	require.NotNil(t, finalResponse)
+	sort.Slice(finalResponse.TagValues, func(i, j int) bool { return finalResponse.TagValues[i].Value < finalResponse.TagValues[j].Value })
+	require.Equal(t, expected.TagValues, finalResponse.TagValues)
+	// assert metrics, and make sure it's non-zero when response is non-empty
+	if len(finalResponse.TagValues) > 0 {
+		require.Greater(t, finalResponse.Metrics.InspectedBytes, uint64(0))
+	}
+}
+
+func naiveTagValuesV2Combine(rNew, rInto *tempopb.SearchTagValuesV2Response) {
+	rIntoTagValues := map[string]*tempopb.TagValue{}
+	for _, val := range rInto.GetTagValues() {
+		rIntoTagValues[val.Type+"="+val.Value] = val
+	}
+
+	for _, newVal := range rNew.GetTagValues() {
+		if _, ok := rIntoTagValues[newVal.Type+"="+newVal.Value]; ok {
+			continue
+		}
+
+		rInto.TagValues = append(rInto.TagValues, newVal)
+	}
+
+	rInto.Metrics.InspectedBytes += rNew.Metrics.InspectedBytes
+	rInto.Metrics.CompletedJobs += rNew.Metrics.CompletedJobs
+}
+
+func callSearchTagsV2AndAssert(t *testing.T, h *util.TempoHarness, scope, query string, expected *tempopb.SearchTagsV2Response, start, end int64) {
+	// search for tag values
+	apiClient := h.APIClientHTTP("")
+
+	response, err := apiClient.SearchTagsV2WithRange(scope, query, start, end)
+	require.NoError(t, err)
+
+	// parse response
+	prepTagsResponse(response)
+	require.Equal(t, expected.Scopes, response.Scopes)
+	assertMetrics(t, response.Metrics, lenWithoutIntrinsic(response))
+
+	// streaming
+	grpcReq := &tempopb.SearchTagsRequest{
+		Scope: scope,
+		Query: query,
+		Start: uint32(start),
+		End:   uint32(end),
+	}
+
+	grpcClient, ctx, err := h.APIClientGRPC("")
+	require.NoError(t, err)
+
+	respTagsValuesV2, err := grpcClient.SearchTagsV2(ctx, grpcReq)
+	require.NoError(t, err)
+	finalResponse := &tempopb.SearchTagsV2Response{
+		Metrics: &tempopb.MetadataMetrics{},
+	}
+	for {
+		resp, err := respTagsValuesV2.Recv()
+		if resp != nil {
+			naiveTagsV2Combine(resp, finalResponse)
+		}
+		if errors.Is(err, io.EOF) {
+			break
+		}
+		require.NoError(t, err)
+	}
+	require.NotNil(t, finalResponse)
+	require.NotNil(t, finalResponse.Metrics)
+
+	prepTagsResponse(finalResponse)
+	require.Equal(t, expected.Scopes, finalResponse.Scopes)
+	// assert metrics, and make sure it's non-zero when response is non-empty
+	if lenWithoutIntrinsic(response) > 0 {
+		require.Greater(t, finalResponse.Metrics.InspectedBytes, uint64(100))
+	}
+}
+
+func naiveTagsV2Combine(rNew, rInto *tempopb.SearchTagsV2Response) {
+	distinctVals := collector.NewScopedDistinctString(0, 0, 0)
+	for _, scope := range rNew.GetScopes() {
+		for _, tag := range scope.GetTags() {
+			distinctVals.Collect(scope.GetName(), tag)
+		}
+	}
+
+	for _, scope := range rInto.GetScopes() {
+		for _, tag := range scope.GetTags() {
+			distinctVals.Collect(scope.GetName(), tag)
+		}
+	}
+
+	scopeToVals := distinctVals.Strings()
+	rInto.Scopes = make([]*tempopb.SearchTagsV2Scope, 0, len(scopeToVals))
+	for scope, vals := range scopeToVals {
+		rInto.Scopes = append(rInto.Scopes, &tempopb.SearchTagsV2Scope{
+			Name: scope,
+			Tags: vals,
+		})
+	}
+
+	rInto.Metrics.InspectedBytes += rNew.Metrics.InspectedBytes
+	rInto.Metrics.CompletedJobs += rNew.Metrics.CompletedJobs
+}
+
+func prepTagsResponse(resp *tempopb.SearchTagsV2Response) {
+	sort.Slice(resp.Scopes, func(i, j int) bool { return resp.Scopes[i].Name < resp.Scopes[j].Name })
+	for _, scope := range resp.Scopes {
+		if len(scope.Tags) == 0 {
+			scope.Tags = nil
+		}
+
+		slices.Sort(scope.Tags)
+	}
+}
+
+func assertMetrics(t *testing.T, metrics *tempopb.MetadataMetrics, respLen int) {
+	// metrics are not present when response is empty, so return
+	if respLen == 0 {
+		return
+	}
+
+	require.NotNil(t, metrics)
+	// if response len is empty, then the inspected bytes should be 0
+	// assert metrics, and make sure it's non-zero
+	require.Greater(t, metrics.InspectedBytes, uint64(300))
+}
+
+func lenWithoutIntrinsic(resp *tempopb.SearchTagsV2Response) int {
+	size := 0
+	for _, scope := range resp.Scopes {
+		// we don't count intrinsics as results for testing
+		if scope.Name == "intrinsic" {
+			continue
+		}
+		size += len(scope.Tags)
+	}
+	return size
+}
+
+func callSearchTagsAndAssert(t *testing.T, h *util.TempoHarness, expectedTagNames []string, start, end int64) {
+	apiClient := h.APIClientHTTP("")
+	var response *tempopb.SearchTagsResponse
+	var err error
+
+	if start == 0 && end == 0 {
+		response, err = apiClient.SearchTags()
+	} else {
+		response, err = apiClient.SearchTagsWithRange(start, end)
+	}
+	require.NoError(t, err)
+
+	// Sort for comparison
+	slices.Sort(response.TagNames)
+	require.Equal(t, expectedTagNames, response.TagNames)
+}
+
+func callSearchTagValuesAndAssert(t *testing.T, h *util.TempoHarness, tagName string, expectedValues []string, start, end int64) {
+	// V1 API doesn't have a client method with range, so we use raw HTTP requests
+	urlPath := fmt.Sprintf("/api/search/tag/%s/values", tagName)
+	req, err := http.NewRequest(http.MethodGet, h.BaseURL()+urlPath, nil)
+	require.NoError(t, err)
+
+	q := req.URL.Query()
+	if start != 0 {
+		q.Set("start", fmt.Sprintf("%d", start))
+	}
+	if end != 0 {
+		q.Set("end", fmt.Sprintf("%d", end))
+	}
+	req.URL.RawQuery = q.Encode()
+
+	res, err := http.DefaultClient.Do(req)
+	require.NoError(t, err)
+	require.Equal(t, http.StatusOK, res.StatusCode)
+
+	body, err := io.ReadAll(res.Body)
+	require.NoError(t, err)
+	defer res.Body.Close()
+
+	var response tempopb.SearchTagValuesResponse
+	require.NoError(t, jsonpb.Unmarshal(bytes.NewReader(body), &response))
+
+	// Sort for comparison
+	sort.Strings(response.TagValues)
+	require.Equal(t, expectedValues, response.TagValues)
+}

integration/api/combine_test.go

@@ -0,0 +1,320 @@
+package api
+
+import (
+	"testing"
+
+	"github.com/grafana/tempo/pkg/tempopb"
+	v1 "github.com/grafana/tempo/pkg/tempopb/common/v1"
+	"github.com/stretchr/testify/require"
+)
+
+func TestNaiveTagValuesV2Combine(t *testing.T) {
+	tests := []struct {
+		name     string
+		rNew     *tempopb.SearchTagValuesV2Response
+		rInto    *tempopb.SearchTagValuesV2Response
+		expected *tempopb.SearchTagValuesV2Response
+	}{
+		{
+			name: "combine unique tag values",
+			rNew: &tempopb.SearchTagValuesV2Response{
+				TagValues: []*tempopb.TagValue{
+					{Type: "string", Value: "value1"},
+					{Type: "string", Value: "value2"},
+				},
+				Metrics: &tempopb.MetadataMetrics{
+					InspectedBytes: 100,
+					CompletedJobs:  1,
+				},
+			},
+			rInto: &tempopb.SearchTagValuesV2Response{
+				TagValues: []*tempopb.TagValue{
+					{Type: "string", Value: "value3"},
+				},
+				Metrics: &tempopb.MetadataMetrics{
+					InspectedBytes: 50,
+					CompletedJobs:  1,
+				},
+			},
+			expected: &tempopb.SearchTagValuesV2Response{
+				TagValues: []*tempopb.TagValue{
+					{Type: "string", Value: "value3"},
+					{Type: "string", Value: "value1"},
+					{Type: "string", Value: "value2"},
+				},
+				Metrics: &tempopb.MetadataMetrics{
+					InspectedBytes: 150,
+					CompletedJobs:  2,
+				},
+			},
+		},
+		{
+			name: "skip duplicate tag values",
+			rNew: &tempopb.SearchTagValuesV2Response{
+				TagValues: []*tempopb.TagValue{
+					{Type: "string", Value: "value1"},
+					{Type: "string", Value: "value2"},
+				},
+				Metrics: &tempopb.MetadataMetrics{
+					InspectedBytes: 100,
+					CompletedJobs:  1,
+				},
+			},
+			rInto: &tempopb.SearchTagValuesV2Response{
+				TagValues: []*tempopb.TagValue{
+					{Type: "string", Value: "value1"},
+				},
+				Metrics: &tempopb.MetadataMetrics{
+					InspectedBytes: 50,
+					CompletedJobs:  1,
+				},
+			},
+			expected: &tempopb.SearchTagValuesV2Response{
+				TagValues: []*tempopb.TagValue{
+					{Type: "string", Value: "value1"},
+					{Type: "string", Value: "value2"},
+				},
+				Metrics: &tempopb.MetadataMetrics{
+					InspectedBytes: 150,
+					CompletedJobs:  2,
+				},
+			},
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			naiveTagValuesV2Combine(tt.rNew, tt.rInto)
+			require.Equal(t, tt.expected, tt.rInto)
+		})
+	}
+}
+
+func TestNaiveTagsV2Combine(t *testing.T) {
+	tests := []struct {
+		name     string
+		rNew     *tempopb.SearchTagsV2Response
+		rInto    *tempopb.SearchTagsV2Response
+		expected int // expected number of unique tags across all scopes
+	}{
+		{
+			name: "combine unique tags from different scopes",
+			rNew: &tempopb.SearchTagsV2Response{
+				Scopes: []*tempopb.SearchTagsV2Scope{
+					{
+						Name: "resource",
+						Tags: []string{"tag1", "tag2"},
+					},
+				},
+				Metrics: &tempopb.MetadataMetrics{
+					InspectedBytes: 100,
+					CompletedJobs:  1,
+				},
+			},
+			rInto: &tempopb.SearchTagsV2Response{
+				Scopes: []*tempopb.SearchTagsV2Scope{
+					{
+						Name: "span",
+						Tags: []string{"tag3"},
+					},
+				},
+				Metrics: &tempopb.MetadataMetrics{
+					InspectedBytes: 50,
+					CompletedJobs:  1,
+				},
+			},
+			expected: 3, // tag1, tag2, tag3
+		},
+		{
+			name: "deduplicate tags within same scope",
+			rNew: &tempopb.SearchTagsV2Response{
+				Scopes: []*tempopb.SearchTagsV2Scope{
+					{
+						Name: "resource",
+						Tags: []string{"tag1", "tag2"},
+					},
+				},
+				Metrics: &tempopb.MetadataMetrics{
+					InspectedBytes: 100,
+					CompletedJobs:  1,
+				},
+			},
+			rInto: &tempopb.SearchTagsV2Response{
+				Scopes: []*tempopb.SearchTagsV2Scope{
+					{
+						Name: "resource",
+						Tags: []string{"tag1", "tag3"},
+					},
+				},
+				Metrics: &tempopb.MetadataMetrics{
+					InspectedBytes: 50,
+					CompletedJobs:  1,
+				},
+			},
+			expected: 3, // tag1, tag2, tag3
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			naiveTagsV2Combine(tt.rNew, tt.rInto)
+
+			// Count total unique tags across all scopes
+			totalTags := 0
+			for _, scope := range tt.rInto.Scopes {
+				totalTags += len(scope.Tags)
+			}
+			require.Equal(t, tt.expected, totalTags)
+
+			// Verify metrics were combined
+			expectedBytes := tt.rNew.Metrics.InspectedBytes + 50 // original rInto value
+			expectedJobs := tt.rNew.Metrics.CompletedJobs + 1    // original rInto value
+			require.Equal(t, expectedBytes, tt.rInto.Metrics.InspectedBytes)
+			require.Equal(t, expectedJobs, tt.rInto.Metrics.CompletedJobs)
+		})
+	}
+}
+
+func TestNaiveQueryRangeCombine(t *testing.T) {
+	tests := []struct {
+		name     string
+		rNew     *tempopb.QueryRangeResponse
+		rInto    *tempopb.QueryRangeResponse
+		expected *tempopb.QueryRangeResponse
+	}{
+		{
+			name: "combine series with different labels",
+			rNew: &tempopb.QueryRangeResponse{
+				Series: []*tempopb.TimeSeries{
+					{
+						Labels: []v1.KeyValue{
+							{Key: "foo", Value: &v1.AnyValue{Value: &v1.AnyValue_StringValue{StringValue: "bar"}}},
+						},
+						Samples: []tempopb.Sample{{TimestampMs: 1000, Value: 1.0}},
+					},
+				},
+				Metrics: &tempopb.SearchMetrics{
+					InspectedBytes:  100,
+					CompletedJobs:   1,
+					InspectedTraces: 10,
+					InspectedSpans:  50,
+				},
+			},
+			rInto: &tempopb.QueryRangeResponse{
+				Series: []*tempopb.TimeSeries{
+					{
+						Labels: []v1.KeyValue{
+							{Key: "baz", Value: &v1.AnyValue{Value: &v1.AnyValue_StringValue{StringValue: "qux"}}},
+						},
+						Samples: []tempopb.Sample{{TimestampMs: 2000, Value: 2.0}},
+					},
+				},
+				Metrics: &tempopb.SearchMetrics{
+					InspectedBytes:  50,
+					CompletedJobs:   1,
+					InspectedTraces: 5,
+					InspectedSpans:  25,
+				},
+			},
+			expected: &tempopb.QueryRangeResponse{
+				Series: []*tempopb.TimeSeries{
+					{
+						Labels: []v1.KeyValue{
+							{Key: "baz", Value: &v1.AnyValue{Value: &v1.AnyValue_StringValue{StringValue: "qux"}}},
+						},
+						Samples: []tempopb.Sample{{TimestampMs: 2000, Value: 2.0}},
+					},
+					{
+						Labels: []v1.KeyValue{
+							{Key: "foo", Value: &v1.AnyValue{Value: &v1.AnyValue_StringValue{StringValue: "bar"}}},
+						},
+						Samples: []tempopb.Sample{{TimestampMs: 1000, Value: 1.0}},
+					},
+				},
+				Metrics: &tempopb.SearchMetrics{
+					InspectedBytes:  150,
+					CompletedJobs:   2,
+					InspectedTraces: 15,
+					InspectedSpans:  75,
+				},
+			},
+		},
+		{
+			name: "combine series with same labels",
+			rNew: &tempopb.QueryRangeResponse{
+				Series: []*tempopb.TimeSeries{
+					{
+						Labels: []v1.KeyValue{
+							{Key: "foo", Value: &v1.AnyValue{Value: &v1.AnyValue_StringValue{StringValue: "bar"}}},
+						},
+						Samples: []tempopb.Sample{{TimestampMs: 2000, Value: 2.0}},
+					},
+				},
+				Metrics: &tempopb.SearchMetrics{
+					InspectedBytes:  100,
+					CompletedJobs:   1,
+					InspectedTraces: 10,
+					InspectedSpans:  50,
+				},
+			},
+			rInto: &tempopb.QueryRangeResponse{
+				Series: []*tempopb.TimeSeries{
+					{
+						Labels: []v1.KeyValue{
+							{Key: "foo", Value: &v1.AnyValue{Value: &v1.AnyValue_StringValue{StringValue: "bar"}}},
+						},
+						Samples: []tempopb.Sample{{TimestampMs: 1000, Value: 1.0}},
+					},
+				},
+				Metrics: &tempopb.SearchMetrics{
+					InspectedBytes:  50,
+					CompletedJobs:   1,
+					InspectedTraces: 5,
+					InspectedSpans:  25,
+				},
+			},
+			expected: &tempopb.QueryRangeResponse{
+				Series: []*tempopb.TimeSeries{
+					{
+						Labels: []v1.KeyValue{
+							{Key: "foo", Value: &v1.AnyValue{Value: &v1.AnyValue_StringValue{StringValue: "bar"}}},
+						},
+						Samples: []tempopb.Sample{
+							{TimestampMs: 1000, Value: 1.0},
+							{TimestampMs: 2000, Value: 2.0},
+						},
+					},
+				},
+				Metrics: &tempopb.SearchMetrics{
+					InspectedBytes:  150,
+					CompletedJobs:   2,
+					InspectedTraces: 15,
+					InspectedSpans:  75,
+				},
+			},
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			naiveQueryRangeCombine(tt.rNew, tt.rInto)
+			require.Equal(t, tt.expected.Metrics, tt.rInto.Metrics)
+			require.Len(t, tt.rInto.Series, len(tt.expected.Series))
+
+			// Build a map of expected series by label key for comparison
+			expectedMap := map[string]*tempopb.TimeSeries{}
+			for _, series := range tt.expected.Series {
+				expectedMap[keyFromLabels(series.Labels)] = series
+			}
+
+			// Check that all series match expected (order doesn't matter due to map iteration)
+			for _, series := range tt.rInto.Series {
+				key := keyFromLabels(series.Labels)
+				expectedSeries, ok := expectedMap[key]
+				require.True(t, ok, "unexpected series with key %s", key)
+				require.Equal(t, expectedSeries.Labels, series.Labels)
+				require.Len(t, series.Samples, len(expectedSeries.Samples))
+			}
+		})
+	}
+}

integration/api/config-mcp.yaml

@@ -0,0 +1,3 @@
+query_frontend:
+  mcp_server:
+    enabled: true

integration/api/config-multi-tenant.yaml

@@ -0,0 +1 @@
+multitenancy_enabled: true

integration/api/config-overrides.yaml

@@ -0,0 +1,6 @@
+multitenancy_enabled: true
+
+overrides:
+  user_configurable_overrides:
+    enabled: true
+    poll_interval: 10s

integration/api/config-query-range-exemplars.yaml

@@ -0,0 +1,2 @@
+query_frontend:
+  query_end_cutoff: -5m # TestQueryRangeExemplars fails if this is not set. i don't know why

integration/api/config-query-range-max-series-disabled.yaml

@@ -0,0 +1,3 @@
+query_frontend:
+  metrics:
+    max_response_series: 0

integration/api/config-query-range-max-series.yaml

@@ -0,0 +1,3 @@
+query_frontend:
+  metrics:
+    max_response_series: 3

integration/api/management_test.go

@@ -0,0 +1,166 @@
+package api
+
+import (
+	"net/http"
+	"testing"
+
+	"github.com/grafana/tempo/integration/util"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+// TestStatusAndRingAPIs performs a basic smoke test of all status and ring management endpoints.
+func TestStatusAndRingAPIs(t *testing.T) {
+	util.RunIntegrationTests(t, util.TestHarnessConfig{
+		DeploymentMode: util.DeploymentModeMicroservices,
+		Components:     util.ComponentsRecentDataQuerying | util.ComponentsMetricsGeneration | util.ComponentsBackendWork | util.ComponentsBackendQuerying,
+	}, func(h *util.TempoHarness) {
+		testCases := []struct {
+			name           string
+			service        string
+			endpoint       string
+			expectedStatus int
+		}{
+			// All status endpoints on query-frontend
+			{
+				name:           "status - all info",
+				service:        util.ServiceQueryFrontend,
+				endpoint:       "/status",
+				expectedStatus: http.StatusOK,
+			},
+			{
+				name:           "status/version",
+				service:        util.ServiceQueryFrontend,
+				endpoint:       "/status/version",
+				expectedStatus: http.StatusOK,
+			},
+			{
+				name:           "status/services",
+				service:        util.ServiceQueryFrontend,
+				endpoint:       "/status/services",
+				expectedStatus: http.StatusOK,
+			},
+			{
+				name:           "status/endpoints",
+				service:        util.ServiceQueryFrontend,
+				endpoint:       "/status/endpoints",
+				expectedStatus: http.StatusOK,
+			},
+			{
+				name:           "status/config",
+				service:        util.ServiceQueryFrontend,
+				endpoint:       "/status/config",
+				expectedStatus: http.StatusOK,
+			},
+			{
+				name:           "status/runtime_config",
+				service:        util.ServiceQueryFrontend,
+				endpoint:       "/status/runtime_config",
+				expectedStatus: http.StatusOK,
+			},
+			{
+				name:           "status/buildinfo",
+				service:        util.ServiceQueryFrontend,
+				endpoint:       "/api/status/buildinfo",
+				expectedStatus: http.StatusOK,
+			},
+			{
+				name:           "status/backendscheduler",
+				service:        util.ServiceBackendScheduler,
+				endpoint:       "/status/backendscheduler",
+				expectedStatus: http.StatusOK,
+			},
+
+			// ring endpoints
+			{
+				name:           "livestore ring",
+				service:        util.ServiceQuerier,
+				endpoint:       "/live-store/ring",
+				expectedStatus: http.StatusOK,
+			},
+			// TODO: research this behavior
+			// { 200s but should 404!
+			// 	name:           "generator ring",
+			// 	service:        util.ServiceQuerier,
+			// 	endpoint:       "/metrics-generator/ring",
+			// 	expectedStatus: http.StatusNotFound,
+			// },
+			// { 404s due to disabled store in config-base.yaml
+			// 	name:           "backend worker ring",
+			// 	service:        util.ServiceBackendWorker,
+			// 	endpoint:       "/backend-worker/ring",
+			// 	expectedStatus: http.StatusOK,
+			// },
+			{
+				name:           "partition ring - distributor",
+				service:        util.ServiceDistributor,
+				endpoint:       "/partition-ring",
+				expectedStatus: http.StatusOK,
+			},
+			{
+				name:           "partition ring - livestore",
+				service:        util.ServiceLiveStoreZoneA,
+				endpoint:       "/partition-ring",
+				expectedStatus: http.StatusOK,
+			},
+			{
+				name:           "partition ring - generator",
+				service:        util.ServiceMetricsGenerator,
+				endpoint:       "/partition-ring",
+				expectedStatus: http.StatusOK,
+			},
+
+			// memberlist endpoint should be on all services in the memberlist cluster
+			{
+				name:           "memberlist on distributor",
+				service:        util.ServiceDistributor,
+				endpoint:       "/memberlist",
+				expectedStatus: http.StatusOK,
+			},
+			{
+				name:           "memberlist on livestore",
+				service:        util.ServiceLiveStoreZoneA,
+				endpoint:       "/memberlist",
+				expectedStatus: http.StatusOK,
+			},
+		}
+
+		// add a status endpoint for all services
+		for _, k := range util.AllTempoServices {
+			testCases = append(testCases, []struct {
+				name           string
+				service        string
+				endpoint       string
+				expectedStatus int
+			}{
+				{
+					name:           "status - " + k,
+					service:        k,
+					endpoint:       "/status",
+					expectedStatus: http.StatusOK,
+				},
+				{
+					name:           "metrics - " + k,
+					service:        k,
+					endpoint:       "/metrics",
+					expectedStatus: http.StatusOK,
+				},
+			}...)
+		}
+
+		client := &http.Client{}
+
+		for _, tc := range testCases {
+			t.Run(tc.name, func(t *testing.T) {
+				svc := h.Services[tc.service]
+
+				url := "http://" + svc.Endpoint(3200) + tc.endpoint
+				resp, err := client.Get(url)
+				require.NoError(t, err, "failed to make request to %s", url)
+				defer resp.Body.Close()
+
+				assert.Equal(t, tc.expectedStatus, resp.StatusCode, "unexpected status code for %s", tc.endpoint)
+			})
+		}
+	})
+}

integration/api/mcp_test.go

@@ -9,7 +9,6 @@ import (
 	"time"
 
 	"github.com/grafana/e2e"
-	e2edb "github.com/grafana/e2e/db"
 	"github.com/grafana/tempo/integration/util"
 	"github.com/grafana/tempo/pkg/api"
 	tempoUtil "github.com/grafana/tempo/pkg/util"
@@ -22,67 +21,62 @@ import (
 // TestMCP is a smoke test that starts up a tempo instance, writes a trace and queries it back via MCP.
 // It also verifies that all expected tools are available.
 func TestMCP(t *testing.T) {
-	s, err := e2e.NewScenario("tempo_e2e_mcp")
-	require.NoError(t, err)
-	defer s.Close()
+	util.RunIntegrationTests(t, util.TestHarnessConfig{
+		ConfigOverlay: "config-mcp.yaml",
+	}, func(h *util.TempoHarness) {
+		h.WaitTracesWritable(t)
 
-	minio := e2edb.NewMinio(9000, "tempo")
-	require.NotNil(t, minio)
-	require.NoError(t, s.StartAndWaitReady(minio))
+		// Write a trace
+		info := tempoUtil.NewTraceInfo(time.Now(), "")
+		require.NoError(t, h.WriteTraceInfo(info, ""))
 
-	require.NoError(t, util.CopyFileToSharedDir(s, configAllInOneLocal, "config.yaml"))
-	tempo := util.NewTempoAllInOne()
-	require.NoError(t, s.StartAndWaitReady(tempo))
+		_, err := info.ConstructTraceFromEpoch()
+		require.NoError(t, err)
 
-	jaegerClient, err := util.NewJaegerToOTLPExporter(tempo.Endpoint(4317))
-	require.NoError(t, err)
-	require.NotNil(t, jaegerClient)
+		// Wait for the trace to be written to the WAL
+		h.WaitTracesQueryable(t, 1)
 
-	info := tempoUtil.NewTraceInfo(time.Now(), "")
-	require.NoError(t, info.EmitAllBatches(jaegerClient))
+		// now query it back with mcp
+		queryFrontend := h.Services[util.ServiceQueryFrontend]
+		mcpClient := createMCPClient(t, queryFrontend)
 
-	_, err = info.ConstructTraceFromEpoch()
-	require.NoError(t, err)
+		tools := listTools(t, mcpClient)
 
-	// now query it back with mcp
-	mcpClient := createMCPClient(t, tempo)
+		// confirm all tools are listed as read only and no open world
+		for _, tool := range tools {
+			require.NotNil(t, tool.Annotations.DestructiveHint, "tool %s doesn't specify destructive", tool.Name)
+			require.False(t, *tool.Annotations.DestructiveHint, "tool %s is marked destructive", tool.Name)
 
-	tools := listTools(t, mcpClient)
+			require.NotNil(t, tool.Annotations.OpenWorldHint, "tool %s doesn't specify open world", tool.Name)
+			require.False(t, *tool.Annotations.OpenWorldHint, "tool %s is marked open world", tool.Name)
 
-	// confirm all tools are listed as read only and no open world
-	for _, tool := range tools {
-		require.NotNil(t, tool.Annotations.DestructiveHint, "tool %s doesn't specify destructive", tool.Name)
-		require.False(t, *tool.Annotations.DestructiveHint, "tool %s is marked destructive", tool.Name)
+			require.NotNil(t, tool.Annotations.ReadOnlyHint, "tool %s doesn't specify read only", tool.Name)
+			require.True(t, *tool.Annotations.ReadOnlyHint, "tool %s is marked write", tool.Name)
+		}
 
-		require.NotNil(t, tool.Annotations.OpenWorldHint, "tool %s doesn't specify open world", tool.Name)
-		require.False(t, *tool.Annotations.OpenWorldHint, "tool %s is marked open world", tool.Name)
+		// Verify all expected tools are available
+		expectedTools := []string{
+			"traceql-search",
+			"traceql-metrics-instant",
+			"traceql-metrics-range",
+			"get-trace",
+			"get-attribute-names",
+			"get-attribute-values",
+			"docs-traceql",
+		}
 
-		require.NotNil(t, tool.Annotations.ReadOnlyHint, "tool %s doesn't specify read only", tool.Name)
-		require.True(t, *tool.Annotations.ReadOnlyHint, "tool %s is marked write", tool.Name)
-	}
+		actualTools := make([]string, len(tools))
+		for i, tool := range tools {
+			actualTools[i] = tool.Name
+		}
 
-	// Verify all expected tools are available
-	expectedTools := []string{
-		"traceql-search",
-		"traceql-metrics-instant",
-		"traceql-metrics-range",
-		"get-trace",
-		"get-attribute-names",
-		"get-attribute-values",
-		"docs-traceql",
-	}
+		// sort both lists
+		sort.Strings(actualTools)
+		sort.Strings(expectedTools)
+		require.Equal(t, expectedTools, actualTools)
 
-	actualTools := make([]string, len(tools))
-	for i, tool := range tools {
-		actualTools[i] = tool.Name
-	}
-
-	// sort both lists
-	sort.Strings(actualTools)
-	sort.Strings(expectedTools)
-	require.Equal(t, expectedTools, actualTools)
-
-	assertTraceOverMCP(t, mcpClient, info.HexID())
+		assertTraceOverMCP(t, mcpClient, info.HexID())
+	})
 }
 
 func createMCPClient(t *testing.T, tempo *e2e.HTTPService) mcpclient.MCPClient {

integration/api/multi_tenant_test.go

@@ -0,0 +1,71 @@
+package api
+
+import (
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/grafana/tempo/integration/util"
+	tempoUtil "github.com/grafana/tempo/pkg/util"
+	"github.com/stretchr/testify/require"
+)
+
+func TestSingleTenantSearch(t *testing.T) {
+	testSearch(t, "test")
+}
+
+func TestWildCardTenantSearch(t *testing.T) {
+	testSearch(t, "*")
+}
+
+func TestTwoTenantsSearch(t *testing.T) {
+	testSearch(t, "test|test2")
+}
+
+func TestThreeTenantsSearch(t *testing.T) {
+	testSearch(t, "test|test2|test3")
+}
+
+func testSearch(t *testing.T, tenant string) {
+	util.RunIntegrationTests(t, util.TestHarnessConfig{
+		ConfigOverlay: "config-multi-tenant.yaml",
+	}, func(h *util.TempoHarness) {
+		h.WaitTracesWritable(t)
+
+		var info *tempoUtil.TraceInfo
+
+		tenants := strings.Split(tenant, "|")
+		tenantSize := len(tenants)
+
+		// write traces for all tenants
+		for _, tenant := range tenants {
+			info = tempoUtil.NewTraceInfo(time.Now(), tenant)
+			require.NoError(t, h.WriteTraceInfo(info, tenant))
+		}
+
+		// assert that we have one trace for each tenant and correct number of spans received
+		h.WaitTracesQueryable(t, tenantSize)
+
+		// check trace by id
+		apiClient := h.APIClientHTTP(tenant)
+		util.SearchTraceQLAndAssertTrace(t, apiClient, info)
+
+		// call search tags endpoints, ensure no errors and results are not empty
+		tagsV2Resp, err := apiClient.SearchTagsV2()
+		require.NoError(t, err)
+		require.Equal(t, 4, len(tagsV2Resp.GetScopes())) // resource, span, event, link, instrumentation intrinsics
+		for _, s := range tagsV2Resp.Scopes {
+			require.NotEmpty(t, s.Tags)
+		}
+
+		tagsValuesV2Resp, err := apiClient.SearchTagValuesV2("span.vulture-0", "{}")
+		require.NoError(t, err)
+		require.NotEmpty(t, tagsValuesV2Resp.TagValues)
+
+		grpcClient, ctx, err := h.APIClientGRPC(tenant)
+		require.NoError(t, err)
+
+		now := time.Now()
+		util.SearchStreamAndAssertTrace(t, ctx, grpcClient, info, now.Add(-time.Hour).Unix(), now.Add(time.Hour).Unix())
+	})
+}

integration/api/overrides_api_test.go

@@ -0,0 +1,712 @@
+package api
+
+import (
+	"fmt"
+	"net/http"
+	"strings"
+	"testing"
+
+	"github.com/grafana/tempo/integration/util"
+	"github.com/grafana/tempo/modules/overrides/histograms"
+	"github.com/grafana/tempo/modules/overrides/userconfigurable/client"
+	"github.com/grafana/tempo/pkg/httpclient"
+	jsoniter "github.com/json-iterator/go"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+const (
+	configOverrides = "./config-overrides.yaml"
+)
+
+func TestOverridesWithObjectStorage(t *testing.T) {
+	util.RunIntegrationTests(t, util.TestHarnessConfig{
+		Backends:       util.BackendObjectStorageAll,
+		DeploymentMode: util.DeploymentModeSingleBinary,
+		ConfigOverlay:  configOverrides,
+	}, func(h *util.TempoHarness) {
+		apiClient := h.APIClientHTTP("single-tenant")
+
+		// Create overrides
+		initialLimits := &client.Limits{
+			MetricsGenerator: client.LimitsMetricsGenerator{
+				DisableCollection: boolPtr(true),
+			},
+		}
+
+		fmt.Println("* Creating overrides with non-0 version")
+		_, err := apiClient.SetOverrides(initialLimits, "123")
+		assert.ErrorContains(t, err, "412") // precondition failed
+
+		fmt.Println("* Creating overrides")
+		_, err = apiClient.SetOverrides(initialLimits, "0")
+		assert.NoError(t, err)
+
+		limits, version, err := apiClient.GetOverrides()
+		assert.NoError(t, err)
+		EnableInstanceLabel, EnableInstanceLabelIsSet := limits.GetMetricsGenerator().GetProcessor().GetSpanMetrics().GetEnableInstanceLabel()
+		assert.True(t, EnableInstanceLabel)
+		assert.False(t, EnableInstanceLabelIsSet)
+		printLimits(limits, version)
+
+		disableCollection, ok := limits.GetMetricsGenerator().GetDisableCollection()
+		assert.True(t, ok)
+		assert.True(t, disableCollection)
+
+		// Update overrides - POST
+		updatedLimits := &client.Limits{
+			MetricsGenerator: client.LimitsMetricsGenerator{
+				DisableCollection: nil,
+				Processors:        map[string]struct{}{"span-metrics": {}},
+			},
+		}
+
+		fmt.Println("* Update overrides with bogus version number")
+		_, err = apiClient.SetOverrides(updatedLimits, "abc")
+		assert.ErrorContains(t, err, "412") // precondition failed
+
+		fmt.Println("* Update overrides with backend.VersionNew")
+		_, err = apiClient.SetOverrides(updatedLimits, "0")
+		assert.ErrorContains(t, err, "412") // precondition failed
+
+		fmt.Println("* Update overrides with wrong version number")
+		_, err = apiClient.SetOverrides(updatedLimits, "123")
+		assert.ErrorContains(t, err, "412") // precondition failed
+
+		fmt.Println("* Update overrides")
+		_, err = apiClient.SetOverrides(updatedLimits, version)
+		assert.NoError(t, err)
+
+		limits, version, err = apiClient.GetOverrides()
+
+		assert.NoError(t, err)
+		printLimits(limits, version)
+
+		_, ok = limits.GetMetricsGenerator().GetDisableCollection()
+		assert.False(t, ok) // is not set anymore
+		processors, ok := limits.GetMetricsGenerator().GetProcessors()
+		assert.True(t, ok)
+		assert.ElementsMatch(t, keys(processors.GetMap()), []string{"span-metrics"})
+
+		// Modify overrides - PATCH
+		patch := &client.Limits{
+			MetricsGenerator: client.LimitsMetricsGenerator{
+				GenerateNativeHistograms:       histogramModePtr(histograms.HistogramMethodNative),
+				NativeHistogramMaxBucketNumber: uint32Ptr(200),
+				DisableCollection:              boolPtr(true),
+				Processor: client.LimitsMetricsGeneratorProcessor{
+					SpanMetrics: client.LimitsMetricsGeneratorProcessorSpanMetrics{
+						EnableInstanceLabel: boolPtr(false),
+					},
+				},
+			},
+		}
+
+		fmt.Println("* Patch overrides")
+		limits, version, err = apiClient.PatchOverrides(patch)
+		assert.NoError(t, err)
+
+		disableCollection, ok = limits.GetMetricsGenerator().GetDisableCollection()
+		assert.True(t, ok)
+		assert.True(t, disableCollection)
+		processors, ok = limits.GetMetricsGenerator().GetProcessors()
+		assert.True(t, ok)
+		assert.ElementsMatch(t, keys(processors.GetMap()), []string{"span-metrics"})
+		EnableInstanceLabel, EnableInstanceLabelIsSet = limits.GetMetricsGenerator().GetProcessor().GetSpanMetrics().GetEnableInstanceLabel()
+		assert.False(t, EnableInstanceLabel)
+		assert.True(t, EnableInstanceLabelIsSet)
+
+		generateNativeHistograms, ok := limits.GetMetricsGenerator().GetGenerateNativeHistograms()
+		assert.True(t, ok)
+		assert.Equal(t, histograms.HistogramMethodNative, generateNativeHistograms)
+		nativeHistogramMaxBucketNumber, ok := limits.GetMetricsGenerator().GetNativeHistogramMaxBucketNumber()
+		assert.True(t, ok)
+		assert.Equal(t, uint32(200), nativeHistogramMaxBucketNumber)
+
+		// Delete overrides
+		cfg, err := h.GetConfig()
+		require.NoError(t, err)
+		if cfg.StorageConfig.Trace.Backend != "gcs" {
+			// Delete with preconditions is not supported by fake-gcs-server https://github.com/fsouza/fake-gcs-server/issues/1282
+			fmt.Println("* Deleting overrides - don't respect version")
+			err = apiClient.DeleteOverrides("123")
+			assert.ErrorContains(t, err, "412") // precondition failed
+		}
+
+		fmt.Println("* Deleting overrides")
+		err = apiClient.DeleteOverrides(version)
+		assert.NoError(t, err)
+
+		// Get overrides - 404
+		fmt.Println("* Get overrides - 404")
+		_, _, err = apiClient.GetOverrides()
+		assert.ErrorIs(t, err, httpclient.ErrNotFound)
+
+		// Recreate overrides - PATCH
+		patch = &client.Limits{
+			MetricsGenerator: client.LimitsMetricsGenerator{
+				DisableCollection: boolPtr(true),
+			},
+		}
+
+		fmt.Println("* Patch overrides - overrides don't exist yet")
+		_, _, err = apiClient.PatchOverrides(patch)
+		assert.NoError(t, err)
+	})
+}
+
+func TestOverridesAPI_GET(t *testing.T) {
+	util.RunIntegrationTests(t, util.TestHarnessConfig{
+		ConfigOverlay:  configOverrides,
+		DeploymentMode: util.DeploymentModeSingleBinary,
+		Backends:       util.BackendObjectStorageS3, // this test fails on other backends b/c it's testing specific etag related code
+	}, func(h *util.TempoHarness) {
+		t.Run("returns 404 when config not found", func(t *testing.T) {
+			apiClient := h.APIClientHTTP("tenant-get-1")
+
+			limits, etag, err := apiClient.GetOverrides()
+			require.Nil(t, limits) // no limits because it doesn't exist for this tenant
+			require.Empty(t, etag) // etag will be ""
+			require.ErrorIs(t, err, httpclient.ErrNotFound)
+		})
+
+		t.Run("returns config with etag", func(t *testing.T) {
+			apiClient := h.APIClientHTTP("tenant-get-2")
+
+			// create initial config with POST
+			initialLimits := &client.Limits{
+				CostAttribution: client.CostAttribution{
+					Dimensions: &map[string]string{"host.name": "host_name"},
+				},
+				MetricsGenerator: client.LimitsMetricsGenerator{
+					DisableCollection: boolPtr(false),
+				},
+			}
+			setEtag, err := apiClient.SetOverrides(initialLimits, "0")
+			require.NoError(t, err)
+			require.NotEmpty(t, setEtag)
+
+			// get should return the config and etag
+			returnedLimits, etag, err := apiClient.GetOverrides()
+			require.NoError(t, err)
+			require.Equal(t, setEtag, etag)
+			require.Equal(t, initialLimits, returnedLimits)
+		})
+	})
+}
+
+func TestOverridesAPI_POST(t *testing.T) {
+	util.RunIntegrationTests(t, util.TestHarnessConfig{
+		ConfigOverlay:  configOverrides,
+		DeploymentMode: util.DeploymentModeSingleBinary,
+		Backends:       util.BackendObjectStorageS3, // this test fails on other backends b/c it's testing specific etag related code
+	}, func(h *util.TempoHarness) {
+		t.Run("API returns 428 without if-match header", func(t *testing.T) {
+			baseURL := h.BaseURL()
+
+			req, err := http.NewRequest(http.MethodPost, fmt.Sprintf("%s/api/overrides", baseURL), strings.NewReader(`{}`))
+			require.NoError(t, err)
+			req.Header.Set("Content-Type", "application/json")
+			req.Header.Set("X-Scope-OrgID", "tenant-post-1")
+
+			resp, err := http.DefaultClient.Do(req)
+			require.NoError(t, err)
+			require.Equal(t, http.StatusPreconditionRequired, resp.StatusCode)
+			err = resp.Body.Close()
+			require.NoError(t, err)
+		})
+
+		t.Run("creates config with If-Match 0 on new tenant", func(t *testing.T) {
+			apiClient := h.APIClientHTTP("tenant-post-2")
+
+			limits := &client.Limits{
+				MetricsGenerator: client.LimitsMetricsGenerator{
+					DisableCollection: boolPtr(true),
+				},
+				CostAttribution: client.CostAttribution{
+					Dimensions: &map[string]string{"host.name": "host_name"},
+				},
+			}
+			setEtag, err := apiClient.SetOverrides(limits, "0")
+			require.NoError(t, err)
+			require.NotEmpty(t, setEtag)
+
+			// verify we can read it back
+			returnedLimits, etag, err := apiClient.GetOverrides()
+			require.NoError(t, err)
+			require.Equal(t, setEtag, etag)
+			require.Equal(t, limits, returnedLimits)
+			disableCollection, ok := returnedLimits.GetMetricsGenerator().GetDisableCollection()
+			require.True(t, ok)
+			require.True(t, disableCollection)
+		})
+
+		t.Run("tenant with existing config returns 412 with If-Match 0", func(t *testing.T) {
+			apiClient := h.APIClientHTTP("tenant-post-3")
+
+			// create initial config for tenant so we can try again
+			limits := &client.Limits{
+				CostAttribution: client.CostAttribution{
+					Dimensions: &map[string]string{"service.name": "service_name"},
+				},
+			}
+			etag, err := apiClient.SetOverrides(limits, "0")
+			require.NotEmpty(t, etag)
+			require.NoError(t, err)
+
+			// try to create config again with If-Match 0
+			etag2, err2 := apiClient.SetOverrides(limits, "0")
+			require.Empty(t, etag2)
+			require.ErrorContains(t, err2, "failed with response: 412 body: version does not match")
+		})
+
+		t.Run("incorrect If-Match returns 412", func(t *testing.T) {
+			apiClient := h.APIClientHTTP("tenant-post-4")
+
+			// Create initial config
+			limits := &client.Limits{
+				MetricsGenerator: client.LimitsMetricsGenerator{
+					DisableCollection: boolPtr(false),
+				},
+			}
+			etag, err := apiClient.SetOverrides(limits, "0")
+			require.NotEmpty(t, etag)
+			require.NoError(t, err)
+
+			// Try to update with wrong version
+			etag2, err2 := apiClient.SetOverrides(limits, "made-up-etag-value")
+			require.Empty(t, etag2)
+			require.ErrorContains(t, err2, "failed with response: 412 body: version does not match")
+		})
+
+		t.Run("with invalid json returns 400", func(t *testing.T) {
+			baseURL := h.BaseURL()
+
+			// invalid config
+			badConfig := strings.NewReader(`{"metrics_generator": {"processor": {"service_graphs": {"histogram_buckets": [0.1, "invalid"]}}}}`)
+			req, _ := http.NewRequest(http.MethodPost, fmt.Sprintf("%s/api/overrides", baseURL), badConfig)
+			req.Header.Set("Content-Type", "application/json")
+			req.Header.Set("X-Scope-OrgID", "tenant-post-5")
+			req.Header.Set("If-Match", "0")
+
+			resp, _ := http.DefaultClient.Do(req)
+			require.Equal(t, http.StatusBadRequest, resp.StatusCode)
+			resp.Body.Close()
+		})
+
+		t.Run("updates config with correct version", func(t *testing.T) {
+			apiClient := h.APIClientHTTP("tenant-post-6")
+
+			// create initial config
+			initialLimits := &client.Limits{
+				MetricsGenerator: client.LimitsMetricsGenerator{
+					DisableCollection: boolPtr(true),
+				},
+			}
+			setEtag, err := apiClient.SetOverrides(initialLimits, "0")
+			require.NoError(t, err)
+
+			// update with correct version
+			updatedLimits := &client.Limits{
+				MetricsGenerator: client.LimitsMetricsGenerator{
+					Processors: map[string]struct{}{"span-metrics": {}},
+				},
+			}
+			newEtag, err := apiClient.SetOverrides(updatedLimits, setEtag)
+			require.NoError(t, err)
+			require.NotEqual(t, setEtag, newEtag)
+
+			// verify the update
+			returnedLimits, _, err := apiClient.GetOverrides()
+			require.NoError(t, err)
+
+			// DisableCollection should not be set anymore because it was wiped out by the update
+			_, ok := returnedLimits.GetMetricsGenerator().GetDisableCollection()
+			require.False(t, ok)
+
+			// Processors should be set because we added then in updated
+			processors, ok := returnedLimits.GetMetricsGenerator().GetProcessors()
+			require.True(t, ok)
+			require.ElementsMatch(t, keys(processors.GetMap()), []string{"span-metrics"})
+		})
+	})
+}
+
+func TestOverridesAPI_PATCH(t *testing.T) {
+	util.RunIntegrationTests(t, util.TestHarnessConfig{
+		ConfigOverlay:  configOverrides,
+		DeploymentMode: util.DeploymentModeSingleBinary,
+		Backends:       util.BackendObjectStorageS3, // this test fails on other backends b/c it's testing specific etag related code
+	}, func(h *util.TempoHarness) {
+		t.Run("with no existing config creates new config", func(t *testing.T) {
+			apiClient := h.APIClientHTTP("tenant-patch-1")
+
+			patch := &client.Limits{
+				MetricsGenerator: client.LimitsMetricsGenerator{
+					DisableCollection: boolPtr(true),
+				},
+			}
+			returnedLimits, etag, err := apiClient.PatchOverrides(patch)
+			require.NoError(t, err)
+			require.NotEmpty(t, etag)
+
+			// verify returned limits match patch
+			disableCollection, ok := returnedLimits.GetMetricsGenerator().GetDisableCollection()
+			require.True(t, ok)
+			require.True(t, disableCollection)
+
+			// verify config was created via GET
+			getLimits, getEtag, err := apiClient.GetOverrides()
+			require.NoError(t, err)
+			require.Equal(t, etag, getEtag)
+			require.Equal(t, returnedLimits, getLimits)
+		})
+
+		t.Run("preserves existing config sections", func(t *testing.T) {
+			apiClient := h.APIClientHTTP("tenant-patch-2")
+
+			// create initial config with processors
+			initialLimits := &client.Limits{
+				MetricsGenerator: client.LimitsMetricsGenerator{
+					DisableCollection: boolPtr(true),
+					Processors:        map[string]struct{}{"span-metrics": {}},
+				},
+			}
+			_, _, err := apiClient.PatchOverrides(initialLimits)
+			require.NoError(t, err)
+
+			// PATCH with additional processor config
+			patch := &client.Limits{
+				MetricsGenerator: client.LimitsMetricsGenerator{
+					Processor: client.LimitsMetricsGeneratorProcessor{
+						SpanMetrics: client.LimitsMetricsGeneratorProcessorSpanMetrics{
+							EnableInstanceLabel: boolPtr(false),
+						},
+					},
+				},
+			}
+			returnedLimits, _, err := apiClient.PatchOverrides(patch)
+			require.NoError(t, err)
+
+			// verify original fields are preserved and not wiped out
+			disableCollection, ok := returnedLimits.GetMetricsGenerator().GetDisableCollection()
+			require.True(t, ok)
+			require.True(t, disableCollection)
+
+			processors, ok := returnedLimits.GetMetricsGenerator().GetProcessors()
+			require.True(t, ok)
+			require.ElementsMatch(t, keys(processors.GetMap()), []string{"span-metrics"})
+
+			// verify new field was added via PATCH
+			enableInstanceLabel, ok := returnedLimits.GetMetricsGenerator().GetProcessor().GetSpanMetrics().GetEnableInstanceLabel()
+			require.True(t, ok)
+			require.False(t, enableInstanceLabel)
+		})
+
+		t.Run("merges nested processor configs", func(t *testing.T) {
+			apiClient := h.APIClientHTTP("tenant-patch-3")
+
+			// create initial config with service graphs
+			initialPatch := &client.Limits{
+				MetricsGenerator: client.LimitsMetricsGenerator{
+					Processor: client.LimitsMetricsGeneratorProcessor{
+						ServiceGraphs: client.LimitsMetricsGeneratorProcessorServiceGraphs{
+							HistogramBuckets: &[]float64{0.1, 0.2},
+						},
+					},
+				},
+			}
+			_, _, err := apiClient.PatchOverrides(initialPatch)
+			require.NoError(t, err)
+
+			// PATCH with span metrics config
+			spanMetricsPatch := &client.Limits{
+				MetricsGenerator: client.LimitsMetricsGenerator{
+					Processor: client.LimitsMetricsGeneratorProcessor{
+						SpanMetrics: client.LimitsMetricsGeneratorProcessorSpanMetrics{
+							HistogramBuckets: &[]float64{0.3, 0.4},
+						},
+					},
+				},
+			}
+			returnedLimits, _, err := apiClient.PatchOverrides(spanMetricsPatch)
+			require.NoError(t, err)
+
+			// verify both processor configs exist in returned limits
+			sgBuckets, ok := returnedLimits.GetMetricsGenerator().GetProcessor().GetServiceGraphs().GetHistogramBuckets()
+			require.True(t, ok)
+			require.Equal(t, []float64{0.1, 0.2}, sgBuckets)
+
+			smBuckets, ok := returnedLimits.GetMetricsGenerator().GetProcessor().GetSpanMetrics().GetHistogramBuckets()
+			require.True(t, ok)
+			require.Equal(t, []float64{0.3, 0.4}, smBuckets)
+		})
+
+		t.Run("overwrites field values", func(t *testing.T) {
+			apiClient := h.APIClientHTTP("tenant-patch-4")
+
+			// create initial config
+			initialPatch := &client.Limits{
+				MetricsGenerator: client.LimitsMetricsGenerator{
+					Processor: client.LimitsMetricsGeneratorProcessor{
+						ServiceGraphs: client.LimitsMetricsGeneratorProcessorServiceGraphs{
+							HistogramBuckets: &[]float64{0.1, 0.2},
+						},
+					},
+				},
+			}
+			_, _, err := apiClient.PatchOverrides(initialPatch)
+			require.NoError(t, err)
+
+			// PATCH with new histogram buckets (should overwrite)
+			updatedPatch := &client.Limits{
+				MetricsGenerator: client.LimitsMetricsGenerator{
+					Processor: client.LimitsMetricsGeneratorProcessor{
+						ServiceGraphs: client.LimitsMetricsGeneratorProcessorServiceGraphs{
+							HistogramBuckets: &[]float64{0.5, 0.6, 0.7},
+						},
+					},
+				},
+			}
+			returnedLimits, _, err := apiClient.PatchOverrides(updatedPatch)
+			require.NoError(t, err)
+
+			// verify histogram buckets were updated in returned limits
+			buckets, ok := returnedLimits.GetMetricsGenerator().GetProcessor().GetServiceGraphs().GetHistogramBuckets()
+			require.True(t, ok)
+			require.Equal(t, []float64{0.5, 0.6, 0.7}, buckets)
+		})
+
+		t.Run("empty top level config doesn't overwrites nested configs", func(t *testing.T) {
+			apiClient := h.APIClientHTTP("tenant-patch-5")
+
+			// create initial config
+			initialPatch := &client.Limits{
+				CostAttribution: client.CostAttribution{
+					Dimensions: &map[string]string{"service.name": "service_name"},
+				},
+				MetricsGenerator: client.LimitsMetricsGenerator{
+					Processor: client.LimitsMetricsGeneratorProcessor{
+						ServiceGraphs: client.LimitsMetricsGeneratorProcessorServiceGraphs{
+							HistogramBuckets: &[]float64{0.1, 0.2},
+						},
+					},
+				},
+			}
+			returnedLimits, _, err := apiClient.PatchOverrides(initialPatch)
+			require.NoError(t, err)
+
+			// verify histogram buckets were updated in returned limits
+			buckets, ok := returnedLimits.GetMetricsGenerator().GetProcessor().GetServiceGraphs().GetHistogramBuckets()
+			require.True(t, ok)
+			require.Equal(t, []float64{0.1, 0.2}, buckets)
+
+			// verify that CostAttribution config exists and not touched
+			dims, ok := returnedLimits.GetCostAttribution().GetDimensions()
+			require.True(t, ok)
+			require.Equal(t, map[string]string{"service.name": "service_name"}, dims)
+
+			// PATCH with empty metric generator and it doesn't unset nested configs
+			updatedPatch := &client.Limits{
+				MetricsGenerator: client.LimitsMetricsGenerator{},
+			}
+			_, _, err2 := apiClient.PatchOverrides(updatedPatch)
+			require.NoError(t, err2)
+			// doing a GET after PATCH to ensure we get fresh config
+			returnedLimits2, _, err := apiClient.GetOverrides()
+			require.NoError(t, err)
+
+			// verify histogram buckets exist in the config
+			config := returnedLimits2.GetMetricsGenerator()
+			require.NotNil(t, config)
+			buckets, ok = returnedLimits2.GetMetricsGenerator().GetProcessor().GetServiceGraphs().GetHistogramBuckets()
+			require.True(t, ok) // should not be wiped out
+			require.Equal(t, []float64{0.1, 0.2}, buckets)
+		})
+
+		t.Run("version changes after patch", func(t *testing.T) {
+			apiClient := h.APIClientHTTP("tenant-patch-6")
+
+			// Create initial config with POST
+			initialLimits := &client.Limits{
+				MetricsGenerator: client.LimitsMetricsGenerator{
+					DisableCollection: boolPtr(true),
+				},
+			}
+			setEtag, err := apiClient.SetOverrides(initialLimits, "0")
+			require.NoError(t, err)
+
+			// Patch config
+			patch := &client.Limits{
+				MetricsGenerator: client.LimitsMetricsGenerator{
+					Processor: client.LimitsMetricsGeneratorProcessor{
+						ServiceGraphs: client.LimitsMetricsGeneratorProcessorServiceGraphs{
+							HistogramBuckets: &[]float64{0.1, 0.2},
+						},
+					},
+				},
+			}
+			_, patchEtag, err := apiClient.PatchOverrides(patch)
+			require.NoError(t, err)
+
+			// Verify version changed
+			require.NotEqual(t, setEtag, patchEtag)
+			require.NotEmpty(t, patchEtag)
+		})
+
+		t.Run("handles complex nested config", func(t *testing.T) {
+			apiClient := h.APIClientHTTP("tenant-patch-7")
+
+			// Create comprehensive config via PATCH
+			patch := &client.Limits{
+				MetricsGenerator: client.LimitsMetricsGenerator{
+					DisableCollection:              boolPtr(true),
+					GenerateNativeHistograms:       histogramModePtr(histograms.HistogramMethodNative),
+					NativeHistogramMaxBucketNumber: uint32Ptr(200),
+					Processors:                     map[string]struct{}{"span-metrics": {}},
+					Processor: client.LimitsMetricsGeneratorProcessor{
+						SpanMetrics: client.LimitsMetricsGeneratorProcessorSpanMetrics{
+							EnableInstanceLabel: boolPtr(false),
+						},
+					},
+				},
+			}
+			returnedLimits, _, err := apiClient.PatchOverrides(patch)
+			require.NoError(t, err)
+
+			// Verify all fields were set correctly
+			disableCollection, ok := returnedLimits.GetMetricsGenerator().GetDisableCollection()
+			require.True(t, ok)
+			require.True(t, disableCollection)
+
+			generateNativeHistograms, ok := returnedLimits.GetMetricsGenerator().GetGenerateNativeHistograms()
+			require.True(t, ok)
+			require.Equal(t, histograms.HistogramMethodNative, generateNativeHistograms)
+
+			maxBucketNumber, ok := returnedLimits.GetMetricsGenerator().GetNativeHistogramMaxBucketNumber()
+			require.True(t, ok)
+			require.Equal(t, uint32(200), maxBucketNumber)
+
+			processors, ok := returnedLimits.GetMetricsGenerator().GetProcessors()
+			require.True(t, ok)
+			require.ElementsMatch(t, keys(processors.GetMap()), []string{"span-metrics"})
+
+			enableInstanceLabel, ok := returnedLimits.GetMetricsGenerator().GetProcessor().GetSpanMetrics().GetEnableInstanceLabel()
+			require.True(t, ok)
+			require.False(t, enableInstanceLabel)
+		})
+	})
+}
+
+func TestOverridesAPI_DELETE(t *testing.T) {
+	util.RunIntegrationTests(t, util.TestHarnessConfig{
+		ConfigOverlay:  configOverrides,
+		DeploymentMode: util.DeploymentModeSingleBinary,
+		Backends:       util.BackendObjectStorageS3, // this test fails on other backends b/c it's testing specific etag related code
+	}, func(h *util.TempoHarness) {
+		t.Run("config is deleted with correct etag", func(t *testing.T) {
+			apiClient := h.APIClientHTTP("tenant-delete-1")
+
+			// create initial config
+			limits := &client.Limits{
+				MetricsGenerator: client.LimitsMetricsGenerator{
+					DisableCollection: boolPtr(true),
+				},
+			}
+			setEtag, err := apiClient.SetOverrides(limits, "0")
+			require.NoError(t, err)
+
+			// delete config
+			err = apiClient.DeleteOverrides(setEtag)
+			require.NoError(t, err)
+
+			// verify config is gone
+			deletedLimits, deletedEtag, err := apiClient.GetOverrides()
+			require.Nil(t, deletedLimits)
+			require.Empty(t, deletedEtag)
+			require.ErrorIs(t, err, httpclient.ErrNotFound)
+		})
+
+		t.Run("API returns 428 without if-match header", func(t *testing.T) {
+			baseURL := h.BaseURL()
+
+			// create config first
+			apiClient := h.APIClientHTTP("tenant-delete-2")
+			limits := &client.Limits{
+				CostAttribution: client.CostAttribution{
+					Dimensions: &map[string]string{"region": "region_label"},
+				},
+			}
+			_, err := apiClient.SetOverrides(limits, "0")
+			require.NoError(t, err)
+
+			// try to delete without If-Match header
+			req, err := http.NewRequest(http.MethodDelete, fmt.Sprintf("%s/api/overrides", baseURL), nil)
+			require.NoError(t, err)
+			req.Header.Set("X-Scope-OrgID", "tenant-delete-2")
+
+			resp, err := http.DefaultClient.Do(req)
+			require.NoError(t, err)
+			defer resp.Body.Close()
+
+			require.Equal(t, http.StatusPreconditionRequired, resp.StatusCode)
+		})
+
+		t.Run("API returns 412 with wrong version ", func(t *testing.T) {
+			apiClient := h.APIClientHTTP("tenant-delete-wrongver")
+
+			// create config
+			limits := &client.Limits{
+				MetricsGenerator: client.LimitsMetricsGenerator{
+					DisableCollection: boolPtr(false),
+				},
+			}
+			setEtag, err := apiClient.SetOverrides(limits, "0")
+			require.NoError(t, err)
+			require.NotEmpty(t, setEtag)
+
+			// try to delete with wrong version
+			err = apiClient.DeleteOverrides("wrong-version")
+			require.ErrorContains(t, err, "412")
+
+			// verify config still exists
+			returnedLimits, gotEtag, err := apiClient.GetOverrides()
+			require.NoError(t, err)
+			require.Equal(t, setEtag, gotEtag)
+			require.Equal(t, limits, returnedLimits)
+		})
+	})
+}
+
+// Helper functions for overrides API tests
+
+func printLimits(limits *client.Limits, version string) {
+	var str string
+	if limits != nil {
+		bytes, err := jsoniter.Marshal(limits)
+		if err == nil {
+			str = string(bytes)
+		}
+	}
+	fmt.Printf("* Overrides (version = %s): %+v\n", version, str)
+}
+
+func boolPtr(b bool) *bool {
+	return &b
+}
+
+func histogramModePtr(h histograms.HistogramMethod) *histograms.HistogramMethod {
+	return &h
+}
+
+func uint32Ptr(u uint32) *uint32 {
+	return &u
+}
+
+func keys(m map[string]struct{}) []string {
+	var keys []string
+	for k := range m {
+		keys = append(keys, k)
+	}
+	return keys
+}

integration/api/query_range_test.go

@@ -0,0 +1,803 @@
+package api
+
+import (
+	"errors"
+	"fmt"
+	"io"
+	"math"
+	"testing"
+	"time"
+
+	"github.com/google/uuid"
+	"github.com/jaegertracing/jaeger-idl/thrift-gen/jaeger"
+
+	"github.com/grafana/tempo/integration/util"
+	"github.com/grafana/tempo/pkg/httpclient"
+	"github.com/grafana/tempo/pkg/tempopb"
+	v1 "github.com/grafana/tempo/pkg/tempopb/common/v1"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+const (
+	configQueryRangeMaxSeries         = "config-query-range-max-series.yaml"
+	configQueryRangeMaxSeriesDisabled = "config-query-range-max-series-disabled.yaml"
+	configQueryRangeExemplars         = "config-query-range-exemplars.yaml"
+)
+
+type queryRangeRequest struct {
+	Query     string    `json:"query"`
+	Start     time.Time `json:"start"`     // default: now - 5m
+	End       time.Time `json:"end"`       // default: now + 1m
+	Step      string    `json:"step"`      // default: 5s
+	Exemplars int       `json:"exemplars"` // default: 100
+	noDefault bool      `json:"-"`         // if true, SetDefaults() will not set defaults`
+}
+
+func (r *queryRangeRequest) SetDefaults() {
+	if r.noDefault {
+		return
+	}
+	if r.Start.IsZero() {
+		r.Start = time.Now().Add(-5 * time.Minute)
+	}
+	if r.End.IsZero() {
+		r.End = time.Now().Add(time.Minute)
+	}
+	if r.Step == "" {
+		r.Step = "5s"
+	}
+	if r.Exemplars == 0 {
+		r.Exemplars = 100
+	}
+}
+
+func TestQueryRangeExemplars(t *testing.T) {
+	util.RunIntegrationTests(t, util.TestHarnessConfig{
+		ConfigOverlay: configQueryRangeExemplars,
+	}, func(h *util.TempoHarness) {
+		h.WaitTracesWritable(t)
+
+		ticker := time.NewTicker(500 * time.Millisecond)
+		defer ticker.Stop()
+		timer := time.NewTimer(10 * time.Second)
+		defer timer.Stop()
+
+		tracesSent := 0
+
+		// send one batch every 500ms for 10 seconds
+	sendLoop:
+		for {
+			select {
+			case <-ticker.C:
+				require.NoError(t, h.WriteJaegerBatch(
+					util.MakeThriftBatchWithSpanCountAttributeAndName(
+						1, "my operation",
+						"res_val", "span_val",
+						"res_attr", "span_attr",
+					), ""),
+				)
+				require.NoError(t, h.WriteJaegerBatch(
+					util.MakeThriftBatchWithSpanCountAttributeAndName(
+						1, "my operation",
+						"res_val2", "span_val2",
+						"res_attr", "span_attr",
+					), ""),
+				)
+				require.NoError(t, h.WriteJaegerBatch(
+					util.MakeThriftBatchWithSpanCountAttributeAndName(
+						1, "operation with high cardinality",
+						uuid.New().String(), uuid.New().String(),
+						"res_high_cardinality", "span_high_cardinality",
+					), ""),
+				)
+				tracesSent += 3
+			case <-timer.C:
+				break sendLoop
+			}
+		}
+
+		h.WaitTracesQueryable(t, tracesSent)
+
+		for _, exemplarsCase := range []struct {
+			name              string
+			exemplars         int
+			expectedExemplars int
+		}{
+			{
+				name:              "default",
+				exemplars:         0,
+				expectedExemplars: 100, // if set to 0, then limits to 100
+			},
+			{
+				name:              "5 exemplar",
+				exemplars:         5,
+				expectedExemplars: 5,
+			},
+			{
+				name:              "25 exemplars",
+				exemplars:         25,
+				expectedExemplars: 25,
+			},
+			{
+				name:              "capped exemplars",
+				exemplars:         1000,
+				expectedExemplars: 100, // capped to 100
+			},
+		} {
+			for _, query := range []string{
+				"{} | rate()",
+				"{} | compare({status=error})",
+				"{} | count_over_time()",
+				"{} | min_over_time(duration)",
+				"{} | max_over_time(duration)",
+				"{} | avg_over_time(duration)",
+				"{} | sum_over_time(duration)",
+				"{} | quantile_over_time(duration, .5)",
+				"{} | quantile_over_time(duration, .5, 0.9, 0.99)",
+
+				"{} | count_over_time() by (span.span_attr)",
+				"{} | count_over_time() by (resource.res_attr)",
+				"{} | count_over_time() by (.span_attr)",
+				"{} | count_over_time() by (.res_attr)",
+
+				"{} | histogram_over_time(duration)",
+				"{} | count_over_time() by (status)",
+				"{status != error} | count_over_time() by (status)",
+			} {
+				t.Run(fmt.Sprintf("%s: %s", exemplarsCase.name, query), func(t *testing.T) {
+					req := queryRangeRequest{
+						Query:     query,
+						Exemplars: exemplarsCase.exemplars,
+					}
+					callQueryRange(t, h, req, func(queryRangeRes *tempopb.QueryRangeResponse, err error) {
+						require.NoError(t, err)
+						require.NotNil(t, queryRangeRes)
+						require.GreaterOrEqual(t, len(queryRangeRes.GetSeries()), 1)
+
+						exemplarCount := 0
+
+						for _, series := range queryRangeRes.GetSeries() {
+							exemplarCount += len(series.GetExemplars())
+						}
+						assert.LessOrEqual(t, exemplarCount, exemplarsCase.expectedExemplars)
+						assert.GreaterOrEqual(t, exemplarCount, 1)
+					})
+				})
+			}
+		}
+
+		// check exemplars in more detail
+		for _, testCase := range []struct {
+			query                   string
+			targetAttribute         string
+			targetExemplarAttribute string
+		}{
+			{
+				query:                   "{} | quantile_over_time(duration, .9) by (span.span_attr)",
+				targetAttribute:         "span.span_attr",
+				targetExemplarAttribute: "span.span_attr",
+			},
+			{
+				query:                   "{} | quantile_over_time(duration, .9) by (resource.res_attr)",
+				targetAttribute:         "resource.res_attr",
+				targetExemplarAttribute: "resource.res_attr",
+			},
+			{
+				query:                   "{} | quantile_over_time(duration, .9) by (.span_attr)",
+				targetAttribute:         ".span_attr",
+				targetExemplarAttribute: "span.span_attr",
+			},
+			{
+				query:                   "{} | quantile_over_time(duration, .9) by (.res_attr)",
+				targetAttribute:         ".res_attr",
+				targetExemplarAttribute: "resource.res_attr",
+			},
+			{
+				query:                   "{} | rate() by (span.span_attr)",
+				targetAttribute:         "span.span_attr",
+				targetExemplarAttribute: "span.span_attr",
+			},
+			{
+				query:                   "{} | count_over_time() by (span.span_attr)",
+				targetAttribute:         "span.span_attr",
+				targetExemplarAttribute: "span.span_attr",
+			},
+			{
+				query:                   "{} | min_over_time(duration) by (span.span_attr)",
+				targetAttribute:         "span.span_attr",
+				targetExemplarAttribute: "span.span_attr",
+			},
+			{
+				query:                   "{} | max_over_time(duration) by (span.span_attr)",
+				targetAttribute:         "span.span_attr",
+				targetExemplarAttribute: "span.span_attr",
+			},
+			{
+				query:                   "{} | avg_over_time(duration) by (span.span_attr)",
+				targetAttribute:         "span.span_attr",
+				targetExemplarAttribute: "span.span_attr",
+			},
+			{
+				query:                   "{} | sum_over_time(duration) by (span.span_attr)",
+				targetAttribute:         "span.span_attr",
+				targetExemplarAttribute: "span.span_attr",
+			},
+		} {
+			t.Run(testCase.query, func(t *testing.T) {
+				req := queryRangeRequest{
+					Query:     testCase.query,
+					Exemplars: 100,
+				}
+				callQueryRange(t, h, req, func(queryRangeRes *tempopb.QueryRangeResponse, err error) {
+					require.NoError(t, err)
+					require.NotNil(t, queryRangeRes)
+					require.Equal(t, len(queryRangeRes.GetSeries()), 3) // value 1, value 2 and nil (high cardinality's span has no such attribute)
+
+					// Verify that all exemplars in this series belongs to the right series
+					// by matching attribute values
+					var skippedForNilAttr bool
+					for _, series := range queryRangeRes.Series {
+						// search attribute value for the series
+						var expectedAttrValue string
+						for _, label := range series.Labels {
+							if label.Key == testCase.targetAttribute {
+								expectedAttrValue = label.Value.GetStringValue()
+								break
+							}
+						}
+						if (expectedAttrValue == "" || expectedAttrValue == "nil") && !skippedForNilAttr { // one attribute is empty, so we skip it
+							skippedForNilAttr = true
+							continue
+						}
+						require.NotEmpty(t, expectedAttrValue)
+
+						// check attribute value in exemplars
+						for _, exemplar := range series.Exemplars {
+							var actualAttrValue string
+							for _, label := range exemplar.Labels {
+								if label.Key == testCase.targetExemplarAttribute {
+									actualAttrValue = label.Value.GetStringValue()
+									break
+								}
+							}
+							require.Equal(t, expectedAttrValue, actualAttrValue)
+						}
+					}
+				})
+			})
+		}
+
+		// invalid query (both http and grpc)
+		for name, req := range map[string]queryRangeRequest{
+			"invalid query": {
+				Query: "{. a}",
+			},
+		} {
+			t.Run(name, func(t *testing.T) {
+				req := req
+				callQueryRange(t, h, req, func(_ *tempopb.QueryRangeResponse, err error) {
+					require.ErrorContains(t, err, "unexpected END_ATTRIBUTE, expecting IDENTIFIER")
+				})
+			})
+		}
+
+		// invalid query (http only)
+		//  grpc accepts 0 step and provides a default
+		for name, req := range map[string]queryRangeRequest{
+			"step=0 (default step)": {
+				Query: "{} | count_over_time()",
+				Step:  "0",
+			},
+			"step=0s (default step)": {
+				Query: "{} | count_over_time()",
+				Step:  "0s",
+			},
+		} {
+			t.Run(name, func(t *testing.T) {
+				req := req
+				apiClient := h.APIClientHTTP("")
+				_, err := apiClient.MetricsQueryRange(req.Query, req.Start.UnixNano(), req.End.UnixNano(), req.Step, req.Exemplars)
+				require.ErrorContains(t, err, "response: 400")
+			})
+		}
+
+		// small step
+		t.Run("small step", func(t *testing.T) {
+			req := queryRangeRequest{Query: "{} | count_over_time()"}
+			req.SetDefaults()
+			req.Step = "35ms"
+
+			callQueryRange(t, h, req, func(_ *tempopb.QueryRangeResponse, err error) {
+				require.ErrorContains(t, err, "step of 35ms is too small, minimum step for given range is 36ms")
+			})
+		})
+
+		// query with empty results
+		for _, query := range []string{
+			// existing attribute, no traces
+			"{status=error} | count_over_time()",
+			// non-existing attribute, no traces
+			`{span.randomattr = "doesnotexist"} | count_over_time()`,
+		} {
+			t.Run(query, func(t *testing.T) {
+				callQueryRange(t, h, queryRangeRequest{Query: query, Exemplars: 100}, func(queryRangeRes *tempopb.QueryRangeResponse, err error) {
+					require.NoError(t, err)
+					require.NotNil(t, queryRangeRes)
+					// it has time series but they are empty and has no exemplars
+					require.GreaterOrEqual(t, len(queryRangeRes.GetSeries()), 1)
+					exemplarCount := 0
+					for _, series := range queryRangeRes.GetSeries() {
+						exemplarCount += len(series.GetExemplars())
+					}
+					require.Equal(t, 0, exemplarCount)
+				})
+			})
+		}
+
+		for _, testCase := range []struct {
+			name      string
+			query     string
+			step      string
+			converter func([]tempopb.Sample) float64
+		}{
+			{
+				name:      "count_over_time",
+				query:     "{} | count_over_time()",
+				step:      "1s",
+				converter: sumSamples,
+			},
+			{
+				name:      "sum_over_time",
+				query:     "{} | sum_over_time(duration)",
+				step:      "1s",
+				converter: sumSamples,
+			},
+			{
+				name:      "max_over_time",
+				query:     "{} | max_over_time(duration)",
+				step:      "1s",
+				converter: maxSamples,
+			},
+			{
+				name:      "min_over_time",
+				query:     "{} | min_over_time(duration)",
+				step:      "1s",
+				converter: minSamples,
+			},
+			{
+				name:      "1m step",
+				query:     "{} | count_over_time()",
+				step:      "1m",
+				converter: sumSamples,
+			},
+		} {
+			t.Run(testCase.name, func(t *testing.T) {
+				req := queryRangeRequest{
+					Query: testCase.query,
+					// Query range truncates the start and end to the step, while instant query does not.
+					// We need to truncate the start and end to the step to align the interval for query range and instant query.
+					Start: time.Now().Add(-5 * time.Minute).Truncate(time.Minute),
+					End:   time.Now().Add(time.Minute).Truncate(time.Minute),
+					Step:  testCase.step,
+				}
+
+				var expectedValue float64
+				callQueryRange(t, h, req, func(queryRangeRes *tempopb.QueryRangeResponse, err error) {
+					require.NoError(t, err)
+					require.NotNil(t, queryRangeRes)
+					require.Equal(t, 1, len(queryRangeRes.GetSeries()))
+
+					expectedValue = testCase.converter(queryRangeRes.Series[0].Samples)
+				})
+
+				instantQueryRes, err := callInstantQuery(h.APIClientHTTP(""), req)
+				require.NoError(t, err)
+				require.NotNil(t, instantQueryRes)
+				require.Equal(t, 1, len(instantQueryRes.GetSeries()))
+				require.InDelta(t, expectedValue, instantQueryRes.GetSeries()[0].Value, 0.000001)
+			})
+		}
+
+		t.Run("avg_over_time instant query", func(t *testing.T) {
+			req := queryRangeRequest{
+				Query: "{} | avg_over_time(duration)",
+				Start: time.Now().Add(-5 * time.Minute),
+				End:   time.Now().Add(time.Minute),
+			}
+
+			countReq := req
+			countReq.Query = "{} | count_over_time()"
+			countRes, err := callInstantQuery(h.APIClientHTTP(""), countReq)
+			require.NoError(t, err)
+			require.NotNil(t, countRes)
+			require.Equal(t, 1, len(countRes.GetSeries()))
+			count := countRes.GetSeries()[0].Value
+
+			sumReq := req
+			sumReq.Query = "{} | sum_over_time(duration)"
+			sumRes, err := callInstantQuery(h.APIClientHTTP(""), sumReq)
+			require.NoError(t, err)
+			require.NotNil(t, sumRes)
+			require.Equal(t, 1, len(sumRes.GetSeries()))
+			sum := sumRes.GetSeries()[0].Value
+
+			res, err := callInstantQuery(h.APIClientHTTP(""), req)
+			require.NoError(t, err)
+			require.NotNil(t, res)
+			require.Equal(t, 1, len(res.GetSeries()))
+			require.InDelta(t, sum/count, res.GetSeries()[0].Value, 0.000001)
+		})
+
+		for _, testCase := range []struct {
+			name        string
+			query       string
+			expectedNum int
+		}{
+			{
+				name:        "top 1 by span attribute",
+				query:       "{ } | rate() by (span.span_high_cardinality) | topk(1)",
+				expectedNum: 1,
+			},
+			{
+				name:        "top 10 by span attribute",
+				query:       "{ } | rate() by (span.span_high_cardinality) | topk(10)",
+				expectedNum: 10,
+			},
+			{
+				name:        "top 2 by resource attribute",
+				query:       "{ } | rate() by (resource.res_high_cardinality) | topk(2)",
+				expectedNum: 2,
+			},
+			{
+				name:        "bottom 1 by resource attribute",
+				query:       "{ } | rate() by (resource.res_high_cardinality) | bottomk(1)",
+				expectedNum: 1,
+			},
+			{
+				name:        "bootom 10 by resource attribute",
+				query:       "{ } | rate() by (resource.res_high_cardinality) | bottomk(10)",
+				expectedNum: 10,
+			},
+			{
+				name:        "bottom 2 by span attribute",
+				query:       "{ } | rate() by (span.span_high_cardinality) | bottomk(2)",
+				expectedNum: 2,
+			},
+		} {
+			t.Run(testCase.name, func(t *testing.T) {
+				req := queryRangeRequest{
+					Query: testCase.query,
+					Start: time.Now().Add(-5 * time.Minute),
+					End:   time.Now().Add(time.Minute),
+					Step:  "1m",
+				}
+
+				instantQueryRes, err := callInstantQuery(h.APIClientHTTP(""), req)
+				require.NoError(t, err)
+				require.NotNil(t, instantQueryRes)
+				require.Equal(t, testCase.expectedNum, len(instantQueryRes.GetSeries()))
+			})
+		}
+
+		for _, testCase := range []struct {
+			name              string
+			end               time.Time
+			step              string
+			expectedIntervals int
+		}{
+			// |---start|---|---end|
+			{name: "aligned", end: time.Now().Truncate(time.Minute), step: "1m", expectedIntervals: 3},
+			// |---|---start---|---|---end---|
+			{name: "unaligned", end: time.Now(), step: "1m", expectedIntervals: 4},
+			{name: "default step", end: time.Now(), step: "", expectedIntervals: 122},
+		} {
+			t.Run(testCase.name, func(t *testing.T) {
+				req := queryRangeRequest{
+					Query:     "{} | count_over_time()",
+					Start:     testCase.end.Add(-2 * time.Minute),
+					End:       testCase.end,
+					Step:      testCase.step,
+					noDefault: true,
+				}
+
+				callQueryRange(t, h, req, func(queryRangeRes *tempopb.QueryRangeResponse, err error) {
+					require.NoError(t, err)
+					require.NotNil(t, queryRangeRes)
+					series := queryRangeRes.GetSeries()
+					require.Equal(t, 1, len(series), "Expected 1 series for count_over_time query")
+					require.Equal(t, testCase.expectedIntervals, len(series[0].Samples))
+				})
+			})
+		}
+	})
+}
+
+func sumSamples(samples []tempopb.Sample) float64 {
+	var sum float64
+	for _, sample := range samples {
+		sum += sample.Value
+	}
+	return sum
+}
+
+func maxSamples(samples []tempopb.Sample) float64 {
+	maxValue := math.Inf(-1)
+	for _, sample := range samples {
+		if sample.Value > maxValue {
+			maxValue = sample.Value
+		}
+	}
+	return maxValue
+}
+
+func minSamples(samples []tempopb.Sample) float64 {
+	minValue := math.Inf(1)
+	for _, sample := range samples {
+		if sample.Value < minValue {
+			minValue = sample.Value
+		}
+	}
+	return minValue
+}
+
+// TestQueryRangeSingleTrace checks count for a single trace
+// Single trace creates a block with startTime == endTime
+// which covers a few edge cases under the hood.
+func TestQueryRangeSingleTrace(t *testing.T) {
+	util.RunIntegrationTests(t, util.TestHarnessConfig{
+		DeploymentMode: util.DeploymentModeSingleBinary, // for unknown reasons this fails on microservices mode. TODO: figure that crap out
+	}, func(h *util.TempoHarness) {
+		h.WaitTracesWritable(t)
+		// Emit a single trace
+		require.NoError(t, h.WriteJaegerBatch(util.MakeThriftBatch(), ""))
+
+		h.WaitTracesQueryable(t, 1)
+
+		// Query the trace by count. As we have only one trace, we should get one dot with value 1
+		query := "{} | count_over_time()"
+		callQueryRange(t, h, queryRangeRequest{Query: query, Exemplars: 100}, func(queryRangeRes *tempopb.QueryRangeResponse, err error) {
+			require.NoError(t, err)
+			require.NotNil(t, queryRangeRes)
+			require.Equal(t, len(queryRangeRes.GetSeries()), 1)
+
+			series := queryRangeRes.GetSeries()[0]
+			assert.Equal(t, len(series.GetExemplars()), 1)
+
+			var sum float64
+			for _, sample := range series.GetSamples() {
+				sum += sample.Value
+			}
+			require.InDelta(t, sum, 1, 0.000001)
+		})
+	})
+}
+
+func TestQueryRangeMaxSeries(t *testing.T) {
+	util.RunIntegrationTests(t, util.TestHarnessConfig{
+		ConfigOverlay: configQueryRangeMaxSeries,
+	}, func(h *util.TempoHarness) {
+		h.WaitTracesWritable(t)
+
+		ticker := time.NewTicker(500 * time.Millisecond)
+		defer ticker.Stop()
+		timer := time.NewTimer(5 * time.Second)
+		defer timer.Stop()
+
+		tracesSent := 0
+	sendLoop:
+		for {
+			select {
+			case <-ticker.C:
+				require.NoError(t, h.WriteJaegerBatch(util.MakeThriftBatch(), ""))
+				tracesSent++
+			case <-timer.C:
+				break sendLoop
+			}
+		}
+
+		require.Greater(t, tracesSent, 3)
+		h.WaitTracesQueryable(t, tracesSent)
+
+		callQueryRange(t, h, queryRangeRequest{
+			Query:     "{} | rate() by (span:id)",
+			Start:     time.Now().Add(-5 * time.Minute),
+			End:       time.Now(),
+			Step:      "5s",
+			Exemplars: 100,
+		}, func(queryRangeRes *tempopb.QueryRangeResponse, err error) {
+			require.NoError(t, err)
+			require.NotNil(t, queryRangeRes)
+			require.Equal(t, tempopb.PartialStatus_PARTIAL, queryRangeRes.GetStatus())
+			require.Equal(t, "Response exceeds maximum series limit of 3, a partial response is returned. Warning: the accuracy of each individual value is not guaranteed.", queryRangeRes.GetMessage())
+			require.Equal(t, 3, len(queryRangeRes.GetSeries()))
+		})
+	})
+}
+
+func TestQueryRangeMaxSeriesDisabled(t *testing.T) {
+	util.RunIntegrationTests(t, util.TestHarnessConfig{
+		ConfigOverlay: configQueryRangeMaxSeriesDisabled,
+	}, func(h *util.TempoHarness) {
+		h.WaitTracesWritable(t)
+
+		ticker := time.NewTicker(500 * time.Millisecond)
+		defer ticker.Stop()
+		timer := time.NewTimer(5 * time.Second)
+		defer timer.Stop()
+		spanCount := 0
+	sendLoop:
+		for {
+			select {
+			case <-ticker.C:
+				require.NoError(t, h.WriteJaegerBatch(util.MakeThriftBatch(), ""))
+				spanCount++
+			case <-timer.C:
+				break sendLoop
+			}
+		}
+
+		// Wait for traces to be flushed to blocks. spanCount happens to make traces count
+		h.WaitTracesQueryable(t, spanCount)
+
+		callQueryRange(t, h, queryRangeRequest{
+			Query:     "{} | rate() by (span:id)",
+			Start:     time.Now().Add(-5 * time.Minute),
+			End:       time.Now(),
+			Step:      "5s",
+			Exemplars: 100,
+		}, func(queryRangeRes *tempopb.QueryRangeResponse, err error) {
+			require.NoError(t, err)
+			require.NotNil(t, queryRangeRes)
+			require.Equal(t, tempopb.PartialStatus_COMPLETE, queryRangeRes.GetStatus())
+			require.Equal(t, spanCount, len(queryRangeRes.GetSeries()))
+		})
+	})
+}
+
+func TestQueryRangeTypeHandling(t *testing.T) {
+	util.RunIntegrationTests(t, util.TestHarnessConfig{}, func(h *util.TempoHarness) {
+		h.WaitTracesWritable(t)
+
+		// Emit spans where the attribute names but the values are
+		// string and int with the same textual representation.
+		t1 := util.MakeThriftBatch()
+		t1.Spans[0].Tags = append(t1.Spans[0].Tags, &jaeger.Tag{
+			Key:   "foo",
+			VType: jaeger.TagType_STRING,
+			VStr:  strptr("123"),
+		})
+		require.NoError(t, h.WriteJaegerBatch(t1, ""))
+
+		t2 := util.MakeThriftBatch()
+		t2.Spans[0].Tags = append(t2.Spans[0].Tags, &jaeger.Tag{
+			Key:   "foo",
+			VType: jaeger.TagType_LONG,
+			VLong: int64ptr(123),
+		})
+		require.NoError(t, h.WriteJaegerBatch(t2, ""))
+
+		// Wait for traces to be flushed to blocks
+		h.WaitTracesQueryable(t, 2)
+
+		callQueryRange(t, h, queryRangeRequest{
+			Query:     "{} | rate() by (span:id)",
+			Start:     time.Now().Add(-5 * time.Minute),
+			End:       time.Now(),
+			Step:      "5s",
+			Exemplars: 100,
+		}, func(queryRangeRes *tempopb.QueryRangeResponse, err error) {
+			require.NoError(t, err)
+			require.NotNil(t, queryRangeRes)
+			require.Equal(t, tempopb.PartialStatus_COMPLETE, queryRangeRes.GetStatus())
+			require.Equal(t, 2, len(queryRangeRes.GetSeries()))
+		})
+	})
+}
+
+func callInstantQuery(apiClient *httpclient.Client, req queryRangeRequest) (*tempopb.QueryInstantResponse, error) {
+	req.SetDefaults()
+	return apiClient.MetricsQueryInstant(req.Query, req.Start.UnixNano(), req.End.UnixNano(), req.Exemplars)
+}
+
+func callQueryRange(t *testing.T, h *util.TempoHarness, req queryRangeRequest, fn func(*tempopb.QueryRangeResponse, error)) {
+	req.SetDefaults()
+
+	apiClient := h.APIClientHTTP("")
+	fn(apiClient.MetricsQueryRange(req.Query, req.Start.UnixNano(), req.End.UnixNano(), req.Step, req.Exemplars))
+
+	step := time.Duration(0)
+	if req.Step != "" {
+		var err error
+		step, err = time.ParseDuration(req.Step)
+		require.NoError(t, err)
+	}
+
+	grpcClient, ctx, err := h.APIClientGRPC("")
+	require.NoError(t, err)
+
+	stream, err := grpcClient.MetricsQueryRange(ctx, &tempopb.QueryRangeRequest{
+		Query:     req.Query,
+		Start:     uint64(req.Start.UnixNano()),
+		End:       uint64(req.End.UnixNano()),
+		Step:      uint64(step.Nanoseconds()),
+		Exemplars: uint32(req.Exemplars),
+	})
+	require.NoError(t, err)
+
+	finalResponse := &tempopb.QueryRangeResponse{
+		Metrics: &tempopb.SearchMetrics{},
+	}
+	var finalError error
+	for {
+		t.Logf("recv")
+		resp, err := stream.Recv()
+		if resp != nil {
+			t.Logf("resp: %+v, count: %d", resp, len(resp.GetSeries()))
+			naiveQueryRangeCombine(resp, finalResponse)
+			t.Logf("resp: %d, finalResponse: %d", len(resp.GetSeries()), len(finalResponse.GetSeries()))
+		}
+		if errors.Is(err, io.EOF) {
+			break
+		}
+		if err != nil {
+			finalError = err
+			break
+		}
+	}
+	fn(finalResponse, finalError)
+}
+
+func strptr(s string) *string {
+	return &s
+}
+
+func int64ptr(i int64) *int64 {
+	return &i
+}
+
+// naiveQueryRangeCombine makes assumptions about the data being sent from Tempo. it assumes that labels orders are always
+// the same and that samples and exemplars do not need to be deduped.
+func naiveQueryRangeCombine(rNew, rInto *tempopb.QueryRangeResponse) {
+	rIntoSeries := map[string]*tempopb.TimeSeries{}
+	for _, series := range rInto.GetSeries() {
+		rIntoSeries[keyFromLabels(series.GetLabels())] = series
+	}
+
+	for _, newSeries := range rNew.GetSeries() {
+		key := keyFromLabels(newSeries.GetLabels())
+		if intoSeries, ok := rIntoSeries[key]; ok {
+			intoSeries.Exemplars = append(intoSeries.Exemplars, newSeries.Exemplars...)
+			intoSeries.Samples = append(intoSeries.Samples, newSeries.Samples...)
+		} else {
+			rIntoSeries[key] = newSeries
+		}
+	}
+
+	// Rebuild the series slice from the map
+	rInto.Series = make([]*tempopb.TimeSeries, 0, len(rIntoSeries))
+	for _, series := range rIntoSeries {
+		rInto.Series = append(rInto.Series, series)
+	}
+
+	if rInto.Message == "" {
+		rInto.Message = rNew.Message
+	}
+	if rInto.Status == 0 {
+		rInto.Status = rNew.Status
+	}
+
+	// metrics?
+	rInto.Metrics.CompletedJobs += rNew.Metrics.CompletedJobs
+	rInto.Metrics.InspectedBytes += rNew.Metrics.InspectedBytes
+	rInto.Metrics.InspectedTraces += rNew.Metrics.InspectedTraces
+	rInto.Metrics.InspectedSpans += rNew.Metrics.InspectedSpans
+}
+
+func keyFromLabels(labels []v1.KeyValue) string {
+	key := ""
+	for _, label := range labels {
+		key += label.Key + "=" + label.Value.GetStringValue() + ","
+	}
+	return key
+}

integration/api/tenant_validation_test.go

@@ -0,0 +1,146 @@
+package api
+
+import (
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/grafana/dskit/tenant"
+	"github.com/grafana/tempo/integration/util"
+	tempoUtil "github.com/grafana/tempo/pkg/util"
+	"github.com/stretchr/testify/require"
+)
+
+func TestInvalidTenants(t *testing.T) {
+	util.RunIntegrationTests(t, util.TestHarnessConfig{
+		ConfigOverlay:  "config-multi-tenant.yaml",
+		DeploymentMode: util.DeploymentModeSingleBinary,
+	}, func(h *util.TempoHarness) {
+		h.WaitTracesWritable(t)
+
+		longTenant := strings.Repeat("a", tenant.MaxTenantIDLength+1)
+
+		testCases := []struct {
+			name        string
+			orgID       string
+			multitenant bool
+			errContains string
+		}{
+			{
+				name:        "invalid unsupported character",
+				orgID:       "tenant#123",
+				errContains: "unsupported character",
+			},
+			{
+				name:        "invalid slash character",
+				orgID:       "tenant/123",
+				errContains: "unsupported character",
+			},
+			{
+				name:        "invalid unsafe path segment",
+				orgID:       "..",
+				errContains: "tenant ID is '.' or '..'",
+			},
+			{
+				name:        "invalid too long tenant id",
+				orgID:       longTenant,
+				errContains: "tenant ID is too long",
+			},
+			{
+				name:        "invalid empty tenant",
+				orgID:       "",
+				errContains: "no org id",
+			},
+			{
+				name:        "invalid leading separator",
+				orgID:       "|tenantA",
+				multitenant: true,
+				errContains: "no org id",
+			},
+			{
+				name:        "invalid trailing separator",
+				orgID:       "tenantA|",
+				multitenant: true,
+				errContains: "no org id",
+			},
+			{
+				name:        "invalid double separator",
+				orgID:       "tenantA||tenantB",
+				multitenant: true,
+				errContains: "no org id",
+			},
+			{
+				name:        "invalid multi tenant unsupported character",
+				orgID:       "tenantA|tenant#B",
+				multitenant: true,
+				errContains: "tenant#B",
+			},
+			{
+				name:        "invalid multi tenant unsafe segment",
+				orgID:       "tenantA|..",
+				multitenant: true,
+				errContains: "tenant ID is '.' or '..'",
+			},
+			{
+				name:        "invalid multi tenant too long segment",
+				orgID:       "tenantA|" + longTenant,
+				multitenant: true,
+				errContains: "tenant ID is too long",
+			},
+		}
+
+		for _, tc := range testCases {
+			t.Run(tc.name, func(t *testing.T) {
+				assertHTTP400 := func(t *testing.T, err error) {
+					require.Error(t, err)
+					// no org ID in single-tenant returns 401 from Auth middleware
+					// others return 400
+					require.Contains(t, err.Error(), "response: 40")
+					require.Contains(t, err.Error(), tc.errContains)
+				}
+
+				apiClient := h.APIClientHTTP(tc.orgID)
+
+				_, err := apiClient.QueryTrace("0047ac2c027451d89e3f3ba6d2a6b7b")
+				assertHTTP400(t, err)
+
+				_, err = apiClient.QueryTraceV2("0047ac2c027451d89e3f3ba6d2a6b7b")
+				assertHTTP400(t, err)
+
+				_, err = apiClient.SearchTraceQL("{}")
+				assertHTTP400(t, err)
+
+				_, err = apiClient.SearchTags()
+				assertHTTP400(t, err)
+
+				_, err = apiClient.SearchTagsV2()
+				assertHTTP400(t, err)
+
+				_, err = apiClient.SearchTagValues("span.name")
+				assertHTTP400(t, err)
+
+				_, err = apiClient.SearchTagValuesV2("span.name", "{}")
+				assertHTTP400(t, err)
+
+				if !tc.multitenant {
+					_, err := apiClient.MetricsSummary("{}", "name", 0, 0)
+					assertHTTP400(t, err)
+
+					_, err = apiClient.MetricsQueryRange("{} | count_over_time()", 0, 0, "", 0)
+					assertHTTP400(t, err)
+				}
+			})
+
+			if !tc.multitenant {
+				t.Run(tc.name+"_write", func(t *testing.T) {
+					writeTraceInfo := tempoUtil.NewTraceInfo(time.Now(), tc.orgID)
+					writeErr := h.WriteTraceInfo(writeTraceInfo, tc.orgID)
+
+					require.Error(t, writeErr)
+					require.Contains(t, writeErr.Error(), "code = InvalidArgument")
+					require.Contains(t, writeErr.Error(), tc.errContains)
+				})
+			}
+		}
+	})
+}

integration/backendscheduler/backendscheduler_test.go

@@ -1,378 +0,0 @@
-package backendscheduler
-
-import (
-	"context"
-	"fmt"
-	"os"
-	"strconv"
-	"testing"
-	"time"
-
-	"github.com/go-kit/log"
-	"github.com/grafana/e2e"
-	"github.com/grafana/tempo/cmd/tempo/app"
-	e2eBackend "github.com/grafana/tempo/integration/e2e/backend"
-	"github.com/grafana/tempo/integration/util"
-	"github.com/grafana/tempo/pkg/model"
-	"github.com/grafana/tempo/pkg/tempopb"
-	"github.com/grafana/tempo/pkg/util/test"
-	"github.com/grafana/tempo/tempodb"
-	"github.com/grafana/tempo/tempodb/backend"
-	"github.com/grafana/tempo/tempodb/encoding"
-	"github.com/grafana/tempo/tempodb/encoding/common"
-	"github.com/grafana/tempo/tempodb/wal"
-	io_prometheus_client "github.com/prometheus/client_model/go"
-	"github.com/prometheus/prometheus/model/labels"
-	"github.com/stretchr/testify/require"
-	"gopkg.in/yaml.v2"
-)
-
-const (
-	tenant = "test"
-)
-
-func TestBackendSchedulerConfigurations(t *testing.T) {
-	cases := []struct {
-		name       string
-		configFile string
-	}{
-		{
-			name:       "default",
-			configFile: "config.yaml",
-		},
-	}
-
-	for _, tc := range cases {
-		t.Run(tc.name, func(t *testing.T) {
-			testWithConfig(t, tc.configFile)
-		})
-	}
-}
-
-func testWithConfig(t *testing.T, configFile string) {
-	ctx, cancel := context.WithCancel(context.Background())
-	defer cancel()
-
-	s, err := e2e.NewScenario("tempo-integration")
-	require.NoError(t, err)
-	defer s.Close()
-
-	cfg := app.Config{}
-	buff, err := os.ReadFile(configFile)
-	require.NoError(t, err)
-	err = yaml.UnmarshalStrict(buff, &cfg)
-	require.NoError(t, err)
-
-	b, err := e2eBackend.New(s, cfg)
-	require.NoError(t, err)
-
-	err = b.WaitReady()
-	require.NoError(t, err)
-
-	err = b.Ready()
-	require.NoError(t, err)
-
-	// Give some time for startup
-	time.Sleep(1 * time.Second)
-
-	require.NoError(t, util.CopyFileToSharedDir(s, configFile, "config.yaml"))
-
-	e := b.Endpoint(b.HTTPPort())
-	t.Logf("Endpoint: %s", e)
-
-	scheduler := util.NewTempoTarget("backend-scheduler", "config.yaml")
-	require.NoError(t, s.StartAndWaitReady(scheduler))
-
-	// Setup tempodb with local backend
-	tempodbWriter := setupBackendWithEndpoint(t, &cfg.StorageConfig.Trace, e)
-
-	cases := []struct {
-		name                      string
-		tenantCount               int
-		blockCount                int
-		expectedBlocks            int // accumulated expected blocks.  Each test adds this to the total
-		expectedOutstandingBlocks int // accumulated expected outstanding blocks.  Each test adds this to the total, but 1 outstanding block is between 2-4 blocks.
-	}{
-		{
-			name:                      "a bunch of tenants with 1 block each is 0 outstanding",
-			tenantCount:               11,
-			blockCount:                1,
-			expectedBlocks:            1,
-			expectedOutstandingBlocks: 0,
-		},
-		{
-			name:                      "a bunch of tenants with 12 blocks each 12 outstanding",
-			tenantCount:               11,
-			blockCount:                12,
-			expectedBlocks:            12,
-			expectedOutstandingBlocks: 12,
-		},
-	}
-	var tenants []string
-
-	type expectations struct {
-		blocks      int
-		outstanding int
-	}
-
-	// We continue to write blocks through these tests, so our expectations need
-	// to grow as well.
-	accumulant := make(map[string]*expectations)
-	totalOutstanding := 0
-	totalBlocksWritten := 0
-
-	for _, tc := range cases {
-		t.Run(tc.name, func(t *testing.T) {
-			// Populate the backend with the specified number of tenants, blocks, and records
-			tenants = populateBackend(ctx, t, tempodbWriter, tc.tenantCount, tc.blockCount, 1)
-
-			totalBlocksWritten += tc.tenantCount * tc.blockCount
-
-			if _, ok := accumulant[tenant]; !ok {
-				accumulant[tenant] = &expectations{}
-			}
-
-			accumulant[tenant].blocks += tc.expectedBlocks
-			accumulant[tenant].outstanding += tc.expectedOutstandingBlocks
-
-			// Check the metrics for each tenant
-			for _, tenantID := range tenants {
-				tenantMatcher := e2e.WithLabelMatchers(&labels.Matcher{Type: labels.MatchEqual, Name: "tenant", Value: tenantID})
-
-				expectedBlocks := accumulant[tenant].blocks
-				expectedOutstanding := accumulant[tenant].outstanding
-				totalOutstanding += expectedOutstanding
-
-				t.Logf("Waiting for %d blocks in the blocklist for tenant: %s", expectedBlocks, tenantID)
-
-				require.NoError(t, scheduler.WaitSumMetricsWithOptions(e2e.Equals(float64(expectedBlocks)), []string{"tempodb_blocklist_length"},
-					e2e.WaitMissingMetrics,
-					tenantMatcher,
-					printMetricValue(t, fmt.Sprintf("%d", expectedBlocks), "tempodb_blocklist_length"),
-				))
-			}
-		})
-	}
-
-	time.Sleep(8 * time.Second) // Wait for polling and measurement to catch up
-
-	// Capture the total blocks which have been written
-	totalBlocksPreCompactions, err := scheduler.SumMetrics([]string{"tempodb_blocklist_length"})
-	require.NoError(t, err)
-	require.Len(t, totalBlocksPreCompactions, 1, "expected only one blocklist length metric")
-	require.Equal(t, float64(totalBlocksWritten), totalBlocksPreCompactions[0], "expected total blocks to match the sum of all tenants")
-
-	// NOTE: the compaction provider has a channel capacity of 1, and the
-	// backendscheduler has a channel capacity of 1.  This means that the
-	// compaction provider can create 4 jobs before they are processed in the
-	// Next call of the scheduler.
-	// - one recorded before pushing into the compaction provider channel
-	// - one in the compaction provider channel
-	// - one between the provider pull and the scheduler push
-	// - one in the merged channel
-
-	// NOTE: Since the measurement of outstanding blocks also skips the blocks
-	// from jobs which have not been processed, we expect our outstanding blocks
-	// to be N-M, where N is the actual outstanding blocks and M is the number of
-	// blocks attached to jobs which have not been recorded by the scheduler. The
-	// order of tenants is non-defined when all tenants have the same block
-	// count, so we don't know which tenant will be first. Measure total
-	// outstanding blocks instead of per tenant for this reason.
-
-	// NOTE: Due to timing and window selection, we may not fully populate a job,
-	// and we may end up with jobs which between 2 and 4 blocks outstanding.
-	// - 4 jobs not yet recorded by the scheduler (no worker running yet)
-	// - between 2 and 4 blocks per job (default settings)
-	expectedTotalOutstandingMin := totalOutstanding - 16
-	expectedTotalOutstandingMax := totalOutstanding - 8
-
-	outstanding, err := scheduler.SumMetrics([]string{"tempodb_compaction_outstanding_blocks"})
-	require.NoError(t, err)
-	require.Len(t, outstanding, 1, "expected only one outstanding block metric")
-	t.Logf("Total outstanding blocks: %f", outstanding[0])
-	t.Logf("Expected outstanding blocks range: [%d, %d]", expectedTotalOutstandingMin, expectedTotalOutstandingMax)
-	t.Logf("Total outstanding blocks: %d", totalOutstanding)
-	require.LessOrEqual(t, outstanding[0], float64(expectedTotalOutstandingMax), "unexpected maximum outstanding blocks count")
-	require.GreaterOrEqual(t, outstanding[0], float64(expectedTotalOutstandingMin), "unexpected minimum outstanding blocks count")
-
-	// Delay starting the work to ensure we have a clean state of the data before
-	// the worker starts processing jobs.
-	worker := util.NewTempoTarget("backend-worker", "config.yaml")
-	require.NoError(t, s.StartAndWaitReady(worker))
-
-	// Allow the worker some time to process the blocks.
-	// Wait until the last tenant has processed its blocks
-
-	for _, tenantID := range tenants {
-		t.Run(fmt.Sprintf("work-finished-check-wait-%s", tenantID), func(t *testing.T) {
-			tenantMatcher := e2e.WithLabelMatchers(&labels.Matcher{Type: labels.MatchEqual, Name: "tenant", Value: tenantID})
-
-			// We should have at least 1 block for each tenant
-			require.NoError(t, scheduler.WaitSumMetricsWithOptions(e2e.GreaterOrEqual(1.0), []string{"tempodb_blocklist_length"},
-				e2e.WaitMissingMetrics,
-				tenantMatcher,
-				printMetricValue(t, fmt.Sprintf("%f+", 1.0), "tempodb_blocklist_length"),
-			))
-		})
-	}
-
-	time.Sleep(5 * time.Second)
-
-	// Stop the worker to ensure it is not running while we check the metrics.
-	require.NoError(t, s.Stop(worker))
-
-	// Require that all jobs which have been processed are not failed.
-	_, err = scheduler.SumMetrics([]string{"tempo_backend_scheduler_jobs_failed_total"})
-	require.Error(t, err, "metric not found")
-
-	totalBlocksPostCompactions, err := scheduler.SumMetrics([]string{"tempodb_blocklist_length"})
-	require.NoError(t, err)
-	require.Len(t, totalBlocksPostCompactions, 1, "expected only one blocklist length metric")
-	require.Less(t, totalBlocksPostCompactions[0], totalBlocksPreCompactions[0], "expected total blocks to be less after compaction")
-
-	// Some variance in the number of expected due to the notes above.  We should
-	// expect that a fully compacted tenant has between 1 and 4 blocks, and
-	// between 0 and 1 outstanding blocks.  The sleep above should be enough to
-	// allow the worker to finish processing all the outstanding blocks.
-	expectedMin := &expectations{
-		blocks:      1,
-		outstanding: 0,
-	}
-
-	expectedMax := &expectations{
-		blocks:      4,
-		outstanding: 1,
-	}
-
-	for _, tenantID := range tenants {
-		t.Run(fmt.Sprintf("prestop-check-%s", tenantID), func(t *testing.T) {
-			tenantMatcher := e2e.WithLabelMatchers(&labels.Matcher{Type: labels.MatchEqual, Name: "tenant", Value: tenantID})
-
-			t.Logf("Waiting for blocklist metric: %s: %d-%d", tenantID, expectedMin.blocks, expectedMax.blocks)
-
-			require.NoError(t, scheduler.WaitSumMetricsWithOptions(e2e.Between(float64(expectedMin.blocks), float64(expectedMax.blocks)), []string{"tempodb_blocklist_length"},
-				e2e.WaitMissingMetrics,
-				tenantMatcher,
-				printMetricValue(t, fmt.Sprintf("%d-%d", expectedMin.blocks, expectedMax.blocks), "tempodb_blocklist_length"),
-			))
-
-			t.Logf("Waiting for outstanding blocks metric: %s: %d-%d", tenantID, expectedMin.outstanding, expectedMax.outstanding)
-
-			require.NoError(t, scheduler.WaitSumMetricsWithOptions(e2e.Between(float64(expectedMin.outstanding), float64(expectedMax.outstanding)), []string{"tempodb_compaction_outstanding_blocks"},
-				e2e.WaitMissingMetrics,
-				tenantMatcher,
-				printMetricValue(t, fmt.Sprintf("%d-%d", expectedMin.outstanding, expectedMax.outstanding), "tempodb_compaction_outstanding_blocks"),
-			))
-		})
-	}
-
-	// Stop and restart the scheduler to ensure we replay the state correctly.
-	// It should match the outstanding blocks above since we have not modified
-	// the backend.
-	require.NoError(t, s.Stop(scheduler))
-	require.NoError(t, s.StartAndWaitReady(scheduler, worker))
-
-	time.Sleep(2 * time.Second)
-
-	for _, tenantID := range tenants {
-		t.Run(fmt.Sprintf("final-check-%s", tenantID), func(t *testing.T) {
-			tenantMatcher := e2e.WithLabelMatchers(&labels.Matcher{Type: labels.MatchEqual, Name: "tenant", Value: tenantID})
-
-			t.Logf("Waiting for blocklist metric: %s: %d-%d", tenantID, expectedMin.blocks, expectedMax.blocks)
-
-			require.NoError(t, scheduler.WaitSumMetricsWithOptions(e2e.Between(float64(expectedMin.blocks), float64(expectedMax.blocks)), []string{"tempodb_blocklist_length"},
-				e2e.WaitMissingMetrics,
-				tenantMatcher,
-				printMetricValue(t, fmt.Sprintf("%d-%d", expectedMin.blocks, expectedMax.blocks), "tempodb_blocklist_length"),
-			))
-
-			t.Logf("Waiting for outstanding blocks metric: %s: %d-%d", tenantID, expectedMin.outstanding, expectedMax.outstanding)
-
-			require.NoError(t, scheduler.WaitSumMetricsWithOptions(e2e.Between(float64(expectedMin.outstanding), float64(expectedMax.outstanding)), []string{"tempodb_compaction_outstanding_blocks"},
-				e2e.WaitMissingMetrics,
-				tenantMatcher,
-				printMetricValue(t, fmt.Sprintf("%d-%d", expectedMin.outstanding, expectedMax.outstanding), "tempodb_compaction_outstanding_blocks"),
-			))
-		})
-	}
-}
-
-func printValues(t *testing.T, expected string, metric string) e2e.GetMetricValueFunc {
-	return func(m *io_prometheus_client.Metric) float64 {
-		v := e2e.DefaultMetricsOptions.GetValue(m)
-		t.Logf("metric %q: label %q: current %f, expected %s", metric, m.GetLabel()[0].GetValue(), v, expected)
-		return v
-	}
-}
-
-func printMetricValue(t *testing.T, expectedValue string, metric string) e2e.MetricsOption {
-	return func(opts *e2e.MetricsOptions) {
-		opts.GetValue = printValues(t, expectedValue, metric)
-	}
-}
-
-func setupBackendWithEndpoint(t testing.TB, cfg *tempodb.Config, endpoint string) tempodb.Writer {
-	cfg.Block = &common.BlockConfig{
-		IndexDownsampleBytes: 11,
-		BloomFP:              .01,
-		BloomShardSizeBytes:  100_000,
-		Version:              encoding.LatestEncoding().Version(),
-		Encoding:             backend.EncNone,
-		IndexPageSizeBytes:   1000,
-		RowGroupSizeBytes:    30_000_000,
-		DedicatedColumns:     backend.DedicatedColumns{{Scope: "span", Name: "key", Type: "string"}},
-	}
-	cfg.WAL = &wal.Config{
-		Filepath: t.TempDir(),
-	}
-	cfg.S3.Endpoint = endpoint
-
-	_, w, _, err := tempodb.New(cfg, nil, log.NewNopLogger())
-	require.NoError(t, err)
-
-	return w
-}
-
-func populateBackend(ctx context.Context, t testing.TB, w tempodb.Writer, tenantCount, blockCount, recordCount int) []string {
-	wal := w.WAL()
-
-	dec := model.MustNewSegmentDecoder(model.CurrentEncoding)
-
-	tenants := make([]string, tenantCount)
-
-	for i := range tenantCount {
-		tenantID := tenant + strconv.Itoa(i)
-
-		tenants[i] = tenantID
-
-		for range blockCount {
-			blockID := backend.NewUUID()
-			meta := &backend.BlockMeta{BlockID: blockID, TenantID: tenantID, DataEncoding: model.CurrentEncoding}
-			head, err := wal.NewBlock(meta, model.CurrentEncoding)
-			require.NoError(t, err)
-
-			for range recordCount {
-				id := test.ValidTraceID(nil)
-				req := test.MakeTrace(10, id)
-
-				writeTraceToWal(t, head, dec, id, req, 0, 0)
-			}
-
-			_, err = w.CompleteBlock(ctx, head)
-			require.NoError(t, err)
-		}
-	}
-
-	return tenants
-}
-
-func writeTraceToWal(t require.TestingT, b common.WALBlock, dec model.SegmentDecoder, id common.ID, tr *tempopb.Trace, start, end uint32) {
-	b1, err := dec.PrepareForWrite(tr, 0, 0)
-	require.NoError(t, err)
-
-	b2, err := dec.ToObject([][]byte{b1})
-	require.NoError(t, err)
-
-	err = b.Append(id, b2, start, end, true)
-	require.NoError(t, err, "unexpected error writing req")
-}

integration/backendscheduler/config.yaml

@@ -1,60 +0,0 @@
-server:
-  http_listen_port: 3200
-  # grpc_server_max_recv_msg_size: 134217728
-  # grpc_server_max_send_msg_size: 134217728
-distributor:
-  receivers:
-    jaeger:
-      protocols:
-        grpc:
-          endpoint: "distributor:14250"
-ingester:
-  lifecycler:
-    ring:
-      kvstore:
-        store: inmemory
-      replication_factor: 3
-    heartbeat_period: 100ms
-    min_ready_duration: 1s
-  trace_idle_period: 2s
-  max_block_duration: 2s
-  complete_block_timeout: 5s
-  flush_check_period: 1s
-storage:
-  trace:
-    blocklist_poll: 1s
-    blocklist_poll_stale_tenant_index: 1s # force components to always fall back to polling
-    backend: s3
-    s3:
-      bucket: tempo
-      endpoint: tempo-integration-minio-9000:9000 # TODO: this is brittle, fix this eventually
-      access_key: Cheescake # TODO: use cortex_e2e.MinioAccessKey
-      secret_key: supersecret # TODO: use cortex_e2e.MinioSecretKey
-      insecure: true
-    pool:
-      max_workers: 10
-      queue_depth: 100
-backend_scheduler:
-  provider:
-    compaction:
-      min_cycle_interval: 50ms
-      measure_interval: 1s
-backend_worker:
-  backend_scheduler_addr: tempo-integration-backend-scheduler:9095
-  finish_on_shutdown_timeout: 1s
-backend_scheduler_client:
-  grpc_client_config:
-    max_send_msg_size: 134217728
-#   max_recv_msg_size: 134217728
-memberlist:
-  abort_if_cluster_join_fails: false
-  bind_port: 7946
-  join_members: []
-  # - tempo_e2e-distributor:7946
-  # - tempo_e2e-ingester-1:7946
-  # - tempo_e2e-ingester-2:7946
-  # - tempo_e2e-ingester-3:7946
-  # - tempo_e2e-querier:7946
-querier:
-  frontend_worker:
-    frontend_address: tempo_e2e-query-frontend:9095

integration/e2e/README.md

@@ -1,40 +0,0 @@
-**Running the integration tests**
-
-
-`-count=1` is passed to disable cache during test runs.
-
-```sh
-# build latest image
-make docker-tempo
-make docker-tempo-query
-
-# run all tests
-go test -count=1 -v ./integration/e2e/...
-
-# run a particular test "TestMicroservices"
-go test -count=1 -v ./integration/e2e/... -run TestMicroservices$
-
-# build and run a particular test "TestMicroservicesWithKVStores"
-make docker-tempo && go test -count=1 -v ./integration/e2e/... -run TestMicroservicesWithKVStores$
-
-# run a single e2e tests with timeout
-go test -timeout 3m -count=1 -v ./integration/e2e/... -run ^TestMultiTenantSearch$
-
-# follow and watch logs while tests are running (assuming e2e test container is named tempo_e2e-tempo)
-docker logs $(docker container ls -f name=tempo_e2e-tempo -q) -f
-```
-
-**How to debug Tempo while running an integration test**
-
-1. Build latest debug image
-    ```sh
-        make docker-tempo-debug
-    ```
-2. Use the function ``NewTempoAllInOneDebug`` in your test to spin a Tempo instance with debug capabilities
-3. Set a breakpoint after ``require.NoError(t, s.StartAndWaitReady(tempo))`` and before the action you want debug
-4. Get the port of Delve debugger inside the container
-    ```sh
-    docker ps --format '{{.Ports}}'  
-        # 0.0.0.0:53467->2345
-    ```
-5. Run the debugger against that port as is specified [here](https://github.com/grafana/tempo/tree/main/example/docker-compose/debug)

integration/e2e/api/config-all-in-one-s3-overrides.yaml

@@ -1,65 +0,0 @@
-target: all
-stream_over_http_enabled: true
-auth_enabled: true
-
-server:
-  http_listen_port: 3200
-
-query_frontend:
-  query_end_cutoff: -1s # Include all recent data
-  search:
-    query_backend_after: 5s
-    query_ingesters_until: 5s
-
-distributor:
-  receivers:
-    jaeger:
-      protocols:
-        grpc:
-          endpoint: "tempo:14250"
-    otlp:
-      protocols:
-        grpc:
-          endpoint: "0.0.0.0:4317"
-ingester:
-  lifecycler:
-    address: 127.0.0.1
-    ring:
-      kvstore:
-        store: inmemory
-      replication_factor: 1
-    final_sleep: 0s
-    min_ready_duration: 1s
-  trace_idle_period: 1s
-  max_block_duration: 5s
-  complete_block_timeout: 5s
-  flush_check_period: 1s
-
-storage:
-  trace:
-    blocklist_poll: 1s
-    backend: s3
-    s3:
-      bucket: tempo
-      endpoint: tempo_e2e-minio-9000:9000 # TODO: this is brittle, fix this eventually
-      access_key: Cheescake # TODO: use cortex_e2e.MinioAccessKey
-      secret_key: supersecret # TODO: use cortex_e2e.MinioSecretKey
-      insecure: true
-      prefix: {{ .Prefix }}
-    pool:
-      max_workers: 10
-      queue_depth: 100
-
-overrides:
-  user_configurable_overrides:
-    enabled: true
-    poll_interval: 10s
-    client:
-      backend: s3
-      s3:
-        # TODO use separate bucket?
-        bucket: tempo
-        endpoint: tempo_e2e-minio-9000:9000 # TODO: this is brittle, fix this eventually
-        access_key: Cheescake # TODO: use cortex_e2e.MinioAccessKey
-        secret_key: supersecret # TODO: use cortex_e2e.MinioSecretKey
-        insecure: true

integration/e2e/api/config-cross-cluster-a.yaml

@@ -1,52 +0,0 @@
-server:
-  http_listen_port: 3200
-
-query_frontend:
-  query_end_cutoff: 0 # Include all recent data
-
-distributor:
-  receivers:
-    jaeger:
-      protocols:
-        grpc:
-          endpoint: "distributor-a:14250"
-    otlp:
-      protocols:
-        grpc:
-          endpoint: "distributor-a:4317"
-
-ingester:
-  lifecycler:
-    ring:
-      replication_factor: 3
-    heartbeat_period: 100ms
-  override_ring_key: "ring-a"
-
-storage:
-  trace:
-    backend: s3
-    s3:
-      bucket: tempo
-      endpoint: tempo_active_active-minio-9000:9000 # TODO: this is brittle, fix this eventually
-      access_key: Cheescake # TODO: use cortex_e2e.MinioAccessKey
-      secret_key: supersecret # TODO: use cortex_e2e.MinioSecretKey
-      insecure: true
-    pool:
-      max_workers: 10
-      queue_depth: 100
-
-memberlist:
-  abort_if_cluster_join_fails: false
-  bind_port: 7946
-  join_members:
-    - tempo_active_active-ingester-a-1:7946
-    - tempo_active_active-ingester-a-2:7946
-    - tempo_active_active-ingester-a-3:7946
-    - tempo_active_active-ingester-b-1:7946
-    - tempo_active_active-ingester-b-2:7946
-    - tempo_active_active-ingester-b-3:7946
-
-querier:
-  secondary_ingester_ring: "ring-b"
-  frontend_worker:
-    frontend_address: tempo_active_active-query-frontend-a:9095

integration/e2e/api/config-cross-cluster-b.yaml

@@ -1,52 +0,0 @@
-server:
-  http_listen_port: 3200
-
-query_frontend:
-  query_end_cutoff: 0 # Include all recent data
-
-distributor:
-  receivers:
-    jaeger:
-      protocols:
-        grpc:
-          endpoint: "distributor-b:14250"
-    otlp:
-      protocols:
-        grpc:
-          endpoint: "distributor-b:4317"
-
-ingester:
-  lifecycler:
-    ring:
-      replication_factor: 3
-    heartbeat_period: 100ms
-  override_ring_key: "ring-b"
-
-storage:
-  trace:
-    backend: s3
-    s3:
-      bucket: tempo
-      endpoint: tempo_active_active-minio-9000:9000 # TODO: this is brittle, fix this eventually
-      access_key: Cheescake # TODO: use cortex_e2e.MinioAccessKey
-      secret_key: supersecret # TODO: use cortex_e2e.MinioSecretKey
-      insecure: true
-    pool:
-      max_workers: 10
-      queue_depth: 100
-
-memberlist:
-  abort_if_cluster_join_fails: false
-  bind_port: 7946
-  join_members:
-    - tempo_active_active-ingester-a-1:7946
-    - tempo_active_active-ingester-a-2:7946
-    - tempo_active_active-ingester-a-3:7946
-    - tempo_active_active-ingester-b-1:7946
-    - tempo_active_active-ingester-b-2:7946
-    - tempo_active_active-ingester-b-3:7946
-
-querier:
-  secondary_ingester_ring: "ring-a"
-  frontend_worker:
-    frontend_address: tempo_active_active-query-frontend-b:9095

integration/e2e/api/config-multi-tenant-local.yaml

@@ -1,76 +0,0 @@
-target: all
-multitenancy_enabled: true
-stream_over_http_enabled: true
-
-server:
-  http_listen_port: 3200
-  log_level: warn
-
-query_frontend:
-  query_end_cutoff: 0 # Include all recent data
-  search:
-    ingester_shards: 1
-  metrics:
-    max_response_series: 0
-
-distributor:
-  receivers:
-    jaeger:
-      protocols:
-        grpc:
-          endpoint: "tempo:14250"
-    otlp:
-      protocols:
-        grpc:
-          endpoint: "tempo:4317"
-    zipkin:
-      endpoint: "tempo:9411"
-  log_received_spans:
-    enabled: true
-
-ingester:
-  lifecycler:
-    address: 127.0.0.1
-    ring:
-      kvstore:
-        store: inmemory
-      replication_factor: 1
-    final_sleep: 0s
-  trace_idle_period: 1s
-  max_block_bytes: 1
-  max_block_duration: 2s
-  complete_block_timeout: 20s
-  flush_check_period: 1s
-
-metrics_generator:
-  processor:
-    service_graphs:
-      histogram_buckets: [1, 2] # seconds
-    span_metrics:
-      histogram_buckets: [1, 2]
-  registry:
-    collection_interval: 1s
-  storage:
-    path: /var/tempo
-    remote_write:
-      - url: http://tempo_e2e-prometheus:9090/api/v1/write
-        send_exemplars: true
-
-storage:
-  trace:
-    blocklist_poll: 1s
-    backend: local
-    local:
-      path: /var/tempo
-    pool:
-      max_workers: 10
-      queue_depth: 100
-
-overrides:
-  user_configurable_overrides:
-    enabled: true
-    poll_interval: 10s
-    client:
-      backend: local
-      local:
-        path: /var/tempo/overrides

integration/e2e/api/config-plugin-test.yaml

@@ -1,49 +0,0 @@
-target: all
-
-server:
-  http_listen_port: 3200
-  log_level: info
-
-query_frontend:
-  query_end_cutoff: -5m # Include all recent data
-
-distributor:
-  receivers:
-    jaeger:
-      protocols:
-        grpc:
-          endpoint: "tempo:14250"
-    otlp:
-      protocols:
-        grpc:
-          endpoint: "tempo:4317"
-    zipkin:
-      endpoint: "tempo:9411"
-  log_received_spans:
-    enabled: true
-
-ingester:
-  lifecycler:
-    address: 127.0.0.1
-    ring:
-      kvstore:
-        store: inmemory
-      replication_factor: 1
-    final_sleep: 0s
-  trace_idle_period: 1s
-  max_block_bytes: 1
-  max_block_duration: 2s
-  complete_block_timeout: 1s
-  flush_check_period: 3s
-
-storage:
-  trace:
-    backend: local
-    local:
-      path: /var/tempo
-    pool:
-      max_workers: 10
-      queue_depth: 100
-    block:
-      version: vParquet3
-    blocklist_poll: 1s

integration/e2e/api/config-query-range-max-series-disabled-querier.yaml

@@ -1,77 +0,0 @@
-target: all
-stream_over_http_enabled: true
-
-server:
-  http_listen_port: 3200
-  log_level: debug
-
-query_frontend:
-  query_end_cutoff: 0 # Include all recent data
-
-distributor:
-  receivers:
-    jaeger:
-      protocols:
-        grpc:
-          endpoint: "tempo:14250"
-    otlp:
-      protocols:
-        grpc:
-          endpoint: "tempo:4317"
-    zipkin:
-      endpoint: "tempo:9411"
-  log_received_spans:
-    enabled: true
-
-metrics_generator:
-  processor:
-    local_blocks:
-      flush_check_period: 1s
-      max_block_duration: 10s
-      flush_to_storage: true
-  storage:
-    path: /var/tempo
-    remote_write:
-      - url: http://tempo_e2e-prometheus:9090/api/v1/write
-        send_exemplars: true
-  traces_storage:
-    path: /var/tempo/generator/traces
-
-ingester:
-  lifecycler:
-    address: 127.0.0.1
-    ring:
-      kvstore:
-        store: inmemory
-      replication_factor: 1
-    final_sleep: 0s
-    min_ready_duration: 1s
-  trace_idle_period: 1s
-  max_block_bytes: 1
-  max_block_duration: 2s
-  complete_block_timeout: 5s
-  flush_check_period: 1s
-
-storage:
-  trace:
-    backend: local
-    local:
-      path: /var/tempo
-    pool:
-      max_workers: 10
-      queue_depth: 100
-    block:
-      version: vParquet4
-    blocklist_poll: 1s
-
-overrides:
-  defaults:
-    metrics_generator:
-      processors: [local-blocks]
-  user_configurable_overrides:
-    enabled: true
-    poll_interval: 10s
-    client:
-      backend: local
-      local:
-        path: /var/tempo/overrides

integration/e2e/api/config-query-range-max-series-disabled.yaml

@@ -1,78 +0,0 @@
-target: all
-stream_over_http_enabled: true
-
-server:
-  http_listen_port: 3200
-  log_level: debug
-
-query_frontend:
-  query_end_cutoff: 0 # Include all recent data
-  metrics:
-    max_response_series: 0
-
-distributor:
-  receivers:
-    jaeger:
-      protocols:
-        grpc:
-          endpoint: "tempo:14250"
-    otlp:
-      protocols:
-        grpc:
-          endpoint: "tempo:4317"
-    zipkin:
-      endpoint: "tempo:9411"
-  log_received_spans:
-    enabled: true
-
-metrics_generator:
-  processor:
-    local_blocks:
-      flush_check_period: 1s
-      max_block_duration: 10s
-      flush_to_storage: true
-  storage:
-    path: /var/tempo
-    remote_write:
-      - url: http://tempo_e2e-prometheus:9090/api/v1/write
-        send_exemplars: true
-  traces_storage:
-    path: /var/tempo/generator/traces
-
-ingester:
-  lifecycler:
-    address: 127.0.0.1
-    ring:
-      kvstore:
-        store: inmemory
-      replication_factor: 1
-    final_sleep: 0s
-  trace_idle_period: 1s
-  max_block_bytes: 1
-  max_block_duration: 2s
-  complete_block_timeout: 20s
-  flush_check_period: 1s
-
-storage:
-  trace:
-    backend: local
-    local:
-      path: /var/tempo
-    pool:
-      max_workers: 10
-      queue_depth: 100
-    block:
-      version: vParquet4
-    blocklist_poll: 1s
-
-overrides:
-  defaults:
-    metrics_generator:
-      processors: [local-blocks]
-  user_configurable_overrides:
-    enabled: true
-    poll_interval: 10s
-    client:
-      backend: local
-      local:
-        path: /var/tempo/overrides

integration/e2e/api/config-query-range-max-series.yaml

@@ -1,78 +0,0 @@
-target: all
-stream_over_http_enabled: true
-
-server:
-  http_listen_port: 3200
-  log_level: debug
-
-query_frontend:
-  query_end_cutoff: 0 # Include all recent data
-  metrics:
-    max_response_series: 3
-
-distributor:
-  receivers:
-    jaeger:
-      protocols:
-        grpc:
-          endpoint: "tempo:14250"
-    otlp:
-      protocols:
-        grpc:
-          endpoint: "tempo:4317"
-    zipkin:
-      endpoint: "tempo:9411"
-  log_received_spans:
-    enabled: true
-
-metrics_generator:
-  processor:
-    local_blocks:
-      flush_check_period: 1s
-      max_block_duration: 10s
-      flush_to_storage: true
-  storage:
-    path: /var/tempo
-    remote_write:
-      - url: http://tempo_e2e-prometheus:9090/api/v1/write
-        send_exemplars: true
-  traces_storage:
-    path: /var/tempo/generator/traces
-
-ingester:
-  lifecycler:
-    address: 127.0.0.1
-    ring:
-      kvstore:
-        store: inmemory
-      replication_factor: 1
-    final_sleep: 0s
-  trace_idle_period: 1s
-  max_block_bytes: 1
-  max_block_duration: 2s
-  complete_block_timeout: 20s
-  flush_check_period: 1s
-
-storage:
-  trace:
-    backend: local
-    local:
-      path: /var/tempo
-    pool:
-      max_workers: 10
-      queue_depth: 100
-    block:
-      version: vParquet4
-    blocklist_poll: 1s
-
-overrides:
-  defaults:
-    metrics_generator:
-      processors: [local-blocks]
-  user_configurable_overrides:
-    enabled: true
-    poll_interval: 10s
-    client:
-      backend: local
-      local:
-        path: /var/tempo/overrides

integration/e2e/api/config-query-range.yaml

@@ -1,76 +0,0 @@
-target: all
-stream_over_http_enabled: true
-
-server:
-  http_listen_port: 3200
-  log_level: debug
-
-query_frontend:
-  query_end_cutoff: -5m # Include all recent data
-
-distributor:
-  receivers:
-    jaeger:
-      protocols:
-        grpc:
-          endpoint: "tempo:14250"
-    otlp:
-      protocols:
-        grpc:
-          endpoint: "tempo:4317"
-    zipkin:
-      endpoint: "tempo:9411"
-  log_received_spans:
-    enabled: true
-
-metrics_generator:
-  processor:
-    local_blocks:
-      flush_check_period: 1s
-      max_block_duration: 10s
-      flush_to_storage: true
-  storage:
-    path: /var/tempo
-    remote_write:
-      - url: http://tempo_e2e-prometheus:9090/api/v1/write
-        send_exemplars: true
-  traces_storage:
-    path: /var/tempo/generator/traces
-
-ingester:
-  lifecycler:
-    address: 127.0.0.1
-    ring:
-      kvstore:
-        store: inmemory
-      replication_factor: 1
-    final_sleep: 0s
-  trace_idle_period: 1s
-  max_block_bytes: 1
-  max_block_duration: 2s
-  complete_block_timeout: 20s
-  flush_check_period: 1s
-
-storage:
-  trace:
-    backend: local
-    local:
-      path: /var/tempo
-    pool:
-      max_workers: 10
-      queue_depth: 100
-    block:
-      version: vParquet4
-    blocklist_poll: 1s
-
-overrides:
-  defaults:
-    metrics_generator:
-      processors: [local-blocks]
-  user_configurable_overrides:
-    enabled: true
-    poll_interval: 10s
-    client:
-      backend: local
-      local:
-        path: /var/tempo/overrides

integration/e2e/api/config-tempo-query.yaml

@@ -1,3 +0,0 @@
-address: "0.0.0.0:7777"
-backend: tempo:3200
-services_query_duration: 1h

integration/e2e/api/cross_cluster_reads_test.go

@@ -1,87 +0,0 @@
-package api
-
-import (
-	"testing"
-	"time"
-
-	"github.com/grafana/e2e"
-	e2edb "github.com/grafana/e2e/db"
-	"github.com/grafana/tempo/integration/util"
-	"github.com/prometheus/prometheus/model/labels"
-	"github.com/stretchr/testify/require"
-
-	"github.com/grafana/tempo/pkg/httpclient"
-	tempoUtil "github.com/grafana/tempo/pkg/util"
-)
-
-// TestCrossClusterReads uses the secondary_ingester_ring querier configuration option. it writes a trace to
-// one cluster and reads it from another. the key elements of this test:
-// - use of the secondary_ingester_ring option in the querier
-// - renaming of the ingester rings using override_ring_key to avoid collisions
-// - using a common kv store by gossiping across both clusters
-func TestCrossClusterReads(t *testing.T) {
-	t.Parallel()
-
-	// start minio
-	s, err := e2e.NewScenario("tempo_active_active")
-	require.NoError(t, err)
-
-	minio := e2edb.NewMinio(9000, "tempo")
-	require.NotNil(t, minio)
-	require.NoError(t, s.StartAndWaitReady(minio))
-
-	// setup clusters
-	tempoDistributorA, _ := createCluster(t, s, "a")
-	_, tempoQueryFrontendB := createCluster(t, s, "b")
-
-	// write to cluster A
-	c, err := util.NewJaegerToOTLPExporter(tempoDistributorA.Endpoint(4317))
-	require.NoError(t, err)
-	require.NotNil(t, c)
-
-	info := tempoUtil.NewTraceInfo(time.Now(), "")
-	require.NoError(t, info.EmitAllBatches(c))
-
-	expected, err := info.ConstructTraceFromEpoch()
-	require.NoError(t, err)
-
-	// test metrics
-	require.NoError(t, tempoDistributorA.WaitSumMetrics(e2e.Equals(util.SpanCount(expected)), "tempo_distributor_spans_received_total"))
-
-	// read from cluster B
-	apiClient := httpclient.New("http://"+tempoQueryFrontendB.Endpoint(tempoPort), "")
-
-	// query an in-memory trace
-	util.QueryAndAssertTrace(t, apiClient, info)
-}
-
-func createCluster(t *testing.T, s *e2e.Scenario, postfix string) (*e2e.HTTPService, *e2e.HTTPService) {
-	require.NoError(t, util.CopyFileToSharedDir(s, "config-cross-cluster-"+postfix+".yaml", "config.yaml"))
-
-	tempoIngester1 := util.NewNamedTempoIngester("ingester-"+postfix, 1)
-	tempoIngester2 := util.NewNamedTempoIngester("ingester-"+postfix, 2)
-	tempoIngester3 := util.NewNamedTempoIngester("ingester-"+postfix, 3)
-
-	tempoDistributor := util.NewNamedTempoDistributor("distributor-" + postfix)
-	tempoQueryFrontend := util.NewNamedTempoQueryFrontend("query-frontend-" + postfix)
-	tempoQuerier := util.NewNamedTempoQuerier("querier-" + postfix)
-	require.NoError(t, s.StartAndWaitReady(tempoIngester1, tempoIngester2, tempoIngester3, tempoDistributor, tempoQueryFrontend, tempoQuerier))
-
-	// wait for active ingesters
-	time.Sleep(1 * time.Second)
-	matchers := []*labels.Matcher{
-		{
-			Type:  labels.MatchEqual,
-			Name:  "name",
-			Value: "ingester",
-		},
-		{
-			Type:  labels.MatchEqual,
-			Name:  "state",
-			Value: "ACTIVE",
-		},
-	}
-	require.NoError(t, tempoDistributor.WaitSumMetricsWithOptions(e2e.Equals(3), []string{`tempo_ring_members`}, e2e.WithLabelMatchers(matchers...), e2e.WaitMissingMetrics))
-
-	return tempoDistributor, tempoQueryFrontend
-}

integration/e2e/api/https_test.go

@@ -1,87 +0,0 @@
-package api
-
-import (
-	"context"
-	"crypto/tls"
-	"net/http"
-	"os"
-	"testing"
-	"time"
-
-	"github.com/grafana/e2e"
-	"github.com/grafana/tempo/cmd/tempo/app"
-	"github.com/grafana/tempo/integration/e2e/backend"
-	e2e_ca "github.com/grafana/tempo/integration/e2e/ca"
-	"github.com/grafana/tempo/integration/util"
-	"github.com/grafana/tempo/pkg/httpclient"
-	tempoUtil "github.com/grafana/tempo/pkg/util"
-	"github.com/stretchr/testify/require"
-	"google.golang.org/grpc/credentials"
-	"gopkg.in/yaml.v2"
-)
-
-const (
-	configHTTPS = "config-https.yaml"
-)
-
-func TestHTTPS(t *testing.T) {
-	km := e2e_ca.SetupCertificates(t)
-
-	s, err := e2e.NewScenario("tempo_e2e")
-	require.NoError(t, err)
-	defer s.Close()
-
-	// set up the backend
-	cfg := app.Config{}
-	buff, err := os.ReadFile(configHTTPS)
-	require.NoError(t, err)
-	err = yaml.UnmarshalStrict(buff, &cfg)
-	require.NoError(t, err)
-	_, err = backend.New(s, cfg)
-	require.NoError(t, err)
-
-	// copy in certs
-	require.NoError(t, util.CopyFileToSharedDir(s, km.ServerCertFile, "tls.crt"))
-	require.NoError(t, util.CopyFileToSharedDir(s, km.ServerKeyFile, "tls.key"))
-	require.NoError(t, util.CopyFileToSharedDir(s, km.CaCertFile, "ca.crt"))
-
-	require.NoError(t, util.CopyFileToSharedDir(s, configHTTPS, "config.yaml"))
-	tempo := util.NewTempoAllInOneWithReadinessProbe(e2e.NewHTTPReadinessProbe(3201, "/ready", 200, 299))
-	require.NoError(t, s.StartAndWaitReady(tempo))
-
-	// Get port for the Jaeger gRPC receiver endpoint
-	c, err := util.NewJaegerToOTLPExporter(tempo.Endpoint(4317))
-	require.NoError(t, err)
-	require.NotNil(t, c)
-
-	time.Sleep(10 * time.Second)
-
-	info := tempoUtil.NewTraceInfo(time.Now(), "")
-	require.NoError(t, info.EmitAllBatches(c))
-
-	apiClient := httpclient.New("https://"+tempo.Endpoint(tempoPort), "")
-
-	// trust bad certs
-	defaultTransport := http.DefaultTransport.(*http.Transport).Clone()
-	defaultTransport.TLSClientConfig = &tls.Config{InsecureSkipVerify: true}
-	apiClient.WithTransport(defaultTransport)
-
-	echoReq, err := http.NewRequest("GET", "https://"+tempo.Endpoint(tempoPort)+"/api/echo", nil)
-	require.NoError(t, err)
-	resp, err := apiClient.Do(echoReq)
-	require.NoError(t, err)
-	require.Equal(t, http.StatusOK, resp.StatusCode)
-
-	// query an in-memory trace
-	util.QueryAndAssertTrace(t, apiClient, info)
-	util.SearchAndAssertTrace(t, apiClient, info)
-	util.SearchTraceQLAndAssertTrace(t, apiClient, info)
-
-	creds := credentials.NewTLS(&tls.Config{InsecureSkipVerify: true})
-	grpcClient, err := util.NewSearchGRPCClientWithCredentials(context.Background(), tempo.Endpoint(tempoPort), creds)
-	require.NoError(t, err)
-
-	time.Sleep(1 * time.Second)
-	now := time.Now()
-	util.SearchStreamAndAssertTrace(t, context.Background(), grpcClient, info, now.Add(-time.Hour).Unix(), now.Unix())
-}

integration/e2e/api/multi_tenant_test.go

@@ -1,248 +0,0 @@
-package api
-
-import (
-	"context"
-	"fmt"
-	"os"
-	"strings"
-	"testing"
-	"time"
-
-	"github.com/google/uuid"
-	"github.com/grafana/dskit/user"
-	"github.com/grafana/e2e"
-	"github.com/grafana/tempo/integration/util"
-	"github.com/grafana/tempo/pkg/collector"
-	"github.com/grafana/tempo/pkg/httpclient"
-	"github.com/grafana/tempo/pkg/tempopb"
-	tempoUtil "github.com/grafana/tempo/pkg/util"
-	"github.com/prometheus/prometheus/model/labels"
-	"github.com/stretchr/testify/assert"
-	"github.com/stretchr/testify/require"
-	"gopkg.in/yaml.v2"
-
-	"github.com/grafana/tempo/cmd/tempo/app"
-	"github.com/grafana/tempo/integration/e2e/backend"
-)
-
-const (
-	configMultiTenant = "config-multi-tenant-local.yaml"
-)
-
-type traceStringsMap struct {
-	rKeys     []string
-	rValues   []string
-	spanNames []string
-}
-
-func TestSingleTenantSearch(t *testing.T) {
-	t.Parallel()
-	testSearch(t, "test", 1)
-}
-
-func TestWildCardTenantSearch(t *testing.T) {
-	t.Parallel()
-	testSearch(t, "*", 1)
-}
-
-func TestTwoTenantsSearch(t *testing.T) {
-	t.Parallel()
-	testSearch(t, "test|test2", 2)
-}
-
-func TestThreeTenantsSearch(t *testing.T) {
-	t.Parallel()
-	testSearch(t, "test|test2|test3", 3)
-}
-
-func generateNetworkName() string {
-	uuid := uuid.New()
-	return uuid.String()
-}
-
-func testSearch(t *testing.T, tenant string, tenantSize int) {
-	s, err := e2e.NewScenario(generateNetworkName())
-	require.NoError(t, err)
-	defer s.Close()
-
-	// set up the backend
-	cfg := app.Config{}
-	buff, err := os.ReadFile(configMultiTenant)
-	require.NoError(t, err)
-	err = yaml.UnmarshalStrict(buff, &cfg)
-	require.NoError(t, err)
-	_, err = backend.New(s, cfg)
-	require.NoError(t, err)
-
-	require.NoError(t, util.CopyFileToSharedDir(s, configMultiTenant, "config.yaml"))
-	tempo := util.NewTempoAllInOne()
-	prometheus := util.NewPrometheus()
-	require.NoError(t, s.StartAndWaitReady(tempo, prometheus))
-
-	// Get port for the Jaeger gRPC receiver endpoint
-	c, err := util.NewJaegerToOTLPExporter(tempo.Endpoint(4317))
-	require.NoError(t, err)
-	require.NotNil(t, c)
-
-	var info *tempoUtil.TraceInfo
-	var traceMap traceStringsMap
-
-	tenants := strings.Split(tenant, "|")
-	require.Equal(t, tenantSize, len(tenants))
-
-	var expectedSpans float64
-	// write traces for all tenants
-	for _, tenant := range tenants {
-		info = tempoUtil.NewTraceInfo(time.Now(), tenant)
-		require.NoError(t, info.EmitAllBatches(c))
-
-		trace, err := info.ConstructTraceFromEpoch()
-		traceMap = getAttrsAndSpanNames(trace) // store it to assert tests
-
-		require.NoError(t, err)
-		expectedSpans = expectedSpans + util.SpanCount(trace)
-	}
-
-	// assert that we have one trace and each tenant and correct number of spans received
-	require.NoError(t, tempo.WaitSumMetrics(e2e.Equals(float64(tenantSize)), "tempo_ingester_traces_created_total"))
-	require.NoError(t, tempo.WaitSumMetrics(e2e.Equals(expectedSpans), "tempo_distributor_spans_received_total"))
-
-	// Wait for the traces to be written to the WAL
-	time.Sleep(time.Second * 3)
-
-	// test echo
-	util.AssertEcho(t, "http://"+tempo.Endpoint(tempoPort)+"/api/echo")
-
-	// client will have testcase tenant id
-	apiClient := httpclient.New("http://"+tempo.Endpoint(tempoPort), tenant)
-
-	// check trace by id
-	resp, err := apiClient.QueryTrace(info.HexID())
-	require.NoError(t, err)
-	respTm := getAttrsAndSpanNames(resp)
-
-	assert.ElementsMatch(t, traceMap.rValues, respTm.rValues)
-	assert.ElementsMatch(t, respTm.rKeys, traceMap.rKeys)
-	assert.ElementsMatch(t, traceMap.spanNames, respTm.spanNames)
-
-	// flush trace to backend
-	util.CallFlush(t, tempo)
-
-	// search and traceql search, note: SearchAndAssertTrace also calls SearchTagValues
-	util.SearchAndAssertTrace(t, apiClient, info)
-	util.SearchTraceQLAndAssertTrace(t, apiClient, info)
-
-	// force clear completed block
-	util.CallFlush(t, tempo)
-
-	// wait for flush to complete for all tenants, each tenant will have one block
-	require.NoError(t, tempo.WaitSumMetrics(e2e.Equals(float64(tenantSize)), "tempo_ingester_blocks_flushed_total"))
-
-	// call search tags endpoints, ensure no errors and results are not empty
-	tagsResp, err := apiClient.SearchTags()
-	require.NoError(t, err)
-	require.NotEmpty(t, tagsResp.TagNames)
-
-	tagsV2Resp, err := apiClient.SearchTagsV2()
-	require.NoError(t, err)
-	require.Equal(t, 4, len(tagsV2Resp.GetScopes())) // resource, span, event, link, instrumentation intrinsics
-	for _, s := range tagsV2Resp.Scopes {
-		require.NotEmpty(t, s.Tags)
-	}
-
-	tagsValuesResp, err := apiClient.SearchTagValues("vulture-0")
-	require.NoError(t, err)
-	require.NotEmpty(t, tagsValuesResp.TagValues)
-
-	tagsValuesV2Resp, err := apiClient.SearchTagValuesV2("span.vulture-0", "{}")
-	require.NoError(t, err)
-	require.NotEmpty(t, tagsValuesV2Resp.TagValues)
-
-	// check metrics for all routes
-	routeTable := []struct {
-		route    string
-		reqCount int
-	}{
-		// query frontend routes
-		{route: "api_search", reqCount: 2}, // called twice
-		{route: "api_traces_traceid", reqCount: 1},
-		{route: "api_search_tags", reqCount: 1},
-		{route: "api_search_tag_tagname_values", reqCount: 2}, // called twice
-		{route: "api_v2_search_tags", reqCount: 1},
-		{route: "api_v2_search_tag_tagname_values", reqCount: 1},
-		// Querier routes, we make one request for each tenant
-		{route: "/tempopb.Querier/SearchRecent", reqCount: 2 * tenantSize}, // called twice
-		{route: "/tempopb.Querier/FindTraceByID", reqCount: tenantSize},
-		{route: "/tempopb.Querier/SearchTags", reqCount: tenantSize},
-		{route: "/tempopb.Querier/SearchTagsV2", reqCount: tenantSize},
-		{route: "/tempopb.Querier/SearchTagValues", reqCount: 2 * tenantSize}, // called twice
-		{route: "/tempopb.Querier/SearchTagValuesV2", reqCount: tenantSize},
-	}
-	for _, rt := range routeTable {
-		assertRequestCountMetric(t, tempo, rt.route, rt.reqCount)
-	}
-
-	// test streaming search over grpc
-	grpcCtx := user.InjectOrgID(context.Background(), tenant)
-	grpcCtx, err = user.InjectIntoGRPCRequest(grpcCtx)
-	require.NoError(t, err)
-
-	grpcClient, err := util.NewSearchGRPCClient(grpcCtx, tempo.Endpoint(tempoPort))
-	require.NoError(t, err)
-
-	time.Sleep(2 * time.Second) // ensure that blocklist poller has built the blocklist
-	now := time.Now()
-	util.SearchStreamAndAssertTrace(t, grpcCtx, grpcClient, info, now.Add(-10*time.Minute).Unix(), now.Add(10*time.Minute).Unix())
-	assertRequestCountMetric(t, tempo, "/tempopb.StreamingQuerier/Search", 1)
-
-	// test unsupported endpoint
-	_, msErr := apiClient.MetricsSummary("{}", "name", 0, 0)
-	if tenantSize > 1 {
-		// error for multi-tenant request for unsupported endpoints
-		require.Error(t, msErr)
-	} else {
-		require.NoError(t, msErr)
-	}
-}
-
-func assertRequestCountMetric(t *testing.T, s *e2e.HTTPService, route string, reqCount int) {
-	fmt.Printf("==== %s, assertRequestCountMetric route: %v, rt.reqCount: %v \n", t.Name(), route, reqCount)
-
-	err := s.WaitSumMetricsWithOptions(e2e.Equals(float64(reqCount)),
-		[]string{"tempo_request_duration_seconds"},
-		e2e.WithLabelMatchers(labels.MustNewMatcher(labels.MatchEqual, "route", route)),
-		e2e.WithMetricCount, // get count from histogram metric
-	)
-	require.NoError(t, err)
-}
-
-// getAttrsAndSpanNames returns trace attrs and span names
-func getAttrsAndSpanNames(trace *tempopb.Trace) traceStringsMap {
-	rAttrsKeys := collector.NewDistinctString(0, 0, 0)
-	rAttrsValues := collector.NewDistinctString(0, 0, 0)
-	spanNames := collector.NewDistinctString(0, 0, 0)
-
-	for _, b := range trace.ResourceSpans {
-		for _, ss := range b.ScopeSpans {
-			for _, s := range ss.Spans {
-				if s.Name != "" {
-					spanNames.Collect(s.Name)
-				}
-			}
-		}
-		for _, a := range b.Resource.Attributes {
-			if a.Key != "" {
-				rAttrsKeys.Collect(a.Key)
-			}
-			if a.Value.GetStringValue() != "" {
-				rAttrsValues.Collect(a.Value.GetStringValue())
-			}
-		}
-	}
-
-	return traceStringsMap{
-		rKeys:     rAttrsKeys.Strings(),
-		rValues:   rAttrsValues.Strings(),
-		spanNames: spanNames.Strings(),
-	}
-}

integration/e2e/api/overrides_api_test.go

@@ -1,761 +0,0 @@
-package api
-
-import (
-	"fmt"
-	"net/http"
-	"os"
-	"strings"
-	"testing"
-
-	"github.com/grafana/e2e"
-	"github.com/grafana/tempo/cmd/tempo/app"
-	"github.com/grafana/tempo/integration/e2e/backend"
-	"github.com/grafana/tempo/integration/util"
-	"github.com/grafana/tempo/modules/overrides/histograms"
-	"github.com/grafana/tempo/modules/overrides/userconfigurable/client"
-	"github.com/grafana/tempo/pkg/httpclient"
-	jsoniter "github.com/json-iterator/go"
-	"github.com/stretchr/testify/assert"
-	"github.com/stretchr/testify/require"
-	"gopkg.in/yaml.v2"
-)
-
-const (
-	configAllInOneS3          = "./../deployments/config-all-in-one-s3.yaml"
-	configAllInOneAzurite     = "./../deployments/config-all-in-one-azurite.yaml"
-	configAllInOneGCS         = "./../deployments/config-all-in-one-gcs.yaml"
-	configAllInOneS3Overrides = "./config-all-in-one-s3-overrides.yaml"
-)
-
-func TestOverrides(t *testing.T) {
-	testBackends := []struct {
-		name           string
-		skipVersioning bool
-		configFile     string
-	}{
-		{
-			name: "local",
-			// local backend does not enforce versioning
-			skipVersioning: true,
-			configFile:     configAllInOneLocal,
-		},
-		{
-			name:       "s3",
-			configFile: configAllInOneS3,
-		},
-		{
-			name:       "azure",
-			configFile: configAllInOneAzurite,
-		},
-		{
-			name:       "gcs",
-			configFile: configAllInOneGCS,
-		},
-	}
-	for _, tc := range testBackends {
-		t.Run(tc.name, func(t *testing.T) {
-			s, tempo := setupTempo(t, tc.configFile)
-			defer s.Close()
-
-			orgID := ""
-			apiClient := httpclient.New("http://"+tempo.Endpoint(3200), orgID)
-
-			// Create overrides
-			initialLimits := &client.Limits{
-				MetricsGenerator: client.LimitsMetricsGenerator{
-					DisableCollection: boolPtr(true),
-				},
-			}
-
-			if !tc.skipVersioning {
-				fmt.Println("* Creating overrides with non-0 version")
-				_, err := apiClient.SetOverrides(initialLimits, "123")
-				assert.ErrorContains(t, err, "412") // precondition failed
-			}
-
-			fmt.Println("* Creating overrides")
-			_, err := apiClient.SetOverrides(initialLimits, "0")
-			assert.NoError(t, err)
-
-			limits, version, err := apiClient.GetOverrides()
-			assert.NoError(t, err)
-			EnableInstanceLabel, EnableInstanceLabelIsSet := limits.GetMetricsGenerator().GetProcessor().GetSpanMetrics().GetEnableInstanceLabel()
-			assert.True(t, EnableInstanceLabel)
-			assert.False(t, EnableInstanceLabelIsSet)
-			printLimits(limits, version)
-
-			disableCollection, ok := limits.GetMetricsGenerator().GetDisableCollection()
-			assert.True(t, ok)
-			assert.True(t, disableCollection)
-
-			// Update overrides - POST
-			updatedLimits := &client.Limits{
-				MetricsGenerator: client.LimitsMetricsGenerator{
-					DisableCollection: nil,
-					Processors:        map[string]struct{}{"span-metrics": {}},
-				},
-			}
-
-			if !tc.skipVersioning {
-				fmt.Println("* Update overrides with bogus version number")
-				_, err = apiClient.SetOverrides(updatedLimits, "abc")
-				assert.ErrorContains(t, err, "412") // precondition failed
-
-				fmt.Println("* Update overrides with backend.VersionNew")
-				_, err = apiClient.SetOverrides(updatedLimits, "0")
-				assert.ErrorContains(t, err, "412") // precondition failed
-
-				fmt.Println("* Update overrides with wrong version number")
-				_, err = apiClient.SetOverrides(updatedLimits, "123")
-				assert.ErrorContains(t, err, "412") // precondition failed
-			}
-
-			fmt.Println("* Update overrides")
-			_, err = apiClient.SetOverrides(updatedLimits, version)
-			assert.NoError(t, err)
-
-			limits, version, err = apiClient.GetOverrides()
-
-			assert.NoError(t, err)
-			printLimits(limits, version)
-
-			_, ok = limits.GetMetricsGenerator().GetDisableCollection()
-			assert.False(t, ok) // is not set anymore
-			processors, ok := limits.GetMetricsGenerator().GetProcessors()
-			assert.True(t, ok)
-			assert.ElementsMatch(t, keys(processors.GetMap()), []string{"span-metrics"})
-
-			// Modify overrides - PATCH
-			patch := &client.Limits{
-				MetricsGenerator: client.LimitsMetricsGenerator{
-					GenerateNativeHistograms:       histogramModePtr(histograms.HistogramMethodNative),
-					NativeHistogramMaxBucketNumber: uint32Ptr(200),
-					DisableCollection:              boolPtr(true),
-					Processor: client.LimitsMetricsGeneratorProcessor{
-						SpanMetrics: client.LimitsMetricsGeneratorProcessorSpanMetrics{
-							EnableInstanceLabel: boolPtr(false),
-						},
-					},
-				},
-			}
-
-			fmt.Println("* Patch overrides")
-			limits, version, err = apiClient.PatchOverrides(patch)
-			assert.NoError(t, err)
-
-			disableCollection, ok = limits.GetMetricsGenerator().GetDisableCollection()
-			assert.True(t, ok)
-			assert.True(t, disableCollection)
-			processors, ok = limits.GetMetricsGenerator().GetProcessors()
-			assert.True(t, ok)
-			assert.ElementsMatch(t, keys(processors.GetMap()), []string{"span-metrics"})
-			EnableInstanceLabel, EnableInstanceLabelIsSet = limits.GetMetricsGenerator().GetProcessor().GetSpanMetrics().GetEnableInstanceLabel()
-			assert.False(t, EnableInstanceLabel)
-			assert.True(t, EnableInstanceLabelIsSet)
-
-			generateNativeHistograms, ok := limits.GetMetricsGenerator().GetGenerateNativeHistograms()
-			assert.True(t, ok)
-			assert.Equal(t, histograms.HistogramMethodNative, generateNativeHistograms)
-			nativeHistogramMaxBucketNumber, ok := limits.GetMetricsGenerator().GetNativeHistogramMaxBucketNumber()
-			assert.True(t, ok)
-			assert.Equal(t, uint32(200), nativeHistogramMaxBucketNumber)
-
-			// Delete overrides
-			if !tc.skipVersioning && tc.name != "gcs" {
-				// Delete with preconditions is not supported by fake-gcs-server https://github.com/fsouza/fake-gcs-server/issues/1282
-				fmt.Println("* Deleting overrides - don't respect version")
-				err = apiClient.DeleteOverrides("123")
-				assert.ErrorContains(t, err, "412") // precondition failed
-			}
-
-			fmt.Println("* Deleting overrides")
-			err = apiClient.DeleteOverrides(version)
-			assert.NoError(t, err)
-
-			// Get overrides - 404
-			fmt.Println("* Get overrides - 404")
-			_, _, err = apiClient.GetOverrides()
-			assert.ErrorIs(t, err, httpclient.ErrNotFound)
-
-			// Recreate overrides - PATCH
-			patch = &client.Limits{
-				MetricsGenerator: client.LimitsMetricsGenerator{
-					DisableCollection: boolPtr(true),
-				},
-			}
-
-			fmt.Println("* Patch overrides - overrides don't exist yet")
-			_, _, err = apiClient.PatchOverrides(patch)
-			assert.NoError(t, err)
-		})
-	}
-}
-
-func TestOverridesAPI_GET(t *testing.T) {
-	s, tempo := setupTempo(t, configAllInOneS3Overrides)
-	defer s.Close()
-
-	t.Run("returns 404 when config not found", func(t *testing.T) {
-		apiClient := httpclient.New("http://"+tempo.Endpoint(3200), "tenant-get-1")
-
-		limits, etag, err := apiClient.GetOverrides()
-		require.Nil(t, limits) // no limits because it doesn't exist for this tenant
-		require.Empty(t, etag) // etag will be ""
-		require.ErrorIs(t, err, httpclient.ErrNotFound)
-	})
-
-	t.Run("returns config with etag", func(t *testing.T) {
-		apiClient := httpclient.New("http://"+tempo.Endpoint(3200), "tenant-get-2")
-
-		// create initial config with POST
-		initialLimits := &client.Limits{
-			CostAttribution: client.CostAttribution{
-				Dimensions: &map[string]string{"host.name": "host_name"},
-			},
-			MetricsGenerator: client.LimitsMetricsGenerator{
-				DisableCollection: boolPtr(false),
-			},
-		}
-		setEtag, err := apiClient.SetOverrides(initialLimits, "0")
-		require.NoError(t, err)
-		require.NotEmpty(t, setEtag)
-
-		// get should return the config and etag
-		returnedLimits, etag, err := apiClient.GetOverrides()
-		require.NoError(t, err)
-		require.Equal(t, setEtag, etag)
-		require.Equal(t, initialLimits, returnedLimits)
-	})
-}
-
-func TestOverridesAPI_POST(t *testing.T) {
-	s, tempo := setupTempo(t, configAllInOneS3Overrides)
-	defer s.Close()
-
-	t.Run("API returns 428 without if-match header", func(t *testing.T) {
-		baseURL := "http://" + tempo.Endpoint(3200)
-
-		req, err := http.NewRequest(http.MethodPost, fmt.Sprintf("%s/api/overrides", baseURL), strings.NewReader(`{}`))
-		require.NoError(t, err)
-		req.Header.Set("Content-Type", "application/json")
-		req.Header.Set("X-Scope-OrgID", "tenant-post-1")
-
-		resp, err := http.DefaultClient.Do(req)
-		require.NoError(t, err)
-		require.Equal(t, http.StatusPreconditionRequired, resp.StatusCode)
-		err = resp.Body.Close()
-		require.NoError(t, err)
-	})
-
-	t.Run("creates config with If-Match 0 on new tenant", func(t *testing.T) {
-		apiClient := httpclient.New("http://"+tempo.Endpoint(3200), "tenant-post-2")
-
-		limits := &client.Limits{
-			MetricsGenerator: client.LimitsMetricsGenerator{
-				DisableCollection: boolPtr(true),
-			},
-			CostAttribution: client.CostAttribution{
-				Dimensions: &map[string]string{"host.name": "host_name"},
-			},
-		}
-		setEtag, err := apiClient.SetOverrides(limits, "0")
-		require.NoError(t, err)
-		require.NotEmpty(t, setEtag)
-
-		// verify we can read it back
-		returnedLimits, etag, err := apiClient.GetOverrides()
-		require.NoError(t, err)
-		require.Equal(t, setEtag, etag)
-		require.Equal(t, limits, returnedLimits)
-		disableCollection, ok := returnedLimits.GetMetricsGenerator().GetDisableCollection()
-		require.True(t, ok)
-		require.True(t, disableCollection)
-	})
-
-	t.Run("tenant with existing config returns 412 with If-Match 0", func(t *testing.T) {
-		apiClient := httpclient.New("http://"+tempo.Endpoint(3200), "tenant-post-3")
-
-		// create initial config for tenant so we can try again
-		limits := &client.Limits{
-			CostAttribution: client.CostAttribution{
-				Dimensions: &map[string]string{"service.name": "service_name"},
-			},
-		}
-		etag, err := apiClient.SetOverrides(limits, "0")
-		require.NotEmpty(t, etag)
-		require.NoError(t, err)
-
-		// try to create config again with If-Match 0
-		etag2, err2 := apiClient.SetOverrides(limits, "0")
-		require.Empty(t, etag2)
-		require.ErrorContains(t, err2, "failed with response: 412 body: version does not match")
-	})
-
-	t.Run("incorrect If-Match returns 412", func(t *testing.T) {
-		apiClient := httpclient.New("http://"+tempo.Endpoint(3200), "tenant-post-4")
-
-		// Create initial config
-		limits := &client.Limits{
-			MetricsGenerator: client.LimitsMetricsGenerator{
-				DisableCollection: boolPtr(false),
-			},
-		}
-		etag, err := apiClient.SetOverrides(limits, "0")
-		require.NotEmpty(t, etag)
-		require.NoError(t, err)
-
-		// Try to update with wrong version
-		etag2, err2 := apiClient.SetOverrides(limits, "made-up-etag-value")
-		require.Empty(t, etag2)
-		require.ErrorContains(t, err2, "failed with response: 412 body: version does not match")
-	})
-
-	t.Run("with invalid json returns 400", func(t *testing.T) {
-		baseURL := "http://" + tempo.Endpoint(3200)
-
-		// invalid config
-		badConfig := strings.NewReader(`{"metrics_generator": {"processor": {"service_graphs": {"histogram_buckets": [0.1, "invalid"]}}}}`)
-		req, _ := http.NewRequest(http.MethodPost, fmt.Sprintf("%s/api/overrides", baseURL), badConfig)
-		req.Header.Set("Content-Type", "application/json")
-		req.Header.Set("X-Scope-OrgID", "tenant-post-5")
-		req.Header.Set("If-Match", "0")
-
-		resp, _ := http.DefaultClient.Do(req)
-		require.Equal(t, http.StatusBadRequest, resp.StatusCode)
-		resp.Body.Close()
-	})
-
-	t.Run("updates config with correct version", func(t *testing.T) {
-		apiClient := httpclient.New("http://"+tempo.Endpoint(3200), "tenant-post-6")
-
-		// create initial config
-		initialLimits := &client.Limits{
-			MetricsGenerator: client.LimitsMetricsGenerator{
-				DisableCollection: boolPtr(true),
-			},
-		}
-		setEtag, err := apiClient.SetOverrides(initialLimits, "0")
-		require.NoError(t, err)
-
-		// update with correct version
-		updatedLimits := &client.Limits{
-			MetricsGenerator: client.LimitsMetricsGenerator{
-				Processors: map[string]struct{}{"span-metrics": {}},
-			},
-		}
-		newEtag, err := apiClient.SetOverrides(updatedLimits, setEtag)
-		require.NoError(t, err)
-		require.NotEqual(t, setEtag, newEtag)
-
-		// verify the update
-		returnedLimits, _, err := apiClient.GetOverrides()
-		require.NoError(t, err)
-
-		// DisableCollection should not be set anymore because it was wiped out by the update
-		_, ok := returnedLimits.GetMetricsGenerator().GetDisableCollection()
-		require.False(t, ok)
-
-		// Processors should be set because we added then in updated
-		processors, ok := returnedLimits.GetMetricsGenerator().GetProcessors()
-		require.True(t, ok)
-		require.ElementsMatch(t, keys(processors.GetMap()), []string{"span-metrics"})
-	})
-}
-
-func TestOverridesAPI_PATCH(t *testing.T) {
-	s, tempo := setupTempo(t, configAllInOneS3Overrides)
-	defer s.Close()
-
-	t.Run("with no existing config creates new config", func(t *testing.T) {
-		apiClient := httpclient.New("http://"+tempo.Endpoint(3200), "tenant-patch-1")
-
-		patch := &client.Limits{
-			MetricsGenerator: client.LimitsMetricsGenerator{
-				DisableCollection: boolPtr(true),
-			},
-		}
-		returnedLimits, etag, err := apiClient.PatchOverrides(patch)
-		require.NoError(t, err)
-		require.NotEmpty(t, etag)
-
-		// verify returned limits match patch
-		disableCollection, ok := returnedLimits.GetMetricsGenerator().GetDisableCollection()
-		require.True(t, ok)
-		require.True(t, disableCollection)
-
-		// verify config was created via GET
-		getLimits, getEtag, err := apiClient.GetOverrides()
-		require.NoError(t, err)
-		require.Equal(t, etag, getEtag)
-		require.Equal(t, returnedLimits, getLimits)
-	})
-
-	t.Run("preserves existing config sections", func(t *testing.T) {
-		apiClient := httpclient.New("http://"+tempo.Endpoint(3200), "tenant-patch-2")
-
-		// create initial config with processors
-		initialLimits := &client.Limits{
-			MetricsGenerator: client.LimitsMetricsGenerator{
-				DisableCollection: boolPtr(true),
-				Processors:        map[string]struct{}{"span-metrics": {}},
-			},
-		}
-		_, _, err := apiClient.PatchOverrides(initialLimits)
-		require.NoError(t, err)
-
-		// PATCH with additional processor config
-		patch := &client.Limits{
-			MetricsGenerator: client.LimitsMetricsGenerator{
-				Processor: client.LimitsMetricsGeneratorProcessor{
-					SpanMetrics: client.LimitsMetricsGeneratorProcessorSpanMetrics{
-						EnableInstanceLabel: boolPtr(false),
-					},
-				},
-			},
-		}
-		returnedLimits, _, err := apiClient.PatchOverrides(patch)
-		require.NoError(t, err)
-
-		// verify original fields are preserved and not wiped out
-		disableCollection, ok := returnedLimits.GetMetricsGenerator().GetDisableCollection()
-		require.True(t, ok)
-		require.True(t, disableCollection)
-
-		processors, ok := returnedLimits.GetMetricsGenerator().GetProcessors()
-		require.True(t, ok)
-		require.ElementsMatch(t, keys(processors.GetMap()), []string{"span-metrics"})
-
-		// verify new field was added via PATCH
-		enableInstanceLabel, ok := returnedLimits.GetMetricsGenerator().GetProcessor().GetSpanMetrics().GetEnableInstanceLabel()
-		require.True(t, ok)
-		require.False(t, enableInstanceLabel)
-	})
-
-	t.Run("merges nested processor configs", func(t *testing.T) {
-		apiClient := httpclient.New("http://"+tempo.Endpoint(3200), "tenant-patch-3")
-
-		// create initial config with service graphs
-		initialPatch := &client.Limits{
-			MetricsGenerator: client.LimitsMetricsGenerator{
-				Processor: client.LimitsMetricsGeneratorProcessor{
-					ServiceGraphs: client.LimitsMetricsGeneratorProcessorServiceGraphs{
-						HistogramBuckets: &[]float64{0.1, 0.2},
-					},
-				},
-			},
-		}
-		_, _, err := apiClient.PatchOverrides(initialPatch)
-		require.NoError(t, err)
-
-		// PATCH with span metrics config
-		spanMetricsPatch := &client.Limits{
-			MetricsGenerator: client.LimitsMetricsGenerator{
-				Processor: client.LimitsMetricsGeneratorProcessor{
-					SpanMetrics: client.LimitsMetricsGeneratorProcessorSpanMetrics{
-						HistogramBuckets: &[]float64{0.3, 0.4},
-					},
-				},
-			},
-		}
-		returnedLimits, _, err := apiClient.PatchOverrides(spanMetricsPatch)
-		require.NoError(t, err)
-
-		// verify both processor configs exist in returned limits
-		sgBuckets, ok := returnedLimits.GetMetricsGenerator().GetProcessor().GetServiceGraphs().GetHistogramBuckets()
-		require.True(t, ok)
-		require.Equal(t, []float64{0.1, 0.2}, sgBuckets)
-
-		smBuckets, ok := returnedLimits.GetMetricsGenerator().GetProcessor().GetSpanMetrics().GetHistogramBuckets()
-		require.True(t, ok)
-		require.Equal(t, []float64{0.3, 0.4}, smBuckets)
-	})
-
-	t.Run("overwrites field values", func(t *testing.T) {
-		apiClient := httpclient.New("http://"+tempo.Endpoint(3200), "tenant-patch-4")
-
-		// create initial config
-		initialPatch := &client.Limits{
-			MetricsGenerator: client.LimitsMetricsGenerator{
-				Processor: client.LimitsMetricsGeneratorProcessor{
-					ServiceGraphs: client.LimitsMetricsGeneratorProcessorServiceGraphs{
-						HistogramBuckets: &[]float64{0.1, 0.2},
-					},
-				},
-			},
-		}
-		_, _, err := apiClient.PatchOverrides(initialPatch)
-		require.NoError(t, err)
-
-		// PATCH with new histogram buckets (should overwrite)
-		updatedPatch := &client.Limits{
-			MetricsGenerator: client.LimitsMetricsGenerator{
-				Processor: client.LimitsMetricsGeneratorProcessor{
-					ServiceGraphs: client.LimitsMetricsGeneratorProcessorServiceGraphs{
-						HistogramBuckets: &[]float64{0.5, 0.6, 0.7},
-					},
-				},
-			},
-		}
-		returnedLimits, _, err := apiClient.PatchOverrides(updatedPatch)
-		require.NoError(t, err)
-
-		// verify histogram buckets were updated in returned limits
-		buckets, ok := returnedLimits.GetMetricsGenerator().GetProcessor().GetServiceGraphs().GetHistogramBuckets()
-		require.True(t, ok)
-		require.Equal(t, []float64{0.5, 0.6, 0.7}, buckets)
-	})
-
-	t.Run("empty top level config doesn't overwrites nested configs", func(t *testing.T) {
-		apiClient := httpclient.New("http://"+tempo.Endpoint(3200), "tenant-patch-5")
-
-		// create initial config
-		initialPatch := &client.Limits{
-			CostAttribution: client.CostAttribution{
-				Dimensions: &map[string]string{"service.name": "service_name"},
-			},
-			MetricsGenerator: client.LimitsMetricsGenerator{
-				Processor: client.LimitsMetricsGeneratorProcessor{
-					ServiceGraphs: client.LimitsMetricsGeneratorProcessorServiceGraphs{
-						HistogramBuckets: &[]float64{0.1, 0.2},
-					},
-				},
-			},
-		}
-		returnedLimits, _, err := apiClient.PatchOverrides(initialPatch)
-		require.NoError(t, err)
-
-		// verify histogram buckets were updated in returned limits
-		buckets, ok := returnedLimits.GetMetricsGenerator().GetProcessor().GetServiceGraphs().GetHistogramBuckets()
-		require.True(t, ok)
-		require.Equal(t, []float64{0.1, 0.2}, buckets)
-
-		// verify that CostAttribution config exists and not touched
-		dims, ok := returnedLimits.GetCostAttribution().GetDimensions()
-		require.True(t, ok)
-		require.Equal(t, map[string]string{"service.name": "service_name"}, dims)
-
-		// PATCH with empty metric generator and it doesn't unset nested configs
-		updatedPatch := &client.Limits{
-			MetricsGenerator: client.LimitsMetricsGenerator{},
-		}
-		_, _, err2 := apiClient.PatchOverrides(updatedPatch)
-		require.NoError(t, err2)
-		// doing a GET after PATCH to ensure we get fresh config
-		returnedLimits2, _, err := apiClient.GetOverrides()
-		require.NoError(t, err)
-
-		// verify histogram buckets exist in the config
-		config := returnedLimits2.GetMetricsGenerator()
-		require.NotNil(t, config)
-		buckets, ok = returnedLimits2.GetMetricsGenerator().GetProcessor().GetServiceGraphs().GetHistogramBuckets()
-		require.True(t, ok) // should not be wiped out
-		require.Equal(t, []float64{0.1, 0.2}, buckets)
-	})
-
-	t.Run("version changes after patch", func(t *testing.T) {
-		apiClient := httpclient.New("http://"+tempo.Endpoint(3200), "tenant-patch-6")
-
-		// Create initial config with POST
-		initialLimits := &client.Limits{
-			MetricsGenerator: client.LimitsMetricsGenerator{
-				DisableCollection: boolPtr(true),
-			},
-		}
-		setEtag, err := apiClient.SetOverrides(initialLimits, "0")
-		require.NoError(t, err)
-
-		// Patch config
-		patch := &client.Limits{
-			MetricsGenerator: client.LimitsMetricsGenerator{
-				Processor: client.LimitsMetricsGeneratorProcessor{
-					ServiceGraphs: client.LimitsMetricsGeneratorProcessorServiceGraphs{
-						HistogramBuckets: &[]float64{0.1, 0.2},
-					},
-				},
-			},
-		}
-		_, patchEtag, err := apiClient.PatchOverrides(patch)
-		require.NoError(t, err)
-
-		// Verify version changed
-		require.NotEqual(t, setEtag, patchEtag)
-		require.NotEmpty(t, patchEtag)
-	})
-
-	t.Run("handles complex nested config", func(t *testing.T) {
-		apiClient := httpclient.New("http://"+tempo.Endpoint(3200), "tenant-patch-7")
-
-		// Create comprehensive config via PATCH
-		patch := &client.Limits{
-			MetricsGenerator: client.LimitsMetricsGenerator{
-				DisableCollection:              boolPtr(true),
-				GenerateNativeHistograms:       histogramModePtr(histograms.HistogramMethodNative),
-				NativeHistogramMaxBucketNumber: uint32Ptr(200),
-				Processors:                     map[string]struct{}{"span-metrics": {}},
-				Processor: client.LimitsMetricsGeneratorProcessor{
-					SpanMetrics: client.LimitsMetricsGeneratorProcessorSpanMetrics{
-						EnableInstanceLabel: boolPtr(false),
-					},
-				},
-			},
-		}
-		returnedLimits, _, err := apiClient.PatchOverrides(patch)
-		require.NoError(t, err)
-
-		// Verify all fields were set correctly
-		disableCollection, ok := returnedLimits.GetMetricsGenerator().GetDisableCollection()
-		require.True(t, ok)
-		require.True(t, disableCollection)
-
-		generateNativeHistograms, ok := returnedLimits.GetMetricsGenerator().GetGenerateNativeHistograms()
-		require.True(t, ok)
-		require.Equal(t, histograms.HistogramMethodNative, generateNativeHistograms)
-
-		maxBucketNumber, ok := returnedLimits.GetMetricsGenerator().GetNativeHistogramMaxBucketNumber()
-		require.True(t, ok)
-		require.Equal(t, uint32(200), maxBucketNumber)
-
-		processors, ok := returnedLimits.GetMetricsGenerator().GetProcessors()
-		require.True(t, ok)
-		require.ElementsMatch(t, keys(processors.GetMap()), []string{"span-metrics"})
-
-		enableInstanceLabel, ok := returnedLimits.GetMetricsGenerator().GetProcessor().GetSpanMetrics().GetEnableInstanceLabel()
-		require.True(t, ok)
-		require.False(t, enableInstanceLabel)
-	})
-}
-
-func TestOverridesAPI_DELETE(t *testing.T) {
-	s, tempo := setupTempo(t, configAllInOneS3Overrides)
-	defer s.Close()
-
-	t.Run("config is deleted with correct etag", func(t *testing.T) {
-		apiClient := httpclient.New("http://"+tempo.Endpoint(3200), "tenant-delete-1")
-
-		// create initial config
-		limits := &client.Limits{
-			MetricsGenerator: client.LimitsMetricsGenerator{
-				DisableCollection: boolPtr(true),
-			},
-		}
-		setEtag, err := apiClient.SetOverrides(limits, "0")
-		require.NoError(t, err)
-
-		// delete config
-		err = apiClient.DeleteOverrides(setEtag)
-		require.NoError(t, err)
-
-		// verify config is gone
-		deletedLimits, deletedEtag, err := apiClient.GetOverrides()
-		require.Nil(t, deletedLimits)
-		require.Empty(t, deletedEtag)
-		require.ErrorIs(t, err, httpclient.ErrNotFound)
-	})
-
-	t.Run("API returns 428 without if-match header", func(t *testing.T) {
-		baseURL := "http://" + tempo.Endpoint(3200)
-
-		// create config first
-		apiClient := httpclient.New(baseURL, "tenant-delete-2")
-		limits := &client.Limits{
-			CostAttribution: client.CostAttribution{
-				Dimensions: &map[string]string{"region": "region_label"},
-			},
-		}
-		_, err := apiClient.SetOverrides(limits, "0")
-		require.NoError(t, err)
-
-		// try to delete without If-Match header
-		req, err := http.NewRequest(http.MethodDelete, fmt.Sprintf("%s/api/overrides", baseURL), nil)
-		require.NoError(t, err)
-		req.Header.Set("X-Scope-OrgID", "tenant-delete-2")
-
-		resp, err := http.DefaultClient.Do(req)
-		require.NoError(t, err)
-		defer resp.Body.Close()
-
-		require.Equal(t, http.StatusPreconditionRequired, resp.StatusCode)
-	})
-
-	t.Run("API returns 412 with wrong version ", func(t *testing.T) {
-		apiClient := httpclient.New("http://"+tempo.Endpoint(3200), "tenant-delete-wrongver")
-
-		// create config
-		limits := &client.Limits{
-			MetricsGenerator: client.LimitsMetricsGenerator{
-				DisableCollection: boolPtr(false),
-			},
-		}
-		setEtag, err := apiClient.SetOverrides(limits, "0")
-		require.NoError(t, err)
-		require.NotEmpty(t, setEtag)
-
-		// try to delete with wrong version
-		err = apiClient.DeleteOverrides("wrong-version")
-		require.ErrorContains(t, err, "412")
-
-		// verify config still exists
-		returnedLimits, gotEtag, err := apiClient.GetOverrides()
-		require.NoError(t, err)
-		require.Equal(t, setEtag, gotEtag)
-		require.Equal(t, limits, returnedLimits)
-	})
-}
-
-// Helper functions for overrides API tests
-
-// setupTempo stands up a new NewTempoAllInOne for e2e test
-func setupTempo(t *testing.T, configFilePath string) (*e2e.Scenario, *e2e.HTTPService) {
-	s, err := e2e.NewScenario("tempo_e2e")
-	require.NoError(t, err)
-
-	// copy config template to shared directory and expand template variables
-	tmplConfig := map[string]any{"Prefix": ""}
-	configFile, err := util.CopyTemplateToSharedDir(s, configFilePath, "config.yaml", tmplConfig)
-	require.NoError(t, err)
-
-	// set up the backend using the config file
-	cfg := app.Config{}
-	buff, err := os.ReadFile(configFile)
-	require.NoError(t, err)
-	err = yaml.UnmarshalStrict(buff, &cfg)
-	require.NoError(t, err)
-	_, err = backend.New(s, cfg)
-	require.NoError(t, err)
-
-	tempo := util.NewTempoAllInOne()
-	require.NoError(t, s.StartAndWaitReady(tempo))
-
-	return s, tempo
-}
-
-func printLimits(limits *client.Limits, version string) {
-	var str string
-	if limits != nil {
-		bytes, err := jsoniter.Marshal(limits)
-		if err == nil {
-			str = string(bytes)
-		}
-	}
-	fmt.Printf("* Overrides (version = %s): %+v\n", version, str)
-}
-
-func boolPtr(b bool) *bool {
-	return &b
-}
-
-func histogramModePtr(h histograms.HistogramMethod) *histograms.HistogramMethod {
-	return &h
-}
-
-func uint32Ptr(u uint32) *uint32 {
-	return &u
-}
-
-func keys(m map[string]struct{}) []string {
-	var keys []string
-	for k := range m {
-		keys = append(keys, k)
-	}
-	return keys
-}

integration/e2e/api/query_range_test.go

@@ -1,922 +0,0 @@
-package api
-
-import (
-	"context"
-	"fmt"
-	"io"
-	"math"
-	"net/http"
-	"net/url"
-	"strings"
-	"testing"
-	"time"
-
-	"github.com/gogo/protobuf/jsonpb"
-	"github.com/google/uuid"
-	"github.com/jaegertracing/jaeger-idl/thrift-gen/jaeger"
-
-	"github.com/grafana/e2e"
-	"github.com/grafana/tempo/integration/util"
-	"github.com/grafana/tempo/pkg/tempopb"
-	"github.com/stretchr/testify/assert"
-	"github.com/stretchr/testify/require"
-)
-
-const (
-	configQueryRange                         = "config-query-range.yaml"
-	configQueryRangeMaxSeries                = "config-query-range-max-series.yaml"
-	configQueryRangeMaxSeriesDisabled        = "config-query-range-max-series-disabled.yaml"
-	configQueryRangeMaxSeriesDisabledQuerier = "config-query-range-max-series-disabled-querier.yaml"
-)
-
-type queryRangeRequest struct {
-	Query     string    `json:"query"`
-	Start     time.Time `json:"start"`     // default: now - 5m
-	End       time.Time `json:"end"`       // default: now + 1m
-	Step      string    `json:"step"`      // default: 5s
-	Exemplars int       `json:"exemplars"` // default: 100
-	noDefault bool      `json:"-"`         // if true, SetDefaults() will not set defaults`
-}
-
-func (r *queryRangeRequest) SetDefaults() {
-	if r.noDefault {
-		return
-	}
-	if r.Start.IsZero() {
-		r.Start = time.Now().Add(-5 * time.Minute)
-	}
-	if r.End.IsZero() {
-		r.End = time.Now().Add(time.Minute)
-	}
-	if r.Step == "" {
-		r.Step = "5s"
-	}
-	if r.Exemplars == 0 {
-		r.Exemplars = 100
-	}
-}
-
-// Set debugMode to true to print the response body
-var debugMode = false
-
-func TestQueryRangeExemplars(t *testing.T) {
-	t.Parallel()
-
-	s, err := e2e.NewScenario("tempo_e2e")
-	require.NoError(t, err)
-	defer s.Close()
-
-	require.NoError(t, util.CopyFileToSharedDir(s, configQueryRange, "config.yaml"))
-	tempo := util.NewTempoAllInOne()
-	require.NoError(t, s.StartAndWaitReady(tempo))
-
-	jaegerClient, err := util.NewJaegerToOTLPExporter(tempo.Endpoint(4317))
-	require.NoError(t, err)
-	require.NotNil(t, jaegerClient)
-
-	ticker := time.NewTicker(500 * time.Millisecond)
-	defer ticker.Stop()
-	timer := time.NewTimer(10 * time.Second)
-	defer timer.Stop()
-	// send one batch every 500ms for 10 seconds
-sendLoop:
-	for {
-		select {
-		case <-ticker.C:
-			require.NoError(t, jaegerClient.EmitBatch(context.Background(),
-				util.MakeThriftBatchWithSpanCountAttributeAndName(
-					1, "my operation",
-					"res_val", "span_val",
-					"res_attr", "span_attr",
-				),
-			))
-			require.NoError(t, jaegerClient.EmitBatch(context.Background(),
-				util.MakeThriftBatchWithSpanCountAttributeAndName(
-					1, "my operation",
-					"res_val2", "span_val2",
-					"res_attr", "span_attr",
-				),
-			))
-			require.NoError(t, jaegerClient.EmitBatch(context.Background(),
-				util.MakeThriftBatchWithSpanCountAttributeAndName(
-					1, "operation with high cardinality",
-					uuid.New().String(), uuid.New().String(),
-					"res_high_cardinality", "span_high_cardinality",
-				),
-			))
-		case <-timer.C:
-			break sendLoop
-		}
-	}
-
-	// Wait for traces to be flushed to blocks
-	require.NoError(t, tempo.WaitSumMetricsWithOptions(e2e.GreaterOrEqual(1), []string{"tempo_metrics_generator_processor_local_blocks_spans_total"}, e2e.WaitMissingMetrics))
-	require.NoError(t, tempo.WaitSumMetricsWithOptions(e2e.GreaterOrEqual(1), []string{"tempo_metrics_generator_processor_local_blocks_cut_blocks"}, e2e.WaitMissingMetrics))
-
-	for _, exemplarsCase := range []struct {
-		name              string
-		exemplars         int
-		expectedExemplars int
-	}{
-		{
-			name:              "default",
-			exemplars:         0,
-			expectedExemplars: 100, // if set to 0, then limits to 100
-		},
-		{
-			name:              "5 exemplar",
-			exemplars:         5,
-			expectedExemplars: 5,
-		},
-		{
-			name:              "25 exemplars",
-			exemplars:         25,
-			expectedExemplars: 25,
-		},
-		{
-			name:              "capped exemplars",
-			exemplars:         1000,
-			expectedExemplars: 100, // capped to 100
-		},
-	} {
-		for _, query := range []string{
-			"{} | rate()",
-			"{} | compare({status=error})",
-			"{} | count_over_time()",
-			"{} | min_over_time(duration)",
-			"{} | max_over_time(duration)",
-			"{} | avg_over_time(duration)",
-			"{} | sum_over_time(duration)",
-			"{} | quantile_over_time(duration, .5)",
-			"{} | quantile_over_time(duration, .5, 0.9, 0.99)",
-
-			"{} | count_over_time() by (span.span_attr)",
-			"{} | count_over_time() by (resource.res_attr)",
-			"{} | count_over_time() by (.span_attr)",
-			"{} | count_over_time() by (.res_attr)",
-
-			"{} | histogram_over_time(duration)",
-			"{} | count_over_time() by (status)",
-			"{status != error} | count_over_time() by (status)",
-		} {
-			t.Run(fmt.Sprintf("%s: %s", exemplarsCase.name, query), func(t *testing.T) {
-				req := queryRangeRequest{
-					Query:     query,
-					Exemplars: exemplarsCase.exemplars,
-				}
-				queryRangeRes := callQueryRange(t, tempo.Endpoint(tempoPort), req)
-				require.NotNil(t, queryRangeRes)
-				require.GreaterOrEqual(t, len(queryRangeRes.GetSeries()), 1)
-
-				exemplarCount := 0
-
-				for _, series := range queryRangeRes.GetSeries() {
-					exemplarCount += len(series.GetExemplars())
-				}
-				assert.LessOrEqual(t, exemplarCount, exemplarsCase.expectedExemplars)
-				assert.GreaterOrEqual(t, exemplarCount, 1)
-			})
-		}
-	}
-
-	// check exemplars in more detail
-	for _, testCase := range []struct {
-		query                   string
-		targetAttribute         string
-		targetExemplarAttribute string
-	}{
-		{
-			query:                   "{} | quantile_over_time(duration, .9) by (span.span_attr)",
-			targetAttribute:         "span.span_attr",
-			targetExemplarAttribute: "span.span_attr",
-		},
-		{
-			query:                   "{} | quantile_over_time(duration, .9) by (resource.res_attr)",
-			targetAttribute:         "resource.res_attr",
-			targetExemplarAttribute: "resource.res_attr",
-		},
-		{
-			query:                   "{} | quantile_over_time(duration, .9) by (.span_attr)",
-			targetAttribute:         ".span_attr",
-			targetExemplarAttribute: "span.span_attr",
-		},
-		{
-			query:                   "{} | quantile_over_time(duration, .9) by (.res_attr)",
-			targetAttribute:         ".res_attr",
-			targetExemplarAttribute: "resource.res_attr",
-		},
-		{
-			query:                   "{} | rate() by (span.span_attr)",
-			targetAttribute:         "span.span_attr",
-			targetExemplarAttribute: "span.span_attr",
-		},
-		{
-			query:                   "{} | count_over_time() by (span.span_attr)",
-			targetAttribute:         "span.span_attr",
-			targetExemplarAttribute: "span.span_attr",
-		},
-		{
-			query:                   "{} | min_over_time(duration) by (span.span_attr)",
-			targetAttribute:         "span.span_attr",
-			targetExemplarAttribute: "span.span_attr",
-		},
-		{
-			query:                   "{} | max_over_time(duration) by (span.span_attr)",
-			targetAttribute:         "span.span_attr",
-			targetExemplarAttribute: "span.span_attr",
-		},
-		{
-			query:                   "{} | avg_over_time(duration) by (span.span_attr)",
-			targetAttribute:         "span.span_attr",
-			targetExemplarAttribute: "span.span_attr",
-		},
-		{
-			query:                   "{} | sum_over_time(duration) by (span.span_attr)",
-			targetAttribute:         "span.span_attr",
-			targetExemplarAttribute: "span.span_attr",
-		},
-	} {
-		t.Run(testCase.query, func(t *testing.T) {
-			req := queryRangeRequest{
-				Query:     testCase.query,
-				Exemplars: 100,
-			}
-			queryRangeRes := callQueryRange(t, tempo.Endpoint(tempoPort), req)
-			require.NotNil(t, queryRangeRes)
-			require.Equal(t, len(queryRangeRes.GetSeries()), 3) // value 1, value 2 and nil (high cardinality's span has no such attribute)
-
-			// Verify that all exemplars in this series belongs to the right series
-			// by matching attribute values
-			var skippedForNilAttr bool
-			for _, series := range queryRangeRes.Series {
-				// search attribute value for the series
-				var expectedAttrValue string
-				for _, label := range series.Labels {
-					if label.Key == testCase.targetAttribute {
-						expectedAttrValue = label.Value.GetStringValue()
-						break
-					}
-				}
-				if (expectedAttrValue == "" || expectedAttrValue == "nil") && !skippedForNilAttr { // one attribute is empty, so we skip it
-					skippedForNilAttr = true
-					continue
-				}
-				require.NotEmpty(t, expectedAttrValue)
-
-				// check attribute value in exemplars
-				for _, exemplar := range series.Exemplars {
-					var actualAttrValue string
-					for _, label := range exemplar.Labels {
-						if label.Key == testCase.targetExemplarAttribute {
-							actualAttrValue = label.Value.GetStringValue()
-							break
-						}
-					}
-					require.Equal(t, expectedAttrValue, actualAttrValue)
-				}
-			}
-		})
-	}
-
-	// invalid query
-	for name, req := range map[string]queryRangeRequest{
-		"invalid query": {
-			Query: "{. a}",
-		},
-		"step=0 (default step)": {
-			Query: "{} | count_over_time()",
-			Step:  "0",
-		},
-		"step=0s (default step)": {
-			Query: "{} | count_over_time()",
-			Step:  "0s",
-		},
-	} {
-		t.Run(name, func(t *testing.T) {
-			req := req
-			req.SetDefaults()
-			res := doRequest(t, tempo.Endpoint(tempoPort), "api/metrics/query_range", req)
-			require.Equal(t, 400, res.StatusCode)
-		})
-	}
-
-	// small step
-	t.Run("small step", func(t *testing.T) {
-		req := queryRangeRequest{Query: "{} | count_over_time()"}
-		req.SetDefaults()
-		req.Step = "35ms"
-		res := doRequest(t, tempo.Endpoint(tempoPort), "api/metrics/query_range", req)
-		require.Equal(t, 400, res.StatusCode)
-		body, err := io.ReadAll(res.Body)
-		require.NoError(t, err)
-		require.Equal(t, string(body), "step of 35ms is too small, minimum step for given range is 36ms")
-	})
-
-	// query with empty results
-	for _, query := range []string{
-		// existing attribute, no traces
-		"{status=error} | count_over_time()",
-		// non-existing attribute, no traces
-		`{span.randomattr = "doesnotexist"} | count_over_time()`,
-	} {
-		t.Run(query, func(t *testing.T) {
-			queryRangeRes := callQueryRange(t, tempo.Endpoint(tempoPort), queryRangeRequest{Query: query, Exemplars: 100})
-			require.NotNil(t, queryRangeRes)
-			// it has time series but they are empty and has no exemplars
-			require.GreaterOrEqual(t, len(queryRangeRes.GetSeries()), 1)
-			exemplarCount := 0
-			for _, series := range queryRangeRes.GetSeries() {
-				exemplarCount += len(series.GetExemplars())
-			}
-			require.Equal(t, 0, exemplarCount)
-		})
-	}
-
-	for _, testCase := range []struct {
-		name      string
-		query     string
-		step      string
-		converter func([]tempopb.Sample) float64
-	}{
-		{
-			name:      "count_over_time",
-			query:     "{} | count_over_time()",
-			step:      "1s",
-			converter: sumSamples,
-		},
-		{
-			name:      "sum_over_time",
-			query:     "{} | sum_over_time(duration)",
-			step:      "1s",
-			converter: sumSamples,
-		},
-		{
-			name:      "max_over_time",
-			query:     "{} | max_over_time(duration)",
-			step:      "1s",
-			converter: maxSamples,
-		},
-		{
-			name:      "min_over_time",
-			query:     "{} | min_over_time(duration)",
-			step:      "1s",
-			converter: minSamples,
-		},
-		{
-			name:      "1m step",
-			query:     "{} | count_over_time()",
-			step:      "1m",
-			converter: sumSamples,
-		},
-	} {
-		t.Run(testCase.name, func(t *testing.T) {
-			req := queryRangeRequest{
-				Query: testCase.query,
-				// Query range truncates the start and end to the step, while instant query does not.
-				// We need to truncate the start and end to the step to align the interval for query range and instant query.
-				Start: time.Now().Add(-5 * time.Minute).Truncate(time.Minute),
-				End:   time.Now().Add(time.Minute).Truncate(time.Minute),
-				Step:  testCase.step,
-			}
-
-			queryRangeRes := callQueryRange(t, tempo.Endpoint(tempoPort), req)
-			require.NotNil(t, queryRangeRes)
-			require.Equal(t, 1, len(queryRangeRes.GetSeries()))
-
-			expectedValue := testCase.converter(queryRangeRes.Series[0].Samples)
-
-			instantQueryRes := callInstantQuery(t, tempo.Endpoint(tempoPort), req)
-			require.NotNil(t, instantQueryRes)
-			require.Equal(t, 1, len(instantQueryRes.GetSeries()))
-			require.InDelta(t, expectedValue, instantQueryRes.GetSeries()[0].Value, 0.000001)
-		})
-	}
-
-	t.Run("avg_over_time instant query", func(t *testing.T) {
-		req := queryRangeRequest{
-			Query: "{} | avg_over_time(duration)",
-			Start: time.Now().Add(-5 * time.Minute),
-			End:   time.Now().Add(time.Minute),
-		}
-
-		countReq := req
-		countReq.Query = "{} | count_over_time()"
-		countRes := callInstantQuery(t, tempo.Endpoint(tempoPort), countReq)
-		require.NotNil(t, countRes)
-		require.Equal(t, 1, len(countRes.GetSeries()))
-		count := countRes.GetSeries()[0].Value
-
-		sumReq := req
-		sumReq.Query = "{} | sum_over_time(duration)"
-		sumRes := callInstantQuery(t, tempo.Endpoint(tempoPort), sumReq)
-		require.NotNil(t, sumRes)
-		require.Equal(t, 1, len(sumRes.GetSeries()))
-		sum := sumRes.GetSeries()[0].Value
-
-		res := callInstantQuery(t, tempo.Endpoint(tempoPort), req)
-		require.NotNil(t, res)
-		require.Equal(t, 1, len(res.GetSeries()))
-		require.InDelta(t, sum/count, res.GetSeries()[0].Value, 0.000001)
-	})
-
-	for _, testCase := range []struct {
-		name        string
-		query       string
-		expectedNum int
-	}{
-		{
-			name:        "top 1 by span attribute",
-			query:       "{ } | rate() by (span.span_high_cardinality) | topk(1)",
-			expectedNum: 1,
-		},
-		{
-			name:        "top 10 by span attribute",
-			query:       "{ } | rate() by (span.span_high_cardinality) | topk(10)",
-			expectedNum: 10,
-		},
-		{
-			name:        "top 2 by resource attribute",
-			query:       "{ } | rate() by (resource.res_high_cardinality) | topk(2)",
-			expectedNum: 2,
-		},
-		{
-			name:        "bottom 1 by resource attribute",
-			query:       "{ } | rate() by (resource.res_high_cardinality) | bottomk(1)",
-			expectedNum: 1,
-		},
-		{
-			name:        "bootom 10 by resource attribute",
-			query:       "{ } | rate() by (resource.res_high_cardinality) | bottomk(10)",
-			expectedNum: 10,
-		},
-		{
-			name:        "bottom 2 by span attribute",
-			query:       "{ } | rate() by (span.span_high_cardinality) | bottomk(2)",
-			expectedNum: 2,
-		},
-	} {
-		t.Run(testCase.name, func(t *testing.T) {
-			req := queryRangeRequest{
-				Query: testCase.query,
-				Start: time.Now().Add(-5 * time.Minute),
-				End:   time.Now().Add(time.Minute),
-				Step:  "1m",
-			}
-
-			instantQueryRes := callInstantQuery(t, tempo.Endpoint(tempoPort), req)
-			require.NotNil(t, instantQueryRes)
-			require.Equal(t, testCase.expectedNum, len(instantQueryRes.GetSeries()))
-		})
-	}
-
-	for _, testCase := range []struct {
-		name              string
-		end               time.Time
-		step              string
-		expectedIntervals int
-	}{
-		// |---start|---|---end|
-		{name: "aligned", end: time.Now().Truncate(time.Minute), step: "1m", expectedIntervals: 3},
-		// |---|---start---|---|---end---|
-		{name: "unaligned", end: time.Now(), step: "1m", expectedIntervals: 4},
-		{name: "default step", end: time.Now(), step: "", expectedIntervals: 122},
-	} {
-		t.Run(testCase.name, func(t *testing.T) {
-			req := queryRangeRequest{
-				Query:     "{} | count_over_time()",
-				Start:     testCase.end.Add(-2 * time.Minute),
-				End:       testCase.end,
-				Step:      testCase.step,
-				noDefault: true,
-			}
-
-			queryRangeRes := callQueryRange(t, tempo.Endpoint(tempoPort), req)
-			require.NotNil(t, queryRangeRes)
-			series := queryRangeRes.GetSeries()
-			require.Equal(t, 1, len(series), "Expected 1 series for count_over_time query")
-			require.Equal(t, testCase.expectedIntervals, len(series[0].Samples))
-		})
-	}
-}
-
-func sumSamples(samples []tempopb.Sample) float64 {
-	var sum float64
-	for _, sample := range samples {
-		sum += sample.Value
-	}
-	return sum
-}
-
-func maxSamples(samples []tempopb.Sample) float64 {
-	maxValue := math.Inf(-1)
-	for _, sample := range samples {
-		if sample.Value > maxValue {
-			maxValue = sample.Value
-		}
-	}
-	return maxValue
-}
-
-func minSamples(samples []tempopb.Sample) float64 {
-	minValue := math.Inf(1)
-	for _, sample := range samples {
-		if sample.Value < minValue {
-			minValue = sample.Value
-		}
-	}
-	return minValue
-}
-
-// TestQueryRangeSingleTrace checks count for a single trace
-// Single trace creates a block with startTime == endTime
-// which covers a few edge cases under the hood.
-func TestQueryRangeSingleTrace(t *testing.T) {
-	t.Parallel()
-
-	s, err := e2e.NewScenario("tempo_e2e_single_trace")
-	require.NoError(t, err)
-	defer s.Close()
-
-	require.NoError(t, util.CopyFileToSharedDir(s, configQueryRange, "config.yaml"))
-	tempo := util.NewTempoAllInOne()
-	require.NoError(t, s.StartAndWaitReady(tempo))
-
-	jaegerClient, err := util.NewJaegerToOTLPExporter(tempo.Endpoint(4317))
-	require.NoError(t, err)
-	require.NotNil(t, jaegerClient)
-
-	// Emit a single trace
-	require.NoError(t, jaegerClient.EmitBatch(context.Background(), util.MakeThriftBatch()))
-
-	// Wait for traces to be flushed to blocks
-	require.NoError(t, tempo.WaitSumMetricsWithOptions(e2e.GreaterOrEqual(1), []string{"tempo_metrics_generator_processor_local_blocks_spans_total"}, e2e.WaitMissingMetrics))
-	require.NoError(t, tempo.WaitSumMetricsWithOptions(e2e.GreaterOrEqual(1), []string{"tempo_metrics_generator_processor_local_blocks_cut_blocks"}, e2e.WaitMissingMetrics))
-
-	// Query the trace by count. As we have only one trace, we should get one dot with value 1
-	query := "{} | count_over_time()"
-	queryRangeRes := callQueryRange(t, tempo.Endpoint(tempoPort), queryRangeRequest{Query: query, Exemplars: 100})
-	require.NotNil(t, queryRangeRes)
-	require.Equal(t, len(queryRangeRes.GetSeries()), 1)
-
-	series := queryRangeRes.GetSeries()[0]
-	assert.Equal(t, len(series.GetExemplars()), 1)
-
-	var sum float64
-	for _, sample := range series.GetSamples() {
-		sum += sample.Value
-	}
-	require.InDelta(t, sum, 1, 0.000001)
-}
-
-func TestQueryRangeMaxSeries(t *testing.T) {
-	t.Parallel()
-
-	s, err := e2e.NewScenario("tempo_e2e_query_range_max_series")
-	require.NoError(t, err)
-	defer s.Close()
-
-	require.NoError(t, util.CopyFileToSharedDir(s, configQueryRangeMaxSeries, "config.yaml"))
-	tempo := util.NewTempoAllInOne()
-	require.NoError(t, s.StartAndWaitReady(tempo))
-
-	jaegerClient, err := util.NewJaegerToOTLPExporter(tempo.Endpoint(4317))
-	require.NoError(t, err)
-	require.NotNil(t, jaegerClient)
-
-	ticker := time.NewTicker(500 * time.Millisecond)
-	defer ticker.Stop()
-	timer := time.NewTimer(5 * time.Second)
-	defer timer.Stop()
-sendLoop:
-	for {
-		select {
-		case <-ticker.C:
-			require.NoError(t, jaegerClient.EmitBatch(context.Background(), util.MakeThriftBatch()))
-		case <-timer.C:
-			break sendLoop
-		}
-	}
-	util.CallFlush(t, tempo)
-	time.Sleep(1 * time.Second)
-	// Wait for traces to be flushed to blocks
-	require.NoError(t, tempo.WaitSumMetricsWithOptions(e2e.GreaterOrEqual(1), []string{"tempo_metrics_generator_processor_local_blocks_spans_total"}, e2e.WaitMissingMetrics))
-	require.NoError(t, tempo.WaitSumMetricsWithOptions(e2e.GreaterOrEqual(1), []string{"tempo_metrics_generator_processor_local_blocks_cut_blocks"}, e2e.WaitMissingMetrics))
-
-	query := "{} | rate() by (span:id)"
-	url := fmt.Sprintf(
-		"http://%s/api/metrics/query_range?q=%s&start=%d&end=%d&step=%s",
-		tempo.Endpoint(3200),
-		url.QueryEscape(query),
-		time.Now().Add(-5*time.Minute).UnixNano(),
-		time.Now().UnixNano(),
-		"5s",
-	)
-
-	req, err := http.NewRequest(http.MethodGet, url, nil)
-	require.NoError(t, err)
-
-	res, err := http.DefaultClient.Do(req)
-	require.NoError(t, err)
-
-	// Read body
-	body, err := io.ReadAll(res.Body)
-	require.NoError(t, err)
-
-	queryRangeRes := &tempopb.QueryRangeResponse{}
-	readBody := strings.NewReader(string(body))
-	err = new(jsonpb.Unmarshaler).Unmarshal(readBody, queryRangeRes)
-	require.NoError(t, err)
-	require.NotNil(t, queryRangeRes)
-
-	// max series is 3 so we should get a partial response with 3 series
-	require.Equal(t, tempopb.PartialStatus_PARTIAL, queryRangeRes.GetStatus())
-	require.Equal(t, "Response exceeds maximum series limit of 3, a partial response is returned. Warning: the accuracy of each individual value is not guaranteed.", queryRangeRes.GetMessage())
-	require.Equal(t, 3, len(queryRangeRes.GetSeries()))
-}
-
-func TestQueryRangeMaxSeriesDisabled(t *testing.T) {
-	t.Parallel()
-
-	s, err := e2e.NewScenario("tempo_e2e_query_range_max_series_disabled")
-	require.NoError(t, err)
-	defer s.Close()
-
-	require.NoError(t, util.CopyFileToSharedDir(s, configQueryRangeMaxSeriesDisabled, "config.yaml"))
-	tempo := util.NewTempoAllInOne()
-	require.NoError(t, s.StartAndWaitReady(tempo))
-
-	jaegerClient, err := util.NewJaegerToOTLPExporter(tempo.Endpoint(4317))
-	require.NoError(t, err)
-	require.NotNil(t, jaegerClient)
-
-	ticker := time.NewTicker(500 * time.Millisecond)
-	defer ticker.Stop()
-	timer := time.NewTimer(5 * time.Second)
-	defer timer.Stop()
-	spanCount := 0
-sendLoop:
-	for {
-		select {
-		case <-ticker.C:
-			require.NoError(t, jaegerClient.EmitBatch(context.Background(), util.MakeThriftBatch()))
-			spanCount++
-		case <-timer.C:
-			break sendLoop
-		}
-	}
-
-	// Wait for traces to be flushed to blocks
-	require.NoError(t, tempo.WaitSumMetricsWithOptions(e2e.GreaterOrEqual(1), []string{"tempo_metrics_generator_processor_local_blocks_spans_total"}, e2e.WaitMissingMetrics))
-	require.NoError(t, tempo.WaitSumMetricsWithOptions(e2e.GreaterOrEqual(1), []string{"tempo_metrics_generator_processor_local_blocks_cut_blocks"}, e2e.WaitMissingMetrics))
-
-	query := "{} | rate() by (span:id)"
-	url := fmt.Sprintf(
-		"http://%s/api/metrics/query_range?q=%s&start=%d&end=%d&step=%s",
-		tempo.Endpoint(3200),
-		url.QueryEscape(query),
-		time.Now().Add(-5*time.Minute).UnixNano(),
-		time.Now().UnixNano(),
-		"5s",
-	)
-
-	req, err := http.NewRequest(http.MethodGet, url, nil)
-	require.NoError(t, err)
-
-	res, err := http.DefaultClient.Do(req)
-	require.NoError(t, err)
-
-	// Read body
-	body, err := io.ReadAll(res.Body)
-	require.NoError(t, err)
-
-	queryRangeRes := &tempopb.QueryRangeResponse{}
-	readBody := strings.NewReader(string(body))
-	err = new(jsonpb.Unmarshaler).Unmarshal(readBody, queryRangeRes)
-	require.NoError(t, err)
-	require.NotNil(t, queryRangeRes)
-
-	// max series is disabled so we should get a complete response with all series
-	require.Equal(t, tempopb.PartialStatus_COMPLETE, queryRangeRes.GetStatus())
-	require.Equal(t, spanCount, len(queryRangeRes.GetSeries()))
-}
-
-func TestQueryRangeMaxSeriesDisabledQuerier(t *testing.T) {
-	t.Parallel()
-
-	s, err := e2e.NewScenario("tempo_e2e_query_range_max_series_disabled_querier")
-	require.NoError(t, err)
-	defer s.Close()
-
-	require.NoError(t, util.CopyFileToSharedDir(s, configQueryRangeMaxSeriesDisabledQuerier, "config.yaml"))
-	tempo := util.NewTempoAllInOne()
-	require.NoError(t, s.StartAndWaitReady(tempo))
-
-	jaegerClient, err := util.NewJaegerToOTLPExporter(tempo.Endpoint(4317))
-	require.NoError(t, err)
-	require.NotNil(t, jaegerClient)
-
-	ticker := time.NewTicker(500 * time.Millisecond)
-	defer ticker.Stop()
-	timer := time.NewTimer(5 * time.Second)
-	defer timer.Stop()
-	spanCount := 0
-sendLoop:
-	for {
-		select {
-		case <-ticker.C:
-			require.NoError(t, jaegerClient.EmitBatch(context.Background(), util.MakeThriftBatch()))
-			spanCount++
-		case <-timer.C:
-			break sendLoop
-		}
-	}
-
-	// Wait for traces to be flushed to blocks
-	require.NoError(t, tempo.WaitSumMetricsWithOptions(e2e.GreaterOrEqual(1), []string{"tempo_metrics_generator_processor_local_blocks_spans_total"}, e2e.WaitMissingMetrics))
-	require.NoError(t, tempo.WaitSumMetricsWithOptions(e2e.GreaterOrEqual(1), []string{"tempo_metrics_generator_processor_local_blocks_cut_blocks"}, e2e.WaitMissingMetrics))
-
-	// Wait for the traces to be written to the WAL
-	time.Sleep(time.Second * 4)
-
-	util.CallFlush(t, tempo)
-	time.Sleep(blockFlushTimeout)
-	util.CallFlush(t, tempo)
-
-	require.NoError(t, tempo.WaitSumMetrics(e2e.GreaterOrEqual(5), "tempo_ingester_blocks_flushed_total"))
-
-	query := "{} | rate() by (span:id)"
-	url := fmt.Sprintf(
-		"http://%s/api/metrics/query_range?q=%s&start=%d&end=%d&step=%s",
-		tempo.Endpoint(3200),
-		url.QueryEscape(query),
-		time.Now().Add(-5*time.Minute).UnixNano(),
-		time.Now().UnixNano(),
-		"5s",
-	)
-
-	req, err := http.NewRequest(http.MethodGet, url, nil)
-	require.NoError(t, err)
-
-	res, err := http.DefaultClient.Do(req)
-	require.NoError(t, err)
-
-	// Read body
-	body, err := io.ReadAll(res.Body)
-	require.NoError(t, err)
-
-	queryRangeRes := &tempopb.QueryRangeResponse{}
-	readBody := strings.NewReader(string(body))
-	err = new(jsonpb.Unmarshaler).Unmarshal(readBody, queryRangeRes)
-	require.NoError(t, err)
-	require.NotNil(t, queryRangeRes)
-
-	// max series is disabled so we should get a complete response with all series
-	require.Equal(t, tempopb.PartialStatus_COMPLETE, queryRangeRes.GetStatus())
-	require.Equal(t, spanCount, len(queryRangeRes.GetSeries()))
-}
-
-func TestQueryRangeTypeHandling(t *testing.T) {
-	t.Parallel()
-
-	ctx := t.Context()
-
-	s, err := e2e.NewScenario("tempo_e2e_query_range_type_handling")
-	require.NoError(t, err)
-	defer s.Close()
-
-	require.NoError(t, util.CopyFileToSharedDir(s, configQueryRangeMaxSeriesDisabledQuerier, "config.yaml"))
-	tempo := util.NewTempoAllInOne()
-	require.NoError(t, s.StartAndWaitReady(tempo))
-
-	jaegerClient, err := util.NewJaegerToOTLPExporter(tempo.Endpoint(4317))
-	require.NoError(t, err)
-	require.NotNil(t, jaegerClient)
-
-	// Emit spans where the attribute names but the values are
-	// string and int with the same textual representation.
-	t1 := util.MakeThriftBatch()
-	t1.Spans[0].Tags = append(t1.Spans[0].Tags, &jaeger.Tag{
-		Key:   "foo",
-		VType: jaeger.TagType_STRING,
-		VStr:  strptr("123"),
-	})
-	require.NoError(t, jaegerClient.EmitBatch(ctx, t1))
-
-	t2 := util.MakeThriftBatch()
-	t2.Spans[0].Tags = append(t2.Spans[0].Tags, &jaeger.Tag{
-		Key:   "foo",
-		VType: jaeger.TagType_LONG,
-		VLong: int64ptr(123),
-	})
-	require.NoError(t, jaegerClient.EmitBatch(ctx, t2))
-
-	// Wait for traces to be flushed to blocks
-	require.NoError(t, tempo.WaitSumMetricsWithOptions(e2e.GreaterOrEqual(1), []string{"tempo_metrics_generator_processor_local_blocks_spans_total"}, e2e.WaitMissingMetrics))
-	require.NoError(t, tempo.WaitSumMetricsWithOptions(e2e.GreaterOrEqual(1), []string{"tempo_metrics_generator_processor_local_blocks_cut_blocks"}, e2e.WaitMissingMetrics))
-
-	// Wait for the traces to be written to the WAL
-	time.Sleep(time.Second * 4)
-
-	query := "{span.foo != nil} | rate() by (span.foo)"
-	url := fmt.Sprintf(
-		"http://%s/api/metrics/query_range?q=%s&start=%d&end=%d&step=%s",
-		tempo.Endpoint(3200),
-		url.QueryEscape(query),
-		time.Now().Add(-5*time.Minute).UnixNano(),
-		time.Now().UnixNano(),
-		"5s",
-	)
-
-	req, err := http.NewRequest(http.MethodGet, url, nil)
-	require.NoError(t, err)
-
-	res, err := http.DefaultClient.Do(req)
-	require.NoError(t, err)
-
-	// Read body
-	body, err := io.ReadAll(res.Body)
-	require.NoError(t, err)
-
-	queryRangeRes := &tempopb.QueryRangeResponse{}
-	readBody := strings.NewReader(string(body))
-	err = new(jsonpb.Unmarshaler).Unmarshal(readBody, queryRangeRes)
-	require.NoError(t, err)
-	require.NotNil(t, queryRangeRes)
-
-	// max series is disabled so we should get a complete response with all series
-	require.Equal(t, tempopb.PartialStatus_COMPLETE, queryRangeRes.GetStatus())
-
-	// Should get 2 series:
-	// foo="123"
-	// foo=int(123)
-	require.Equal(t, 2, len(queryRangeRes.GetSeries()))
-}
-
-func callInstantQuery(t *testing.T, endpoint string, req queryRangeRequest) tempopb.QueryInstantResponse {
-	req.SetDefaults()
-	res := doRequest(t, endpoint, "api/metrics/query", req)
-	require.Equal(t, http.StatusOK, res.StatusCode)
-
-	// Read body and print it
-	body, err := io.ReadAll(res.Body)
-	require.NoError(t, err)
-	if debugMode {
-		t.Logf("Response body: %s", string(body))
-	}
-
-	instantQueryRes := tempopb.QueryInstantResponse{}
-	readBody := strings.NewReader(string(body))
-	err = new(jsonpb.Unmarshaler).Unmarshal(readBody, &instantQueryRes)
-	require.NoError(t, err)
-	return instantQueryRes
-}
-
-func callQueryRange(t *testing.T, endpoint string, req queryRangeRequest) tempopb.QueryRangeResponse {
-	req.SetDefaults()
-	res := doRequest(t, endpoint, "api/metrics/query_range", req)
-	require.Equal(t, http.StatusOK, res.StatusCode)
-
-	// Read body and print it
-	body, err := io.ReadAll(res.Body)
-	require.NoError(t, err)
-	if debugMode {
-		t.Logf("Response body: %s", string(body))
-	}
-
-	queryRangeRes := tempopb.QueryRangeResponse{}
-	readBody := strings.NewReader(string(body))
-	err = new(jsonpb.Unmarshaler).Unmarshal(readBody, &queryRangeRes)
-	require.NoError(t, err)
-	return queryRangeRes
-}
-
-func doRequest(t *testing.T, host, endpoint string, req queryRangeRequest) *http.Response {
-	req.Query = fmt.Sprintf("%s with(exemplars=true)", req.Query)
-	url := buildURL(host, endpoint, req)
-	rawReq, err := http.NewRequest(http.MethodGet, url, nil)
-	require.NoError(t, err)
-
-	res, err := http.DefaultClient.Do(rawReq)
-	require.NoError(t, err)
-	return res
-}
-
-func buildURL(host, endpoint string, req queryRangeRequest) string {
-	return fmt.Sprintf(
-		"http://%s/%s?query=%s&start=%d&end=%d&step=%s&exemplars=%d",
-		host, endpoint,
-		url.QueryEscape(req.Query),
-		req.Start.UnixNano(),
-		req.End.UnixNano(),
-		req.Step,
-		req.Exemplars,
-	)
-}
-
-func strptr(s string) *string {
-	return &s
-}
-
-func int64ptr(i int64) *int64 {
-	return &i
-}

integration/e2e/api/tenant_validation_test.go

@@ -1,171 +0,0 @@
-package api
-
-import (
-	"os"
-	"strings"
-	"testing"
-	"time"
-
-	"github.com/grafana/dskit/tenant"
-	"github.com/grafana/e2e"
-	"github.com/grafana/tempo/integration/util"
-	"github.com/grafana/tempo/pkg/httpclient"
-	tempoUtil "github.com/grafana/tempo/pkg/util"
-	"github.com/stretchr/testify/require"
-	"gopkg.in/yaml.v2"
-
-	"github.com/grafana/tempo/cmd/tempo/app"
-	"github.com/grafana/tempo/integration/e2e/backend"
-)
-
-func TestInvalidTenants(t *testing.T) {
-	t.Parallel()
-
-	s, err := e2e.NewScenario(generateNetworkName())
-	require.NoError(t, err)
-	defer s.Close()
-
-	cfg := app.Config{}
-	buff, err := os.ReadFile(configMultiTenant)
-	require.NoError(t, err)
-	err = yaml.UnmarshalStrict(buff, &cfg)
-	require.NoError(t, err)
-	_, err = backend.New(s, cfg)
-	require.NoError(t, err)
-
-	require.NoError(t, util.CopyFileToSharedDir(s, configMultiTenant, "config.yaml"))
-	tempo := util.NewTempoAllInOne()
-	prometheus := util.NewPrometheus()
-	require.NoError(t, s.StartAndWaitReady(tempo, prometheus))
-
-	baseURL := "http://" + tempo.Endpoint(tempoPort)
-
-	longTenant := strings.Repeat("a", tenant.MaxTenantIDLength+1)
-
-	testCases := []struct {
-		name        string
-		orgID       string
-		multitenant bool
-		errContains string
-	}{
-		{
-			name:        "invalid unsupported character",
-			orgID:       "tenant#123",
-			errContains: "unsupported character",
-		},
-		{
-			name:        "invalid slash character",
-			orgID:       "tenant/123",
-			errContains: "unsupported character",
-		},
-		{
-			name:        "invalid unsafe path segment",
-			orgID:       "..",
-			errContains: "tenant ID is '.' or '..'",
-		},
-		{
-			name:        "invalid too long tenant id",
-			orgID:       longTenant,
-			errContains: "tenant ID is too long",
-		},
-		{
-			name:        "invalid empty tenant",
-			orgID:       "",
-			errContains: "no org id",
-		},
-		{
-			name:        "invalid leading separator",
-			orgID:       "|tenantA",
-			multitenant: true,
-			errContains: "no org id",
-		},
-		{
-			name:        "invalid trailing separator",
-			orgID:       "tenantA|",
-			multitenant: true,
-			errContains: "no org id",
-		},
-		{
-			name:        "invalid double separator",
-			orgID:       "tenantA||tenantB",
-			multitenant: true,
-			errContains: "no org id",
-		},
-		{
-			name:        "invalid multi tenant unsupported character",
-			orgID:       "tenantA|tenant#B",
-			multitenant: true,
-			errContains: "tenant#B",
-		},
-		{
-			name:        "invalid multi tenant unsafe segment",
-			orgID:       "tenantA|..",
-			multitenant: true,
-			errContains: "tenant ID is '.' or '..'",
-		},
-		{
-			name:        "invalid multi tenant too long segment",
-			orgID:       "tenantA|" + longTenant,
-			multitenant: true,
-			errContains: "tenant ID is too long",
-		},
-	}
-
-	for _, tc := range testCases {
-		tc := tc
-		t.Run(tc.name, func(t *testing.T) {
-			client := httpclient.New(baseURL, tc.orgID)
-
-			assertHTTP400 := func(t *testing.T, err error) {
-				require.Error(t, err)
-				// no org ID in single-tenant returns 401 from Auth middleware
-				// others return 400
-				require.Contains(t, err.Error(), "response: 40")
-				require.Contains(t, err.Error(), tc.errContains)
-			}
-
-			_, err := client.QueryTrace("0047ac2c027451d89e3f3ba6d2a6b7b")
-			assertHTTP400(t, err)
-
-			_, err = client.QueryTraceV2("0047ac2c027451d89e3f3ba6d2a6b7b")
-			assertHTTP400(t, err)
-
-			_, err = client.SearchTraceQL("{}")
-			assertHTTP400(t, err)
-
-			_, err = client.SearchTags()
-			assertHTTP400(t, err)
-
-			_, err = client.SearchTagsV2()
-			assertHTTP400(t, err)
-
-			_, err = client.SearchTagValues("span.name")
-			assertHTTP400(t, err)
-
-			_, err = client.SearchTagValuesV2("span.name", "{}")
-			assertHTTP400(t, err)
-
-			if !tc.multitenant {
-				_, err := client.MetricsSummary("{}", "name", 0, 0)
-				assertHTTP400(t, err)
-
-				_, err = client.MetricsQueryRange("{} | count_over_time()", 0, 0, "", 0)
-				assertHTTP400(t, err)
-			}
-		})
-
-		if !tc.multitenant {
-			t.Run(tc.name+"_write", func(t *testing.T) {
-				exporter, err := util.NewJaegerToOTLPExporterWithAuth(tempo.Endpoint(4317), tc.orgID, "", false)
-				require.NoError(t, err)
-				require.NotNil(t, exporter)
-
-				writeTraceInfo := tempoUtil.NewTraceInfo(time.Now(), tc.orgID)
-				writeErr := writeTraceInfo.EmitAllBatches(exporter)
-				require.Error(t, writeErr)
-				require.Contains(t, writeErr.Error(), "code = InvalidArgument")
-				require.Contains(t, writeErr.Error(), tc.errContains)
-			})
-		}
-	}
-}

integration/e2e/backend/backend.go

@@ -1,112 +0,0 @@
-package backend
-
-import (
-	"context"
-	"fmt"
-	"strconv"
-	"strings"
-
-	"github.com/grafana/e2e"
-	e2e_db "github.com/grafana/e2e/db"
-	"github.com/grafana/tempo/integration/util"
-
-	"github.com/grafana/tempo/cmd/tempo/app"
-	"github.com/grafana/tempo/tempodb/backend"
-	"github.com/grafana/tempo/tempodb/backend/azure"
-)
-
-const (
-	azuriteImage = "mcr.microsoft.com/azure-storage/azurite:3.35.0"
-	gcsImage     = "fsouza/fake-gcs-server:1.52.2"
-)
-
-func parsePort(endpoint string) (int, error) {
-	substrings := strings.Split(endpoint, ":")
-	portStrings := strings.Split(substrings[len(substrings)-1], "/")
-	port, err := strconv.Atoi(portStrings[0])
-	if err != nil {
-		return 0, err
-	}
-	return port, nil
-}
-
-func New(scenario *e2e.Scenario, cfg app.Config) (*e2e.HTTPService, error) {
-	var backendService *e2e.HTTPService
-	switch cfg.StorageConfig.Trace.Backend {
-	case backend.S3:
-		port, err := parsePort(cfg.StorageConfig.Trace.S3.Endpoint)
-		if err != nil {
-			return nil, err
-		}
-		backendService = e2e_db.NewMinio(port, "tempo")
-		if backendService == nil {
-			return nil, fmt.Errorf("error creating minio backend")
-		}
-		err = scenario.StartAndWaitReady(backendService)
-		if err != nil {
-			return nil, err
-		}
-	case backend.Azure:
-		port, err := parsePort(cfg.StorageConfig.Trace.Azure.Endpoint)
-		if err != nil {
-			return nil, err
-		}
-		backendService = NewAzurite(port)
-		err = scenario.StartAndWaitReady(backendService)
-		if err != nil {
-			return nil, err
-		}
-		cfg.StorageConfig.Trace.Azure.Endpoint = backendService.Endpoint(port)
-		_, err = azure.CreateContainer(context.TODO(), cfg.StorageConfig.Trace.Azure)
-		if err != nil {
-			return nil, err
-		}
-	case backend.GCS:
-		port, err := parsePort(cfg.StorageConfig.Trace.GCS.Endpoint)
-		if err != nil {
-			return nil, err
-		}
-		backendService = NewGCS(port)
-		if backendService == nil {
-			return nil, fmt.Errorf("error creating gcs backend")
-		}
-		err = scenario.StartAndWaitReady(backendService)
-		if err != nil {
-			return nil, err
-		}
-	}
-
-	return backendService, nil
-}
-
-func NewAzurite(port int) *e2e.HTTPService {
-	s := e2e.NewHTTPService(
-		"azurite",
-		azuriteImage, // Create the azurite container
-		e2e.NewCommandWithoutEntrypoint("sh", "-c", "azurite -l /data --blobHost 0.0.0.0"),
-		e2e.NewHTTPReadinessProbe(port, "/devstoreaccount1?comp=list", 403, 403), // If we get 403 the Azurite is ready
-		port, // blob storage port
-	)
-
-	s.SetBackoff(util.TempoBackoff())
-
-	return s
-}
-
-func NewGCS(port int) *e2e.HTTPService {
-	commands := []string{
-		"mkdir -p /data/tempo",
-		"/bin/fake-gcs-server -data /data -public-host=tempo_e2e-gcs -port=4443",
-	}
-	s := e2e.NewHTTPService(
-		"gcs",
-		gcsImage, // Create the the gcs container
-		e2e.NewCommandWithoutEntrypoint("sh", "-c", strings.Join(commands, " && ")),
-		e2e.NewHTTPReadinessProbe(port, "/", 400, 400), // for lack of a better way, readiness probe does not support https at the moment
-		port,
-	)
-
-	s.SetBackoff(util.TempoBackoff())
-
-	return s
-}

integration/e2e/config-encodings.tmpl.yaml

@@ -1,56 +0,0 @@
-target: all
-stream_over_http_enabled: true
-
-server:
-  http_listen_port: 3200
-
-query_frontend:
-  query_end_cutoff: -5m # Include all recent data
-  search:
-    query_backend_after: 5s
-    query_ingesters_until: 5s
-distributor:
-  receivers:
-    jaeger:
-      protocols:
-        grpc:
-          endpoint: "tempo:14250"
-    otlp:
-      protocols:
-        grpc:
-          endpoint: "tempo:4317"
-    zipkin:
-      endpoint: "tempo:9411"
-  log_received_spans:
-    enabled: true
-
-ingester:
-  lifecycler:
-    address: 127.0.0.1
-    ring:
-      kvstore:
-        store: inmemory
-      replication_factor: 1
-    final_sleep: 0s
-    min_ready_duration: 1s
-  trace_idle_period: 1s
-  max_block_duration: 5s
-  complete_block_timeout: 5s
-  flush_check_period: 1s
-
-storage:
-  trace:
-    blocklist_poll: 1s
-    backend: local
-    local:
-      path: /var/tempo
-    pool:
-      max_workers: 10
-      queue_depth: 100
-    block:
-      version: "{{.Version}}"
-      parquet_dedicated_columns:
-        - { scope: resource, name: vulture-process-0, type: string }
-        - { scope: resource, name: vulture-process-1, type: string }
-        - { scope: span, name: vulture-0, type: string }
-        - { scope: span, name: vulture-1, type: string }

integration/e2e/config-limits-429.yaml

@@ -1,48 +0,0 @@
-target: all
-stream_over_http_enabled: true
-
-server:
-  http_listen_port: 3200
-
-distributor:
-  receivers:
-    jaeger:
-      protocols:
-        grpc:
-          endpoint: "tempo:14250"
-    otlp:
-      protocols:
-        grpc:
-          endpoint: "0.0.0.0:4317"
-
-overrides:
-  defaults:
-    ingestion:
-      max_traces_per_user: 1
-      rate_limit_bytes: 500
-      burst_size_bytes: 500
-    global:
-      max_bytes_per_trace: 130
-
-ingester:
-  lifecycler:
-    address: 127.0.0.1
-    ring:
-      kvstore:
-        store: inmemory
-      replication_factor: 1
-    final_sleep: 0s
-    min_ready_duration: 1s
-  trace_idle_period: 3600s
-
-query_frontend:
-  max_outstanding_per_tenant: 0 # forces everything to 429
-
-storage:
-  trace:
-    backend: local
-    local:
-      path: /var/tempo
-    pool:
-      max_workers: 10
-      queue_depth: 100

integration/e2e/config-limits-partial-success.yaml

@@ -1,38 +0,0 @@
-target: all
-
-server:
-  http_listen_port: 3200
-
-distributor:
-  receivers:
-    otlp:
-      protocols:
-        grpc:
-          endpoint: "tempo:4317"
-
-overrides:
-  max_bytes_per_trace: 600
-  max_traces_per_user: 6
-  ingestion_rate_limit_bytes: 7000
-  ingestion_burst_size_bytes: 7000
-
-
-ingester:
-  lifecycler:
-    address: 127.0.0.1
-    ring:
-      kvstore:
-        store: inmemory
-      replication_factor: 1
-    final_sleep: 0s
-    min_ready_duration: 1s
-  trace_idle_period: 3600s
-
-storage:
-  trace:
-    backend: local
-    local:
-      path: /var/tempo
-    pool:
-      max_workers: 10
-      queue_depth: 100

integration/e2e/config-limits-query.yaml

@@ -1,60 +0,0 @@
-target: all
-
-server:
-  http_listen_port: 3200
-
-query_frontend:
-  query_end_cutoff: 0 # Include all recent data
-  search:
-    query_ingesters_until: 30m
-
-distributor:
-  receivers:
-    jaeger:
-      protocols:
-        grpc:
-          endpoint: "tempo:14250"
-    otlp:
-      protocols:
-        grpc:
-          endpoint: "0.0.0.0:4317"
-
-compactor:
-  compaction:
-    block_retention: 1h
-
-overrides:
-  defaults:
-    compaction:
-      block_retention: 1h
-    ingestion:
-      max_traces_per_user: 1
-      rate_limit_bytes: 500
-      burst_size_bytes: 500
-    global:
-      max_bytes_per_trace: 130
-
-ingester:
-  lifecycler:
-    address: 127.0.0.1
-    ring:
-      kvstore:
-        store: inmemory
-      replication_factor: 1
-    final_sleep: 0s
-    min_ready_duration: 1s
-  trace_idle_period: 1ms
-  flush_check_period: 1ms
-  complete_block_timeout: 10s
-
-storage:
-  trace:
-    blocklist_poll: 1s
-    backend: local
-    local:
-      path: /var/tempo/blocks
-    wal:
-      path: /var/tempo/wal
-    pool:
-      max_workers: 10
-      queue_depth: 100

integration/e2e/config-limits.yaml

@@ -1,47 +0,0 @@
-target: all
-
-server:
-  http_listen_port: 3200
-
-distributor:
-  retry_after_on_resource_exhausted: 1s
-  receivers:
-    jaeger:
-      protocols:
-        grpc:
-          endpoint: "tempo:14250"
-    otlp:
-      protocols:
-        grpc:
-          endpoint: "tempo:4317"
-        http:
-          endpoint: "tempo:4318"
-
-overrides:
-  defaults:
-    ingestion:
-      max_traces_per_user: 1
-      rate_limit_bytes: 500
-      burst_size_bytes: 500
-    global:
-      max_bytes_per_trace: 130
-
-ingester:
-  lifecycler:
-    address: 127.0.0.1
-    ring:
-      kvstore:
-        store: inmemory
-      replication_factor: 1
-    final_sleep: 0s
-    min_ready_duration: 1s
-  trace_idle_period: 3600s
-
-storage:
-  trace:
-    backend: local
-    local:
-      path: /var/tempo
-    pool:
-      max_workers: 10
-      queue_depth: 100

integration/e2e/config-metrics-generator-messaging-system.yaml

@@ -1,55 +0,0 @@
-server:
-  http_listen_port: 3200
-
-distributor:
-  receivers:
-    jaeger:
-      protocols:
-        grpc:
-          endpoint: "distributor:14250"
-    otlp:
-      protocols:
-        grpc:
-          endpoint: "0.0.0.0:4317"
-  log_received_spans:
-    enabled: true
-
-ingester:
-  lifecycler:
-    ring:
-      replication_factor: 1
-    min_ready_duration: 1s
-
-metrics_generator:
-  processor:
-    service_graphs:
-      histogram_buckets: [1, 2]  # seconds
-      enable_messaging_system_latency_histogram: true
-    span_metrics:
-      histogram_buckets: [1, 2]
-  registry:
-    collection_interval: 1s
-  storage:
-    path: /var/tempo
-    remote_write:
-      - url: http://tempo_e2e-prometheus:9090/api/v1/write
-        send_exemplars: true
-
-storage:
-  trace:
-    backend: local
-    local:
-      path: /var/tempo
-
-memberlist:
-  bind_port: 7946
-  join_members:
-    - tempo_e2e-distributor:7946
-    - tempo_e2e-ingester-1:7946
-    - tempo_e2e-metrics-generator:7946
-
-overrides:
-  defaults:
-    metrics_generator:
-      processors: [service-graphs, span-metrics]
-      max_active_series: 1000

integration/e2e/config-metrics-generator-targetinfo.yaml

@@ -1,57 +0,0 @@
-server:
-  http_listen_port: 3200
-
-distributor:
-  receivers:
-    jaeger:
-      protocols:
-        grpc:
-          endpoint: "distributor:14250"
-    otlp:
-      protocols:
-        grpc:
-          endpoint: "0.0.0.0:4317"
-  log_received_spans:
-    enabled: true
-
-ingester:
-  lifecycler:
-    ring:
-      replication_factor: 1
-    min_ready_duration: 1s
-
-metrics_generator:
-  processor:
-    service_graphs:
-      histogram_buckets: [1, 2]  # seconds
-    span_metrics:
-      histogram_buckets: [1, 2]
-  registry:
-    collection_interval: 1s
-  storage:
-    path: /var/tempo
-    remote_write:
-      - url: http://tempo_e2e-prometheus:9090/api/v1/write
-        send_exemplars: true
-
-storage:
-  trace:
-    backend: local
-    local:
-      path: /var/tempo
-
-memberlist:
-  bind_port: 7946
-  join_members:
-    - tempo_e2e-distributor:7946
-    - tempo_e2e-ingester-1:7946
-    - tempo_e2e-metrics-generator:7946
-
-overrides:
-  defaults:
-    metrics_generator:
-      processors: [service-graphs, span-metrics]
-      max_active_series: 1000
-      processor:
-        span_metrics:
-          enable_target_info: true  # seconds

integration/e2e/config-metrics-generator.yaml

@@ -1,54 +0,0 @@
-server:
-  http_listen_port: 3200
-
-distributor:
-  receivers:
-    jaeger:
-      protocols:
-        grpc:
-          endpoint: "distributor:14250"
-    otlp:
-      protocols:
-        grpc:
-          endpoint: "0.0.0.0:4317"
-  log_received_spans:
-    enabled: true
-
-ingester:
-  lifecycler:
-    min_ready_duration: 1s
-    ring:
-      replication_factor: 1
-
-metrics_generator:
-  processor:
-    service_graphs:
-      histogram_buckets: [1, 2]  # seconds
-    span_metrics:
-      histogram_buckets: [1, 2]  # seconds
-  registry:
-    collection_interval: 1s
-  storage:
-    path: /var/tempo
-    remote_write:
-      - url: http://tempo_e2e-prometheus:9090/api/v1/write
-        send_exemplars: true
-
-storage:
-  trace:
-    backend: local
-    local:
-      path: /var/tempo
-
-memberlist:
-  bind_port: 7946
-  join_members:
-    - tempo_e2e-distributor:7946
-    - tempo_e2e-ingester-1:7946
-    - tempo_e2e-metrics-generator:7946
-
-overrides:
-  defaults:
-    metrics_generator:
-      processors: [service-graphs, span-metrics]
-      max_active_series: 1000

integration/e2e/deployments/config-all-in-one-azurite.yaml

@@ -1,60 +0,0 @@
-target: all
-stream_over_http_enabled: true
-
-server:
-  http_listen_port: 3200
-
-query_frontend:
-  query_end_cutoff: -1s # Include all recent data
-  search:
-    query_backend_after: 5s
-    query_ingesters_until: 5s
-distributor:
-  receivers:
-    jaeger:
-      protocols:
-        grpc:
-          endpoint: "tempo:14250"
-    otlp:
-      protocols:
-        grpc:
-          endpoint: "0.0.0.0:4317"
-ingester:
-  lifecycler:
-    address: 127.0.0.1
-    ring:
-      kvstore:
-        store: inmemory
-      replication_factor: 1
-    final_sleep: 0s
-    min_ready_duration: 1s
-  trace_idle_period: 1s
-  max_block_duration: 5s
-  complete_block_timeout: 5s
-  flush_check_period: 1s
-
-storage:
-  trace:
-    blocklist_poll: 1s
-    backend: azure
-    azure:
-      container_name: tempo # how to store data in azure
-      endpoint_suffix: tempo_e2e-azurite:10000
-      storage_account_name: "devstoreaccount1"
-      storage_account_key: "Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw=="
-      prefix: {{ .Prefix }}
-    pool:
-      max_workers: 10
-      queue_depth: 100
-
-overrides:
-  user_configurable_overrides:
-    enabled: true
-    poll_interval: 10s
-    client:
-      backend: azure
-      azure:
-        container_name: tempo
-        endpoint_suffix: tempo_e2e-azurite:10000
-        storage_account_name: "devstoreaccount1"
-        storage_account_key: "Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw=="

integration/e2e/deployments/config-all-in-one-gcs.yaml

@@ -1,62 +0,0 @@
-target: all
-stream_over_http_enabled: true
-
-server:
-  http_listen_port: 3200
-
-query_frontend:
-  query_end_cutoff: -1s # Include all recent data
-  search:
-    query_backend_after: 5s
-    query_ingesters_until: 5s
-
-distributor:
-  receivers:
-    jaeger:
-      protocols:
-        grpc:
-          endpoint: "tempo:14250"
-    otlp:
-      protocols:
-        grpc:
-          endpoint: "0.0.0.0:4317"
-
-ingester:
-  lifecycler:
-    address: 127.0.0.1
-    ring:
-      kvstore:
-        store: inmemory
-      replication_factor: 1
-    final_sleep: 0s
-    min_ready_duration: 1s
-  trace_idle_period: 1s
-  max_block_duration: 5s
-  complete_block_timeout: 5s
-  flush_check_period: 1s
-
-storage:
-  trace:
-    blocklist_poll: 1s
-    backend: gcs
-    gcs:
-      bucket_name: tempo
-      endpoint: https://tempo_e2e-gcs:4443/storage/v1/
-      insecure: true
-      prefix: {{ .Prefix }}
-    pool:
-      max_workers: 10
-      queue_depth: 1000
-
-overrides:
-  user_configurable_overrides:
-    enabled: true
-    poll_interval: 10s
-    client:
-      backend: gcs
-      # fsouza/fake-gcs-server does not support versioning
-      confirm_versioning: false
-      gcs:
-        bucket_name: tempo
-        endpoint: https://tempo_e2e-gcs:4443/storage/v1/
-        insecure: true

integration/e2e/deployments/config-all-in-one-local.yaml

@@ -1,69 +0,0 @@
-target: all
-stream_over_http_enabled: true
-
-server:
-  http_listen_port: 3200
-  log_level: info
-
-query_frontend:
-  query_end_cutoff: -5m # Include all recent data
-  mcp_server:
-    enabled: true
-  search:
-    query_backend_after: 5s
-    query_ingesters_until: 5s
-  metrics:
-    max_response_series: 0
-
-distributor:
-  receivers:
-    jaeger:
-      protocols:
-        grpc:
-          endpoint: "tempo:14250"
-    otlp:
-      protocols:
-        grpc:
-          endpoint: "tempo:4317"
-    zipkin:
-      endpoint: "tempo:9411"
-  log_received_spans:
-    enabled: true
-  log_discarded_spans:
-    enabled: true
-
-ingester:
-  lifecycler:
-    address: 127.0.0.1
-    ring:
-      kvstore:
-        store: inmemory
-      replication_factor: 1
-    final_sleep: 0s
-    min_ready_duration: 1s
-  trace_idle_period: 1s
-  max_block_bytes: 1
-  max_block_duration: 2s
-  complete_block_timeout: 5s
-  flush_check_period: 1s
-
-storage:
-  trace:
-    backend: local
-    local:
-      path: /var/tempo/traces
-    pool:
-      max_workers: 10
-      queue_depth: 100
-    block:
-      version: vParquet3
-    blocklist_poll: 1s
-
-overrides:
-  user_configurable_overrides:
-    enabled: true
-    poll_interval: 10s
-    client:
-      backend: local
-      local:
-        path: /var/tempo/overrides

integration/e2e/deployments/config-all-in-one-s3.yaml

@@ -1,64 +0,0 @@
-target: all
-stream_over_http_enabled: true
-
-server:
-  http_listen_port: 3200
-
-query_frontend:
-  query_end_cutoff: -1s # Include all recent data
-  search:
-    query_backend_after: 5s
-    query_ingesters_until: 5s
-
-distributor:
-  receivers:
-    jaeger:
-      protocols:
-        grpc:
-          endpoint: "tempo:14250"
-    otlp:
-      protocols:
-        grpc:
-          endpoint: "0.0.0.0:4317"
-ingester:
-  lifecycler:
-    address: 127.0.0.1
-    ring:
-      kvstore:
-        store: inmemory
-      replication_factor: 1
-    final_sleep: 0s
-    min_ready_duration: 1s
-  trace_idle_period: 1s
-  max_block_duration: 5s
-  complete_block_timeout: 5s
-  flush_check_period: 1s
-
-storage:
-  trace:
-    blocklist_poll: 1s
-    backend: s3
-    s3:
-      bucket: tempo
-      endpoint: tempo_e2e-minio-9000:9000 # TODO: this is brittle, fix this eventually
-      access_key: Cheescake # TODO: use cortex_e2e.MinioAccessKey
-      secret_key: supersecret # TODO: use cortex_e2e.MinioSecretKey
-      insecure: true
-      prefix: {{ .Prefix }}
-    pool:
-      max_workers: 10
-      queue_depth: 100
-
-overrides:
-  user_configurable_overrides:
-    enabled: true
-    poll_interval: 10s
-    client:
-      backend: s3
-      s3:
-        # TODO use separate bucket?
-        bucket: tempo
-        endpoint: tempo_e2e-minio-9000:9000 # TODO: this is brittle, fix this eventually
-        access_key: Cheescake # TODO: use cortex_e2e.MinioAccessKey
-        secret_key: supersecret # TODO: use cortex_e2e.MinioSecretKey
-        insecure: true

integration/e2e/deployments/config-microservices.tmpl.yaml

@@ -1,55 +0,0 @@
-server:
-  http_listen_port: 3200
-query_frontend:
-  query_end_cutoff: -1m # Include all recent data
-distributor:
-  receivers:
-    jaeger:
-      protocols:
-        grpc:
-          endpoint: "distributor:14250"
-    otlp:
-      protocols:
-        grpc:
-          endpoint: "0.0.0.0:4317"
-
-ingester:
-  lifecycler:
-    ring:
-      kvstore: {{ .KVStore }}
-      replication_factor: 3
-    heartbeat_period: 100ms
-    min_ready_duration: 1s
-  trace_idle_period: 2s
-  max_block_duration: 2s
-  complete_block_timeout: 5s
-  flush_check_period: 1s
-
-storage:
-  trace:
-    blocklist_poll: 2s
-    blocklist_poll_stale_tenant_index: 1s # force components to always fall back to polling
-    backend: s3
-    s3:
-      bucket: tempo
-      endpoint: tempo_e2e-minio-9000:9000 # TODO: this is brittle, fix this eventually
-      access_key: Cheescake # TODO: use cortex_e2e.MinioAccessKey
-      secret_key: supersecret # TODO: use cortex_e2e.MinioSecretKey
-      insecure: true
-    pool:
-      max_workers: 10
-      queue_depth: 100
-
-memberlist:
-  abort_if_cluster_join_fails: false
-  bind_port: 7946
-  join_members:
-  - tempo_e2e-distributor:7946
-  - tempo_e2e-ingester-1:7946
-  - tempo_e2e-ingester-2:7946
-  - tempo_e2e-ingester-3:7946
-  - tempo_e2e-querier:7946
-
-querier:
-  frontend_worker:
-    frontend_address: tempo_e2e-query-frontend:9095

integration/e2e/deployments/config-scalable-single-binary.yaml

@@ -1,54 +0,0 @@
-server:
-  http_listen_port: 3200
-
-distributor:
-  receivers:
-    jaeger:
-      protocols:
-        grpc:
-          endpoint: "0.0.0.0:14250"
-    otlp:
-      protocols:
-        grpc:
-          endpoint: "0.0.0.0:4317"
-
-ingester:
-  lifecycler:
-    # address: 127.0.0.1
-    # address: tempo-1
-    ring:
-      kvstore:
-        store: memberlist
-      replication_factor: 3
-    heartbeat_period: 100ms
-    min_ready_duration: 1s
-  max_block_bytes: 1
-  max_block_duration: 2s
-  flush_check_period: 1s
-  trace_idle_period: 1s
-
-storage:
-  trace:
-    blocklist_poll: 2s
-    backend: s3
-    s3:
-      bucket: tempo
-      endpoint: tempo_e2e-minio-9000:9000 # TODO: this is brittle, fix this eventually
-      access_key: Cheescake # TODO: use cortex_e2e.MinioAccessKey
-      secret_key: supersecret # TODO: use cortex_e2e.MinioSecretKey
-      insecure: true
-    pool:
-      max_workers: 10
-      queue_depth: 100
-
-memberlist:
-  abort_if_cluster_join_fails: false
-  bind_port: 7946
-  join_members:
-    - tempo-1:7946
-    - tempo-2:7946
-    - tempo-3:7946
-
-querier:
-  frontend_worker:
-    frontend_address: tempo-1:9095

integration/e2e/deployments/microservices_test.go

@@ -1,192 +0,0 @@
-package deployments
-
-import (
-	"context"
-	"fmt"
-	"testing"
-	"time"
-
-	"github.com/grafana/e2e"
-	e2edb "github.com/grafana/e2e/db"
-	"github.com/grafana/tempo/integration/util"
-	"github.com/grafana/tempo/pkg/httpclient"
-	tempoUtil "github.com/grafana/tempo/pkg/util"
-	"github.com/prometheus/prometheus/model/labels"
-	"github.com/stretchr/testify/require"
-)
-
-const configMicroservices = "config-microservices.tmpl.yaml"
-
-func TestMicroservicesWithKVStores(t *testing.T) {
-	testKVStores := []struct {
-		name     string
-		kvconfig func(hostname string, port int) string
-	}{
-		{
-			name: "memberlist",
-			kvconfig: func(string, int) string {
-				return `
-        store: memberlist`
-			},
-		},
-		{
-			name: "etcd",
-			kvconfig: func(hostname string, port int) string {
-				return fmt.Sprintf(`
-        store: etcd
-        etcd:
-          endpoints:
-            - http://%s:%d`, hostname, port)
-			},
-		},
-		{
-			name: "consul",
-			kvconfig: func(hostname string, port int) string {
-				return fmt.Sprintf(`
-        store: consul
-        consul:
-          host: http://%s:%d`, hostname, port)
-			},
-		},
-	}
-
-	for _, tc := range testKVStores {
-		t.Run(tc.name, func(t *testing.T) {
-			s, err := e2e.NewScenario("tempo_e2e")
-			require.NoError(t, err)
-			defer s.Close()
-
-			// Set up KVStore
-			var kvstore *e2e.HTTPService
-			switch tc.name {
-			case "etcd":
-				kvstore = e2edb.NewETCD()
-				require.NoError(t, s.StartAndWaitReady(kvstore))
-			case "consul":
-				kvstore = e2edb.NewConsul()
-				require.NoError(t, s.StartAndWaitReady(kvstore))
-			case "memberlist":
-			default:
-				t.Errorf("unknown KVStore %s", tc.name)
-			}
-
-			KVStoreConfig := tc.kvconfig("", 0)
-			if kvstore != nil {
-				KVStoreConfig = tc.kvconfig(kvstore.Name(), kvstore.HTTPPort())
-			}
-
-			// copy config template to shared directory and expand template variables
-			tmplConfig := map[string]any{"KVStore": KVStoreConfig}
-			_, err = util.CopyTemplateToSharedDir(s, configMicroservices, "config.yaml", tmplConfig)
-			require.NoError(t, err)
-
-			minio := e2edb.NewMinio(9000, "tempo")
-			require.NotNil(t, minio)
-			require.NoError(t, s.StartAndWaitReady(minio))
-
-			tempoIngester1 := util.NewTempoIngester(1)
-			tempoIngester2 := util.NewTempoIngester(2)
-			tempoIngester3 := util.NewTempoIngester(3)
-
-			tempoDistributor := util.NewTempoDistributor()
-			tempoQueryFrontend := util.NewTempoQueryFrontend()
-			tempoQuerier := util.NewTempoQuerier()
-			require.NoError(t, s.StartAndWaitReady(tempoIngester1, tempoIngester2, tempoIngester3, tempoDistributor, tempoQueryFrontend, tempoQuerier))
-
-			// wait for active ingesters
-			time.Sleep(1 * time.Second)
-			matchers := []*labels.Matcher{
-				{
-					Type:  labels.MatchEqual,
-					Name:  "name",
-					Value: "ingester",
-				},
-				{
-					Type:  labels.MatchEqual,
-					Name:  "state",
-					Value: "ACTIVE",
-				},
-			}
-			require.NoError(t, tempoDistributor.WaitSumMetricsWithOptions(e2e.Equals(3), []string{`tempo_ring_members`}, e2e.WithLabelMatchers(matchers...), e2e.WaitMissingMetrics))
-
-			// Get port for the Jaeger gRPC receiver endpoint
-			c, err := util.NewJaegerToOTLPExporter(tempoDistributor.Endpoint(4317))
-			require.NoError(t, err)
-			require.NotNil(t, c)
-
-			info := tempoUtil.NewTraceInfo(time.Now(), "")
-			require.NoError(t, info.EmitAllBatches(c))
-
-			expected, err := info.ConstructTraceFromEpoch()
-			require.NoError(t, err)
-
-			// test metrics
-			require.NoError(t, tempoDistributor.WaitSumMetrics(e2e.Equals(util.SpanCount(expected)), "tempo_distributor_spans_received_total"))
-
-			// test echo
-			util.AssertEcho(t, "http://"+tempoQueryFrontend.Endpoint(3200)+"/api/echo")
-
-			apiClient := httpclient.New("http://"+tempoQueryFrontend.Endpoint(3200), "")
-
-			// query an in-memory trace
-			util.QueryAndAssertTrace(t, apiClient, info)
-
-			// wait trace_idle_time and ensure trace is created in ingester
-			require.NoError(t, tempoIngester1.WaitSumMetricsWithOptions(e2e.Less(3), []string{"tempo_ingester_traces_created_total"}, e2e.WaitMissingMetrics))
-			require.NoError(t, tempoIngester2.WaitSumMetricsWithOptions(e2e.Less(3), []string{"tempo_ingester_traces_created_total"}, e2e.WaitMissingMetrics))
-			require.NoError(t, tempoIngester3.WaitSumMetricsWithOptions(e2e.Less(3), []string{"tempo_ingester_traces_created_total"}, e2e.WaitMissingMetrics))
-
-			// flush trace to backend
-			util.CallFlush(t, tempoIngester1)
-			util.CallFlush(t, tempoIngester2)
-			util.CallFlush(t, tempoIngester3)
-
-			// search for trace
-			util.SearchAndAssertTrace(t, apiClient, info)
-			util.SearchTraceQLAndAssertTrace(t, apiClient, info)
-
-			// sleep for one maintenance cycle
-			time.Sleep(5 * time.Second)
-
-			// test metrics
-			for _, i := range []*e2e.HTTPService{tempoIngester1, tempoIngester2, tempoIngester3} {
-				require.NoError(t, i.WaitSumMetrics(e2e.Equals(1), "tempo_ingester_blocks_flushed_total"))
-			}
-			require.NoError(t, tempoQuerier.WaitSumMetrics(e2e.Equals(3), "tempodb_blocklist_length"))
-			require.NoError(t, tempoQueryFrontend.WaitSumMetrics(e2e.Equals(4), "tempo_query_frontend_queries_total"))
-
-			// query trace - should fetch from backend
-			util.QueryAndAssertTrace(t, apiClient, info)
-
-			// stop an ingester and confirm we can still write and query
-			err = tempoIngester2.Kill()
-			require.NoError(t, err)
-
-			// sleep for heartbeat timeout
-			time.Sleep(1 * time.Second)
-
-			info = tempoUtil.NewTraceInfo(time.Now(), "")
-			require.NoError(t, info.EmitAllBatches(c))
-
-			// query by id
-			util.QueryAndAssertTrace(t, apiClient, info)
-
-			// wait trace_idle_time and ensure trace is created in ingester
-			require.NoError(t, tempoIngester1.WaitSumMetricsWithOptions(e2e.Less(4), []string{"tempo_ingester_traces_created_total"}, e2e.WaitMissingMetrics))
-			require.NoError(t, tempoIngester3.WaitSumMetricsWithOptions(e2e.Less(4), []string{"tempo_ingester_traces_created_total"}, e2e.WaitMissingMetrics))
-
-			// flush trace to backend
-			util.CallFlush(t, tempoIngester1)
-			util.CallFlush(t, tempoIngester3)
-
-			// search for trace
-			util.SearchAndAssertTrace(t, apiClient, info)
-
-			// stop another ingester and confirm things fail
-			err = tempoIngester1.Kill()
-			require.NoError(t, err)
-
-			require.Error(t, info.EmitBatches(context.Background(), c))
-		})
-	}
-}

integration/e2e/deployments/single_binary_test.go

@@ -1,214 +0,0 @@
-package deployments
-
-import (
-	"context"
-	"fmt"
-	"net/http"
-	"os"
-	"sync"
-	"testing"
-	"time"
-
-	"github.com/grafana/e2e"
-	"github.com/grafana/tempo/cmd/tempo/app"
-	"github.com/grafana/tempo/integration/e2e/backend"
-	"github.com/grafana/tempo/integration/util"
-	"github.com/grafana/tempo/pkg/httpclient"
-	tempoUtil "github.com/grafana/tempo/pkg/util"
-	"github.com/prometheus/prometheus/model/labels"
-	"github.com/stretchr/testify/require"
-	"gopkg.in/yaml.v2"
-)
-
-const (
-	configAllInOneS3      = "config-all-in-one-s3.yaml"
-	configAllInOneAzurite = "config-all-in-one-azurite.yaml"
-	configAllInOneGCS     = "config-all-in-one-gcs.yaml"
-)
-
-func TestAllInOne(t *testing.T) {
-	testBackends := []struct {
-		name       string
-		configFile string
-	}{
-		{
-			name:       "s3",
-			configFile: configAllInOneS3,
-		},
-		{
-			name:       "azure",
-			configFile: configAllInOneAzurite,
-		},
-		{
-			name:       "gcs",
-			configFile: configAllInOneGCS,
-		},
-	}
-
-	storageBackendTestPermutations := []struct {
-		name   string
-		prefix string
-	}{
-		{
-			name: "no-prefix",
-		},
-		{
-			name:   "prefix",
-			prefix: "a/b/c/",
-		},
-	}
-
-	for _, tc := range testBackends {
-		for _, pc := range storageBackendTestPermutations {
-			t.Run(tc.name+"-"+pc.name, func(t *testing.T) {
-				s, err := e2e.NewScenario("tempo_e2e")
-				require.NoError(t, err)
-				defer s.Close()
-
-				// copy config template to shared directory and expand template variables
-				tmplConfig := map[string]any{"Prefix": pc.prefix}
-				configFile, err := util.CopyTemplateToSharedDir(s, tc.configFile, "config.yaml", tmplConfig)
-				require.NoError(t, err)
-
-				// set up the backend
-				cfg := app.Config{}
-				buff, err := os.ReadFile(configFile)
-				require.NoError(t, err)
-				err = yaml.UnmarshalStrict(buff, &cfg)
-				require.NoError(t, err)
-				_, err = backend.New(s, cfg)
-				require.NoError(t, err)
-
-				tempo := util.NewTempoAllInOne()
-				require.NoError(t, s.StartAndWaitReady(tempo))
-
-				// Get port for the Jaeger gRPC receiver endpoint
-				c, err := util.NewJaegerToOTLPExporter(tempo.Endpoint(4317))
-				require.NoError(t, err)
-				require.NotNil(t, c)
-
-				info := tempoUtil.NewTraceInfo(time.Now(), "")
-				require.NoError(t, info.EmitAllBatches(c))
-
-				expected, err := info.ConstructTraceFromEpoch()
-				require.NoError(t, err)
-
-				// test metrics
-				require.NoError(t, tempo.WaitSumMetrics(e2e.Equals(util.SpanCount(expected)), "tempo_distributor_spans_received_total"))
-
-				// test echo
-				// nolint:goconst
-				util.AssertEcho(t, "http://"+tempo.Endpoint(3200)+"/api/echo")
-
-				apiClient := httpclient.New("http://"+tempo.Endpoint(3200), "")
-
-				// query an in-memory trace
-				util.QueryAndAssertTrace(t, apiClient, info)
-
-				// wait trace_idle_time and ensure trace is created in ingester
-				require.NoError(t, tempo.WaitSumMetricsWithOptions(e2e.Less(3), []string{"tempo_ingester_traces_created_total"}, e2e.WaitMissingMetrics))
-
-				// flush trace to backend
-				util.CallFlush(t, tempo)
-
-				// search for trace in backend
-				util.SearchAndAssertTrace(t, apiClient, info)
-				util.SearchTraceQLAndAssertTrace(t, apiClient, info)
-
-				// sleep
-				time.Sleep(10 * time.Second)
-
-				// force clear completed block
-				util.CallFlush(t, tempo)
-
-				fmt.Println(tempo.Endpoint(3200))
-				// test metrics
-				require.NoError(t, tempo.WaitSumMetrics(e2e.Equals(1), "tempo_ingester_blocks_flushed_total"))
-				require.NoError(t, tempo.WaitSumMetricsWithOptions(e2e.Equals(1), []string{"tempodb_blocklist_length"}, e2e.WaitMissingMetrics))
-				require.NoError(t, tempo.WaitSumMetrics(e2e.Equals(4), "tempo_query_frontend_queries_total"))
-
-				matchers := []*labels.Matcher{
-					{
-						Type:  labels.MatchEqual,
-						Name:  "receiver",
-						Value: "otlp/otlp_receiver",
-					},
-					{
-						Type:  labels.MatchEqual,
-						Name:  "transport",
-						Value: "grpc",
-					},
-				}
-
-				require.NoError(t, tempo.WaitSumMetricsWithOptions(e2e.Greater(1), []string{"tempo_receiver_accepted_spans"}, e2e.WithLabelMatchers(matchers...)))
-				require.NoError(t, tempo.WaitSumMetricsWithOptions(e2e.Equals(0), []string{"tempo_receiver_refused_spans"}, e2e.WithLabelMatchers(matchers...)))
-
-				// query trace - should fetch from backend
-				util.QueryAndAssertTrace(t, apiClient, info)
-
-				// search the backend. this works b/c we're passing a start/end AND setting query ingesters within min/max to 0
-				now := time.Now()
-				util.SearchAndAssertTraceBackend(t, apiClient, info, now.Add(-20*time.Minute).Unix(), now.Unix())
-
-				util.SearchAndAsserTagsBackend(t, apiClient, now.Add(-20*time.Minute).Unix(), now.Unix())
-
-				// find the trace with streaming. using the http server b/c that's what Grafana will do
-				grpcClient, err := util.NewSearchGRPCClient(context.Background(), tempo.Endpoint(3200))
-				require.NoError(t, err)
-
-				util.SearchStreamAndAssertTrace(t, context.Background(), grpcClient, info, now.Add(-20*time.Minute).Unix(), now.Unix())
-			})
-		}
-	}
-}
-
-func TestShutdownDelay(t *testing.T) {
-	s, err := e2e.NewScenario("tempo_e2e")
-	require.NoError(t, err)
-	defer s.Close()
-
-	// copy config template to shared directory and expand template variables
-	tmplConfig := map[string]any{"Prefix": ""}
-	configFile, err := util.CopyTemplateToSharedDir(s, configAllInOneS3, "config.yaml", tmplConfig)
-	require.NoError(t, err)
-
-	// set up the backend
-	cfg := app.Config{}
-	buff, err := os.ReadFile(configFile)
-	require.NoError(t, err)
-	err = yaml.UnmarshalStrict(buff, &cfg)
-	require.NoError(t, err)
-	_, err = backend.New(s, cfg)
-	require.NoError(t, err)
-
-	tempo := util.NewTempoAllInOne("-shutdown-delay=5s")
-
-	// this line tests confirms that the readiness flag is up
-	require.NoError(t, s.StartAndWaitReady(tempo))
-
-	// if we're here the readiness flag is up. now call kill and check the readiness flag is down
-	wg := sync.WaitGroup{}
-	wg.Add(1)
-	go func() {
-		defer wg.Done()
-
-		for i := 0; i < 10; i++ {
-			res, err := e2e.DoGet("http://" + tempo.Endpoint(3200) + "/ready")
-			require.NoError(t, err)
-			res.Body.Close()
-
-			if res.StatusCode == http.StatusServiceUnavailable {
-				// found it!
-				return
-			}
-			time.Sleep(time.Second)
-		}
-
-		require.Fail(t, "readiness flag never went down")
-	}()
-
-	// call stop and allow the code above to test for a unavailable readiness flag
-	_ = tempo.Stop()
-
-	wg.Wait()
-}

integration/e2e/deployments/ssb_test.go

@@ -1,124 +0,0 @@
-package deployments
-
-import (
-	"context"
-	"sync"
-	"testing"
-	"time"
-
-	"github.com/grafana/e2e"
-	e2edb "github.com/grafana/e2e/db"
-	"github.com/grafana/tempo/integration/util"
-	"github.com/prometheus/prometheus/model/labels"
-	"github.com/stretchr/testify/require"
-
-	"github.com/grafana/tempo/pkg/httpclient"
-	tempoUtil "github.com/grafana/tempo/pkg/util"
-)
-
-const (
-	configHA = "config-scalable-single-binary.yaml"
-)
-
-func TestScalableSingleBinary(t *testing.T) {
-	s, err := e2e.NewScenario("tempo_e2e")
-	require.NoError(t, err)
-	defer s.Close()
-
-	minio := e2edb.NewMinio(9000, "tempo")
-	require.NotNil(t, minio)
-	require.NoError(t, s.StartAndWaitReady(minio))
-
-	// copy configuration file over to shared dir
-	require.NoError(t, util.CopyFileToSharedDir(s, configHA, "config.yaml"))
-
-	// start three scalable single binary tempos in parallel
-	var wg sync.WaitGroup
-	var tempo1, tempo2, tempo3 *e2e.HTTPService
-	wg.Add(3)
-	go func() {
-		tempo1 = util.NewTempoScalableSingleBinary(1)
-		wg.Done()
-	}()
-	go func() {
-		tempo2 = util.NewTempoScalableSingleBinary(2)
-		wg.Done()
-	}()
-	go func() {
-		tempo3 = util.NewTempoScalableSingleBinary(3)
-		wg.Done()
-	}()
-	wg.Wait()
-	require.NoError(t, s.StartAndWaitReady(tempo1, tempo2, tempo3))
-
-	// wait for 2 active ingesters
-	time.Sleep(1 * time.Second)
-	matchers := []*labels.Matcher{
-		{
-			Type:  labels.MatchEqual,
-			Name:  "name",
-			Value: "ingester",
-		},
-		{
-			Type:  labels.MatchEqual,
-			Name:  "state",
-			Value: "ACTIVE",
-		},
-	}
-
-	t.Logf("tempo1.Endpoint(): %+v", tempo1.Endpoint(3200))
-
-	require.NoError(t, tempo1.WaitSumMetricsWithOptions(e2e.Equals(3), []string{`tempo_ring_members`}, e2e.WithLabelMatchers(matchers...), e2e.WaitMissingMetrics))
-	require.NoError(t, tempo2.WaitSumMetricsWithOptions(e2e.Equals(3), []string{`tempo_ring_members`}, e2e.WithLabelMatchers(matchers...), e2e.WaitMissingMetrics))
-	require.NoError(t, tempo3.WaitSumMetricsWithOptions(e2e.Equals(3), []string{`tempo_ring_members`}, e2e.WithLabelMatchers(matchers...), e2e.WaitMissingMetrics))
-
-	c1, err := util.NewJaegerToOTLPExporter(tempo1.Endpoint(4317))
-	require.NoError(t, err)
-	require.NotNil(t, c1)
-
-	c2, err := util.NewJaegerToOTLPExporter(tempo2.Endpoint(4317))
-	require.NoError(t, err)
-	require.NotNil(t, c2)
-
-	c3, err := util.NewJaegerToOTLPExporter(tempo3.Endpoint(4317))
-	require.NoError(t, err)
-	require.NotNil(t, c3)
-
-	info := tempoUtil.NewTraceInfo(time.Unix(1632169410, 0), "")
-	require.NoError(t, info.EmitBatches(context.Background(), c1))
-
-	expected, err := info.ConstructTraceFromEpoch()
-	require.NoError(t, err)
-
-	// test metrics
-	require.NoError(t, tempo1.WaitSumMetrics(e2e.Equals(util.SpanCount(expected)), "tempo_distributor_spans_received_total"))
-
-	// wait trace_idle_time and ensure trace is created in ingester
-	time.Sleep(1 * time.Second)
-	require.NoError(t, tempo1.WaitSumMetricsWithOptions(e2e.Less(3), []string{"tempo_ingester_traces_created_total"}, e2e.WaitMissingMetrics))
-
-	for _, i := range []*e2e.HTTPService{tempo1, tempo2, tempo3} {
-		util.CallFlush(t, i)
-		require.NoError(t, i.WaitSumMetrics(e2e.Equals(1), "tempo_ingester_blocks_flushed_total"))
-		util.CallIngesterRing(t, i)
-		util.CallCompactorRing(t, i)
-		util.CallStatus(t, i)
-		util.CallBuildinfo(t, i)
-	}
-
-	apiClient1 := httpclient.New("http://"+tempo1.Endpoint(3200), "")
-
-	util.QueryAndAssertTrace(t, apiClient1, info)
-
-	err = tempo1.Kill()
-	require.NoError(t, err)
-
-	// Push to one of the instances that are still running.
-	require.NoError(t, info.EmitBatches(context.Background(), c2))
-
-	err = tempo2.Kill()
-	require.NoError(t, err)
-
-	err = tempo3.Kill()
-	require.NoError(t, err)
-}

integration/e2e/encodings_test.go

@@ -1,129 +0,0 @@
-package e2e
-
-import (
-	"context"
-	"os"
-	"testing"
-	"time"
-
-	util2 "github.com/grafana/tempo/integration/util"
-	v2 "github.com/grafana/tempo/tempodb/encoding/v2"
-
-	"github.com/grafana/e2e"
-	"github.com/stretchr/testify/require"
-	"gopkg.in/yaml.v2"
-
-	"github.com/grafana/tempo/cmd/tempo/app"
-	"github.com/grafana/tempo/integration/e2e/backend"
-	"github.com/grafana/tempo/integration/util"
-	"github.com/grafana/tempo/pkg/httpclient"
-	tempoUtil "github.com/grafana/tempo/pkg/util"
-	"github.com/grafana/tempo/tempodb/encoding"
-)
-
-const (
-	configAllEncodings = "./config-encodings.tmpl.yaml"
-)
-
-func TestEncodings(t *testing.T) {
-	const repeatedSearchCount = 10
-
-	for _, enc := range encoding.AllEncodingsForWrites() {
-		t.Run(enc.Version(), func(t *testing.T) {
-			s, err := e2e.NewScenario("tempo_e2e")
-			require.NoError(t, err)
-			defer s.Close()
-
-			// copy config template to shared directory and expand template variables
-			tmplConfig := map[string]any{"Version": enc.Version()}
-			config, err := util2.CopyTemplateToSharedDir(s, configAllEncodings, "config.yaml", tmplConfig)
-			require.NoError(t, err)
-
-			// load final config
-			var cfg app.Config
-			buff, err := os.ReadFile(config)
-			require.NoError(t, err)
-			err = yaml.UnmarshalStrict(buff, &cfg)
-			require.NoError(t, err)
-
-			// set up the backend
-			_, err = backend.New(s, cfg)
-			require.NoError(t, err)
-
-			tempo := util2.NewTempoAllInOne()
-			require.NoError(t, s.StartAndWaitReady(tempo))
-
-			// Get port for the Jaeger gRPC receiver endpoint
-			c, err := util2.NewJaegerToOTLPExporter(tempo.Endpoint(4317))
-			require.NoError(t, err)
-			require.NotNil(t, c)
-
-			info := tempoUtil.NewTraceInfo(time.Now(), "")
-			require.NoError(t, info.EmitAllBatches(c))
-
-			expected, err := info.ConstructTraceFromEpoch()
-			require.NoError(t, err)
-
-			// test metrics
-			require.NoError(t, tempo.WaitSumMetrics(e2e.Equals(util.SpanCount(expected)), "tempo_distributor_spans_received_total"))
-
-			// test echo
-			util.AssertEcho(t, "http://"+tempo.Endpoint(3200)+"/api/echo")
-
-			apiClient := httpclient.New("http://"+tempo.Endpoint(3200), "")
-
-			// query an in-memory trace
-			util.QueryAndAssertTrace(t, apiClient, info)
-
-			// wait trace_idle_time and ensure trace is created in ingester
-			require.NoError(t, tempo.WaitSumMetricsWithOptions(e2e.Less(3), []string{"tempo_ingester_traces_created_total"}, e2e.WaitMissingMetrics))
-
-			// flush trace to backend
-			util.CallFlush(t, tempo)
-
-			// v2 does not support querying and must be skipped
-			if enc.Version() != v2.VersionString {
-				// search for trace in backend multiple times with different attributes to make sure
-				// we search with different scopes and with attributes from dedicated columns
-				for range repeatedSearchCount {
-					util2.SearchAndAssertTrace(t, apiClient, info)
-					util2.SearchTraceQLAndAssertTrace(t, apiClient, info)
-				}
-			}
-
-			// sleep
-			time.Sleep(10 * time.Second)
-
-			// force clear completed block
-			util.CallFlush(t, tempo)
-
-			// test metrics
-			require.NoError(t, tempo.WaitSumMetrics(e2e.Equals(1), "tempo_ingester_blocks_flushed_total"))
-			require.NoError(t, tempo.WaitSumMetricsWithOptions(e2e.Equals(1), []string{"tempodb_blocklist_length"}, e2e.WaitMissingMetrics))
-			if enc.Version() != v2.VersionString {
-				require.NoError(t, tempo.WaitSumMetrics(e2e.Greater(15), "tempo_query_frontend_queries_total"))
-			}
-
-			// query trace - should fetch from backend
-			util.QueryAndAssertTrace(t, apiClient, info)
-
-			// create grpc client used for streaming
-			grpcClient, err := util2.NewSearchGRPCClient(context.Background(), tempo.Endpoint(3200))
-			require.NoError(t, err)
-
-			if enc.Version() == v2.VersionString {
-				return // v2 does not support querying and must be skipped
-			}
-
-			// search for trace in backend multiple times with different attributes to make sure
-			// we search with different scopes and with attributes from dedicated columns
-			now := time.Now()
-			for range repeatedSearchCount {
-				// search the backend. this works b/c we're passing a start/end AND setting query ingesters within min/max to 0
-				util2.SearchAndAssertTraceBackend(t, apiClient, info, now.Add(-20*time.Minute).Unix(), now.Unix())
-				// find the trace with streaming. using the http server b/c that's what Grafana will do
-				util2.SearchStreamAndAssertTrace(t, context.Background(), grpcClient, info, now.Add(-20*time.Minute).Unix(), now.Unix())
-			}
-		})
-	}
-}

integration/e2e/ingest/config-kafka.yaml

@@ -1,83 +0,0 @@
-target: all-3.0
-stream_over_http_enabled: true
-partition_ring_live_store: true
-
-server:
-  http_listen_port: 3200
-  log_level: info
-
-querier:
-  query_live_store: true
-
-query_frontend:
-  rf1_after: "2025-01-01T00:00:00Z"
-  query_end_cutoff: 0 # Include all recent data
-  search:
-    query_backend_after: 30s
-    query_ingesters_until: 30s
-
-distributor:
-  ingester_write_path_enabled: false
-  kafka_write_path_enabled: true
-  receivers:
-    jaeger:
-      protocols:
-        grpc:
-          endpoint: "tempo:14250"
-    otlp:
-      protocols:
-        grpc:
-          endpoint: "0.0.0.0:4317"
-  log_received_spans:
-    enabled: true
-  log_discarded_spans:
-    enabled: true
-
-ingester:
-  lifecycler:
-    address: 127.0.0.1
-    ring:
-      kvstore:
-        store: inmemory
-      replication_factor: 1
-    final_sleep: 0s
-    min_ready_duration: 1s
-  trace_idle_period: 1s
-  max_block_bytes: 1
-  max_block_duration: 2s
-  complete_block_timeout: 20s
-  flush_check_period: 1s
-  partition_ring:
-    kvstore:
-      store: inmemory
-  flush_object_storage: false
-
-storage:
-  trace:
-    backend: local
-    local:
-      path: /var/tempo
-    pool:
-      max_workers: 10
-      queue_depth: 100
-    block:
-      version: vParquet3
-    blocklist_poll: 1s
-
-overrides:
-  user_configurable_overrides:
-    enabled: true
-    poll_interval: 10s
-    client:
-      backend: local
-      local:
-        path: /var/tempo/overrides
-
-block_builder:
-  consume_cycle_duration: 1m
-
-ingest:
-  enabled: true
-  kafka:
-    address: kafka:9092
-    topic: tempo-ingest

integration/e2e/ingest/config-live-store.yaml

@@ -1,71 +0,0 @@
-stream_over_http_enabled: true
-partition_ring_live_store: true
-
-server:
-  http_listen_port: 3200
-  log_level: info
-
-query_frontend:
-  rf1_after: "2025-01-01T00:00:00Z"
-  query_end_cutoff: -5m # Include all recent data
-
-distributor:
-  ingester_write_path_enabled: false
-  kafka_write_path_enabled: true
-  receivers:
-    otlp:
-      protocols:
-        grpc:
-          endpoint: "0.0.0.0:4317"
-  log_received_spans:
-    enabled: true
-  log_discarded_spans:
-    enabled: true
-
-live_store:
-  partition_ring:
-    kvstore:
-      store: memberlist
-    delete_inactive_partition_after: 1s
-
-storage:
-  trace:
-    backend: local
-    local:
-      path: /var/tempo
-    pool:
-      max_workers: 10
-      queue_depth: 100
-    block:
-      version: vParquet3
-    blocklist_poll: 1s
-
-memberlist:
-  abort_if_cluster_join_fails: false
-  bind_port: 7946
-  join_members:
-    - live-store-0:7946
-    - live-store-1:7946
-
-overrides:
-  user_configurable_overrides:
-    enabled: true
-    poll_interval: 10s
-    client:
-      backend: local
-      local:
-        path: /var/tempo/overrides
-
-block_builder:
-  consume_cycle_duration: 1m
-
-ingest:
-  enabled: true
-  kafka:
-    address: kafka:9092
-    topic: tempo-ingest
-
-querier:
-  frontend_worker:
-    frontend_address: query-frontend:9095
-  query_live_store: true

integration/e2e/ingest/config-partition-downscale.yaml

@@ -1,94 +0,0 @@
-server:
-  http_listen_port: 3200
-  log_level: info
-
-distributor:
-  receivers:
-    otlp:
-      protocols:
-        grpc:
-          endpoint: "distributor:4317"
-
-query_frontend:
-  rf1_after: "1970-01-01T00:00:00Z"
-  query_end_cutoff: 0 # Include all recent data
-
-ingester:
-  lifecycler:
-    min_ready_duration: 1s
-    ring:
-      kvstore:
-        store: memberlist
-      replication_factor: 1
-  partition_ring:
-    kvstore:
-      store: memberlist
-  flush_object_storage: false
-
-querier:
-  frontend_worker:
-    frontend_address: tempo_e2e-query-frontend:9095
-
-metrics_generator:
-  registry:
-    external_labels:
-      source: tempo
-      cluster: docker-compose
-  storage:
-    path: /var/tempo/generator/wal
-    remote_write:
-      - url: http://prometheus:9090/api/v1/write
-        send_exemplars: true
-  traces_storage:
-    path: /var/tempo/generator/traces
-  traces_query_storage:
-    path: /var/tempo/generator/query_traces
-  processor:
-    local_blocks:
-      flush_to_storage: true
-
-memberlist:
-  bind_port: 7946
-  join_members:
-    - tempo_e2e-distributor:7946
-    - tempo_e2e-ingester-1:7946
-    - tempo_e2e-ingester-2:7946
-    - tempo_e2e-metrics-generator:7946
-
-storage:
-  trace:
-    blocklist_poll: 2s
-    blocklist_poll_stale_tenant_index: 1s # force components to always fall back to polling
-    backend: s3
-    s3:
-      bucket: tempo
-      endpoint: tempo_e2e-minio-9000:9000 # TODO: this is brittle, fix this eventually
-      access_key: Cheescake # TODO: use cortex_e2e.MinioAccessKey
-      secret_key: supersecret # TODO: use cortex_e2e.MinioSecretKey
-      insecure: true
-    pool:
-      max_workers: 10
-      queue_depth: 100
-
-overrides:
-  defaults:
-    metrics_generator:
-      processors: [local-blocks]
-  user_configurable_overrides:
-    enabled: true
-    poll_interval: 10s
-    client:
-      backend: local
-      local:
-        path: /var/tempo/overrides
-
-block_builder:
-  consume_cycle_duration: 1m
-  assigned_partitions:
-    block-builder-0: [0]
-
-ingest:
-  enabled: true
-  kafka:
-    address: kafka:9092
-    topic: tempo-ingest

integration/e2e/ingest/ingest_storage_test.go

@@ -1,70 +0,0 @@
-package ingest
-
-import (
-	"net/http"
-	"testing"
-	"time"
-
-	"github.com/grafana/e2e"
-	e2edb "github.com/grafana/e2e/db"
-	"github.com/grafana/tempo/integration/util"
-	"github.com/grafana/tempo/pkg/httpclient"
-	tempoUtil "github.com/grafana/tempo/pkg/util"
-	"github.com/prometheus/prometheus/model/labels"
-	"github.com/stretchr/testify/require"
-)
-
-func TestIngest(t *testing.T) {
-	s, err := e2e.NewScenario("tempo_e2e")
-	require.NoError(t, err)
-	defer s.Close()
-
-	// copy config template to shared directory and expand template variables
-	require.NoError(t, util.CopyFileToSharedDir(s, "config-kafka.yaml", "config.yaml"))
-
-	kafka := e2edb.NewKafka()
-	require.NoError(t, s.StartAndWaitReady(kafka))
-
-	tempo := util.NewTempoAllInOne()
-	require.NoError(t, s.StartAndWaitReady(tempo))
-
-	// Wait until joined to partition ring
-	matchers := []*labels.Matcher{
-		{Type: labels.MatchEqual, Name: "state", Value: "Active"},
-		{Type: labels.MatchEqual, Name: "name", Value: "livestore-partitions"},
-	}
-	require.NoError(t, tempo.WaitSumMetricsWithOptions(e2e.Equals(1), []string{"tempo_partition_ring_partitions"}, e2e.WithLabelMatchers(matchers...)))
-
-	// Get port for the Jaeger gRPC receiver endpoint
-	c, err := util.NewJaegerToOTLPExporter(tempo.Endpoint(4317))
-	require.NoError(t, err)
-	require.NotNil(t, c)
-
-	info := tempoUtil.NewTraceInfo(time.Now(), "")
-	require.NoError(t, info.EmitAllBatches(c))
-
-	expected, err := info.ConstructTraceFromEpoch()
-	require.NoError(t, err)
-
-	// test metrics
-	require.NoError(t, tempo.WaitSumMetrics(e2e.Equals(util.SpanCount(expected)), "tempo_distributor_spans_received_total"))
-
-	// test echo
-	util.AssertEcho(t, "http://"+tempo.Endpoint(3200)+"/api/echo")
-
-	apiClient := httpclient.New("http://"+tempo.Endpoint(3200), "")
-
-	// wait until the block-builder block is flushed
-	require.NoError(t, tempo.WaitSumMetricsWithOptions(e2e.Equals(1), []string{"tempo_block_builder_flushed_blocks"}, e2e.WaitMissingMetrics))
-	require.NoError(t, tempo.WaitSumMetricsWithOptions(e2e.GreaterOrEqual(1), []string{"tempodb_blocklist_length"},
-		e2e.WaitMissingMetrics, e2e.WithLabelMatchers(&labels.Matcher{Type: labels.MatchEqual, Name: "tenant", Value: "single-tenant"})))
-
-	// search the backend. this works b/c we're passing a start/end AND setting query ingesters within min/max to 0
-	now := time.Now()
-	util.SearchAndAssertTraceBackend(t, apiClient, info, now.Add(-20*time.Minute).Unix(), now.Unix())
-
-	// Call /metrics
-	res, err := e2e.DoGet("http://" + tempo.Endpoint(3200) + "/metrics")
-	require.NoError(t, err)
-	require.Equal(t, http.StatusOK, res.StatusCode)
-}

integration/e2e/ingest/live_store_test.go

@@ -1,241 +0,0 @@
-package ingest
-
-import (
-	"testing"
-	"time"
-
-	"github.com/grafana/e2e"
-	e2edb "github.com/grafana/e2e/db"
-	"github.com/grafana/tempo/integration/util"
-	"github.com/grafana/tempo/pkg/httpclient"
-	tempoUtil "github.com/grafana/tempo/pkg/util"
-	"github.com/prometheus/prometheus/model/labels"
-	"github.com/stretchr/testify/assert"
-	"github.com/stretchr/testify/require"
-)
-
-func TestLiveStore(t *testing.T) {
-	s, err := e2e.NewScenario("tempo_e2e")
-	require.NoError(t, err)
-	defer s.Close()
-
-	// copy config template to shared directory and expand template variables
-	require.NoError(t, util.CopyFileToSharedDir(s, "config-live-store.yaml", "config.yaml"))
-
-	kafka := e2edb.NewKafka()
-	require.NoError(t, s.StartAndWaitReady(kafka))
-
-	liveStore0 := util.NewTempoLiveStore(0)
-	liveStore1 := util.NewTempoLiveStore(1)
-	require.NoError(t, s.StartAndWaitReady(liveStore0, liveStore1))
-	waitUntilJoinedToPartitionRing(t, liveStore0, 2)
-
-	distributor := util.NewTempoDistributor()
-	require.NoError(t, s.StartAndWaitReady(distributor))
-
-	// Get port for the Jaeger gRPC receiver endpoint
-	c, err := util.NewJaegerToOTLPExporter(distributor.Endpoint(4317))
-	require.NoError(t, err)
-	require.NotNil(t, c)
-
-	info := tempoUtil.NewTraceInfo(time.Now(), "")
-	require.NoError(t, info.EmitAllBatches(c))
-
-	expected, err := info.ConstructTraceFromEpoch()
-	require.NoError(t, err)
-
-	// test metrics
-	require.NoError(t, distributor.WaitSumMetrics(e2e.Equals(util.SpanCount(expected)), "tempo_distributor_spans_received_total"))
-
-	liveStoreProcessedRecords := waitForTraceInLiveStore(t, 1, liveStore0, liveStore1)
-	// the number of processed records should be reasonable
-	assert.NoError(t, liveStoreProcessedRecords.WaitSumMetrics(e2e.Between(1, 25), "tempo_live_store_records_processed_total"))
-}
-
-// TestLiveStoreAPISmoke tests the API endpoints that will hit live store.
-// It will be deleted after api tests will start using Rhythm.
-func TestSmokeLiveStoreAPI(t *testing.T) {
-	s, err := e2e.NewScenario("tempo_e2e")
-	require.NoError(t, err)
-	defer s.Close()
-
-	// copy config template to shared directory and expand template variables
-	require.NoError(t, util.CopyFileToSharedDir(s, "config-live-store.yaml", "config.yaml"))
-
-	frontend, distributor := StartTempoWithLiveStore(t, s)
-	client := httpclient.New("http://"+frontend.Endpoint(3200), "")
-
-	// Get port for the Jaeger gRPC receiver endpoint
-	c, err := util.NewJaegerToOTLPExporter(distributor.Endpoint(4317))
-	require.NoError(t, err)
-	require.NotNil(t, c)
-
-	info := tempoUtil.NewTraceInfo(time.Now(), "")
-	require.NoError(t, info.EmitAllBatches(c))
-	time.Sleep(10 * time.Second) // wait for the trace to be ingested
-
-	t.Run("get trace by id", func(t *testing.T) {
-		tr, err := client.QueryTrace(info.HexID())
-		require.NoError(t, err)
-		require.NotNil(t, tr)
-		require.Greater(t, int(util.SpanCount(tr)), 0)
-	})
-
-	t.Run("get trace by id v2", func(t *testing.T) {
-		resp, err := client.QueryTraceV2(info.HexID())
-		require.NoError(t, err)
-		require.NotNil(t, resp)
-		require.NotNil(t, resp.Trace)
-		require.Greater(t, int(util.SpanCount(resp.Trace)), 0)
-	})
-
-	t.Run("search tags v1", func(t *testing.T) {
-		resp, err := client.SearchTags()
-		require.NoError(t, err)
-		require.NotNil(t, resp)
-		require.Greater(t, len(resp.TagNames), 0)
-	})
-
-	t.Run("search tag values v1", func(t *testing.T) {
-		resp, err := client.SearchTagValues("service.name")
-		require.NoError(t, err)
-		require.NotNil(t, resp)
-		require.Greater(t, len(resp.TagValues), 0)
-	})
-
-	t.Run("search tags v2", func(t *testing.T) {
-		resp, err := client.SearchTagsV2()
-		require.NoError(t, err)
-		require.NotNil(t, resp)
-		total := 0
-		for _, sc := range resp.Scopes {
-			total += len(sc.Tags)
-		}
-		require.Greater(t, total, 0)
-	})
-
-	t.Run("search tag values v2", func(t *testing.T) {
-		resp, err := client.SearchTagValuesV2("resource.service.name", "")
-		require.NoError(t, err)
-		require.NotNil(t, resp)
-		require.Greater(t, len(resp.TagValues), 0)
-	})
-
-	t.Run("metrics query range count_over_time", func(t *testing.T) {
-		qr, err := client.MetricsQueryRange("{} | count_over_time()", 0, 0, "", 0)
-		require.NoError(t, err)
-		require.NotNil(t, qr)
-		require.Equal(t, 1, len(qr.Series))
-		var count int
-		for _, s := range qr.Series[0].Samples {
-			count += int(s.Value)
-		}
-		require.Greater(t, count, 0)
-	})
-}
-
-func TestLiveStoreLookback(t *testing.T) {
-	for _, testCase := range []struct {
-		name              string
-		argument          string // if test becomes unstable, increase sleeps and argument value
-		startNewLiveStore bool
-		expectedTraces    float64
-	}{
-		{
-			name:              "restart_2s",
-			argument:          "-live-store.complete-block-timeout=1s", // lookback period twice greater
-			startNewLiveStore: false,
-			expectedTraces:    2, // fresh and after start
-		},
-		{
-			name:              "restart_default",
-			argument:          "", // default is 1h
-			startNewLiveStore: false,
-			expectedTraces:    3, // old, fresh and after start, but not already committed
-		},
-		{
-			name:              "start_2s",
-			argument:          "-live-store.complete-block-timeout=1s", // lookback period twice greater
-			startNewLiveStore: true,
-			expectedTraces:    2, // fresh and after start
-		},
-		{
-			name:              "start_default",
-			argument:          "", // default is 1h
-			startNewLiveStore: true,
-			expectedTraces:    4, // all traces
-		},
-	} {
-		t.Run(testCase.name, func(t *testing.T) {
-			s, err := e2e.NewScenario("tempo_e2e_TestLiveStoreLookback_" + testCase.name)
-			t.Parallel()
-			require.NoError(t, err)
-			defer s.Close()
-
-			// copy config template to shared directory and expand template variables
-			require.NoError(t, util.CopyFileToSharedDir(s, "config-live-store.yaml", "config.yaml"))
-
-			kafka := e2edb.NewKafka()
-			require.NoError(t, s.StartAndWaitReady(kafka))
-
-			liveStore := util.NewTempoLiveStore(0, testCase.argument)
-			require.NoError(t, s.StartAndWaitReady(liveStore))
-			waitUntilJoinedToPartitionRing(t, liveStore, 1)
-
-			distributor := util.NewTempoDistributor()
-			require.NoError(t, s.StartAndWaitReady(distributor))
-
-			// Get port for the Jaeger gRPC receiver endpoint
-			c, err := util.NewJaegerToOTLPExporter(distributor.Endpoint(4317))
-			require.NoError(t, err)
-			require.NotNil(t, c)
-
-			info := tempoUtil.NewTraceInfo(time.Now(), "")
-			require.NoError(t, info.EmitAllBatches(c)) // committed by the first live store before shutdown
-
-			expected, err := info.ConstructTraceFromEpoch()
-			require.NoError(t, err)
-
-			// should process the trace
-			require.NoError(t, distributor.WaitSumMetrics(e2e.Equals(util.SpanCount(expected)), "tempo_distributor_spans_received_total"))
-			require.NoError(t, liveStore.WaitSumMetrics(e2e.Equals(1), "tempo_live_store_traces_created_total"))
-
-			require.NoError(t, s.Stop(liveStore))
-			require.NoError(t, tempoUtil.NewTraceInfo(time.Now(), "").EmitAllBatches(c)) // old trace
-			time.Sleep(3 * time.Second)
-			require.NoError(t, tempoUtil.NewTraceInfo(time.Now(), "").EmitAllBatches(c)) // fresh trace
-
-			if testCase.startNewLiveStore { // new live store without committed offset
-				liveStore = util.NewNamedTempoLiveStore("live-store-zone-b", 0, testCase.argument)
-			}
-			require.NoError(t, s.StartAndWaitReady(liveStore))
-
-			require.NoError(t, tempoUtil.NewTraceInfo(time.Now(), "").EmitAllBatches(c)) // after start
-			require.NoError(t, liveStore.WaitSumMetrics(e2e.Equals(testCase.expectedTraces), "tempo_live_store_traces_created_total"))
-		})
-	}
-}
-
-func waitUntilJoinedToPartitionRing(t *testing.T, liveStore *e2e.HTTPService, numPartitions float64) {
-	matchers := []*labels.Matcher{
-		{Type: labels.MatchEqual, Name: "state", Value: "Active"},
-		{Type: labels.MatchEqual, Name: "name", Value: "livestore-partitions"},
-	}
-	require.NoError(t, liveStore.WaitSumMetricsWithOptions(e2e.Equals(numPartitions), []string{"tempo_partition_ring_partitions"}, e2e.WithLabelMatchers(matchers...)))
-}
-
-func StartTempoWithLiveStore(t *testing.T, s *e2e.Scenario) (*e2e.HTTPService, *e2e.HTTPService) {
-	kafka := e2edb.NewKafka()
-	require.NoError(t, s.StartAndWaitReady(kafka))
-
-	liveStore1 := util.NewTempoLiveStore(0)
-	liveStore2 := util.NewTempoLiveStore(1)
-	require.NoError(t, s.StartAndWaitReady(liveStore1, liveStore2))
-	waitUntilJoinedToPartitionRing(t, liveStore1, 2)
-
-	distributor := util.NewTempoDistributor()
-	frontend := util.NewTempoQueryFrontend()
-	require.NoError(t, s.StartAndWaitReady(distributor, frontend, util.NewTempoQuerier()))
-
-	return frontend, distributor
-}

integration/e2e/ingest/partition_downscale_test.go

@@ -1,537 +0,0 @@
-package ingest
-
-import (
-	"encoding/json"
-	"fmt"
-	"net/http"
-	"testing"
-	"time"
-
-	"github.com/grafana/e2e"
-	e2edb "github.com/grafana/e2e/db"
-	"github.com/grafana/tempo/integration/util"
-	"github.com/grafana/tempo/pkg/httpclient"
-	tempoUtil "github.com/grafana/tempo/pkg/util"
-	"github.com/prometheus/prometheus/model/labels"
-	"github.com/stretchr/testify/require"
-)
-
-func TestPartitionDownscale(t *testing.T) {
-	s, err := e2e.NewScenario("tempo_e2e")
-	require.NoError(t, err)
-	defer s.Close()
-
-	// copy config template to shared directory and expand template variables
-	require.NoError(t, util.CopyFileToSharedDir(s, "config-partition-downscale.yaml", "config.yaml"))
-
-	// Start dependencies
-	kafka := e2edb.NewKafka()
-	require.NoError(t, s.StartAndWaitReady(kafka))
-
-	minio := e2edb.NewMinio(9000, "tempo")
-	require.NotNil(t, minio)
-	require.NoError(t, s.StartAndWaitReady(minio))
-
-	// Start Tempo components
-	distributor := util.NewTempoDistributor()
-	ingester := util.NewTempoIngester(0)
-	querier := util.NewTempoQuerier()
-	queryFrontend := util.NewTempoQueryFrontend()
-
-	require.NoError(t, s.StartAndWaitReady(distributor, ingester, querier, queryFrontend))
-
-	// Wait until ingester and metrics-generator are active
-	isServiceActiveMatcher := func(service string) []*labels.Matcher {
-		return []*labels.Matcher{
-			labels.MustNewMatcher(labels.MatchEqual, "name", service),
-			labels.MustNewMatcher(labels.MatchEqual, "state", "ACTIVE"),
-		}
-	}
-	require.NoError(t, distributor.WaitSumMetricsWithOptions(e2e.Equals(1), []string{`tempo_ring_members`}, e2e.WithLabelMatchers(isServiceActiveMatcher("ingester")...), e2e.WaitMissingMetrics))
-
-	// Wait until joined to partition ring
-	partitionStateMatchers := func(state string) []*labels.Matcher {
-		return []*labels.Matcher{
-			{Type: labels.MatchEqual, Name: "state", Value: state},
-			{Type: labels.MatchEqual, Name: "name", Value: "ingester-partitions"},
-		}
-	}
-	require.NoError(t, distributor.WaitSumMetricsWithOptions(e2e.Equals(1), []string{"tempo_partition_ring_partitions"}, e2e.WithLabelMatchers(partitionStateMatchers("Active")...)))
-
-	// Get port for the Jaeger gRPC receiver endpoint
-	c, err := util.NewJaegerToOTLPExporter(distributor.Endpoint(4317))
-	require.NoError(t, err)
-	require.NotNil(t, c)
-
-	// Generate and emit initial traces
-	info := tempoUtil.NewTraceInfo(time.Now(), "")
-	require.NoError(t, info.EmitAllBatches(c))
-
-	// Wait for traces to be received
-	expected, err := info.ConstructTraceFromEpoch()
-	require.NoError(t, err)
-	require.NoError(t, distributor.WaitSumMetrics(e2e.Equals(util.SpanCount(expected)), "tempo_distributor_spans_received_total"))
-
-	// Create API client
-	apiClient := httpclient.New("http://"+queryFrontend.Endpoint(3200), "")
-
-	// Set ingester's partition to INACTIVE state (prepare for downscale)
-	req, err := http.NewRequest("POST", "http://"+ingester.Endpoint(3200)+"/ingester/prepare-partition-downscale", nil)
-	require.NoError(t, err)
-	httpResp, err := apiClient.Do(req)
-	require.NoError(t, err)
-	require.Equal(t, 200, httpResp.StatusCode)
-
-	// Verify ingester's partition is INACTIVE by checking the timestamp
-	req, err = http.NewRequest("GET", "http://"+ingester.Endpoint(3200)+"/ingester/prepare-partition-downscale", nil)
-	require.NoError(t, err)
-	httpResp, err = apiClient.Do(req)
-	require.NoError(t, err)
-	require.Equal(t, 200, httpResp.StatusCode)
-	var result map[string]any
-	require.NoError(t, json.NewDecoder(httpResp.Body).Decode(&result))
-	require.Greater(t, result["timestamp"].(float64), float64(0)) // ts > 0 ==> INACTIVE (when it was marked for downscale)
-
-	require.NoError(t, distributor.WaitSumMetricsWithOptions(e2e.Equals(1), []string{"tempo_partition_ring_partitions"}, e2e.WithLabelMatchers(partitionStateMatchers("Inactive")...)))
-
-	// Start block-builder (it should consume data from the downscaled partition)
-	blockbuilder := util.NewTempoBlockBuilder(0)
-	require.NoError(t, s.StartAndWaitReady(blockbuilder))
-
-	// Wait for blocks to be flushed from the downscaled partition
-	require.NoError(t, blockbuilder.WaitSumMetricsWithOptions(e2e.GreaterOrEqual(1), []string{"tempo_block_builder_flushed_blocks"}, e2e.WaitMissingMetrics))
-	require.NoError(t, queryFrontend.WaitSumMetricsWithOptions(e2e.GreaterOrEqual(1), []string{"tempodb_blocklist_length"},
-		e2e.WaitMissingMetrics, e2e.WithLabelMatchers(&labels.Matcher{Type: labels.MatchEqual, Name: "tenant", Value: "single-tenant"})))
-
-	// Verify initial traces can be queried from backend storage
-	trace, err := apiClient.QueryTrace(info.HexID())
-	require.NoError(t, err)
-	require.NotNil(t, trace)
-
-	// Set ingester's partition back to ACTIVE state
-	req, err = http.NewRequest("DELETE", "http://"+ingester.Endpoint(3200)+"/ingester/prepare-partition-downscale", nil)
-	require.NoError(t, err)
-	httpResp, err = apiClient.Do(req)
-	require.NoError(t, err)
-	require.Equal(t, 200, httpResp.StatusCode)
-
-	// Verify ingester's partition is ACTIVE by checking the timestamp is 0
-	req, err = http.NewRequest("GET", "http://"+ingester.Endpoint(3200)+"/ingester/prepare-partition-downscale", nil)
-	require.NoError(t, err)
-	httpResp, err = apiClient.Do(req)
-	require.NoError(t, err)
-	require.Equal(t, 200, httpResp.StatusCode)
-	require.NoError(t, json.NewDecoder(httpResp.Body).Decode(&result))
-	require.Equal(t, float64(0), result["timestamp"].(float64)) // ts == 0 ==> ACTIVE
-
-	require.NoError(t, distributor.WaitSumMetricsWithOptions(e2e.Equals(1), []string{"tempo_partition_ring_partitions"}, e2e.WithLabelMatchers(partitionStateMatchers("Active")...)))
-
-	// Generate and emit more traces after reactivating the partition
-	info2 := tempoUtil.NewTraceInfo(time.Now(), "")
-	require.NoError(t, info2.EmitAllBatches(c))
-
-	// Wait for new traces to be received
-	expected2, err := info2.ConstructTraceFromEpoch()
-	require.NoError(t, err)
-	require.NoError(t, distributor.WaitSumMetrics(e2e.Equals(util.SpanCount(expected)+util.SpanCount(expected2)), "tempo_distributor_spans_received_total"))
-
-	// Wait for the new traces to be flushed by block-builder
-	require.NoError(t, blockbuilder.WaitSumMetricsWithOptions(e2e.GreaterOrEqual(2), []string{"tempo_block_builder_flushed_blocks"}, e2e.WaitMissingMetrics))
-
-	// Verify all traces using trace ID lookup
-	trace, err = apiClient.QueryTrace(info.HexID())
-	require.NoError(t, err)
-	require.NotNil(t, trace)
-	require.Equal(t, util.SpanCount(expected), util.SpanCount(trace))
-
-	trace2, err := apiClient.QueryTrace(info2.HexID())
-	require.NoError(t, err)
-	require.NotNil(t, trace2)
-	require.Equal(t, util.SpanCount(expected2), util.SpanCount(trace2))
-}
-
-func TestLiveStorePartitionDownscale(t *testing.T) {
-	s, err := e2e.NewScenario("tempo_e2e")
-	require.NoError(t, err)
-	defer s.Close()
-
-	require.NoError(t, util.CopyFileToSharedDir(s, "config-live-store.yaml", "config.yaml"))
-
-	// Start dependencies
-	kafka := e2edb.NewKafka()
-	require.NoError(t, s.StartAndWaitReady(kafka))
-
-	// Start live-store instances
-	liveStore0 := util.NewTempoLiveStore(0)
-	liveStore1 := util.NewTempoLiveStore(1)
-	require.NoError(t, s.StartAndWaitReady(liveStore0, liveStore1))
-	waitUntilJoinedToPartitionRing(t, liveStore0, 2) // wait for both to join
-
-	// Start other Tempo components
-	distributor := util.NewTempoDistributor()
-	querier := util.NewTempoQuerier()
-	queryFrontend := util.NewTempoQueryFrontend()
-	require.NoError(t, s.StartAndWaitReady(distributor, querier, queryFrontend))
-
-	// Generate and emit initial traces
-	c, err := util.NewJaegerToOTLPExporter(distributor.Endpoint(4317))
-	require.NoError(t, err)
-	require.NotNil(t, c)
-
-	tracePool := newTracePool()
-	firstTrace, err := tracePool.Generate()
-	require.NoError(t, err)
-	require.NoError(t, firstTrace.EmitAllBatches(c))
-
-	// Wait for traces to be received
-	expected, err := firstTrace.ConstructTraceFromEpoch()
-	require.NoError(t, err)
-	distributorSpanCount := util.SpanCount(expected)
-	require.NoError(t, distributor.WaitSumMetrics(e2e.Equals(distributorSpanCount), "tempo_distributor_spans_received_total"))
-
-	// Verify traces are processed by one of the live-stores
-	// First try to find which live-store processed the records
-	// We will shutdown this instance later
-	var liveStoreInactive, liveStoreActive *e2e.HTTPService
-
-	ls := waitForTraceInLiveStore(t, 1, liveStore0, liveStore1)
-	if ls == liveStore0 {
-		liveStoreInactive = liveStore0
-		liveStoreActive = liveStore1
-	} else {
-		liveStoreInactive = liveStore1
-		liveStoreActive = liveStore0
-	}
-
-	apiClient := httpclient.New("http://"+queryFrontend.Endpoint(3200), "")
-
-	t.Run("verify partition is ACTIVE", func(t *testing.T) {
-		for _, liveStore := range []*e2e.HTTPService{liveStoreInactive, liveStoreActive} {
-			res := preparePartitionDownscale(t, http.MethodGet, liveStore)
-			require.Equal(t, "PartitionActive", res.State)
-		}
-	})
-
-	t.Run("prepare partition for downscale", func(t *testing.T) {
-		// Set live-store's partition to INACTIVE state (prepare for downscale)
-		for range 2 { // repeated action will be noop
-			res := preparePartitionDownscale(t, http.MethodPost, liveStoreInactive)
-			require.Greater(t, res.Timestamp, int64(0)) // ts > 0 ==> INACTIVE
-			require.Equal(t, "PartitionInactive", res.State)
-		}
-
-		// Test GET method
-		res := preparePartitionDownscale(t, http.MethodGet, liveStoreInactive)
-		require.Greater(t, res.Timestamp, int64(0)) // ts > 0 ==> INACTIVE
-		require.Equal(t, "PartitionInactive", res.State)
-		res = preparePartitionDownscale(t, http.MethodGet, liveStoreActive)
-		require.Equal(t, "PartitionActive", res.State) // still active
-
-		for _, component := range []*e2e.HTTPService{liveStoreInactive, liveStoreActive, distributor} {
-			verifyPartitionState(t, component, "Inactive", 1)
-			verifyPartitionState(t, component, "Active", 1)
-		}
-	})
-
-	t.Run("verify data is still accessible during downscale", func(t *testing.T) {
-		// Even with partition INACTIVE, existing data should still be queryable
-		// from the live-store during the grace period
-		trace, err := apiClient.QueryTrace(firstTrace.HexID())
-		require.NoError(t, err)
-		require.NotNil(t, trace)
-		require.Equal(t, util.SpanCount(expected), util.SpanCount(trace))
-	})
-
-	t.Run("generate new traces during downscale", func(t *testing.T) {
-		inactiveCount, err := liveStoreInactive.SumMetrics([]string{"tempo_live_store_records_processed_total"}, e2e.SkipMissingMetrics)
-		require.NoError(t, err)
-
-		// Send 10 traces. In case of a bug when it sends to both live-stores,
-		// possibility of false pass is 1/1024.
-		const numTraces = 10
-		traces := make([]string, 0, numTraces)
-		spanCounts := make([]float64, 0, numTraces)
-
-		// Generate new traces - these should be processed by the other live-store
-		// since the first one is in INACTIVE state
-		for range numTraces {
-			trace, err := tracePool.Generate()
-			require.NoError(t, err)
-			require.NoError(t, trace.EmitAllBatches(c))
-			expected, err := trace.ConstructTraceFromEpoch()
-			require.NoError(t, err)
-			traces = append(traces, trace.HexID())
-			spanCounts = append(spanCounts, util.SpanCount(expected))
-		}
-
-		for _, spanCount := range spanCounts {
-			distributorSpanCount += spanCount
-		}
-		// Wait for new traces to be received
-		require.NoError(t, distributor.WaitSumMetrics(e2e.Equals(distributorSpanCount), "tempo_distributor_spans_received_total"))
-		require.NoError(t, liveStoreActive.WaitSumMetrics(e2e.GreaterOrEqual(float64(1)), "tempo_live_store_records_processed_total"))
-
-		// Verify inactive live-store still has the same number of processed records
-		inactiveCountAfter, err := liveStoreInactive.SumMetrics([]string{"tempo_live_store_records_processed_total"}, e2e.SkipMissingMetrics)
-		require.NoError(t, err)
-		require.Equal(t, inactiveCount, inactiveCountAfter, "inactive count should be the same")
-
-		// The other live-store (not the one being downscaled) should process new records
-		for i, traceID := range traces {
-			actualTrace, err := apiClient.QueryTrace(traceID)
-			require.NoError(t, err)
-			require.Equal(t, spanCounts[i], util.SpanCount(actualTrace))
-		}
-	})
-
-	t.Run("cancel downscale preparation", func(t *testing.T) {
-		// Cancel partition downscale
-		res := preparePartitionDownscale(t, http.MethodDelete, liveStoreInactive)
-		require.Equal(t, int64(0), res.Timestamp) // ts == 0 ==> ACTIVE
-		require.Equal(t, "PartitionActive", res.State)
-
-		// Verify partition is back to ACTIVE
-		res = preparePartitionDownscale(t, http.MethodGet, liveStoreInactive)
-		require.Equal(t, int64(0), res.Timestamp) // ts == 0 ==> ACTIVE
-		require.Equal(t, "PartitionActive", res.State)
-
-		for _, component := range []*e2e.HTTPService{liveStoreInactive, liveStoreActive, distributor} {
-			verifyPartitionState(t, component, "Active", 2)
-		}
-		for _, liveStore := range []*e2e.HTTPService{liveStoreInactive, liveStoreActive} {
-			res := preparePartitionDownscale(t, http.MethodGet, liveStore)
-			require.Equal(t, "PartitionActive", res.State)
-		}
-	})
-
-	t.Run("verify normal operation after cancellation", func(t *testing.T) {
-		inactiveCount, err := liveStoreInactive.SumMetrics([]string{"tempo_live_store_records_processed_total"}, e2e.SkipMissingMetrics)
-		require.NoError(t, err)
-		activeCount, err := liveStoreActive.SumMetrics([]string{"tempo_live_store_records_processed_total"}, e2e.SkipMissingMetrics)
-		require.NoError(t, err)
-
-		// Send 10 traces. Possibility of false positive is 1/1024.
-		const numTraces = 10
-		traces := make([]string, 0, numTraces)
-		spanCounts := make([]float64, 0, numTraces)
-
-		for range numTraces {
-			trace, err := tracePool.Generate()
-			require.NoError(t, err)
-			require.NoError(t, trace.EmitAllBatches(c))
-			expected, err := trace.ConstructTraceFromEpoch()
-			require.NoError(t, err)
-			traces = append(traces, trace.HexID())
-			spanCounts = append(spanCounts, util.SpanCount(expected))
-		}
-
-		// Wait for new traces to be received
-		for _, spanCount := range spanCounts {
-			distributorSpanCount += spanCount
-		}
-		// Verify distributor and both live-stores have processed new records
-		require.NoError(t, distributor.WaitSumMetrics(e2e.Equals(distributorSpanCount), "tempo_distributor_spans_received_total"))
-		require.NoError(t, liveStoreActive.WaitSumMetrics(e2e.Greater(activeCount[0]), "tempo_live_store_records_processed_total"))
-		require.NoError(t, liveStoreInactive.WaitSumMetrics(e2e.Greater(inactiveCount[0]), "tempo_live_store_records_processed_total"))
-
-		// The other live-store (not the one being downscaled) should process new records
-		for i, traceID := range traces {
-			actualTrace, err := apiClient.QueryTrace(traceID)
-			require.NoError(t, err)
-			require.Equal(t, spanCounts[i], util.SpanCount(actualTrace))
-		}
-	})
-}
-
-func TestLiveStoreDownscaleHappyPath(t *testing.T) {
-	s, err := e2e.NewScenario("tempo_e2e")
-	require.NoError(t, err)
-	defer s.Close()
-
-	require.NoError(t, util.CopyFileToSharedDir(s, "config-live-store.yaml", "config.yaml"))
-
-	// Start dependencies
-	kafka := e2edb.NewKafka()
-	require.NoError(t, s.StartAndWaitReady(kafka))
-
-	liveStoreActive := util.NewTempoLiveStore(0)
-	liveStoreInactive := util.NewTempoLiveStore(1)
-	require.NoError(t, s.StartAndWaitReady(liveStoreActive, liveStoreInactive))
-	waitUntilJoinedToPartitionRing(t, liveStoreActive, 2) // wait for both to join
-
-	distributor := util.NewTempoDistributor()
-	require.NoError(t, s.StartAndWaitReady(distributor))
-
-	// Wait until both live-stores are active in the ring
-	isServiceActiveMatcher := func(service, state string) []*labels.Matcher {
-		return []*labels.Matcher{
-			labels.MustNewMatcher(labels.MatchEqual, "name", service),
-			labels.MustNewMatcher(labels.MatchEqual, "state", state),
-		}
-	}
-	require.NoError(t, distributor.WaitSumMetricsWithOptions(
-		e2e.Equals(2),
-		[]string{`tempo_ring_members`},
-		e2e.WithLabelMatchers(isServiceActiveMatcher("live-store", "ACTIVE")...),
-		e2e.WaitMissingMetrics,
-	))
-
-	// Change partition state to INACTIVE
-	preparePartitionDownscale(t, http.MethodPost, liveStoreInactive)
-
-	// Prepare downscale, right before shutdown
-	req, err := http.NewRequest("POST", "http://"+liveStoreInactive.Endpoint(3200)+"/live-store/prepare-downscale", nil)
-	require.NoError(t, err)
-	httpResp, err := http.DefaultClient.Do(req)
-	require.NoError(t, err)
-	require.Equal(t, 204, httpResp.StatusCode) // PrepareDownscaleHandler returns StatusNoContent (204)
-
-	// Shutdown inactive live-store
-	require.NoError(t, liveStoreInactive.Stop())
-	time.Sleep(5 * time.Second)
-
-	// Only one ACTIVE live-store remains
-	partitions := getRingStatus(t, distributor).Partitions
-	require.Equal(t, 1, len(partitions), "only one ACTIVE live-store should remain, actual result: %v", partitions)
-	partition := partitions[0]
-
-	require.Equal(t, partitionData{ID: 0, Corrupted: false, State: 2, OwnerIDs: []string{"live-store-0"}}, partition)
-}
-
-func TestLiveStoreRestart(t *testing.T) {
-	s, err := e2e.NewScenario("tempo_e2e")
-	require.NoError(t, err)
-	defer s.Close()
-
-	require.NoError(t, util.CopyFileToSharedDir(s, "config-live-store.yaml", "config.yaml"))
-
-	// Start dependencies
-	kafka := e2edb.NewKafka()
-	require.NoError(t, s.StartAndWaitReady(kafka))
-
-	liveStore := util.NewTempoLiveStore(0)
-	distributor := util.NewTempoDistributor()
-	require.NoError(t, s.StartAndWaitReady(liveStore, distributor))
-	waitUntilJoinedToPartitionRing(t, liveStore, 1)
-
-	// Change partition state to INACTIVE
-	preparePartitionDownscale(t, http.MethodPost, liveStore)
-
-	// Check that after stop (restart), it is not removed from the ring
-	require.NoError(t, liveStore.Stop())
-	time.Sleep(5 * time.Second)
-	rs := getRingStatus(t, distributor)
-	require.Equal(t, 1, len(rs.Partitions))
-}
-
-type tracePool struct {
-	pool map[string]*tempoUtil.TraceInfo
-	ts   time.Time
-	idx  int
-}
-
-func newTracePool() *tracePool {
-	return &tracePool{
-		pool: make(map[string]*tempoUtil.TraceInfo),
-		ts:   time.Now(),
-		idx:  0,
-	}
-}
-
-func (p *tracePool) Generate() (*tempoUtil.TraceInfo, error) {
-	p.idx++
-	// We add a second to make sure the trace ID is different
-	// as seed of the rand is based on the timestamp
-	info := tempoUtil.NewTraceInfo(p.ts.Add(time.Second*time.Duration(p.idx)), "")
-	if p.pool[info.HexID()] != nil {
-		return nil, fmt.Errorf("test is invalid, generated the same trace ID: %s", info.HexID())
-	}
-	p.pool[info.HexID()] = info
-	return info, nil
-}
-
-func TestTracePool(t *testing.T) {
-	pool := newTracePool()
-	for i := range 50 {
-		trace, err := pool.Generate()
-		require.NoError(t, err, "error generating trace %d", i)
-		require.NotNil(t, trace, "trace is nil for %d", i)
-	}
-}
-
-func waitForTraceInLiveStore(t *testing.T, expectedRecords int, liveStores ...*e2e.HTTPService) *e2e.HTTPService {
-	ch := make(chan *e2e.HTTPService)
-	timeout := 30 * time.Second
-
-	for _, liveStore := range liveStores {
-		go func(liveStore *e2e.HTTPService) {
-			err := liveStore.WaitSumMetrics(e2e.GreaterOrEqual(float64(expectedRecords)), "tempo_live_store_records_processed_total")
-			if err == nil {
-				ch <- liveStore
-			}
-		}(liveStore)
-	}
-
-	select {
-	case liveStore := <-ch:
-		{
-			close(ch)
-			return liveStore
-		}
-	case <-time.After(timeout):
-		t.Fatalf("timeout waiting for trace to appear in any live-store after %s", timeout)
-	}
-	return nil
-}
-
-func verifyPartitionState(t *testing.T, liveStore *e2e.HTTPService, expectedState string, expectedCount int) {
-	partitionStateMatchers := []*labels.Matcher{
-		{Type: labels.MatchEqual, Name: "state", Value: expectedState},
-		{Type: labels.MatchEqual, Name: "name", Value: "livestore-partitions"},
-	}
-	require.NoError(t, liveStore.WaitSumMetricsWithOptions(e2e.Equals(float64(expectedCount)), []string{"tempo_partition_ring_partitions"}, e2e.WithLabelMatchers(partitionStateMatchers...)))
-}
-
-type preparePartitionDownscaleResponse struct {
-	Timestamp int64  `json:"timestamp"`
-	State     string `json:"state"`
-}
-
-type ringStatus struct {
-	Partitions []partitionData `json:"partitions"`
-}
-
-type partitionData struct {
-	ID        int32    `json:"id"`
-	Corrupted bool     `json:"corrupted"`
-	State     int      `json:"state"`
-	OwnerIDs  []string `json:"owner_ids"`
-	// skip the fields below
-	// StateTimestamp string   `json:"state_timestamp"`
-	// Tokens         []uint32  `json:"tokens"`
-}
-
-func getRingStatus(t *testing.T, service *e2e.HTTPService) ringStatus {
-	req, err := http.NewRequest("GET", "http://"+service.Endpoint(3200)+"/partition-ring", nil)
-	req.Header.Set("Accept", "application/json")
-	require.NoError(t, err)
-	httpResp, err := http.DefaultClient.Do(req)
-	require.NoError(t, err)
-	require.Equal(t, 200, httpResp.StatusCode)
-
-	var result ringStatus
-	require.NoError(t, json.NewDecoder(httpResp.Body).Decode(&result))
-	return result
-}
-
-func preparePartitionDownscale(t *testing.T, method string, liveStore *e2e.HTTPService) preparePartitionDownscaleResponse {
-	req, err := http.NewRequest(method, "http://"+liveStore.Endpoint(3200)+"/live-store/prepare-partition-downscale", nil)
-	require.NoError(t, err)
-	httpResp, err := http.DefaultClient.Do(req)
-	require.NoError(t, err)
-	require.Equal(t, 200, httpResp.StatusCode)
-
-	var result preparePartitionDownscaleResponse
-	require.NoError(t, json.NewDecoder(httpResp.Body).Decode(&result))
-	return result
-}

integration/e2e/limits_test.go

@@ -1,395 +0,0 @@
-package e2e
-
-import (
-	"bytes"
-	"context"
-	crand "crypto/rand"
-	"encoding/binary"
-	"fmt"
-	"io"
-	"net/http"
-	"strings"
-	"testing"
-	"time"
-
-	"github.com/grafana/dskit/user"
-	util2 "github.com/grafana/tempo/integration/util"
-	"go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc"
-	"google.golang.org/grpc/codes"
-	"google.golang.org/grpc/status"
-
-	"github.com/grafana/e2e"
-	"github.com/prometheus/prometheus/model/labels"
-	"github.com/stretchr/testify/assert"
-	"github.com/stretchr/testify/require"
-	"go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp"
-	"google.golang.org/genproto/googleapis/rpc/errdetails"
-
-	"github.com/grafana/tempo/integration/util"
-	"github.com/grafana/tempo/pkg/httpclient"
-	"github.com/grafana/tempo/pkg/model/trace"
-	"github.com/grafana/tempo/pkg/tempopb"
-	tempoUtil "github.com/grafana/tempo/pkg/util"
-	"github.com/grafana/tempo/pkg/util/test"
-
-	"go.opentelemetry.io/collector/pdata/ptrace"
-)
-
-const (
-	configLimits             = "config-limits.yaml"
-	configLimitsQuery        = "config-limits-query.yaml"
-	configLimitsPartialError = "config-limits-partial-success.yaml"
-	configLimits429          = "config-limits-429.yaml"
-)
-
-func TestLimits(t *testing.T) {
-	s, err := e2e.NewScenario("tempo_e2e")
-	require.NoError(t, err)
-	defer s.Close()
-
-	require.NoError(t, util2.CopyFileToSharedDir(s, configLimits, "config.yaml"))
-	tempo := util2.NewTempoAllInOne()
-	require.NoError(t, s.StartAndWaitReady(tempo))
-
-	// Get port for the otlp receiver endpoint
-	c, err := util2.NewJaegerToOTLPExporter(tempo.Endpoint(4317))
-	require.NoError(t, err)
-	require.NotNil(t, c)
-
-	// should fail b/c the trace is too large. each batch should be ~70 bytes
-	batch := util.MakeThriftBatchWithSpanCount(2)
-	require.NoError(t, c.EmitBatch(context.Background(), batch), "max trace size")
-
-	// push a trace
-	require.NoError(t, c.EmitBatch(context.Background(), util.MakeThriftBatchWithSpanCount(1)))
-
-	// should fail b/c this will be too many traces
-	batch = util.MakeThriftBatch()
-	require.NoError(t, c.EmitBatch(context.Background(), batch), "too many traces")
-
-	// should fail b/c due to ingestion rate limit
-	batch = util.MakeThriftBatchWithSpanCount(10)
-	err = c.EmitBatch(context.Background(), batch)
-	require.Error(t, err)
-
-	// this error must have a retryinfo as expected in otel collector code: https://github.com/open-telemetry/opentelemetry-collector/blob/d7b49df5d9e922df6ce56ad4b64ee1c79f9dbdbe/exporter/otlpexporter/otlp.go#L172
-	st, ok := status.FromError(err)
-	require.True(t, ok)
-	foundRetryInfo := false
-	for _, detail := range st.Details() {
-		if _, ok := detail.(*errdetails.RetryInfo); ok {
-			foundRetryInfo = true
-			break
-		}
-	}
-	require.True(t, foundRetryInfo)
-
-	// test limit metrics
-	err = tempo.WaitSumMetricsWithOptions(e2e.Equals(2),
-		[]string{"tempo_discarded_spans_total"},
-		e2e.WithLabelMatchers(labels.MustNewMatcher(labels.MatchEqual, "reason", "trace_too_large")),
-	)
-	require.NoError(t, err)
-	err = tempo.WaitSumMetricsWithOptions(e2e.Equals(1),
-		[]string{"tempo_discarded_spans_total"},
-		e2e.WithLabelMatchers(labels.MustNewMatcher(labels.MatchEqual, "reason", "live_traces_exceeded")),
-	)
-	require.NoError(t, err)
-	err = tempo.WaitSumMetricsWithOptions(e2e.Equals(10),
-		[]string{"tempo_discarded_spans_total"},
-		e2e.WithLabelMatchers(labels.MustNewMatcher(labels.MatchEqual, "reason", "rate_limited")),
-	)
-	require.NoError(t, err)
-}
-
-func TestOTLPLimits(t *testing.T) {
-	s, err := e2e.NewScenario("tempo_e2e")
-	require.NoError(t, err)
-	defer s.Close()
-
-	require.NoError(t, util2.CopyFileToSharedDir(s, configLimits, "config.yaml"))
-	tempo := util2.NewTempoAllInOne()
-	require.NoError(t, s.StartAndWaitReady(tempo))
-
-	protoSpans := test.MakeProtoSpans(100)
-
-	// gRPC
-	grpcClient := otlptracegrpc.NewClient(
-		otlptracegrpc.WithEndpoint(tempo.Endpoint(4317)),
-		otlptracegrpc.WithInsecure(),
-		otlptracegrpc.WithRetry(otlptracegrpc.RetryConfig{Enabled: false}),
-	)
-	require.NoError(t, grpcClient.Start(context.Background()))
-
-	grpcErr := grpcClient.UploadTraces(context.Background(), protoSpans)
-	assert.Error(t, grpcErr)
-	require.Equal(t, codes.ResourceExhausted, status.Code(grpcErr))
-
-	// HTTP
-	httpClient := otlptracehttp.NewClient(
-		otlptracehttp.WithEndpoint(tempo.Endpoint(4318)),
-		otlptracehttp.WithInsecure(),
-		otlptracehttp.WithRetry(otlptracehttp.RetryConfig{Enabled: false}),
-	)
-	require.NoError(t, httpClient.Start(context.Background()))
-
-	httpErr := httpClient.UploadTraces(context.Background(), protoSpans)
-	assert.Error(t, httpErr)
-	require.Contains(t, httpErr.Error(), "retry-able request failure")
-}
-
-func TestOTLPLimitsVanillaClient(t *testing.T) {
-	s, err := e2e.NewScenario("tempo_e2e")
-	require.NoError(t, err)
-	defer s.Close()
-
-	require.NoError(t, util2.CopyFileToSharedDir(s, configLimits, "config.yaml"))
-	tempo := util2.NewTempoAllInOne()
-	require.NoError(t, s.StartAndWaitReady(tempo))
-
-	trace := test.MakeTrace(10, []byte{1, 2, 3, 4, 5, 6, 7, 8, 9, 10})
-
-	testCases := []struct {
-		name    string
-		payload func() []byte
-		headers map[string]string
-	}{
-		// TODO There is an issue when sending the payload in json format. The server returns a 200 instead of a 429.
-		// {
-		// 	"JSON format",
-		// 	func() []byte {
-		// 		b := &bytes.Buffer{}
-		// 		err := (&jsonpb.Marshaler{}).Marshal(b, trace)
-		// 		require.NoError(t, err)
-		// 		return b.Bytes()
-		// 	},
-		// 	map[string]string{
-		// 		"Content-Type": "application/json",
-		// 	},
-		// },
-		{
-			"Proto format",
-			func() []byte {
-				b, err := trace.Marshal()
-				require.NoError(t, err)
-				return b
-			},
-			map[string]string{
-				"Content-Type": "application/x-protobuf",
-			},
-		},
-	}
-
-	for _, tc := range testCases {
-		t.Run(tc.name, func(t *testing.T) {
-			req, err := http.NewRequest(http.MethodPost, "http://"+tempo.Endpoint(4318)+"/v1/traces", bytes.NewReader(tc.payload()))
-			require.NoError(t, err)
-			for k, v := range tc.headers {
-				req.Header.Set(k, v)
-			}
-
-			resp, err := http.DefaultClient.Do(req)
-			require.NoError(t, err)
-			defer func() { _ = resp.Body.Close() }()
-			bodyBytes, err := io.ReadAll(resp.Body)
-			require.NoError(t, err)
-			fmt.Println(string(bodyBytes))
-
-			assert.Equal(t, http.StatusTooManyRequests, resp.StatusCode)
-		})
-	}
-}
-
-func TestQueryLimits(t *testing.T) {
-	s, err := e2e.NewScenario("tempo_e2e")
-	require.NoError(t, err)
-	defer s.Close()
-
-	require.NoError(t, util2.CopyFileToSharedDir(s, configLimitsQuery, "config.yaml"))
-	tempo := util2.NewTempoAllInOne()
-	require.NoError(t, s.StartAndWaitReady(tempo))
-
-	// Get port for the otlp receiver endpoint
-	c, err := util2.NewJaegerToOTLPExporter(tempo.Endpoint(4317))
-	require.NoError(t, err)
-	require.NotNil(t, c)
-
-	// make a trace with 10 spans and push them one at a time, flush in between each one to force different blocks
-	batch := util.MakeThriftBatchWithSpanCount(5)
-	allSpans := batch.Spans
-	for i := range batch.Spans {
-		batch.Spans = allSpans[i : i+1]
-		require.NoError(t, c.EmitBatch(context.Background(), batch))
-		util.CallFlush(t, tempo)
-		// this push along with the double flush is required to forget the too large trace
-		require.NoError(t, c.EmitBatch(context.Background(), util.MakeThriftBatchWithSpanCount(1)))
-		util.CallFlush(t, tempo)
-		time.Sleep(2 * time.Second) // trace idle and flush time are both 1ms
-	}
-
-	// calc trace id
-	traceID := [16]byte{}
-	binary.BigEndian.PutUint64(traceID[:8], uint64(batch.Spans[0].TraceIdHigh))
-	binary.BigEndian.PutUint64(traceID[8:], uint64(batch.Spans[0].TraceIdLow))
-
-	// now try to query it back. this should fail b/c the trace is too large
-	client := httpclient.New("http://"+tempo.Endpoint(3200), tempoUtil.FakeTenantID)
-	querierClient := httpclient.New("http://"+tempo.Endpoint(3200)+"/querier", tempoUtil.FakeTenantID)
-
-	require.Eventually(t, func() bool {
-		_, err = client.QueryTrace(tempoUtil.TraceIDToHexString(traceID[:]))
-		if err == nil {
-			return false
-		}
-		return strings.Contains(err.Error(), trace.ErrTraceTooLarge.Error())
-	}, time.Minute, time.Second)
-
-	require.ErrorContains(t, err, "failed with response: 422") // confirm frontend returns 422
-
-	_, err = querierClient.QueryTrace(tempoUtil.TraceIDToHexString(traceID[:]))
-	require.ErrorContains(t, err, trace.ErrTraceTooLarge.Error())
-	require.ErrorContains(t, err, "failed with response: 422")
-
-	// complete block timeout  is 10 seconds
-	time.Sleep(15 * time.Second)
-	_, err = client.QueryTrace(tempoUtil.TraceIDToHexString(traceID[:]))
-	require.ErrorContains(t, err, trace.ErrTraceTooLarge.Error())
-	require.ErrorContains(t, err, "failed with response: 422") // confirm frontend returns 422
-
-	_, err = querierClient.QueryTrace(tempoUtil.TraceIDToHexString(traceID[:]))
-	require.ErrorContains(t, err, trace.ErrTraceTooLarge.Error())
-	require.ErrorContains(t, err, "failed with response: 422") // confirm querier returns 422
-}
-
-func TestLimitsPartialSuccess(t *testing.T) {
-	s, err := e2e.NewScenario("tempo_e2e")
-	require.NoError(t, err)
-	defer s.Close()
-	require.NoError(t, util2.CopyFileToSharedDir(s, configLimitsPartialError, "config.yaml"))
-	tempo := util2.NewTempoAllInOne()
-	require.NoError(t, s.StartAndWaitReady(tempo))
-
-	// otel grpc exporter
-	exporter, err := util2.NewOtelGRPCExporter(tempo.Endpoint(4317))
-	require.NoError(t, err)
-
-	// make request
-	traceIDs := make([][]byte, 6)
-	for index := range traceIDs {
-		traceID := make([]byte, 16)
-		_, err = crand.Read(traceID)
-		require.NoError(t, err)
-		traceIDs[index] = traceID
-	}
-
-	// 3 traces with trace_too_large and 3 with no error
-	spanCountsByTrace := []int{1, 4, 1, 5, 6, 1}
-	req := test.MakeReqWithMultipleTraceWithSpanCount(spanCountsByTrace, traceIDs)
-
-	b, err := req.Marshal()
-	require.NoError(t, err)
-
-	// unmarshal into otlp proto
-	traces, err := (&ptrace.ProtoUnmarshaler{}).UnmarshalTraces(b)
-	require.NoError(t, err)
-	require.NotNil(t, traces)
-
-	ctx := user.InjectOrgID(context.Background(), tempoUtil.FakeTenantID)
-	ctx, err = user.InjectIntoGRPCRequest(ctx)
-	require.NoError(t, err)
-
-	// send traces to tempo
-	// partial success = no error
-	err = exporter.ConsumeTraces(ctx, traces)
-	require.NoError(t, err)
-
-	// shutdown to ensure traces are flushed
-	require.NoError(t, exporter.Shutdown(context.Background()))
-
-	// query for the one trace that didn't trigger an error
-	client := httpclient.New("http://"+tempo.Endpoint(3200), tempoUtil.FakeTenantID)
-	for i, count := range spanCountsByTrace {
-		if count == 1 {
-			result, err := client.QueryTrace(tempoUtil.TraceIDToHexString(traceIDs[i]))
-			require.NoError(t, err)
-			assert.Equal(t, 1, len(result.ResourceSpans))
-		}
-	}
-
-	// test metrics
-	// 3 traces with trace_too_large each with 4+5+6 spans
-	err = tempo.WaitSumMetricsWithOptions(e2e.Equals(15),
-		[]string{"tempo_discarded_spans_total"},
-		e2e.WithLabelMatchers(labels.MustNewMatcher(labels.MatchEqual, "reason", "trace_too_large")),
-	)
-	require.NoError(t, err)
-
-	// this metric should never exist
-	err = tempo.WaitSumMetricsWithOptions(e2e.Equals(0),
-		[]string{"tempo_discarded_spans_total"},
-		e2e.WithLabelMatchers(labels.MustNewMatcher(labels.MatchEqual, "reason", "unknown_error")),
-	)
-	require.NoError(t, err)
-}
-
-func TestQueryRateLimits(t *testing.T) {
-	s, err := e2e.NewScenario("tempo_e2e")
-	require.NoError(t, err)
-	defer s.Close()
-
-	require.NoError(t, util2.CopyFileToSharedDir(s, configLimits429, "config.yaml"))
-	tempo := util2.NewTempoAllInOne()
-	require.NoError(t, s.StartAndWaitReady(tempo))
-
-	// Get port for the otlp receiver endpoint
-	c, err := util2.NewJaegerToOTLPExporter(tempo.Endpoint(4317))
-	require.NoError(t, err)
-	require.NotNil(t, c)
-
-	// make a trace with 10 spans and push them one at a time, flush in between each one to force different blocks
-	batch := util.MakeThriftBatchWithSpanCount(5)
-	allSpans := batch.Spans
-	for i := range batch.Spans {
-		batch.Spans = allSpans[i : i+1]
-		require.NoError(t, c.EmitBatch(context.Background(), batch))
-		util.CallFlush(t, tempo)
-		time.Sleep(2 * time.Second) // trace idle and flush time are both 1ms
-	}
-	// now try to query it back. this should fail b/c the frontend queue doesn't have room
-	client := httpclient.New("http://"+tempo.Endpoint(3200), tempoUtil.FakeTenantID)
-
-	// 429 HTTP Trace ID Lookup
-	traceID := []byte{0x01, 0x02}
-	_, err = client.QueryTrace(tempoUtil.TraceIDToHexString(traceID))
-	require.ErrorContains(t, err, "job queue full")
-	require.ErrorContains(t, err, "failed with response: 429")
-
-	start := time.Now().Add(-1 * time.Hour).Unix()
-	end := time.Now().Add(1 * time.Hour).Unix()
-
-	// 429 HTTP Search
-	_, err = client.SearchTraceQLWithRange("{}", start, end)
-	require.ErrorContains(t, err, "job queue full")
-	require.ErrorContains(t, err, "failed with response: 429")
-
-	// 429 GRPC Search
-	grpcClient, err := util2.NewSearchGRPCClient(context.Background(), tempo.Endpoint(3200))
-	require.NoError(t, err)
-
-	resp, err := grpcClient.Search(context.Background(), &tempopb.SearchRequest{
-		Query: "{}",
-		Start: uint32(start),
-		End:   uint32(end),
-	})
-	require.NoError(t, err)
-
-	// loop until we get io.EOF or an error
-	for {
-		_, err = resp.Recv()
-		if err != nil {
-			break
-		}
-	}
-	require.ErrorContains(t, err, "job queue full")
-	require.ErrorContains(t, err, "code = ResourceExhausted")
-}

integration/e2e/metrics_generator_test.go

@@ -1,552 +0,0 @@
-package e2e
-
-import (
-	"context"
-	"fmt"
-	"io"
-	"math/rand"
-	"net/http"
-	"net/url"
-	"strings"
-	"testing"
-	"time"
-
-	"github.com/grafana/e2e"
-	"github.com/grafana/tempo/integration/util"
-	thrift "github.com/jaegertracing/jaeger-idl/thrift-gen/jaeger"
-	io_prometheus_client "github.com/prometheus/client_model/go"
-	"github.com/prometheus/common/expfmt"
-	"github.com/prometheus/common/model"
-	"github.com/prometheus/prometheus/model/labels"
-	"github.com/stretchr/testify/assert"
-	"github.com/stretchr/testify/require"
-)
-
-const (
-	configMetricsGenerator                = "config-metrics-generator.yaml"
-	configMetricsGeneratorTargetInfo      = "config-metrics-generator-targetinfo.yaml"
-	configMetricsGeneratorMessagingSystem = "config-metrics-generator-messaging-system.yaml"
-)
-
-func TestMetricsGenerator(t *testing.T) {
-	s, err := e2e.NewScenario("tempo_e2e")
-	require.NoError(t, err)
-	defer s.Close()
-
-	require.NoError(t, util.CopyFileToSharedDir(s, configMetricsGenerator, "config.yaml"))
-	tempoDistributor := util.NewTempoDistributor()
-	tempoIngester := util.NewTempoIngester(1)
-	tempoMetricsGenerator := util.NewTempoMetricsGenerator()
-	prometheus := util.NewPrometheus()
-	require.NoError(t, s.StartAndWaitReady(tempoDistributor, tempoIngester, tempoMetricsGenerator, prometheus))
-
-	// Wait until ingester and metrics-generator are active
-	isServiceActiveMatcher := func(service string) []*labels.Matcher {
-		return []*labels.Matcher{
-			labels.MustNewMatcher(labels.MatchEqual, "name", service),
-			labels.MustNewMatcher(labels.MatchEqual, "state", "ACTIVE"),
-		}
-	}
-	require.NoError(t, tempoDistributor.WaitSumMetricsWithOptions(e2e.Equals(1), []string{`tempo_ring_members`}, e2e.WithLabelMatchers(isServiceActiveMatcher("ingester")...), e2e.WaitMissingMetrics))
-	require.NoError(t, tempoDistributor.WaitSumMetricsWithOptions(e2e.Equals(1), []string{`tempo_ring_members`}, e2e.WithLabelMatchers(isServiceActiveMatcher("metrics-generator")...), e2e.WaitMissingMetrics))
-
-	// Get port for the Jaeger gRPC receiver endpoint
-	c, err := util.NewJaegerToOTLPExporter(tempoDistributor.Endpoint(4317))
-	require.NoError(t, err)
-	require.NotNil(t, c)
-
-	// Send two spans that have a client-server relationship
-	r := rand.New(rand.NewSource(time.Now().UnixMilli()))
-	traceIDLow := r.Int63()
-	traceIDHigh := r.Int63()
-	parentSpanID := r.Int63()
-
-	err = c.EmitBatch(context.Background(), &thrift.Batch{
-		Process: &thrift.Process{ServiceName: "lb"},
-		Spans: []*thrift.Span{
-			{
-				TraceIdLow:    traceIDLow,
-				TraceIdHigh:   traceIDHigh,
-				SpanId:        parentSpanID,
-				ParentSpanId:  0,
-				OperationName: "lb-get",
-				StartTime:     time.Now().UnixMicro(),
-				Duration:      int64(2 * time.Second / time.Microsecond),
-				Tags:          []*thrift.Tag{{Key: "span.kind", VStr: stringPtr("client")}},
-			},
-		},
-	})
-	require.NoError(t, err)
-
-	err = c.EmitBatch(context.Background(), &thrift.Batch{
-		Process: &thrift.Process{ServiceName: "app"},
-		Spans: []*thrift.Span{
-			{
-				TraceIdLow:    traceIDLow,
-				TraceIdHigh:   traceIDHigh,
-				SpanId:        r.Int63(),
-				ParentSpanId:  parentSpanID,
-				OperationName: "app-handle",
-				StartTime:     time.Now().UnixMicro(),
-				Duration:      int64(1 * time.Second / time.Microsecond),
-				Tags:          []*thrift.Tag{{Key: "span.kind", VStr: stringPtr("server")}},
-			},
-		},
-	})
-	require.NoError(t, err)
-
-	// also send one with 5 minutes old timestamp
-	err = c.EmitBatch(context.Background(), &thrift.Batch{
-		Process: &thrift.Process{ServiceName: "app"},
-		Spans: []*thrift.Span{
-			{
-				TraceIdLow:    traceIDLow,
-				TraceIdHigh:   traceIDHigh,
-				SpanId:        r.Int63(),
-				ParentSpanId:  parentSpanID,
-				OperationName: "app-handle",
-				StartTime:     time.Now().Add(-5 * time.Minute).UnixMicro(),
-				Duration:      int64(1 * time.Second / time.Microsecond),
-				Tags:          []*thrift.Tag{{Key: "span.kind", VStr: stringPtr("server")}},
-			},
-		},
-	})
-	require.NoError(t, err)
-
-	// also send one with timestamp 10 days in the future
-	err = c.EmitBatch(context.Background(), &thrift.Batch{
-		Process: &thrift.Process{ServiceName: "app"},
-		Spans: []*thrift.Span{
-			{
-				TraceIdLow:    traceIDLow,
-				TraceIdHigh:   traceIDHigh,
-				SpanId:        r.Int63(),
-				ParentSpanId:  parentSpanID,
-				OperationName: "app-handle",
-				StartTime:     time.Now().Add(10 * 24 * time.Hour).UnixMicro(),
-				Duration:      int64(1 * time.Second / time.Microsecond),
-				Tags:          []*thrift.Tag{{Key: "span.kind", VStr: stringPtr("server")}},
-			},
-		},
-	})
-	require.NoError(t, err)
-
-	// also send one with an invalid label value
-	err = c.EmitBatch(context.Background(), &thrift.Batch{
-		Process: &thrift.Process{ServiceName: "app"},
-		Spans: []*thrift.Span{
-			{
-				TraceIdLow:    traceIDLow,
-				TraceIdHigh:   traceIDHigh,
-				SpanId:        r.Int63(),
-				ParentSpanId:  parentSpanID,
-				OperationName: "\xff\xff",
-				StartTime:     time.Now().UnixMicro(),
-				Duration:      int64(2 * time.Second / time.Microsecond),
-				Tags:          []*thrift.Tag{{Key: "span.kind", VStr: stringPtr("server")}},
-			},
-		},
-	})
-	require.NoError(t, err)
-
-	// Fetch metrics from Prometheus once they are received
-	var metricFamilies map[string]*io_prometheus_client.MetricFamily
-	for {
-		metricFamilies, err = extractMetricsFromPrometheus(prometheus, `{__name__=~"traces_.+"}`)
-		require.NoError(t, err)
-		if len(metricFamilies) > 0 {
-			break
-		}
-		time.Sleep(time.Second)
-	}
-
-	// Print collected metrics for easier debugging
-	fmt.Println()
-	for key, family := range metricFamilies {
-		fmt.Println(key)
-		for _, metric := range family.Metric {
-			fmt.Println(metric)
-		}
-	}
-	fmt.Println()
-
-	// Service graphs
-	lbls := []string{"client", "lb", "server", "app"}
-	assert.Equal(t, 1.0, sumValues(metricFamilies, "traces_service_graph_request_total", lbls))
-
-	assert.Equal(t, 0.0, sumValues(metricFamilies, "traces_service_graph_request_client_seconds_bucket", append(lbls, "le", "1")))
-	assert.Equal(t, 1.0, sumValues(metricFamilies, "traces_service_graph_request_client_seconds_bucket", append(lbls, "le", "2")))
-	assert.Equal(t, 1.0, sumValues(metricFamilies, "traces_service_graph_request_client_seconds_bucket", append(lbls, "le", "+Inf")))
-	assert.Equal(t, 1.0, sumValues(metricFamilies, "traces_service_graph_request_client_seconds_count", lbls))
-	assert.Equal(t, 2.0, sumValues(metricFamilies, "traces_service_graph_request_client_seconds_sum", lbls))
-
-	assert.Equal(t, 1.0, sumValues(metricFamilies, "traces_service_graph_request_server_seconds_bucket", append(lbls, "le", "1")))
-	assert.Equal(t, 1.0, sumValues(metricFamilies, "traces_service_graph_request_server_seconds_bucket", append(lbls, "le", "2")))
-	assert.Equal(t, 1.0, sumValues(metricFamilies, "traces_service_graph_request_server_seconds_bucket", append(lbls, "le", "+Inf")))
-	assert.Equal(t, 1.0, sumValues(metricFamilies, "traces_service_graph_request_server_seconds_count", lbls))
-	assert.Equal(t, 1.0, sumValues(metricFamilies, "traces_service_graph_request_server_seconds_sum", lbls))
-
-	assert.Equal(t, 0.0, sumValues(metricFamilies, "traces_service_graph_request_failed_total", nil))
-	assert.Equal(t, 0.0, sumValues(metricFamilies, "traces_service_graph_unpaired_spans_total", nil))
-	assert.Equal(t, 0.0, sumValues(metricFamilies, "traces_service_graph_dropped_spans_total", nil))
-
-	// Span metrics
-	lbls = []string{"service", "lb", "span_name", "lb-get", "span_kind", "SPAN_KIND_CLIENT", "status_code", "STATUS_CODE_UNSET"}
-	assert.Equal(t, 1.0, sumValues(metricFamilies, "traces_spanmetrics_calls_total", lbls))
-	assert.NotEqual(t, 0, sumValues(metricFamilies, "traces_spanmetrics_size_total", lbls))
-	assert.Equal(t, 0.0, sumValues(metricFamilies, "traces_spanmetrics_latency_bucket", append(lbls, "le", "1")))
-	assert.Equal(t, 1.0, sumValues(metricFamilies, "traces_spanmetrics_latency_bucket", append(lbls, "le", "2")))
-	assert.Equal(t, 1.0, sumValues(metricFamilies, "traces_spanmetrics_latency_bucket", append(lbls, "le", "+Inf")))
-	assert.Equal(t, 1.0, sumValues(metricFamilies, "traces_spanmetrics_latency_count", lbls))
-	assert.Equal(t, 2.0, sumValues(metricFamilies, "traces_spanmetrics_latency_sum", lbls))
-
-	lbls = []string{"service", "app", "span_name", "app-handle", "span_kind", "SPAN_KIND_SERVER", "status_code", "STATUS_CODE_UNSET"}
-	assert.Equal(t, 1.0, sumValues(metricFamilies, "traces_spanmetrics_calls_total", lbls))
-	assert.NotEqual(t, 0, sumValues(metricFamilies, "traces_spanmetrics_size_total", lbls))
-	assert.Equal(t, 1.0, sumValues(metricFamilies, "traces_spanmetrics_latency_bucket", append(lbls, "le", "1")))
-	assert.Equal(t, 1.0, sumValues(metricFamilies, "traces_spanmetrics_latency_bucket", append(lbls, "le", "2")))
-	assert.Equal(t, 1.0, sumValues(metricFamilies, "traces_spanmetrics_latency_bucket", append(lbls, "le", "+Inf")))
-	assert.Equal(t, 1.0, sumValues(metricFamilies, "traces_spanmetrics_latency_count", lbls))
-	assert.Equal(t, 1.0, sumValues(metricFamilies, "traces_spanmetrics_latency_sum", lbls))
-
-	// Verify metrics
-	assert.NoError(t, tempoMetricsGenerator.WaitSumMetrics(e2e.Equals(5), "tempo_metrics_generator_spans_received_total"))
-	assert.NoError(t, tempoMetricsGenerator.WaitSumMetrics(e2e.Equals(3), "tempo_metrics_generator_spans_discarded_total"))
-	assert.NoError(t, tempoMetricsGenerator.WaitSumMetrics(e2e.Equals(25), "tempo_metrics_generator_registry_active_series"))
-	assert.NoError(t, tempoMetricsGenerator.WaitSumMetrics(e2e.Equals(1000), "tempo_metrics_generator_registry_max_active_series"))
-	assert.NoError(t, tempoMetricsGenerator.WaitSumMetrics(e2e.Equals(25), "tempo_metrics_generator_registry_series_added_total"))
-}
-
-func TestMetricsGeneratorTargetInfoEnabled(t *testing.T) {
-	s, err := e2e.NewScenario("tempo_e2e")
-	require.NoError(t, err)
-	defer s.Close()
-
-	require.NoError(t, util.CopyFileToSharedDir(s, configMetricsGeneratorTargetInfo, "config.yaml"))
-	tempoDistributor := util.NewTempoDistributor()
-	tempoIngester := util.NewTempoIngester(1)
-	tempoMetricsGenerator := util.NewTempoMetricsGenerator()
-	prometheus := util.NewPrometheus()
-	require.NoError(t, s.StartAndWaitReady(tempoDistributor, tempoIngester, tempoMetricsGenerator, prometheus))
-
-	// Wait until ingester and metrics-generator are active
-	isServiceActiveMatcher := func(service string) []*labels.Matcher {
-		return []*labels.Matcher{
-			labels.MustNewMatcher(labels.MatchEqual, "name", service),
-			labels.MustNewMatcher(labels.MatchEqual, "state", "ACTIVE"),
-		}
-	}
-	require.NoError(t, tempoDistributor.WaitSumMetricsWithOptions(e2e.Equals(1), []string{`tempo_ring_members`}, e2e.WithLabelMatchers(isServiceActiveMatcher("ingester")...), e2e.WaitMissingMetrics))
-	require.NoError(t, tempoDistributor.WaitSumMetricsWithOptions(e2e.Equals(1), []string{`tempo_ring_members`}, e2e.WithLabelMatchers(isServiceActiveMatcher("metrics-generator")...), e2e.WaitMissingMetrics))
-
-	// Get port for the Jaeger gRPC receiver endpoint
-	c, err := util.NewJaegerToOTLPExporter(tempoDistributor.Endpoint(4317))
-	require.NoError(t, err)
-	require.NotNil(t, c)
-
-	// Send two spans that have a client-server relationship
-	r := rand.New(rand.NewSource(time.Now().UnixMilli()))
-	traceIDLow := r.Int63()
-	traceIDHigh := r.Int63()
-	parentSpanID := r.Int63()
-
-	err = c.EmitBatch(context.Background(), &thrift.Batch{
-		Process: &thrift.Process{ServiceName: "lb"},
-		Spans: []*thrift.Span{
-			{
-				TraceIdLow:    traceIDLow,
-				TraceIdHigh:   traceIDHigh,
-				SpanId:        parentSpanID,
-				ParentSpanId:  0,
-				OperationName: "lb-get",
-				StartTime:     time.Now().UnixMicro(),
-				Duration:      int64(2 * time.Second / time.Microsecond),
-				Tags:          []*thrift.Tag{{Key: "span.kind", VStr: stringPtr("client")}},
-			},
-		},
-	})
-	require.NoError(t, err)
-
-	err = c.EmitBatch(context.Background(), &thrift.Batch{
-		Process: &thrift.Process{ServiceName: "app"},
-		Spans: []*thrift.Span{
-			{
-				TraceIdLow:    traceIDLow,
-				TraceIdHigh:   traceIDHigh,
-				SpanId:        r.Int63(),
-				ParentSpanId:  parentSpanID,
-				OperationName: "app-handle",
-				StartTime:     time.Now().UnixMicro(),
-				Duration:      int64(1 * time.Second / time.Microsecond),
-				Tags:          []*thrift.Tag{{Key: "span.kind", VStr: stringPtr("server")}},
-			},
-		},
-	})
-	require.NoError(t, err)
-
-	// also send one with 5 minutes old timestamp
-	err = c.EmitBatch(context.Background(), &thrift.Batch{
-		Process: &thrift.Process{ServiceName: "app"},
-		Spans: []*thrift.Span{
-			{
-				TraceIdLow:    traceIDLow,
-				TraceIdHigh:   traceIDHigh,
-				SpanId:        r.Int63(),
-				ParentSpanId:  parentSpanID,
-				OperationName: "app-handle",
-				StartTime:     time.Now().Add(-5 * time.Minute).UnixMicro(),
-				Duration:      int64(1 * time.Second / time.Microsecond),
-				Tags:          []*thrift.Tag{{Key: "span.kind", VStr: stringPtr("server")}},
-			},
-		},
-	})
-	require.NoError(t, err)
-
-	// also send one with timestamp 10 days in the future
-	err = c.EmitBatch(context.Background(), &thrift.Batch{
-		Process: &thrift.Process{ServiceName: "app"},
-		Spans: []*thrift.Span{
-			{
-				TraceIdLow:    traceIDLow,
-				TraceIdHigh:   traceIDHigh,
-				SpanId:        r.Int63(),
-				ParentSpanId:  parentSpanID,
-				OperationName: "app-handle",
-				StartTime:     time.Now().Add(10 * 24 * time.Hour).UnixMicro(),
-				Duration:      int64(1 * time.Second / time.Microsecond),
-				Tags:          []*thrift.Tag{{Key: "span.kind", VStr: stringPtr("server")}},
-			},
-		},
-	})
-	require.NoError(t, err)
-
-	// Fetch metrics from Prometheus once they are received
-	var metricFamilies map[string]*io_prometheus_client.MetricFamily
-	for {
-		metricFamilies, err = extractMetricsFromPrometheus(prometheus, `{__name__=~"traces_.+"}`)
-		require.NoError(t, err)
-		if len(metricFamilies) > 0 {
-			break
-		}
-		time.Sleep(time.Second)
-	}
-
-	// Print collected metrics for easier debugging
-	fmt.Println()
-	for key, family := range metricFamilies {
-		fmt.Println(key)
-		for _, metric := range family.Metric {
-			fmt.Println(metric)
-		}
-	}
-	fmt.Println()
-
-	// Service graphs
-	lbls := []string{"client", "lb", "server", "app"}
-	assert.Equal(t, 1.0, sumValues(metricFamilies, "traces_service_graph_request_total", lbls))
-
-	assert.Equal(t, 0.0, sumValues(metricFamilies, "traces_service_graph_request_client_seconds_bucket", append(lbls, "le", "1")))
-	assert.Equal(t, 1.0, sumValues(metricFamilies, "traces_service_graph_request_client_seconds_bucket", append(lbls, "le", "2")))
-	assert.Equal(t, 1.0, sumValues(metricFamilies, "traces_service_graph_request_client_seconds_bucket", append(lbls, "le", "+Inf")))
-	assert.Equal(t, 1.0, sumValues(metricFamilies, "traces_service_graph_request_client_seconds_count", lbls))
-	assert.Equal(t, 2.0, sumValues(metricFamilies, "traces_service_graph_request_client_seconds_sum", lbls))
-
-	assert.Equal(t, 1.0, sumValues(metricFamilies, "traces_service_graph_request_server_seconds_bucket", append(lbls, "le", "1")))
-	assert.Equal(t, 1.0, sumValues(metricFamilies, "traces_service_graph_request_server_seconds_bucket", append(lbls, "le", "2")))
-	assert.Equal(t, 1.0, sumValues(metricFamilies, "traces_service_graph_request_server_seconds_bucket", append(lbls, "le", "+Inf")))
-	assert.Equal(t, 1.0, sumValues(metricFamilies, "traces_service_graph_request_server_seconds_count", lbls))
-	assert.Equal(t, 1.0, sumValues(metricFamilies, "traces_service_graph_request_server_seconds_sum", lbls))
-
-	assert.Equal(t, 0.0, sumValues(metricFamilies, "traces_service_graph_request_failed_total", nil))
-	assert.Equal(t, 0.0, sumValues(metricFamilies, "traces_service_graph_unpaired_spans_total", nil))
-	assert.Equal(t, 0.0, sumValues(metricFamilies, "traces_service_graph_dropped_spans_total", nil))
-
-	// Span metrics
-	lbls = []string{"service", "lb", "span_name", "lb-get", "span_kind", "SPAN_KIND_CLIENT", "status_code", "STATUS_CODE_UNSET", "job", "lb"}
-	assert.Equal(t, 1.0, sumValues(metricFamilies, "traces_spanmetrics_calls_total", lbls))
-	assert.NotEqual(t, 0, sumValues(metricFamilies, "traces_spanmetrics_size_total", lbls))
-	assert.Equal(t, 0.0, sumValues(metricFamilies, "traces_spanmetrics_latency_bucket", append(lbls, "le", "1")))
-	assert.Equal(t, 1.0, sumValues(metricFamilies, "traces_spanmetrics_latency_bucket", append(lbls, "le", "2")))
-	assert.Equal(t, 1.0, sumValues(metricFamilies, "traces_spanmetrics_latency_bucket", append(lbls, "le", "+Inf")))
-	assert.Equal(t, 1.0, sumValues(metricFamilies, "traces_spanmetrics_latency_count", lbls))
-	assert.Equal(t, 2.0, sumValues(metricFamilies, "traces_spanmetrics_latency_sum", lbls))
-
-	lbls = []string{"service", "app", "span_name", "app-handle", "span_kind", "SPAN_KIND_SERVER", "status_code", "STATUS_CODE_UNSET", "job", "app"}
-	assert.Equal(t, 1.0, sumValues(metricFamilies, "traces_spanmetrics_calls_total", lbls))
-	assert.NotEqual(t, 0, sumValues(metricFamilies, "traces_spanmetrics_size_total", lbls))
-	assert.Equal(t, 1.0, sumValues(metricFamilies, "traces_spanmetrics_latency_bucket", append(lbls, "le", "1")))
-	assert.Equal(t, 1.0, sumValues(metricFamilies, "traces_spanmetrics_latency_bucket", append(lbls, "le", "2")))
-	assert.Equal(t, 1.0, sumValues(metricFamilies, "traces_spanmetrics_latency_bucket", append(lbls, "le", "+Inf")))
-	assert.Equal(t, 1.0, sumValues(metricFamilies, "traces_spanmetrics_latency_count", lbls))
-	assert.Equal(t, 1.0, sumValues(metricFamilies, "traces_spanmetrics_latency_sum", lbls))
-
-	// Verify metrics
-	assert.NoError(t, tempoMetricsGenerator.WaitSumMetrics(e2e.Equals(4), "tempo_metrics_generator_spans_received_total"))
-	assert.NoError(t, tempoMetricsGenerator.WaitSumMetrics(e2e.Equals(2), "tempo_metrics_generator_spans_discarded_total"))
-	assert.NoError(t, tempoMetricsGenerator.WaitSumMetrics(e2e.Equals(25), "tempo_metrics_generator_registry_active_series"))
-	assert.NoError(t, tempoMetricsGenerator.WaitSumMetrics(e2e.Equals(1000), "tempo_metrics_generator_registry_max_active_series"))
-	assert.NoError(t, tempoMetricsGenerator.WaitSumMetrics(e2e.Equals(25), "tempo_metrics_generator_registry_series_added_total"))
-}
-
-func TestMetricsGeneratorMessagingSystemLatencyHistogramEnabled(t *testing.T) {
-	s, err := e2e.NewScenario("tempo_e2e")
-	require.NoError(t, err)
-	defer s.Close()
-
-	// Use a config that enables the messaging system latency histogram
-	require.NoError(t, util.CopyFileToSharedDir(s, configMetricsGeneratorMessagingSystem, "config.yaml"))
-	tempoDistributor := util.NewTempoDistributor()
-	tempoIngester := util.NewTempoIngester(1)
-	tempoMetricsGenerator := util.NewTempoMetricsGenerator()
-	prometheus := util.NewPrometheus()
-	require.NoError(t, s.StartAndWaitReady(tempoDistributor, tempoIngester, tempoMetricsGenerator, prometheus))
-
-	isServiceActiveMatcher := func(service string) []*labels.Matcher {
-		return []*labels.Matcher{
-			labels.MustNewMatcher(labels.MatchEqual, "name", service),
-			labels.MustNewMatcher(labels.MatchEqual, "state", "ACTIVE"),
-		}
-	}
-	require.NoError(t, tempoDistributor.WaitSumMetricsWithOptions(e2e.Equals(1), []string{"tempo_ring_members"}, e2e.WithLabelMatchers(isServiceActiveMatcher("ingester")...), e2e.WaitMissingMetrics))
-	require.NoError(t, tempoDistributor.WaitSumMetricsWithOptions(e2e.Equals(1), []string{"tempo_ring_members"}, e2e.WithLabelMatchers(isServiceActiveMatcher("metrics-generator")...), e2e.WaitMissingMetrics))
-
-	c, err := util.NewJaegerToOTLPExporter(tempoDistributor.Endpoint(4317))
-	require.NoError(t, err)
-	require.NotNil(t, c)
-
-	// Send a pair of spans with a messaging system relationship (producer -> consumer)
-	// ignore the gosec linter because we are using a random number generator to create trace IDs
-	// and span IDs, which is not a security risk in this context.
-	// #nosec G404 -- nosemgrep: math-random-used
-	r := rand.New(rand.NewSource(time.Now().UnixMilli()))
-	traceIDLow := r.Int63()
-	traceIDHigh := r.Int63()
-	parentSpanID := r.Int63()
-
-	// Set explicit times to ensure producer ends before consumer starts
-	now := time.Now()
-	producerStart := now
-	producerDuration := 2 * time.Second
-	producerEnd := producerStart.Add(producerDuration)
-	consumerStart := producerEnd.Add(1 * time.Second) // ensure consumer starts after producer ends
-	consumerDuration := 2 * time.Second
-
-	// Producer span
-	err = c.EmitBatch(context.Background(), &thrift.Batch{
-		Process: &thrift.Process{ServiceName: "producer"},
-		Spans: []*thrift.Span{
-			{
-				TraceIdLow:    traceIDLow,
-				TraceIdHigh:   traceIDHigh,
-				SpanId:        parentSpanID,
-				ParentSpanId:  0,
-				OperationName: "produce-message",
-				StartTime:     producerStart.UnixMicro(),
-				Duration:      int64(producerDuration / time.Microsecond),
-				Tags: []*thrift.Tag{
-					{Key: "span.kind", VStr: stringPtr("producer")},
-					{Key: "messaging.system", VStr: stringPtr("kafka")},
-				},
-			},
-		},
-	})
-	require.NoError(t, err)
-
-	// Consumer span
-	err = c.EmitBatch(context.Background(), &thrift.Batch{
-		Process: &thrift.Process{ServiceName: "consumer"},
-		Spans: []*thrift.Span{
-			{
-				TraceIdLow:    traceIDLow,
-				TraceIdHigh:   traceIDHigh,
-				SpanId:        r.Int63(),
-				ParentSpanId:  parentSpanID,
-				OperationName: "consume-message",
-				StartTime:     consumerStart.UnixMicro(),
-				Duration:      int64(consumerDuration / time.Microsecond),
-				Tags: []*thrift.Tag{
-					{Key: "span.kind", VStr: stringPtr("consumer")},
-					{Key: "messaging.system", VStr: stringPtr("kafka")},
-				},
-			},
-		},
-	})
-	require.NoError(t, err)
-
-	// Wait for the metric to be produced
-	var metricFamilies map[string]*io_prometheus_client.MetricFamily
-	for {
-		metricFamilies, err = extractMetricsFromPrometheus(prometheus, `{__name__=~"traces_.+"}`)
-		require.NoError(t, err)
-		if _, ok := metricFamilies["traces_service_graph_request_messaging_system_seconds_bucket"]; ok {
-			break
-		}
-		time.Sleep(30 * time.Second)
-	}
-
-	// Check that the metric exists and has a non-zero count
-	lbls := []string{"client", "producer", "server", "consumer", "connection_type", "messaging_system"}
-	assert.NotEqual(t, 0.0, sumValues(metricFamilies, "traces_service_graph_request_messaging_system_seconds_count", lbls))
-	assert.NotEqual(t, 0.0, sumValues(metricFamilies, "traces_service_graph_request_messaging_system_seconds_sum", lbls))
-}
-
-// extractMetricsFromPrometheus extracts metrics stored in Prometheus using the /federate endpoint.
-func extractMetricsFromPrometheus(prometheus *e2e.HTTPService, matcher string) (map[string]*io_prometheus_client.MetricFamily, error) {
-	url := fmt.Sprintf("http://%s/federate?match[]=%s", prometheus.HTTPEndpoint(), url.QueryEscape(matcher))
-
-	res, err := http.Get(url)
-	if err != nil {
-		return nil, err
-	}
-	defer res.Body.Close()
-
-	if res.StatusCode < 200 || res.StatusCode >= 300 {
-		return nil, fmt.Errorf("unexpected status code %d while fetching federate metrics", res.StatusCode)
-	}
-
-	body, err := io.ReadAll(res.Body)
-	if err != nil {
-		return nil, err
-	}
-
-	tp := expfmt.NewTextParser(model.UTF8Validation)
-	return tp.TextToMetricFamilies(strings.NewReader(string(body)))
-}
-
-// sumValues calculates the sum of all metrics in the metricFamily that contain the given labels.
-// filterLabels must be key-value pairs.
-func sumValues(metricFamily map[string]*io_prometheus_client.MetricFamily, metric string, filterLabels []string) float64 {
-	if len(filterLabels)%2 != 0 {
-		panic(fmt.Sprintf("filterLabels must be pairs: %v", filterLabels))
-	}
-	filterLabelsMap := map[string]string{}
-	for i := 0; i < len(filterLabels); i += 2 {
-		filterLabelsMap[filterLabels[i]] = filterLabels[i+1]
-	}
-
-	sum := 0.0
-
-outer:
-	for _, metric := range metricFamily[metric].GetMetric() {
-		labelMap := map[string]string{}
-		for _, label := range metric.GetLabel() {
-			labelMap[label.GetName()] = label.GetValue()
-		}
-
-		for key, expectedValue := range filterLabelsMap {
-			value, ok := labelMap[key]
-			if !ok || value != expectedValue {
-				continue outer
-			}
-		}
-
-		// since we fetch metrics using /federate they are all untyped
-		sum += metric.GetUntyped().GetValue()
-	}
-
-	return sum
-}
-
-func stringPtr(s string) *string {
-	return &s
-}

integration/e2e/receivers_test.go

@@ -1,154 +0,0 @@
-package e2e
-
-import (
-	"context"
-	crand "crypto/rand"
-	"testing"
-
-	"github.com/grafana/dskit/user"
-	"github.com/grafana/e2e"
-	"github.com/open-telemetry/opentelemetry-collector-contrib/exporter/zipkinexporter"
-	"github.com/stretchr/testify/assert"
-	"github.com/stretchr/testify/require"
-	"go.opentelemetry.io/collector/component"
-	"go.opentelemetry.io/collector/component/componenttest"
-	"go.opentelemetry.io/collector/config/configgrpc"
-	"go.opentelemetry.io/collector/config/confighttp"
-	"go.opentelemetry.io/collector/config/configtls"
-	"go.opentelemetry.io/collector/exporter"
-	"go.opentelemetry.io/collector/exporter/otlpexporter"
-	"go.opentelemetry.io/collector/pdata/ptrace"
-	metricnoop "go.opentelemetry.io/otel/metric/noop"
-	tracenoop "go.opentelemetry.io/otel/trace/noop"
-	"go.uber.org/zap"
-
-	"github.com/grafana/tempo/integration/util"
-	"github.com/grafana/tempo/pkg/httpclient"
-	tempoUtil "github.com/grafana/tempo/pkg/util"
-	"github.com/grafana/tempo/pkg/util/test"
-)
-
-const (
-	configAllInOneLocal = "deployments/config-all-in-one-local.yaml"
-)
-
-func TestReceivers(t *testing.T) {
-	s, err := e2e.NewScenario("tempo_e2e")
-	require.NoError(t, err)
-	defer s.Close()
-
-	require.NoError(t, util.CopyFileToSharedDir(s, configAllInOneLocal, "config.yaml"))
-	tempo := util.NewTempoAllInOne()
-	require.NoError(t, s.StartAndWaitReady(tempo))
-
-	testReceivers := []struct {
-		name string
-
-		factory  exporter.Factory
-		config   func(exporter.Factory, string) component.Config
-		endpoint string
-	}{
-		{
-			"otlp gRPC",
-			otlpexporter.NewFactory(),
-			func(factory exporter.Factory, endpoint string) component.Config {
-				exporterCfg := factory.CreateDefaultConfig()
-				otlpCfg := exporterCfg.(*otlpexporter.Config)
-				otlpCfg.ClientConfig = configgrpc.ClientConfig{
-					Endpoint: endpoint,
-					TLS: configtls.ClientConfig{
-						Insecure: true,
-					},
-				}
-				return otlpCfg
-			},
-			tempo.Endpoint(4317),
-		},
-		{
-			"zipkin",
-			zipkinexporter.NewFactory(),
-			func(factory exporter.Factory, endpoint string) component.Config {
-				exporterCfg := factory.CreateDefaultConfig()
-				zipkinCfg := exporterCfg.(*zipkinexporter.Config)
-				zipkinCfg.ClientConfig = confighttp.ClientConfig{
-					Endpoint: endpoint,
-					TLS: configtls.ClientConfig{
-						Insecure: true,
-					},
-				}
-				zipkinCfg.Format = "json"
-				return zipkinCfg
-			},
-			"http://" + tempo.Endpoint(9411),
-		},
-	}
-
-	for _, tc := range testReceivers {
-		t.Run(tc.name, func(t *testing.T) {
-			// create exporter
-			logger, _ := zap.NewDevelopment()
-			exporter, err := tc.factory.CreateTraces(
-				context.Background(),
-				exporter.Settings{
-					ID: component.NewID(tc.factory.Type()),
-					TelemetrySettings: component.TelemetrySettings{
-						Logger:         logger,
-						TracerProvider: tracenoop.NewTracerProvider(),
-						MeterProvider:  metricnoop.NewMeterProvider(),
-					},
-					BuildInfo: component.NewDefaultBuildInfo(),
-				},
-				tc.config(tc.factory, tc.endpoint),
-			)
-			require.NoError(t, err)
-
-			err = exporter.Start(context.Background(), componenttest.NewNopHost())
-			require.NoError(t, err)
-
-			// make request
-			traceID := make([]byte, 16)
-			_, err = crand.Read(traceID)
-			require.NoError(t, err)
-			req := test.MakeTrace(20, traceID)
-
-			// zipkin doesn't support events and will 400 if you attempt to push one. just strip
-			// all events from the trace here
-			if tc.name == "zipkin" {
-				for _, b := range req.ResourceSpans {
-					for _, ss := range b.ScopeSpans {
-						for _, s := range ss.Spans {
-							s.Events = nil
-						}
-					}
-				}
-			}
-
-			b, err := req.Marshal()
-			require.NoError(t, err)
-
-			// unmarshal into otlp proto
-			traces, err := (&ptrace.ProtoUnmarshaler{}).UnmarshalTraces(b)
-			require.NoError(t, err)
-			require.NotNil(t, traces)
-
-			ctx := user.InjectOrgID(context.Background(), tempoUtil.FakeTenantID)
-			ctx, err = user.InjectIntoGRPCRequest(ctx)
-			require.NoError(t, err)
-
-			// send traces to tempo
-			err = exporter.ConsumeTraces(ctx, traces)
-			require.NoError(t, err)
-
-			// shutdown to ensure traces are flushed
-			require.NoError(t, exporter.Shutdown(context.Background()))
-
-			// query for the trace
-			client := httpclient.New("http://"+tempo.Endpoint(3200), tempoUtil.FakeTenantID)
-			trace, err := client.QueryTrace(tempoUtil.TraceIDToHexString(traceID))
-			require.NoError(t, err)
-
-			// just compare spanCount because otel flattens all ILS into one
-			assert.Equal(t, util.SpanCount(req), util.SpanCount(trace))
-		})
-	}
-}

integration/limits/config-ingest-partial-success.yaml

@@ -0,0 +1,8 @@
+overrides:
+  defaults:
+    ingestion:
+      max_traces_per_user: 6
+      rate_limit_bytes: 7000
+      burst_size_bytes: 7000
+    global:
+      max_bytes_per_trace: 600

integration/limits/config-ingest.yaml

@@ -0,0 +1,16 @@
+distributor:
+  retry_after_on_resource_exhausted: 1s
+  receivers:
+    otlp:
+      protocols:
+        http:
+          endpoint: "0.0.0.0:4318"
+
+overrides:
+  defaults:
+    ingestion:
+      max_traces_per_user: 1
+      rate_limit_bytes: 500
+      burst_size_bytes: 500
+    global:
+      max_bytes_per_trace: 130

integration/limits/config-query-rate.yaml

@@ -0,0 +1,2 @@
+query_frontend:
+  max_outstanding_per_tenant: 0 # forces everything to 429

integration/limits/limits_test.go

@@ -0,0 +1,329 @@
+package e2e
+
+import (
+	"bytes"
+	"context"
+	crand "crypto/rand"
+	"encoding/binary"
+	"fmt"
+	"io"
+	"net/http"
+	"testing"
+	"time"
+
+	"github.com/grafana/tempo/modules/overrides"
+	"go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc"
+	"google.golang.org/grpc/codes"
+	"google.golang.org/grpc/status"
+
+	"github.com/grafana/e2e"
+	"github.com/prometheus/prometheus/model/labels"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+	"go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp"
+	"google.golang.org/genproto/googleapis/rpc/errdetails"
+
+	"github.com/grafana/tempo/integration/util"
+	"github.com/grafana/tempo/pkg/model/trace"
+	"github.com/grafana/tempo/pkg/tempopb"
+	tempoUtil "github.com/grafana/tempo/pkg/util"
+	"github.com/grafana/tempo/pkg/util/test"
+)
+
+const (
+	configIngestPartialSucess = "config-ingest-partial-success.yaml"
+	configQueryRate           = "config-query-rate.yaml"
+	configIngest              = "config-ingest.yaml"
+)
+
+func TestIngestionLimits(t *testing.T) {
+	util.RunIntegrationTests(t, util.TestHarnessConfig{
+		ConfigOverlay: configIngest,
+	}, func(h *util.TempoHarness) {
+		h.WaitTracesWritable(t)
+
+		// should fail b/c the trace is too large. each batch should be ~70 bytes
+		batch := util.MakeThriftBatchWithSpanCount(2)
+		require.NoError(t, h.WriteJaegerBatch(batch, ""))
+
+		// push a trace
+		require.NoError(t, h.WriteJaegerBatch(util.MakeThriftBatchWithSpanCount(1), ""))
+
+		// should fail b/c this will be too many traces
+		batch = util.MakeThriftBatch()
+		require.NoError(t, h.WriteJaegerBatch(batch, ""))
+
+		// should fail b/c due to ingestion rate limit
+		batch = util.MakeThriftBatchWithSpanCount(10)
+		err := h.WriteJaegerBatch(batch, "")
+
+		// this error must have a retryinfo as expected in otel collector code: https://github.com/open-telemetry/opentelemetry-collector/blob/d7b49df5d9e922df6ce56ad4b64ee1c79f9dbdbe/exporter/otlpexporter/otlp.go#L172
+		st, ok := status.FromError(err)
+		require.True(t, ok)
+		foundRetryInfo := false
+		for _, detail := range st.Details() {
+			if _, ok := detail.(*errdetails.RetryInfo); ok {
+				foundRetryInfo = true
+				break
+			}
+		}
+		require.True(t, foundRetryInfo)
+
+		// test limit metrics
+		liveStore := h.Services[util.ServiceLiveStoreZoneA]
+		err = liveStore.WaitSumMetricsWithOptions(e2e.Equals(2),
+			[]string{"tempo_discarded_spans_total"},
+			e2e.WithLabelMatchers(labels.MustNewMatcher(labels.MatchEqual, "reason", "trace_too_large")),
+			e2e.WaitMissingMetrics,
+		)
+		require.NoError(t, err)
+		err = liveStore.WaitSumMetricsWithOptions(e2e.Equals(1),
+			[]string{"tempo_discarded_spans_total"},
+			e2e.WithLabelMatchers(labels.MustNewMatcher(labels.MatchEqual, "reason", "live_traces_exceeded")),
+			e2e.WaitMissingMetrics,
+		)
+		require.NoError(t, err)
+		err = h.Services[util.ServiceDistributor].WaitSumMetricsWithOptions(e2e.Equals(10),
+			[]string{"tempo_discarded_spans_total"},
+			e2e.WithLabelMatchers(labels.MustNewMatcher(labels.MatchEqual, "reason", "rate_limited")),
+			e2e.WaitMissingMetrics,
+		)
+		require.NoError(t, err)
+	})
+}
+
+func TestOTLPLimits(t *testing.T) {
+	util.RunIntegrationTests(t, util.TestHarnessConfig{
+		ConfigOverlay: configIngest,
+	}, func(h *util.TempoHarness) {
+		h.WaitTracesWritable(t)
+		protoSpans := test.MakeProtoSpans(100)
+
+		// gRPC
+		grpcClient := otlptracegrpc.NewClient(
+			otlptracegrpc.WithEndpoint(h.Services[util.ServiceDistributor].Endpoint(4317)),
+			otlptracegrpc.WithInsecure(),
+			otlptracegrpc.WithRetry(otlptracegrpc.RetryConfig{Enabled: false}),
+		)
+		require.NoError(t, grpcClient.Start(context.Background()))
+
+		grpcErr := grpcClient.UploadTraces(context.Background(), protoSpans)
+		assert.Error(t, grpcErr)
+		require.Equal(t, codes.ResourceExhausted, status.Code(grpcErr))
+
+		// HTTP
+		httpClient := otlptracehttp.NewClient(
+			otlptracehttp.WithEndpoint(h.Services[util.ServiceDistributor].Endpoint(4318)),
+			otlptracehttp.WithInsecure(),
+			otlptracehttp.WithRetry(otlptracehttp.RetryConfig{Enabled: false}),
+		)
+		require.NoError(t, httpClient.Start(context.Background()))
+
+		httpErr := httpClient.UploadTraces(context.Background(), protoSpans)
+		assert.Error(t, httpErr)
+		require.Contains(t, httpErr.Error(), "retry-able request failure")
+	})
+}
+
+func TestOTLPLimitsVanillaClient(t *testing.T) {
+	util.RunIntegrationTests(t, util.TestHarnessConfig{
+		ConfigOverlay: configIngest,
+	}, func(h *util.TempoHarness) {
+		h.WaitTracesWritable(t)
+
+		trace := test.MakeTrace(10, []byte{1, 2, 3, 4, 5, 6, 7, 8, 9, 10})
+
+		testCases := []struct {
+			name    string
+			payload func() []byte
+			headers map[string]string
+		}{
+			// TODO There is an issue when sending the payload in json format. The server returns a 200 instead of a 429.
+			// {
+			// 	"JSON format",
+			// 	func() []byte {
+			// 		b := &bytes.Buffer{}
+			// 		err := (&jsonpb.Marshaler{}).Marshal(b, trace)
+			// 		require.NoError(t, err)
+			// 		return b.Bytes()
+			// 	},
+			// 	map[string]string{
+			// 		"Content-Type": "application/json",
+			// 	},
+			// },
+			{
+				"Proto format",
+				func() []byte {
+					b, err := trace.Marshal()
+					require.NoError(t, err)
+					return b
+				},
+				map[string]string{
+					"Content-Type": "application/x-protobuf",
+				},
+			},
+		}
+
+		for _, tc := range testCases {
+			t.Run(tc.name, func(t *testing.T) {
+				req, err := http.NewRequest(http.MethodPost, "http://"+h.Services[util.ServiceDistributor].Endpoint(4318)+"/v1/traces", bytes.NewReader(tc.payload()))
+				require.NoError(t, err)
+				for k, v := range tc.headers {
+					req.Header.Set(k, v)
+				}
+
+				resp, err := http.DefaultClient.Do(req)
+				require.NoError(t, err)
+				defer func() { _ = resp.Body.Close() }()
+				bodyBytes, err := io.ReadAll(resp.Body)
+				require.NoError(t, err)
+				fmt.Println(string(bodyBytes))
+
+				assert.Equal(t, http.StatusTooManyRequests, resp.StatusCode)
+			})
+		}
+	})
+}
+
+func TestQueryLimits(t *testing.T) {
+	util.RunIntegrationTests(t, util.TestHarnessConfig{}, func(h *util.TempoHarness) {
+		h.WaitTracesWritable(t)
+
+		batch := util.MakeThriftBatchWithSpanCount(5)
+		require.NoError(t, h.WriteJaegerBatch(batch, ""))
+
+		// confirm the trace is queryable before adjusting overrides. otherwise the override
+		// update might beat the ingestion into livestore and the test will be flaky
+		h.WaitTracesQueryable(t, 1)
+
+		// retroactively make the trace too large so it will fail on querying
+		err := h.UpdateOverrides(map[string]*overrides.Overrides{
+			"single-tenant": {
+				Global: overrides.GlobalOverrides{
+					MaxBytesPerTrace: 1,
+				},
+			},
+		})
+		require.NoError(t, err)
+
+		// calc trace id
+		traceID := [16]byte{}
+		binary.BigEndian.PutUint64(traceID[:8], uint64(batch.Spans[0].TraceIdHigh))
+		binary.BigEndian.PutUint64(traceID[8:], uint64(batch.Spans[0].TraceIdLow))
+
+		// now try to query it back. this should fail b/c the trace is too large
+		apiClient := h.APIClientHTTP("")
+
+		_, err = apiClient.QueryTrace(tempoUtil.TraceIDToHexString(traceID[:]))
+		require.ErrorContains(t, err, trace.ErrTraceTooLarge.Error())
+		require.ErrorContains(t, err, "failed with response: 422") // confirm frontend returns 422
+
+		// wait for live store to complete the block
+		liveStore := h.Services[util.ServiceLiveStoreZoneA]
+		err = liveStore.WaitSumMetricsWithOptions(e2e.Greater(0),
+			[]string{"tempo_live_store_blocks_completed_total"},
+			e2e.WaitMissingMetrics,
+		)
+		require.NoError(t, err)
+
+		_, err = apiClient.QueryTrace(tempoUtil.TraceIDToHexString(traceID[:]))
+		require.ErrorContains(t, err, trace.ErrTraceTooLarge.Error())
+		require.ErrorContains(t, err, "failed with response: 422") // confirm frontend returns 422
+	})
+}
+
+func TestLimitsPartialSuccess(t *testing.T) {
+	util.RunIntegrationTests(t, util.TestHarnessConfig{
+		ConfigOverlay: configIngestPartialSucess,
+	}, func(h *util.TempoHarness) {
+		h.WaitTracesWritable(t)
+		// make request
+		traceIDs := make([][]byte, 6)
+		for index := range traceIDs {
+			traceID := make([]byte, 16)
+			_, err := crand.Read(traceID)
+			require.NoError(t, err)
+			traceIDs[index] = traceID
+		}
+
+		// 3 traces with trace_too_large and 3 with no error
+		spanCountsByTrace := []int{1, 4, 1, 5, 6, 1}
+		trace := test.MakeReqWithMultipleTraceWithSpanCount(spanCountsByTrace, traceIDs)
+
+		// send traces to tempo
+		// partial success = no error
+		require.NoError(t, h.WriteTempoProtoTraces(trace, ""))
+
+		// wait for live store to ingest traces
+		h.WaitTracesQueryable(t, 3) // only 3 traces are small enough to be ingested
+
+		// query for the traces that didn't trigger an error
+		apiClient := h.APIClientHTTP("")
+		for i, count := range spanCountsByTrace {
+			if count == 1 {
+				result, err := apiClient.QueryTrace(tempoUtil.TraceIDToHexString(traceIDs[i]))
+				require.NoError(t, err)
+				assert.Equal(t, 1, len(result.ResourceSpans))
+			}
+		}
+
+		// test metrics
+		// 3 traces with trace_too_large each with 4+5+6 spans
+		liveStore := h.Services[util.ServiceLiveStoreZoneA]
+		err := liveStore.WaitSumMetricsWithOptions(e2e.Equals(15),
+			[]string{"tempo_discarded_spans_total"},
+			e2e.WithLabelMatchers(labels.MustNewMatcher(labels.MatchEqual, "reason", "trace_too_large")),
+			e2e.WaitMissingMetrics,
+		)
+		require.NoError(t, err)
+	})
+}
+
+func TestQueryRateLimits(t *testing.T) {
+	util.RunIntegrationTests(t, util.TestHarnessConfig{
+		ConfigOverlay: configQueryRate,
+	}, func(h *util.TempoHarness) {
+		h.WaitTracesWritable(t)
+		// todo: do we even need to push a trace?
+		batch := util.MakeThriftBatchWithSpanCount(5)
+		require.NoError(t, h.WriteJaegerBatch(batch, ""))
+
+		// now try to query it back. this should fail b/c the frontend queue doesn't have room
+		apiClient := h.APIClientHTTP("")
+
+		// 429 HTTP Trace ID Lookup
+		traceID := []byte{0x01, 0x02}
+		_, err := apiClient.QueryTrace(tempoUtil.TraceIDToHexString(traceID))
+		require.ErrorContains(t, err, "job queue full")
+		require.ErrorContains(t, err, "failed with response: 429")
+
+		start := time.Now().Add(-1 * time.Hour).Unix()
+		end := time.Now().Add(1 * time.Hour).Unix()
+
+		// 429 HTTP Search
+		_, err = apiClient.SearchTraceQLWithRange("{}", start, end)
+		require.ErrorContains(t, err, "job queue full")
+		require.ErrorContains(t, err, "failed with response: 429")
+
+		// 429 GRPC Search
+		grpcClient, ctx, err := h.APIClientGRPC("")
+		require.NoError(t, err)
+
+		resp, err := grpcClient.Search(ctx, &tempopb.SearchRequest{
+			Query: "{}",
+			Start: uint32(start),
+			End:   uint32(end),
+		})
+		require.NoError(t, err)
+
+		// loop until we get io.EOF or an error
+		for {
+			_, err = resp.Recv()
+			if err != nil {
+				break
+			}
+		}
+		require.ErrorContains(t, err, "job queue full")
+		require.ErrorContains(t, err, "code = ResourceExhausted")
+	})
+}

integration/metrics-generator/config-messaging-system.yaml

@@ -0,0 +1,7 @@
+metrics_generator:
+  processor:
+    service_graphs:
+      histogram_buckets: [1, 2]  # seconds
+      enable_messaging_system_latency_histogram: true
+    span_metrics:
+      histogram_buckets: [1, 2]

integration/metrics-generator/config-remote-write.yaml

@@ -0,0 +1,11 @@
+metrics_generator:
+  processor:
+    service_graphs:
+      histogram_buckets: [1, 2]  # seconds
+    span_metrics:
+      histogram_buckets: [1, 2]  # seconds
+
+overrides:
+  defaults:
+    metrics_generator:
+      max_active_series: 1000

integration/metrics-generator/config-targetinfo.yaml

@@ -0,0 +1,14 @@
+metrics_generator:
+  processor:
+    service_graphs:
+      histogram_buckets: [1, 2]  # seconds
+    span_metrics:
+      histogram_buckets: [1, 2]
+
+overrides:
+  defaults:
+    metrics_generator:
+      max_active_series: 1000
+      processor:
+        span_metrics:
+          enable_target_info: true

integration/metrics-generator/metrics_generator_test.go

@@ -0,0 +1,417 @@
+package e2e
+
+import (
+	"fmt"
+	"io"
+	"math/rand"
+	"net/http"
+	"net/url"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/grafana/e2e"
+	"github.com/grafana/tempo/integration/util"
+	thrift "github.com/jaegertracing/jaeger-idl/thrift-gen/jaeger"
+	io_prometheus_client "github.com/prometheus/client_model/go"
+	"github.com/prometheus/common/expfmt"
+	"github.com/prometheus/common/model"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+const (
+	configMetricsGenerator                = "config-remote-write.yaml"
+	configMetricsGeneratorTargetInfo      = "config-targetinfo.yaml"
+	configMetricsGeneratorMessagingSystem = "config-messaging-system.yaml"
+)
+
+func TestMetricsGeneratorRemoteWrite(t *testing.T) {
+	util.RunIntegrationTests(t, util.TestHarnessConfig{
+		ConfigOverlay: configMetricsGenerator,
+		Components:    util.ComponentsMetricsGeneration,
+	}, func(h *util.TempoHarness) {
+		h.WaitTracesWritable(t)
+
+		// Send two spans that have a client-server relationship
+		r := rand.New(rand.NewSource(time.Now().UnixMilli()))
+		traceIDLow := r.Int63()   // nolint:gosec // G404: Use of weak random number generator
+		traceIDHigh := r.Int63()  // nolint:gosec // G404: Use of weak random number generator
+		parentSpanID := r.Int63() // nolint:gosec // G404: Use of weak random number generator
+
+		require.NoError(t, h.WriteJaegerBatch(&thrift.Batch{
+			Process: &thrift.Process{ServiceName: "lb"},
+			Spans: []*thrift.Span{
+				{
+					TraceIdLow:    traceIDLow,
+					TraceIdHigh:   traceIDHigh,
+					SpanId:        parentSpanID,
+					ParentSpanId:  0,
+					OperationName: "lb-get",
+					StartTime:     time.Now().UnixMicro(),
+					Duration:      int64(2 * time.Second / time.Microsecond),
+					Tags:          []*thrift.Tag{{Key: "span.kind", VStr: stringPtr("client")}},
+				},
+			},
+		}, ""))
+
+		require.NoError(t, h.WriteJaegerBatch(&thrift.Batch{
+			Process: &thrift.Process{ServiceName: "app"},
+			Spans: []*thrift.Span{
+				{
+					TraceIdLow:    traceIDLow,
+					TraceIdHigh:   traceIDHigh,
+					SpanId:        r.Int63(), // nolint:gosec // G404: Use of weak random number generator
+					ParentSpanId:  parentSpanID,
+					OperationName: "app-handle",
+					StartTime:     time.Now().UnixMicro(),
+					Duration:      int64(1 * time.Second / time.Microsecond),
+					Tags:          []*thrift.Tag{{Key: "span.kind", VStr: stringPtr("server")}},
+				},
+			},
+		}, ""))
+
+		// also send one with 5 minutes old timestamp
+		require.NoError(t, h.WriteJaegerBatch(&thrift.Batch{
+			Process: &thrift.Process{ServiceName: "app"},
+			Spans: []*thrift.Span{
+				{
+					TraceIdLow:    traceIDLow,
+					TraceIdHigh:   traceIDHigh,
+					SpanId:        r.Int63(), // nolint:gosec // G404: Use of weak random number generator
+					ParentSpanId:  parentSpanID,
+					OperationName: "app-handle",
+					StartTime:     time.Now().Add(-5 * time.Minute).UnixMicro(),
+					Duration:      int64(1 * time.Second / time.Microsecond),
+					Tags:          []*thrift.Tag{{Key: "span.kind", VStr: stringPtr("server")}},
+				},
+			},
+		}, ""))
+
+		// also send one with timestamp 10 days in the future
+		require.NoError(t, h.WriteJaegerBatch(&thrift.Batch{
+			Process: &thrift.Process{ServiceName: "app"},
+			Spans: []*thrift.Span{
+				{
+					TraceIdLow:    traceIDLow,
+					TraceIdHigh:   traceIDHigh,
+					SpanId:        r.Int63(), // nolint:gosec // G404: Use of weak random number generator
+					ParentSpanId:  parentSpanID,
+					OperationName: "app-handle",
+					StartTime:     time.Now().Add(10 * 24 * time.Hour).UnixMicro(),
+					Duration:      int64(1 * time.Second / time.Microsecond),
+					Tags:          []*thrift.Tag{{Key: "span.kind", VStr: stringPtr("server")}},
+				},
+			},
+		}, ""))
+
+		// also send one with an invalid label value
+		require.NoError(t, h.WriteJaegerBatch(&thrift.Batch{
+			Process: &thrift.Process{ServiceName: "app"},
+			Spans: []*thrift.Span{
+				{
+					TraceIdLow:    traceIDLow,
+					TraceIdHigh:   traceIDHigh,
+					SpanId:        r.Int63(), // nolint:gosec // G404: Use of weak random number generator
+					ParentSpanId:  parentSpanID,
+					OperationName: "\xff\xff",
+					StartTime:     time.Now().UnixMicro(),
+					Duration:      int64(2 * time.Second / time.Microsecond),
+					Tags:          []*thrift.Tag{{Key: "span.kind", VStr: stringPtr("server")}},
+				},
+			},
+		}, ""))
+
+		// Fetch metrics from Prometheus once they are received
+		var metricFamilies map[string]*io_prometheus_client.MetricFamily
+		var err error
+		for {
+			metricFamilies, err = extractMetricsFromPrometheus(h.Services[util.ServicePrometheus], `{__name__=~"traces_.+"}`)
+			require.NoError(t, err)
+			if len(metricFamilies) > 0 {
+				break
+			}
+			time.Sleep(time.Second)
+		}
+
+		// Print collected metrics for easier debugging
+		fmt.Println()
+		for key, family := range metricFamilies {
+			fmt.Println(key)
+			for _, metric := range family.Metric {
+				fmt.Println(metric)
+			}
+		}
+		fmt.Println()
+
+		// Service graphs
+		lbls := []string{"client", "lb", "server", "app"}
+		assert.Equal(t, 1.0, sumValues(metricFamilies, "traces_service_graph_request_total", lbls))
+
+		assert.Equal(t, 0.0, sumValues(metricFamilies, "traces_service_graph_request_client_seconds_bucket", append(lbls, "le", "1")))
+		assert.Equal(t, 1.0, sumValues(metricFamilies, "traces_service_graph_request_client_seconds_bucket", append(lbls, "le", "2")))
+		assert.Equal(t, 1.0, sumValues(metricFamilies, "traces_service_graph_request_client_seconds_bucket", append(lbls, "le", "+Inf")))
+		assert.Equal(t, 1.0, sumValues(metricFamilies, "traces_service_graph_request_client_seconds_count", lbls))
+		assert.Equal(t, 2.0, sumValues(metricFamilies, "traces_service_graph_request_client_seconds_sum", lbls))
+
+		assert.Equal(t, 1.0, sumValues(metricFamilies, "traces_service_graph_request_server_seconds_bucket", append(lbls, "le", "1")))
+		assert.Equal(t, 1.0, sumValues(metricFamilies, "traces_service_graph_request_server_seconds_bucket", append(lbls, "le", "2")))
+		assert.Equal(t, 1.0, sumValues(metricFamilies, "traces_service_graph_request_server_seconds_bucket", append(lbls, "le", "+Inf")))
+		assert.Equal(t, 1.0, sumValues(metricFamilies, "traces_service_graph_request_server_seconds_count", lbls))
+		assert.Equal(t, 1.0, sumValues(metricFamilies, "traces_service_graph_request_server_seconds_sum", lbls))
+
+		assert.Equal(t, 0.0, sumValues(metricFamilies, "traces_service_graph_request_failed_total", nil))
+		assert.Equal(t, 0.0, sumValues(metricFamilies, "traces_service_graph_unpaired_spans_total", nil))
+		assert.Equal(t, 0.0, sumValues(metricFamilies, "traces_service_graph_dropped_spans_total", nil))
+
+		// Span metrics
+		lbls = []string{"service", "lb", "span_name", "lb-get", "span_kind", "SPAN_KIND_CLIENT", "status_code", "STATUS_CODE_UNSET"}
+		assert.Equal(t, 1.0, sumValues(metricFamilies, "traces_spanmetrics_calls_total", lbls))
+		assert.NotEqual(t, 0, sumValues(metricFamilies, "traces_spanmetrics_size_total", lbls))
+		assert.Equal(t, 0.0, sumValues(metricFamilies, "traces_spanmetrics_latency_bucket", append(lbls, "le", "1")))
+		assert.Equal(t, 1.0, sumValues(metricFamilies, "traces_spanmetrics_latency_bucket", append(lbls, "le", "2")))
+		assert.Equal(t, 1.0, sumValues(metricFamilies, "traces_spanmetrics_latency_bucket", append(lbls, "le", "+Inf")))
+		assert.Equal(t, 1.0, sumValues(metricFamilies, "traces_spanmetrics_latency_count", lbls))
+		assert.Equal(t, 2.0, sumValues(metricFamilies, "traces_spanmetrics_latency_sum", lbls))
+
+		lbls = []string{"service", "app", "span_name", "app-handle", "span_kind", "SPAN_KIND_SERVER", "status_code", "STATUS_CODE_UNSET"}
+		assert.Equal(t, 1.0, sumValues(metricFamilies, "traces_spanmetrics_calls_total", lbls))
+		assert.NotEqual(t, 0, sumValues(metricFamilies, "traces_spanmetrics_size_total", lbls))
+		assert.Equal(t, 1.0, sumValues(metricFamilies, "traces_spanmetrics_latency_bucket", append(lbls, "le", "1")))
+		assert.Equal(t, 1.0, sumValues(metricFamilies, "traces_spanmetrics_latency_bucket", append(lbls, "le", "2")))
+		assert.Equal(t, 1.0, sumValues(metricFamilies, "traces_spanmetrics_latency_bucket", append(lbls, "le", "+Inf")))
+		assert.Equal(t, 1.0, sumValues(metricFamilies, "traces_spanmetrics_latency_count", lbls))
+		assert.Equal(t, 1.0, sumValues(metricFamilies, "traces_spanmetrics_latency_sum", lbls))
+
+		// Verify metrics
+		metricsGenerator := h.Services[util.ServiceMetricsGenerator]
+		assert.NoError(t, metricsGenerator.WaitSumMetrics(e2e.Equals(5), "tempo_metrics_generator_spans_received_total"))
+		assert.NoError(t, metricsGenerator.WaitSumMetrics(e2e.Equals(3), "tempo_metrics_generator_spans_discarded_total"))
+		assert.NoError(t, metricsGenerator.WaitSumMetrics(e2e.Equals(25), "tempo_metrics_generator_registry_active_series"))
+		assert.NoError(t, metricsGenerator.WaitSumMetrics(e2e.Equals(1000), "tempo_metrics_generator_registry_max_active_series"))
+		assert.NoError(t, metricsGenerator.WaitSumMetrics(e2e.Equals(25), "tempo_metrics_generator_registry_series_added_total"))
+	})
+}
+
+func TestMetricsGeneratorTargetInfoEnabled(t *testing.T) {
+	util.RunIntegrationTests(t, util.TestHarnessConfig{
+		ConfigOverlay: configMetricsGeneratorTargetInfo,
+		Components:    util.ComponentsMetricsGeneration,
+	}, func(h *util.TempoHarness) {
+		h.WaitTracesWritable(t)
+
+		// Send two spans that have a client-server relationship
+		r := rand.New(rand.NewSource(time.Now().UnixMilli()))
+		traceIDLow := r.Int63()   // nolint:gosec // G404: Use of weak random number generator
+		traceIDHigh := r.Int63()  // nolint:gosec // G404: Use of weak random number generator
+		parentSpanID := r.Int63() // nolint:gosec // G404: Use of weak random number generator
+
+		require.NoError(t, h.WriteJaegerBatch(&thrift.Batch{
+			Process: &thrift.Process{
+				ServiceName: "lb",
+				Tags:        []*thrift.Tag{{Key: "target_info", VStr: stringPtr("lb")}},
+			},
+			Spans: []*thrift.Span{
+				{
+					TraceIdLow:    traceIDLow,
+					TraceIdHigh:   traceIDHigh,
+					SpanId:        parentSpanID,
+					ParentSpanId:  0,
+					OperationName: "lb-get",
+					StartTime:     time.Now().UnixMicro(),
+					Duration:      int64(2 * time.Second / time.Microsecond),
+					Tags:          []*thrift.Tag{{Key: "span.kind", VStr: stringPtr("client")}},
+				},
+			},
+		}, ""))
+
+		require.NoError(t, h.WriteJaegerBatch(&thrift.Batch{
+			Process: &thrift.Process{
+				ServiceName: "app",
+				Tags:        []*thrift.Tag{{Key: "target_info", VStr: stringPtr("app")}},
+			},
+			Spans: []*thrift.Span{
+				{
+					TraceIdLow:    traceIDLow,
+					TraceIdHigh:   traceIDHigh,
+					SpanId:        r.Int63(), // nolint:gosec // G404: Use of weak random number generator
+					ParentSpanId:  parentSpanID,
+					OperationName: "app-handle",
+					StartTime:     time.Now().UnixMicro(),
+					Duration:      int64(1 * time.Second / time.Microsecond),
+					Tags:          []*thrift.Tag{{Key: "span.kind", VStr: stringPtr("server")}},
+				},
+			},
+		}, ""))
+
+		// Fetch metrics from Prometheus once they are received
+		var metricFamilies map[string]*io_prometheus_client.MetricFamily
+		var err error
+		for {
+			metricFamilies, err = extractMetricsFromPrometheus(h.Services[util.ServicePrometheus], `{__name__=~"traces_.+"}`)
+			require.NoError(t, err)
+			if len(metricFamilies) > 0 {
+				break
+			}
+			time.Sleep(time.Second)
+		}
+
+		// Print collected metrics for easier debugging
+		fmt.Println()
+		for key, family := range metricFamilies {
+			fmt.Println(key)
+			for _, metric := range family.Metric {
+				fmt.Println(metric)
+			}
+		}
+		fmt.Println()
+
+		assert.Equal(t, 1.0, sumValues(metricFamilies, "traces_target_info", []string{"job", "app", "target_info", "app", "instance", ""}))
+		assert.Equal(t, 1.0, sumValues(metricFamilies, "traces_target_info", []string{"job", "lb", "target_info", "lb", "instance", ""}))
+	})
+}
+
+func TestMetricsGeneratorMessagingSystemLatencyHistogramEnabled(t *testing.T) {
+	// Use a config that enables the messaging system latency histogram
+	util.RunIntegrationTests(t, util.TestHarnessConfig{
+		ConfigOverlay: configMetricsGeneratorMessagingSystem,
+		Components:    util.ComponentsMetricsGeneration,
+	}, func(h *util.TempoHarness) {
+		h.WaitTracesWritable(t)
+
+		// Send a pair of spans with a messaging system relationship (producer -> consumer)
+		// ignore the gosec linter because we are using a random number generator to create trace IDs
+		// and span IDs, which is not a security risk in this context.
+		// #nosec G404 -- nosemgrep: math-random-used
+		r := rand.New(rand.NewSource(time.Now().UnixMilli()))
+		traceIDLow := r.Int63()   // nolint:gosec // G404: Use of weak random number generator
+		traceIDHigh := r.Int63()  // nolint:gosec // G404: Use of weak random number generator
+		parentSpanID := r.Int63() // nolint:gosec // G404: Use of weak random number generator
+
+		// Set explicit times to ensure producer ends before consumer starts
+		now := time.Now()
+		producerStart := now
+		producerDuration := 2 * time.Second
+		producerEnd := producerStart.Add(producerDuration)
+		consumerStart := producerEnd.Add(1 * time.Second) // ensure consumer starts after producer ends
+		consumerDuration := 2 * time.Second
+
+		// Producer span
+		require.NoError(t, h.WriteJaegerBatch(&thrift.Batch{
+			Process: &thrift.Process{ServiceName: "producer"},
+			Spans: []*thrift.Span{
+				{
+					TraceIdLow:    traceIDLow,
+					TraceIdHigh:   traceIDHigh,
+					SpanId:        parentSpanID,
+					ParentSpanId:  0,
+					OperationName: "produce-message",
+					StartTime:     producerStart.UnixMicro(),
+					Duration:      int64(producerDuration / time.Microsecond),
+					Tags: []*thrift.Tag{
+						{Key: "span.kind", VStr: stringPtr("producer")},
+						{Key: "messaging.system", VStr: stringPtr("kafka")},
+					},
+				},
+			},
+		}, ""))
+
+		// Consumer span
+		require.NoError(t, h.WriteJaegerBatch(&thrift.Batch{
+			Process: &thrift.Process{ServiceName: "consumer"},
+			Spans: []*thrift.Span{
+				{
+					TraceIdLow:    traceIDLow,
+					TraceIdHigh:   traceIDHigh,
+					SpanId:        r.Int63(), // nolint:gosec // G404: Use of weak random number generator
+					ParentSpanId:  parentSpanID,
+					OperationName: "consume-message",
+					StartTime:     consumerStart.UnixMicro(),
+					Duration:      int64(consumerDuration / time.Microsecond),
+					Tags: []*thrift.Tag{
+						{Key: "span.kind", VStr: stringPtr("consumer")},
+						{Key: "messaging.system", VStr: stringPtr("kafka")},
+					},
+				},
+			},
+		}, ""))
+
+		// Wait for the metric to be produced
+		var metricFamilies map[string]*io_prometheus_client.MetricFamily
+		var err error
+		for {
+			metricFamilies, err = extractMetricsFromPrometheus(h.Services[util.ServicePrometheus], `{__name__=~"traces_.+"}`)
+			require.NoError(t, err)
+			if _, ok := metricFamilies["traces_service_graph_request_messaging_system_seconds_bucket"]; ok {
+				break
+			}
+			time.Sleep(30 * time.Second)
+		}
+
+		// Check that the metric exists and has a non-zero count
+		lbls := []string{"client", "producer", "server", "consumer", "connection_type", "messaging_system"}
+		assert.NotEqual(t, 0.0, sumValues(metricFamilies, "traces_service_graph_request_messaging_system_seconds_count", lbls))
+		assert.NotEqual(t, 0.0, sumValues(metricFamilies, "traces_service_graph_request_messaging_system_seconds_sum", lbls))
+	})
+}
+
+// extractMetricsFromPrometheus extracts metrics stored in Prometheus using the /federate endpoint.
+func extractMetricsFromPrometheus(prometheus *e2e.HTTPService, matcher string) (map[string]*io_prometheus_client.MetricFamily, error) {
+	url := fmt.Sprintf("http://%s/federate?match[]=%s", prometheus.HTTPEndpoint(), url.QueryEscape(matcher))
+
+	res, err := http.Get(url)
+	if err != nil {
+		return nil, err
+	}
+	defer res.Body.Close()
+
+	if res.StatusCode < 200 || res.StatusCode >= 300 {
+		return nil, fmt.Errorf("unexpected status code %d while fetching federate metrics", res.StatusCode)
+	}
+
+	body, err := io.ReadAll(res.Body)
+	if err != nil {
+		return nil, err
+	}
+
+	tp := expfmt.NewTextParser(model.UTF8Validation)
+	return tp.TextToMetricFamilies(strings.NewReader(string(body)))
+}
+
+// sumValues calculates the sum of all metrics in the metricFamily that contain the given labels.
+// filterLabels must be key-value pairs.
+func sumValues(metricFamily map[string]*io_prometheus_client.MetricFamily, metric string, filterLabels []string) float64 {
+	if len(filterLabels)%2 != 0 {
+		panic(fmt.Sprintf("filterLabels must be pairs: %v", filterLabels))
+	}
+	filterLabelsMap := map[string]string{}
+	for i := 0; i < len(filterLabels); i += 2 {
+		filterLabelsMap[filterLabels[i]] = filterLabels[i+1]
+	}
+
+	sum := 0.0
+
+outer:
+	for _, metric := range metricFamily[metric].GetMetric() {
+		labelMap := map[string]string{}
+		for _, label := range metric.GetLabel() {
+			labelMap[label.GetName()] = label.GetValue()
+		}
+
+		for key, expectedValue := range filterLabelsMap {
+			value, ok := labelMap[key]
+			if !ok || value != expectedValue {
+				continue outer
+			}
+		}
+
+		// since we fetch metrics using /federate they are all untyped
+		sum += metric.GetUntyped().GetValue()
+	}
+
+	return sum
+}
+
+func stringPtr(s string) *string {
+	return &s
+}

integration/operations/compression_test.go

@@ -1,14 +1,16 @@
-package api
+package deployments
 
 import (
+	"bytes"
 	"compress/gzip"
 	"io"
 	"net/http"
 	"testing"
 	"time"
 
-	"github.com/grafana/e2e"
+	"github.com/gogo/protobuf/jsonpb"
 	"github.com/grafana/tempo/integration/util"
+	"github.com/klauspost/compress/gzhttp"
 	"github.com/stretchr/testify/require"
 
 	"github.com/grafana/tempo/pkg/httpclient"
@@ -16,33 +18,26 @@ import (
 	tempoUtil "github.com/grafana/tempo/pkg/util"
 )
 
-const (
-	configCompression = "../deployments/config-all-in-one-local.yaml"
-)
-
 func TestCompression(t *testing.T) {
-	s, err := e2e.NewScenario("tempo_e2e")
-	require.NoError(t, err)
-	defer s.Close()
+	util.RunIntegrationTests(t, util.TestHarnessConfig{
+		DeploymentMode: util.DeploymentModeSingleBinary,
+	}, func(h *util.TempoHarness) {
+		h.WaitTracesWritable(t)
 
-	require.NoError(t, util.CopyFileToSharedDir(s, configCompression, "config.yaml"))
-	tempo := util.NewTempoAllInOne()
-	require.NoError(t, s.StartAndWaitReady(tempo))
+		// Send a trace
+		info := tempoUtil.NewTraceInfo(time.Now(), "")
+		require.NoError(t, h.WriteTraceInfo(info, ""))
 
-	// Get port for the Jaeger gRPC receiver endpoint
-	c, err := util.NewJaegerToOTLPExporter(tempo.Endpoint(4317))
-	require.NoError(t, err)
-	require.NotNil(t, c)
+		h.WaitTracesQueryable(t, 1)
 
-	info := tempoUtil.NewTraceInfo(time.Now(), "")
-	require.NoError(t, info.EmitAllBatches(c))
+		// Create client with compression
+		apiClient := h.APIClientHTTP("")
+		util.QueryAndAssertTrace(t, apiClient, info)
 
-	apiClient := httpclient.New("http://"+tempo.Endpoint(tempoPort), "")
-
-	apiClientWithCompression := httpclient.NewWithCompression("http://"+tempo.Endpoint(tempoPort), "")
-
-	util.QueryAndAssertTrace(t, apiClient, info)
-	queryAndAssertTraceCompression(t, apiClientWithCompression, info)
+		// Query and assert trace with compression
+		apiClient.WithTransport(gzhttp.Transport(http.DefaultTransport))
+		queryAndAssertTraceCompression(t, apiClient, info)
+	})
 }
 
 func queryAndAssertTraceCompression(t *testing.T, client *httpclient.Client, info *tempoUtil.TraceInfo) {
@@ -59,7 +54,7 @@ func queryAndAssertTraceCompression(t *testing.T, client *httpclient.Client, inf
 	// response, to disable this behaviour you have to explicitly set the Accept-Encoding header.
 
 	// Make the call directly so we have a chance to inspect the response header and manually un-gzip it ourselves to confirm the content.
-	request, err := http.NewRequest("GET", client.BaseURL+httpclient.QueryTraceEndpoint+"/"+info.HexID(), nil)
+	request, err := http.NewRequest("GET", client.BaseURL+httpclient.QueryTraceV2Endpoint+"/"+info.HexID(), nil)
 	require.NoError(t, err)
 	request.Header.Add("Accept-Encoding", "gzip")
 
@@ -73,11 +68,11 @@ func queryAndAssertTraceCompression(t *testing.T, client *httpclient.Client, inf
 	require.NoError(t, err)
 	defer gzipReader.Close()
 
-	m := &tempopb.Trace{}
+	m := &tempopb.TraceByIDResponse{}
 
 	bodyBytes, _ := io.ReadAll(gzipReader)
-	err = tempopb.UnmarshalFromJSONV1(bodyBytes, m)
-
+	err = jsonpb.Unmarshal(bytes.NewReader(bodyBytes), m)
 	require.NoError(t, err)
-	util.AssertEqualTrace(t, expected, m)
+
+	util.AssertEqualTrace(t, expected, m.Trace)
 }

integration/operations/config-https.yaml

@@ -1,13 +1,3 @@
-target: all
-stream_over_http_enabled: true
-
-query_frontend:
-  query_end_cutoff: -1m # Include all recent data
-  search:
-    ingester_shards: 1
-  metrics:
-    max_response_series: 3
-
 server:
   http_listen_port: 3200
   tls_cipher_suites: TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384
@@ -27,55 +17,14 @@ internal_server:
   enable: true
   http_listen_port: 3201
 
-distributor:
-  receivers:
-    jaeger:
-      protocols:
-        grpc:
-          endpoint: "tempo:14250"
-    otlp:
-      protocols:
-        grpc:
-          endpoint: "tempo:4317"
+live_store_client:
+  grpc_client_config:
+    tls_enabled: true
+    tls_insecure_skip_verify: true
+    tls_cipher_suites: TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384
+    tls_min_version: VersionTLS12
 
-ingester:
-  lifecycler:
-    address: 127.0.0.1
-    ring:
-      kvstore:
-        store: inmemory
-      replication_factor: 1
-    final_sleep: 0s
-  trace_idle_period: 1ms
-  max_block_duration: 1h
-  complete_block_timeout: 1h
-  flush_check_period: 1ms
-
-storage:
-  trace:
-    backend: gcs
-    gcs:
-      bucket_name: tempo
-      endpoint: https://tempo_e2e-gcs:4443/storage/v1/
-      insecure: true
-    pool:
-      max_workers: 10
-      queue_depth: 1000
-
-overrides:
-  user_configurable_overrides:
-    enabled: true
-    poll_interval: 10s
-    client:
-      backend: gcs
-      # fsouza/fake-gcs-server does not support versioning
-      confirm_versioning: false
-      gcs:
-        bucket_name: tempo
-        endpoint: https://tempo_e2e-gcs:4443/storage/v1/
-        insecure: true
-
-ingester_client:
+backend_scheduler_client:
   grpc_client_config:
     tls_enabled: true
     tls_insecure_skip_verify: true
@@ -83,6 +32,7 @@ ingester_client:
     tls_min_version: VersionTLS12
 
 querier:
+  query_live_store: true
   frontend_worker:
     grpc_client_config:
       tls_enabled: true

integration/operations/config-kvstore.yaml

@@ -0,0 +1,3 @@
+live_store:
+  partition_ring:
+    kvstore:{{ .KVStoreConfig }}

integration/operations/config-livestore-short-timeout.yaml

@@ -0,0 +1,2 @@
+live_store:
+  complete_block_timeout: 1s

integration/operations/config-query-plugin.yaml

@@ -0,0 +1,8 @@
+distributor:
+  receivers:
+    jaeger:
+      protocols:
+        grpc:
+          endpoint: "0.0.0.0:14250"
+    zipkin:
+      endpoint: "0.0.0.0:9411"

integration/operations/config-receivers.yaml

@@ -0,0 +1,12 @@
+distributor:
+  receivers:
+    otlp:
+      protocols:
+        http:
+          endpoint: "0.0.0.0:4318"
+    zipkin:
+      endpoint: "0.0.0.0:9411"
+    jaeger:
+      protocols:
+        grpc:
+          endpoint: "0.0.0.0:14250"

integration/operations/config-tempo-query.yaml

@@ -0,0 +1,3 @@
+address: "0.0.0.0:7777"
+backend: tempo:3200 # magically matches the name and port setup in the harness for a single binary deployment
+services_query_duration: 1h

integration/operations/failure_modes_test.go

@@ -0,0 +1,55 @@
+package deployments
+
+import (
+	"testing"
+	"time"
+
+	"github.com/grafana/tempo/integration/util"
+	tempoUtil "github.com/grafana/tempo/pkg/util"
+	"github.com/stretchr/testify/require"
+)
+
+func TestFailureModes(t *testing.T) {
+	util.RunIntegrationTests(t, util.TestHarnessConfig{
+		Components: util.ComponentsRecentDataQuerying | util.ComponentsBackendQuerying,
+	}, func(h *util.TempoHarness) {
+		h.WaitTracesWritable(t)
+
+		info := tempoUtil.NewTraceInfo(time.Now(), "")
+		require.NoError(t, h.WriteTraceInfo(info, ""))
+
+		h.WaitTracesQueryable(t, 1)
+
+		apiClient := h.APIClientHTTP("")
+		util.QueryAndAssertTrace(t, apiClient, info)
+
+		// stop one live store. data should still be queryable
+		liveStoreB := h.Services[util.ServiceLiveStoreZoneB]
+		err := liveStoreB.Stop()
+		require.NoError(t, err)
+
+		_, err = apiClient.QueryTraceV2(info.HexID())
+		require.NoError(t, err)
+
+		// stop the second live store. querying should fail
+		liveStoreA := h.Services[util.ServiceLiveStoreZoneA]
+		err = liveStoreA.Stop()
+		require.NoError(t, err)
+		_, err = apiClient.QueryTraceV2(info.HexID())
+		require.Error(t, err)
+
+		h.WaitTracesWrittenToBackend(t, 1)
+
+		// stop the block builder now that the trace is in the backend. ingestion should still work
+		blockBuilder := h.Services[util.ServiceBlockBuilder]
+		err = blockBuilder.Stop()
+		require.NoError(t, err)
+		require.NoError(t, h.WriteTraceInfo(tempoUtil.NewTraceInfo(time.Now(), ""), ""))
+
+		// restart the query frontend to do backend querying only. querying should work again
+		h.ForceBackendQuerying(t)
+		apiClient = h.APIClientHTTP("")
+		_, err = apiClient.SearchTraceQLWithRange("{}", time.Now().Add(-time.Hour).Unix(), time.Now().Unix())
+		require.NoError(t, err)
+	})
+}

integration/operations/https_test.go

@@ -1,24 +1,90 @@
-package test
+package deployments
 
 import (
+	"bytes"
+	"context"
 	"crypto/ecdsa"
 	"crypto/elliptic"
 	"crypto/rand"
+	"crypto/tls"
 	"crypto/x509"
 	"crypto/x509/pkix"
 	"encoding/pem"
 	"io"
 	"math/big"
+	"net/http"
 	"os"
 	"path/filepath"
 	"testing"
 	"time"
 
 	"github.com/grafana/dskit/runutil"
+	"github.com/grafana/e2e"
+	"github.com/grafana/tempo/integration/util"
+	"github.com/grafana/tempo/pkg/httpclient"
+	tempoUtil "github.com/grafana/tempo/pkg/util"
 	"github.com/stretchr/testify/require"
+	"google.golang.org/grpc/credentials"
 )
 
-type KeyMaterial struct {
+const (
+	configHTTPS = "config-https.yaml"
+
+	tempoPort = 3200 // magically matches the port in config-base.yaml
+)
+
+// TestHTTPS tests the use of unsigned certs with Tempo. Due to this we run the special "internal server"
+// on port 3201 which requires us to pass custome readiness probe. Additionally we have to create custom
+// a custom API client that uses https, but doesn't validate the certs.
+// Finally note that we actually push over an unencrypted connection, using the default harness functions.
+// This works b/c the TLS configuration for ingestion is configured through the OTEL receiver config.
+func TestHTTPS(t *testing.T) {
+	km := setupCertificates(t)
+
+	util.RunIntegrationTests(t, util.TestHarnessConfig{
+		ConfigOverlay:  configHTTPS,
+		ReadinessProbe: e2e.NewHTTPReadinessProbe(3201, "/ready", 200, 299), // this works b/c the service creation code in ../util/services.go adds a 3201 port to the services. we could also use a custom readiness probe.
+		PreStartHook: func(s *e2e.Scenario, _ map[string]any) error {
+			require.NoError(t, util.CopyFileToSharedDir(s, km.ServerCertFile, "tls.crt"))
+			require.NoError(t, util.CopyFileToSharedDir(s, km.ServerKeyFile, "tls.key"))
+			require.NoError(t, util.CopyFileToSharedDir(s, km.CaCertFile, "ca.crt"))
+
+			return nil
+		},
+	}, func(h *util.TempoHarness) {
+		// wait for traces to be writable
+		require.True(t, scrapeMetrics(t, h.Services[util.ServiceDistributor], tempoPort, "tempo_partition_ring_partitions{name=\"livestore-partitions\",state=\"Active\"} 1"))
+
+		// write a trace
+		info := tempoUtil.NewTraceInfo(time.Now(), "")
+		require.NoError(t, h.WriteTraceInfo(info, ""))
+
+		queryFrontend := h.Services[util.ServiceQueryFrontend]
+		apiClient := httpclient.New("https://"+queryFrontend.Endpoint(tempoPort), "")
+
+		// trust bad certs
+		defaultTransport := http.DefaultTransport.(*http.Transport).Clone()
+		defaultTransport.TLSClientConfig = &tls.Config{InsecureSkipVerify: true}
+		apiClient.WithTransport(defaultTransport)
+
+		util.QueryAndAssertTrace(t, apiClient, info)
+
+		// wait for the traces to be queryable
+		require.True(t, scrapeMetrics(t, h.Services[util.ServiceLiveStoreZoneA], tempoPort, "tempo_live_store_traces_created_total{tenant=\"single-tenant\"} 1"))
+		require.True(t, scrapeMetrics(t, h.Services[util.ServiceLiveStoreZoneB], tempoPort, "tempo_live_store_traces_created_total{tenant=\"single-tenant\"} 1"))
+
+		util.SearchTraceQLAndAssertTrace(t, apiClient, info)
+
+		creds := credentials.NewTLS(&tls.Config{InsecureSkipVerify: true})
+		grpcClient, err := util.NewSearchGRPCClient(queryFrontend.Endpoint(tempoPort), creds)
+		require.NoError(t, err)
+
+		now := time.Now()
+		util.SearchStreamAndAssertTrace(t, context.Background(), grpcClient, info, now.Add(-time.Hour).Unix(), now.Unix())
+	})
+}
+
+type keyMaterial struct {
 	CaCertFile                string
 	ServerCertFile            string
 	ServerKeyFile             string
@@ -32,7 +98,7 @@ type KeyMaterial struct {
 	Client2KeyFile            string
 }
 
-func SetupCertificates(t *testing.T) KeyMaterial {
+func setupCertificates(t *testing.T) keyMaterial {
 	testCADir := t.TempDir()
 
 	// create server side CA
@@ -116,7 +182,7 @@ func SetupCertificates(t *testing.T) KeyMaterial {
 		client2KeyFile,
 	))
 
-	return KeyMaterial{
+	return keyMaterial{
 		CaCertFile:                caCertFile,
 		ServerCertFile:            serverCertFile,
 		ServerKeyFile:             serverKeyFile,
@@ -178,7 +244,7 @@ func (ca *ca) writeCACertificate(path string) error {
 		return err
 	}
 
-	return writeExclusivePEMFile(path, "CERTIFICATE", 0o600, derBytes)
+	return writeExclusivePEMFile(path, "CERTIFICATE", 0o644, derBytes) // nolint:gosec // G306: Expect WriteFile permissions to be 0600 or less
 }
 
 func (ca *ca) writeCertificate(template *x509.Certificate, certPath string, keyPath string) error {
@@ -192,7 +258,7 @@ func (ca *ca) writeCertificate(template *x509.Certificate, certPath string, keyP
 		return err
 	}
 
-	if err := writeExclusivePEMFile(keyPath, "PRIVATE KEY", 0o600, keyBytes); err != nil {
+	if err := writeExclusivePEMFile(keyPath, "PRIVATE KEY", 0o644, keyBytes); err != nil { // nolint:gosec // G306: Expect WriteFile permissions to be 0600 or less
 		return err
 	}
 
@@ -208,5 +274,45 @@ func (ca *ca) writeCertificate(template *x509.Certificate, certPath string, keyP
 		return err
 	}
 
-	return writeExclusivePEMFile(certPath, "CERTIFICATE", 0o600, derBytes)
+	return writeExclusivePEMFile(certPath, "CERTIFICATE", 0o644, derBytes) // nolint:gosec // G306: Expect WriteFile permissions to be 0600 or less
+}
+
+func scrapeMetrics(t *testing.T, service *e2e.HTTPService, port int, searchString string) bool {
+	t.Helper()
+
+	// create HTTPS client with insecure skip verify
+	client := &http.Client{
+		Transport: &http.Transport{
+			TLSClientConfig: &tls.Config{InsecureSkipVerify: true},
+		},
+		Timeout: util.MetricsTimeout,
+	}
+
+	found := false
+
+	require.Eventually(t, func() bool {
+		url := "https://" + service.Endpoint(port) + "/metrics"
+		resp, err := client.Get(url)
+		if err != nil {
+			t.Logf("failed to scrape metrics: %v", err)
+			return false
+		}
+		defer resp.Body.Close()
+
+		if resp.StatusCode != http.StatusOK {
+			t.Logf("unexpected status code: %d", resp.StatusCode)
+			return false
+		}
+
+		body, err := io.ReadAll(resp.Body)
+		if err != nil {
+			t.Logf("failed to read response body: %v", err)
+			return false
+		}
+
+		found = bytes.Contains(body, []byte(searchString))
+		return found
+	}, time.Minute, time.Second, "could not write trace to tempo")
+
+	return found
 }

integration/operations/kvstores_test.go

@@ -0,0 +1,103 @@
+package deployments
+
+import (
+	"fmt"
+	"testing"
+	"time"
+
+	"github.com/grafana/e2e"
+	e2edb "github.com/grafana/e2e/db"
+	"github.com/grafana/tempo/integration/util"
+	tempoUtil "github.com/grafana/tempo/pkg/util"
+	"github.com/prometheus/prometheus/model/labels"
+	"github.com/stretchr/testify/require"
+)
+
+const configKVStore = "config-kvstore.yaml"
+
+func TestKVStores(t *testing.T) {
+	testKVStores := []struct {
+		name          string
+		setupKVStore  func(*e2e.Scenario, map[string]any) error
+		kvstoreConfig string
+	}{
+		{
+			name: "memberlist",
+			setupKVStore: func(_ *e2e.Scenario, templateData map[string]any) error {
+				templateData["KVStoreConfig"] = `
+      store: memberlist`
+				return nil
+			},
+		},
+		{
+			name: "etcd",
+			setupKVStore: func(s *e2e.Scenario, templateData map[string]any) error {
+				etcd := e2edb.NewETCD()
+				if err := s.StartAndWaitReady(etcd); err != nil {
+					return err
+				}
+				templateData["KVStoreConfig"] = fmt.Sprintf(`
+      store: etcd
+      etcd:
+        endpoints:
+          - http://%s:%d`, etcd.Name(), etcd.HTTPPort())
+				return nil
+			},
+		},
+		{
+			name: "consul",
+			setupKVStore: func(s *e2e.Scenario, templateData map[string]any) error {
+				consul := e2edb.NewConsul()
+				if err := s.StartAndWaitReady(consul); err != nil {
+					return err
+				}
+				templateData["KVStoreConfig"] = fmt.Sprintf(`
+      store: consul
+      consul:
+        host: http://%s:%d`, consul.Name(), consul.HTTPPort())
+				return nil
+			},
+		},
+	}
+
+	for _, tc := range testKVStores {
+		t.Run(tc.name, func(t *testing.T) {
+			util.RunIntegrationTests(t, util.TestHarnessConfig{
+				ConfigOverlay: configKVStore,
+				PreStartHook:  tc.setupKVStore,
+			}, func(h *util.TempoHarness) {
+				h.WaitTracesWritable(t)
+
+				liveStoreA := h.Services[util.ServiceLiveStoreZoneA]
+				liveStoreB := h.Services[util.ServiceLiveStoreZoneB]
+
+				matchers := []*labels.Matcher{
+					{Type: labels.MatchEqual, Name: "state", Value: "ACTIVE"},
+					{Type: labels.MatchEqual, Name: "name", Value: "live-store"},
+				}
+				require.NoError(t, liveStoreA.WaitSumMetricsWithOptions(
+					e2e.Equals(float64(2)),
+					[]string{"tempo_ring_members"},
+					e2e.WithLabelMatchers(matchers...),
+				), "live stores failed to join ring with %s", tc.name)
+				require.NoError(t, liveStoreB.WaitSumMetricsWithOptions(
+					e2e.Equals(float64(2)),
+					[]string{"tempo_ring_members"},
+					e2e.WithLabelMatchers(matchers...),
+				), "live stores failed to join ring with %s", tc.name)
+
+				// Send a trace
+				info := tempoUtil.NewTraceInfo(time.Now(), "")
+				require.NoError(t, h.WriteTraceInfo(info, ""))
+
+				// Wait for trace to be created in live stores
+				h.WaitTracesQueryable(t, 1)
+
+				// Find trace
+				apiClient := h.APIClientHTTP("")
+				util.QueryAndAssertTrace(t, apiClient, info)
+				util.SearchTraceQLAndAssertTrace(t, apiClient, info)
+			})
+		})
+	}
+}

integration/operations/livestore_test.go

@@ -0,0 +1,339 @@
+package deployments
+
+import (
+	"encoding/json"
+	"net/http"
+	"testing"
+	"time"
+
+	"github.com/grafana/e2e"
+	"github.com/grafana/tempo/integration/util"
+	tempoUtil "github.com/grafana/tempo/pkg/util"
+	"github.com/prometheus/prometheus/model/labels"
+	"github.com/stretchr/testify/require"
+)
+
+// TestLiveStorePartitionDownscale tests the complete partition downscale workflow:
+// - Marking a partition as INACTIVE
+// - Verifying new traffic goes to active partitions
+// - Cancelling the downscale
+// - Verifying normal operation resumes
+func TestLiveStorePartitionDownscale(t *testing.T) {
+	util.RunIntegrationTests(t, util.TestHarnessConfig{}, func(h *util.TempoHarness) {
+		// harness creates 2 live stores that own a first partition, shutdown B we don't want it for this test
+		require.NoError(t, h.Services[util.ServiceLiveStoreZoneB].Stop())
+
+		// start a new livestorea that own a second partition. -1 postfix is important to start a new partition!
+		liveStorePartition1 := newLiveStore("live-store-zone-a-1", "zone-a")
+		require.NoError(t, h.TestScenario.StartAndWaitReady(liveStorePartition1))
+
+		// wait for 2 active partitions
+		waitActivePartitions(t, h.Services[util.ServiceDistributor], 2)
+
+		info := tempoUtil.NewTraceInfo(time.Now(), "")
+		require.NoError(t, h.WriteTraceInfo(info, ""))
+
+		liveStorePartition0 := h.Services[util.ServiceLiveStoreZoneA]
+		liveStoreActive := waitForTraceInLiveStore(t, 1, liveStorePartition0, liveStorePartition1)
+		var liveStoreDownscale *e2e.HTTPService
+		if liveStoreActive == liveStorePartition0 {
+			liveStoreDownscale = liveStorePartition1
+		} else {
+			liveStoreDownscale = liveStorePartition0
+		}
+
+		apiClient := h.APIClientHTTP("")
+		distributor := h.Services[util.ServiceDistributor]
+
+		t.Run("verify partition is ACTIVE", func(t *testing.T) {
+			for _, liveStore := range []*e2e.HTTPService{liveStoreDownscale, liveStoreActive} {
+				res := preparePartitionDownscale(t, http.MethodGet, liveStore)
+				require.Equal(t, "PartitionActive", res.State)
+			}
+		})
+
+		t.Run("prepare partition for downscale", func(t *testing.T) {
+			// Set live-store's partition to INACTIVE
+			res := preparePartitionDownscale(t, http.MethodPost, liveStoreDownscale)
+			require.Greater(t, res.Timestamp, int64(0))
+			require.Equal(t, "PartitionInactive", res.State)
+
+			// Verify state
+			verifyPartitionState(t, distributor, "Inactive", 1)
+			verifyPartitionState(t, distributor, "Active", 1)
+		})
+
+		t.Run("verify data is still accessible during downscale", func(t *testing.T) {
+			util.QueryAndAssertTrace(t, apiClient, info)
+		})
+
+		t.Run("generate new trace during downscale", func(t *testing.T) {
+			inactiveCount, err := liveStoreDownscale.SumMetrics([]string{"tempo_live_store_records_processed_total"}, e2e.SkipMissingMetrics)
+			require.NoError(t, err)
+
+			info := tempoUtil.NewTraceInfo(time.Now(), "")
+			require.NoError(t, h.WriteTraceInfo(info, ""))
+
+			require.NoError(t, liveStoreActive.WaitSumMetrics(e2e.Equals(float64(2)), "tempo_live_store_traces_created_total"))
+
+			// Verify inactive live-store didn't process new records
+			inactiveCountAfter, err := liveStoreDownscale.SumMetrics([]string{"tempo_live_store_records_processed_total"}, e2e.SkipMissingMetrics)
+			require.NoError(t, err)
+			require.Equal(t, inactiveCount, inactiveCountAfter)
+
+			util.QueryAndAssertTrace(t, apiClient, info)
+		})
+
+		t.Run("cancel downscale preparation", func(t *testing.T) {
+			res := preparePartitionDownscale(t, http.MethodDelete, liveStoreDownscale)
+			require.Equal(t, int64(0), res.Timestamp)
+			require.Equal(t, "PartitionActive", res.State)
+
+			verifyPartitionState(t, distributor, "Active", 2)
+		})
+
+		t.Run("verify normal operation after cancellation", func(t *testing.T) {
+			info := tempoUtil.NewTraceInfo(time.Now(), "")
+			require.NoError(t, h.WriteTraceInfo(info, ""))
+
+			// poll the live stores until their total traces created >= 3. this is the third traced written. that means the above trace was consumed and we can query it
+			require.Eventually(t, func() bool {
+				totalA, err := liveStoreActive.SumMetrics([]string{"tempo_live_store_traces_created_total"}, e2e.SkipMissingMetrics)
+				require.NoError(t, err)
+				totalB, err := liveStoreDownscale.SumMetrics([]string{"tempo_live_store_traces_created_total"}, e2e.SkipMissingMetrics)
+				require.NoError(t, err)
+
+				return totalA[0]+totalB[0] >= 3.
+			}, 10*time.Second, 100*time.Millisecond)
+
+			util.QueryAndAssertTrace(t, apiClient, info)
+		})
+	})
+}
+
+// TestLiveStoreDownscaleHappyPath tests the complete downscale flow where
+// an inactive live-store is properly shut down
+func TestLiveStoreDownscaleHappyPath(t *testing.T) {
+	util.RunIntegrationTests(t, util.TestHarnessConfig{}, func(h *util.TempoHarness) {
+		// harness creates 2 live stores that own a first partition, shutdown B we don't want it for this test
+		require.NoError(t, h.Services[util.ServiceLiveStoreZoneB].Stop())
+
+		// start a new livestore that own a second partition. -1 postfix is important to start a new partition!
+		liveStorePartition1 := newLiveStore("live-store-zone-a-1", "zone-a")
+		require.NoError(t, h.TestScenario.StartAndWaitReady(liveStorePartition1))
+
+		// wait for 2 active partitions
+		distributor := h.Services[util.ServiceDistributor]
+		waitActivePartitions(t, distributor, 2)
+
+		// Mark partition as INACTIVE
+		preparePartitionDownscale(t, http.MethodPost, liveStorePartition1)
+
+		// Prepare for shutdown
+		req, err := http.NewRequest("POST", "http://"+liveStorePartition1.Endpoint(3200)+"/live-store/prepare-downscale", nil)
+		require.NoError(t, err)
+		httpResp, err := http.DefaultClient.Do(req)
+		require.NoError(t, err)
+		require.Equal(t, 204, httpResp.StatusCode)
+
+		// Stop inactive live-store
+		require.NoError(t, liveStorePartition1.Stop())
+
+		waitActivePartitions(t, distributor, 1)
+
+		// Verify only one active partition remains
+		partitions := getRingStatus(t, distributor).Partitions
+
+		activeCount := 0
+		var activePartition *partitionData
+
+		for _, partition := range partitions {
+			if partition.State == activePartitionState {
+				activePartition = &partition
+				activeCount++
+			}
+		}
+		require.Equal(t, 1, activeCount)
+		require.NotNil(t, activePartition)
+		require.Equal(t, int32(0), activePartition.ID)
+		require.False(t, activePartition.Corrupted)
+		require.Equal(t, activePartitionState, activePartition.State)
+		require.Contains(t, activePartition.OwnerIDs, "live-store-zone-a-0")
+	})
+}
+
+func TestLiveStoreLookback(t *testing.T) {
+	for _, testCase := range []struct {
+		name              string
+		configOverlay     string
+		startNewLiveStore bool
+		expectedTraces    float64
+	}{
+		{
+			name:              "restart_2s",
+			configOverlay:     "config-livestore-short-timeout.yaml", // lookback period twice greater
+			startNewLiveStore: false,
+			expectedTraces:    2, // fresh and after start
+		},
+		{
+			name:              "restart_default",
+			configOverlay:     "", // default is 1h
+			startNewLiveStore: false,
+			expectedTraces:    3, // old, fresh and after start, but not already committed
+		},
+		{
+			name:              "start_2s",
+			configOverlay:     "config-livestore-short-timeout.yaml", // lookback period twice greater
+			startNewLiveStore: true,
+			expectedTraces:    2, // fresh and after start
+		},
+		{
+			name:              "start_default",
+			configOverlay:     "", // default is 1h
+			startNewLiveStore: true,
+			expectedTraces:    4, // all traces
+		},
+	} {
+		t.Run(testCase.name, func(t *testing.T) {
+			util.RunIntegrationTests(t, util.TestHarnessConfig{
+				ConfigOverlay: testCase.configOverlay,
+			}, func(h *util.TempoHarness) {
+				// Start initial live store
+				liveStoreB := h.Services[util.ServiceLiveStoreZoneB]
+				require.NoError(t, liveStoreB.Stop())
+
+				h.WaitTracesWritable(t)
+
+				// Write first trace - will be committed by the first live store before shutdown
+				info := tempoUtil.NewTraceInfo(time.Now(), "")
+				require.NoError(t, h.WriteTraceInfo(info, ""))
+
+				// wait queryable
+				liveStoreA := h.Services[util.ServiceLiveStoreZoneA]
+				require.NoError(t, liveStoreA.WaitSumMetricsWithOptions(e2e.GreaterOrEqual(float64(1)), []string{"tempo_live_store_traces_created_total"}, e2e.WaitMissingMetrics))
+
+				// Stop the live store
+				require.NoError(t, liveStoreA.Stop())
+
+				// Write old trace (during downtime)
+				require.NoError(t, h.WriteTraceInfo(tempoUtil.NewTraceInfo(time.Now(), ""), ""))
+				time.Sleep(3 * time.Second)
+
+				// Write fresh trace (during downtime)
+				require.NoError(t, h.WriteTraceInfo(tempoUtil.NewTraceInfo(time.Now(), ""), ""))
+
+				// Restart the same live store or start a new one
+				liveStore := liveStoreA
+				if testCase.startNewLiveStore {
+					liveStore = liveStoreB
+				}
+				require.NoError(t, liveStore.Start(h.TestScenario.NetworkName(), h.TestScenario.SharedDir()))
+				require.NoError(t, liveStore.WaitReady())
+
+				h.WaitTracesWritable(t)
+
+				// Write trace after restart
+				require.NoError(t, h.WriteTraceInfo(tempoUtil.NewTraceInfo(time.Now(), ""), ""))
+
+				// Verify expected number of traces were processed
+				require.NoError(t, liveStore.WaitSumMetricsWithOptions(e2e.GreaterOrEqual(float64(testCase.expectedTraces)), []string{"tempo_live_store_traces_created_total"}, e2e.WaitMissingMetrics))
+				if testCase.startNewLiveStore {
+					require.NoError(t, liveStore.WaitSumMetricsWithOptions(e2e.GreaterOrEqual(float64(testCase.expectedTraces)), []string{"tempo_live_store_traces_created_total"}, e2e.WaitMissingMetrics))
+				}
+			})
+		})
+	}
+}
+
+const activePartitionState = 2 // apparently state == 2 is active
+
+type preparePartitionDownscaleResponse struct {
+	Timestamp int64  `json:"timestamp"`
+	State     string `json:"state"`
+}
+
+type ringStatus struct {
+	Partitions []partitionData `json:"partitions"`
+}
+
+type partitionData struct {
+	ID        int32    `json:"id"`
+	Corrupted bool     `json:"corrupted"`
+	State     int      `json:"state"`
+	OwnerIDs  []string `json:"owner_ids"`
+}
+
+func getRingStatus(t *testing.T, service *e2e.HTTPService) ringStatus {
+	req, err := http.NewRequest("GET", "http://"+service.Endpoint(3200)+"/partition-ring", nil)
+	req.Header.Set("Accept", "application/json")
+	require.NoError(t, err)
+	httpResp, err := http.DefaultClient.Do(req)
+	require.NoError(t, err)
+	require.Equal(t, 200, httpResp.StatusCode)
+
+	var result ringStatus
+	require.NoError(t, json.NewDecoder(httpResp.Body).Decode(&result))
+	return result
+}
+
+func preparePartitionDownscale(t *testing.T, method string, liveStore *e2e.HTTPService) preparePartitionDownscaleResponse {
+	req, err := http.NewRequest(method, "http://"+liveStore.Endpoint(3200)+"/live-store/prepare-partition-downscale", nil)
+	require.NoError(t, err)
+	httpResp, err := http.DefaultClient.Do(req)
+	require.NoError(t, err)
+	require.Equal(t, 200, httpResp.StatusCode)
+
+	var result preparePartitionDownscaleResponse
+	require.NoError(t, json.NewDecoder(httpResp.Body).Decode(&result))
+	return result
+}
+
+func verifyPartitionState(t *testing.T, service *e2e.HTTPService, expectedState string, expectedCount int) {
+	partitionStateMatchers := []*labels.Matcher{
+		{Type: labels.MatchEqual, Name: "state", Value: expectedState},
+		{Type: labels.MatchEqual, Name: "name", Value: "livestore-partitions"},
+	}
+	require.NoError(t, service.WaitSumMetricsWithOptions(
+		e2e.Equals(float64(expectedCount)),
+		[]string{"tempo_partition_ring_partitions"},
+		e2e.WithLabelMatchers(partitionStateMatchers...),
+	))
+}
+
+func waitForTraceInLiveStore(t *testing.T, expectedTraces int, liveStores ...*e2e.HTTPService) *e2e.HTTPService {
+	ch := make(chan *e2e.HTTPService)
+	timeout := 30 * time.Second
+
+	for _, liveStore := range liveStores {
+		go func(liveStore *e2e.HTTPService) {
+			err := liveStore.WaitSumMetrics(e2e.GreaterOrEqual(float64(expectedTraces)), "tempo_live_store_traces_created_total")
+			if err == nil {
+				ch <- liveStore
+			}
+		}(liveStore)
+	}
+
+	select {
+	case liveStore := <-ch:
+		close(ch)
+		return liveStore
+	case <-time.After(timeout):
+		t.Fatalf("timeout waiting for trace in live-store after %s", timeout)
+	}
+	return nil
+}
+
+func waitActivePartitions(t *testing.T, service *e2e.HTTPService, count int) {
+	matchers := []*labels.Matcher{
+		{Type: labels.MatchEqual, Name: "state", Value: "Active"},
+		{Type: labels.MatchEqual, Name: "name", Value: "livestore-partitions"},
+	}
+	require.NoError(t, service.WaitSumMetricsWithOptions(
+		e2e.Equals(float64(count)),
+		[]string{"tempo_partition_ring_partitions"},
+		e2e.WithLabelMatchers(matchers...)), "distributor failed to see the partition ring")
+}
+
+func newLiveStore(name string, zone string) *e2e.HTTPService {
+	return util.NewTempoService(name, "live-store", e2e.NewHTTPReadinessProbe(3200, "/ready", 200, 299), nil, "-live-store.instance-availability-zone="+zone)
+}

integration/operations/operational_metrics_test.go

@@ -0,0 +1,267 @@
+package deployments
+
+import (
+	"errors"
+	"io"
+	"testing"
+	"time"
+
+	"github.com/grafana/e2e"
+	"github.com/grafana/tempo/integration/util"
+	"github.com/grafana/tempo/pkg/tempopb"
+	tempoUtil "github.com/grafana/tempo/pkg/util"
+	"github.com/grafana/tempo/pkg/util/test"
+	"github.com/prometheus/prometheus/model/labels"
+	"github.com/stretchr/testify/require"
+)
+
+func TestWriteMetrics(t *testing.T) {
+	util.RunIntegrationTests(t, util.TestHarnessConfig{
+		Components: util.ComponentsRecentDataQuerying | util.ComponentsBackendQuerying,
+	}, func(h *util.TempoHarness) {
+		h.WaitTracesWritable(t)
+
+		countTraces := 10
+		countSpans := 3 * 3 * countTraces // batches * spans per batch * traces
+		countBytes := 0
+
+		for range countTraces {
+			trace := test.MakeTraceWithSpanCount(3, 3, test.ValidTraceID(nil))
+			countBytes += trace.Size()
+
+			require.NoError(t, h.WriteTempoProtoTraces(trace, ""))
+		}
+
+		h.WaitTracesQueryable(t, countTraces)
+
+		// distributor
+		distributor := h.Services[util.ServiceDistributor]
+		assertMetricEquals(t, distributor, "tempo_distributor_produce_batches_total", float64(countTraces), nil)
+		assertMetricEquals(t, distributor, "tempo_distributor_spans_received_total", float64(countSpans), nil)
+
+		sums, err := distributor.SumMetrics([]string{"tempo_distributor_bytes_received_total"})
+		require.NoError(t, err)
+		actualBytes := sums[0]
+		assertMetricInDelta(t, distributor, "tempo_distributor_bytes_received_total", float64(countBytes), .1*float64(countBytes), nil)
+		assertMetricInDelta(t, distributor, "tempo_distributor_ingress_bytes_total", float64(countBytes), .1*float64(countBytes), nil)
+
+		// livestore
+		livestore := h.Services[util.ServiceLiveStoreZoneA]
+		assertMetricInDelta(t, livestore, "tempo_live_store_bytes_received_total", float64(countBytes), actualBytes, nil)
+		assertMetricEquals(t, livestore, "tempo_live_store_partition_owned", float64(1), nil)
+
+		// we may not have yet completed a block so wait for this one
+		err = livestore.WaitSumMetricsWithOptions(e2e.Greater(0),
+			[]string{"tempo_live_store_blocks_completed_total"},
+			e2e.WaitMissingMetrics,
+		)
+		require.NoError(t, err)
+
+		h.WaitTracesWrittenToBackend(t, countTraces)
+
+		// blockbuilder
+		blockbuilder := h.Services[util.ServiceBlockBuilder]
+		assertMetricEquals(t, blockbuilder, "tempo_block_builder_flushed_blocks", float64(1), nil)
+		assertMetricEquals(t, blockbuilder, "tempo_block_builder_owned_partitions", float64(1), nil)
+	})
+}
+
+func TestReadMetrics(t *testing.T) {
+	util.RunIntegrationTests(t, util.TestHarnessConfig{}, func(h *util.TempoHarness) {
+		h.WaitTracesWritable(t)
+
+		info := tempoUtil.NewTraceInfo(time.Now(), "")
+		require.NoError(t, h.WriteTraceInfo(info, ""))
+
+		h.WaitTracesQueryable(t, 1)
+
+		// http
+		apiClient := h.APIClientHTTP("")
+
+		_, err := apiClient.MetricsQueryInstant("{} | rate()", time.Now().Add(-time.Hour).Unix(), time.Now().Unix(), 0)
+		require.NoError(t, err)
+		_, err = apiClient.MetricsQueryRange("{} | rate()", time.Now().Add(-time.Hour).Unix(), time.Now().Unix(), "1m", 0)
+		require.NoError(t, err)
+		id, err := info.TraceID()
+		require.NoError(t, err)
+		_, err = apiClient.QueryTraceV2(tempoUtil.TraceIDToHexString(id))
+		require.NoError(t, err)
+		_, err = apiClient.SearchTagValuesV2("span.foo", "")
+		require.NoError(t, err)
+		_, err = apiClient.SearchTagsV2()
+		require.NoError(t, err)
+		_, err = apiClient.SearchTraceQL("{}")
+		require.NoError(t, err)
+
+		// grpc
+		grpcClient, ctx, err := h.APIClientGRPC("")
+		require.NoError(t, err)
+		instantClient, err := grpcClient.MetricsQueryInstant(ctx, &tempopb.QueryInstantRequest{
+			Query: "{} | rate()",
+			Start: uint64(time.Now().Add(-time.Hour).Local().UnixNano()),
+			End:   uint64(time.Now().Local().UnixNano()),
+		})
+		require.NoError(t, err)
+		err = drainStreamingClient(instantClient)
+		require.NoError(t, err)
+
+		rangeClient, err := grpcClient.MetricsQueryRange(ctx, &tempopb.QueryRangeRequest{
+			Query:     "{} | rate()",
+			Start:     uint64(time.Now().Add(-time.Hour).Local().UnixNano()),
+			End:       uint64(time.Now().Local().UnixNano()),
+			Step:      uint64(time.Minute.Nanoseconds()),
+			Exemplars: 0,
+		})
+		require.NoError(t, err)
+		err = drainStreamingClient(rangeClient)
+		require.NoError(t, err)
+
+		searchTagValuesClient, err := grpcClient.SearchTagValuesV2(ctx, &tempopb.SearchTagValuesRequest{
+			TagName: "span.foo",
+			Query:   "",
+		})
+		require.NoError(t, err)
+		err = drainStreamingClient(searchTagValuesClient)
+		require.NoError(t, err)
+
+		searchTagsClient, err := grpcClient.SearchTagsV2(ctx, &tempopb.SearchTagsRequest{
+			Query: "{}",
+		})
+		require.NoError(t, err)
+		err = drainStreamingClient(searchTagsClient)
+		require.NoError(t, err)
+
+		searchClient, err := grpcClient.Search(ctx, &tempopb.SearchRequest{
+			Query: "{}",
+		})
+		require.NoError(t, err)
+		err = drainStreamingClient(searchClient)
+		require.NoError(t, err)
+
+		// query-frontend
+		queryFrontend := h.Services[util.ServiceQueryFrontend]
+
+		assertMetricEquals(t, queryFrontend, "tempo_query_frontend_queries_total", float64(4), map[string]string{"op": "metrics", "result": "completed"})
+		assertMetricEquals(t, queryFrontend, "tempo_query_frontend_queries_total", float64(4), map[string]string{"op": "metadata", "result": "completed"})
+		assertMetricEquals(t, queryFrontend, "tempo_query_frontend_queries_total", float64(2), map[string]string{"op": "search", "result": "completed"})
+		assertMetricEquals(t, queryFrontend, "tempo_query_frontend_queries_total", float64(1), map[string]string{"op": "traces", "result": "completed"})
+
+		assertMetricEquals(t, queryFrontend, "tempo_query_frontend_queries_within_slo_total", float64(4), map[string]string{"op": "metrics", "result": "completed"})
+		assertMetricEquals(t, queryFrontend, "tempo_query_frontend_queries_within_slo_total", float64(4), map[string]string{"op": "metadata", "result": "completed"})
+		assertMetricEquals(t, queryFrontend, "tempo_query_frontend_queries_within_slo_total", float64(2), map[string]string{"op": "search", "result": "completed"})
+		assertMetricEquals(t, queryFrontend, "tempo_query_frontend_queries_within_slo_total", float64(1), map[string]string{"op": "traces", "result": "completed"})
+
+		assertMetricGreater(t, queryFrontend, "tempo_query_frontend_bytes_inspected_total", float64(0), map[string]string{"op": "metadata"})
+		assertMetricEquals(t, queryFrontend, "tempo_query_frontend_bytes_inspected_total", float64(0), map[string]string{"op": "metrics"}) // metrics is 0? possibly a bug?
+		assertMetricGreater(t, queryFrontend, "tempo_query_frontend_bytes_inspected_total", float64(0), map[string]string{"op": "search"})
+		assertMetricGreater(t, queryFrontend, "tempo_query_frontend_bytes_inspected_total", float64(0), map[string]string{"op": "traces"})
+
+		assertMetricCountEquals(t, queryFrontend, "tempo_request_duration_seconds", float64(1), map[string]string{"method": "GET", "route": "api_metrics_query", "status_code": "200", "ws": "false"})
+		assertMetricCountEquals(t, queryFrontend, "tempo_request_duration_seconds", float64(1), map[string]string{"method": "GET", "route": "api_metrics_query_range", "status_code": "200", "ws": "false"})
+		assertMetricCountEquals(t, queryFrontend, "tempo_request_duration_seconds", float64(1), map[string]string{"method": "GET", "route": "api_v2_traces_traceid", "status_code": "200", "ws": "false"})
+		assertMetricCountEquals(t, queryFrontend, "tempo_request_duration_seconds", float64(1), map[string]string{"method": "GET", "route": "api_v2_search_tag_tagname_values", "status_code": "200", "ws": "false"})
+		assertMetricCountEquals(t, queryFrontend, "tempo_request_duration_seconds", float64(1), map[string]string{"method": "GET", "route": "api_v2_search_tags", "status_code": "200", "ws": "false"})
+		assertMetricCountEquals(t, queryFrontend, "tempo_request_duration_seconds", float64(1), map[string]string{"method": "GET", "route": "api_search", "status_code": "200", "ws": "false"})
+
+		assertMetricCountEquals(t, queryFrontend, "tempo_request_duration_seconds", float64(1), map[string]string{"method": "gRPC", "route": "/tempopb.StreamingQuerier/MetricsQueryInstant", "status_code": "success", "ws": "false"})
+		assertMetricCountEquals(t, queryFrontend, "tempo_request_duration_seconds", float64(1), map[string]string{"method": "gRPC", "route": "/tempopb.StreamingQuerier/MetricsQueryRange", "status_code": "success", "ws": "false"})
+		assertMetricCountEquals(t, queryFrontend, "tempo_request_duration_seconds", float64(1), map[string]string{"method": "gRPC", "route": "/tempopb.StreamingQuerier/SearchTagValuesV2", "status_code": "success", "ws": "false"})
+		assertMetricCountEquals(t, queryFrontend, "tempo_request_duration_seconds", float64(1), map[string]string{"method": "gRPC", "route": "/tempopb.StreamingQuerier/SearchTagsV2", "status_code": "success", "ws": "false"})
+		assertMetricCountEquals(t, queryFrontend, "tempo_request_duration_seconds", float64(1), map[string]string{"method": "gRPC", "route": "/tempopb.StreamingQuerier/Search", "status_code": "success", "ws": "false"})
+
+		// querier
+		querier := h.Services[util.ServiceQuerier]
+		assertMetricGreater(t, querier, "tempo_querier_worker_request_executed_total", float64(0), nil)
+		assertMetricEquals(t, querier, "tempo_querier_livestore_clients", float64(2), nil)
+
+		// testing these counts might be a little brittle and we find it's not worthwhile
+		assertMetricCountEquals(t, querier, "tempo_request_duration_seconds", float64(4), map[string]string{"method": "GET", "route": "querier_api_metrics_query_range", "status_code": "200", "ws": "false"})
+		assertMetricCountEquals(t, querier, "tempo_request_duration_seconds", float64(50), map[string]string{"method": "GET", "route": "querier_api_v2_traces_traceid", "status_code": "200", "ws": "false"})
+		assertMetricCountEquals(t, querier, "tempo_request_duration_seconds", float64(2), map[string]string{"method": "GET", "route": "querier_api_v2_search_tag_tagname_values", "status_code": "200", "ws": "false"})
+		assertMetricCountEquals(t, querier, "tempo_request_duration_seconds", float64(2), map[string]string{"method": "GET", "route": "querier_api_v2_search_tags", "status_code": "200", "ws": "false"})
+		assertMetricCountEquals(t, querier, "tempo_request_duration_seconds", float64(6), map[string]string{"method": "GET", "route": "querier_api_search", "status_code": "200", "ws": "false"})
+	})
+}
+
+type streamingClient[T any] interface {
+	Recv() (*T, error)
+}
+
+func drainStreamingClient[T any](stream streamingClient[T]) error {
+	for {
+		resp, err := stream.Recv()
+		if resp != nil {
+			break
+		}
+		if errors.Is(err, io.EOF) {
+			return nil
+		}
+		if err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
+func assertMetricEquals(t *testing.T, service *e2e.HTTPService, metric string, expected float64, labelValues map[string]string) {
+	t.Helper()
+	opts := []e2e.MetricsOption{}
+	if len(labelValues) > 0 {
+		matchers := make([]*labels.Matcher, 0, len(labelValues))
+		for name, value := range labelValues {
+			matchers = append(matchers, &labels.Matcher{Type: labels.MatchEqual, Name: name, Value: value})
+		}
+		opts = append(opts, e2e.WithLabelMatchers(matchers...))
+	}
+	sums, err := service.SumMetrics([]string{metric}, opts...)
+	require.NoError(t, err)
+	require.Equal(t, expected, sums[0])
+}
+
+// nolint:unparam
+func assertMetricCountEquals(t *testing.T, service *e2e.HTTPService, metric string, expected float64, labelValues map[string]string) {
+	t.Helper()
+	opts := []e2e.MetricsOption{e2e.WithMetricCount}
+	if len(labelValues) > 0 {
+		matchers := make([]*labels.Matcher, 0, len(labelValues))
+		for name, value := range labelValues {
+			matchers = append(matchers, &labels.Matcher{Type: labels.MatchEqual, Name: name, Value: value})
+		}
+		opts = append(opts, e2e.WithLabelMatchers(matchers...))
+	}
+	sums, err := service.SumMetrics([]string{metric}, opts...)
+	require.NoError(t, err)
+	require.Equal(t, expected, sums[0])
+}
+
+func assertMetricInDelta(t *testing.T, service *e2e.HTTPService, metric string, expected, delta float64, labelValues map[string]string) {
+	t.Helper()
+	opts := []e2e.MetricsOption{}
+	if len(labelValues) > 0 {
+		matchers := make([]*labels.Matcher, 0, len(labelValues))
+		for name, value := range labelValues {
+			matchers = append(matchers, &labels.Matcher{Type: labels.MatchEqual, Name: name, Value: value})
+		}
+		opts = append(opts, e2e.WithLabelMatchers(matchers...))
+	}
+	sums, err := service.SumMetrics([]string{metric}, opts...)
+	require.NoError(t, err)
+	require.InDelta(t, expected, sums[0], delta)
+}
+
+// nolint:unparam
+func assertMetricGreater(t *testing.T, service *e2e.HTTPService, metric string, minValue float64, labelValues map[string]string) {
+	t.Helper()
+	opts := []e2e.MetricsOption{}
+	if len(labelValues) > 0 {
+		matchers := make([]*labels.Matcher, 0, len(labelValues))
+		for name, value := range labelValues {
+			matchers = append(matchers, &labels.Matcher{Type: labels.MatchEqual, Name: name, Value: value})
+		}
+		opts = append(opts, e2e.WithLabelMatchers(matchers...))
+	}
+	sums, err := service.SumMetrics([]string{metric}, opts...)
+	require.NoError(t, err)
+	require.Greater(t, sums[0], minValue)
+}

integration/operations/query_plugin_test.go

@@ -1,7 +1,6 @@
-package api
+package deployments
 
 import (
-	"context"
 	"encoding/json"
 	"fmt"
 	"io"
@@ -19,107 +18,60 @@ import (
 )
 
 func TestSearchUsingJaegerPlugin(t *testing.T) {
-	t.Parallel()
+	util.RunIntegrationTests(t, util.TestHarnessConfig{
+		DeploymentMode: util.DeploymentModeSingleBinary,
+	}, func(h *util.TempoHarness) {
+		h.WaitTracesWritable(t)
 
-	s, err := e2e.NewScenario("tempo_query_plugin_e2e")
-	require.NoError(t, err)
-	defer s.Close()
+		require.NoError(t, util.CopyFileToSharedDir(h.TestScenario, "config-tempo-query.yaml", "config-tempo-query.yaml"))
 
-	require.NoError(t, util.CopyFileToSharedDir(s, "config-plugin-test.yaml", "config.yaml"))
-	require.NoError(t, util.CopyFileToSharedDir(s, "config-tempo-query.yaml", "config-tempo-query.yaml"))
+		// Start tempo-query and jaeger-query services
+		tempoQuery := util.NewTempoQuery()
+		jaegerQuery := util.NewJaegerQuery()
 
-	tempo := util.NewTempoAllInOne()
-	tempoQuery := util.NewTempoQuery()
-	jaegerQuery := util.NewJaegerQuery()
+		err := h.TestScenario.StartAndWaitReady(tempoQuery, jaegerQuery)
+		require.NoError(t, err)
 
-	require.NoError(t, s.StartAndWaitReady(tempo))
-	require.NoError(t, s.StartAndWaitReady(tempoQuery))
-	require.NoError(t, s.StartAndWaitReady(jaegerQuery))
+		batch := makeThriftBatchWithSpanCountForServiceAndOp(2, "execute", "backend")
+		require.NoError(t, h.WriteJaegerBatch(batch, ""))
 
-	jaegerClient, err := util.NewJaegerToOTLPExporter(tempo.Endpoint(4317))
-	require.NoError(t, err)
-	require.NotNil(t, jaegerClient)
+		batch = makeThriftBatchWithSpanCountForServiceAndOp(2, "request", "frontend")
+		require.NoError(t, h.WriteJaegerBatch(batch, ""))
 
-	batch := makeThriftBatchWithSpanCountForServiceAndOp(2, "execute", "backend")
-	require.NoError(t, jaegerClient.EmitBatch(context.Background(), batch))
+		// wait for the 2 traces to be written to the live stores
+		h.WaitTracesQueryable(t, 2)
 
-	batch = makeThriftBatchWithSpanCountForServiceAndOp(2, "request", "frontend")
-	require.NoError(t, jaegerClient.EmitBatch(context.Background(), batch))
+		callJaegerQuerySearchServicesAssert(t, jaegerQuery, servicesOrOpJaegerQueryResponse{
+			Data: []string{
+				"frontend",
+				"backend",
+			},
+			Total: 2,
+		})
 
-	// wait for the 2 traces to be written to the WAL
-	require.NoError(t, tempo.WaitSumMetricsWithOptions(e2e.Equals(2), []string{"tempo_ingester_traces_created_total"}, e2e.WaitMissingMetrics))
+		callJaegerQuerySearchOperationAssert(t, jaegerQuery, "frontend", servicesOrOpJaegerQueryResponse{
+			Data: []string{
+				"execute",
+				"request",
+			},
+			Total: 2,
+		})
 
-	callJaegerQuerySearchServicesAssert(t, tempo, jaegerQuery, servicesOrOpJaegerQueryResponse{
-		Data: []string{
-			"frontend",
-			"backend",
-		},
-		Total: 2,
-	})
+		callJaegerQuerySearchOperationAssert(t, jaegerQuery, "backend", servicesOrOpJaegerQueryResponse{
+			Data: []string{
+				"execute",
+				"request",
+			},
+			Total: 2,
+		})
 
-	callJaegerQuerySearchOperationAssert(t, jaegerQuery, "frontend", servicesOrOpJaegerQueryResponse{
-		Data: []string{
-			"execute",
-			"request",
-		},
-		Total: 2,
-	})
-
-	callJaegerQuerySearchOperationAssert(t, jaegerQuery, "backend", servicesOrOpJaegerQueryResponse{
-		Data: []string{
-			"execute",
-			"request",
-		},
-		Total: 2,
-	})
-
-	callJaegerQuerySearchTraceAssert(t, jaegerQuery, "request", "frontend")
-	callJaegerQuerySearchTraceAssert(t, jaegerQuery, "execute", "backend")
-}
-
-func TestSearchUsingBackendTagsService(t *testing.T) {
-	t.Parallel()
-
-	s, err := e2e.NewScenario("tempo_query_plugin_backend_e2e")
-	require.NoError(t, err)
-	defer s.Close()
-
-	require.NoError(t, util.CopyFileToSharedDir(s, "config-plugin-test.yaml", "config.yaml"))
-	require.NoError(t, util.CopyFileToSharedDir(s, "config-tempo-query.yaml", "config-tempo-query.yaml"))
-
-	tempo := util.NewTempoAllInOne()
-	tempoQuery := util.NewTempoQuery()
-	jaegerQuery := util.NewJaegerQuery()
-
-	require.NoError(t, s.StartAndWaitReady(tempo))
-	require.NoError(t, s.StartAndWaitReady(tempoQuery))
-	require.NoError(t, s.StartAndWaitReady(jaegerQuery))
-
-	jaegerClient, err := util.NewJaegerToOTLPExporter(tempo.Endpoint(4317))
-	require.NoError(t, err)
-	require.NotNil(t, jaegerClient)
-
-	batch := makeThriftBatchWithSpanCountForServiceAndOp(2, "execute", "backend")
-	require.NoError(t, jaegerClient.EmitBatch(context.Background(), batch))
-
-	batch = makeThriftBatchWithSpanCountForServiceAndOp(2, "request", "frontend")
-	require.NoError(t, jaegerClient.EmitBatch(context.Background(), batch))
-
-	// wait for the 2 traces to be written to the WAL
-	require.NoError(t, tempo.WaitSumMetricsWithOptions(e2e.Equals(2), []string{"tempo_ingester_traces_created_total"}, e2e.WaitMissingMetrics))
-
-	callJaegerQuerySearchServicesAssert(t, tempo, jaegerQuery, servicesOrOpJaegerQueryResponse{
-		Data: []string{
-			"frontend",
-			"backend",
-		},
-		Total: 2,
+		callJaegerQuerySearchTraceAssert(t, jaegerQuery, "request", "frontend")
+		callJaegerQuerySearchTraceAssert(t, jaegerQuery, "execute", "backend")
 	})
 }
 
-func callJaegerQuerySearchServicesAssert(t *testing.T, tempo, svc *e2e.HTTPService, expected servicesOrOpJaegerQueryResponse) {
+func callJaegerQuerySearchServicesAssert(t *testing.T, svc *e2e.HTTPService, expected servicesOrOpJaegerQueryResponse) {
 	assert.Eventually(t, func() bool {
-		util.CallFlush(t, tempo)
 		// search for tag values
 		req, err := http.NewRequest(http.MethodGet, "http://"+svc.Endpoint(16686)+"/api/services", nil)
 		require.NoError(t, err)

integration/operations/receivers_test.go

@@ -0,0 +1,206 @@
+package deployments
+
+import (
+	"context"
+	crand "crypto/rand"
+	"testing"
+
+	"github.com/grafana/dskit/user"
+	"github.com/open-telemetry/opentelemetry-collector-contrib/exporter/zipkinexporter"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+	"go.opentelemetry.io/collector/component"
+	"go.opentelemetry.io/collector/component/componenttest"
+	"go.opentelemetry.io/collector/config/configgrpc"
+	"go.opentelemetry.io/collector/config/confighttp"
+	"go.opentelemetry.io/collector/config/configtls"
+	"go.opentelemetry.io/collector/exporter"
+	"go.opentelemetry.io/collector/exporter/otlpexporter"
+	"go.opentelemetry.io/collector/exporter/otlphttpexporter"
+	"go.opentelemetry.io/collector/pdata/ptrace"
+	metricnoop "go.opentelemetry.io/otel/metric/noop"
+	tracenoop "go.opentelemetry.io/otel/trace/noop"
+	"go.uber.org/zap"
+
+	"github.com/grafana/tempo/integration/util"
+	"github.com/grafana/tempo/pkg/tempopb"
+	tempoUtil "github.com/grafana/tempo/pkg/util"
+	"github.com/grafana/tempo/pkg/util/test"
+)
+
+const configReceivers = `config-receivers.yaml`
+
+func TestReceivers(t *testing.T) {
+	util.RunIntegrationTests(t, util.TestHarnessConfig{
+		DeploymentMode: util.DeploymentModeSingleBinary,
+		ConfigOverlay:  configReceivers,
+	}, func(h *util.TempoHarness) {
+		h.WaitTracesWritable(t)
+
+		testReceivers := []struct {
+			name           string
+			createExporter func() (exporter.Traces, error)
+		}{
+			{
+				name: "otlp gRPC",
+				createExporter: func() (exporter.Traces, error) {
+					factory := otlpexporter.NewFactory()
+					cfg := factory.CreateDefaultConfig().(*otlpexporter.Config)
+					cfg.ClientConfig = configgrpc.ClientConfig{
+						Endpoint: h.Services[util.ServiceDistributor].Endpoint(4317),
+						TLS: configtls.ClientConfig{
+							Insecure: true,
+						},
+					}
+
+					logger, _ := zap.NewDevelopment()
+					return factory.CreateTraces(
+						context.Background(),
+						exporter.Settings{
+							ID: component.NewID(factory.Type()),
+							TelemetrySettings: component.TelemetrySettings{
+								Logger:         logger,
+								TracerProvider: tracenoop.NewTracerProvider(),
+								MeterProvider:  metricnoop.NewMeterProvider(),
+							},
+							BuildInfo: component.NewDefaultBuildInfo(),
+						},
+						cfg,
+					)
+				},
+			},
+			{
+				name: "otlp HTTP",
+				createExporter: func() (exporter.Traces, error) {
+					factory := otlphttpexporter.NewFactory()
+					cfg := factory.CreateDefaultConfig().(*otlphttpexporter.Config)
+					cfg.ClientConfig = confighttp.ClientConfig{
+						Endpoint: "http://" + h.Services[util.ServiceDistributor].Endpoint(4318),
+						TLS: configtls.ClientConfig{
+							Insecure: true,
+						},
+					}
+
+					logger, _ := zap.NewDevelopment()
+					return factory.CreateTraces(
+						context.Background(),
+						exporter.Settings{
+							ID: component.NewID(factory.Type()),
+							TelemetrySettings: component.TelemetrySettings{
+								Logger:         logger,
+								TracerProvider: tracenoop.NewTracerProvider(),
+								MeterProvider:  metricnoop.NewMeterProvider(),
+							},
+							BuildInfo: component.NewDefaultBuildInfo(),
+						},
+						cfg,
+					)
+				},
+			},
+			{
+				name: "zipkin",
+				createExporter: func() (exporter.Traces, error) {
+					factory := zipkinexporter.NewFactory()
+					cfg := factory.CreateDefaultConfig().(*zipkinexporter.Config)
+					cfg.ClientConfig = confighttp.ClientConfig{
+						Endpoint: "http://" + h.Services[util.ServiceDistributor].Endpoint(9411),
+						TLS: configtls.ClientConfig{
+							Insecure: true,
+						},
+					}
+					cfg.Format = "json"
+
+					logger, _ := zap.NewDevelopment()
+					return factory.CreateTraces(
+						context.Background(),
+						exporter.Settings{
+							ID: component.NewID(factory.Type()),
+							TelemetrySettings: component.TelemetrySettings{
+								Logger:         logger,
+								TracerProvider: tracenoop.NewTracerProvider(),
+								MeterProvider:  metricnoop.NewMeterProvider(),
+							},
+							BuildInfo: component.NewDefaultBuildInfo(),
+						},
+						cfg,
+					)
+				},
+			},
+			{
+				name: "jaeger gRPC",
+				createExporter: func() (exporter.Traces, error) {
+					return util.NewJaegerGRPCExporter(h.Services[util.ServiceDistributor].Endpoint(14250))
+				},
+			},
+		}
+
+		for i, tc := range testReceivers {
+			t.Run(tc.name, func(t *testing.T) {
+				// create exporter
+				exp, err := tc.createExporter()
+				require.NoError(t, err)
+
+				err = exp.Start(context.Background(), componenttest.NewNopHost())
+				require.NoError(t, err)
+
+				// make request
+				traceID := make([]byte, 16)
+				_, err = crand.Read(traceID)
+				require.NoError(t, err)
+				req := test.MakeTrace(20, traceID)
+
+				// zipkin doesn't support events and will 400 if you attempt to push one. just strip
+				// all events from the trace here
+				if tc.name == "zipkin" {
+					for _, b := range req.ResourceSpans {
+						for _, ss := range b.ScopeSpans {
+							for _, s := range ss.Spans {
+								s.Events = nil
+							}
+						}
+					}
+				}
+
+				b, err := req.Marshal()
+				require.NoError(t, err)
+
+				// unmarshal into otlp proto
+				traces, err := (&ptrace.ProtoUnmarshaler{}).UnmarshalTraces(b)
+				require.NoError(t, err)
+				require.NotNil(t, traces)
+
+				ctx := user.InjectOrgID(context.Background(), tempoUtil.FakeTenantID)
+				ctx, err = user.InjectIntoGRPCRequest(ctx)
+				require.NoError(t, err)
+
+				// send traces to tempo
+				err = exp.ConsumeTraces(ctx, traces)
+				require.NoError(t, err)
+
+				// shutdown to ensure traces are flushed
+				require.NoError(t, exp.Shutdown(context.Background()))
+
+				expectedTraces := i + 1
+				h.WaitTracesQueryable(t, expectedTraces)
+
+				// query for the trace
+				trace, err := h.APIClientHTTP("").QueryTrace(tempoUtil.TraceIDToHexString(traceID))
+				require.NoError(t, err)
+
+				// just compare spanCount because otel flattens all ILS into one
+				assert.Equal(t, spanCount(req), spanCount(trace))
+			})
+		}
+	})
+}
+
+func spanCount(a *tempopb.Trace) float64 {
+	count := 0
+	for _, batch := range a.ResourceSpans {
+		for _, spans := range batch.ScopeSpans {
+			count += len(spans.Spans)
+		}
+	}
+
+	return float64(count)
+}

integration/poller/config-azurite.yaml

@@ -1,54 +0,0 @@
-target: all
-stream_over_http_enabled: true
-
-server:
-  http_listen_port: 3200
-
-query_frontend:
-  search:
-    query_backend_after: 0 # setting these both to 0 will force all range searches to hit the backend
-    query_ingesters_until: 0
-
-distributor:
-  receivers:
-    jaeger:
-      protocols:
-        grpc:
-
-ingester:
-  lifecycler:
-    address: 127.0.0.1
-    ring:
-      kvstore:
-        store: inmemory
-      replication_factor: 1
-    final_sleep: 0s
-  trace_idle_period: 1s
-  max_block_duration: 5s
-  complete_block_timeout: 5s
-  flush_check_period: 1s
-
-storage:
-  trace:
-    blocklist_poll: 1s
-    backend: azure
-    azure:
-      container_name: tempo                    # how to store data in azure
-      endpoint_suffix: tempo_e2e-azurite:10000
-      storage_account_name: "devstoreaccount1"
-      storage_account_key: "Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw=="
-    pool:
-      max_workers: 10
-      queue_depth: 100
-
-overrides:
-  user_configurable_overrides:
-    enabled: true
-    poll_interval: 10s
-    client:
-      backend: azure
-      azure:
-        container_name: tempo
-        endpoint_suffix: tempo_e2e-azurite:10000
-        storage_account_name: "devstoreaccount1"
-        storage_account_key: "Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw=="

integration/poller/config-gcs.yaml

@@ -1,54 +0,0 @@
-target: all
-stream_over_http_enabled: true
-
-server:
-  http_listen_port: 3200
-
-query_frontend:
-  search:
-    query_ingesters_until: 0 # setting these both to 0 will force all range searches to hit the backend
-    query_backend_after: 0
-
-distributor:
-  receivers:
-    jaeger:
-      protocols:
-        grpc:
-
-ingester:
-  lifecycler:
-    address: 127.0.0.1
-    ring:
-      kvstore:
-        store: inmemory
-      replication_factor: 1
-    final_sleep: 0s
-  trace_idle_period: 1s
-  max_block_duration: 5s
-  complete_block_timeout: 5s
-  flush_check_period: 1s
-
-storage:
-  trace:
-    blocklist_poll: 1s
-    backend: gcs
-    gcs:
-      bucket_name: tempo
-      endpoint: https://tempo_e2e-gcs:4443/storage/v1/
-      insecure: true
-    pool:
-      max_workers: 10
-      queue_depth: 1000
-
-overrides:
-  user_configurable_overrides:
-    enabled: true
-    poll_interval: 10s
-    client:
-      backend: gcs
-      # fsouza/fake-gcs-server does not support versioning
-      confirm_versioning: false
-      gcs:
-        bucket_name: tempo
-        endpoint: https://tempo_e2e-gcs:4443/storage/v1/
-        insecure: true

integration/poller/config-s3.yaml

@@ -1,57 +0,0 @@
-target: all
-stream_over_http_enabled: true
-
-server:
-  http_listen_port: 3200
-
-query_frontend:
-  search:
-    query_backend_after: 0 # setting these both to 0 will force all range searches to hit the backend
-    query_ingesters_until: 0
-
-distributor:
-  receivers:
-    jaeger:
-      protocols:
-        grpc:
-
-ingester:
-  lifecycler:
-    address: 127.0.0.1
-    ring:
-      kvstore:
-        store: inmemory
-      replication_factor: 1
-    final_sleep: 0s
-  trace_idle_period: 1s
-  max_block_duration: 5s
-  complete_block_timeout: 5s
-  flush_check_period: 1s
-
-storage:
-  trace:
-    blocklist_poll: 1s
-    backend: s3
-    s3:
-      bucket: tempo
-      endpoint: tempo-integration-minio-9000:9000  # TODO: this is brittle, fix this eventually
-      access_key: Cheescake # TODO: use cortex_e2e.MinioAccessKey
-      secret_key: supersecret # TODO: use cortex_e2e.MinioSecretKey
-      insecure: true
-    pool:
-      max_workers: 10
-      queue_depth: 100
-
-overrides:
-  user_configurable_overrides:
-    enabled: true
-    poll_interval: 10s
-    client:
-      backend: s3
-      s3:
-        # TODO use separate bucket?
-        bucket: tempo
-        endpoint: tempo-integration-minio-9000:9000  # TODO: this is brittle, fix this eventually
-        access_key: Cheescake # TODO: use cortex_e2e.MinioAccessKey
-        secret_key: supersecret # TODO: use cortex_e2e.MinioSecretKey
-        insecure: true

integration/storage/backend_scheduler_test.go

@@ -0,0 +1,352 @@
+package storage
+
+import (
+	"context"
+	"fmt"
+	"strconv"
+	"testing"
+	"time"
+
+	"github.com/go-kit/log"
+	"github.com/grafana/e2e"
+	"github.com/grafana/tempo/integration/util"
+	"github.com/grafana/tempo/pkg/model"
+	"github.com/grafana/tempo/pkg/tempopb"
+	"github.com/grafana/tempo/pkg/util/test"
+	"github.com/grafana/tempo/tempodb"
+	"github.com/grafana/tempo/tempodb/backend"
+	"github.com/grafana/tempo/tempodb/encoding"
+	"github.com/grafana/tempo/tempodb/encoding/common"
+	"github.com/grafana/tempo/tempodb/wal"
+	io_prometheus_client "github.com/prometheus/client_model/go"
+	"github.com/prometheus/prometheus/model/labels"
+	"github.com/stretchr/testify/require"
+)
+
+func TestBackendScheduler(t *testing.T) {
+	util.RunIntegrationTests(t, util.TestHarnessConfig{
+		Components:    util.ComponentsBackendWork,
+		Backends:      util.BackendObjectStorageAll,
+		ConfigOverlay: "config-backend-scheduler.yaml",
+	}, func(h *util.TempoHarness) {
+		ctx, cancel := context.WithCancel(context.Background())
+		defer cancel()
+
+		s := h.TestScenario
+
+		// Get the config from the harness
+		cfg, err := h.GetConfig()
+		require.NoError(t, err)
+
+		objStorage := h.Services[util.ServiceObjectStorage]
+		backendEndpoint := objStorage.HTTPEndpoint()
+		t.Logf("Endpoint: %s", backendEndpoint)
+
+		// Get the scheduler and worker services from the harness
+		scheduler := h.Services[util.ServiceBackendScheduler]
+		worker := h.Services[util.ServiceBackendWorker]
+
+		// Stop the worker initially - we'll start it later after populating data
+		require.NoError(t, s.Stop(worker))
+
+		// Setup tempodb with local backend
+		tempodbWriter := setupBackendWithEndpoint(t, &cfg.StorageConfig.Trace, backendEndpoint)
+
+		cases := []struct {
+			name                      string
+			tenantCount               int
+			blockCount                int
+			expectedBlocks            int // accumulated expected blocks.  Each test adds this to the total
+			expectedOutstandingBlocks int // accumulated expected outstanding blocks.  Each test adds this to the total, but 1 outstanding block is between 2-4 blocks.
+		}{
+			{
+				name:                      "a bunch of tenants with 1 block each is 0 outstanding",
+				tenantCount:               11,
+				blockCount:                1,
+				expectedBlocks:            1,
+				expectedOutstandingBlocks: 0,
+			},
+			{
+				name:                      "a bunch of tenants with 12 blocks each 12 outstanding",
+				tenantCount:               11,
+				blockCount:                12,
+				expectedBlocks:            12,
+				expectedOutstandingBlocks: 12,
+			},
+		}
+		var tenants []string
+
+		type expectations struct {
+			blocks      int
+			outstanding int
+		}
+
+		// We continue to write blocks through these tests, so our expectations need
+		// to grow as well.
+		accumulant := make(map[string]*expectations)
+		totalOutstanding := 0
+		totalBlocksWritten := 0
+
+		for _, tc := range cases {
+			t.Run(tc.name, func(t *testing.T) {
+				// Populate the backend with the specified number of tenants, blocks, and records
+				tenants = populateBackend(ctx, t, tempodbWriter, tc.tenantCount, tc.blockCount, 1)
+
+				totalBlocksWritten += tc.tenantCount * tc.blockCount
+
+				if _, ok := accumulant[tenant]; !ok {
+					accumulant[tenant] = &expectations{}
+				}
+
+				accumulant[tenant].blocks += tc.expectedBlocks
+				accumulant[tenant].outstanding += tc.expectedOutstandingBlocks
+
+				// Check the metrics for each tenant
+				for _, tenantID := range tenants {
+					tenantMatcher := e2e.WithLabelMatchers(&labels.Matcher{Type: labels.MatchEqual, Name: "tenant", Value: tenantID})
+
+					expectedBlocks := accumulant[tenant].blocks
+					expectedOutstanding := accumulant[tenant].outstanding
+					totalOutstanding += expectedOutstanding
+
+					t.Logf("Waiting for %d blocks in the blocklist for tenant: %s", expectedBlocks, tenantID)
+
+					require.NoError(t, scheduler.WaitSumMetricsWithOptions(e2e.Equals(float64(expectedBlocks)), []string{"tempodb_blocklist_length"},
+						e2e.WaitMissingMetrics,
+						tenantMatcher,
+						printMetricValue(t, fmt.Sprintf("%d", expectedBlocks), "tempodb_blocklist_length"),
+					))
+				}
+			})
+		}
+
+		time.Sleep(8 * time.Second) // Wait for polling and measurement to catch up
+
+		// Capture the total blocks which have been written
+		totalBlocksPreCompactions, err := scheduler.SumMetrics([]string{"tempodb_blocklist_length"})
+		require.NoError(t, err)
+		require.Len(t, totalBlocksPreCompactions, 1, "expected only one blocklist length metric")
+		require.Equal(t, float64(totalBlocksWritten), totalBlocksPreCompactions[0], "expected total blocks to match the sum of all tenants")
+
+		// NOTE: the compaction provider has a channel capacity of 1, and the
+		// backendscheduler has a channel capacity of 1.  This means that the
+		// compaction provider can create 4 jobs before they are processed in the
+		// Next call of the scheduler.
+		// - one recorded before pushing into the compaction provider channel
+		// - one in the compaction provider channel
+		// - one between the provider pull and the scheduler push
+		// - one in the merged channel
+
+		// NOTE: Since the measurement of outstanding blocks also skips the blocks
+		// from jobs which have not been processed, we expect our outstanding blocks
+		// to be N-M, where N is the actual outstanding blocks and M is the number of
+		// blocks attached to jobs which have not been recorded by the scheduler. The
+		// order of tenants is non-defined when all tenants have the same block
+		// count, so we don't know which tenant will be first. Measure total
+		// outstanding blocks instead of per tenant for this reason.
+
+		// NOTE: Due to timing and window selection, we may not fully populate a job,
+		// and we may end up with jobs which between 2 and 4 blocks outstanding.
+		// - 4 jobs not yet recorded by the scheduler (no worker running yet)
+		// - between 2 and 4 blocks per job (default settings)
+		expectedTotalOutstandingMin := totalOutstanding - 16
+		expectedTotalOutstandingMax := totalOutstanding - 8
+
+		outstanding, err := scheduler.SumMetrics([]string{"tempodb_compaction_outstanding_blocks"})
+		require.NoError(t, err)
+		require.Len(t, outstanding, 1, "expected only one outstanding block metric")
+		t.Logf("Total outstanding blocks: %f", outstanding[0])
+		t.Logf("Expected outstanding blocks range: [%d, %d]", expectedTotalOutstandingMin, expectedTotalOutstandingMax)
+		t.Logf("Total outstanding blocks: %d", totalOutstanding)
+		require.LessOrEqual(t, outstanding[0], float64(expectedTotalOutstandingMax), "unexpected maximum outstanding blocks count")
+		require.GreaterOrEqual(t, outstanding[0], float64(expectedTotalOutstandingMin), "unexpected minimum outstanding blocks count")
+
+		// Delay starting the work to ensure we have a clean state of the data before
+		// the worker starts processing jobs.
+		require.NoError(t, s.StartAndWaitReady(worker))
+
+		// Allow the worker some time to process the blocks.
+		// Wait until the last tenant has processed its blocks
+
+		for _, tenantID := range tenants {
+			t.Run(fmt.Sprintf("work-finished-check-wait-%s", tenantID), func(t *testing.T) {
+				tenantMatcher := e2e.WithLabelMatchers(&labels.Matcher{Type: labels.MatchEqual, Name: "tenant", Value: tenantID})
+
+				// We should have at least 1 block for each tenant
+				require.NoError(t, scheduler.WaitSumMetricsWithOptions(e2e.GreaterOrEqual(1.0), []string{"tempodb_blocklist_length"},
+					e2e.WaitMissingMetrics,
+					tenantMatcher,
+					printMetricValue(t, fmt.Sprintf("%f+", 1.0), "tempodb_blocklist_length"),
+				))
+			})
+		}
+
+		time.Sleep(5 * time.Second)
+
+		// Stop the worker to ensure it is not running while we check the metrics.
+		require.NoError(t, s.Stop(worker))
+
+		// Require that all jobs which have been processed are not failed.
+		_, err = scheduler.SumMetrics([]string{"tempo_backend_scheduler_jobs_failed_total"})
+		require.Error(t, err, "metric not found")
+
+		totalBlocksPostCompactions, err := scheduler.SumMetrics([]string{"tempodb_blocklist_length"})
+		require.NoError(t, err)
+		require.Len(t, totalBlocksPostCompactions, 1, "expected only one blocklist length metric")
+		require.Less(t, totalBlocksPostCompactions[0], totalBlocksPreCompactions[0], "expected total blocks to be less after compaction")
+
+		// Some variance in the number of expected due to the notes above.  We should
+		// expect that a fully compacted tenant has between 1 and 4 blocks, and
+		// between 0 and 1 outstanding blocks.  The sleep above should be enough to
+		// allow the worker to finish processing all the outstanding blocks.
+		expectedMin := &expectations{
+			blocks:      1,
+			outstanding: 0,
+		}
+
+		expectedMax := &expectations{
+			blocks:      4,
+			outstanding: 1,
+		}
+
+		for _, tenantID := range tenants {
+			t.Run(fmt.Sprintf("prestop-check-%s", tenantID), func(t *testing.T) {
+				tenantMatcher := e2e.WithLabelMatchers(&labels.Matcher{Type: labels.MatchEqual, Name: "tenant", Value: tenantID})
+
+				t.Logf("Waiting for blocklist metric: %s: %d-%d", tenantID, expectedMin.blocks, expectedMax.blocks)
+
+				require.NoError(t, scheduler.WaitSumMetricsWithOptions(e2e.Between(float64(expectedMin.blocks), float64(expectedMax.blocks)), []string{"tempodb_blocklist_length"},
+					e2e.WaitMissingMetrics,
+					tenantMatcher,
+					printMetricValue(t, fmt.Sprintf("%d-%d", expectedMin.blocks, expectedMax.blocks), "tempodb_blocklist_length"),
+				))
+
+				t.Logf("Waiting for outstanding blocks metric: %s: %d-%d", tenantID, expectedMin.outstanding, expectedMax.outstanding)
+
+				require.NoError(t, scheduler.WaitSumMetricsWithOptions(e2e.Between(float64(expectedMin.outstanding), float64(expectedMax.outstanding)), []string{"tempodb_compaction_outstanding_blocks"},
+					e2e.WaitMissingMetrics,
+					tenantMatcher,
+					printMetricValue(t, fmt.Sprintf("%d-%d", expectedMin.outstanding, expectedMax.outstanding), "tempodb_compaction_outstanding_blocks"),
+				))
+			})
+		}
+
+		// Stop and restart the scheduler to ensure we replay the state correctly.
+		// It should match the outstanding blocks above since we have not modified
+		// the backend.
+		require.NoError(t, s.Stop(scheduler))
+		require.NoError(t, s.StartAndWaitReady(scheduler, worker))
+
+		time.Sleep(2 * time.Second)
+
+		for _, tenantID := range tenants {
+			t.Run(fmt.Sprintf("final-check-%s", tenantID), func(t *testing.T) {
+				tenantMatcher := e2e.WithLabelMatchers(&labels.Matcher{Type: labels.MatchEqual, Name: "tenant", Value: tenantID})
+
+				t.Logf("Waiting for blocklist metric: %s: %d-%d", tenantID, expectedMin.blocks, expectedMax.blocks)
+
+				require.NoError(t, scheduler.WaitSumMetricsWithOptions(e2e.Between(float64(expectedMin.blocks), float64(expectedMax.blocks)), []string{"tempodb_blocklist_length"},
+					e2e.WaitMissingMetrics,
+					tenantMatcher,
+					printMetricValue(t, fmt.Sprintf("%d-%d", expectedMin.blocks, expectedMax.blocks), "tempodb_blocklist_length"),
+				))
+
+				t.Logf("Waiting for outstanding blocks metric: %s: %d-%d", tenantID, expectedMin.outstanding, expectedMax.outstanding)
+
+				require.NoError(t, scheduler.WaitSumMetricsWithOptions(e2e.Between(float64(expectedMin.outstanding), float64(expectedMax.outstanding)), []string{"tempodb_compaction_outstanding_blocks"},
+					e2e.WaitMissingMetrics,
+					tenantMatcher,
+					printMetricValue(t, fmt.Sprintf("%d-%d", expectedMin.outstanding, expectedMax.outstanding), "tempodb_compaction_outstanding_blocks"),
+				))
+			})
+		}
+	})
+}
+
+func printValues(t *testing.T, expected string, metric string) e2e.GetMetricValueFunc {
+	return func(m *io_prometheus_client.Metric) float64 {
+		v := e2e.DefaultMetricsOptions.GetValue(m)
+		t.Logf("metric %q: label %q: current %f, expected %s", metric, m.GetLabel()[0].GetValue(), v, expected)
+		return v
+	}
+}
+
+func printMetricValue(t *testing.T, expectedValue string, metric string) e2e.MetricsOption {
+	return func(opts *e2e.MetricsOptions) {
+		opts.GetValue = printValues(t, expectedValue, metric)
+	}
+}
+
+func setupBackendWithEndpoint(t testing.TB, cfg *tempodb.Config, endpoint string) tempodb.Writer {
+	cfg.Block = &common.BlockConfig{
+		IndexDownsampleBytes: 11,
+		BloomFP:              .01,
+		BloomShardSizeBytes:  100_000,
+		Version:              encoding.LatestEncoding().Version(),
+		Encoding:             backend.EncNone,
+		IndexPageSizeBytes:   1000,
+		RowGroupSizeBytes:    30_000_000,
+		DedicatedColumns:     backend.DedicatedColumns{{Scope: "span", Name: "key", Type: "string"}},
+	}
+	cfg.WAL = &wal.Config{
+		Filepath: t.TempDir(),
+	}
+	if cfg.S3 != nil {
+		cfg.S3.Endpoint = endpoint
+	}
+	if cfg.Azure != nil {
+		cfg.Azure.Endpoint = endpoint
+	}
+	if cfg.GCS != nil {
+		cfg.GCS.Endpoint = endpoint
+	}
+
+	_, w, _, err := tempodb.New(cfg, nil, log.NewNopLogger())
+	require.NoError(t, err)
+
+	return w
+}
+
+func populateBackend(ctx context.Context, t testing.TB, w tempodb.Writer, tenantCount, blockCount, recordCount int) []string {
+	wal := w.WAL()
+
+	dec := model.MustNewSegmentDecoder(model.CurrentEncoding)
+
+	tenants := make([]string, tenantCount)
+
+	for i := range tenantCount {
+		tenantID := tenant + strconv.Itoa(i)
+
+		tenants[i] = tenantID
+
+		for range blockCount {
+			blockID := backend.NewUUID()
+			meta := &backend.BlockMeta{BlockID: blockID, TenantID: tenantID, DataEncoding: model.CurrentEncoding}
+			head, err := wal.NewBlock(meta, model.CurrentEncoding)
+			require.NoError(t, err)
+
+			for range recordCount {
+				id := test.ValidTraceID(nil)
+				req := test.MakeTrace(10, id)
+
+				writeTraceToWal(t, head, dec, id, req, 0, 0)
+			}
+
+			_, err = w.CompleteBlock(ctx, head)
+			require.NoError(t, err)
+		}
+	}
+
+	return tenants
+}
+
+func writeTraceToWal(t require.TestingT, b common.WALBlock, dec model.SegmentDecoder, id common.ID, tr *tempopb.Trace, start, end uint32) {
+	b1, err := dec.PrepareForWrite(tr, 0, 0)
+	require.NoError(t, err)
+
+	b2, err := dec.ToObject([][]byte{b1})
+	require.NoError(t, err)
+
+	err = b.Append(id, b2, start, end, true)
+	require.NoError(t, err, "unexpected error writing req")
+}

integration/storage/config-backend-scheduler.yaml

@@ -0,0 +1,3 @@
+storage:
+  trace:
+    blocklist_poll_stale_tenant_index: 1s # force components to always fall back to polling

integration/storage/config-encodings.yaml

@@ -0,0 +1,19 @@
+storage:
+  trace:
+    block:
+      version: "{{.Version}}"
+      parquet_dedicated_columns:
+        - { scope: resource, name: vulture-process-0, type: string }
+        - { scope: resource, name: vulture-process-1, type: string }
+        - { scope: span, name: vulture-0, type: string }
+        - { scope: span, name: vulture-1, type: string }
+
+block_builder:
+  block:
+    version: "{{.Version}}"
+  wal:
+    version: "{{.Version}}"
+
+live_store:
+  block_config:
+    version: "{{.Version}}"