hanzo/traces
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

[Tempo 3.0] Remove compactor and v2 block encoding code (#6273)

Browse Source 
* o7 compactor

Signed-off-by: Joe Elliott <number101010@gmail.com>

* wip: o7 v2

Signed-off-by: Joe Elliott <number101010@gmail.com>

* test cleanup

Signed-off-by: Joe Elliott <number101010@gmail.com>

* remove compactor from jsonnet

Signed-off-by: Joe Elliott <number101010@gmail.com>

* changelog

Signed-off-by: Joe Elliott <number101010@gmail.com>

* gen manifest

Signed-off-by: Joe Elliott <number101010@gmail.com>

* config cleanup

Signed-off-by: Joe Elliott <number101010@gmail.com>

* docs

Signed-off-by: Joe Elliott <number101010@gmail.com>

* jsonnet

Signed-off-by: Joe Elliott <number101010@gmail.com>

* remove encoding

Signed-off-by: Joe Elliott <number101010@gmail.com>

* remove data encoding

Signed-off-by: Joe Elliott <number101010@gmail.com>

* lint and cleanup

Signed-off-by: Joe Elliott <number101010@gmail.com>

* unflake TestWorker\?

Signed-off-by: Joe Elliott <number101010@gmail.com>

---------

Signed-off-by: Joe Elliott <number101010@gmail.com>
This commit is contained in:
Joe Elliott
2026-02-22 00:55:18 -08:00
committed by Zach Kelling
parent c63f424583
commit 3ae918ff89
186 changed files with 623 additions and 10232 deletions
Download Patch File Download Diff File Expand all files Collapse all files
CHANGELOG.md
+5
View File
@@ -4,6 +4,11 @@
* [FEATURE] Add span_multiplier_key to overrides. This allows tenants to specify the attribute key used for span multiplier values to compensate for head-based sampling. [#6260](https://github.com/grafana/tempo/pull/6260) (@carles-grafana)
* [BUGFIX] Correct avg_over_time calculation [#6252](https://github.com/grafana/tempo/pull/6252) (@ruslan-mikhailov)
### 3.0 Cleanup
* [CHANGE] **BREAKING CHANGE** Removed `v2` block encoding and compactor component. [#6273](https://github.com/grafana/tempo/pull/6273) (@joe-elliott)
This includes the removal of the following CLI commands which were `v2` specific: `list block`, `list index`, `view index`, `gen index`, `gen bloom`.
# v2.10.0-rc.0
* [CHANGE] **BREAKING CHANGE** Validate tenant ID in frontend and distributor [#5786](https://github.com/grafana/tempo/pull/5786) (@carles-grafana)
CONTRIBUTING.md
+2 -1
View File
@@ -42,7 +42,8 @@ example/ - great place to get started running Tempo
tk/
integration/ - e2e tests
modules/ - top level Tempo components
compactor/
backend-worker/
backend-scheduler/
distributor/
ingester/
overrides/
cmd/tempo-cli/cmd-gen-bloom.go
-163
View File
@@ -1,163 +0,0 @@
package main
import (
"bytes"
"context"
"errors"
"fmt"
"io"
"os"
"strconv"
"github.com/google/uuid"
willf_bloom "github.com/willf/bloom"
"github.com/grafana/tempo/tempodb/backend"
"github.com/grafana/tempo/tempodb/encoding/common"
v2 "github.com/grafana/tempo/tempodb/encoding/v2"
)
type bloomCmd struct {
TenantID string `arg:"" help:"tenant-id within the bucket"`
BlockID string `arg:"" help:"block ID to list"`
BloomFP float64 `arg:"" help:"bloom filter false positive rate (use prod settings!)"`
BloomShardSize int `arg:"" help:"bloom filter shard size (use prod settings!)"`
backendOptions
}
type forEachRecord func(id common.ID) error
func ReplayBlockAndDoForEachRecord(meta *backend.BlockMeta, filepath string, forEach forEachRecord) error {
// replay file to extract records
f, err := os.OpenFile(filepath, os.O_RDONLY, 0o600)
if err != nil {
return err
}
dataReader, err := v2.NewDataReader(backend.NewContextReaderWithAllReader(f), meta.Encoding)
if err != nil {
return fmt.Errorf("error creating data reader: %w", err)
}
defer dataReader.Close()
var buffer []byte
objectRW := v2.NewObjectReaderWriter()
for {
buffer, _, err := dataReader.NextPage(buffer)
if errors.Is(err, io.EOF) {
break
}
if err != nil {
return fmt.Errorf("error reading page from datareader: %w", err)
}
iter := v2.NewIterator(bytes.NewReader(buffer), objectRW)
var iterErr error
for {
var id common.ID
id, _, iterErr = iter.NextBytes(context.TODO())
if iterErr != nil {
break
}
err := forEach(id)
if err != nil {
return fmt.Errorf("error adding to bloom filter: %w", err)
}
}
if !errors.Is(iterErr, io.EOF) {
return iterErr
}
}
return nil
}
func (cmd *bloomCmd) Run(ctx *globalOptions) error {
blockID, err := uuid.Parse(cmd.BlockID)
if err != nil {
return err
}
r, w, _, err := loadBackend(&cmd.backendOptions, ctx)
if err != nil {
return err
}
meta, err := r.BlockMeta(context.TODO(), blockID, cmd.TenantID)
if err != nil {
return err
}
if meta.Version != v2.VersionString {
return fmt.Errorf("unsupported block version: %s", meta.Version)
}
// replay file and add records to bloom filter
bloom := common.NewBloom(cmd.BloomFP, uint(cmd.BloomShardSize), uint(meta.TotalObjects))
if bloom.GetShardCount() != int(meta.BloomShardCount) {
err := fmt.Errorf("shards in generated bloom filter do not match block meta, please use prod settings for bloom shard size and FP")
fmt.Println(err.Error())
return err
}
addToBloom := func(id common.ID) error {
bloom.Add(id)
return nil
}
err = ReplayBlockAndDoForEachRecord(meta, cmd.backendOptions.Bucket+cmd.TenantID+"/"+cmd.BlockID+"/"+dataFilename, addToBloom)
if err != nil {
fmt.Println("error replaying block", err)
return err
}
// write to the local backend
bloomBytes, err := bloom.Marshal()
if err != nil {
fmt.Println("error marshalling bloom filter")
return err
}
for i := 0; i < len(bloomBytes); i++ {
err = w.Write(context.TODO(), bloomFilePrefix+strconv.Itoa(i), blockID, cmd.TenantID, bloomBytes[i], nil)
if err != nil {
fmt.Println("error writing bloom filter to backend", err)
return err
}
}
fmt.Println("bloom written to backend successfully")
// verify generated bloom
shardedBloomFilter := make([]*willf_bloom.BloomFilter, meta.BloomShardCount)
for i := 0; i < int(meta.BloomShardCount); i++ {
bloomBytes, err := r.Read(context.TODO(), bloomFilePrefix+strconv.Itoa(i), blockID, cmd.TenantID, nil)
if err != nil {
fmt.Println("error reading bloom from backend")
return nil
}
shardedBloomFilter[i] = &willf_bloom.BloomFilter{}
_, err = shardedBloomFilter[i].ReadFrom(bytes.NewReader(bloomBytes))
if err != nil {
fmt.Println("error parsing bloom")
return nil
}
}
testBloom := func(id common.ID) error {
key := common.ShardKeyForTraceID(id, int(meta.BloomShardCount))
if !shardedBloomFilter[key].Test(id) {
return fmt.Errorf("id not added to bloom, filter is likely corrupt")
}
return nil
}
err = ReplayBlockAndDoForEachRecord(meta, cmd.backendOptions.Bucket+cmd.TenantID+"/"+cmd.BlockID+"/"+dataFilename, testBloom)
if err != nil {
fmt.Println("error replaying block", err)
return err
}
fmt.Println("bloom filter verified")
return nil
}
cmd/tempo-cli/cmd-gen-index.go
-189
View File
@@ -1,189 +0,0 @@
package main
import (
"bytes"
"context"
"errors"
"fmt"
"io"
"os"
"github.com/google/uuid"
"github.com/grafana/tempo/tempodb/backend"
"github.com/grafana/tempo/tempodb/encoding/common"
v2 "github.com/grafana/tempo/tempodb/encoding/v2"
)
type indexCmd struct {
TenantID string `arg:"" help:"tenant-id within the bucket"`
BlockID string `arg:"" help:"block ID to list"`
backendOptions
}
func ReplayBlockAndGetRecords(meta *backend.BlockMeta, filepath string) ([]v2.Record, error, error) {
var replayError error
// replay file to extract records
f, err := os.OpenFile(filepath, os.O_RDONLY, 0o600)
if err != nil {
return nil, nil, err
}
dataReader, err := v2.NewDataReader(backend.NewContextReaderWithAllReader(f), meta.Encoding)
if err != nil {
return nil, nil, err
}
defer dataReader.Close()
var buffer []byte
var records []v2.Record
objectRW := v2.NewObjectReaderWriter()
currentOffset := uint64(0)
for {
buffer, pageLen, err := dataReader.NextPage(buffer)
if errors.Is(err, io.EOF) {
break
}
if err != nil {
replayError = err
break
}
iter := v2.NewIterator(bytes.NewReader(buffer), objectRW)
var lastID common.ID
var iterErr error
for {
var id common.ID
id, _, iterErr = iter.NextBytes(context.TODO())
if iterErr != nil {
break
}
lastID = id
}
if !errors.Is(iterErr, io.EOF) {
replayError = iterErr
break
}
// make a copy so we don't hold onto the iterator buffer
recordID := append([]byte(nil), lastID...)
records = append(records, v2.Record{
ID: recordID,
Start: currentOffset,
Length: pageLen,
})
currentOffset += uint64(pageLen)
}
return records, replayError, nil
}
func VerifyIndex(indexReader v2.IndexReader, dataReader v2.DataReader) error {
for i := 0; ; i++ {
record, err := indexReader.At(context.TODO(), i)
if err != nil {
return err
}
if record == nil {
break
}
// read data file at record position
_, _, err = dataReader.Read(context.TODO(), []v2.Record{*record}, nil, nil)
if err != nil {
fmt.Println("index/data is corrupt, record/data mismatch")
return err
}
}
return nil
}
func (cmd *indexCmd) Run(ctx *globalOptions) error {
blockID, err := uuid.Parse(cmd.BlockID)
if err != nil {
return err
}
r, w, _, err := loadBackend(&cmd.backendOptions, ctx)
if err != nil {
return err
}
meta, err := r.BlockMeta(context.TODO(), blockID, cmd.TenantID)
if err != nil {
return err
}
if meta.Version != v2.VersionString {
return fmt.Errorf("unsupported block version: %s", meta.Version)
}
// replay file to extract records
records, replayError, err := ReplayBlockAndGetRecords(meta, cmd.backendOptions.Bucket+cmd.TenantID+"/"+cmd.BlockID+"/"+dataFilename)
if replayError != nil {
fmt.Println("error replaying block. data file likely corrupt", replayError)
return replayError
}
if err != nil {
fmt.Println("error accessing data/meta file")
return err
}
// write using IndexWriter
indexWriter := v2.NewIndexWriter(int(meta.IndexPageSize))
indexBytes, err := indexWriter.Write(records)
if err != nil {
fmt.Println("error writing records to indexWriter", err)
return err
}
// write to the local backend
err = w.Write(context.TODO(), "index", blockID, cmd.TenantID, indexBytes, nil)
if err != nil {
fmt.Println("error writing index to backend", err)
return err
}
fmt.Println("index written to backend successfully")
// verify generated index
// get index file with records
indexFilePath := cmd.backendOptions.Bucket + cmd.TenantID + "/" + cmd.BlockID + "/" + indexFilename
indexFile, err := os.OpenFile(indexFilePath, os.O_RDONLY, 0o600)
if err != nil {
fmt.Println("error opening index file")
return err
}
indexReader, err := v2.NewIndexReader(backend.NewContextReaderWithAllReader(indexFile), int(meta.IndexPageSize), len(records))
if err != nil {
fmt.Println("error reading index file")
return err
}
// data reader
dataFilePath := cmd.backendOptions.Bucket + cmd.TenantID + "/" + cmd.BlockID + "/" + dataFilename
dataFile, err := os.OpenFile(dataFilePath, os.O_RDONLY, 0o600)
if err != nil {
fmt.Println("error opening data file")
return err
}
dataReader, err := v2.NewDataReader(backend.NewContextReaderWithAllReader(dataFile), meta.Encoding)
if err != nil {
fmt.Println("error reading data file")
return err
}
defer dataReader.Close()
err = VerifyIndex(indexReader, dataReader)
if err != nil {
return err
}
fmt.Println("index verified!")
return nil
}
cmd/tempo-cli/cmd-list-block.go
-258
View File
@@ -1,258 +0,0 @@
package main
import (
"bytes"
"context"
"errors"
"fmt"
"io"
"os"
"os/signal"
"sort"
"syscall"
"time"
"github.com/dustin/go-humanize"
"github.com/google/uuid"
"github.com/grafana/tempo/pkg/model"
"github.com/grafana/tempo/pkg/tempopb"
"github.com/grafana/tempo/pkg/util"
tempodb_backend "github.com/grafana/tempo/tempodb/backend"
"github.com/grafana/tempo/tempodb/encoding/common"
v2 "github.com/grafana/tempo/tempodb/encoding/v2"
)
type valueStats struct {
count int
}
type values struct {
all map[string]valueStats
key string
count int
}
type kvPairs map[string]values
type listBlockCmd struct {
backendOptions
TenantID string `arg:"" help:"tenant-id within the bucket"`
BlockID string `arg:"" help:"block ID to list"`
Scan bool `help:"scan contents of block for duplicate trace IDs and other info (warning, can be intense)"`
}
func (cmd *listBlockCmd) Run(ctx *globalOptions) error {
r, _, c, err := loadBackend(&cmd.backendOptions, ctx)
if err != nil {
return err
}
return dumpBlock(r, c, cmd.TenantID, time.Hour, cmd.BlockID, cmd.Scan)
}
func dumpBlock(r tempodb_backend.Reader, c tempodb_backend.Compactor, tenantID string, windowRange time.Duration, blockID string, scan bool) error {
id := uuid.MustParse(blockID)
meta, err := r.BlockMeta(context.TODO(), id, tenantID)
if err != nil && !errors.Is(err, tempodb_backend.ErrDoesNotExist) {
return err
}
compactedMeta, err := c.CompactedBlockMeta(id, tenantID)
if err != nil && !errors.Is(err, tempodb_backend.ErrDoesNotExist) {
return err
}
if meta == nil && compactedMeta == nil {
fmt.Println("Unable to load any meta for block", blockID)
return nil
}
unifiedMeta := getMeta(meta, compactedMeta, windowRange)
fmt.Println("ID : ", unifiedMeta.BlockID)
fmt.Println("Version : ", unifiedMeta.Version)
fmt.Println("Total Objects : ", unifiedMeta.TotalObjects)
fmt.Println("Data Size : ", humanize.Bytes(unifiedMeta.Size_))
fmt.Println("Encoding : ", unifiedMeta.Encoding)
fmt.Println("Level : ", unifiedMeta.CompactionLevel)
fmt.Println("Window : ", unifiedMeta.window)
fmt.Println("Start : ", unifiedMeta.StartTime)
fmt.Println("End : ", unifiedMeta.EndTime)
fmt.Println("Duration : ", fmt.Sprint(unifiedMeta.EndTime.Sub(unifiedMeta.StartTime).Round(time.Second)))
fmt.Println("Age : ", fmt.Sprint(time.Since(unifiedMeta.EndTime).Round(time.Second)))
if scan {
if unifiedMeta.Version != v2.VersionString {
return fmt.Errorf("cannot scan block contents. unsupported block version: %s", unifiedMeta.Version)
}
fmt.Println("Scanning block contents. Press CRTL+C to quit ...")
block, err := v2.NewBackendBlock(&unifiedMeta.BlockMeta, r)
if err != nil {
return err
}
iter, err := block.Iterator(uint32(2 * 1024 * 1024))
if err != nil {
return err
}
defer iter.Close()
// Scanning stats
i := 0
dupe := 0
maxObjSize := 0
minObjSize := 0
maxObjID := common.ID{}
minObjID := common.ID{}
allKVP := kvPairs{}
printStats := func() {
fmt.Println()
fmt.Println("Scanning results:")
fmt.Println("Objects scanned : ", i)
fmt.Println("Duplicates : ", dupe)
fmt.Println("Smallest object : ", humanize.Bytes(uint64(minObjSize)), " : ", util.TraceIDToHexString(minObjID))
fmt.Println("Largest object : ", humanize.Bytes(uint64(maxObjSize)), " : ", util.TraceIDToHexString(maxObjID))
fmt.Println("")
printKVPairs(allKVP)
}
// Print stats on ctrl+c
c := make(chan os.Signal, 1)
// nolint:govet
signal.Notify(c, os.Interrupt, syscall.SIGTERM)
go func() {
<-c
printStats()
os.Exit(0)
}()
ctx := context.Background()
prevID := make([]byte, 16)
for {
objID, obj, err := iter.NextBytes(ctx)
if errors.Is(err, io.EOF) {
break
} else if err != nil {
return err
}
if len(obj) > maxObjSize {
maxObjSize = len(obj)
maxObjID = objID
}
if len(obj) < minObjSize || minObjSize == 0 {
minObjSize = len(obj)
minObjID = objID
}
if bytes.Equal(objID, prevID) {
dupe++
}
copy(prevID, objID)
trace, err := model.MustNewObjectDecoder(unifiedMeta.DataEncoding).PrepareForRead(obj)
if err != nil {
return err
}
kvp := extractKVPairs(trace)
for k, vs := range kvp {
addKey(allKVP, k, 1)
for v := range vs.all {
addVal(allKVP, k, v, 1)
}
}
i++
if i%100000 == 0 {
fmt.Println("Record: ", i)
}
}
printStats()
}
return nil
}
// helper methods for calculating label stats
func printKVPairs(kvp kvPairs) {
allValues := make([]values, 0, len(kvp))
for _, vs := range kvp {
allValues = append(allValues, vs)
}
sort.Slice(allValues, func(i, j int) bool {
return relativeValue(allValues[i]) > relativeValue(allValues[j])
})
for _, vs := range allValues {
fmt.Println("key:", vs.key, "count:", vs.count, "len:", len(vs.all), "value:", relativeValue(vs))
for a, c := range vs.all {
fmt.Printf(" %s:\t%.2f\n", a, float64(c.count)/float64(vs.count))
}
}
}
// attempts to calculate the "value" that storing a given label would provide by. currently (number of times appeared)^2 / cardinality
// this is not researched and could definitely be improved
func relativeValue(v values) float64 {
return (float64(v.count) * float64(v.count)) / float64(len(v.all))
}
func extractKVPairs(t *tempopb.Trace) kvPairs {
kvp := kvPairs{}
for _, b := range t.ResourceSpans {
spanCount := 0
for _, ils := range b.ScopeSpans {
for _, s := range ils.Spans {
spanCount++
for _, a := range s.Attributes {
val := util.StringifyAnyValue(a.Value)
if val == "" {
continue
}
addKey(kvp, a.Key, 1)
addVal(kvp, a.Key, val, 1)
}
}
}
for _, a := range b.Resource.Attributes {
val := util.StringifyAnyValue(a.Value)
if val == "" {
continue
}
addKey(kvp, a.Key, spanCount)
addVal(kvp, a.Key, val, spanCount)
}
}
return kvp
}
func addKey(kvp kvPairs, key string, count int) {
v, ok := kvp[key]
if !ok {
v = values{
all: map[string]valueStats{},
key: key,
}
}
v.count += count
kvp[key] = v
}
func addVal(kvp kvPairs, key, val string, count int) {
v := kvp[key]
stats, ok := v.all[val]
if !ok {
stats = valueStats{
count: 0,
}
}
stats.count += count
v.all[val] = stats
kvp[key] = v
}
cmd/tempo-cli/cmd-list-blocks.go
+1 -3
View File
@@ -35,7 +35,7 @@ func (l *listBlocksCmd) Run(ctx *globalOptions) error {
}
func displayResults(results []blockStats, windowDuration time.Duration, includeCompacted bool) {
columns := []string{"id", "lvl", "objects", "size", "encoding", "vers", "window", "start", "end", "duration", "age"}
columns := []string{"id", "lvl", "objects", "size", "vers", "window", "start", "end", "duration", "age"}
if includeCompacted {
columns = append(columns, "cmp")
}
@@ -59,8 +59,6 @@ func displayResults(results []blockStats, windowDuration time.Duration, includeC
s = strconv.Itoa(int(r.TotalObjects))
case "size":
s = fmt.Sprintf("%v", humanize.Bytes(r.Size_))
case "encoding":
s = r.Encoding.String()
case "vers":
s = r.Version
case "window":
cmd/tempo-cli/cmd-list-index.go
-85
View File
@@ -1,85 +0,0 @@
package main
import (
"context"
"encoding/hex"
"fmt"
"github.com/google/uuid"
v2 "github.com/grafana/tempo/tempodb/encoding/v2"
)
type listIndexCmd struct {
backendOptions
TenantID string `arg:"" help:"tenant-id within the bucket"`
BlockID string `arg:"" help:"block ID to list"`
}
func (cmd *listIndexCmd) Run(ctx *globalOptions) error {
blockID, err := uuid.Parse(cmd.BlockID)
if err != nil {
return err
}
r, _, _, err := loadBackend(&cmd.backendOptions, ctx)
if err != nil {
return err
}
meta, err := r.BlockMeta(context.TODO(), blockID, cmd.TenantID)
if err != nil {
return err
}
if meta.Version != v2.VersionString {
return fmt.Errorf("unsupported block version: %s", meta.Version)
}
b, err := v2.NewBackendBlock(meta, r)
if err != nil {
return err
}
reader, err := b.NewIndexReader()
if err != nil {
return err
}
var minRecord *v2.Record
var maxRecord *v2.Record
count := 0
for i := 0; ; i++ {
record, err := reader.At(context.TODO(), i)
if err != nil {
return err
}
if record == nil {
break
}
count++
if minRecord == nil || record.Length < minRecord.Length {
minRecord = record
}
if maxRecord == nil || record.Length > maxRecord.Length {
maxRecord = record
}
}
fmt.Println("Index entries:", count)
if minRecord != nil {
fmt.Printf("Min record: ID:%s Start:%v Length:%v\n", hex.EncodeToString(minRecord.ID), minRecord.Start, minRecord.Length)
}
if maxRecord != nil {
fmt.Printf("Max record: ID:%s Start:%v Length:%v\n", hex.EncodeToString(maxRecord.ID), maxRecord.Start, maxRecord.Length)
}
return nil
}
cmd/tempo-cli/cmd-rewrite-blocks.go
+2 -16
View File
@@ -13,7 +13,6 @@ import (
"github.com/go-kit/log/level"
"github.com/google/uuid"
"github.com/grafana/tempo/pkg/boundedwaitgroup"
"github.com/grafana/tempo/pkg/model"
"github.com/grafana/tempo/pkg/util"
"github.com/grafana/tempo/tempodb"
"github.com/grafana/tempo/tempodb/backend"
@@ -115,27 +114,14 @@ func rewriteBlock(ctx context.Context, r backend.Reader, w backend.Writer, meta
BloomShardSizeBytes: common.DefaultBloomShardSizeBytes,
Version: meta.Version,
// these fields aren't in use anymore. we need to remove the old flatbuffer search. setting them for completeness
SearchEncoding: backend.EncSnappy,
SearchPageSizeBytes: 1024 * 1024,
// v2 fields
IndexDownsampleBytes: common.DefaultIndexDownSampleBytes,
IndexPageSizeBytes: common.DefaultIndexPageSizeBytes,
Encoding: backend.EncZstd,
// parquet fields
RowGroupSizeBytes: 100_000_000, // default
// vParquet3 fields
DedicatedColumns: meta.DedicatedColumns,
},
ChunkSizeBytes: tempodb.DefaultChunkSizeBytes,
FlushSizeBytes: tempodb.DefaultFlushSizeBytes,
IteratorBufferSize: tempodb.DefaultIteratorBufferSize,
OutputBlocks: 1,
Combiner: model.StaticCombiner, // this should never be necessary b/c we are only compacting one block
MaxBytesPerTrace: 0, // disable for this process
OutputBlocks: 1,
MaxBytesPerTrace: 0, // disable for this process
// hook to drop the trace
DropObject: func(id common.ID) bool {
cmd/tempo-cli/cmd-rewrite-blocks_test.go
+1 -1
View File
@@ -81,7 +81,7 @@ func generateTestBlocks(t *testing.T, tempDir string, tenantID string, blockCoun
for bn := 0; bn < blockCount; bn++ {
traces := newTestTraces(traceCount)
iter := &testIterator{traces: traces}
meta := backend.NewBlockMeta(tenantID, uuid.New(), vparquet4.VersionString, backend.EncNone, "")
meta := backend.NewBlockMeta(tenantID, uuid.New(), vparquet4.VersionString)
meta.TotalObjects = int64(len(iter.traces))
_, err := vparquet4.CreateBlock(ctx, cfg, meta, iter, r, w)
require.NoError(t, err)
cmd/tempo-cli/cmd-view-index.go
-68
View File
@@ -1,68 +0,0 @@
package main
import (
"context"
"encoding/hex"
"fmt"
"github.com/google/uuid"
v2 "github.com/grafana/tempo/tempodb/encoding/v2"
)
type viewIndexCmd struct {
backendOptions
TenantID string `arg:"" help:"tenant-id within the bucket"`
BlockID string `arg:"" help:"block ID to list"`
}
func (cmd *viewIndexCmd) Run(ctx *globalOptions) error {
blockID, err := uuid.Parse(cmd.BlockID)
if err != nil {
return err
}
r, _, _, err := loadBackend(&cmd.backendOptions, ctx)
if err != nil {
return err
}
meta, err := r.BlockMeta(context.TODO(), blockID, cmd.TenantID)
if err != nil {
return err
}
if meta.Version != v2.VersionString {
return fmt.Errorf("unsupported block version: %s", meta.Version)
}
b, err := v2.NewBackendBlock(meta, r)
if err != nil {
return err
}
reader, err := b.NewIndexReader()
if err != nil {
return err
}
pageSize := 20
for i := 0; ; i++ {
record, err := reader.At(context.TODO(), i)
if err != nil {
return err
}
if record == nil {
return nil
}
fmt.Printf("Index entry: %10v ID: %s Start: %10v Length: %10v\n", i, hex.EncodeToString(record.ID), record.Start, record.Length)
if (i+1)%pageSize == 0 {
fmt.Printf("Press enter to continue\r")
fmt.Scanln()
}
}
}
cmd/tempo-cli/main.go
-5
View File
@@ -47,11 +47,9 @@ var cli struct {
globalOptions
List struct {
Block listBlockCmd `cmd:"" help:"List information about a block"`
Blocks listBlocksCmd `cmd:"" help:"List information about all blocks in a bucket"`
CompactionSummary listCompactionSummaryCmd `cmd:"" help:"List summary of data by compaction level"`
CacheSummary listCacheSummaryCmd `cmd:"" help:"List summary of bloom sizes per day per compaction level"`
Index listIndexCmd `cmd:"" help:"List information about a block index"`
Column listColumnCmd `cmd:"" help:"List values in a given column"`
} `cmd:""`
@@ -61,13 +59,10 @@ var cli struct {
} `cmd:""`
View struct {
Index viewIndexCmd `cmd:"" help:"View contents of block index"`
Schema viewSchemaCmd `cmd:"" help:"View parquet schema"`
} `cmd:""`
Gen struct {
Index indexCmd `cmd:"" help:"Generate index for a block"`
Bloom bloomCmd `cmd:"" help:"Generate bloom for a block"`
AttrIndex attrIndexCmd `cmd:"" help:"Generate an attribute index for a parquet block (EXPERIMENTAL)"`
} `cmd:""`
cmd/tempo/app/app.go
-2
View File
@@ -34,7 +34,6 @@ import (
"github.com/grafana/tempo/cmd/tempo/build"
"github.com/grafana/tempo/modules/backendscheduler"
"github.com/grafana/tempo/modules/compactor"
"github.com/grafana/tempo/modules/distributor"
"github.com/grafana/tempo/modules/distributor/receiver"
frontend_v1 "github.com/grafana/tempo/modules/frontend/v1"
@@ -76,7 +75,6 @@ type App struct {
distributor *distributor.Distributor
querier *querier.Querier
frontend *frontend_v1.Frontend
compactor *compactor.Compactor
ingester *ingester.Ingester
generator *generator.Generator
blockBuilder *blockbuilder.BlockBuilder
cmd/tempo/app/config.go
+5 -45
View File
@@ -13,7 +13,6 @@ import (
"github.com/grafana/tempo/modules/backendworker"
"github.com/grafana/tempo/modules/blockbuilder"
"github.com/grafana/tempo/modules/cache"
"github.com/grafana/tempo/modules/compactor"
"github.com/grafana/tempo/modules/distributor"
"github.com/grafana/tempo/modules/frontend"
"github.com/grafana/tempo/modules/generator"
@@ -31,7 +30,6 @@ import (
"github.com/grafana/tempo/pkg/util"
"github.com/grafana/tempo/tempodb"
"github.com/grafana/tempo/tempodb/backend"
"github.com/grafana/tempo/tempodb/encoding/common"
)
const defaultGRPCCompression = "snappy"
@@ -55,7 +53,6 @@ type Config struct {
LiveStoreClient livestore_client.Config `yaml:"live_store_client,omitempty"`
Querier querier.Config `yaml:"querier,omitempty"`
Frontend frontend.Config `yaml:"query_frontend,omitempty"`
Compactor compactor.Config `yaml:"compactor,omitempty"`
Ingester ingester.Config `yaml:"ingester,omitempty"`
Generator generator.Config `yaml:"metrics_generator,omitempty"`
Ingest ingest.Config `yaml:"ingest,omitempty"`
@@ -149,7 +146,6 @@ func (c *Config) RegisterFlagsAndApplyDefaults(prefix string, f *flag.FlagSet) {
c.BlockBuilder.RegisterFlagsAndApplyDefaults(util.PrefixConfig(prefix, "block-builder"), f)
c.Querier.RegisterFlagsAndApplyDefaults(util.PrefixConfig(prefix, "querier"), f)
c.Frontend.RegisterFlagsAndApplyDefaults(util.PrefixConfig(prefix, "frontend"), f)
c.Compactor.RegisterFlagsAndApplyDefaults(util.PrefixConfig(prefix, "compactor"), f)
c.StorageConfig.RegisterFlagsAndApplyDefaults(util.PrefixConfig(prefix, "storage"), f)
c.UsageReport.RegisterFlagsAndApplyDefaults(util.PrefixConfig(prefix, "reporting"), f)
c.CacheProvider.RegisterFlagsAndApplyDefaults(util.PrefixConfig(prefix, "cache"), f)
@@ -170,18 +166,14 @@ func (c *Config) CheckConfig() []ConfigWarning {
warnings = append(warnings, warnCompleteBlockTimeout)
}
if c.Compactor.Compactor.BlockRetention < c.StorageConfig.Trace.BlocklistPoll {
if c.BackendWorker.Compactor.BlockRetention < c.StorageConfig.Trace.BlocklistPoll {
warnings = append(warnings, warnBlockRetention)
}
if c.Compactor.Compactor.RetentionConcurrency == 0 {
if c.BackendWorker.Compactor.RetentionConcurrency == 0 {
warnings = append(warnings, warnRetentionConcurrency)
}
if c.StorageConfig.Trace.Backend == backend.S3 && c.Compactor.Compactor.FlushSizeBytes < 5242880 {
warnings = append(warnings, warnStorageTraceBackendS3)
}
if c.StorageConfig.Trace.BlocklistPollConcurrency == 0 {
warnings = append(warnings, warnBlocklistPollConcurrency)
}
@@ -209,27 +201,6 @@ func (c *Config) CheckConfig() []ConfigWarning {
})
}
// check v2 specific settings
if c.StorageConfig.Trace.Block.Version != "v2" && c.StorageConfig.Trace.Block.IndexDownsampleBytes != common.DefaultIndexDownSampleBytes {
warnings = append(warnings, newV2Warning("v2_index_downsample_bytes"))
}
if c.StorageConfig.Trace.Block.Version != "v2" && c.StorageConfig.Trace.Block.IndexPageSizeBytes != common.DefaultIndexPageSizeBytes {
warnings = append(warnings, newV2Warning("v2_index_page_size_bytes"))
}
if c.StorageConfig.Trace.Block.Version != "v2" && c.Compactor.Compactor.ChunkSizeBytes != tempodb.DefaultChunkSizeBytes {
warnings = append(warnings, newV2Warning("v2_in_buffer_bytes"))
}
if c.StorageConfig.Trace.Block.Version != "v2" && c.Compactor.Compactor.FlushSizeBytes != tempodb.DefaultFlushSizeBytes {
warnings = append(warnings, newV2Warning("v2_out_buffer_bytes"))
}
if c.StorageConfig.Trace.Block.Version != "v2" && c.Compactor.Compactor.IteratorBufferSize != tempodb.DefaultIteratorBufferSize {
warnings = append(warnings, newV2Warning("v2_prefetch_traces_count"))
}
if c.tracesAndOverridesStorageConflict() {
warnings = append(warnings, warnTracesAndUserConfigurableOverridesStorageConflict)
}
@@ -277,17 +248,13 @@ var (
Explain: "You may receive 404s between the time the ingesters have flushed a trace and the querier is aware of the new block",
}
warnBlockRetention = ConfigWarning{
Message: "compactor.compaction.compacted_block_timeout < storage.trace.blocklist_poll",
Explain: "Queriers and Compactors may attempt to read a block that no longer exists",
Message: "backend_worker.compaction.compacted_block_timeout < storage.trace.blocklist_poll",
Explain: "Queriers and Backend-workers may attempt to read a block that no longer exists",
}
warnRetentionConcurrency = ConfigWarning{
Message: "c.Compactor.Compactor.RetentionConcurrency must be greater than zero. Using default.",
Message: "backend_worker.Compactor.RetentionConcurrency must be greater than zero. Using default.",
Explain: fmt.Sprintf("default=%d", tempodb.DefaultRetentionConcurrency),
}
warnStorageTraceBackendS3 = ConfigWarning{
Message: "c.Compactor.Compactor.FlushSizeBytes < 5242880",
Explain: "Compaction flush size should be 5MB or higher for S3 backend",
}
warnBlocklistPollConcurrency = ConfigWarning{
Message: "c.StorageConfig.Trace.BlocklistPollConcurrency must be greater than zero. Using default.",
Explain: fmt.Sprintf("default=%d", tempodb.DefaultBlocklistPollConcurrency),
@@ -345,13 +312,6 @@ var (
}
)
func newV2Warning(setting string) ConfigWarning {
return ConfigWarning{
Message: "c.StorageConfig.Trace.Block.Version != \"v2\" but " + setting + " is set",
Explain: "This setting is only used in v2 blocks",
}
}
func (c *Config) tracesAndOverridesStorageConflict() bool {
traceStorage := c.StorageConfig.Trace
overridesStorage := c.Overrides.UserConfigurableOverridesConfig.Client
cmd/tempo/app/config_test.go
-37
View File
@@ -14,8 +14,6 @@ import (
"github.com/grafana/tempo/tempodb"
"github.com/grafana/tempo/tempodb/backend"
"github.com/grafana/tempo/tempodb/encoding/common"
v2 "github.com/grafana/tempo/tempodb/encoding/v2"
"github.com/grafana/tempo/tempodb/encoding/vparquet4"
)
func TestConfig_CheckConfig(t *testing.T) {
@@ -74,7 +72,6 @@ func TestConfig_CheckConfig(t *testing.T) {
warnCompleteBlockTimeout,
warnBlockRetention,
warnRetentionConcurrency,
warnStorageTraceBackendS3,
warnBlocklistPollConcurrency,
warnLogReceivedTraces,
warnLogDiscardedTraces,
@@ -102,40 +99,6 @@ func TestConfig_CheckConfig(t *testing.T) {
}(),
expect: []ConfigWarning{warnStorageTraceBackendLocal},
},
{
name: "warnings for v2 settings when they drift from default",
config: func() *Config {
cfg := NewDefaultConfig()
cfg.StorageConfig.Trace.Block.Version = vparquet4.VersionString
cfg.StorageConfig.Trace.Block.IndexDownsampleBytes = 1
cfg.StorageConfig.Trace.Block.IndexPageSizeBytes = 1
cfg.Compactor.Compactor.ChunkSizeBytes = 1
cfg.Compactor.Compactor.FlushSizeBytes = 1
cfg.Compactor.Compactor.IteratorBufferSize = 1
return cfg
}(),
expect: []ConfigWarning{
newV2Warning("v2_index_downsample_bytes"),
newV2Warning("v2_index_page_size_bytes"),
newV2Warning("v2_in_buffer_bytes"),
newV2Warning("v2_out_buffer_bytes"),
newV2Warning("v2_prefetch_traces_count"),
},
},
{
name: "no warnings for v2 settings when they drift from default and v2 is the block version",
config: func() *Config {
cfg := NewDefaultConfig()
cfg.StorageConfig.Trace.Block.Version = v2.VersionString
cfg.StorageConfig.Trace.Block.IndexDownsampleBytes = 1
cfg.StorageConfig.Trace.Block.IndexPageSizeBytes = 1
cfg.Compactor.Compactor.ChunkSizeBytes = 1
cfg.Compactor.Compactor.FlushSizeBytes = 1
cfg.Compactor.Compactor.IteratorBufferSize = 1
return cfg
}(),
expect: nil,
},
{
name: "trace storage conflicts with overrides storage - local",
config: func() *Config {
cmd/tempo/app/modules.go
+2 -25
View File
@@ -29,7 +29,6 @@ import (
"github.com/grafana/tempo/modules/backendworker"
"github.com/grafana/tempo/modules/blockbuilder"
"github.com/grafana/tempo/modules/cache"
"github.com/grafana/tempo/modules/compactor"
"github.com/grafana/tempo/modules/distributor"
"github.com/grafana/tempo/modules/frontend"
"github.com/grafana/tempo/modules/frontend/interceptor"
@@ -81,7 +80,6 @@ const (
MetricsGeneratorNoLocalBlocks string = "metrics-generator-no-local-blocks"
Querier string = "querier"
QueryFrontend string = "query-frontend"
Compactor string = "compactor"
BlockBuilder string = "block-builder"
BackendScheduler string = "backend-scheduler"
BackendWorker string = "backend-worker"
@@ -460,7 +458,7 @@ func (t *App) initQuerier() (services.Service, error) {
level.Warn(log.Logger).Log("msg", "Worker address is empty in single binary mode. Attempting automatic worker configuration. If queries are unresponsive consider configuring the worker explicitly.", "address", t.cfg.Querier.Worker.FrontendAddress)
}
// do not enable polling if this is the single binary. in that case the compactor will take care of polling
// do not enable polling if this is the single binary. in that case the backend-worker will take care of polling
if t.cfg.Target == Querier {
t.store.EnablePolling(context.Background(), nil, false)
}
@@ -595,24 +593,6 @@ func (t *App) initQueryFrontend() (services.Service, error) {
//go:embed static
var staticFiles embed.FS
func (t *App) initCompactor() (services.Service, error) {
if t.cfg.Target == ScalableSingleBinary && t.cfg.Compactor.ShardingRing.KVStore.Store == "" {
t.cfg.Compactor.ShardingRing.KVStore.Store = "memberlist"
}
compactor, err := compactor.New(t.cfg.Compactor, t.store, t.Overrides, prometheus.DefaultRegisterer)
if err != nil {
return nil, fmt.Errorf("failed to create compactor: %w", err)
}
t.compactor = compactor
if t.compactor.Ring != nil {
t.Server.HTTPRouter().Handle("/compactor/ring", t.compactor.Ring)
}
return t.compactor, nil
}
func (t *App) initOptionalStore() (services.Service, error) {
// Used by the local-blocs processor to flush RF1 blocks to storage.
// Only initialize if it's configured.
@@ -664,7 +644,6 @@ func (t *App) initMemberlistKV() (services.Service, error) {
t.cfg.Ingester.IngesterPartitionRing.KVStore.MemberlistKV = t.MemberlistKV.GetMemberlistKV
t.cfg.Generator.Ring.KVStore.MemberlistKV = t.MemberlistKV.GetMemberlistKV
t.cfg.Distributor.DistributorRing.KVStore.MemberlistKV = t.MemberlistKV.GetMemberlistKV
t.cfg.Compactor.ShardingRing.KVStore.MemberlistKV = t.MemberlistKV.GetMemberlistKV
t.cfg.BackendWorker.Ring.KVStore.MemberlistKV = t.MemberlistKV.GetMemberlistKV
t.cfg.LiveStore.PartitionRing.KVStore.MemberlistKV = t.MemberlistKV.GetMemberlistKV
t.cfg.LiveStore.Ring.KVStore.MemberlistKV = t.MemberlistKV.GetMemberlistKV
@@ -850,7 +829,6 @@ func (t *App) setupModuleManager() error {
mm.RegisterModule(Ingester, t.initIngester)
mm.RegisterModule(Querier, t.initQuerier)
mm.RegisterModule(QueryFrontend, t.initQueryFrontend)
mm.RegisterModule(Compactor, t.initCompactor)
mm.RegisterModule(MetricsGenerator, t.initGenerator)
mm.RegisterModule(MetricsGeneratorNoLocalBlocks, t.initGeneratorNoLocalBlocks)
mm.RegisterModule(BlockBuilder, t.initBlockBuilder)
@@ -887,14 +865,13 @@ func (t *App) setupModuleManager() error {
MetricsGenerator: {Common, OptionalStore, MemberlistKV, PartitionRing},
MetricsGeneratorNoLocalBlocks: {Common, GeneratorRingWatcher},
Querier: {Common, Store, IngesterRing, MetricsGeneratorRing, SecondaryIngesterRing, PartitionRing},
Compactor: {Common, Store, MemberlistKV},
BlockBuilder: {Common, Store, MemberlistKV, PartitionRing},
BackendScheduler: {Common, Store},
BackendWorker: {Common, Store, MemberlistKV},
LiveStore: {Common, MemberlistKV, PartitionRing},
// composite targets
SingleBinary: {Compactor, QueryFrontend, Querier, Ingester, Distributor, MetricsGenerator},
SingleBinary: {QueryFrontend, Querier, Ingester, Distributor, MetricsGenerator},
SingleBinary3_0: {BackendScheduler, BackendWorker, QueryFrontend, Querier, Distributor, MetricsGenerator, BlockBuilder, LiveStore}, // TODO: when we cut 3.0 remove SingleBinary and replace with this
ScalableSingleBinary: {SingleBinary},
}
docs/sources/tempo/configuration/_index.md
-31
View File
@@ -1464,8 +1464,6 @@ storage:
# configuration block for the Write Ahead Log (WAL)
wal: <WAL config>
[path: <string> | default = "/var/tempo/wal"]
[v2_encoding: <string> | default = snappy]
[search_encoding: <string> | default = none]
[ingestion_time_range_slack: <duration> | default = 2m]
# block configuration
@@ -1575,15 +1573,6 @@ Defines re-used configuration blocks.
# maximum size of each bloom filter shard
[bloom_filter_shard_size_bytes: <int> | default = 100KiB]
# number of bytes per index record
[v2_index_downsample_bytes: <uint64> | default = 1MiB]
# block encoding/compression. options: none, gzip, lz4-64k, lz4-256k, lz4-1M, lz4, snappy, zstd, s2
[v2_encoding: <string> | default = zstd]
# search data encoding/compression. same options as block encoding.
[search_encoding: <string> | default = snappy]
# number of bytes per search page
[search_page_size_bytes: <int> | default = 1MiB]
@@ -1646,18 +1635,6 @@ The `compaction` configuration block is used by the compactor, scheduler, and wo
# The time between compaction cycles.
# Note: The default will be used if the value is set to 0.
[compaction_cycle: <duration> | default=30s]
# Optional
# Amount of data to buffer from input blocks.
[v2_in_buffer_bytes: <int> | default=5242880]
# Optional
# Flush data to backend when buffer is this large.
[v2_out_buffer_bytes: <int> | default=20971520]
# Optional
# Number of traces to buffer in memory during compaction. Increasing may improve performance but will also increase memory usage. Default is 1000.
[v2_prefetch_traces_count: <int> | default=1000]
```
### Filter policies
@@ -1829,14 +1806,6 @@ The storage WAL configuration block.
# Example: "/var/tempo/wal
[path: <string> | default = ""]
# WAL encoding/compression.
# options: none, gzip, lz4-64k, lz4-256k, lz4-1M, lz4, snappy, zstd, s2
[v2_encoding: <string> | default = "zstd" ]
# Defines the search data encoding/compression protocol.
# Options: none, gzip, lz4-64k, lz4-256k, lz4-1M, lz4, snappy, zstd, s2
[search_encoding: <string> | default = "snappy"]
# When a span is written to the WAL it adjusts the start and end times of the block it is written to.
# This block start and end time range is then used when choosing blocks for search.
# This is also used for querying traces by ID when the start and end parameters are specified. To prevent spans too far
docs/sources/tempo/configuration/compression.md
-52
View File
@@ -1,52 +0,0 @@
---
title: Compression and encoding
description: Learn about compression and encoding options available for Tempo.
weight: 200
---
<!-- Page needs to be updated. -->
# Compression and encoding
Tempo can compress traces that it pushes to backend storage. This requires extra
memory and CPU, but it reduces the quantity of stored data.
Anecdotal tests suggest that `zstd` will cut your storage costs to ~15% of the uncompressed amount.
It is _highly_ recommended to use the default `zstd`. (The compression field is used for the old v2 format. the vParquet* formats compress columns individually.)
Compression is configured under storage like so:
```yaml
storage:
trace:
block:
v2_encoding: zstd
```
The following options are supported:
- none
- gzip
- lz4-64k
- lz4-256k
- lz4-1M
- lz4
- snappy
- zstd
- s2
Although all of these compression formats are supported in Tempo, at Grafana
we use `zstd`. It's possible/probable that the other compression algorithms may have issue at scale.
File an issue if you have any problems.
## WAL
The WAL also supports compression. By default, this is configured to use `snappy`. This comes with a small performance
penalty but reduces disk I/O and and adds checksums to the WAL. All of the above configuration options are supported
but only `snappy` has been tested at scale.
```
storage:
trace:
wal:
v2_encoding: snappy
```
docs/sources/tempo/configuration/manifest.md
-93
View File
@@ -424,63 +424,6 @@ query_frontend:
enabled: false
max_query_expression_size_bytes: 131072
rf1_after: 0001-01-01T00:00:00Z
compactor:
ring:
kvstore:
store: ""
prefix: collectors/
consul:
host: localhost:8500
acl_token: ""
http_client_timeout: 20s
consistent_reads: false
watch_rate_limit: 1
watch_burst_size: 1
cas_retry_delay: 1s
etcd:
endpoints: []
dial_timeout: 10s
max_retries: 10
tls_enabled: false
tls_cert_path: ""
tls_key_path: ""
tls_ca_path: ""
tls_server_name: ""
tls_insecure_skip_verify: false
tls_cipher_suites: ""
tls_min_version: ""
username: ""
password: ""
multi:
primary: ""
secondary: ""
mirror_enabled: false
mirror_timeout: 2s
heartbeat_period: 5s
heartbeat_timeout: 1m0s
wait_stability_min_duration: 1m0s
wait_stability_max_duration: 5m0s
instance_id: hostname
instance_interface_names:
- eth0
- en0
instance_port: 0
instance_addr: ""
enable_inet6: false
wait_active_instance_timeout: 10m0s
compaction:
v2_in_buffer_bytes: 5242880
v2_out_buffer_bytes: 20971520
v2_prefetch_traces_count: 1000
compaction_window: 1h0m0s
max_compaction_objects: 6000000
max_block_bytes: 107374182400
block_retention: 336h0m0s
compacted_block_retention: 1h0m0s
retention_concurrency: 10
max_time_per_tenant: 5m0s
compaction_cycle: 30s
override_ring_key: compactor
ingester:
lifecycler:
ring:
@@ -686,11 +629,6 @@ metrics_generator:
bloom_filter_false_positive: 0.01
bloom_filter_shard_size_bytes: 102400
version: vParquet4
search_encoding: snappy
search_page_size_bytes: 1048576
v2_index_downsample_bytes: 1048576
v2_index_page_size_bytes: 256000
v2_encoding: zstd
parquet_row_group_size_bytes: 100000000
parquet_dedicated_columns:
- scope: resource
@@ -795,14 +733,10 @@ metrics_generator:
remote_write_add_org_id_header: true
traces_storage:
path: ""
v2_encoding: none
search_encoding: none
ingestion_time_range_slack: 2m0s
version: vParquet4
traces_query_storage:
path: ""
v2_encoding: none
search_encoding: none
ingestion_time_range_slack: 2m0s
version: vParquet4
metrics_ingestion_time_range_slack: 30s
@@ -846,11 +780,6 @@ block_builder:
bloom_filter_false_positive: 0.01
bloom_filter_shard_size_bytes: 102400
version: vParquet4
search_encoding: snappy
search_page_size_bytes: 1048576
v2_index_downsample_bytes: 1048576
v2_index_page_size_bytes: 256000
v2_encoding: zstd
parquet_row_group_size_bytes: 100000000
parquet_dedicated_columns:
- scope: resource
@@ -911,8 +840,6 @@ block_builder:
options: []
wal:
path: /var/tempo/block-builder/traces
v2_encoding: none
search_encoding: none
ingestion_time_range_slack: 2m0s
version: vParquet4
storage:
@@ -922,18 +849,11 @@ storage:
queue_depth: 20000
wal:
path: /var/tempo/wal
v2_encoding: snappy
search_encoding: none
ingestion_time_range_slack: 2m0s
block:
bloom_filter_false_positive: 0.01
bloom_filter_shard_size_bytes: 102400
version: vParquet4
search_encoding: snappy
search_page_size_bytes: 1048576
v2_index_downsample_bytes: 1048576
v2_index_page_size_bytes: 256000
v2_encoding: zstd
parquet_row_group_size_bytes: 100000000
parquet_dedicated_columns:
- scope: resource
@@ -1239,9 +1159,6 @@ backend_scheduler:
compaction:
measure_interval: 1m0s
compaction:
v2_in_buffer_bytes: 5242880
v2_out_buffer_bytes: 20971520
v2_prefetch_traces_count: 1000
compaction_window: 1h0m0s
max_compaction_objects: 6000000
max_block_bytes: 107374182400
@@ -1291,9 +1208,6 @@ backend_worker:
max_period: 1m0s
max_retries: 0
compaction:
v2_in_buffer_bytes: 5242880
v2_out_buffer_bytes: 20971520
v2_prefetch_traces_count: 1000
compaction_window: 1h0m0s
max_compaction_objects: 6000000
max_block_bytes: 107374182400
@@ -1427,8 +1341,6 @@ live_store:
commit_interval: 5s
wal:
path: /var/tempo/live-store/traces
v2_encoding: none
search_encoding: none
ingestion_time_range_slack: 2m0s
version: vParquet4
query_block_concurrency: 10
@@ -1446,11 +1358,6 @@ live_store:
bloom_filter_false_positive: 0.01
bloom_filter_shard_size_bytes: 102400
version: ""
search_encoding: snappy
search_page_size_bytes: 1048576
v2_index_downsample_bytes: 1048576
v2_index_page_size_bytes: 256000
v2_encoding: zstd
parquet_row_group_size_bytes: 100000000
parquet_dedicated_columns:
- scope: resource
integration/storage/backend_scheduler_test.go
+6 -9
View File
@@ -279,14 +279,11 @@ func printMetricValue(t *testing.T, expectedValue string, metric string) e2e.Met
func setupBackendWithEndpoint(t testing.TB, cfg *tempodb.Config, endpoint string) tempodb.Writer {
cfg.Block = &common.BlockConfig{
IndexDownsampleBytes: 11,
BloomFP: .01,
BloomShardSizeBytes: 100_000,
Version: encoding.LatestEncoding().Version(),
Encoding: backend.EncNone,
IndexPageSizeBytes: 1000,
RowGroupSizeBytes: 30_000_000,
DedicatedColumns: backend.DedicatedColumns{{Scope: "span", Name: "key", Type: "string"}},
BloomFP: .01,
BloomShardSizeBytes: 100_000,
Version: encoding.LatestEncoding().Version(),
RowGroupSizeBytes: 30_000_000,
DedicatedColumns: backend.DedicatedColumns{{Scope: "span", Name: "key", Type: "string"}},
}
cfg.WAL = &wal.Config{
Filepath: t.TempDir(),
@@ -321,7 +318,7 @@ func populateBackend(ctx context.Context, t testing.TB, w tempodb.Writer, tenant
for range blockCount {
blockID := backend.NewUUID()
meta := &backend.BlockMeta{BlockID: blockID, TenantID: tenantID, DataEncoding: model.CurrentEncoding}
meta := &backend.BlockMeta{BlockID: blockID, TenantID: tenantID}
head, err := wal.NewBlock(meta, model.CurrentEncoding)
require.NoError(t, err)
integration/storage/encodings_test.go
-6
View File
@@ -9,7 +9,6 @@ import (
"github.com/grafana/tempo/integration/util"
tempoUtil "github.com/grafana/tempo/pkg/util"
"github.com/grafana/tempo/tempodb/encoding"
v2 "github.com/grafana/tempo/tempodb/encoding/v2"
)
func TestEncodings(t *testing.T) {
@@ -34,11 +33,6 @@ func TestEncodings(t *testing.T) {
apiClient := h.APIClientHTTP("")
util.QueryAndAssertTrace(t, apiClient, info)
// v2 does not support querying and must be skipped
if enc.Version() == v2.VersionString {
return
}
// search for trace in backend multiple times with different attributes to make sure
// we search with different scopes and with attributes from dedicated columns
for range repeatedSearchCount {
modules/backendscheduler/backendscheduler_test.go
+3 -6
View File
@@ -269,12 +269,9 @@ func newStoreWithLogger(ctx context.Context, t testing.TB, log log.Logger, tmpDi
Path: tmpDir + "/traces",
},
Block: &common.BlockConfig{
IndexDownsampleBytes: 2,
BloomFP: 0.01,
BloomShardSizeBytes: 100_000,
Version: encoding.LatestEncoding().Version(),
Encoding: backend.EncLZ4_1M,
IndexPageSizeBytes: 1000,
BloomFP: 0.01,
BloomShardSizeBytes: 100_000,
Version: encoding.LatestEncoding().Version(),
},
WAL: &wal.Config{
Filepath: tmpDir + "/wal",
modules/backendscheduler/provider/compaction_test.go
+3 -6
View File
@@ -393,12 +393,9 @@ func newStoreWithLogger(ctx context.Context, t testing.TB, log log.Logger, tmpDi
Path: tmpDir + "/traces",
},
Block: &common.BlockConfig{
IndexDownsampleBytes: 2,
BloomFP: 0.01,
BloomShardSizeBytes: 100_000,
Version: encoding.LatestEncoding().Version(),
Encoding: backend.EncLZ4_1M,
IndexPageSizeBytes: 1000,
BloomFP: 0.01,
BloomShardSizeBytes: 100_000,
Version: encoding.LatestEncoding().Version(),
},
WAL: &wal.Config{
Filepath: tmpDir + "/wal",
modules/backendworker/backendworker.go
+7 -68
View File
@@ -17,9 +17,7 @@ import (
backendscheduler_client "github.com/grafana/tempo/modules/backendscheduler/client"
"github.com/grafana/tempo/modules/overrides"
"github.com/grafana/tempo/modules/storage"
"github.com/grafana/tempo/pkg/model"
"github.com/grafana/tempo/pkg/tempopb"
tempo_util "github.com/grafana/tempo/pkg/util"
"github.com/grafana/tempo/pkg/util/log"
"github.com/grafana/tempo/tempodb"
"github.com/grafana/tempo/tempodb/backend"
@@ -118,7 +116,7 @@ func New(cfg Config, schedulerClientCfg backendscheduler_client.Config, store st
w.Ring, err = ring.New(cfg.Ring.ToLifecyclerConfig().RingConfig, backendWorkerRingKey, cfg.OverrideRingKey, log.Logger, reg)
if err != nil {
return nil, fmt.Errorf("unable to initialize compactor ring: %w", err)
return nil, fmt.Errorf("unable to initialize backend-worker ring: %w", err)
}
}
@@ -152,26 +150,26 @@ func (w *BackendWorker) starting(ctx context.Context) (err error) {
}
// Wait until the ring client detected this instance in the ACTIVE state.
level.Info(log.Logger).Log("msg", "waiting until compactor is ACTIVE in the ring")
level.Info(log.Logger).Log("msg", "waiting until backend-worker is ACTIVE in the ring")
ctxWithTimeout, cancel := context.WithTimeout(ctx, w.cfg.Ring.WaitActiveInstanceTimeout)
defer cancel()
if err := ring.WaitInstanceState(ctxWithTimeout, w.Ring, w.ringLifecycler.GetInstanceID(), ring.ACTIVE); err != nil {
return err
}
level.Info(log.Logger).Log("msg", "compactor is ACTIVE in the ring")
level.Info(log.Logger).Log("msg", "backend-worker is ACTIVE in the ring")
// In the event of a cluster cold start we may end up in a situation where each new compactor
// In the event of a cluster cold start we may end up in a situation where each new backend-worker
// instance starts at a slightly different time and thus each one starts with a different state
// of the ring. It's better to just wait the ring stability for a short time.
if w.cfg.Ring.WaitStabilityMinDuration > 0 {
minWaiting := w.cfg.Ring.WaitStabilityMinDuration
maxWaiting := w.cfg.Ring.WaitStabilityMaxDuration
level.Info(log.Logger).Log("msg", "waiting until compactor ring topology is stable", "min_waiting", minWaiting.String(), "max_waiting", maxWaiting.String())
level.Info(log.Logger).Log("msg", "waiting until backend-worker ring topology is stable", "min_waiting", minWaiting.String(), "max_waiting", maxWaiting.String())
if err := ring.WaitRingStability(ctx, w.Ring, ringOp, minWaiting, maxWaiting); err != nil {
level.Warn(log.Logger).Log("msg", "compactor ring topology is not stable after the max waiting time, proceeding anyway")
level.Warn(log.Logger).Log("msg", "backend-worker ring topology is not stable after the max waiting time, proceeding anyway")
} else {
level.Info(log.Logger).Log("msg", "compactor ring topology is stable")
level.Info(log.Logger).Log("msg", "backend-worker ring topology is stable")
}
}
}
@@ -383,61 +381,6 @@ func (w *BackendWorker) compact(ctx context.Context, blockMetas []*backend.Block
return w.store.CompactWithConfig(ctx, blockMetas, tenantID, &w.cfg.Compactor, w, w)
}
// Combine implements tempodb.CompactorSharder
func (w *BackendWorker) Combine(dataEncoding string, tenantID string, objs ...[]byte) ([]byte, bool, error) {
combinedObj, wasCombined, err := model.StaticCombiner.Combine(dataEncoding, objs...)
if err != nil {
return nil, false, err
}
maxBytes := w.overrides.MaxBytesPerTrace(tenantID)
if maxBytes == 0 || len(combinedObj) < maxBytes {
return combinedObj, wasCombined, nil
}
// technically neither of these conditions should ever be true, we are adding them as guard code
// for the following logic
if len(objs) == 0 {
return []byte{}, wasCombined, nil
}
if len(objs) == 1 {
return objs[0], wasCombined, nil
}
totalDiscarded := countSpans(dataEncoding, objs[1:]...)
overrides.RecordDiscardedSpans(totalDiscarded, overrides.ReasonCompactorDiscardedSpans, tenantID)
return objs[0], wasCombined, nil
}
// Copied from compactor module. Centralize?
func countSpans(dataEncoding string, objs ...[]byte) (total int) {
var traceID string
decoder, err := model.NewObjectDecoder(dataEncoding)
if err != nil {
return 0
}
for _, o := range objs {
t, err := decoder.PrepareForRead(o)
if err != nil {
continue
}
for _, b := range t.ResourceSpans {
for _, ilm := range b.ScopeSpans {
if len(ilm.Spans) > 0 && traceID == "" {
traceID = tempo_util.TraceIDToHexString(ilm.Spans[0].TraceId)
}
total += len(ilm.Spans)
}
}
}
level.Debug(log.Logger).Log("msg", "max size of trace exceeded", "traceId", traceID, "discarded_span_count", total)
return
}
func (w *BackendWorker) Owns(hash string) bool {
if !w.isSharded() {
return true
@@ -579,10 +522,6 @@ func (s ownsEverythingSharder) RecordDiscardedSpans(count int, tenantID string,
s.w.RecordDiscardedSpans(count, tenantID, traceID, rootSpanName, rootServiceName)
}
func (s ownsEverythingSharder) Combine(dataEncoding string, tenantID string, objs ...[]byte) ([]byte, bool, error) {
return s.w.Combine(dataEncoding, tenantID, objs...)
}
// createShutdownContext creates a context that starts a timeout only after parentCtx is cancelled
func createShutdownContext(parentCtx context.Context, shutdownTimeout time.Duration) (context.Context, context.CancelFunc) {
jobsCtx, jobsCancel := context.WithCancel(context.Background())
modules/backendworker/backendworker_ring.go
+12 -12
View File
@@ -14,7 +14,7 @@ import (
)
// RingConfig masks the ring lifecycler config which contains
// many options not really required by the compactors ring. This config
// many options not really required by the worker's ring. This config
// is used to strip down the config to the minimum, and avoid confusion
// to the user.
type RingConfig struct {
@@ -50,26 +50,26 @@ func (cfg *RingConfig) RegisterFlags(f *flag.FlagSet) {
}
// Ring flags
cfg.KVStore.RegisterFlagsWithPrefix("compactor.ring.", "collectors/", f)
f.DurationVar(&cfg.HeartbeatPeriod, "compactor.ring.heartbeat-period", 5*time.Second, "Period at which to heartbeat to the ring. 0 = disabled.")
f.DurationVar(&cfg.HeartbeatTimeout, "compactor.ring.heartbeat-timeout", time.Minute, "The heartbeat timeout after which compactors are considered unhealthy within the ring. 0 = never (timeout disabled).")
cfg.KVStore.RegisterFlagsWithPrefix("worker.ring.", "collectors/", f)
f.DurationVar(&cfg.HeartbeatPeriod, "worker.ring.heartbeat-period", 5*time.Second, "Period at which to heartbeat to the ring. 0 = disabled.")
f.DurationVar(&cfg.HeartbeatTimeout, "worker.ring.heartbeat-timeout", time.Minute, "The heartbeat timeout after which workers are considered unhealthy within the ring. 0 = never (timeout disabled).")
// Wait stability flags.
f.DurationVar(&cfg.WaitStabilityMinDuration, "compactor.ring.wait-stability-min-duration", time.Minute, "Minimum time to wait for ring stability at startup. 0 to disable.")
f.DurationVar(&cfg.WaitStabilityMaxDuration, "compactor.ring.wait-stability-max-duration", 5*time.Minute, "Maximum time to wait for ring stability at startup. If the compactor ring keeps changing after this period of time, the compactor will start anyway.")
f.DurationVar(&cfg.WaitStabilityMinDuration, "worker.ring.wait-stability-min-duration", time.Minute, "Minimum time to wait for ring stability at startup. 0 to disable.")
f.DurationVar(&cfg.WaitStabilityMaxDuration, "worker.ring.wait-stability-max-duration", 5*time.Minute, "Maximum time to wait for ring stability at startup. If the backend-worker ring keeps changing after this period of time, the backend-worker will start anyway.")
// Instance flags
cfg.InstanceInterfaceNames = []string{"eth0", "en0"}
f.Var((*flagext.StringSlice)(&cfg.InstanceInterfaceNames), "compactor.ring.instance-interface-names", "Name of network interface to read address from.")
f.StringVar(&cfg.InstanceAddr, "compactor.ring.instance-addr", "", "IP address to advertise in the ring.")
f.IntVar(&cfg.InstancePort, "compactor.ring.instance-port", 0, "Port to advertise in the ring (defaults to server.grpc-listen-port).")
f.StringVar(&cfg.InstanceID, "compactor.ring.instance-id", hostname, "Instance ID to register in the ring.")
f.Var((*flagext.StringSlice)(&cfg.InstanceInterfaceNames), "worker.ring.instance-interface-names", "Name of network interface to read address from.")
f.StringVar(&cfg.InstanceAddr, "worker.ring.instance-addr", "", "IP address to advertise in the ring.")
f.IntVar(&cfg.InstancePort, "worker.ring.instance-port", 0, "Port to advertise in the ring (defaults to server.grpc-listen-port).")
f.StringVar(&cfg.InstanceID, "worker.ring.instance-id", hostname, "Instance ID to register in the ring.")
// Timeout durations
f.DurationVar(&cfg.WaitActiveInstanceTimeout, "compactor.ring.wait-active-instance-timeout", 10*time.Minute, "Timeout for waiting on compactor to become ACTIVE in the ring.")
f.DurationVar(&cfg.WaitActiveInstanceTimeout, "worker.ring.wait-active-instance-timeout", 10*time.Minute, "Timeout for waiting on backend-worker to become ACTIVE in the ring.")
}
// ToLifecyclerConfig returns a LifecyclerConfig based on the compactor
// ToLifecyclerConfig returns a LifecyclerConfig based on the backend-worker
// ring config.
func (cfg *RingConfig) ToLifecyclerConfig() ring.LifecyclerConfig {
// We have to make sure that the ring.LifecyclerConfig and ring.Config
modules/backendworker/backendworker_test.go
+7 -6
View File
@@ -12,6 +12,7 @@ import (
"github.com/google/uuid"
"github.com/grafana/dskit/flagext"
"github.com/grafana/dskit/kv"
"github.com/grafana/dskit/services"
backendscheduler_client "github.com/grafana/tempo/modules/backendscheduler/client"
"github.com/grafana/tempo/modules/overrides"
"github.com/grafana/tempo/modules/storage"
@@ -58,6 +59,9 @@ func TestWorker(t *testing.T) {
err = w.processJobs(ctx)
require.NoError(t, err)
err = services.StopAndAwaitTerminated(ctx, w)
require.NoError(t, err)
}
func setupDependencies(ctx context.Context, t *testing.T, limits overrides.Config) (Config, backendscheduler_client.Config, overrides.Service, *mockScheduler, storage.Store) {
@@ -171,12 +175,9 @@ func newStoreWithLogger(ctx context.Context, t testing.TB, log log.Logger, tmpDi
Path: tmpDir + "/traces",
},
Block: &common.BlockConfig{
IndexDownsampleBytes: 2,
BloomFP: 0.01,
BloomShardSizeBytes: 100_000,
Version: encoding.LatestEncoding().Version(),
Encoding: backend.EncLZ4_1M,
IndexPageSizeBytes: 1000,
BloomFP: 0.01,
BloomShardSizeBytes: 100_000,
Version: encoding.LatestEncoding().Version(),
},
WAL: &wal.Config{
Filepath: tmpDir + "/wal",
modules/blockbuilder/blockbuilder_test.go
+3 -6
View File
@@ -916,12 +916,9 @@ func newStoreWithLogger(ctx context.Context, t testing.TB, log log.Logger, skipN
Path: tmpDir,
},
Block: &common.BlockConfig{
IndexDownsampleBytes: 2,
BloomFP: 0.01,
BloomShardSizeBytes: 100_000,
Version: encoding.LatestEncoding().Version(),
Encoding: backend.EncLZ4_1M,
IndexPageSizeBytes: 1000,
BloomFP: 0.01,
BloomShardSizeBytes: 100_000,
Version: encoding.LatestEncoding().Version(),
},
WAL: &wal.Config{
Filepath: tmpDir,
modules/blockbuilder/config_test.go
+6 -27
View File
@@ -8,8 +8,6 @@ import (
"github.com/grafana/tempo/tempodb/backend"
"github.com/grafana/tempo/tempodb/encoding"
"github.com/grafana/tempo/tempodb/encoding/common"
v2 "github.com/grafana/tempo/tempodb/encoding/v2"
"github.com/grafana/tempo/tempodb/encoding/vparquet4"
"github.com/grafana/tempo/tempodb/wal"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
@@ -36,11 +34,9 @@ func TestConfig_validate(t *testing.T) {
cfg: Config{
BlockConfig: BlockConfig{
BlockCfg: common.BlockConfig{
Version: encoding.LatestEncoding().Version(),
IndexDownsampleBytes: 1,
IndexPageSizeBytes: 1,
BloomFP: 0.1,
BloomShardSizeBytes: 1,
Version: encoding.LatestEncoding().Version(),
BloomFP: 0.1,
BloomShardSizeBytes: 1,
DedicatedColumns: backend.DedicatedColumns{
{Scope: backend.DedicatedColumnScopeResource, Name: "foo", Type: backend.DedicatedColumnTypeString},
},
@@ -53,21 +49,6 @@ func TestConfig_validate(t *testing.T) {
},
expectedErr: nil,
},
{
name: "InvalidBlockConfig",
cfg: Config{
BlockConfig: BlockConfig{
BlockCfg: common.BlockConfig{
Version: vparquet4.VersionString,
IndexDownsampleBytes: 0,
},
},
WAL: wal.Config{
Version: v2.VersionString,
},
},
expectedErr: errors.New("block config validation failed: positive index downsample required"),
},
{
name: "InvalidBlockVersion",
cfg: Config{
@@ -85,11 +66,9 @@ func TestConfig_validate(t *testing.T) {
cfg: Config{
BlockConfig: BlockConfig{
BlockCfg: common.BlockConfig{
Version: encoding.LatestEncoding().Version(),
IndexDownsampleBytes: 1,
IndexPageSizeBytes: 1,
BloomFP: 0.1,
BloomShardSizeBytes: 1,
Version: encoding.LatestEncoding().Version(),
BloomFP: 0.1,
BloomShardSizeBytes: 1,
DedicatedColumns: backend.DedicatedColumns{
{Scope: backend.DedicatedColumnScopeResource, Name: "foo", Type: backend.DedicatedColumnTypeString},
},
modules/blockbuilder/partition_writer_test.go
-2
View File
@@ -6,7 +6,6 @@ import (
"github.com/go-kit/log"
"github.com/grafana/tempo/pkg/util/test"
"github.com/grafana/tempo/tempodb/backend"
"github.com/grafana/tempo/tempodb/encoding"
"github.com/grafana/tempo/tempodb/wal"
"github.com/stretchr/testify/require"
@@ -25,7 +24,6 @@ func getPartitionWriter(t *testing.T) *writer {
)
w, err := wal.New(&wal.Config{
Filepath: tmpDir,
Encoding: backend.EncNone,
IngestionSlack: 3 * time.Minute,
Version: encoding.DefaultEncoding().Version(),
})
modules/blockbuilder/tenant_store.go
+1 -1
View File
@@ -122,7 +122,7 @@ func (s *tenantStore) Flush(ctx context.Context, r tempodb.Reader, w tempodb.Wri
}
// Initial meta for creating the block
meta := backend.NewBlockMeta(s.tenantID, (uuid.UUID)(blockID), s.enc.Version(), backend.EncNone, "")
meta := backend.NewBlockMeta(s.tenantID, (uuid.UUID)(blockID), s.enc.Version())
meta.DedicatedColumns = s.overrides.DedicatedColumns(s.tenantID)
meta.ReplicationFactor = 1
meta.TotalObjects = int64(s.liveTraces.Len())
modules/blockbuilder/tenant_store_test.go
-1
View File
@@ -26,7 +26,6 @@ func getTenantStore(t *testing.T, startTime time.Time, cycleDuration, slackDurat
w, err := wal.New(&wal.Config{
Filepath: tmpDir,
Encoding: backend.EncNone,
IngestionSlack: 3 * time.Minute,
Version: encoding.DefaultEncoding().Version(),
})
modules/compactor/compactor.go
-335
View File
@@ -1,335 +0,0 @@
package compactor
import (
"context"
"fmt"
"hash/fnv"
"time"
"github.com/go-kit/log/level"
"github.com/grafana/dskit/kv"
"github.com/grafana/dskit/ring"
"github.com/grafana/dskit/services"
"github.com/prometheus/client_golang/prometheus"
"github.com/grafana/tempo/modules/overrides"
"github.com/grafana/tempo/modules/storage"
"github.com/grafana/tempo/pkg/model"
tempoUtil "github.com/grafana/tempo/pkg/util"
"github.com/grafana/tempo/pkg/util/log"
)
const (
// ringAutoForgetUnhealthyPeriods is how many consecutive timeout periods an unhealthy instance
// in the ring will be automatically removed.
ringAutoForgetUnhealthyPeriods = 2
// We use a safe default instead of exposing to config option to the user
// in order to simplify the config.
ringNumTokens = 512
compactorRingKey = "compactor"
)
var ringOp = ring.NewOp([]ring.InstanceState{ring.ACTIVE}, nil)
type Compactor struct {
services.Service
cfg *Config
store storage.Store
overrides overrides.Interface
// Ring used for sharding compactions.
ringLifecycler *ring.BasicLifecycler
Ring *ring.Ring
subservices *services.Manager
subservicesWatcher *services.FailureWatcher
}
// New makes a new Compactor.
func New(cfg Config, store storage.Store, overrides overrides.Interface, reg prometheus.Registerer) (*Compactor, error) {
c := &Compactor{
cfg: &cfg,
store: store,
overrides: overrides,
}
if c.isSharded() {
reg = prometheus.WrapRegistererWithPrefix("tempo_", reg)
lifecyclerStore, err := kv.NewClient(
cfg.ShardingRing.KVStore,
ring.GetCodec(),
kv.RegistererWithKVName(reg, compactorRingKey+"-lifecycler"),
log.Logger,
)
if err != nil {
return nil, err
}
delegate := ring.BasicLifecyclerDelegate(c)
delegate = ring.NewLeaveOnStoppingDelegate(delegate, log.Logger)
delegate = ring.NewAutoForgetDelegate(ringAutoForgetUnhealthyPeriods*cfg.ShardingRing.HeartbeatTimeout, delegate, log.Logger)
bcfg, err := toBasicLifecyclerConfig(cfg.ShardingRing, log.Logger)
if err != nil {
return nil, err
}
c.ringLifecycler, err = ring.NewBasicLifecycler(bcfg, compactorRingKey, cfg.OverrideRingKey, lifecyclerStore, delegate, log.Logger, reg)
if err != nil {
return nil, fmt.Errorf("unable to initialize compactor ring lifecycler: %w", err)
}
c.Ring, err = ring.New(c.cfg.ShardingRing.ToLifecyclerConfig().RingConfig, compactorRingKey, cfg.OverrideRingKey, log.Logger, reg)
if err != nil {
return nil, fmt.Errorf("unable to initialize compactor ring: %w", err)
}
}
c.Service = services.NewBasicService(c.starting, c.running, c.stopping)
return c, nil
}
func (c *Compactor) starting(ctx context.Context) (err error) {
// In case this function will return error we want to unregister the instance
// from the ring. We do it ensuring dependencies are gracefully stopped if they
// were already started.
defer func() {
if err == nil || c.subservices == nil {
return
}
if stopErr := services.StopManagerAndAwaitStopped(context.Background(), c.subservices); stopErr != nil {
level.Error(log.Logger).Log("msg", "failed to gracefully stop compactor dependencies", "err", stopErr)
}
}()
if c.isSharded() {
c.subservices, err = services.NewManager(c.ringLifecycler, c.Ring)
if err != nil {
return fmt.Errorf("failed to create subservices: %w", err)
}
c.subservicesWatcher = services.NewFailureWatcher()
c.subservicesWatcher.WatchManager(c.subservices)
err := services.StartManagerAndAwaitHealthy(ctx, c.subservices)
if err != nil {
return fmt.Errorf("failed to start subservices: %w", err)
}
// Wait until the ring client detected this instance in the ACTIVE state.
level.Info(log.Logger).Log("msg", "waiting until compactor is ACTIVE in the ring")
ctxWithTimeout, cancel := context.WithTimeout(ctx, c.cfg.ShardingRing.WaitActiveInstanceTimeout)
defer cancel()
if err := ring.WaitInstanceState(ctxWithTimeout, c.Ring, c.ringLifecycler.GetInstanceID(), ring.ACTIVE); err != nil {
return err
}
level.Info(log.Logger).Log("msg", "compactor is ACTIVE in the ring")
// In the event of a cluster cold start we may end up in a situation where each new compactor
// instance starts at a slightly different time and thus each one starts with a different state
// of the ring. It's better to just wait the ring stability for a short time.
if c.cfg.ShardingRing.WaitStabilityMinDuration > 0 {
minWaiting := c.cfg.ShardingRing.WaitStabilityMinDuration
maxWaiting := c.cfg.ShardingRing.WaitStabilityMaxDuration
level.Info(log.Logger).Log("msg", "waiting until compactor ring topology is stable", "min_waiting", minWaiting.String(), "max_waiting", maxWaiting.String())
if err := ring.WaitRingStability(ctx, c.Ring, ringOp, minWaiting, maxWaiting); err != nil {
level.Warn(log.Logger).Log("msg", "compactor ring topology is not stable after the max waiting time, proceeding anyway")
} else {
level.Info(log.Logger).Log("msg", "compactor ring topology is stable")
}
}
}
// this will block until one poll cycle is complete
c.store.EnablePolling(ctx, c, true)
return nil
}
func (c *Compactor) running(ctx context.Context) error {
if !c.cfg.Disabled {
level.Info(log.Logger).Log("msg", "enabling compaction")
err := c.store.EnableCompaction(ctx, &c.cfg.Compactor, c, c)
if err != nil {
return fmt.Errorf("failed to enable compaction: %w", err)
}
}
if c.subservices != nil {
select {
case <-ctx.Done():
return nil
case err := <-c.subservicesWatcher.Chan():
return fmt.Errorf("compactor subservices failed: %w", err)
}
} else {
<-ctx.Done()
}
return nil
}
// Called after compactor is asked to stop via StopAsync.
func (c *Compactor) stopping(_ error) error {
if c.subservices != nil {
return services.StopManagerAndAwaitStopped(context.Background(), c.subservices)
}
return nil
}
// Owns implements tempodb.CompactorSharder
func (c *Compactor) Owns(hash string) bool {
if !c.isSharded() {
return true
}
level.Debug(log.Logger).Log("msg", "checking hash", "hash", hash)
hasher := fnv.New32a()
_, _ = hasher.Write([]byte(hash))
hash32 := hasher.Sum32()
rs, err := c.Ring.Get(hash32, ringOp, []ring.InstanceDesc{}, nil, nil)
if err != nil {
level.Error(log.Logger).Log("msg", "failed to get ring", "err", err)
return false
}
if len(rs.Instances) != 1 {
level.Error(log.Logger).Log("msg", "unexpected number of compactors in the shard (expected 1, got %d)", len(rs.Instances))
return false
}
ringAddr := c.ringLifecycler.GetInstanceAddr()
level.Debug(log.Logger).Log("msg", "checking addresses", "owning_addr", rs.Instances[0].Addr, "this_addr", ringAddr)
return rs.Instances[0].Addr == ringAddr
}
// Combine implements tempodb.CompactorSharder
func (c *Compactor) Combine(dataEncoding string, tenantID string, objs ...[]byte) ([]byte, bool, error) {
combinedObj, wasCombined, err := model.StaticCombiner.Combine(dataEncoding, objs...)
if err != nil {
return nil, false, err
}
maxBytes := c.overrides.MaxBytesPerTrace(tenantID)
if maxBytes == 0 || len(combinedObj) < maxBytes {
return combinedObj, wasCombined, nil
}
// technically neither of these conditions should ever be true, we are adding them as guard code
// for the following logic
if len(objs) == 0 {
return []byte{}, wasCombined, nil
}
if len(objs) == 1 {
return objs[0], wasCombined, nil
}
totalDiscarded := countSpans(dataEncoding, objs[1:]...)
overrides.RecordDiscardedSpans(totalDiscarded, overrides.ReasonCompactorDiscardedSpans, tenantID)
return objs[0], wasCombined, nil
}
// RecordDiscardedSpans implements tempodb.CompactorSharder
func (c *Compactor) RecordDiscardedSpans(count int, tenantID string, traceID string, rootSpanName string, rootServiceName string) {
level.Warn(log.Logger).Log("msg", "max size of trace exceeded", "tenant", tenantID, "traceId", traceID,
"rootSpanName", rootSpanName, "rootServiceName", rootServiceName, "discarded_span_count", count)
overrides.RecordDiscardedSpans(count, overrides.ReasonCompactorDiscardedSpans, tenantID)
}
// BlockRetentionForTenant implements CompactorOverrides
func (c *Compactor) BlockRetentionForTenant(tenantID string) time.Duration {
return c.overrides.BlockRetention(tenantID)
}
// CompactionDisabledForTenant implements CompactorOverrides
func (c *Compactor) CompactionDisabledForTenant(tenantID string) bool {
return c.overrides.CompactionDisabled(tenantID)
}
func (c *Compactor) MaxBytesPerTraceForTenant(tenantID string) int {
return c.overrides.MaxBytesPerTrace(tenantID)
}
func (c *Compactor) MaxCompactionRangeForTenant(tenantID string) time.Duration {
return c.overrides.MaxCompactionRange(tenantID)
}
func (c *Compactor) isSharded() bool {
store := c.cfg.ShardingRing.KVStore.Store
return store != "" && store != "inmemory"
}
// OnRingInstanceRegister is called while the lifecycler is registering the
// instance within the ring and should return the state and set of tokens to
// use for the instance itself.
func (c *Compactor) OnRingInstanceRegister(_ *ring.BasicLifecycler, ringDesc ring.Desc, instanceExists bool, _ string, instanceDesc ring.InstanceDesc) (ring.InstanceState, ring.Tokens) {
// When we initialize the compactor instance in the ring we want to start from
// a clean situation, so whatever is the state we set it ACTIVE, while we keep existing
// tokens (if any) or the ones loaded from file.
var tokens []uint32
if instanceExists {
tokens = instanceDesc.GetTokens()
}
takenTokens := ringDesc.GetTokens()
gen := ring.NewRandomTokenGenerator()
newTokens := gen.GenerateTokens(ringNumTokens-len(tokens), takenTokens)
// Tokens sorting will be enforced by the parent caller.
tokens = append(tokens, newTokens...)
return ring.ACTIVE, tokens
}
// OnRingInstanceTokens is called once the instance tokens are set and are
// stable within the ring (honoring the observe period, if set).
func (c *Compactor) OnRingInstanceTokens(*ring.BasicLifecycler, ring.Tokens) {}
// OnRingInstanceStopping is called while the lifecycler is stopping. The lifecycler
// will continue to hearbeat the ring the this function is executing and will proceed
// to unregister the instance from the ring only after this function has returned.
func (c *Compactor) OnRingInstanceStopping(*ring.BasicLifecycler) {}
// OnRingInstanceHeartbeat is called while the instance is updating its heartbeat
// in the ring.
func (c *Compactor) OnRingInstanceHeartbeat(*ring.BasicLifecycler, *ring.Desc, *ring.InstanceDesc) {
}
func countSpans(dataEncoding string, objs ...[]byte) (total int) {
var traceID string
decoder, err := model.NewObjectDecoder(dataEncoding)
if err != nil {
return 0
}
for _, o := range objs {
t, err := decoder.PrepareForRead(o)
if err != nil {
continue
}
for _, b := range t.ResourceSpans {
for _, ilm := range b.ScopeSpans {
if len(ilm.Spans) > 0 && traceID == "" {
traceID = tempoUtil.TraceIDToHexString(ilm.Spans[0].TraceId)
}
total += len(ilm.Spans)
}
}
}
level.Debug(log.Logger).Log("msg", "max size of trace exceeded", "traceId", traceID, "discarded_span_count", total)
return
}
modules/compactor/compactor_ring.go
-108
View File
@@ -1,108 +0,0 @@
package compactor
import (
"flag"
"os"
"time"
"github.com/go-kit/log/level"
"github.com/grafana/dskit/flagext"
"github.com/grafana/dskit/kv"
"github.com/grafana/dskit/ring"
util_log "github.com/grafana/tempo/pkg/util/log"
)
// RingConfig masks the ring lifecycler config which contains
// many options not really required by the compactors ring. This config
// is used to strip down the config to the minimum, and avoid confusion
// to the user.
type RingConfig struct {
KVStore kv.Config `yaml:"kvstore"`
HeartbeatPeriod time.Duration `yaml:"heartbeat_period"`
HeartbeatTimeout time.Duration `yaml:"heartbeat_timeout"`
// Wait ring stability.
WaitStabilityMinDuration time.Duration `yaml:"wait_stability_min_duration"`
WaitStabilityMaxDuration time.Duration `yaml:"wait_stability_max_duration"`
// Instance details
InstanceID string `yaml:"instance_id" doc:"hidden"`
InstanceInterfaceNames []string `yaml:"instance_interface_names"`
InstancePort int `yaml:"instance_port" doc:"hidden"`
InstanceAddr string `yaml:"instance_addr" doc:"hidden"`
EnableInet6 bool `yaml:"enable_inet6"`
// Injected internally
ListenPort int `yaml:"-"`
WaitActiveInstanceTimeout time.Duration `yaml:"wait_active_instance_timeout"`
ObservePeriod time.Duration `yaml:"-"`
}
// RegisterFlags adds the flags required to config this to the given FlagSet
func (cfg *RingConfig) RegisterFlags(f *flag.FlagSet) {
hostname, err := os.Hostname()
if err != nil {
level.Error(util_log.Logger).Log("msg", "failed to get hostname", "err", err)
os.Exit(1)
}
// Ring flags
cfg.KVStore.RegisterFlagsWithPrefix("compactor.ring.", "collectors/", f)
f.DurationVar(&cfg.HeartbeatPeriod, "compactor.ring.heartbeat-period", 5*time.Second, "Period at which to heartbeat to the ring. 0 = disabled.")
f.DurationVar(&cfg.HeartbeatTimeout, "compactor.ring.heartbeat-timeout", time.Minute, "The heartbeat timeout after which compactors are considered unhealthy within the ring. 0 = never (timeout disabled).")
// Wait stability flags.
f.DurationVar(&cfg.WaitStabilityMinDuration, "compactor.ring.wait-stability-min-duration", time.Minute, "Minimum time to wait for ring stability at startup. 0 to disable.")
f.DurationVar(&cfg.WaitStabilityMaxDuration, "compactor.ring.wait-stability-max-duration", 5*time.Minute, "Maximum time to wait for ring stability at startup. If the compactor ring keeps changing after this period of time, the compactor will start anyway.")
// Instance flags
cfg.InstanceInterfaceNames = []string{"eth0", "en0"}
f.Var((*flagext.StringSlice)(&cfg.InstanceInterfaceNames), "compactor.ring.instance-interface-names", "Name of network interface to read address from.")
f.StringVar(&cfg.InstanceAddr, "compactor.ring.instance-addr", "", "IP address to advertise in the ring.")
f.IntVar(&cfg.InstancePort, "compactor.ring.instance-port", 0, "Port to advertise in the ring (defaults to server.grpc-listen-port).")
f.StringVar(&cfg.InstanceID, "compactor.ring.instance-id", hostname, "Instance ID to register in the ring.")
// Timeout durations
f.DurationVar(&cfg.WaitActiveInstanceTimeout, "compactor.ring.wait-active-instance-timeout", 10*time.Minute, "Timeout for waiting on compactor to become ACTIVE in the ring.")
}
// ToLifecyclerConfig returns a LifecyclerConfig based on the compactor
// ring config.
func (cfg *RingConfig) ToLifecyclerConfig() ring.LifecyclerConfig {
// We have to make sure that the ring.LifecyclerConfig and ring.Config
// defaults are preserved
lc := ring.LifecyclerConfig{}
rc := ring.Config{}
flagext.DefaultValues(&lc)
flagext.DefaultValues(&rc)
// Configure ring
rc.KVStore = cfg.KVStore
rc.HeartbeatTimeout = cfg.HeartbeatTimeout
rc.ReplicationFactor = 1
// Configure lifecycler
lc.RingConfig = rc
lc.RingConfig.SubringCacheDisabled = true
lc.ListenPort = cfg.ListenPort
lc.Addr = cfg.InstanceAddr
lc.Port = cfg.InstancePort
lc.ID = cfg.InstanceID
lc.InfNames = cfg.InstanceInterfaceNames
lc.UnregisterOnShutdown = true
lc.HeartbeatPeriod = cfg.HeartbeatPeriod
lc.ObservePeriod = cfg.ObservePeriod
lc.JoinAfter = 0
lc.MinReadyDuration = 0
lc.FinalSleep = 0
// We use a safe default instead of exposing to config option to the user
// in order to simplify the config.
lc.NumTokens = 512
return lc
}
modules/compactor/compactor_test.go
-237
View File
@@ -1,237 +0,0 @@
package compactor
import (
"math"
"testing"
"github.com/grafana/dskit/kv"
"github.com/prometheus/client_golang/prometheus"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"github.com/grafana/tempo/modules/overrides"
"github.com/grafana/tempo/pkg/model"
"github.com/grafana/tempo/pkg/model/trace"
"github.com/grafana/tempo/pkg/tempopb"
v1 "github.com/grafana/tempo/pkg/tempopb/trace/v1"
"github.com/grafana/tempo/pkg/util/test"
"github.com/grafana/tempo/tempodb/backend"
)
func TestCombineLimitsNotHit(t *testing.T) {
o, err := overrides.NewOverrides(overrides.Config{
Defaults: overrides.Overrides{
Global: overrides.GlobalOverrides{
MaxBytesPerTrace: math.MaxInt,
},
},
}, nil, prometheus.DefaultRegisterer)
require.NoError(t, err)
c := &Compactor{
overrides: o,
}
trace := test.MakeTraceWithSpanCount(2, 10, nil)
t1 := &tempopb.Trace{
ResourceSpans: []*v1.ResourceSpans{
trace.ResourceSpans[0],
},
}
t2 := &tempopb.Trace{
ResourceSpans: []*v1.ResourceSpans{
trace.ResourceSpans[1],
},
}
obj1 := encode(t, t1)
obj2 := encode(t, t2)
actual, wasCombined, err := c.Combine(model.CurrentEncoding, "test", obj1, obj2)
assert.NoError(t, err)
assert.Equal(t, true, wasCombined)
assert.Equal(t, encode(t, trace), actual) // entire trace should be returned
}
func TestCombineLimitsHit(t *testing.T) {
o, err := overrides.NewOverrides(overrides.Config{
Defaults: overrides.Overrides{
Global: overrides.GlobalOverrides{
MaxBytesPerTrace: 1,
},
},
}, nil, prometheus.DefaultRegisterer)
require.NoError(t, err)
c := &Compactor{
overrides: o,
}
trace := test.MakeTraceWithSpanCount(2, 10, nil)
t1 := &tempopb.Trace{
ResourceSpans: []*v1.ResourceSpans{
trace.ResourceSpans[0],
},
}
t2 := &tempopb.Trace{
ResourceSpans: []*v1.ResourceSpans{
trace.ResourceSpans[1],
},
}
obj1 := encode(t, t1)
obj2 := encode(t, t2)
actual, wasCombined, err := c.Combine(model.CurrentEncoding, "test", obj1, obj2)
assert.NoError(t, err)
assert.Equal(t, true, wasCombined)
assert.Equal(t, encode(t, t1), actual) // only t1 was returned b/c the combined trace was greater than the threshold
}
func TestCombineDoesntEnforceZero(t *testing.T) {
o, err := overrides.NewOverrides(overrides.Config{
Defaults: overrides.Overrides{
Global: overrides.GlobalOverrides{
MaxBytesPerTrace: math.MaxInt,
},
},
}, nil, prometheus.DefaultRegisterer)
require.NoError(t, err)
c := &Compactor{
overrides: o,
}
trace := test.MakeTraceWithSpanCount(2, 10, nil)
t1 := &tempopb.Trace{
ResourceSpans: []*v1.ResourceSpans{
trace.ResourceSpans[0],
},
}
t2 := &tempopb.Trace{
ResourceSpans: []*v1.ResourceSpans{
trace.ResourceSpans[1],
},
}
obj1 := encode(t, t1)
obj2 := encode(t, t2)
actual, wasCombined, err := c.Combine(model.CurrentEncoding, "test", obj1, obj2)
assert.NoError(t, err)
assert.Equal(t, true, wasCombined)
assert.Equal(t, encode(t, trace), actual) // entire trace should be returned
}
func TestCountSpans(t *testing.T) {
t1 := test.MakeTraceWithSpanCount(1, 10, nil)
t2 := test.MakeTraceWithSpanCount(2, 13, nil)
t1ExpectedSpans := 10
t2ExpectedSpans := 26
b1 := encode(t, t1)
b2 := encode(t, t2)
b1Total := countSpans(model.CurrentEncoding, b1)
b2Total := countSpans(model.CurrentEncoding, b2)
total := countSpans(model.CurrentEncoding, b1, b2)
assert.Equal(t, t1ExpectedSpans, b1Total)
assert.Equal(t, t2ExpectedSpans, b2Total)
assert.Equal(t, t1ExpectedSpans+t2ExpectedSpans, total)
}
func TestDedicatedColumns(t *testing.T) {
o, err := overrides.NewOverrides(overrides.Config{
Defaults: overrides.Overrides{
Storage: overrides.StorageOverrides{
DedicatedColumns: backend.DedicatedColumns{
{Scope: "resource", Name: "dedicated.resource.1", Type: "string"},
{Scope: "span", Name: "dedicated.span.1", Type: "string"},
},
},
},
}, nil, prometheus.DefaultRegisterer)
require.NoError(t, err)
c := &Compactor{overrides: o}
tr := test.AddDedicatedAttributes(test.MakeTraceWithSpanCount(2, 10, nil))
t1 := &tempopb.Trace{
ResourceSpans: []*v1.ResourceSpans{
tr.ResourceSpans[0],
},
}
t2 := &tempopb.Trace{
ResourceSpans: []*v1.ResourceSpans{
tr.ResourceSpans[1],
},
}
obj1 := encode(t, t1)
obj2 := encode(t, t2)
actual, wasCombined, err := c.Combine(model.CurrentEncoding, "test", obj1, obj2)
assert.NoError(t, err)
assert.Equal(t, true, wasCombined)
assert.Equal(t, encode(t, tr), actual) // entire trace should be returned
}
func encode(t *testing.T, tr *tempopb.Trace) []byte {
trace.SortTrace(tr)
sd := model.MustNewSegmentDecoder(model.CurrentEncoding)
segment, err := sd.PrepareForWrite(tr, 0, 0)
require.NoError(t, err)
obj, err := sd.ToObject([][]byte{segment})
require.NoError(t, err)
return obj
}
func TestIsSharded(t *testing.T) {
tests := []struct {
name string
store string
expected bool
}{
{
name: "empty store is not sharded",
store: "",
expected: false,
},
{
name: "inmemory store is not sharded",
store: "inmemory",
expected: false,
},
{
name: "memberlist store is sharded",
store: "memberlist",
expected: true,
},
{
name: "consul store is sharded",
store: "consul",
expected: true,
},
{
name: "etcd store is sharded",
store: "etcd",
expected: true,
},
}
for _, tc := range tests {
t.Run(tc.name, func(t *testing.T) {
c := &Compactor{
cfg: &Config{
ShardingRing: RingConfig{
KVStore: kv.Config{
Store: tc.store,
},
},
},
}
assert.Equal(t, tc.expected, c.isSharded())
})
}
}
modules/compactor/config.go
-50
View File
@@ -1,50 +0,0 @@
package compactor
import (
"flag"
"net"
"strconv"
"github.com/go-kit/log"
"github.com/grafana/dskit/flagext"
"github.com/grafana/dskit/ring"
"github.com/grafana/tempo/pkg/util"
"github.com/grafana/tempo/tempodb"
)
type Config struct {
Disabled bool `yaml:"disabled,omitempty"`
ShardingRing RingConfig `yaml:"ring,omitempty"`
Compactor tempodb.CompactorConfig `yaml:"compaction"`
OverrideRingKey string `yaml:"override_ring_key"`
}
// RegisterFlagsAndApplyDefaults registers the flags.
func (cfg *Config) RegisterFlagsAndApplyDefaults(prefix string, f *flag.FlagSet) {
cfg.Compactor = tempodb.CompactorConfig{}
cfg.Compactor.RegisterFlagsAndApplyDefaults(util.PrefixConfig(prefix, "compaction"), f)
flagext.DefaultValues(&cfg.ShardingRing)
cfg.ShardingRing.KVStore.Store = "" // by default compactor is not sharded
f.BoolVar(&cfg.Disabled, util.PrefixConfig(prefix, "disabled"), false, "Disable compaction.")
cfg.OverrideRingKey = compactorRingKey
}
func toBasicLifecyclerConfig(cfg RingConfig, logger log.Logger) (ring.BasicLifecyclerConfig, error) {
instanceAddr, err := ring.GetInstanceAddr(cfg.InstanceAddr, cfg.InstanceInterfaceNames, logger, cfg.EnableInet6)
if err != nil {
return ring.BasicLifecyclerConfig{}, err
}
instancePort := ring.GetInstancePort(cfg.InstancePort, cfg.ListenPort)
instanceAddrPort := net.JoinHostPort(instanceAddr, strconv.Itoa(instancePort))
return ring.BasicLifecyclerConfig{
ID: cfg.InstanceID,
Addr: instanceAddrPort,
HeartbeatPeriod: cfg.HeartbeatPeriod,
NumTokens: ringNumTokens,
}, nil
}
modules/frontend/metrics_query_range_sharder.go
-1
View File
@@ -291,7 +291,6 @@ func (s *queryRangeSharder) buildBackendRequests(ctx context.Context, tenantID s
StartPage: uint32(startPage),
PagesToSearch: uint32(pages),
Version: m.Version,
Encoding: m.Encoding.String(),
Size_: m.Size_,
FooterSize: m.FooterSize,
// DedicatedColumns: dc, for perf reason we pass dedicated columns json in directly to not have to realloc object -> proto -> json
modules/frontend/search_sharder.go
-2
View File
@@ -319,10 +319,8 @@ func buildBackendRequests(ctx context.Context, tenantID string, parent pipeline.
BlockID: blockID,
StartPage: uint32(startPage),
PagesToSearch: uint32(pages),
Encoding: m.Encoding.String(),
IndexPageSize: m.IndexPageSize,
TotalRecords: m.TotalRecords,
DataEncoding: m.DataEncoding,
Version: m.Version,
Size_: m.Size_,
FooterSize: m.FooterSize,
modules/frontend/search_sharder_test.go
+17 -24
View File
@@ -131,14 +131,12 @@ func TestBuildBackendRequests(t *testing.T) {
Size_: 1000,
TotalRecords: 100,
BlockID: backend.MustParse("00000000-0000-0000-0000-000000000000"),
DataEncoding: "json",
Encoding: backend.EncGZIP,
IndexPageSize: 13,
Version: "glarg",
},
},
expectedURIs: []string{
"/querier?blockID=00000000-0000-0000-0000-000000000000&dataEncoding=json&encoding=gzip&end=20&footerSize=0&indexPageSize=13&k=test&pagesToSearch=100&size=1000&start=10&startPage=0&totalRecords=100&v=test&version=glarg",
"/querier?blockID=00000000-0000-0000-0000-000000000000&encoding=none&end=20&footerSize=0&indexPageSize=13&k=test&pagesToSearch=100&size=1000&start=10&startPage=0&totalRecords=100&v=test&version=glarg",
},
},
// meta.json with dedicated columns
@@ -149,7 +147,6 @@ func TestBuildBackendRequests(t *testing.T) {
Size_: 1000,
TotalRecords: 10,
BlockID: backend.MustParse("00000000-0000-0000-0000-000000000000"),
Encoding: backend.EncNone,
IndexPageSize: 13,
Version: "vParquet3",
DedicatedColumns: backend.DedicatedColumns{
@@ -158,7 +155,7 @@ func TestBuildBackendRequests(t *testing.T) {
},
},
expectedURIs: []string{
"/querier?blockID=00000000-0000-0000-0000-000000000000&dataEncoding=&dc=%5B%7B%22name%22%3A%22net.sock.host.addr%22%7D%5D&encoding=none&end=20&footerSize=0&indexPageSize=13&k=test&pagesToSearch=10&size=1000&start=10&startPage=0&totalRecords=10&v=test&version=vParquet3",
"/querier?blockID=00000000-0000-0000-0000-000000000000&dc=%5B%7B%22name%22%3A%22net.sock.host.addr%22%7D%5D&encoding=none&end=20&footerSize=0&indexPageSize=13&k=test&pagesToSearch=10&size=1000&start=10&startPage=0&totalRecords=10&v=test&version=vParquet3",
},
},
// bytes/per request is too small for the page size
@@ -172,9 +169,9 @@ func TestBuildBackendRequests(t *testing.T) {
},
},
expectedURIs: []string{
"/querier?blockID=00000000-0000-0000-0000-000000000000&dataEncoding=&encoding=none&end=20&footerSize=0&indexPageSize=0&k=test&pagesToSearch=1&size=1000&start=10&startPage=0&totalRecords=3&v=test&version=",
"/querier?blockID=00000000-0000-0000-0000-000000000000&dataEncoding=&encoding=none&end=20&footerSize=0&indexPageSize=0&k=test&pagesToSearch=1&size=1000&start=10&startPage=1&totalRecords=3&v=test&version=",
"/querier?blockID=00000000-0000-0000-0000-000000000000&dataEncoding=&encoding=none&end=20&footerSize=0&indexPageSize=0&k=test&pagesToSearch=1&size=1000&start=10&startPage=2&totalRecords=3&v=test&version=",
"/querier?blockID=00000000-0000-0000-0000-000000000000&encoding=none&end=20&footerSize=0&indexPageSize=0&k=test&pagesToSearch=1&size=1000&start=10&startPage=0&totalRecords=3&v=test&version=",
"/querier?blockID=00000000-0000-0000-0000-000000000000&encoding=none&end=20&footerSize=0&indexPageSize=0&k=test&pagesToSearch=1&size=1000&start=10&startPage=1&totalRecords=3&v=test&version=",
"/querier?blockID=00000000-0000-0000-0000-000000000000&encoding=none&end=20&footerSize=0&indexPageSize=0&k=test&pagesToSearch=1&size=1000&start=10&startPage=2&totalRecords=3&v=test&version=",
},
},
// 100 pages, 10 bytes per page, 1k allowed per request
@@ -188,7 +185,7 @@ func TestBuildBackendRequests(t *testing.T) {
},
},
expectedURIs: []string{
"/querier?blockID=00000000-0000-0000-0000-000000000000&dataEncoding=&encoding=none&end=20&footerSize=0&indexPageSize=0&k=test&pagesToSearch=100&size=1000&start=10&startPage=0&totalRecords=100&v=test&version=",
"/querier?blockID=00000000-0000-0000-0000-000000000000&encoding=none&end=20&footerSize=0&indexPageSize=0&k=test&pagesToSearch=100&size=1000&start=10&startPage=0&totalRecords=100&v=test&version=",
},
},
// 100 pages, 10 bytes per page, 900 allowed per request
@@ -202,8 +199,8 @@ func TestBuildBackendRequests(t *testing.T) {
},
},
expectedURIs: []string{
"/querier?blockID=00000000-0000-0000-0000-000000000000&dataEncoding=&encoding=none&end=20&footerSize=0&indexPageSize=0&k=test&pagesToSearch=90&size=1000&start=10&startPage=0&totalRecords=100&v=test&version=",
"/querier?blockID=00000000-0000-0000-0000-000000000000&dataEncoding=&encoding=none&end=20&footerSize=0&indexPageSize=0&k=test&pagesToSearch=90&size=1000&start=10&startPage=90&totalRecords=100&v=test&version=",
"/querier?blockID=00000000-0000-0000-0000-000000000000&encoding=none&end=20&footerSize=0&indexPageSize=0&k=test&pagesToSearch=90&size=1000&start=10&startPage=0&totalRecords=100&v=test&version=",
"/querier?blockID=00000000-0000-0000-0000-000000000000&encoding=none&end=20&footerSize=0&indexPageSize=0&k=test&pagesToSearch=90&size=1000&start=10&startPage=90&totalRecords=100&v=test&version=",
},
},
// two blocks
@@ -222,10 +219,10 @@ func TestBuildBackendRequests(t *testing.T) {
},
},
expectedURIs: []string{
"/querier?blockID=00000000-0000-0000-0000-000000000000&dataEncoding=&encoding=none&end=20&footerSize=0&indexPageSize=0&k=test&pagesToSearch=90&size=1000&start=10&startPage=0&totalRecords=100&v=test&version=",
"/querier?blockID=00000000-0000-0000-0000-000000000000&dataEncoding=&encoding=none&end=20&footerSize=0&indexPageSize=0&k=test&pagesToSearch=90&size=1000&start=10&startPage=90&totalRecords=100&v=test&version=",
"/querier?blockID=00000000-0000-0000-0000-000000000001&dataEncoding=&encoding=none&end=20&footerSize=0&indexPageSize=0&k=test&pagesToSearch=180&size=1000&start=10&startPage=0&totalRecords=200&v=test&version=",
"/querier?blockID=00000000-0000-0000-0000-000000000001&dataEncoding=&encoding=none&end=20&footerSize=0&indexPageSize=0&k=test&pagesToSearch=180&size=1000&start=10&startPage=180&totalRecords=200&v=test&version=",
"/querier?blockID=00000000-0000-0000-0000-000000000000&encoding=none&end=20&footerSize=0&indexPageSize=0&k=test&pagesToSearch=90&size=1000&start=10&startPage=0&totalRecords=100&v=test&version=",
"/querier?blockID=00000000-0000-0000-0000-000000000000&encoding=none&end=20&footerSize=0&indexPageSize=0&k=test&pagesToSearch=90&size=1000&start=10&startPage=90&totalRecords=100&v=test&version=",
"/querier?blockID=00000000-0000-0000-0000-000000000001&encoding=none&end=20&footerSize=0&indexPageSize=0&k=test&pagesToSearch=180&size=1000&start=10&startPage=0&totalRecords=200&v=test&version=",
"/querier?blockID=00000000-0000-0000-0000-000000000001&encoding=none&end=20&footerSize=0&indexPageSize=0&k=test&pagesToSearch=180&size=1000&start=10&startPage=180&totalRecords=200&v=test&version=",
},
},
}
@@ -273,8 +270,6 @@ func TestBuildBackendRequestsShardNumbers(t *testing.T) {
Size_: 1000,
TotalRecords: 10,
BlockID: backend.MustParse("00000000-0000-0000-0000-000000000000"),
DataEncoding: "json",
Encoding: backend.EncGZIP,
IndexPageSize: 13,
Version: "glarg",
},
@@ -290,8 +285,6 @@ func TestBuildBackendRequestsShardNumbers(t *testing.T) {
Size_: 1000,
TotalRecords: 10,
BlockID: backend.MustParse("00000000-0000-0000-0000-000000000000"),
DataEncoding: "json",
Encoding: backend.EncGZIP,
IndexPageSize: 13,
Version: "glarg",
},
@@ -384,7 +377,7 @@ func TestBuildBackendRequestsShardNumbers(t *testing.T) {
}
func TestBackendRequests(t *testing.T) {
bm := backend.NewBlockMeta("test", uuid.New(), "wdwad", backend.EncGZIP, "asdf")
bm := backend.NewBlockMeta("test", uuid.New(), "wdwad")
bm.StartTime = time.Unix(100, 0)
bm.EndTime = time.Unix(200, 0)
bm.Size_ = defaultTargetBytesPerRequest * 2
@@ -409,8 +402,8 @@ func TestBackendRequests(t *testing.T) {
name: "start and end same as block",
request: "/?tags=foo%3Dbar&minDuration=10ms&maxDuration=30ms&limit=50&start=100&end=200",
expectedReqsURIs: []string{
"/querier?blockID=" + bm.BlockID.String() + "&dataEncoding=asdf&encoding=gzip&end=200&footerSize=0&indexPageSize=0&limit=50&maxDuration=30ms&minDuration=10ms&pagesToSearch=1&size=209715200&start=100&startPage=0&tags=foo%3Dbar&totalRecords=2&version=wdwad",
"/querier?blockID=" + bm.BlockID.String() + "&dataEncoding=asdf&encoding=gzip&end=200&footerSize=0&indexPageSize=0&limit=50&maxDuration=30ms&minDuration=10ms&pagesToSearch=1&size=209715200&start=100&startPage=1&tags=foo%3Dbar&totalRecords=2&version=wdwad",
"/querier?blockID=" + bm.BlockID.String() + "&encoding=none&end=200&footerSize=0&indexPageSize=0&limit=50&maxDuration=30ms&minDuration=10ms&pagesToSearch=1&size=209715200&start=100&startPage=0&tags=foo%3Dbar&totalRecords=2&version=wdwad",
"/querier?blockID=" + bm.BlockID.String() + "&encoding=none&end=200&footerSize=0&indexPageSize=0&limit=50&maxDuration=30ms&minDuration=10ms&pagesToSearch=1&size=209715200&start=100&startPage=1&tags=foo%3Dbar&totalRecords=2&version=wdwad",
},
expectedJobs: 2,
expectedBlocks: 1,
@@ -420,8 +413,8 @@ func TestBackendRequests(t *testing.T) {
name: "start and end in block",
request: "/?tags=foo%3Dbar&minDuration=10ms&maxDuration=30ms&limit=50&start=110&end=150",
expectedReqsURIs: []string{
"/querier?blockID=" + bm.BlockID.String() + "&dataEncoding=asdf&encoding=gzip&end=150&footerSize=0&indexPageSize=0&limit=50&maxDuration=30ms&minDuration=10ms&pagesToSearch=1&size=209715200&start=110&startPage=0&tags=foo%3Dbar&totalRecords=2&version=wdwad",
"/querier?blockID=" + bm.BlockID.String() + "&dataEncoding=asdf&encoding=gzip&end=150&footerSize=0&indexPageSize=0&limit=50&maxDuration=30ms&minDuration=10ms&pagesToSearch=1&size=209715200&start=110&startPage=1&tags=foo%3Dbar&totalRecords=2&version=wdwad",
"/querier?blockID=" + bm.BlockID.String() + "&encoding=none&end=150&footerSize=0&indexPageSize=0&limit=50&maxDuration=30ms&minDuration=10ms&pagesToSearch=1&size=209715200&start=110&startPage=0&tags=foo%3Dbar&totalRecords=2&version=wdwad",
"/querier?blockID=" + bm.BlockID.String() + "&encoding=none&end=150&footerSize=0&indexPageSize=0&limit=50&maxDuration=30ms&minDuration=10ms&pagesToSearch=1&size=209715200&start=110&startPage=1&tags=foo%3Dbar&totalRecords=2&version=wdwad",
},
expectedJobs: 2,
expectedBlocks: 1,
modules/frontend/tag_sharder.go
-4
View File
@@ -63,10 +63,8 @@ func (r *tagsSearchRequest) buildTagSearchBlockRequest(subR *http.Request, block
BlockID: blockID,
StartPage: uint32(startPage),
PagesToSearch: uint32(pages),
Encoding: m.Encoding.String(),
IndexPageSize: m.IndexPageSize,
TotalRecords: m.TotalRecords,
DataEncoding: m.DataEncoding,
Version: m.Version,
Size_: m.Size_,
FooterSize: m.FooterSize,
@@ -120,10 +118,8 @@ func (r *tagValueSearchRequest) buildTagSearchBlockRequest(subR *http.Request, b
BlockID: blockID,
StartPage: uint32(startPage),
PagesToSearch: uint32(pages),
Encoding: m.Encoding.String(),
IndexPageSize: m.IndexPageSize,
TotalRecords: m.TotalRecords,
DataEncoding: m.DataEncoding,
Version: m.Version,
Size_: m.Size_,
FooterSize: m.FooterSize,
modules/frontend/tag_sharder_test.go
+6 -7
View File
@@ -78,7 +78,6 @@ func (r *fakeReq) buildTagSearchBlockRequest(subR *http.Request, blockID string,
q.Set("encoding", "gzip")
q.Set("indexPageSize", strconv.FormatUint(0, 10))
q.Set("totalRecords", strconv.FormatUint(2, 10))
q.Set("dataEncoding", "asdf")
q.Set("version", "wdwad")
q.Set("footerSize", strconv.FormatUint(0, 10))
@@ -88,7 +87,7 @@ func (r *fakeReq) buildTagSearchBlockRequest(subR *http.Request, blockID string,
}
func TestTagsBackendRequestsDoNotHitBackendIfStartIsAfterQueryBackendAfter(t *testing.T) {
bm := backend.NewBlockMeta("test", uuid.New(), "wdwad", backend.EncGZIP, "asdf")
bm := backend.NewBlockMeta("test", uuid.New(), "wdwad")
startTime := time.Now().Add(-1 * time.Minute).Unix()
endTime := time.Now().Unix()
s := &searchTagSharder{
@@ -112,7 +111,7 @@ func TestTagsBackendRequestsDoNotHitBackendIfStartIsAfterQueryBackendAfter(t *te
}
func TestTagsBackendRequests(t *testing.T) {
bm := backend.NewBlockMeta("test", uuid.New(), "wdwad", backend.EncGZIP, "asdf")
bm := backend.NewBlockMeta("test", uuid.New(), "wdwad")
bm.StartTime = time.Unix(100, 0)
bm.EndTime = time.Unix(200, 0)
bm.Size_ = defaultTargetBytesPerRequest * 2
@@ -140,8 +139,8 @@ func TestTagsBackendRequests(t *testing.T) {
100, 200,
},
expectedReqsURIs: []string{
"/querier?blockID=" + bm.BlockID.String() + "&dataEncoding=asdf&encoding=gzip&end=200&footerSize=0&indexPageSize=0&pagesToSearch=1&size=209715200&start=100&startPage=0&totalRecords=2&version=wdwad",
"/querier?blockID=" + bm.BlockID.String() + "&dataEncoding=asdf&encoding=gzip&end=200&footerSize=0&indexPageSize=0&pagesToSearch=1&size=209715200&start=100&startPage=1&totalRecords=2&version=wdwad",
"/querier?blockID=" + bm.BlockID.String() + "&encoding=gzip&end=200&footerSize=0&indexPageSize=0&pagesToSearch=1&size=209715200&start=100&startPage=0&totalRecords=2&version=wdwad",
"/querier?blockID=" + bm.BlockID.String() + "&encoding=gzip&end=200&footerSize=0&indexPageSize=0&pagesToSearch=1&size=209715200&start=100&startPage=1&totalRecords=2&version=wdwad",
},
expectedError: nil,
},
@@ -151,8 +150,8 @@ func TestTagsBackendRequests(t *testing.T) {
110, 150,
},
expectedReqsURIs: []string{
"/querier?blockID=" + bm.BlockID.String() + "&dataEncoding=asdf&encoding=gzip&end=150&footerSize=0&indexPageSize=0&pagesToSearch=1&size=209715200&start=110&startPage=0&totalRecords=2&version=wdwad",
"/querier?blockID=" + bm.BlockID.String() + "&dataEncoding=asdf&encoding=gzip&end=150&footerSize=0&indexPageSize=0&pagesToSearch=1&size=209715200&start=110&startPage=1&totalRecords=2&version=wdwad",
"/querier?blockID=" + bm.BlockID.String() + "&encoding=gzip&end=150&footerSize=0&indexPageSize=0&pagesToSearch=1&size=209715200&start=110&startPage=0&totalRecords=2&version=wdwad",
"/querier?blockID=" + bm.BlockID.String() + "&encoding=gzip&end=150&footerSize=0&indexPageSize=0&pagesToSearch=1&size=209715200&start=110&startPage=1&totalRecords=2&version=wdwad",
},
expectedError: nil,
},
modules/generator/processor/localblocks/query_range_test.go
+4 -6
View File
@@ -47,12 +47,10 @@ func TestProcessor(t *testing.T) {
Path: path.Join(tempDir, "traces"),
},
Block: &common.BlockConfig{
IndexDownsampleBytes: 17,
BloomFP: .01,
BloomShardSizeBytes: 100_000,
Version: blockVersion,
IndexPageSizeBytes: 1000,
RowGroupSizeBytes: 10000,
BloomFP: .01,
BloomShardSizeBytes: 100_000,
Version: blockVersion,
RowGroupSizeBytes: 10000,
},
WAL: &wal.Config{
Filepath: path.Join(tempDir, "wal"),
modules/ingester/ingester_test.go
+3 -6
View File
@@ -522,12 +522,9 @@ func defaultIngesterStore(t testing.TB, tmpDir string) storage.Store {
Path: tmpDir,
},
Block: &common.BlockConfig{
IndexDownsampleBytes: 2,
BloomFP: 0.01,
BloomShardSizeBytes: 100_000,
Version: encoding.LatestEncoding().Version(),
Encoding: backend.EncLZ4_1M,
IndexPageSizeBytes: 1000,
BloomFP: 0.01,
BloomShardSizeBytes: 100_000,
Version: encoding.LatestEncoding().Version(),
},
WAL: &wal.Config{
Filepath: tmpDir,
modules/overrides/config.go
+1 -1
View File
@@ -173,7 +173,7 @@ type ReadOverrides struct {
}
type CompactionOverrides struct {
// Compactor enforced overrides.
// Backend-worker/scheduler enforced overrides.
BlockRetention model.Duration `yaml:"block_retention,omitempty" json:"block_retention,omitempty"`
CompactionWindow model.Duration `yaml:"compaction_window,omitempty" json:"compaction_window,omitempty"`
CompactionDisabled bool `yaml:"compaction_disabled,omitempty" json:"compaction_disabled,omitempty"`
modules/overrides/config_legacy.go
+1 -1
View File
@@ -147,7 +147,7 @@ type LegacyOverrides struct {
MetricsGeneratorProcessorHostInfoMetricName string `yaml:"metrics_generator_processor_host_info_metric_name" json:"metrics_generator_processor_host_info_metric_name"`
MetricsGeneratorIngestionSlack time.Duration `yaml:"metrics_generator_ingestion_time_range_slack" json:"metrics_generator_ingestion_time_range_slack,omitempty"`
// Compactor enforced limits.
// Backend-worker/scheduler enforced limits.
BlockRetention model.Duration `yaml:"block_retention" json:"block_retention"`
CompactionDisabled bool `yaml:"compaction_disabled" json:"compaction_disabled"`
CompactionWindow model.Duration `yaml:"compaction_window" json:"compaction_window"`
modules/overrides/discarded_spans.go
+1 -1
View File
@@ -17,7 +17,7 @@ const (
ReasonLiveTracesExceeded = "live_traces_exceeded"
// ReasonUnknown indicates an unknown error when pushing spans.
ReasonUnknown = "unknown_error"
// ReasonTraceTooLargeToCompact indicates a trace is too large for the compactor to combine/compact.
// ReasonTraceTooLargeToCompact indicates a trace is too large for the backend-worker to combine/compact.
ReasonCompactorDiscardedSpans = "trace_too_large_to_compact"
)
modules/querier/querier.go
-28
View File
@@ -904,11 +904,6 @@ func (q *Querier) SearchBlock(ctx context.Context, req *tempopb.SearchBlockReque
return nil, err
}
enc, err := backend.ParseEncoding(req.Encoding)
if err != nil {
return nil, err
}
dc, err := backend.DedicatedColumnsFromTempopb(req.DedicatedColumns)
if err != nil {
return nil, err
@@ -917,12 +912,10 @@ func (q *Querier) SearchBlock(ctx context.Context, req *tempopb.SearchBlockReque
meta := &backend.BlockMeta{
Version: req.Version,
TenantID: tenantID,
Encoding: enc,
Size_: req.Size_,
IndexPageSize: req.IndexPageSize,
TotalRecords: req.TotalRecords,
BlockID: blockID,
DataEncoding: req.DataEncoding,
FooterSize: req.FooterSize,
DedicatedColumns: dc,
}
@@ -960,11 +953,6 @@ func (q *Querier) internalTagsSearchBlockV2(ctx context.Context, req *tempopb.Se
return nil, err
}
enc, err := backend.ParseEncoding(req.Encoding)
if err != nil {
return nil, err
}
dc, err := backend.DedicatedColumnsFromTempopb(req.DedicatedColumns)
if err != nil {
return nil, err
@@ -973,12 +961,10 @@ func (q *Querier) internalTagsSearchBlockV2(ctx context.Context, req *tempopb.Se
meta := &backend.BlockMeta{
Version: req.Version,
TenantID: tenantID,
Encoding: enc,
Size_: req.Size_,
IndexPageSize: req.IndexPageSize,
TotalRecords: req.TotalRecords,
BlockID: blockID,
DataEncoding: req.DataEncoding,
FooterSize: req.FooterSize,
DedicatedColumns: dc,
}
@@ -1041,11 +1027,6 @@ func (q *Querier) internalTagValuesSearchBlock(ctx context.Context, req *tempopb
return &tempopb.SearchTagValuesResponse{}, err
}
enc, err := backend.ParseEncoding(req.Encoding)
if err != nil {
return &tempopb.SearchTagValuesResponse{}, err
}
dc, err := backend.DedicatedColumnsFromTempopb(req.DedicatedColumns)
if err != nil {
return &tempopb.SearchTagValuesResponse{}, err
@@ -1054,12 +1035,10 @@ func (q *Querier) internalTagValuesSearchBlock(ctx context.Context, req *tempopb
meta := &backend.BlockMeta{
Version: req.Version,
TenantID: tenantID,
Encoding: enc,
Size_: req.Size_,
IndexPageSize: req.IndexPageSize,
TotalRecords: req.TotalRecords,
BlockID: blockID,
DataEncoding: req.DataEncoding,
FooterSize: req.FooterSize,
DedicatedColumns: dc,
}
@@ -1087,11 +1066,6 @@ func (q *Querier) internalTagValuesSearchBlockV2(ctx context.Context, req *tempo
return &tempopb.SearchTagValuesV2Response{}, err
}
enc, err := backend.ParseEncoding(req.Encoding)
if err != nil {
return &tempopb.SearchTagValuesV2Response{}, err
}
dc, err := backend.DedicatedColumnsFromTempopb(req.DedicatedColumns)
if err != nil {
return &tempopb.SearchTagValuesV2Response{}, err
@@ -1100,12 +1074,10 @@ func (q *Querier) internalTagValuesSearchBlockV2(ctx context.Context, req *tempo
meta := &backend.BlockMeta{
Version: req.Version,
TenantID: tenantID,
Encoding: enc,
Size_: req.Size_,
IndexPageSize: req.IndexPageSize,
TotalRecords: req.TotalRecords,
BlockID: blockID,
DataEncoding: req.DataEncoding,
FooterSize: req.FooterSize,
DedicatedColumns: dc,
}
modules/querier/querier_query_range.go
+1 -8
View File
@@ -59,11 +59,6 @@ func (q *Querier) queryBlock(ctx context.Context, req *tempopb.QueryRangeRequest
return nil, err
}
enc, err := backend.ParseEncoding(req.Encoding)
if err != nil {
return nil, err
}
dc, err := backend.DedicatedColumnsFromTempopb(req.DedicatedColumns)
if err != nil {
return nil, err
@@ -74,11 +69,9 @@ func (q *Querier) queryBlock(ctx context.Context, req *tempopb.QueryRangeRequest
TenantID: tenantID,
StartTime: time.Unix(0, int64(req.Start)),
EndTime: time.Unix(0, int64(req.End)),
Encoding: enc,
// IndexPageSize: req.IndexPageSize,
// TotalRecords: req.TotalRecords,
BlockID: blockID,
// DataEncoding: req.DataEncoding,
BlockID: blockID,
Size_: req.Size_,
FooterSize: req.FooterSize,
DedicatedColumns: dc,
modules/storage/config.go
-3
View File
@@ -6,7 +6,6 @@ import (
"github.com/grafana/tempo/pkg/cache"
"github.com/grafana/tempo/pkg/util"
"github.com/grafana/tempo/tempodb"
"github.com/grafana/tempo/tempodb/backend"
"github.com/grafana/tempo/tempodb/backend/azure"
"github.com/grafana/tempo/tempodb/backend/gcs"
"github.com/grafana/tempo/tempodb/backend/local"
@@ -36,8 +35,6 @@ func (cfg *Config) RegisterFlagsAndApplyDefaults(prefix string, f *flag.FlagSet)
cfg.Trace.WAL = &wal.Config{}
cfg.Trace.WAL.RegisterFlags(f)
f.StringVar(&cfg.Trace.WAL.Filepath, util.PrefixConfig(prefix, "trace.wal.path"), "/var/tempo/wal", "Path at which store WAL blocks.")
cfg.Trace.WAL.Encoding = backend.EncSnappy
cfg.Trace.WAL.SearchEncoding = backend.EncNone
cfg.Trace.Search = &tempodb.SearchConfig{}
cfg.Trace.Search.RegisterFlagsAndApplyDefaults(prefix, f)
modules/storage/store.go
+2 -10
View File
@@ -12,12 +12,8 @@ import (
)
var (
statCache = usagestats.NewString("storage_cache")
statBackend = usagestats.NewString("storage_backend")
statWalEncoding = usagestats.NewString("storage_wal_encoding")
statWalSearchEncoding = usagestats.NewString("storage_wal_search_encoding")
statBlockEncoding = usagestats.NewString("storage_block_encoding")
statBlockSearchEncoding = usagestats.NewString("storage_block_search_encoding")
statCache = usagestats.NewString("storage_cache")
statBackend = usagestats.NewString("storage_backend")
)
// Store wraps the tempodb storage layer
@@ -43,10 +39,6 @@ type store struct {
func NewStore(cfg Config, cacheProvider cache.Provider, logger log.Logger) (Store, error) {
statCache.Set(cfg.Trace.Cache)
statBackend.Set(cfg.Trace.Backend)
statWalEncoding.Set(cfg.Trace.WAL.Encoding.String())
statWalSearchEncoding.Set(cfg.Trace.WAL.SearchEncoding.String())
statBlockEncoding.Set(cfg.Trace.Block.Encoding.String())
statBlockSearchEncoding.Set(cfg.Trace.Block.SearchEncoding.String())
r, w, c, err := tempodb.New(&cfg.Trace, cacheProvider, logger)
if err != nil {
operations/jsonnet-compiled/microservices/gen/ConfigMap-tempo-backend-scheduler.yaml
-1
View File
@@ -1,7 +1,6 @@
apiVersion: v1
data:
tempo.yaml: |
compactor: {}
distributor: {}
http_api_prefix: ""
ingester:
operations/jsonnet-compiled/microservices/gen/ConfigMap-tempo-backend-worker.yaml
-1
View File
@@ -3,7 +3,6 @@ data:
tempo.yaml: |
backend_worker:
backend_scheduler_addr: backend-scheduler.tracing.svc.cluster.local.:9095
compactor: {}
distributor: {}
http_api_prefix: ""
ingester:
operations/jsonnet-compiled/microservices/gen/ConfigMap-tempo-block-builder.yaml
-1
View File
@@ -1,7 +1,6 @@
apiVersion: v1
data:
tempo.yaml: |
compactor: {}
distributor: {}
http_api_prefix: ""
ingester:
operations/jsonnet-compiled/microservices/gen/ConfigMap-tempo-compactor.yaml
-50
View File
@@ -1,50 +0,0 @@
apiVersion: v1
data:
tempo.yaml: |
compactor:
compaction:
block_retention: 144h
v2_in_buffer_bytes: 1.048576e+07
ring:
kvstore:
store: memberlist
distributor: {}
http_api_prefix: ""
ingester:
lifecycler:
ring:
replication_factor: 3
memberlist:
abort_if_cluster_join_fails: false
bind_port: 7946
join_members:
- dns+gossip-ring.tracing.svc.cluster.local.:7946
overrides:
per_tenant_override_config: /overrides/overrides.yaml
server:
http_listen_port: 3200
storage:
trace:
azure:
container_name: tempo
backend: gcs
blocklist_poll: 5m
cache: memcached
gcs:
bucket_name: tempo
chunk_buffer_size: 1.048576e+07
memcached:
consistent_hash: true
host: memcached
service: memcached-client
timeout: 200ms
pool:
queue_depth: 2000
s3:
bucket: tempo
wal:
path: /var/tempo/wal
kind: ConfigMap
metadata:
name: tempo-compactor
namespace: tracing
operations/jsonnet-compiled/microservices/gen/ConfigMap-tempo-distributor.yaml
-1
View File
@@ -1,7 +1,6 @@
apiVersion: v1
data:
tempo.yaml: |
compactor: {}
distributor:
receivers:
jaeger:
operations/jsonnet-compiled/microservices/gen/ConfigMap-tempo-ingester.yaml
-1
View File
@@ -1,7 +1,6 @@
apiVersion: v1
data:
tempo.yaml: |
compactor: {}
distributor: {}
http_api_prefix: ""
ingester:
operations/jsonnet-compiled/microservices/gen/ConfigMap-tempo-live-store.yaml
-1
View File
@@ -1,7 +1,6 @@
apiVersion: v1
data:
tempo.yaml: |
compactor: {}
distributor: {}
http_api_prefix: ""
ingester:
operations/jsonnet-compiled/microservices/gen/ConfigMap-tempo-metrics-generator.yaml
-1
View File
@@ -1,7 +1,6 @@
apiVersion: v1
data:
tempo.yaml: |
compactor: {}
distributor: {}
http_api_prefix: ""
ingester:
operations/jsonnet-compiled/microservices/gen/ConfigMap-tempo-querier.yaml
-1
View File
@@ -1,7 +1,6 @@
apiVersion: v1
data:
tempo.yaml: |
compactor: {}
distributor: {}
http_api_prefix: ""
ingester:
operations/jsonnet-compiled/microservices/gen/ConfigMap-tempo-query-frontend.yaml
-1
View File
@@ -1,7 +1,6 @@
apiVersion: v1
data:
tempo.yaml: |
compactor: {}
distributor: {}
http_api_prefix: ""
ingester:
operations/jsonnet-compiled/microservices/gen/Deployment-compactor.yaml
-64
View File
@@ -1,64 +0,0 @@
apiVersion: apps/v1
kind: Deployment
metadata:
name: compactor
namespace: tracing
spec:
minReadySeconds: 10
replicas: 5
revisionHistoryLimit: 10
selector:
matchLabels:
app: compactor
name: compactor
strategy:
rollingUpdate:
maxSurge: 50%
maxUnavailable: 100%
template:
metadata:
annotations:
config_hash: 004e7f005ba26495e1943b7948696f5c
labels:
app: compactor
name: compactor
spec:
containers:
- args:
- -config.file=/conf/tempo.yaml
- -mem-ballast-size-mbs=1024
- -target=compactor
env:
- name: GOMEMLIMIT
value: 5GiB
image: grafana/tempo:latest
imagePullPolicy: IfNotPresent
name: compactor
ports:
- containerPort: 3200
name: prom-metrics
readinessProbe:
httpGet:
path: /ready
port: 3200
initialDelaySeconds: 15
timeoutSeconds: 1
resources:
limits:
cpu: "1"
memory: 5Gi
requests:
cpu: 500m
memory: 3Gi
volumeMounts:
- mountPath: /conf
name: tempo-conf
- mountPath: /overrides
name: overrides
volumes:
- configMap:
name: tempo-compactor
name: tempo-conf
- configMap:
name: tempo-overrides
name: overrides
operations/jsonnet-compiled/microservices/gen/Deployment-distributor.yaml
+1 -1
View File
@@ -19,7 +19,7 @@ spec:
template:
metadata:
annotations:
config_hash: 028b09123416ff106e411d57dafeadd8
config_hash: 2a507013d89d866f2eb6c7e4fa347b80
labels:
app: distributor
name: distributor
operations/jsonnet-compiled/microservices/gen/Deployment-metrics-generator.yaml
+1 -1
View File
@@ -17,7 +17,7 @@ spec:
template:
metadata:
annotations:
config_hash: 1c46db8f00d0094a1d213fb87e3fd4ba
config_hash: eeb2912a9914d007cc3b1a00de7932d6
labels:
app: metrics-generator
name: metrics-generator
operations/jsonnet-compiled/microservices/gen/Deployment-querier.yaml
+1 -1
View File
@@ -19,7 +19,7 @@ spec:
template:
metadata:
annotations:
config_hash: 4eceda35dd62a16a63572f339b7f15f4
config_hash: 65dd44be32e4c0f7675650cc9aac75d8
labels:
app: querier
name: querier
operations/jsonnet-compiled/microservices/gen/Deployment-query-frontend.yaml
+1 -1
View File
@@ -18,7 +18,7 @@ spec:
template:
metadata:
annotations:
config_hash: 46188d18f0d8adfa8586e9dbeb744db2
config_hash: 96a49bace28831351f5e1c9f6671556a
labels:
app: query-frontend
name: query-frontend
operations/jsonnet-compiled/microservices/gen/Service-compactor.yaml
-15
View File
@@ -1,15 +0,0 @@
apiVersion: v1
kind: Service
metadata:
labels:
name: compactor
name: compactor
namespace: tracing
spec:
ports:
- name: compactor-prom-metrics
port: 3200
targetPort: 3200
selector:
app: compactor
name: compactor
operations/jsonnet-compiled/microservices/gen/StatefulSet-backend-scheduler.yaml
+1 -1
View File
@@ -14,7 +14,7 @@ spec:
template:
metadata:
annotations:
config_hash: 46188d18f0d8adfa8586e9dbeb744db2
config_hash: 96a49bace28831351f5e1c9f6671556a
labels:
app: backend-scheduler
name: backend-scheduler
operations/jsonnet-compiled/microservices/gen/StatefulSet-backend-worker.yaml
+1 -1
View File
@@ -14,7 +14,7 @@ spec:
template:
metadata:
annotations:
config_hash: b0788f1dcd73338f86e9c38714e3ea68
config_hash: 227d31fc84f6b9d198b6b1c16bf936df
labels:
app: backend-worker
name: backend-worker
operations/jsonnet-compiled/microservices/gen/StatefulSet-block-builder.yaml
+1 -1
View File
@@ -14,7 +14,7 @@ spec:
template:
metadata:
annotations:
config_hash: 46188d18f0d8adfa8586e9dbeb744db2
config_hash: 96a49bace28831351f5e1c9f6671556a
labels:
app: block-builder
name: block-builder
operations/jsonnet-compiled/microservices/gen/StatefulSet-ingester.yaml
+1 -1
View File
@@ -15,7 +15,7 @@ spec:
template:
metadata:
annotations:
config_hash: 46188d18f0d8adfa8586e9dbeb744db2
config_hash: 96a49bace28831351f5e1c9f6671556a
labels:
app: ingester
name: ingester
operations/jsonnet-compiled/microservices/gen/StatefulSet-live-store-zone-a.yaml
+1 -1
View File
@@ -27,7 +27,7 @@ spec:
template:
metadata:
annotations:
config_hash: 699cc4b0c5f8f06f2115a375105318c8
config_hash: 3968d7442703d11707aa8025d96a8e56
labels:
app: live-store-zone-a
name: live-store-zone-a
operations/jsonnet-compiled/microservices/gen/StatefulSet-live-store-zone-b.yaml
+1 -1
View File
@@ -28,7 +28,7 @@ spec:
template:
metadata:
annotations:
config_hash: 699cc4b0c5f8f06f2115a375105318c8
config_hash: 3968d7442703d11707aa8025d96a8e56
labels:
app: live-store-zone-b
name: live-store-zone-b
operations/jsonnet-compiled/microservices/gen/StatefulSet-metrics-generator.yaml
+1 -1
View File
@@ -15,7 +15,7 @@ spec:
template:
metadata:
annotations:
config_hash: 1c46db8f00d0094a1d213fb87e3fd4ba
config_hash: eeb2912a9914d007cc3b1a00de7932d6
labels:
app: metrics-generator
name: metrics-generator
operations/jsonnet/microservices/common.libsonnet
-10
View File
@@ -38,8 +38,6 @@
container.mixin.readinessProbe.withTimeoutSeconds(1),
withInet6():: {
tempo_compactor_service+:
service.mixin.spec.withIpFamilies(['IPv6']),
tempo_distributor_service+:
service.mixin.spec.withIpFamilies(['IPv6']),
tempo_ingester_service+:
@@ -83,14 +81,6 @@
},
},
tempo_compactor_config+: {
compactor+: {
ring+: {
enable_inet6: true,
},
},
},
tempo_backend_worker_config+: {
backend_worker+: {
ring+: {
operations/jsonnet/microservices/compactor.libsonnet
-55
View File
@@ -1,55 +0,0 @@
{
local k = import 'ksonnet-util/kausal.libsonnet',
local container = k.core.v1.container,
local containerPort = k.core.v1.containerPort,
local volumeMount = k.core.v1.volumeMount,
local deployment = k.apps.v1.deployment,
local volume = k.core.v1.volume,
local envVar = k.core.v1.envVar,
local target_name = 'compactor',
local tempo_config_volume = 'tempo-conf',
local tempo_data_volume = 'tempo-data',
local tempo_overrides_config_volume = 'overrides',
tempo_compactor_ports:: [containerPort.new('prom-metrics', $._config.port)],
tempo_compactor_args:: {
target: target_name,
'config.file': '/conf/tempo.yaml',
'mem-ballast-size-mbs': $._config.ballast_size_mbs,
},
tempo_compactor_container::
container.new(target_name, $._images.tempo_compactor) +
container.withPorts($.tempo_compactor_ports) +
container.withArgs($.util.mapToFlags($.tempo_compactor_args)) +
(if $._config.variables_expansion then container.withEnvMixin($._config.variables_expansion_env_mixin) else {}) +
container.withVolumeMounts([
volumeMount.new(tempo_config_volume, '/conf'),
volumeMount.new(tempo_overrides_config_volume, '/overrides'),
]) +
$.util.withResources($._config.compactor.resources) +
$.util.readinessProbe +
(if $._config.variables_expansion then container.withArgsMixin(['-config.expand-env=true']) else {}) +
(if $._config.compactor.resources.limits.memory != null then container.withEnvMixin([envVar.new('GOMEMLIMIT', $._config.compactor.resources.limits.memory + 'B')]) else {}),
tempo_compactor_deployment:
deployment.new(target_name,
$._config.compactor.replicas,
[
$.tempo_compactor_container,
],
{ app: target_name }) +
deployment.mixin.spec.strategy.rollingUpdate.withMaxSurge('50%') +
deployment.mixin.spec.strategy.rollingUpdate.withMaxUnavailable('100%') +
deployment.mixin.spec.template.metadata.withAnnotations({
config_hash: std.md5(std.toString($.tempo_compactor_configmap.data['tempo.yaml'])),
}) +
deployment.mixin.spec.template.spec.withVolumes([
volume.fromConfigMap(tempo_config_volume, $.tempo_compactor_configmap.metadata.name),
volume.fromConfigMap(tempo_overrides_config_volume, $._config.overrides_configmap_name),
]),
tempo_compactor_service:
k.util.serviceFor($.tempo_compactor_deployment),
}
operations/jsonnet/microservices/config.libsonnet
-14
View File
@@ -6,7 +6,6 @@
memcached: 'memcached:1.6.40-alpine',
memcachedExporter: 'prom/memcached-exporter:v0.15.5',
tempo_compactor: self.tempo,
tempo_distributor: self.tempo,
tempo_ingester: self.tempo,
tempo_querier: self.tempo,
@@ -39,19 +38,6 @@
tempo_query: {
enabled: false,
},
compactor: {
replicas: 1,
resources: {
requests: {
cpu: '500m',
memory: '3Gi',
},
limits: {
cpu: '1',
memory: '5Gi',
},
},
},
query_frontend: {
replicas: 1,
resources: {
operations/jsonnet/microservices/configmap.libsonnet
-26
View File
@@ -16,7 +16,6 @@
},
},
},
compactor: {},
storage: {
trace: {
blocklist_poll: '0',
@@ -72,25 +71,6 @@
},
},
tempo_compactor_config:: $.tempo_config {
compactor+: {
compaction+: {
v2_in_buffer_bytes: 10485760,
block_retention: '144h',
},
ring+: {
kvstore+: {
store: 'memberlist',
},
},
},
storage+: {
trace+: {
blocklist_poll: '5m',
},
},
},
tempo_querier_config:: $.tempo_config {
storage+: {
trace+: {
@@ -150,12 +130,6 @@
'tempo.yaml': $.util.manifestYaml($.tempo_metrics_generator_config),
}),
tempo_compactor_configmap:
configMap.new('tempo-compactor') +
configMap.withData({
'tempo.yaml': k.util.manifestYaml($.tempo_compactor_config),
}),
tempo_querier_configmap:
configMap.new('tempo-querier') +
configMap.withData({
operations/jsonnet/microservices/tempo.libsonnet
-1
View File
@@ -1,7 +1,6 @@
(import 'common.libsonnet') +
(import 'configmap.libsonnet') +
(import 'config.libsonnet') +
(import 'compactor.libsonnet') +
(import 'distributor.libsonnet') +
(import 'ingester.libsonnet') +
(import 'generator.libsonnet') +
operations/jsonnet/microservices/test/environments/default/main.jsonnet
-3
View File
@@ -8,7 +8,6 @@ tempo {
_config+:: {
cluster: 'k3d',
namespace: 'default',
compactor+: {},
querier+: {},
ingester+: {
pvc_size: '5Gi',
@@ -76,8 +75,6 @@ tempo {
local container = k.core.v1.container,
local containerPort = k.core.v1.containerPort,
tempo_compactor_container+::
k.util.resourcesRequests('500m', '500Mi'),
tempo_distributor_container+::
k.util.resourcesRequests('500m', '500Mi') +
operations/jsonnet/single-binary/configmap.libsonnet
-5
View File
@@ -11,11 +11,6 @@
},
ingester: {
},
compactor: {
compaction: {
block_retention: '24h',
},
},
memberlist: {
abort_if_cluster_join_fails: false,
bind_port: 7946,
pkg/api/http.go
+2 -8
View File
@@ -45,10 +45,8 @@ const (
urlParamStartPage = "startPage"
urlParamPagesToSearch = "pagesToSearch"
urlParamBlockID = "blockID"
urlParamEncoding = "encoding"
urlParamIndexPageSize = "indexPageSize"
urlParamTotalRecords = "totalRecords"
urlParamDataEncoding = "dataEncoding"
urlParamVersion = "version"
urlParamSize = "size"
urlParamFooterSize = "footerSize"
@@ -442,9 +440,6 @@ func ParseQueryRangeRequest(r *http.Request) (*tempopb.QueryRangeRequest, error)
version, _ := extractQueryParam(vals, urlParamVersion)
req.Version = version
encoding, _ := extractQueryParam(vals, urlParamEncoding)
req.Encoding = encoding
size, _ := extractQueryParam(vals, urlParamSize)
if size, err := strconv.Atoi(size); err == nil {
req.Size_ = uint64(size)
@@ -527,7 +522,7 @@ func BuildQueryRangeRequest(req *http.Request, searchReq *tempopb.QueryRangeRequ
qb.addParam(urlParamStartPage, strconv.Itoa(int(searchReq.StartPage)))
qb.addParam(urlParamPagesToSearch, strconv.Itoa(int(searchReq.PagesToSearch)))
qb.addParam(urlParamVersion, searchReq.Version)
qb.addParam(urlParamEncoding, searchReq.Encoding)
qb.addParam("encoding", "none")
qb.addParam(urlParamSize, strconv.Itoa(int(searchReq.Size_)))
qb.addParam(urlParamFooterSize, strconv.Itoa(int(searchReq.FooterSize)))
@@ -812,10 +807,9 @@ func BuildSearchBlockRequest(req *http.Request, searchReq *tempopb.SearchBlockRe
qb.addParam(urlParamPagesToSearch, strconv.FormatUint(uint64(searchReq.PagesToSearch), 10))
qb.addParam(urlParamSize, strconv.FormatUint(searchReq.Size_, 10))
qb.addParam(urlParamStartPage, strconv.FormatUint(uint64(searchReq.StartPage), 10))
qb.addParam(urlParamEncoding, searchReq.Encoding)
qb.addParam("encoding", "none") // todo: remove. encoding was removed b/c its unused but we still add it here to make rollouts seamless
qb.addParam(urlParamIndexPageSize, strconv.FormatUint(uint64(searchReq.IndexPageSize), 10))
qb.addParam(urlParamTotalRecords, strconv.FormatUint(uint64(searchReq.TotalRecords), 10))
qb.addParam(urlParamDataEncoding, searchReq.DataEncoding)
qb.addParam(urlParamVersion, searchReq.Version)
qb.addParam(urlParamFooterSize, strconv.FormatUint(uint64(searchReq.FooterSize), 10))
if len(dedicatedColumnsJSON) > 0 && dedicatedColumnsJSON != "null" { // if a caller marshals a nil dedicated cols we will receive the string "null"
pkg/api/http_test.go
+5 -23
View File
@@ -327,14 +327,6 @@ func TestParseSearchBlockRequest(t *testing.T) {
url: "/?start=10&end=20&startPage=0&pagesToSearch=10&blockID=adsf",
expectedError: "invalid blockID: invalid UUID length: 4",
},
{
url: "/?start=10&end=20&startPage=0&pagesToSearch=10&blockID=b92ec614-3fd7-4299-b6db-f657e7025a9b",
expectedError: "invalid encoding: , supported: none, gzip, lz4-64k, lz4-256k, lz4-1M, lz4, snappy, zstd, s2",
},
{
url: "/?start=10&end=20&startPage=0&pagesToSearch=10&blockID=b92ec614-3fd7-4299-b6db-f657e7025a9b&encoding=blerg",
expectedError: "invalid encoding: blerg, supported: none, gzip, lz4-64k, lz4-256k, lz4-1M, lz4, snappy, zstd, s2",
},
{
url: "/?start=10&end=20&startPage=0&pagesToSearch=10&blockID=b92ec614-3fd7-4299-b6db-f657e7025a9b&encoding=s2",
expectedError: "invalid indexPageSize : strconv.ParseInt: parsing \"\": invalid syntax",
@@ -356,7 +348,7 @@ func TestParseSearchBlockRequest(t *testing.T) {
expectedError: "invalid footerSize : strconv.ParseUint: parsing \"\": invalid syntax",
},
{
url: "/?tags=foo%3Dbar&start=10&end=20&startPage=0&pagesToSearch=10&blockID=b92ec614-3fd7-4299-b6db-f657e7025a9b&encoding=s2&footerSize=2000&indexPageSize=10&totalRecords=11&dataEncoding=v1&version=v2&size=1000",
url: "/?tags=foo%3Dbar&start=10&end=20&startPage=0&pagesToSearch=10&blockID=b92ec614-3fd7-4299-b6db-f657e7025a9b&encoding=none&footerSize=2000&indexPageSize=10&totalRecords=11&dataEncoding=v1&version=v2&size=1000",
expected: &tempopb.SearchBlockRequest{
SearchReq: &tempopb.SearchRequest{
Tags: map[string]string{
@@ -370,10 +362,8 @@ func TestParseSearchBlockRequest(t *testing.T) {
StartPage: 0,
PagesToSearch: 10,
BlockID: "b92ec614-3fd7-4299-b6db-f657e7025a9b",
Encoding: "s2",
IndexPageSize: 10,
TotalRecords: 11,
DataEncoding: "v1",
Version: "v2",
Size_: 1000,
FooterSize: 2000,
@@ -394,7 +384,6 @@ func TestParseSearchBlockRequest(t *testing.T) {
StartPage: 0,
PagesToSearch: 10,
BlockID: "b92ec614-3fd7-4299-b6db-f657e7025a9b",
Encoding: "none",
IndexPageSize: 0,
TotalRecords: 2,
Version: "vParquet3",
@@ -431,31 +420,27 @@ func TestBuildSearchBlockRequest(t *testing.T) {
StartPage: 0,
PagesToSearch: 10,
BlockID: "b92ec614-3fd7-4299-b6db-f657e7025a9b",
Encoding: "s2",
IndexPageSize: 10,
TotalRecords: 11,
DataEncoding: "v1",
Version: "v2",
Size_: 1000,
FooterSize: 2000,
},
query: "?blockID=b92ec614-3fd7-4299-b6db-f657e7025a9b&pagesToSearch=10&size=1000&startPage=0&encoding=s2&indexPageSize=10&totalRecords=11&dataEncoding=v1&version=v2&footerSize=2000",
query: "?blockID=b92ec614-3fd7-4299-b6db-f657e7025a9b&pagesToSearch=10&size=1000&startPage=0&encoding=none&indexPageSize=10&totalRecords=11&version=v2&footerSize=2000",
},
{
req: &tempopb.SearchBlockRequest{
StartPage: 0,
PagesToSearch: 10,
BlockID: "b92ec614-3fd7-4299-b6db-f657e7025a9b",
Encoding: "s2",
IndexPageSize: 10,
TotalRecords: 11,
DataEncoding: "v1",
Version: "v2",
Size_: 1000,
FooterSize: 2000,
},
httpReq: httptest.NewRequest("GET", "/test/path", nil),
query: "/test/path?blockID=b92ec614-3fd7-4299-b6db-f657e7025a9b&pagesToSearch=10&size=1000&startPage=0&encoding=s2&indexPageSize=10&totalRecords=11&dataEncoding=v1&version=v2&footerSize=2000",
query: "/test/path?blockID=b92ec614-3fd7-4299-b6db-f657e7025a9b&pagesToSearch=10&size=1000&startPage=0&encoding=none&indexPageSize=10&totalRecords=11&version=v2&footerSize=2000",
},
{
req: &tempopb.SearchBlockRequest{
@@ -472,22 +457,19 @@ func TestBuildSearchBlockRequest(t *testing.T) {
StartPage: 0,
PagesToSearch: 10,
BlockID: "b92ec614-3fd7-4299-b6db-f657e7025a9b",
Encoding: "s2",
IndexPageSize: 10,
TotalRecords: 11,
DataEncoding: "v1",
Version: "v2",
Size_: 1000,
FooterSize: 2000,
},
query: "?start=10&end=20&limit=50&maxDuration=40ms&minDuration=30ms&spss=0&tags=foo%3Dbar&blockID=b92ec614-3fd7-4299-b6db-f657e7025a9b&pagesToSearch=10&size=1000&startPage=0&encoding=s2&indexPageSize=10&totalRecords=11&dataEncoding=v1&version=v2&footerSize=2000",
query: "?start=10&end=20&limit=50&maxDuration=40ms&minDuration=30ms&spss=0&tags=foo%3Dbar&blockID=b92ec614-3fd7-4299-b6db-f657e7025a9b&pagesToSearch=10&size=1000&startPage=0&encoding=none&indexPageSize=10&totalRecords=11&version=v2&footerSize=2000",
},
{
req: &tempopb.SearchBlockRequest{
StartPage: 0,
PagesToSearch: 10,
BlockID: "b92ec614-3fd7-4299-b6db-f657e7025a9b",
Encoding: "none",
IndexPageSize: 0,
TotalRecords: 2,
Version: "vParquet3",
@@ -498,7 +480,7 @@ func TestBuildSearchBlockRequest(t *testing.T) {
},
},
httpReq: httptest.NewRequest("GET", "/test/path", nil),
query: "/test/path?blockID=b92ec614-3fd7-4299-b6db-f657e7025a9b&pagesToSearch=10&size=1000&startPage=0&encoding=none&indexPageSize=0&totalRecords=2&dataEncoding=&version=vParquet3&footerSize=2000&dc=%5B%7B%22scope%22%3A1%2C%22name%22%3A%22net.sock.host.addr%22%7D%5D",
query: "/test/path?blockID=b92ec614-3fd7-4299-b6db-f657e7025a9b&pagesToSearch=10&size=1000&startPage=0&encoding=none&indexPageSize=0&totalRecords=2&version=vParquet3&footerSize=2000&dc=%5B%7B%22scope%22%3A1%2C%22name%22%3A%22net.sock.host.addr%22%7D%5D",
},
}
pkg/api/search_tags.go
+2 -44
View File
@@ -14,7 +14,6 @@ import (
"github.com/grafana/tempo/pkg/tempopb"
"github.com/grafana/tempo/pkg/traceql"
"github.com/grafana/tempo/tempodb/backend"
)
const (
@@ -72,13 +71,6 @@ func ParseSearchBlockRequest(r *http.Request) (*tempopb.SearchBlockRequest, erro
}
req.BlockID = blockID.String()
s = vals.Get(urlParamEncoding)
encoding, err := backend.ParseEncoding(s)
if err != nil {
return nil, err
}
req.Encoding = encoding.String()
s = vals.Get(urlParamIndexPageSize)
indexPageSize, err := strconv.ParseInt(s, 10, 32)
if err != nil {
@@ -96,12 +88,6 @@ func ParseSearchBlockRequest(r *http.Request) (*tempopb.SearchBlockRequest, erro
}
req.TotalRecords = uint32(totalRecords)
// Data encoding can be blank for some block formats, therefore
// no validation on the param here. Eventually we may be able
// to remove this parameter entirely.
dataEncoding := vals.Get(urlParamDataEncoding)
req.DataEncoding = dataEncoding
version := vals.Get(urlParamVersion)
if version == "" {
return nil, errors.New("version required")
@@ -196,13 +182,6 @@ func parseSearchTagValuesBlockRequest(r *http.Request, enforceTraceQL bool) (*te
}
req.BlockID = blockID.String()
s = vals.Get(urlParamEncoding)
encoding, err := backend.ParseEncoding(s)
if err != nil {
return nil, err
}
req.Encoding = encoding.String()
s = vals.Get(urlParamIndexPageSize)
indexPageSize, err := strconv.ParseInt(s, 10, 32)
if err != nil {
@@ -220,12 +199,6 @@ func parseSearchTagValuesBlockRequest(r *http.Request, enforceTraceQL bool) (*te
}
req.TotalRecords = uint32(totalRecords)
// Data encoding can be blank for some block formats, therefore
// no validation on the param here. Eventually we may be able
// to remove this parameter entirely.
dataEncoding := vals.Get(urlParamDataEncoding)
req.DataEncoding = dataEncoding
version := vals.Get(urlParamVersion)
if version == "" {
return nil, errors.New("version required")
@@ -305,13 +278,6 @@ func ParseSearchTagsBlockRequest(r *http.Request) (*tempopb.SearchTagsBlockReque
}
req.BlockID = blockID.String()
s = vals.Get(urlParamEncoding)
encoding, err := backend.ParseEncoding(s)
if err != nil {
return nil, err
}
req.Encoding = encoding.String()
s = vals.Get(urlParamIndexPageSize)
indexPageSize, err := strconv.ParseInt(s, 10, 32)
if err != nil {
@@ -329,12 +295,6 @@ func ParseSearchTagsBlockRequest(r *http.Request) (*tempopb.SearchTagsBlockReque
}
req.TotalRecords = uint32(totalRecords)
// Data encoding can be blank for some block formats, therefore
// no validation on the param here. Eventually we may be able
// to remove this parameter entirely.
dataEncoding := vals.Get(urlParamDataEncoding)
req.DataEncoding = dataEncoding
version := vals.Get(urlParamVersion)
if version == "" {
return nil, errors.New("version required")
@@ -517,10 +477,9 @@ func BuildSearchTagsBlockRequest(req *http.Request, searchReq *tempopb.SearchTag
q.addParam(urlParamBlockID, searchReq.BlockID)
q.addParam(urlParamStartPage, strconv.FormatUint(uint64(searchReq.StartPage), 10))
q.addParam(urlParamPagesToSearch, strconv.FormatUint(uint64(searchReq.PagesToSearch), 10))
q.addParam(urlParamEncoding, searchReq.Encoding)
q.addParam("encoding", "none")
q.addParam(urlParamIndexPageSize, strconv.FormatUint(uint64(searchReq.IndexPageSize), 10))
q.addParam(urlParamTotalRecords, strconv.FormatUint(uint64(searchReq.TotalRecords), 10))
q.addParam(urlParamDataEncoding, searchReq.DataEncoding)
q.addParam(urlParamVersion, searchReq.Version)
q.addParam(urlParamFooterSize, strconv.FormatUint(uint64(searchReq.FooterSize), 10))
@@ -571,10 +530,9 @@ func BuildSearchTagValuesBlockRequest(req *http.Request, searchReq *tempopb.Sear
qb.addParam(urlParamBlockID, searchReq.BlockID)
qb.addParam(urlParamStartPage, strconv.FormatUint(uint64(searchReq.StartPage), 10))
qb.addParam(urlParamPagesToSearch, strconv.FormatUint(uint64(searchReq.PagesToSearch), 10))
qb.addParam(urlParamEncoding, searchReq.Encoding)
qb.addParam("encoding", "none")
qb.addParam(urlParamIndexPageSize, strconv.FormatUint(uint64(searchReq.IndexPageSize), 10))
qb.addParam(urlParamTotalRecords, strconv.FormatUint(uint64(searchReq.TotalRecords), 10))
qb.addParam(urlParamDataEncoding, searchReq.DataEncoding)
qb.addParam(urlParamVersion, searchReq.Version)
qb.addParam(urlParamFooterSize, strconv.FormatUint(uint64(searchReq.FooterSize), 10))
pkg/docsgen/generate_manifest.go
-1
View File
@@ -37,7 +37,6 @@ func main() {
// Override values that depend on the host specifics
const hostname = "hostname"
newConfig.Distributor.DistributorRing.InstanceID = hostname
newConfig.Compactor.ShardingRing.InstanceID = hostname
newConfig.Ingester.LifecyclerConfig.ID = hostname
newConfig.Ingester.LifecyclerConfig.InfNames = []string{"eth0"}
newConfig.Generator.Ring.InstanceID = hostname
pkg/tempopb/tempo.pb.go
+226 -578
View File
File diff suppressed because it is too large Load Diff
pkg/tempopb/tempo.proto
+7 -7
View File
@@ -98,10 +98,10 @@ message SearchBlockRequest {
string blockID = 2;
uint32 startPage = 3;
uint32 pagesToSearch = 4;
string encoding = 5;
// string encoding = 5;
uint32 indexPageSize = 6;
uint32 totalRecords = 7;
string dataEncoding = 8;
// string dataEncoding = 8;
string version = 9;
uint64 size = 10; // total size of data file
uint32 footerSize = 11; // size of file footer (parquet)
@@ -195,10 +195,10 @@ message SearchTagsBlockRequest {
string blockID = 2;
uint32 startPage = 3;
uint32 pagesToSearch = 4;
string encoding = 5;
// string encoding = 5;
uint32 indexPageSize = 6;
uint32 totalRecords = 7;
string dataEncoding = 8;
// string dataEncoding = 8;
string version = 9;
uint64 size = 10; // total size of data file
uint32 footerSize = 11; // size of file footer (parquet)
@@ -212,10 +212,10 @@ message SearchTagValuesBlockRequest {
string blockID = 2;
uint32 startPage = 3;
uint32 pagesToSearch = 4;
string encoding = 5;
// string encoding = 5;
uint32 indexPageSize = 6;
uint32 totalRecords = 7;
string dataEncoding = 8;
// string dataEncoding = 8;
string version = 9;
uint64 size = 10; // total size of data file
uint32 footerSize = 11; // size of file footer (parquet)
@@ -450,7 +450,7 @@ message QueryRangeRequest {
uint32 startPage = 9;
uint32 pagesToSearch = 10;
string version = 11;
string encoding = 12;
// string encoding = 12;
uint64 size = 13; // total size of data file
uint32 footerSize = 14; // size of file footer (parquet)
repeated DedicatedColumn dedicatedColumns = 15;
pkg/usagestats/stats_test.go
+4 -4
View File
@@ -23,7 +23,7 @@ func Test_BuildReport(t *testing.T) {
Edition("non-OSS")
Edition("OSS")
Target("distributor")
Target("compactor")
Target("backend-worker")
NewString("compression").Set("snappy")
NewString("compression").Set("lz4")
NewInt("compression_ratio").Set(50)
@@ -48,7 +48,7 @@ func Test_BuildReport(t *testing.T) {
require.Equal(t, r.Os, runtime.GOOS)
require.Equal(t, r.PrometheusVersion, build.GetVersion())
require.Equal(t, r.Edition, "OSS")
require.Equal(t, r.Target, "compactor")
require.Equal(t, r.Target, "backend-worker")
require.Equal(t, r.Metrics["num_cpu"], runtime.NumCPU())
// Don't check num_goroutine because it could have changed since the report was created.
require.Equal(t, r.Metrics["compression"], "lz4")
@@ -69,7 +69,7 @@ func Test_BuildStats(t *testing.T) {
Edition("non-OSS")
Edition("OSS")
Target("distributor")
Target("compactor")
Target("backend-worker")
NewString("compression").Set("snappy")
NewString("compression").Set("lz4")
NewInt("compression_ratio").Set(50)
@@ -94,7 +94,7 @@ func Test_BuildStats(t *testing.T) {
require.Equal(t, r.Os, runtime.GOOS)
require.Equal(t, r.PrometheusVersion, build.GetVersion())
require.Equal(t, r.Edition, "OSS")
require.Equal(t, r.Target, "compactor")
require.Equal(t, r.Target, "backend-worker")
require.Equal(t, r.Metrics["num_cpu"], runtime.NumCPU())
// Don't check num_goroutine because it could have changed since the report was created.
require.Equal(t, r.Metrics["compression"], "lz4")
tempodb/backend/block_meta.go
+3 -5
View File
@@ -219,17 +219,15 @@ func (dcs *DedicatedColumns) UnmarshalJSON(b []byte) error {
return nil
}
func NewBlockMeta(tenantID string, blockID uuid.UUID, version string, encoding Encoding, dataEncoding string) *BlockMeta {
return NewBlockMetaWithDedicatedColumns(tenantID, blockID, version, encoding, dataEncoding, nil)
func NewBlockMeta(tenantID string, blockID uuid.UUID, version string) *BlockMeta {
return NewBlockMetaWithDedicatedColumns(tenantID, blockID, version, nil)
}
func NewBlockMetaWithDedicatedColumns(tenantID string, blockID uuid.UUID, version string, encoding Encoding, dataEncoding string, dc DedicatedColumns) *BlockMeta {
func NewBlockMetaWithDedicatedColumns(tenantID string, blockID uuid.UUID, version string, dc DedicatedColumns) *BlockMeta {
b := &BlockMeta{
Version: version,
BlockID: UUID(blockID),
TenantID: tenantID,
Encoding: encoding,
DataEncoding: dataEncoding,
DedicatedColumns: dc,
}
tempodb/backend/block_meta_test.go
+1 -9
View File
@@ -20,17 +20,13 @@ const (
func TestNewBlockMeta(t *testing.T) {
testVersion := "blerg"
testEncoding := EncLZ4_256k
testDataEncoding := "blarg"
id := uuid.New()
b := NewBlockMeta(testTenantID, id, testVersion, testEncoding, testDataEncoding)
b := NewBlockMeta(testTenantID, id, testVersion)
assert.Equal(t, id, (uuid.UUID)(b.BlockID))
assert.Equal(t, testTenantID, b.TenantID)
assert.Equal(t, testVersion, b.Version)
assert.Equal(t, testEncoding, b.Encoding)
assert.Equal(t, testDataEncoding, b.DataEncoding)
}
func TestBlockMetaObjectAdded(t *testing.T) {
@@ -107,10 +103,8 @@ func TestBlockMetaJSONProtoRoundTrip(t *testing.T) {
TotalObjects: 10,
Size_: 12345,
CompactionLevel: 1,
Encoding: EncZstd,
IndexPageSize: 250000,
TotalRecords: 124356,
DataEncoding: "",
BloomShardCount: 244,
FooterSize: 15775,
DedicatedColumns: DedicatedColumns{
@@ -131,10 +125,8 @@ func TestBlockMetaJSONProtoRoundTrip(t *testing.T) {
"totalObjects": 10,
"size": 12345,
"compactionLevel": 1,
"encoding": "zstd",
"indexPageSize": 250000,
"totalRecords": 124356,
"dataEncoding": "",
"bloomShards": 244,
"footerSize": 15775,
"dedicatedColumns": [
tempodb/backend/encoding.go
-152
View File
@@ -1,152 +0,0 @@
package backend
import (
"bytes"
"encoding/json"
"fmt"
"strings"
)
// Encoding is the identifier for a chunk encoding.
type Encoding byte
// The different available encodings.
// Make sure to preserve the order, as these numeric values are written to the chunks!
const (
EncNone Encoding = iota
EncGZIP
EncLZ4_64k
EncLZ4_256k
EncLZ4_1M
EncLZ4_4M
EncSnappy
EncZstd
EncS2
)
// SupportedEncoding is a slice of all supported encodings
var SupportedEncoding = []Encoding{
EncNone,
EncGZIP,
EncLZ4_64k,
EncLZ4_256k,
EncLZ4_1M,
EncLZ4_4M,
EncSnappy,
EncZstd,
EncS2,
}
func (e Encoding) String() string {
switch e {
case EncNone:
return "none"
case EncGZIP:
return "gzip"
case EncLZ4_64k:
return "lz4-64k"
case EncLZ4_256k:
return "lz4-256k"
case EncLZ4_1M:
return "lz4-1M"
case EncLZ4_4M:
return "lz4"
case EncSnappy:
return "snappy"
case EncZstd:
return "zstd"
case EncS2:
return "s2"
default:
return "unsupported"
}
}
// UnmarshalYAML implements the Unmarshaler interface of the yaml pkg.
func (e *Encoding) UnmarshalYAML(unmarshal func(interface{}) error) error {
var encString string
err := unmarshal(&encString)
if err != nil {
return err
}
*e, err = ParseEncoding(encString)
if err != nil {
return err
}
return nil
}
// MarshalYAML implements the Marshaler interface of the yaml pkg
func (e Encoding) MarshalYAML() (interface{}, error) {
return e.String(), nil
}
// UnmarshalJSON implements the Unmarshaler interface of the json pkg.
func (e *Encoding) UnmarshalJSON(b []byte) error {
var encString string
err := json.Unmarshal(b, &encString)
if err != nil {
return err
}
*e, err = ParseEncoding(encString)
if err != nil {
return err
}
return nil
}
// MarshalJSON implements the marshaler interface of the json pkg.
func (e Encoding) MarshalJSON() ([]byte, error) {
buffer := bytes.NewBufferString("\"" + e.String() + "\"")
return buffer.Bytes(), nil
}
func (e Encoding) Marshal() ([]byte, error) {
return []byte{byte(e)}, nil
}
func (e *Encoding) MarshalTo(data []byte) (n int, err error) {
data[0] = byte(*e)
return 1, nil
}
func (e *Encoding) Unmarshal(data []byte) error {
if len(data) != 1 {
return fmt.Errorf("invalid data: %s", data)
}
x := Encoding(data[0])
*e = x
return nil
}
func (e *Encoding) Size() int {
return 1
}
// ParseEncoding parses a chunk encoding (compression algorithm) by its name.
func ParseEncoding(enc string) (Encoding, error) {
for _, e := range SupportedEncoding {
if strings.EqualFold(e.String(), enc) {
return e, nil
}
}
return 0, fmt.Errorf("invalid encoding: %s, supported: %s", enc, SupportedEncodingString())
}
// SupportedEncodingString returns the list of supported Encoding.
func SupportedEncodingString() string {
var sb strings.Builder
for i := range SupportedEncoding {
sb.WriteString(SupportedEncoding[i].String())
if i != len(SupportedEncoding)-1 {
sb.WriteString(", ")
}
}
return sb.String()
}
tempodb/backend/encoding_test.go
-45
View File
@@ -1,45 +0,0 @@
package backend
import (
"encoding/json"
"testing"
"github.com/stretchr/testify/assert"
"gopkg.in/yaml.v3"
)
type marshalTest struct {
Test Encoding
}
func TestUnmarshalMarshalYaml(t *testing.T) {
for _, enc := range SupportedEncoding {
expected := marshalTest{
Test: enc,
}
actual := marshalTest{}
buff, err := yaml.Marshal(expected)
assert.NoError(t, err)
err = yaml.Unmarshal(buff, &actual)
assert.NoError(t, err)
assert.Equal(t, expected, actual)
}
}
func TestUnmarshalMarshalJson(t *testing.T) {
for _, enc := range SupportedEncoding {
expected := marshalTest{
Test: enc,
}
actual := marshalTest{}
buff, err := json.Marshal(expected)
assert.NoError(t, err)
err = json.Unmarshal(buff, &actual)
assert.NoError(t, err)
assert.Equal(t, expected, actual)
}
}
tempodb/backend/raw_test.go
+4 -4
View File
@@ -44,7 +44,7 @@ func TestWriter(t *testing.T) {
u = uuid.New()
expectedPath := filepath.Join("test", u.String(), MetaName)
meta := NewBlockMeta("test", u, "blerg", EncGZIP, "glarg")
meta := NewBlockMeta("test", u, "blerg")
jsonBytes, err := json.Marshal(meta)
assert.NoError(t, err)
assert.NoError(t, err)
@@ -134,7 +134,7 @@ func TestReader(t *testing.T) {
assert.Error(t, err)
assert.Nil(t, meta)
expectedMeta := NewBlockMeta("test", uuid.New(), "blerg", EncGZIP, "glarg")
expectedMeta := NewBlockMeta("test", uuid.New(), "blerg")
m.R, _ = json.Marshal(expectedMeta)
meta, err = r.BlockMeta(ctx, uuid.New(), "test")
assert.NoError(t, err)
@@ -210,7 +210,7 @@ func TestRootPath(t *testing.T) {
func TestRoundTripMeta(t *testing.T) {
// RoundTrip with empty DedicatedColumns
meta := NewBlockMeta("test", uuid.New(), "blerg", EncGZIP, "glarg")
meta := NewBlockMeta("test", uuid.New(), "blerg")
// RoundTrip with empty DedicatedColumns
expectedPb, err := meta.Marshal()
assert.NoError(t, err)
@@ -252,7 +252,7 @@ func TestTenantIndexFallback(t *testing.T) {
tenantID = "test"
u = uuid.New()
meta = NewBlockMeta(tenantID, u, "blerg", EncGZIP, "glarg")
meta = NewBlockMeta(tenantID, u, "blerg")
expectedIdx = newTenantIndex([]*BlockMeta{meta}, nil)
)
tempodb/backend/tenantindex_benchmark_test.go
+2 -2
View File
@@ -108,12 +108,12 @@ func makeTestTenantIndex(numBlocks int) *TenantIndex {
blocks := make([]*BlockMeta, 0, numBlocks)
compactedBlocks := make([]*CompactedBlockMeta, 0, numBlocks)
for i := range numBlocks {
meta := NewBlockMeta("test-tenant", uuid.New(), "vParquet4", EncNone, "")
meta := NewBlockMeta("test-tenant", uuid.New(), "vParquet4")
meta.DedicatedColumns = dedicatedCols[i%numDistinctDedicatedCols]
blocks = append(blocks, meta)
compactedMeta := &CompactedBlockMeta{
BlockMeta: *NewBlockMeta("test-tenant", uuid.New(), "vParquet4", EncNone, ""),
BlockMeta: *NewBlockMeta("test-tenant", uuid.New(), "vParquet4"),
CompactedTime: time.Now(),
}
compactedMeta.DedicatedColumns = dedicatedCols[i%numDistinctDedicatedCols]
tempodb/backend/tenantindex_test.go
+12 -12
View File
@@ -21,9 +21,9 @@ func TestIndexMarshalUnmarshal(t *testing.T) {
idx: &TenantIndex{
CreatedAt: time.Now(),
Meta: []*BlockMeta{
NewBlockMeta("test", uuid.New(), "v1", EncGZIP, "adsf"),
NewBlockMeta("test", uuid.New(), "v2", EncNone, "adsf"),
NewBlockMeta("test", uuid.New(), "v3", EncLZ4_4M, "adsf"),
NewBlockMeta("test", uuid.New(), "v1"),
NewBlockMeta("test", uuid.New(), "v2"),
NewBlockMeta("test", uuid.New(), "v3"),
},
},
},
@@ -32,15 +32,15 @@ func TestIndexMarshalUnmarshal(t *testing.T) {
CreatedAt: time.Now(),
CompactedMeta: []*CompactedBlockMeta{
{
BlockMeta: *NewBlockMeta("test", uuid.New(), "v1", EncGZIP, "adsf"),
BlockMeta: *NewBlockMeta("test", uuid.New(), "v1"),
CompactedTime: time.Now(),
},
{
BlockMeta: *NewBlockMeta("test", uuid.New(), "v1", EncZstd, "adsf"),
BlockMeta: *NewBlockMeta("test", uuid.New(), "v1"),
CompactedTime: time.Now(),
},
{
BlockMeta: *NewBlockMeta("test", uuid.New(), "v1", EncSnappy, "adsf"),
BlockMeta: *NewBlockMeta("test", uuid.New(), "v1"),
CompactedTime: time.Now(),
},
},
@@ -49,21 +49,21 @@ func TestIndexMarshalUnmarshal(t *testing.T) {
{
idx: &TenantIndex{
Meta: []*BlockMeta{
NewBlockMeta("test", uuid.New(), "v1", EncGZIP, "adsf"),
NewBlockMeta("test", uuid.New(), "v2", EncNone, "adsf"),
NewBlockMeta("test", uuid.New(), "v3", EncLZ4_4M, "adsf"),
NewBlockMeta("test", uuid.New(), "v1"),
NewBlockMeta("test", uuid.New(), "v2"),
NewBlockMeta("test", uuid.New(), "v3"),
},
CompactedMeta: []*CompactedBlockMeta{
{
BlockMeta: *NewBlockMeta("test", uuid.New(), "v1", EncGZIP, "adsf"),
BlockMeta: *NewBlockMeta("test", uuid.New(), "v1"),
CompactedTime: time.Now(),
},
{
BlockMeta: *NewBlockMeta("test", uuid.New(), "v1", EncZstd, "adsf"),
BlockMeta: *NewBlockMeta("test", uuid.New(), "v1"),
CompactedTime: time.Now(),
},
{
BlockMeta: *NewBlockMeta("test", uuid.New(), "v1", EncSnappy, "adsf"),
BlockMeta: *NewBlockMeta("test", uuid.New(), "v1"),
CompactedTime: time.Now(),
},
},
tempodb/backend/test/backend_test.go
+4 -4
View File
@@ -31,10 +31,10 @@ func TestFixtures(t *testing.T) {
// To regenerate the fixtures, uncomment the write path.
// metas := []*backend.BlockMeta{
// backend.NewBlockMeta(tenant, uuid.New(), "v1", backend.EncGZIP, "adsf"),
// backend.NewBlockMeta(tenant, uuid.New(), "v2", backend.EncNone, "adsf"),
// backend.NewBlockMeta(tenant, uuid.New(), "v3", backend.EncLZ4_4M, "adsf"),
// backend.NewBlockMeta(tenant, uuid.New(), "v4", backend.EncLZ4_1M, "adsf"),
// backend.NewBlockMeta(tenant, uuid.New(), "v1"),
// backend.NewBlockMeta(tenant, uuid.New(), "v2"),
// backend.NewBlockMeta(tenant, uuid.New(), "v3"),
// backend.NewBlockMeta(tenant, uuid.New(), "v4"),
// }
//
// for _, meta := range metas {
tempodb/backend/test/benchmark_test.go
-2
View File
@@ -27,10 +27,8 @@ func BenchmarkIndexLoad(b *testing.B) {
TotalObjects: 10,
Size_: 12345,
CompactionLevel: 1,
Encoding: backend.EncZstd,
IndexPageSize: 250000,
TotalRecords: 124356,
DataEncoding: "",
BloomShardCount: 244,
FooterSize: 15775,
DedicatedColumns: backend.DedicatedColumns{
tempodb/backend/test/test-data/single-tenant/f5ac87ed-77f1-408e-9c98-785692900e81/meta.json
+1 -1
View File
@@ -1 +1 @@
{"format":"v2","blockID":"f5ac87ed-77f1-408e-9c98-785692900e81","tenantID":"single-tenant","startTime":"0001-01-01T00:00:00Z","endTime":"0001-01-01T00:00:00Z","totalObjects":0,"compactionLevel":0,"encoding":"none","indexPageSize":0,"totalRecords":0,"dataEncoding":"adsf","bloomShards":0,"footerSize":0}
{"format":"vParquet5","blockID":"f5ac87ed-77f1-408e-9c98-785692900e81","tenantID":"single-tenant","startTime":"0001-01-01T00:00:00Z","endTime":"0001-01-01T00:00:00Z","totalObjects":0,"compactionLevel":0,"encoding":"none","indexPageSize":0,"totalRecords":0,"dataEncoding":"adsf","bloomShards":0,"footerSize":0}
tempodb/backend/test/test-data/single-tenant/index.pb.zst
BIN
View File
Binary file not shown.
tempodb/backend/v1.pb.go
+60 -158
View File
@@ -28,18 +28,18 @@ var _ = time.Kitchen
const _ = proto.GoGoProtoPackageIsVersion3 // please upgrade the proto package
type BlockMeta struct {
Version string `protobuf:"bytes,1,opt,name=version,proto3" json:"format"`
BlockID UUID `protobuf:"bytes,2,opt,name=block_id,json=blockId,proto3,customtype=UUID" json:"blockID"`
TenantID string `protobuf:"bytes,5,opt,name=tenant_id,json=tenantId,proto3" json:"tenantID"`
StartTime time.Time `protobuf:"bytes,6,opt,name=start_time,json=startTime,proto3,stdtime" json:"startTime"`
EndTime time.Time `protobuf:"bytes,7,opt,name=end_time,json=endTime,proto3,stdtime" json:"endTime"`
TotalObjects int64 `protobuf:"varint,8,opt,name=total_objects,json=totalObjects,proto3" json:"totalObjects"`
Size_ uint64 `protobuf:"varint,9,opt,name=size,proto3" json:"size,omitempty"`
CompactionLevel uint32 `protobuf:"varint,10,opt,name=compaction_level,json=compactionLevel,proto3" json:"compactionLevel"`
Encoding Encoding `protobuf:"bytes,11,opt,name=encoding,proto3,customtype=Encoding" json:"encoding"`
IndexPageSize uint32 `protobuf:"varint,12,opt,name=index_page_size,json=indexPageSize,proto3" json:"indexPageSize"`
TotalRecords uint32 `protobuf:"varint,13,opt,name=total_records,json=totalRecords,proto3" json:"totalRecords"`
DataEncoding string `protobuf:"bytes,14,opt,name=data_encoding,json=dataEncoding,proto3" json:"dataEncoding"`
Version string `protobuf:"bytes,1,opt,name=version,proto3" json:"format"`
BlockID UUID `protobuf:"bytes,2,opt,name=block_id,json=blockId,proto3,customtype=UUID" json:"blockID"`
TenantID string `protobuf:"bytes,5,opt,name=tenant_id,json=tenantId,proto3" json:"tenantID"`
StartTime time.Time `protobuf:"bytes,6,opt,name=start_time,json=startTime,proto3,stdtime" json:"startTime"`
EndTime time.Time `protobuf:"bytes,7,opt,name=end_time,json=endTime,proto3,stdtime" json:"endTime"`
TotalObjects int64 `protobuf:"varint,8,opt,name=total_objects,json=totalObjects,proto3" json:"totalObjects"`
Size_ uint64 `protobuf:"varint,9,opt,name=size,proto3" json:"size,omitempty"`
CompactionLevel uint32 `protobuf:"varint,10,opt,name=compaction_level,json=compactionLevel,proto3" json:"compactionLevel"`
// bytes encoding = 11[(gogoproto.customtype) = "Encoding", (gogoproto.nullable) = false];
IndexPageSize uint32 `protobuf:"varint,12,opt,name=index_page_size,json=indexPageSize,proto3" json:"indexPageSize"`
TotalRecords uint32 `protobuf:"varint,13,opt,name=total_records,json=totalRecords,proto3" json:"totalRecords"`
// string data_encoding = 14[(gogoproto.jsontag) = "dataEncoding"];
BloomShardCount uint32 `protobuf:"varint,15,opt,name=bloom_shard_count,json=bloomShardCount,proto3" json:"bloomShards"`
FooterSize uint32 `protobuf:"varint,16,opt,name=footer_size,json=footerSize,proto3" json:"footerSize"`
DedicatedColumns DedicatedColumns `protobuf:"bytes,17,opt,name=dedicated_columns,json=dedicatedColumns,proto3,customtype=DedicatedColumns" json:"dedicatedColumns,omitempty"`
@@ -143,13 +143,6 @@ func (m *BlockMeta) GetTotalRecords() uint32 {
return 0
}
func (m *BlockMeta) GetDataEncoding() string {
if m != nil {
return m.DataEncoding
}
return ""
}
func (m *BlockMeta) GetBloomShardCount() uint32 {
if m != nil {
return m.BloomShardCount
@@ -285,57 +278,54 @@ func init() {
func init() { proto.RegisterFile("v1/v1.proto", fileDescriptor_39a831d85a350775) }
var fileDescriptor_39a831d85a350775 = []byte{
// 788 bytes of a gzipped FileDescriptorProto
0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x84, 0x54, 0xcd, 0x8e, 0xe3, 0x44,
0x10, 0x8e, 0x67, 0xc2, 0x24, 0xe9, 0x24, 0x93, 0xa4, 0x57, 0x8b, 0x4c, 0x90, 0xd2, 0x51, 0xc4,
0x21, 0x48, 0x8b, 0xa3, 0xd9, 0xd5, 0x22, 0x21, 0x04, 0x12, 0x9e, 0x01, 0x69, 0x11, 0x3f, 0x8b,
0x77, 0xf6, 0x82, 0x90, 0xac, 0xb6, 0xbb, 0xe3, 0x35, 0x6b, 0xbb, 0x23, 0xbb, 0x13, 0xc1, 0x3e,
0xc5, 0x3e, 0x0d, 0xcf, 0x30, 0xe2, 0x34, 0x47, 0xc4, 0xa1, 0x41, 0x99, 0x9b, 0x79, 0x09, 0xd4,
0x65, 0xc7, 0x4e, 0x66, 0x04, 0x73, 0x89, 0xaa, 0xea, 0xab, 0xaf, 0xaa, 0xbe, 0x6a, 0x57, 0x50,
0x77, 0x73, 0xb6, 0xd8, 0x9c, 0x59, 0xab, 0x54, 0x48, 0x81, 0x91, 0x47, 0xfd, 0xd7, 0x3c, 0x61,
0xd6, 0xe6, 0x6c, 0x4c, 0x02, 0x21, 0x82, 0x88, 0x2f, 0x00, 0xf1, 0xd6, 0xcb, 0x85, 0x0c, 0x63,
0x9e, 0x49, 0x1a, 0xaf, 0x8a, 0xe4, 0xf1, 0x47, 0x41, 0x28, 0x5f, 0xad, 0x3d, 0xcb, 0x17, 0xf1,
0x22, 0x10, 0x81, 0xa8, 0x33, 0xb5, 0x07, 0x0e, 0x58, 0x45, 0xfa, 0xec, 0xf7, 0x16, 0xea, 0xd8,
0x91, 0xf0, 0x5f, 0x7f, 0xcb, 0x25, 0xc5, 0x1f, 0xa0, 0xd6, 0x86, 0xa7, 0x59, 0x28, 0x12, 0xd3,
0x98, 0x1a, 0xf3, 0x8e, 0x8d, 0x72, 0x45, 0x4e, 0x96, 0x22, 0x8d, 0xa9, 0x74, 0x76, 0x10, 0xfe,
0x0c, 0xb5, 0x3d, 0x4d, 0x71, 0x43, 0x66, 0x1e, 0x4d, 0x8d, 0x79, 0xcf, 0x9e, 0x5d, 0x29, 0xd2,
0xf8, 0x53, 0x91, 0xe6, 0xcb, 0x97, 0xcf, 0x2e, 0xb6, 0x8a, 0xb4, 0xa0, 0xe4, 0xb3, 0x8b, 0x5c,
0x91, 0x96, 0x57, 0x98, 0x4e, 0x69, 0x30, 0xfc, 0x14, 0x75, 0x24, 0x4f, 0x68, 0x22, 0x35, 0xff,
0x1d, 0x68, 0x63, 0x6e, 0x15, 0x69, 0x5f, 0x42, 0x10, 0x48, 0x6d, 0x59, 0xda, 0xce, 0xce, 0x62,
0xf8, 0x39, 0x42, 0x99, 0xa4, 0xa9, 0x74, 0xb5, 0x62, 0xf3, 0x64, 0x6a, 0xcc, 0xbb, 0x8f, 0xc7,
0x56, 0xb1, 0x0e, 0x6b, 0x27, 0xd2, 0xba, 0xdc, 0xad, 0xc3, 0x7e, 0xa8, 0x67, 0xca, 0x15, 0xe9,
0x00, 0x4b, 0xc7, 0xdf, 0xfe, 0x45, 0x0c, 0xa7, 0x76, 0xf1, 0xd7, 0xa8, 0xcd, 0x13, 0x56, 0xd4,
0x6b, 0xdd, 0x5b, 0xef, 0x41, 0x59, 0xaf, 0xc5, 0x13, 0x56, 0x55, 0xdb, 0x39, 0xf8, 0x29, 0xea,
0x4b, 0x21, 0x69, 0xe4, 0x0a, 0xef, 0x67, 0xee, 0xcb, 0xcc, 0x6c, 0x4f, 0x8d, 0xf9, 0xb1, 0x3d,
0xcc, 0x15, 0xe9, 0x01, 0xf0, 0x7d, 0x11, 0x77, 0x0e, 0x3c, 0x8c, 0x51, 0x33, 0x0b, 0xdf, 0x70,
0xb3, 0x33, 0x35, 0xe6, 0x4d, 0x07, 0x6c, 0xfc, 0x39, 0x1a, 0xfa, 0x22, 0x5e, 0x51, 0x5f, 0x86,
0x22, 0x71, 0x23, 0xbe, 0xe1, 0x91, 0x89, 0xa6, 0xc6, 0xbc, 0x6f, 0x3f, 0xc8, 0x15, 0x19, 0xd4,
0xd8, 0x37, 0x1a, 0x72, 0x6e, 0x07, 0xf0, 0x23, 0x2d, 0xcb, 0x17, 0x2c, 0x4c, 0x02, 0xb3, 0x0b,
0xcf, 0x33, 0x2c, 0x9f, 0xa7, 0xfd, 0x65, 0x19, 0x77, 0xaa, 0x0c, 0xfc, 0x09, 0x1a, 0x84, 0x09,
0xe3, 0xbf, 0xb8, 0x2b, 0x1a, 0x70, 0x17, 0x86, 0xe9, 0x41, 0xb3, 0x51, 0xae, 0x48, 0x1f, 0xa0,
0xe7, 0x34, 0xe0, 0x2f, 0xc2, 0x37, 0xdc, 0x39, 0x74, 0x6b, 0xcd, 0x29, 0xf7, 0x45, 0xca, 0x32,
0xb3, 0x0f, 0xc4, 0x5a, 0xb3, 0x53, 0xc4, 0x9d, 0x03, 0x4f, 0xd3, 0x18, 0x95, 0xd4, 0xad, 0x86,
0x3c, 0x85, 0x6f, 0x00, 0x68, 0x1a, 0xa8, 0x86, 0x3c, 0xf0, 0xf0, 0xa7, 0x68, 0xe4, 0x45, 0x42,
0xc4, 0x6e, 0xf6, 0x8a, 0xa6, 0xcc, 0xf5, 0xc5, 0x3a, 0x91, 0xe6, 0x00, 0x3a, 0x0e, 0x72, 0x45,
0xba, 0x00, 0xbe, 0xd0, 0x58, 0xe6, 0x0c, 0x6a, 0xe7, 0x5c, 0xe7, 0xe1, 0x05, 0xea, 0x2e, 0x85,
0x90, 0x3c, 0x2d, 0x14, 0x0e, 0x81, 0x76, 0x9a, 0x2b, 0x82, 0x8a, 0x30, 0xc8, 0xdb, 0xb3, 0xb1,
0x8f, 0x46, 0x8c, 0xb3, 0xd0, 0xa7, 0x92, 0xeb, 0x5e, 0xd1, 0x3a, 0x4e, 0x32, 0x73, 0x04, 0xdb,
0xfc, 0xb8, 0xdc, 0xe6, 0xf0, 0x62, 0x97, 0x70, 0x5e, 0xe0, 0xb9, 0x22, 0x63, 0x76, 0x2b, 0xf6,
0x48, 0xc4, 0xa1, 0xe4, 0xf1, 0x4a, 0xfe, 0xea, 0x0c, 0x6f, 0x63, 0xf8, 0x3b, 0x84, 0x53, 0xbe,
0x8a, 0x74, 0x50, 0x3f, 0xf5, 0x92, 0xfa, 0x52, 0xa4, 0x26, 0x86, 0xe1, 0x48, 0xae, 0xc8, 0xfb,
0x7b, 0xe8, 0x57, 0x00, 0xee, 0x95, 0x1b, 0xdd, 0x01, 0x67, 0xbf, 0x19, 0x08, 0x9f, 0x17, 0x5f,
0x03, 0x67, 0xf5, 0x55, 0xdb, 0x08, 0x15, 0xf7, 0x1a, 0x73, 0x49, 0xe1, 0xb0, 0xbb, 0x8f, 0x1f,
0x5a, 0xf5, 0x9f, 0x8a, 0x55, 0xa5, 0xda, 0x3d, 0xad, 0xed, 0x5a, 0x11, 0x23, 0x57, 0xa4, 0xe1,
0x74, 0xbc, 0xaa, 0xc6, 0x4f, 0xe8, 0xd4, 0xdf, 0x55, 0x2e, 0x2e, 0xe6, 0xe8, 0xde, 0x8b, 0x79,
0xaf, 0xbc, 0x98, 0x7e, 0xc5, 0xac, 0xee, 0xe6, 0x30, 0x34, 0xfb, 0xc7, 0x40, 0xdd, 0xf2, 0xfc,
0xf5, 0x17, 0x86, 0x7f, 0x40, 0xc8, 0x4f, 0x39, 0xec, 0x9e, 0xca, 0x72, 0xe2, 0xff, 0xeb, 0xf4,
0x6e, 0xd9, 0x69, 0x8f, 0x55, 0x1c, 0x7b, 0xe9, 0x7f, 0x21, 0xf1, 0x13, 0xd4, 0x04, 0xf9, 0x47,
0xd3, 0xe3, 0xff, 0x96, 0xdf, 0xce, 0x15, 0x81, 0x34, 0x07, 0x7e, 0xf1, 0xe5, 0xbe, 0x6a, 0xa0,
0x1f, 0x03, 0x7d, 0xb2, 0x4f, 0xbf, 0xbb, 0x71, 0xbb, 0xaf, 0xff, 0x77, 0x2a, 0xe6, 0x9e, 0x5a,
0x40, 0x3f, 0xbc, 0xda, 0x4e, 0x8c, 0xeb, 0xed, 0xc4, 0xf8, 0x7b, 0x3b, 0x31, 0xde, 0xde, 0x4c,
0x1a, 0xd7, 0x37, 0x93, 0xc6, 0x1f, 0x37, 0x93, 0xc6, 0x8f, 0x03, 0xfd, 0xb6, 0x82, 0x79, 0x8b,
0xb2, 0xbc, 0x77, 0x02, 0x62, 0x9f, 0xfc, 0x1b, 0x00, 0x00, 0xff, 0xff, 0x2e, 0xae, 0x67, 0xf2,
0x10, 0x06, 0x00, 0x00,
// 746 bytes of a gzipped FileDescriptorProto
0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x84, 0x54, 0xdd, 0x6e, 0xd3, 0x30,
0x14, 0x6e, 0xb6, 0xd2, 0x1f, 0xb7, 0x5d, 0x5b, 0x4f, 0x43, 0xa1, 0x48, 0x75, 0x55, 0x71, 0x51,
0x24, 0x48, 0xb5, 0x4d, 0x43, 0x42, 0x08, 0x24, 0xb2, 0x09, 0x69, 0x88, 0x9f, 0x91, 0x6d, 0x37,
0x08, 0x29, 0x72, 0x12, 0xb7, 0x0b, 0x4b, 0xe2, 0x2a, 0x71, 0x2b, 0xd8, 0x53, 0xec, 0x69, 0x78,
0x86, 0x5d, 0xee, 0x12, 0x71, 0x61, 0x50, 0x27, 0x6e, 0xc2, 0x4b, 0x20, 0x3b, 0x69, 0xd3, 0x6e,
0x82, 0xdd, 0x54, 0xe7, 0x9c, 0xef, 0x7c, 0x9f, 0xfd, 0x39, 0xe7, 0x14, 0x54, 0x26, 0x9b, 0xfd,
0xc9, 0xa6, 0x36, 0x0a, 0x29, 0xa3, 0x10, 0x58, 0xd8, 0x3e, 0x25, 0x81, 0xa3, 0x4d, 0x36, 0x5b,
0x68, 0x48, 0xe9, 0xd0, 0x23, 0x7d, 0x89, 0x58, 0xe3, 0x41, 0x9f, 0xb9, 0x3e, 0x89, 0x18, 0xf6,
0x47, 0x49, 0x73, 0xeb, 0xf1, 0xd0, 0x65, 0x27, 0x63, 0x4b, 0xb3, 0xa9, 0xdf, 0x1f, 0xd2, 0x21,
0xcd, 0x3a, 0x45, 0x26, 0x13, 0x19, 0x25, 0xed, 0xdd, 0xdf, 0x05, 0x50, 0xd6, 0x3d, 0x6a, 0x9f,
0xbe, 0x25, 0x0c, 0xc3, 0x07, 0xa0, 0x38, 0x21, 0x61, 0xe4, 0xd2, 0x40, 0x55, 0x3a, 0x4a, 0xaf,
0xac, 0x83, 0x98, 0xa3, 0xc2, 0x80, 0x86, 0x3e, 0x66, 0xc6, 0x0c, 0x82, 0xcf, 0x41, 0xc9, 0x12,
0x14, 0xd3, 0x75, 0xd4, 0x95, 0x8e, 0xd2, 0xab, 0xea, 0xdd, 0x0b, 0x8e, 0x72, 0x3f, 0x38, 0xca,
0x1f, 0x1f, 0xef, 0xef, 0x4d, 0x39, 0x2a, 0x4a, 0xc9, 0xfd, 0xbd, 0x98, 0xa3, 0xa2, 0x95, 0x84,
0x46, 0x1a, 0x38, 0x70, 0x07, 0x94, 0x19, 0x09, 0x70, 0xc0, 0x04, 0xff, 0x8e, 0x3c, 0x46, 0x9d,
0x72, 0x54, 0x3a, 0x92, 0x45, 0x49, 0x2a, 0xb1, 0x34, 0x36, 0x66, 0x91, 0x03, 0x0f, 0x00, 0x88,
0x18, 0x0e, 0x99, 0x29, 0x1c, 0xab, 0x85, 0x8e, 0xd2, 0xab, 0x6c, 0xb5, 0xb4, 0xe4, 0x39, 0xb4,
0x99, 0x49, 0xed, 0x68, 0xf6, 0x1c, 0xfa, 0x86, 0xb8, 0x53, 0xcc, 0x51, 0x59, 0xb2, 0x44, 0xfd,
0xfc, 0x27, 0x52, 0x8c, 0x2c, 0x85, 0xaf, 0x41, 0x89, 0x04, 0x4e, 0xa2, 0x57, 0xbc, 0x55, 0x6f,
0x3d, 0xd5, 0x2b, 0x92, 0xc0, 0x99, 0xab, 0xcd, 0x12, 0xb8, 0x03, 0x6a, 0x8c, 0x32, 0xec, 0x99,
0xd4, 0xfa, 0x4c, 0x6c, 0x16, 0xa9, 0xa5, 0x8e, 0xd2, 0x5b, 0xd5, 0x1b, 0x31, 0x47, 0x55, 0x09,
0xbc, 0x4f, 0xea, 0xc6, 0x52, 0x06, 0x21, 0xc8, 0x47, 0xee, 0x19, 0x51, 0xcb, 0x1d, 0xa5, 0x97,
0x37, 0x64, 0x0c, 0x5f, 0x80, 0x86, 0x4d, 0xfd, 0x11, 0xb6, 0x99, 0x4b, 0x03, 0xd3, 0x23, 0x13,
0xe2, 0xa9, 0xa0, 0xa3, 0xf4, 0x6a, 0xfa, 0x7a, 0xcc, 0x51, 0x3d, 0xc3, 0xde, 0x08, 0xc8, 0xb8,
0x5e, 0x80, 0x4f, 0x41, 0xdd, 0x0d, 0x1c, 0xf2, 0xc5, 0x1c, 0xe1, 0x21, 0x31, 0xa5, 0x7c, 0x55,
0xd2, 0x9b, 0x31, 0x47, 0x35, 0x09, 0x1d, 0xe0, 0x21, 0x39, 0x74, 0xcf, 0x88, 0xb1, 0x9c, 0x66,
0x2e, 0x42, 0x62, 0xd3, 0xd0, 0x89, 0xd4, 0x9a, 0x24, 0x66, 0x2e, 0x8c, 0xa4, 0x6e, 0x2c, 0x65,
0xf0, 0x19, 0x68, 0x5a, 0x1e, 0xa5, 0xbe, 0x19, 0x9d, 0xe0, 0xd0, 0x31, 0x6d, 0x3a, 0x0e, 0x98,
0x5a, 0x97, 0xd4, 0x7a, 0xcc, 0x51, 0x45, 0x82, 0x87, 0x02, 0x8b, 0x8c, 0x7a, 0x96, 0xec, 0x8a,
0x3e, 0xd8, 0x07, 0x95, 0x01, 0xa5, 0x8c, 0x84, 0xc9, 0x55, 0x1b, 0x92, 0xb6, 0x16, 0x73, 0x04,
0x92, 0xb2, 0xbc, 0xe7, 0x42, 0x0c, 0x6d, 0xd0, 0x74, 0x88, 0xe3, 0xda, 0x98, 0x11, 0x71, 0x96,
0x37, 0xf6, 0x83, 0x48, 0x6d, 0xca, 0x39, 0x7c, 0x92, 0xce, 0x61, 0x63, 0x6f, 0xd6, 0xb0, 0x9b,
0xe0, 0x31, 0x47, 0x2d, 0xe7, 0x5a, 0xed, 0x11, 0xf5, 0x5d, 0x46, 0xfc, 0x11, 0xfb, 0x6a, 0x34,
0xae, 0x63, 0xf0, 0x1d, 0x80, 0x21, 0x19, 0x79, 0xa2, 0x28, 0xbe, 0xc2, 0x00, 0xdb, 0x8c, 0x86,
0x2a, 0x94, 0x97, 0x43, 0x31, 0x47, 0xf7, 0x17, 0xd0, 0x57, 0x12, 0x5c, 0x90, 0x6b, 0xde, 0x00,
0xbb, 0xdf, 0x14, 0x00, 0x77, 0x93, 0x0f, 0x45, 0x9c, 0x6c, 0xe1, 0x74, 0x00, 0x92, 0x55, 0xf2,
0x09, 0xc3, 0x72, 0xe7, 0x2a, 0x5b, 0x1b, 0x5a, 0xb6, 0xef, 0xda, 0xbc, 0x55, 0xaf, 0x0a, 0x6f,
0x97, 0x1c, 0x29, 0x31, 0x47, 0x39, 0xa3, 0x6c, 0xcd, 0x35, 0x3e, 0x81, 0x35, 0x7b, 0xa6, 0x9c,
0x0c, 0xf3, 0xca, 0xad, 0xc3, 0x7c, 0x2f, 0x1d, 0xe6, 0xda, 0x9c, 0x39, 0x1f, 0xe9, 0xe5, 0x52,
0xf7, 0x8f, 0x02, 0x2a, 0xe9, 0x66, 0x8a, 0x51, 0x81, 0x1f, 0x00, 0xb0, 0x43, 0x22, 0xdf, 0x1e,
0xb3, 0xf4, 0xc6, 0xff, 0x3b, 0xe9, 0x6e, 0x7a, 0xd2, 0x02, 0x2b, 0xd9, 0xc3, 0x34, 0x7f, 0xc9,
0xe0, 0x36, 0xc8, 0x4b, 0xfb, 0x2b, 0x9d, 0xd5, 0x7f, 0xdb, 0x2f, 0xc5, 0x1c, 0xc9, 0x36, 0x43,
0xfe, 0xc2, 0xa3, 0x45, 0xd7, 0x92, 0xbe, 0x2a, 0xe9, 0xed, 0x45, 0xfa, 0xcd, 0x17, 0xd7, 0x6b,
0xe2, 0x2f, 0x61, 0xce, 0x5c, 0x70, 0x2b, 0xd1, 0x87, 0x17, 0xd3, 0xb6, 0x72, 0x39, 0x6d, 0x2b,
0xbf, 0xa6, 0x6d, 0xe5, 0xfc, 0xaa, 0x9d, 0xbb, 0xbc, 0x6a, 0xe7, 0xbe, 0x5f, 0xb5, 0x73, 0x1f,
0xeb, 0xe2, 0xdb, 0x52, 0xc7, 0xea, 0xa7, 0xf2, 0x56, 0x41, 0x9a, 0xdd, 0xfe, 0x1b, 0x00, 0x00,
0xff, 0xff, 0x90, 0x60, 0x1b, 0x42, 0xab, 0x05, 0x00, 0x00,
}
func (m *BlockMeta) Marshal() (dAtA []byte, err error) {
@@ -389,13 +379,6 @@ func (m *BlockMeta) MarshalToSizedBuffer(dAtA []byte) (int, error) {
i--
dAtA[i] = 0x78
}
if len(m.DataEncoding) > 0 {
i -= len(m.DataEncoding)
copy(dAtA[i:], m.DataEncoding)
i = encodeVarintV1(dAtA, i, uint64(len(m.DataEncoding)))
i--
dAtA[i] = 0x72
}
if m.TotalRecords != 0 {
i = encodeVarintV1(dAtA, i, uint64(m.TotalRecords))
i--
@@ -406,16 +389,6 @@ func (m *BlockMeta) MarshalToSizedBuffer(dAtA []byte) (int, error) {
i--
dAtA[i] = 0x60
}
{
size := m.Encoding.Size()
i -= size
if _, err := m.Encoding.MarshalTo(dAtA[i:]); err != nil {
return 0, err
}
i = encodeVarintV1(dAtA, i, uint64(size))
}
i--
dAtA[i] = 0x5a
if m.CompactionLevel != 0 {
i = encodeVarintV1(dAtA, i, uint64(m.CompactionLevel))
i--
@@ -614,18 +587,12 @@ func (m *BlockMeta) Size() (n int) {
if m.CompactionLevel != 0 {
n += 1 + sovV1(uint64(m.CompactionLevel))
}
l = m.Encoding.Size()
n += 1 + l + sovV1(uint64(l))
if m.IndexPageSize != 0 {
n += 1 + sovV1(uint64(m.IndexPageSize))
}
if m.TotalRecords != 0 {
n += 1 + sovV1(uint64(m.TotalRecords))
}
l = len(m.DataEncoding)
if l > 0 {
n += 1 + l + sovV1(uint64(l))
}
if m.BloomShardCount != 0 {
n += 1 + sovV1(uint64(m.BloomShardCount))
}
@@ -931,39 +898,6 @@ func (m *BlockMeta) Unmarshal(dAtA []byte) error {
break
}
}
case 11:
if wireType != 2 {
return fmt.Errorf("proto: wrong wireType = %d for field Encoding", wireType)
}
var byteLen int
for shift := uint(0); ; shift += 7 {
if shift >= 64 {
return ErrIntOverflowV1
}
if iNdEx >= l {
return io.ErrUnexpectedEOF
}
b := dAtA[iNdEx]
iNdEx++
byteLen |= int(b&0x7F) << shift
if b < 0x80 {
break
}
}
if byteLen < 0 {
return ErrInvalidLengthV1
}
postIndex := iNdEx + byteLen
if postIndex < 0 {
return ErrInvalidLengthV1
}
if postIndex > l {
return io.ErrUnexpectedEOF
}
if err := m.Encoding.Unmarshal(dAtA[iNdEx:postIndex]); err != nil {
return err
}
iNdEx = postIndex
case 12:
if wireType != 0 {
return fmt.Errorf("proto: wrong wireType = %d for field IndexPageSize", wireType)
@@ -1002,38 +936,6 @@ func (m *BlockMeta) Unmarshal(dAtA []byte) error {
break
}
}
case 14:
if wireType != 2 {
return fmt.Errorf("proto: wrong wireType = %d for field DataEncoding", wireType)
}
var stringLen uint64
for shift := uint(0); ; shift += 7 {
if shift >= 64 {
return ErrIntOverflowV1
}
if iNdEx >= l {
return io.ErrUnexpectedEOF
}
b := dAtA[iNdEx]
iNdEx++
stringLen |= uint64(b&0x7F) << shift
if b < 0x80 {
break
}
}
intStringLen := int(stringLen)
if intStringLen < 0 {
return ErrInvalidLengthV1
}
postIndex := iNdEx + intStringLen
if postIndex < 0 {
return ErrInvalidLengthV1
}
if postIndex > l {
return io.ErrUnexpectedEOF
}
m.DataEncoding = string(dAtA[iNdEx:postIndex])
iNdEx = postIndex
case 15:
if wireType != 0 {
return fmt.Errorf("proto: wrong wireType = %d for field BloomShardCount", wireType)
tempodb/backend/v1/v1.proto
+2 -2
View File
@@ -16,10 +16,10 @@ message BlockMeta {
int64 total_objects = 8[(gogoproto.jsontag) = "totalObjects"];
uint64 size = 9;
uint32 compaction_level = 10[(gogoproto.jsontag) = "compactionLevel"];
bytes encoding = 11[(gogoproto.customtype) = "Encoding", (gogoproto.nullable) = false];
// bytes encoding = 11[(gogoproto.customtype) = "Encoding", (gogoproto.nullable) = false];
uint32 index_page_size = 12[(gogoproto.jsontag) = "indexPageSize"];
uint32 total_records = 13[(gogoproto.jsontag) = "totalRecords"];
string data_encoding = 14[(gogoproto.jsontag) = "dataEncoding"];
// string data_encoding = 14[(gogoproto.jsontag) = "dataEncoding"];
uint32 bloom_shard_count = 15[(gogoproto.jsontag) = "bloomShards"];
uint32 footer_size = 16[(gogoproto.jsontag) = "footerSize"];
bytes dedicated_columns = 17 [(gogoproto.customtype) = "DedicatedColumns", (gogoproto.jsontag) = "dedicatedColumns,omitempty", (gogoproto.nullable) = false];
tempodb/blockselector/compaction_block_selector.go
-1
View File
@@ -146,7 +146,6 @@ func (twbs *timeWindowBlockSelector) BlocksToCompact() ([]*backend.BlockMeta, st
stripe := twbs.entries[i : j+1]
if twbs.entries[i].group == twbs.entries[j].group &&
twbs.entries[i].meta.DataEncoding == twbs.entries[j].meta.DataEncoding &&
twbs.entries[i].meta.Version == twbs.entries[j].meta.Version && // update after parquet: only compact blocks of the same version
twbs.entries[i].meta.DedicatedColumnsHash() == twbs.entries[j].meta.DedicatedColumnsHash() && // update after vParquet3: only compact blocks of the same dedicated columns
len(stripe) <= twbs.MaxInputBlocks &&
tempodb/blockselector/compaction_block_selector_test.go
+16 -32
View File
@@ -613,29 +613,13 @@ func TestTimeWindowBlockSelectorBlocksToCompact(t *testing.T) {
},
},
},
{
name: "don't compact across dataEncodings",
blocklist: []*backend.BlockMeta{
{
BlockID: backend.MustParse("00000000-0000-0000-0000-000000000000"),
EndTime: now,
DataEncoding: "bar",
},
{
BlockID: backend.MustParse("00000000-0000-0000-0000-000000000001"),
EndTime: now,
DataEncoding: "foo",
},
},
expected: nil,
},
{
name: "ensures blocks of different versions are not compacted",
blocklist: []*backend.BlockMeta{
{
BlockID: backend.MustParse("00000000-0000-0000-0000-000000000001"),
EndTime: now,
Version: "v2",
Version: "vParquet5",
},
{
BlockID: backend.MustParse("00000000-0000-0000-0000-000000000002"),
@@ -654,7 +638,7 @@ func TestTimeWindowBlockSelectorBlocksToCompact(t *testing.T) {
{
BlockID: backend.MustParse("00000000-0000-0000-0000-000000000001"),
EndTime: now,
Version: "v2",
Version: "vParquet5",
},
{
BlockID: backend.MustParse("00000000-0000-0000-0000-000000000002"),
@@ -664,7 +648,7 @@ func TestTimeWindowBlockSelectorBlocksToCompact(t *testing.T) {
{
BlockID: backend.MustParse("00000000-0000-0000-0000-000000000003"),
EndTime: now,
Version: "v2",
Version: "vParquet5",
},
{
BlockID: backend.MustParse("00000000-0000-0000-0000-000000000004"),
@@ -673,19 +657,6 @@ func TestTimeWindowBlockSelectorBlocksToCompact(t *testing.T) {
},
},
expected: []*backend.BlockMeta{
{
BlockID: backend.MustParse("00000000-0000-0000-0000-000000000001"),
EndTime: now,
Version: "v2",
},
{
BlockID: backend.MustParse("00000000-0000-0000-0000-000000000003"),
EndTime: now,
Version: "v2",
},
},
expectedHash: fmt.Sprintf("%v-%v-%v-%v", tenantID, 0, now.Unix(), 0),
expectedSecond: []*backend.BlockMeta{
{
BlockID: backend.MustParse("00000000-0000-0000-0000-000000000002"),
EndTime: now,
@@ -697,6 +668,19 @@ func TestTimeWindowBlockSelectorBlocksToCompact(t *testing.T) {
Version: "vParquet3",
},
},
expectedHash: fmt.Sprintf("%v-%v-%v-%v", tenantID, 0, now.Unix(), 0),
expectedSecond: []*backend.BlockMeta{
{
BlockID: backend.MustParse("00000000-0000-0000-0000-000000000001"),
EndTime: now,
Version: "vParquet5",
},
{
BlockID: backend.MustParse("00000000-0000-0000-0000-000000000003"),
EndTime: now,
Version: "vParquet5",
},
},
expectedHash2: fmt.Sprintf("%v-%v-%v-%v", tenantID, 0, now.Unix(), 0),
},
{

Some files were not shown because too many files have changed in this diff Show More