Compare commits

...
2 Commits
Author SHA1 Message Date
3e2a967628 [TraceQL Metrics] Remove all obsolete settings and code for RF3 metrics (#3995) (#4010)
* Remove all obsolete settings and code for RF3 metrics. The only path going forward is RF1-based

* Remove remaining references to rf1_read_path

* changelog

* Remove another reference to obsolete sampling rate

* Update error messages for correctness

(cherry picked from commit 43d9326477)

Co-authored-by: Martin Disibio <martin.disibio@grafana.com>
2024-08-27 08:27:08 -04:00
caebd39103 Ingestion data: usage stats (#3983) (#4009)
* usage stats

Signed-off-by: Joe Elliott <number101010@gmail.com>

* changelog

Signed-off-by: Joe Elliott <number101010@gmail.com>

---------

Signed-off-by: Joe Elliott <number101010@gmail.com>
(cherry picked from commit 64127ea7bd)

Co-authored-by: Joe Elliott <number101010@gmail.com>
2024-08-26 14:40:39 -04:00
27 changed files with 249 additions and 1477 deletions
+4
View File
@@ -1,5 +1,8 @@
## main / unreleased
* [FEATURE] TraceQL support for instrumentation scope [#3967](https://github.com/grafana/tempo/pull/3967) (@ie-pham)
* [ENHANCEMENT] Add bytes and spans received to usage stats [#3983](https://github.com/grafana/tempo/pull/3983) (@joe-elliott)
# v2.6.0-rc.0
* [CHANGE] **BREAKING CHANGE** Remove `autocomplete_filtering_enabled` feature flag [#3729](https://github.com/grafana/tempo/pull/3729) (@mapno)
@@ -16,6 +19,7 @@
* [FEATURE] TraceQL support for event:timeSinceStart [#3908](https://github.com/grafana/tempo/pull/3908) (@ie-pham)
* [FEATURE] Autocomplete support for events and links [#3846](https://github.com/grafana/tempo/pull/3846) (@ie-pham)
* [FEATURE] Flush and query RF1 blocks for TraceQL metric queries [#3628](https://github.com/grafana/tempo/pull/3628) [#3691](https://github.com/grafana/tempo/pull/3691) [#3723](https://github.com/grafana/tempo/pull/3723) (@mapno)
* [FEATURE] Flush and query RF1 blocks for TraceQL metric queries [#3628](https://github.com/grafana/tempo/pull/3628) [#3691](https://github.com/grafana/tempo/pull/3691) [#3723](https://github.com/grafana/tempo/pull/3723) (@mapno) [#3995](https://github.com/grafana/tempo/pull/3995) (@mdisibio)
* [FEATURE] Add new compare() metrics function [#3695](https://github.com/grafana/tempo/pull/3695) (@mdisibio)
* [FEATURE] Add new api `/api/metrics/query` for instant metrics queries [#3859](https://github.com/grafana/tempo/pull/3859) (@mdisibio)
* [FEATURE] Add a `q` parameter to `/api/v2/serach/tags` for tag name filtering [#3822](https://github.com/grafana/tempo/pull/3822) (@joe-elliott)
@@ -654,9 +654,6 @@ query_frontend:
# If set to a non-zero value, it's value will be used to decide if query is within SLO or not.
# Query is within SLO if it returned 200 within duration_slo seconds OR processed throughput_slo bytes/s data.
[throughput_bytes_slo: <float> | default = 0 ]
# If set to true, TraceQL metric queries will use RF1 blocks built and flushed by the metrics-generator.
[rf1_read_path: <bool> | default = false]
```
## Querier
-1
View File
@@ -11,7 +11,6 @@ query_frontend:
query_ingesters_until: 0
metrics:
exemplars: true
rf1_read_path: true
distributor:
receivers:
+6
View File
@@ -34,6 +34,7 @@ import (
"github.com/grafana/tempo/pkg/model"
"github.com/grafana/tempo/pkg/tempopb"
v1 "github.com/grafana/tempo/pkg/tempopb/trace/v1"
"github.com/grafana/tempo/pkg/usagestats"
tempo_util "github.com/grafana/tempo/pkg/util"
"github.com/grafana/tempo/pkg/validation"
@@ -109,6 +110,9 @@ var (
Name: "distributor_metrics_generator_clients",
Help: "The current number of metrics-generator clients.",
})
statBytesReceived = usagestats.NewCounter("distributor_bytes_received")
statSpansReceived = usagestats.NewCounter("distributor_spans_received")
)
// rebatchedTrace is used to more cleanly pass the set of data
@@ -353,6 +357,8 @@ func (d *Distributor) PushTraces(ctx context.Context, traces ptrace.Traces) (*te
metricBytesIngested.WithLabelValues(userID).Add(float64(size))
metricSpansIngested.WithLabelValues(userID).Add(float64(spanCount))
statBytesReceived.Inc(int64(size))
statSpansReceived.Inc(int64(spanCount))
keys, rebatchedTraces, err := requestsByTraceID(batches, userID, spanCount)
if err != nil {
@@ -31,22 +31,6 @@ func NewQueryRange(req *tempopb.QueryRangeRequest, trackDiffs bool) (Combiner, e
}
}
samplingRate := resp.RequestData()
if samplingRate != nil {
fRate := samplingRate.(float64)
if fRate <= 1.0 {
// Set final sampling rate after integer rounding
// Multiply up the sampling rate
for _, series := range partial.Series {
for i, sample := range series.Samples {
sample.Value *= 1.0 / fRate
series.Samples[i] = sample
}
}
}
}
combiner.Combine(partial)
return nil
-1
View File
@@ -90,7 +90,6 @@ func (cfg *Config) RegisterFlagsAndApplyDefaults(string, *flag.FlagSet) {
ConcurrentRequests: defaultConcurrentRequests,
TargetBytesPerRequest: defaultTargetBytesPerRequest,
Interval: 5 * time.Minute,
RF1ReadPath: false,
Exemplars: false, // TODO: Remove?
MaxExemplars: 100,
},
@@ -47,6 +47,7 @@ func TestQueryRangeHandlerSucceeds(t *testing.T) {
}, nil, nil, nil, func(c *Config) {
c.Metrics.Sharder.Interval = time.Hour
})
tenant := "foo"
httpReq := httptest.NewRequest("GET", api.PathMetricsQueryRange, nil)
@@ -66,10 +67,9 @@ func TestQueryRangeHandlerSucceeds(t *testing.T) {
require.Equal(t, 200, httpResp.Code)
// for reasons I don't understand, this query turns into 408 jobs.
expectedResp := &tempopb.QueryRangeResponse{
Metrics: &tempopb.SearchMetrics{
CompletedJobs: 4,
CompletedJobs: 4, // 2 blocks, each with 2 row groups that take 1 job
InspectedTraces: 4,
InspectedBytes: 4,
TotalJobs: 4,
@@ -101,90 +101,3 @@ func TestQueryRangeHandlerSucceeds(t *testing.T) {
require.NoError(t, err)
require.Equal(t, expectedResp, actualResp)
}
func TestQueryRangeHandlerRespectsSamplingRate(t *testing.T) {
resp := &tempopb.QueryRangeResponse{
Metrics: &tempopb.SearchMetrics{
InspectedTraces: 1,
InspectedBytes: 1,
},
Series: []*tempopb.TimeSeries{
{
PromLabels: "foo",
Labels: []v1.KeyValue{
{Key: "foo", Value: &v1.AnyValue{Value: &v1.AnyValue_StringValue{StringValue: "bar"}}},
},
Samples: []tempopb.Sample{
{
TimestampMs: 1200_000,
Value: 2,
},
{
TimestampMs: 1100_000,
Value: 1,
},
},
},
},
}
f := frontendWithSettings(t, &mockRoundTripper{
responseFn: func() proto.Message {
return resp
},
}, nil, nil, nil, func(c *Config) {
c.Metrics.Sharder.Interval = time.Hour
})
tenant := "foo"
httpReq := httptest.NewRequest("GET", api.PathMetricsQueryRange, nil)
httpReq = api.BuildQueryRangeRequest(httpReq, &tempopb.QueryRangeRequest{
Query: "{} | rate() with (sample=.2)",
Start: uint64(1100 * time.Second),
End: uint64(1200 * time.Second),
Step: uint64(100 * time.Second),
})
ctx := user.InjectOrgID(httpReq.Context(), tenant)
httpReq = httpReq.WithContext(ctx)
httpResp := httptest.NewRecorder()
f.MetricsQueryRangeHandler.ServeHTTP(httpResp, httpReq)
require.Equal(t, 200, httpResp.Code)
expectedResp := &tempopb.QueryRangeResponse{
Metrics: &tempopb.SearchMetrics{
CompletedJobs: 1,
InspectedTraces: 1,
InspectedBytes: 1,
TotalJobs: 1,
TotalBlocks: 2,
TotalBlockBytes: 419430400,
},
Series: []*tempopb.TimeSeries{
{
PromLabels: "foo",
Labels: []v1.KeyValue{
{Key: "foo", Value: &v1.AnyValue{Value: &v1.AnyValue_StringValue{StringValue: "bar"}}},
},
Samples: []tempopb.Sample{
{
TimestampMs: 1100_000,
Value: 5,
},
{
TimestampMs: 1200_000,
Value: 10,
},
},
},
},
}
actualResp := &tempopb.QueryRangeResponse{}
err := jsonpb.Unmarshal(httpResp.Body, actualResp)
require.NoError(t, err)
require.Equal(t, expectedResp, actualResp)
}
+14 -208
View File
@@ -27,12 +27,11 @@ import (
)
type queryRangeSharder struct {
next pipeline.AsyncRoundTripper[combiner.PipelineResponse]
reader tempodb.Reader
overrides overrides.Interface
cfg QueryRangeSharderConfig
logger log.Logger
replicationFactor uint8
next pipeline.AsyncRoundTripper[combiner.PipelineResponse]
reader tempodb.Reader
overrides overrides.Interface
cfg QueryRangeSharderConfig
logger log.Logger
}
type QueryRangeSharderConfig struct {
@@ -41,17 +40,12 @@ type QueryRangeSharderConfig struct {
MaxDuration time.Duration `yaml:"max_duration"`
QueryBackendAfter time.Duration `yaml:"query_backend_after,omitempty"`
Interval time.Duration `yaml:"interval,omitempty"`
RF1ReadPath bool `yaml:"rf1_read_path,omitempty"`
Exemplars bool `yaml:"exemplars,omitempty"`
MaxExemplars int `yaml:"max_exemplars,omitempty"`
}
// newAsyncQueryRangeSharder creates a sharding middleware for search
func newAsyncQueryRangeSharder(reader tempodb.Reader, o overrides.Interface, cfg QueryRangeSharderConfig, logger log.Logger) pipeline.AsyncMiddleware[combiner.PipelineResponse] {
var replicationFactor uint8
if cfg.RF1ReadPath {
replicationFactor = 1
}
return pipeline.AsyncMiddlewareFunc[combiner.PipelineResponse](func(next pipeline.AsyncRoundTripper[combiner.PipelineResponse]) pipeline.AsyncRoundTripper[combiner.PipelineResponse] {
return queryRangeSharder{
next: next,
@@ -60,8 +54,6 @@ func newAsyncQueryRangeSharder(reader tempodb.Reader, o overrides.Interface, cfg
cfg: cfg,
logger: logger,
replicationFactor: replicationFactor,
}
})
}
@@ -103,18 +95,10 @@ func (s queryRangeSharder) RoundTrip(pipelineRequest pipeline.Request) (pipeline
var (
allowUnsafe = s.overrides.UnsafeQueryHints(tenantID)
samplingRate = s.samplingRate(expr, allowUnsafe)
targetBytesPerRequest = s.jobSize(expr, samplingRate, allowUnsafe)
interval = s.jobInterval(expr, allowUnsafe)
targetBytesPerRequest = s.jobSize(expr, allowUnsafe)
cutoff = time.Now().Add(-s.cfg.QueryBackendAfter)
)
// if interval is 0 then the backend requests code will loop forever. technically if we are here with a 0 interval it should mean a bad request
// b/c it was specified using a query hint. we're going to assume that and return 400
if interval == 0 {
return pipeline.NewBadRequest(errors.New("invalid interval specified: 0")), nil
}
generatorReq := s.generatorRequest(*req, r, tenantID, cutoff)
reqCh := make(chan pipeline.Request, 2) // buffer of 2 allows us to insert generatorReq and metrics
@@ -122,15 +106,7 @@ func (s queryRangeSharder) RoundTrip(pipelineRequest pipeline.Request) (pipeline
reqCh <- pipeline.NewHTTPRequest(generatorReq)
}
var (
totalJobs, totalBlocks uint32
totalBlockBytes uint64
)
if s.cfg.RF1ReadPath {
totalJobs, totalBlocks, totalBlockBytes = s.backendRequests(ctx, tenantID, r, *req, cutoff, samplingRate, targetBytesPerRequest, interval, reqCh)
} else {
totalJobs, totalBlocks, totalBlockBytes = s.shardedBackendRequests(ctx, tenantID, r, *req, cutoff, samplingRate, targetBytesPerRequest, interval, reqCh, nil)
}
totalJobs, totalBlocks, totalBlockBytes := s.backendRequests(ctx, tenantID, r, *req, cutoff, targetBytesPerRequest, reqCh)
span.SetTag("totalJobs", totalJobs)
span.SetTag("totalBlocks", totalBlocks)
@@ -167,7 +143,7 @@ func (s *queryRangeSharder) blockMetas(start, end int64, tenantID string) []*bac
for _, m := range allMetas {
if m.StartTime.UnixNano() <= end &&
m.EndTime.UnixNano() >= start &&
m.ReplicationFactor == s.replicationFactor {
m.ReplicationFactor == 1 { // We always only query RF1 blocks
metas = append(metas, m)
}
}
@@ -175,141 +151,6 @@ func (s *queryRangeSharder) blockMetas(start, end int64, tenantID string) []*bac
return metas
}
func (s *queryRangeSharder) shardedBackendRequests(ctx context.Context, tenantID string, parent *http.Request, searchReq tempopb.QueryRangeRequest, cutoff time.Time, samplingRate float64, targetBytesPerRequest int, interval time.Duration, reqCh chan pipeline.Request, _ func(error)) (totalJobs, totalBlocks uint32, totalBlockBytes uint64) {
// request without start or end, search only in generator
if searchReq.Start == 0 || searchReq.End == 0 {
close(reqCh)
return
}
// Make a copy and limit to backend time range.
backendReq := searchReq
traceql.TrimToBefore(&backendReq, cutoff)
// If empty window then no need to search backend
if backendReq.Start == backendReq.End {
close(reqCh)
return
}
// Blocks within overall time range. This is just for instrumentation, more precise time
// range is checked for each window.
blocks := s.blockMetas(int64(backendReq.Start), int64(backendReq.End), tenantID)
if len(blocks) == 0 {
// no need to search backend
close(reqCh)
return
}
// count blocks
totalBlocks = uint32(len(blocks))
for _, b := range blocks {
totalBlockBytes += b.Size
}
// count jobs. same loops as below
var (
start = backendReq.Start
end = backendReq.End
timeWindowSize = uint64(interval.Nanoseconds())
)
for start < end {
thisStart := start
thisEnd := start + timeWindowSize
if thisEnd > end {
thisEnd = end
}
blocks := s.blockMetas(int64(thisStart), int64(thisEnd), tenantID)
if len(blocks) == 0 {
start = thisEnd
continue
}
totalBlockSize := uint64(0)
for _, b := range blocks {
totalBlockSize += b.Size
}
shards := uint32(math.Ceil(float64(totalBlockSize) / float64(targetBytesPerRequest)))
for i := uint32(1); i <= shards; i++ {
totalJobs++
}
start = thisEnd
}
go func() {
s.buildShardedBackendRequests(ctx, tenantID, parent, backendReq, samplingRate, targetBytesPerRequest, interval, reqCh)
}()
return
}
func (s *queryRangeSharder) buildShardedBackendRequests(ctx context.Context, tenantID string, parent *http.Request, searchReq tempopb.QueryRangeRequest, samplingRate float64, targetBytesPerRequest int, interval time.Duration, reqCh chan pipeline.Request) {
defer close(reqCh)
var (
start = searchReq.Start
end = searchReq.End
timeWindowSize = uint64(interval.Nanoseconds())
)
for start < end {
thisStart := start
thisEnd := start + timeWindowSize
if thisEnd > end {
thisEnd = end
}
blocks := s.blockMetas(int64(thisStart), int64(thisEnd), tenantID)
if len(blocks) == 0 {
start = thisEnd
continue
}
totalBlockSize := uint64(0)
for _, b := range blocks {
totalBlockSize += b.Size
}
shards := uint32(math.Ceil(float64(totalBlockSize) / float64(targetBytesPerRequest)))
exemplars := max(s.exemplarsPerShard(shards), 1)
for i := uint32(1); i <= shards; i++ {
shardR := searchReq
shardR.Start = thisStart
shardR.End = thisEnd
shardR.ShardID = i
shardR.ShardCount = shards
shardR.Exemplars = exemplars
httpReq := s.toUpstreamRequest(ctx, shardR, parent, tenantID)
pipelineR := pipeline.NewHTTPRequest(httpReq)
if samplingRate != 1.0 {
shardR.ShardID *= uint32(1.0 / samplingRate)
shardR.ShardCount *= uint32(1.0 / samplingRate)
// Set final sampling rate after integer rounding
samplingRate = float64(shards) / float64(shardR.ShardCount)
pipelineR.SetResponseData(samplingRate)
}
select {
case reqCh <- pipelineR:
case <-ctx.Done():
return
}
}
start = thisEnd
}
}
func (s *queryRangeSharder) exemplarsPerShard(total uint32) uint32 {
if !s.cfg.Exemplars {
return 0
@@ -317,7 +158,7 @@ func (s *queryRangeSharder) exemplarsPerShard(total uint32) uint32 {
return uint32(math.Ceil(float64(s.cfg.MaxExemplars)*1.2)) / total
}
func (s *queryRangeSharder) backendRequests(ctx context.Context, tenantID string, parent *http.Request, searchReq tempopb.QueryRangeRequest, cutoff time.Time, _ float64, targetBytesPerRequest int, _ time.Duration, reqCh chan pipeline.Request) (totalJobs, totalBlocks uint32, totalBlockBytes uint64) {
func (s *queryRangeSharder) backendRequests(ctx context.Context, tenantID string, parent *http.Request, searchReq tempopb.QueryRangeRequest, cutoff time.Time, targetBytesPerRequest int, reqCh chan pipeline.Request) (totalJobs, totalBlocks uint32, totalBlockBytes uint64) {
// request without start or end, search only in generator
if searchReq.Start == 0 || searchReq.End == 0 {
close(reqCh)
@@ -406,12 +247,10 @@ func (s *queryRangeSharder) buildBackendRequests(ctx context.Context, tenantID s
}
queryRangeReq := &tempopb.QueryRangeRequest{
Query: searchReq.Query,
Start: start,
End: end,
Step: step,
// ShardID: uint32, // No sharding with RF=1
// ShardCount: uint32, // No sharding with RF=1
Query: searchReq.Query,
Start: start,
End: end,
Step: step,
QueryMode: searchReq.QueryMode,
// New RF1 fields
BlockID: m.BlockID.String(),
@@ -487,17 +326,7 @@ func (s *queryRangeSharder) maxDuration(tenantID string) time.Duration {
return s.cfg.MaxDuration
}
func (s *queryRangeSharder) samplingRate(expr *traceql.RootExpr, allowUnsafe bool) float64 {
samplingRate := 1.0
if v, ok := expr.Hints.GetFloat(traceql.HintSample, allowUnsafe); ok {
if v > 0 && v < 1.0 {
samplingRate = v
}
}
return samplingRate
}
func (s *queryRangeSharder) jobSize(expr *traceql.RootExpr, samplingRate float64, allowUnsafe bool) int {
func (s *queryRangeSharder) jobSize(expr *traceql.RootExpr, allowUnsafe bool) int {
// If we have a query hint then use it
if v, ok := expr.Hints.GetInt(traceql.HintJobSize, allowUnsafe); ok && v > 0 {
return v
@@ -506,32 +335,9 @@ func (s *queryRangeSharder) jobSize(expr *traceql.RootExpr, samplingRate float64
// Else use configured value.
size := s.cfg.TargetBytesPerRequest
// Automatically scale job size when sampling less than 100%
// This improves performance.
if samplingRate < 1.0 {
factor := 1.0 / samplingRate
// Keep it within reason
if factor > 10.0 {
factor = 10.0
}
size = int(float64(size) * factor)
}
return size
}
func (s *queryRangeSharder) jobInterval(expr *traceql.RootExpr, allowUnsafe bool) time.Duration {
// If we have a query hint then use it
if v, ok := expr.Hints.GetDuration(traceql.HintJobInterval, allowUnsafe); ok && v > 0 {
return v
}
// Else use configured value
return s.cfg.Interval
}
func hashForQueryRangeRequest(req *tempopb.QueryRangeRequest) uint64 {
if req.Query == "" {
return 0
+17
View File
@@ -742,6 +742,23 @@ func frontendWithSettings(t require.TestingT, next http.RoundTripper, rdr tempod
TotalRecords: 2,
BlockID: uuid.MustParse("00000000-0000-0000-0000-000000000001"),
},
// These are RF1 metrics blocks
{
StartTime: time.Unix(1100, 0),
EndTime: time.Unix(1200, 0),
Size: defaultTargetBytesPerRequest * 2,
TotalRecords: 2,
BlockID: uuid.MustParse("00000000-0000-0000-0000-000000000002"),
ReplicationFactor: 1,
},
{
StartTime: time.Unix(1100, 0),
EndTime: time.Unix(1200, 0),
Size: defaultTargetBytesPerRequest * 2,
TotalRecords: 2,
BlockID: uuid.MustParse("00000000-0000-0000-0000-000000000003"),
ReplicationFactor: 1,
},
},
}
}
@@ -55,7 +55,7 @@ func (p *Processor) QueryRange(ctx context.Context, req *tempopb.QueryRangeReque
// Compile the raw version of the query for wal blocks
// These aren't cached and we put them all into the same evaluator
// for efficiency.
eval, err := e.CompileMetricsQueryRange(req, false, int(req.Exemplars), timeOverlapCutoff, unsafe)
eval, err := e.CompileMetricsQueryRange(req, int(req.Exemplars), timeOverlapCutoff, unsafe)
if err != nil {
return nil, err
}
@@ -192,7 +192,7 @@ func (p *Processor) queryRangeCompleteBlock(ctx context.Context, b *ingester.Loc
}
// Not in cache or not cacheable, so execute
eval, err := traceql.NewEngine().CompileMetricsQueryRange(&req, false, exemplars, timeOverlapCutoff, unsafe)
eval, err := traceql.NewEngine().CompileMetricsQueryRange(&req, exemplars, timeOverlapCutoff, unsafe)
if err != nil {
return nil, err
}
-2
View File
@@ -394,8 +394,6 @@ func (q *Querier) QueryRangeHandler(w http.ResponseWriter, r *http.Request) {
}
span.SetTag("query", req.Query)
span.SetTag("shard", req.ShardID)
span.SetTag("shardCount", req.ShardCount)
span.SetTag("step", time.Duration(req.Step))
span.SetTag("interval", time.Unix(0, int64(req.End)).Sub(time.Unix(0, int64(req.Start))))
+6 -111
View File
@@ -9,15 +9,12 @@ import (
"github.com/google/uuid"
"github.com/grafana/dskit/ring"
"github.com/grafana/dskit/user"
"github.com/grafana/tempo/pkg/boundedwaitgroup"
"github.com/grafana/tempo/pkg/tempopb"
v1 "github.com/grafana/tempo/pkg/tempopb/common/v1"
"github.com/grafana/tempo/pkg/traceql"
"github.com/grafana/tempo/pkg/util/log"
"github.com/grafana/tempo/tempodb/backend"
"github.com/grafana/tempo/tempodb/encoding/common"
"github.com/opentracing/opentracing-go"
"github.com/uber-go/atomic"
)
func (q *Querier) QueryRange(ctx context.Context, req *tempopb.QueryRangeRequest) (*tempopb.QueryRangeResponse, error) {
@@ -25,19 +22,14 @@ func (q *Querier) QueryRange(ctx context.Context, req *tempopb.QueryRangeRequest
return q.queryRangeRecent(ctx, req)
}
if req.BlockID != "" { // RF1 search
return q.queryBlock(ctx, req)
}
// Backend requests go here
return q.queryBackend(ctx, req)
return q.queryBlock(ctx, req)
}
func (q *Querier) queryRangeRecent(ctx context.Context, req *tempopb.QueryRangeRequest) (*tempopb.QueryRangeResponse, error) {
// // Get results from all generators
replicationSet, err := q.generatorRing.GetReplicationSetForOperation(ring.Read)
if err != nil {
return nil, fmt.Errorf("error finding generators in Querier.SpanMetricsSummary: %w", err)
return nil, fmt.Errorf("error finding generators in Querier.queryRangeRecent: %w", err)
}
lookupResults, err := q.forGivenGenerators(
ctx,
@@ -47,9 +39,9 @@ func (q *Querier) queryRangeRecent(ctx context.Context, req *tempopb.QueryRangeR
},
)
if err != nil {
_ = level.Error(log.Logger).Log("error querying generators in Querier.MetricsQueryRange", "err", err)
_ = level.Error(log.Logger).Log("error querying generators in Querier.queryRangeRecent", "err", err)
return nil, fmt.Errorf("error querying generators in Querier.MetricsQueryRange: %w", err)
return nil, fmt.Errorf("error querying generators in Querier.queryRangeRecent: %w", err)
}
c, err := traceql.QueryRangeCombinerFor(req, traceql.AggregateModeSum, false)
@@ -67,7 +59,7 @@ func (q *Querier) queryRangeRecent(ctx context.Context, req *tempopb.QueryRangeR
func (q *Querier) queryBlock(ctx context.Context, req *tempopb.QueryRangeRequest) (*tempopb.QueryRangeResponse, error) {
tenantID, err := user.ExtractOrgID(ctx)
if err != nil {
return nil, fmt.Errorf("error extracting org id in Querier.BackendSearch: %w", err)
return nil, fmt.Errorf("error extracting org id in Querier.queryBlock: %w", err)
}
blockID, err := uuid.Parse(req.BlockID)
@@ -116,7 +108,7 @@ func (q *Querier) queryBlock(ctx context.Context, req *tempopb.QueryRangeRequest
timeOverlapCutoff = v
}
eval, err := traceql.NewEngine().CompileMetricsQueryRange(req, false, int(req.Exemplars), timeOverlapCutoff, unsafe)
eval, err := traceql.NewEngine().CompileMetricsQueryRange(req, int(req.Exemplars), timeOverlapCutoff, unsafe)
if err != nil {
return nil, err
}
@@ -142,103 +134,6 @@ func (q *Querier) queryBlock(ctx context.Context, req *tempopb.QueryRangeRequest
}, nil
}
func (q *Querier) queryBackend(ctx context.Context, req *tempopb.QueryRangeRequest) (*tempopb.QueryRangeResponse, error) {
ctx, cancel := context.WithCancel(ctx)
defer cancel()
tenantID, err := user.ExtractOrgID(ctx)
if err != nil {
return nil, err
}
// Get blocks that overlap this time range
metas := q.store.BlockMetas(tenantID)
withinTimeRange := metas[:0]
for _, m := range metas {
if m.StartTime.UnixNano() <= int64(req.End) && m.EndTime.UnixNano() > int64(req.Start) {
withinTimeRange = append(withinTimeRange, m)
}
}
if len(withinTimeRange) == 0 {
return nil, nil
}
unsafe := q.limits.UnsafeQueryHints(tenantID)
// Optimization
// If there's only 1 block then dedupe not needed.
dedupe := len(withinTimeRange) > 1
expr, err := traceql.Parse(req.Query)
if err != nil {
return nil, err
}
timeOverlapCutoff := q.cfg.Metrics.TimeOverlapCutoff
if v, ok := expr.Hints.GetFloat(traceql.HintTimeOverlapCutoff, unsafe); ok && v >= 0 && v <= 1.0 {
timeOverlapCutoff = v
}
concurrency := q.cfg.Metrics.ConcurrentBlocks
if v, ok := expr.Hints.GetInt(traceql.HintConcurrentBlocks, unsafe); ok && v > 0 && v < 100 {
concurrency = v
}
eval, err := traceql.NewEngine().CompileMetricsQueryRange(req, dedupe, int(req.Exemplars), timeOverlapCutoff, unsafe)
if err != nil {
return nil, err
}
wg := boundedwaitgroup.New(uint(concurrency))
jobErr := atomic.Error{}
for _, m := range withinTimeRange {
// If a job errored then quit immediately.
if err := jobErr.Load(); err != nil {
return nil, err
}
wg.Add(1)
go func(m *backend.BlockMeta) {
defer wg.Done()
span, ctx := opentracing.StartSpanFromContext(ctx, "querier.queryBackEnd.Block", opentracing.Tags{
"block": m.BlockID.String(),
"blockSize": m.Size,
})
defer span.Finish()
f := traceql.NewSpansetFetcherWrapper(func(ctx context.Context, req traceql.FetchSpansRequest) (traceql.FetchSpansResponse, error) {
return q.store.Fetch(ctx, m, req, common.DefaultSearchOptions())
})
// TODO handle error
err := eval.Do(ctx, f, uint64(m.StartTime.UnixNano()), uint64(m.EndTime.UnixNano()))
if err != nil {
jobErr.Store(err)
}
}(m)
}
wg.Wait()
if err := jobErr.Load(); err != nil {
return nil, err
}
res := eval.Results()
inspectedBytes, spansTotal, _ := eval.Metrics()
return &tempopb.QueryRangeResponse{
Series: queryRangeTraceQLToProto(res, req),
Metrics: &tempopb.SearchMetrics{
InspectedBytes: inspectedBytes,
InspectedSpans: spansTotal,
},
}, nil
}
func queryRangeTraceQLToProto(set traceql.SeriesSet, req *tempopb.QueryRangeRequest) []*tempopb.TimeSeries {
resp := make([]*tempopb.TimeSeries, 0, len(set))
-13
View File
@@ -37,8 +37,6 @@ const (
urlParamEnd = "end"
urlParamSpansPerSpanSet = "spss"
urlParamStep = "step"
urlParamShard = "shard"
urlParamShardCount = "shardCount"
urlParamSince = "since"
urlParamExemplars = "exemplars"
@@ -382,15 +380,6 @@ func ParseQueryRangeRequest(r *http.Request) (*tempopb.QueryRangeRequest, error)
}
req.Step = uint64(step.Nanoseconds())
shardCount, _ := extractQueryParam(vals, urlParamShardCount)
if shardCount, err := strconv.Atoi(shardCount); err == nil {
req.ShardCount = uint32(shardCount)
}
shard, _ := extractQueryParam(vals, urlParamShard)
if shard, err := strconv.Atoi(shard); err == nil {
req.ShardID = uint32(shard)
}
// New RF1 params
blockID, _ := extractQueryParam(vals, urlParamBlockID)
if blockID, err := uuid.Parse(blockID); err == nil {
@@ -475,8 +464,6 @@ func BuildQueryRangeRequest(req *http.Request, searchReq *tempopb.QueryRangeRequ
qb.addParam(urlParamStart, strconv.FormatUint(searchReq.Start, 10))
qb.addParam(urlParamEnd, strconv.FormatUint(searchReq.End, 10))
qb.addParam(urlParamStep, time.Duration(searchReq.Step).String())
qb.addParam(urlParamShard, strconv.FormatUint(uint64(searchReq.ShardID), 10))
qb.addParam(urlParamShardCount, strconv.FormatUint(uint64(searchReq.ShardCount), 10))
qb.addParam(QueryModeKey, searchReq.QueryMode)
// New RF1 params
qb.addParam(urlParamBlockID, searchReq.BlockID)
+5 -7
View File
@@ -710,13 +710,11 @@ func TestQueryRangeRoundtrip(t *testing.T) {
{
name: "not empty!",
req: &tempopb.QueryRangeRequest{
Query: "{ foo = `bar` }",
Start: uint64(24 * time.Hour),
End: uint64(25 * time.Hour),
Step: uint64(30 * time.Second),
ShardID: 1,
ShardCount: 2,
QueryMode: "foo",
Query: "{ foo = `bar` }",
Start: uint64(24 * time.Hour),
End: uint64(25 * time.Hour),
Step: uint64(30 * time.Second),
QueryMode: "foo",
},
},
}
+160 -230
View File
@@ -2925,13 +2925,13 @@ func (m *InstantSeries) GetPromLabels() string {
}
type QueryRangeRequest struct {
Query string `protobuf:"bytes,1,opt,name=query,proto3" json:"query,omitempty"`
Start uint64 `protobuf:"varint,2,opt,name=start,proto3" json:"start,omitempty"`
End uint64 `protobuf:"varint,3,opt,name=end,proto3" json:"end,omitempty"`
Step uint64 `protobuf:"varint,4,opt,name=step,proto3" json:"step,omitempty"`
ShardID uint32 `protobuf:"varint,5,opt,name=shardID,proto3" json:"shardID,omitempty"`
ShardCount uint32 `protobuf:"varint,6,opt,name=shardCount,proto3" json:"shardCount,omitempty"`
QueryMode string `protobuf:"bytes,7,opt,name=queryMode,proto3" json:"queryMode,omitempty"`
Query string `protobuf:"bytes,1,opt,name=query,proto3" json:"query,omitempty"`
Start uint64 `protobuf:"varint,2,opt,name=start,proto3" json:"start,omitempty"`
End uint64 `protobuf:"varint,3,opt,name=end,proto3" json:"end,omitempty"`
Step uint64 `protobuf:"varint,4,opt,name=step,proto3" json:"step,omitempty"`
//uint32 shardID = 5; // removed
//uint32 shardCount = 6; // removed
QueryMode string `protobuf:"bytes,7,opt,name=queryMode,proto3" json:"queryMode,omitempty"`
// New RF1 fields
BlockID string `protobuf:"bytes,8,opt,name=blockID,proto3" json:"blockID,omitempty"`
StartPage uint32 `protobuf:"varint,9,opt,name=startPage,proto3" json:"startPage,omitempty"`
@@ -3006,20 +3006,6 @@ func (m *QueryRangeRequest) GetStep() uint64 {
return 0
}
func (m *QueryRangeRequest) GetShardID() uint32 {
if m != nil {
return m.ShardID
}
return 0
}
func (m *QueryRangeRequest) GetShardCount() uint32 {
if m != nil {
return m.ShardCount
}
return 0
}
func (m *QueryRangeRequest) GetQueryMode() string {
if m != nil {
return m.QueryMode
@@ -3389,104 +3375,104 @@ func init() {
func init() { proto.RegisterFile("pkg/tempopb/tempo.proto", fileDescriptor_f22805646f4f62b6) }
var fileDescriptor_f22805646f4f62b6 = []byte{
// 2756 bytes of a gzipped FileDescriptorProto
// 2735 bytes of a gzipped FileDescriptorProto
0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xec, 0x3a, 0xcd, 0x6f, 0x1b, 0xc7,
0xf5, 0x5a, 0xf1, 0x43, 0xe4, 0x23, 0x29, 0x51, 0x63, 0x45, 0xa1, 0x69, 0x47, 0xd6, 0x6f, 0x63,
0xe4, 0xa7, 0xe6, 0x83, 0x92, 0x19, 0x1b, 0x89, 0x93, 0x36, 0x85, 0x65, 0xb1, 0x8e, 0x12, 0x49,
0x56, 0x86, 0x8a, 0x12, 0x14, 0x01, 0x84, 0x15, 0x39, 0xa6, 0x17, 0x22, 0x77, 0x99, 0xdd, 0xa1,
0x6a, 0xf6, 0x50, 0xa0, 0x05, 0x5a, 0xa0, 0x40, 0x0f, 0x3d, 0xb4, 0x87, 0x1e, 0x7b, 0x2a, 0x7a,
0xee, 0x9f, 0x50, 0xa0, 0x08, 0x50, 0x34, 0x08, 0xd0, 0x4b, 0xd0, 0x43, 0x50, 0x24, 0x87, 0xa2,
0xff, 0x45, 0x31, 0x6f, 0x66, 0x76, 0x67, 0xc9, 0x95, 0x14, 0x37, 0x0e, 0x9a, 0x43, 0x4e, 0x9a,
0xf7, 0xe6, 0xed, 0x9b, 0x37, 0xef, 0xfb, 0x0d, 0x05, 0x4f, 0x0f, 0x4f, 0x7a, 0xeb, 0x9c, 0x0d,
0x86, 0xfe, 0xf0, 0x58, 0xfe, 0x6d, 0x0c, 0x03, 0x9f, 0xfb, 0x64, 0x4e, 0x21, 0xeb, 0xcb, 0x1d,
0x7f, 0x30, 0xf0, 0xbd, 0xf5, 0xd3, 0x1b, 0xeb, 0x72, 0x25, 0x09, 0xea, 0x2f, 0xf5, 0x5c, 0xfe,
0x70, 0x74, 0xdc, 0xe8, 0xf8, 0x83, 0xf5, 0x9e, 0xdf, 0xf3, 0xd7, 0x11, 0x7d, 0x3c, 0x7a, 0x80,
0x10, 0x02, 0xb8, 0x52, 0xe4, 0x4b, 0x3c, 0x70, 0x3a, 0x4c, 0x70, 0xc1, 0x85, 0xc4, 0xda, 0xbf,
0xb0, 0xa0, 0x7a, 0x20, 0xe0, 0xcd, 0xf1, 0xf6, 0x16, 0x65, 0x1f, 0x8e, 0x58, 0xc8, 0x49, 0x0d,
0xe6, 0x90, 0x66, 0x7b, 0xab, 0x66, 0xad, 0x5a, 0x6b, 0x65, 0xaa, 0x41, 0xb2, 0x02, 0x70, 0xdc,
0xf7, 0x3b, 0x27, 0x6d, 0xee, 0x04, 0xbc, 0x36, 0xbb, 0x6a, 0xad, 0x15, 0xa9, 0x81, 0x21, 0x75,
0x28, 0x20, 0xd4, 0xf2, 0xba, 0xb5, 0x0c, 0xee, 0x46, 0x30, 0xb9, 0x0a, 0xc5, 0x0f, 0x47, 0x2c,
0x18, 0xef, 0xfa, 0x5d, 0x56, 0xcb, 0xe1, 0x66, 0x8c, 0xb0, 0x3d, 0x58, 0x34, 0xe4, 0x08, 0x87,
0xbe, 0x17, 0x32, 0x72, 0x1d, 0x72, 0x78, 0x32, 0x8a, 0x51, 0x6a, 0xce, 0x37, 0x94, 0x4e, 0x1a,
0x48, 0x4a, 0xe5, 0x26, 0x79, 0x19, 0xe6, 0x06, 0x8c, 0x07, 0x6e, 0x27, 0x44, 0x89, 0x4a, 0xcd,
0xcb, 0x49, 0x3a, 0xc1, 0x72, 0x57, 0x12, 0x50, 0x4d, 0x69, 0x13, 0xe3, 0xde, 0x6a, 0xd3, 0xfe,
0x78, 0x16, 0x2a, 0x6d, 0xe6, 0x04, 0x9d, 0x87, 0x5a, 0x13, 0xaf, 0x41, 0xf6, 0xc0, 0xe9, 0x85,
0x35, 0x6b, 0x35, 0xb3, 0x56, 0x6a, 0xae, 0x46, 0x7c, 0x13, 0x54, 0x0d, 0x41, 0xd2, 0xf2, 0x78,
0x30, 0xde, 0xcc, 0x7e, 0xf4, 0xd9, 0xb5, 0x19, 0x8a, 0xdf, 0x90, 0xeb, 0x50, 0xd9, 0x75, 0xbd,
0xad, 0x51, 0xe0, 0x70, 0xd7, 0xf7, 0x76, 0xa5, 0x70, 0x15, 0x9a, 0x44, 0x22, 0x95, 0xf3, 0xc8,
0xa0, 0xca, 0x28, 0x2a, 0x13, 0x49, 0x96, 0x20, 0xb7, 0xe3, 0x0e, 0x5c, 0x5e, 0xcb, 0xe2, 0xae,
0x04, 0x04, 0x36, 0x44, 0x43, 0xe4, 0x24, 0x16, 0x01, 0x52, 0x85, 0x0c, 0xf3, 0xba, 0xb5, 0x3c,
0xe2, 0xc4, 0x52, 0xd0, 0xbd, 0x23, 0x14, 0x5d, 0x2b, 0xa0, 0xd6, 0x25, 0x40, 0xd6, 0x60, 0xa1,
0x3d, 0x74, 0xbc, 0x70, 0x9f, 0x05, 0xe2, 0x6f, 0x9b, 0xf1, 0x5a, 0x11, 0xbf, 0x99, 0x44, 0xd7,
0x5f, 0x81, 0x62, 0x74, 0x45, 0xc1, 0xfe, 0x84, 0x8d, 0xd1, 0x22, 0x45, 0x2a, 0x96, 0x82, 0xfd,
0xa9, 0xd3, 0x1f, 0x31, 0xe5, 0x0f, 0x12, 0x78, 0x6d, 0xf6, 0x55, 0xcb, 0xfe, 0x4b, 0x06, 0x88,
0x54, 0xd5, 0xa6, 0xf0, 0x02, 0xad, 0xd5, 0x9b, 0x50, 0x0c, 0xb5, 0x02, 0x95, 0x69, 0x97, 0xd3,
0x55, 0x4b, 0x63, 0x42, 0xe1, 0x95, 0xe8, 0x4b, 0xdb, 0x5b, 0xea, 0x20, 0x0d, 0x0a, 0xcf, 0xc2,
0xab, 0xef, 0x3b, 0x3d, 0xa6, 0xf4, 0x17, 0x23, 0x84, 0x86, 0x87, 0x4e, 0x8f, 0x85, 0x07, 0xbe,
0x64, 0xad, 0x74, 0x98, 0x44, 0x0a, 0xcf, 0x65, 0x5e, 0xc7, 0xef, 0xba, 0x5e, 0x4f, 0x39, 0x67,
0x04, 0x0b, 0x0e, 0xae, 0xd7, 0x65, 0x8f, 0x04, 0xbb, 0xb6, 0xfb, 0x63, 0xa6, 0x74, 0x9b, 0x44,
0x12, 0x1b, 0xca, 0xdc, 0xe7, 0x4e, 0x9f, 0xb2, 0x8e, 0x1f, 0x74, 0xc3, 0xda, 0x1c, 0x12, 0x25,
0x70, 0x82, 0xa6, 0xeb, 0x70, 0xa7, 0xa5, 0x4f, 0x92, 0x06, 0x49, 0xe0, 0xc4, 0x3d, 0x4f, 0x59,
0x10, 0xba, 0xbe, 0x87, 0xf6, 0x28, 0x52, 0x0d, 0x12, 0x02, 0xd9, 0x50, 0x1c, 0x0f, 0xab, 0xd6,
0x5a, 0x96, 0xe2, 0x5a, 0x44, 0xe4, 0x03, 0xdf, 0xe7, 0x2c, 0x40, 0xc1, 0x4a, 0x78, 0xa6, 0x81,
0x21, 0x5b, 0x50, 0xed, 0xb2, 0xae, 0xdb, 0x71, 0x38, 0xeb, 0xde, 0xf5, 0xfb, 0xa3, 0x81, 0x17,
0xd6, 0xca, 0xe8, 0xcd, 0xb5, 0x48, 0xe5, 0x5b, 0x49, 0x02, 0x3a, 0xf5, 0x85, 0xfd, 0x67, 0x0b,
0x16, 0x26, 0xa8, 0xc8, 0x4d, 0xc8, 0x85, 0x1d, 0x7f, 0x28, 0x35, 0x3e, 0xdf, 0x5c, 0x39, 0x8b,
0x5d, 0xa3, 0x2d, 0xa8, 0xa8, 0x24, 0x16, 0x77, 0xf0, 0x9c, 0x81, 0xf6, 0x15, 0x5c, 0x93, 0x1b,
0x90, 0xe5, 0xe3, 0xa1, 0x8c, 0xf2, 0xf9, 0xe6, 0x33, 0x67, 0x32, 0x3a, 0x18, 0x0f, 0x19, 0x45,
0x52, 0xfb, 0x1a, 0xe4, 0x90, 0x2d, 0x29, 0x40, 0xb6, 0xbd, 0x7f, 0x67, 0xaf, 0x3a, 0x43, 0xca,
0x50, 0xa0, 0xad, 0xf6, 0xfd, 0x77, 0xe9, 0xdd, 0x56, 0xd5, 0xb2, 0x09, 0x64, 0x05, 0x39, 0x01,
0xc8, 0xb7, 0x0f, 0xe8, 0xf6, 0xde, 0xbd, 0xea, 0x8c, 0xfd, 0x08, 0xe6, 0xb5, 0x77, 0xa9, 0x04,
0x73, 0x13, 0xf2, 0x98, 0x43, 0x74, 0x84, 0x5f, 0x4d, 0x66, 0x0e, 0x49, 0xbd, 0xcb, 0xb8, 0x23,
0x2c, 0x44, 0x15, 0x2d, 0xd9, 0x98, 0x4c, 0x38, 0x93, 0xde, 0x3b, 0x95, 0x6d, 0xfe, 0x9e, 0x81,
0x4b, 0x29, 0x1c, 0x27, 0x33, 0x6d, 0x31, 0xce, 0xb4, 0x6b, 0xb0, 0x10, 0xf8, 0x3e, 0x6f, 0xb3,
0xe0, 0xd4, 0xed, 0xb0, 0xbd, 0x58, 0x65, 0x93, 0x68, 0xe1, 0x9d, 0x02, 0x85, 0xec, 0x91, 0x4e,
0x26, 0xde, 0x24, 0x92, 0xbc, 0x08, 0x8b, 0x18, 0x12, 0x07, 0xee, 0x80, 0xbd, 0xeb, 0xb9, 0x8f,
0xf6, 0x1c, 0xcf, 0xc7, 0x48, 0xc8, 0xd2, 0xe9, 0x0d, 0xe1, 0x55, 0xdd, 0x38, 0x25, 0xc9, 0xf4,
0x62, 0x60, 0xc8, 0xf3, 0x30, 0x17, 0xaa, 0x9c, 0x91, 0x47, 0x0d, 0x54, 0x63, 0x0d, 0x48, 0x3c,
0xd5, 0x04, 0xe4, 0x45, 0x28, 0xa8, 0xa5, 0x88, 0x89, 0x4c, 0x2a, 0x71, 0x44, 0x41, 0x28, 0x94,
0x43, 0x79, 0xb9, 0x36, 0x77, 0x78, 0x58, 0x2b, 0xe0, 0x17, 0x8d, 0xf3, 0xec, 0xd2, 0x68, 0x1b,
0x1f, 0x60, 0x92, 0xa2, 0x09, 0x1e, 0xf5, 0x43, 0x58, 0x9c, 0x22, 0x49, 0xc9, 0x63, 0x2f, 0x98,
0x79, 0xac, 0xd4, 0x7c, 0xca, 0x30, 0x6a, 0xfc, 0xb1, 0x99, 0xde, 0x76, 0xa0, 0x6c, 0x6e, 0x61,
0x1e, 0x1a, 0x3a, 0xde, 0x5d, 0x7f, 0xe4, 0x71, 0x64, 0x2c, 0xf2, 0x90, 0x46, 0x08, 0x9d, 0xb2,
0x20, 0xf0, 0x03, 0xb9, 0x2d, 0x8b, 0x81, 0x81, 0xb1, 0x7f, 0x6e, 0xc1, 0x9c, 0xd2, 0x07, 0x79,
0x16, 0x72, 0xe2, 0x43, 0xed, 0x96, 0x95, 0x84, 0xc2, 0xa8, 0xdc, 0x13, 0xce, 0x33, 0x70, 0x78,
0xe7, 0x21, 0xeb, 0x2a, 0x6e, 0x1a, 0x24, 0xaf, 0x03, 0x38, 0x9c, 0x07, 0xee, 0xf1, 0x88, 0x33,
0x51, 0x51, 0x04, 0x8f, 0x2b, 0x11, 0x0f, 0xd5, 0x45, 0x9c, 0xde, 0x68, 0xbc, 0xcd, 0xc6, 0x87,
0xe2, 0x36, 0xd4, 0x20, 0x17, 0xb1, 0x9e, 0x15, 0xc7, 0x90, 0x65, 0xc8, 0x8b, 0x83, 0x22, 0xdf,
0x54, 0x50, 0x6a, 0x08, 0xa7, 0xba, 0x57, 0xe6, 0x2c, 0xf7, 0xba, 0x0e, 0x15, 0xed, 0x4c, 0x02,
0x0e, 0x95, 0x23, 0x26, 0x91, 0x13, 0xb7, 0xc8, 0x3d, 0xde, 0x2d, 0x7e, 0x17, 0xd5, 0x72, 0x15,
0x8c, 0x22, 0xa2, 0x5c, 0x2f, 0x1c, 0xb2, 0x0e, 0x67, 0xdd, 0x03, 0x1d, 0xf4, 0x58, 0xef, 0x26,
0xd0, 0xe4, 0x39, 0x98, 0x8f, 0x50, 0x9b, 0x63, 0x71, 0xf8, 0x2c, 0xca, 0x37, 0x81, 0x25, 0xab,
0x50, 0xc2, 0xec, 0x8e, 0xc5, 0x4d, 0x57, 0x6e, 0x13, 0x25, 0x2e, 0xda, 0xf1, 0x07, 0xc3, 0x3e,
0xe3, 0xac, 0xfb, 0x96, 0x7f, 0x1c, 0xea, 0xda, 0x93, 0x40, 0x0a, 0xbf, 0xc1, 0x8f, 0x90, 0x42,
0x06, 0x5b, 0x8c, 0x10, 0x72, 0xc7, 0x2c, 0xa5, 0x38, 0x79, 0x14, 0x67, 0x12, 0x9d, 0x90, 0x1b,
0x6b, 0x38, 0xd6, 0x20, 0x53, 0x6e, 0xc4, 0xda, 0x3d, 0x11, 0x0f, 0x42, 0x35, 0xa2, 0xaa, 0xeb,
0xa2, 0xbc, 0xa4, 0xd3, 0xb9, 0x34, 0xb6, 0x4a, 0xd7, 0x4b, 0x90, 0xc3, 0x1e, 0x4d, 0xd7, 0x76,
0x04, 0xe2, 0xc6, 0x23, 0x93, 0xd2, 0x78, 0x64, 0xa3, 0xc6, 0xc3, 0xfe, 0x38, 0x03, 0xcb, 0xf1,
0x49, 0x89, 0x1e, 0xe0, 0xd5, 0xe9, 0x1e, 0xa0, 0x3e, 0x91, 0x45, 0x0d, 0xe9, 0xbe, 0xed, 0x03,
0xbe, 0x19, 0x7d, 0xc0, 0xa7, 0x19, 0xb8, 0x12, 0x19, 0x07, 0x83, 0x2e, 0x69, 0xd5, 0xef, 0x4d,
0x5b, 0xf5, 0xda, 0xb4, 0x55, 0xe5, 0x87, 0xdf, 0x9a, 0xf6, 0x1b, 0x65, 0xda, 0x0d, 0xdd, 0xaa,
0xcb, 0xb0, 0x53, 0x0d, 0x52, 0x1d, 0x0a, 0xdc, 0xe9, 0x89, 0x0e, 0x42, 0xd6, 0xa2, 0x22, 0x8d,
0x60, 0xfb, 0x2d, 0x58, 0x8a, 0xbf, 0x38, 0x6c, 0x46, 0xdf, 0x34, 0x21, 0x8f, 0xc9, 0x43, 0x57,
0xaf, 0xb4, 0xb8, 0x3e, 0x6c, 0xca, 0xae, 0x50, 0x51, 0xda, 0xaf, 0x9b, 0x29, 0x49, 0x6d, 0x46,
0x85, 0xc6, 0x32, 0x0a, 0x0d, 0x81, 0x2c, 0x17, 0x13, 0xd9, 0x2c, 0x0a, 0x83, 0x6b, 0x7b, 0x68,
0x64, 0x99, 0x84, 0x6f, 0x61, 0x7f, 0x25, 0xc5, 0x8d, 0xfa, 0x2b, 0x09, 0x5e, 0x94, 0xd8, 0xb2,
0x29, 0x89, 0x2d, 0x17, 0x27, 0xb6, 0x57, 0xe0, 0xe9, 0xa9, 0x13, 0xd5, 0xed, 0x45, 0x32, 0xd7,
0x48, 0xa5, 0xb2, 0x18, 0x61, 0xdf, 0x84, 0x82, 0xfe, 0x04, 0xaf, 0x32, 0x8e, 0x12, 0x2e, 0xae,
0xd3, 0x67, 0x29, 0x7b, 0x07, 0x2e, 0x4f, 0x1c, 0x67, 0xa8, 0x7b, 0x7d, 0xf2, 0xc0, 0x52, 0x73,
0x31, 0x6e, 0x97, 0xd4, 0x8e, 0x29, 0xc3, 0x1e, 0xe4, 0xb0, 0xd0, 0x91, 0x16, 0x54, 0x02, 0x16,
0xfa, 0xa3, 0xa0, 0xc3, 0xda, 0x46, 0xb7, 0x11, 0x47, 0xac, 0x7c, 0x29, 0x38, 0xbd, 0xd1, 0xa0,
0x26, 0x19, 0x4d, 0x7e, 0x65, 0xef, 0x41, 0x79, 0x7f, 0x14, 0xc6, 0x4d, 0xf5, 0x1b, 0x50, 0xc1,
0xb6, 0x26, 0xdc, 0x1c, 0x1f, 0xa8, 0xe9, 0x3d, 0xb3, 0x36, 0x6f, 0x38, 0xa3, 0xa0, 0x6e, 0x09,
0x0a, 0xca, 0x9c, 0xd0, 0xf7, 0x68, 0x92, 0xdc, 0xfe, 0xbd, 0x05, 0x55, 0x41, 0x82, 0x45, 0x4d,
0x5b, 0xf2, 0xa5, 0xa8, 0x53, 0x17, 0x96, 0x2f, 0x6f, 0x3e, 0x25, 0x26, 0xed, 0x7f, 0x7c, 0x76,
0xad, 0xb2, 0x1f, 0x30, 0xa7, 0xdf, 0xf7, 0x3b, 0x92, 0x5a, 0xb7, 0xe8, 0xff, 0x0f, 0x19, 0xb7,
0x2b, 0x5b, 0x9f, 0x33, 0x69, 0x05, 0x05, 0xb9, 0x05, 0x20, 0xf3, 0xcf, 0x96, 0xc3, 0x9d, 0x5a,
0xf6, 0x3c, 0x7a, 0x83, 0xd0, 0xde, 0x95, 0x22, 0x4a, 0x7d, 0x28, 0x11, 0x6f, 0xc3, 0xdc, 0x31,
0x36, 0x60, 0x5f, 0x5a, 0x91, 0x9a, 0xde, 0xbe, 0x0e, 0xa0, 0x5e, 0x23, 0x44, 0x1d, 0x5f, 0x4e,
0x4c, 0x25, 0x65, 0x7d, 0x29, 0xfb, 0x0d, 0x28, 0xee, 0xb8, 0xde, 0x49, 0xbb, 0xef, 0x76, 0xc4,
0xd0, 0x94, 0xeb, 0xbb, 0xde, 0x89, 0x3e, 0xeb, 0xca, 0xf4, 0x59, 0xe2, 0x8c, 0x86, 0xf8, 0x80,
0xea, 0xa1, 0x7f, 0x42, 0x81, 0x22, 0x40, 0xd1, 0x20, 0x40, 0x2f, 0x41, 0x0f, 0x41, 0x91, 0x1c,
0xfa, 0x6f, 0x14, 0xf3, 0x66, 0x66, 0x77, 0x96, 0x5c, 0x49, 0x71, 0xe3, 0xa0, 0x39, 0xe4, 0xa4,
0x7d, 0x6f, 0xde, 0xbc, 0x79, 0xf3, 0xbe, 0xdf, 0x50, 0xf0, 0xf4, 0xf0, 0xa4, 0xb7, 0xce, 0xd9,
0x60, 0xe8, 0x0f, 0x8f, 0xe5, 0xdf, 0xc6, 0x30, 0xf0, 0xb9, 0x4f, 0xe6, 0x14, 0xb2, 0xbe, 0xdc,
0xf1, 0x07, 0x03, 0xdf, 0x5b, 0x3f, 0xbd, 0xb1, 0x2e, 0xbf, 0x24, 0x41, 0xfd, 0xa5, 0x9e, 0xcb,
0x1f, 0x8e, 0x8e, 0x1b, 0x1d, 0x7f, 0xb0, 0xde, 0xf3, 0x7b, 0xfe, 0x3a, 0xa2, 0x8f, 0x47, 0x0f,
0x10, 0x42, 0x00, 0xbf, 0x14, 0xf9, 0x12, 0x0f, 0x9c, 0x0e, 0x13, 0x5c, 0xf0, 0x43, 0x62, 0xed,
0x5f, 0x58, 0x50, 0x3d, 0x10, 0xf0, 0xe6, 0x78, 0x7b, 0x8b, 0xb2, 0x0f, 0x47, 0x2c, 0xe4, 0xa4,
0x06, 0x73, 0x48, 0xb3, 0xbd, 0x55, 0xb3, 0x56, 0xad, 0xb5, 0x32, 0xd5, 0x20, 0x59, 0x01, 0x38,
0xee, 0xfb, 0x9d, 0x93, 0x36, 0x77, 0x02, 0x5e, 0x9b, 0x5d, 0xb5, 0xd6, 0x8a, 0xd4, 0xc0, 0x90,
0x3a, 0x14, 0x10, 0x6a, 0x79, 0xdd, 0x5a, 0x06, 0x57, 0x23, 0x98, 0x5c, 0x85, 0xe2, 0x87, 0x23,
0x16, 0x8c, 0x77, 0xfd, 0x2e, 0xab, 0xe5, 0x70, 0x31, 0x46, 0xd8, 0x1e, 0x2c, 0x1a, 0x72, 0x84,
0x43, 0xdf, 0x0b, 0x19, 0xb9, 0x0e, 0x39, 0x3c, 0x19, 0xc5, 0x28, 0x35, 0xe7, 0x1b, 0x4a, 0x27,
0x0d, 0x24, 0xa5, 0x72, 0x91, 0xbc, 0x0c, 0x73, 0x03, 0xc6, 0x03, 0xb7, 0x13, 0xa2, 0x44, 0xa5,
0xe6, 0xe5, 0x24, 0x9d, 0x60, 0xb9, 0x2b, 0x09, 0xa8, 0xa6, 0xb4, 0x89, 0x71, 0x6f, 0xb5, 0x68,
0x7f, 0x3c, 0x0b, 0x95, 0x36, 0x73, 0x82, 0xce, 0x43, 0xad, 0x89, 0xd7, 0x20, 0x7b, 0xe0, 0xf4,
0xc2, 0x9a, 0xb5, 0x9a, 0x59, 0x2b, 0x35, 0x57, 0x23, 0xbe, 0x09, 0xaa, 0x86, 0x20, 0x69, 0x79,
0x3c, 0x18, 0x6f, 0x66, 0x3f, 0xfa, 0xec, 0xda, 0x0c, 0xc5, 0x3d, 0xe4, 0x3a, 0x54, 0x76, 0x5d,
0x6f, 0x6b, 0x14, 0x38, 0xdc, 0xf5, 0xbd, 0x5d, 0x29, 0x5c, 0x85, 0x26, 0x91, 0x48, 0xe5, 0x3c,
0x32, 0xa8, 0x32, 0x8a, 0xca, 0x44, 0x92, 0x25, 0xc8, 0xed, 0xb8, 0x03, 0x97, 0xd7, 0xb2, 0xb8,
0x2a, 0x01, 0x81, 0x0d, 0xd1, 0x10, 0x39, 0x89, 0x45, 0x80, 0x54, 0x21, 0xc3, 0xbc, 0x6e, 0x2d,
0x8f, 0x38, 0xf1, 0x29, 0xe8, 0xde, 0x11, 0x8a, 0xae, 0x15, 0x50, 0xeb, 0x12, 0x20, 0x6b, 0xb0,
0xd0, 0x1e, 0x3a, 0x5e, 0xb8, 0xcf, 0x02, 0xf1, 0xb7, 0xcd, 0x78, 0xad, 0x88, 0x7b, 0x26, 0xd1,
0xf5, 0x57, 0xa0, 0x18, 0x5d, 0x51, 0xb0, 0x3f, 0x61, 0x63, 0xb4, 0x48, 0x91, 0x8a, 0x4f, 0xc1,
0xfe, 0xd4, 0xe9, 0x8f, 0x98, 0xf2, 0x07, 0x09, 0xbc, 0x36, 0xfb, 0xaa, 0x65, 0xff, 0x35, 0x03,
0x44, 0xaa, 0x6a, 0x53, 0x78, 0x81, 0xd6, 0xea, 0x4d, 0x28, 0x86, 0x5a, 0x81, 0xca, 0xb4, 0xcb,
0xe9, 0xaa, 0xa5, 0x31, 0xa1, 0xf0, 0x4a, 0xf4, 0xa5, 0xed, 0x2d, 0x75, 0x90, 0x06, 0x85, 0x67,
0xe1, 0xd5, 0xf7, 0x9d, 0x1e, 0x53, 0xfa, 0x8b, 0x11, 0x42, 0xc3, 0x43, 0xa7, 0xc7, 0xc2, 0x03,
0x5f, 0xb2, 0x56, 0x3a, 0x4c, 0x22, 0x85, 0xe7, 0x32, 0xaf, 0xe3, 0x77, 0x5d, 0xaf, 0xa7, 0x9c,
0x33, 0x82, 0x05, 0x07, 0xd7, 0xeb, 0xb2, 0x47, 0x82, 0x5d, 0xdb, 0xfd, 0x31, 0x53, 0xba, 0x4d,
0x22, 0x89, 0x0d, 0x65, 0xee, 0x73, 0xa7, 0x4f, 0x59, 0xc7, 0x0f, 0xba, 0x61, 0x6d, 0x0e, 0x89,
0x12, 0x38, 0x41, 0xd3, 0x75, 0xb8, 0xd3, 0xd2, 0x27, 0x49, 0x83, 0x24, 0x70, 0xe2, 0x9e, 0xa7,
0x2c, 0x08, 0x5d, 0xdf, 0x43, 0x7b, 0x14, 0xa9, 0x06, 0x09, 0x81, 0x6c, 0x28, 0x8e, 0x87, 0x55,
0x6b, 0x2d, 0x4b, 0xf1, 0x5b, 0x44, 0xe4, 0x03, 0xdf, 0xe7, 0x2c, 0x40, 0xc1, 0x4a, 0x78, 0xa6,
0x81, 0x21, 0x5b, 0x50, 0xed, 0xb2, 0xae, 0xdb, 0x71, 0x38, 0xeb, 0xde, 0xf5, 0xfb, 0xa3, 0x81,
0x17, 0xd6, 0xca, 0xe8, 0xcd, 0xb5, 0x48, 0xe5, 0x5b, 0x49, 0x02, 0x3a, 0xb5, 0xc3, 0xfe, 0x8b,
0x05, 0x0b, 0x13, 0x54, 0xe4, 0x26, 0xe4, 0xc2, 0x8e, 0x3f, 0x94, 0x1a, 0x9f, 0x6f, 0xae, 0x9c,
0xc5, 0xae, 0xd1, 0x16, 0x54, 0x54, 0x12, 0x8b, 0x3b, 0x78, 0xce, 0x40, 0xfb, 0x0a, 0x7e, 0x93,
0x1b, 0x90, 0xe5, 0xe3, 0xa1, 0x8c, 0xf2, 0xf9, 0xe6, 0x33, 0x67, 0x32, 0x3a, 0x18, 0x0f, 0x19,
0x45, 0x52, 0xfb, 0x1a, 0xe4, 0x90, 0x2d, 0x29, 0x40, 0xb6, 0xbd, 0x7f, 0x67, 0xaf, 0x3a, 0x43,
0xca, 0x50, 0xa0, 0xad, 0xf6, 0xfd, 0x77, 0xe9, 0xdd, 0x56, 0xd5, 0xb2, 0x09, 0x64, 0x05, 0x39,
0x01, 0xc8, 0xb7, 0x0f, 0xe8, 0xf6, 0xde, 0xbd, 0xea, 0x8c, 0xfd, 0x08, 0xe6, 0xb5, 0x77, 0xa9,
0x04, 0x73, 0x13, 0xf2, 0x98, 0x43, 0x74, 0x84, 0x5f, 0x4d, 0x66, 0x0e, 0x49, 0xbd, 0xcb, 0xb8,
0x23, 0x2c, 0x44, 0x15, 0x2d, 0xd9, 0x98, 0x4c, 0x38, 0x93, 0xde, 0x3b, 0x95, 0x6d, 0xfe, 0x91,
0x81, 0x4b, 0x29, 0x1c, 0x27, 0x33, 0x6d, 0x31, 0xce, 0xb4, 0x6b, 0xb0, 0x10, 0xf8, 0x3e, 0x6f,
0xb3, 0xe0, 0xd4, 0xed, 0xb0, 0xbd, 0x58, 0x65, 0x93, 0x68, 0xe1, 0x9d, 0x02, 0x85, 0xec, 0x91,
0x4e, 0x26, 0xde, 0x24, 0x92, 0xbc, 0x08, 0x8b, 0x18, 0x12, 0x07, 0xee, 0x80, 0xbd, 0xeb, 0xb9,
0x8f, 0xf6, 0x1c, 0xcf, 0xc7, 0x48, 0xc8, 0xd2, 0xe9, 0x05, 0xe1, 0x55, 0xdd, 0x38, 0x25, 0xc9,
0xf4, 0x62, 0x60, 0xc8, 0xf3, 0x30, 0x17, 0xaa, 0x9c, 0x91, 0x47, 0x0d, 0x54, 0x63, 0x0d, 0x48,
0x3c, 0xd5, 0x04, 0xe4, 0x45, 0x28, 0xa8, 0x4f, 0x11, 0x13, 0x99, 0x54, 0xe2, 0x88, 0x82, 0x50,
0x28, 0x87, 0xf2, 0x72, 0x6d, 0xee, 0xf0, 0xb0, 0x56, 0xc0, 0x1d, 0x8d, 0xf3, 0xec, 0xd2, 0x68,
0x1b, 0x1b, 0x30, 0x49, 0xd1, 0x04, 0x8f, 0xfa, 0x21, 0x2c, 0x4e, 0x91, 0xa4, 0xe4, 0xb1, 0x17,
0xcc, 0x3c, 0x56, 0x6a, 0x3e, 0x65, 0x18, 0x35, 0xde, 0x6c, 0xa6, 0xb7, 0x1d, 0x28, 0x9b, 0x4b,
0x98, 0x87, 0x86, 0x8e, 0x77, 0xd7, 0x1f, 0x79, 0x1c, 0x19, 0x8b, 0x3c, 0xa4, 0x11, 0x42, 0xa7,
0x2c, 0x08, 0xfc, 0x40, 0x2e, 0xcb, 0x62, 0x60, 0x60, 0xec, 0x9f, 0x5b, 0x30, 0xa7, 0xf4, 0x41,
0x9e, 0x85, 0x9c, 0xd8, 0xa8, 0xdd, 0xb2, 0x92, 0x50, 0x18, 0x95, 0x6b, 0xc2, 0x79, 0x06, 0x0e,
0xef, 0x3c, 0x64, 0x5d, 0xc5, 0x4d, 0x83, 0xe4, 0x75, 0x00, 0x87, 0xf3, 0xc0, 0x3d, 0x1e, 0x71,
0x26, 0x2a, 0x8a, 0xe0, 0x71, 0x25, 0xe2, 0xa1, 0xba, 0x88, 0xd3, 0x1b, 0x8d, 0xb7, 0xd9, 0xf8,
0x50, 0xdc, 0x86, 0x1a, 0xe4, 0x22, 0xd6, 0xb3, 0xe2, 0x18, 0xb2, 0x0c, 0x79, 0x71, 0x50, 0xe4,
0x9b, 0x0a, 0x4a, 0x0d, 0xe1, 0x54, 0xf7, 0xca, 0x9c, 0xe5, 0x5e, 0xd7, 0xa1, 0xa2, 0x9d, 0x49,
0xc0, 0xa1, 0x72, 0xc4, 0x24, 0x72, 0xe2, 0x16, 0xb9, 0xc7, 0xbb, 0xc5, 0xef, 0xa2, 0x5a, 0xae,
0x82, 0x51, 0x44, 0x94, 0xeb, 0x85, 0x43, 0xd6, 0xe1, 0xac, 0x7b, 0xa0, 0x83, 0x1e, 0xeb, 0xdd,
0x04, 0x9a, 0x3c, 0x07, 0xf3, 0x11, 0x6a, 0x73, 0x2c, 0x0e, 0x9f, 0x45, 0xf9, 0x26, 0xb0, 0x64,
0x15, 0x4a, 0x98, 0xdd, 0xb1, 0xb8, 0xe9, 0xca, 0x6d, 0xa2, 0xc4, 0x45, 0x3b, 0xfe, 0x60, 0xd8,
0x67, 0x9c, 0x75, 0xdf, 0xf2, 0x8f, 0x43, 0x5d, 0x7b, 0x12, 0x48, 0xe1, 0x37, 0xb8, 0x09, 0x29,
0x64, 0xb0, 0xc5, 0x08, 0x21, 0x77, 0xcc, 0x52, 0x8a, 0x93, 0x47, 0x71, 0x26, 0xd1, 0x09, 0xb9,
0xb1, 0x86, 0x63, 0x0d, 0x32, 0xe5, 0x46, 0xac, 0xdd, 0x13, 0xf1, 0x20, 0x54, 0x23, 0xaa, 0xba,
0x2e, 0xca, 0x4b, 0x3a, 0x9d, 0x4b, 0x63, 0xab, 0x74, 0xbd, 0x04, 0x39, 0xec, 0xd1, 0x74, 0x6d,
0x47, 0x20, 0x6e, 0x3c, 0x32, 0x29, 0x8d, 0x47, 0x36, 0x6a, 0x3c, 0xec, 0x8f, 0x33, 0xb0, 0x1c,
0x9f, 0x94, 0xe8, 0x01, 0x5e, 0x9d, 0xee, 0x01, 0xea, 0x13, 0x59, 0xd4, 0x90, 0xee, 0xdb, 0x3e,
0xe0, 0x9b, 0xd1, 0x07, 0x7c, 0x9a, 0x81, 0x2b, 0x91, 0x71, 0x30, 0xe8, 0x92, 0x56, 0xfd, 0xde,
0xb4, 0x55, 0xaf, 0x4d, 0x5b, 0x55, 0x6e, 0xfc, 0xd6, 0xb4, 0xdf, 0x28, 0xd3, 0x6e, 0xe8, 0x56,
0x5d, 0x86, 0x9d, 0x6a, 0x90, 0xea, 0x50, 0xe0, 0x4e, 0x4f, 0x74, 0x10, 0xb2, 0x16, 0x15, 0x69,
0x04, 0xdb, 0x6f, 0xc1, 0x52, 0xbc, 0xe3, 0xb0, 0x19, 0xed, 0x69, 0x42, 0x1e, 0x93, 0x87, 0xae,
0x5e, 0x69, 0x71, 0x7d, 0xd8, 0x94, 0x5d, 0xa1, 0xa2, 0xb4, 0x5f, 0x37, 0x53, 0x92, 0x5a, 0x8c,
0x0a, 0x8d, 0x65, 0x14, 0x1a, 0x02, 0x59, 0x2e, 0x26, 0xb2, 0x59, 0x14, 0x06, 0xbf, 0xed, 0xa1,
0x91, 0x65, 0x12, 0xbe, 0x85, 0xfd, 0x95, 0x14, 0x37, 0xea, 0xaf, 0x24, 0x78, 0x51, 0x62, 0xcb,
0xa6, 0x24, 0xb6, 0x5c, 0x9c, 0xd8, 0x5e, 0x81, 0xa7, 0xa7, 0x4e, 0x54, 0xb7, 0x17, 0xc9, 0x5c,
0x23, 0x95, 0xca, 0x62, 0x84, 0x7d, 0x13, 0x0a, 0x7a, 0x0b, 0x5e, 0x65, 0x1c, 0x25, 0x5c, 0xfc,
0x4e, 0x9f, 0xa5, 0xec, 0x1d, 0xb8, 0x3c, 0x71, 0x9c, 0xa1, 0xee, 0xf5, 0xc9, 0x03, 0x4b, 0xcd,
0xc5, 0xb8, 0x5d, 0x52, 0x2b, 0xa6, 0x0c, 0x7b, 0x90, 0xc3, 0x42, 0x47, 0x5a, 0x50, 0x09, 0x58,
0xe8, 0x8f, 0x82, 0x0e, 0x6b, 0x1b, 0xdd, 0x46, 0x1c, 0xb1, 0xf2, 0xa5, 0xe0, 0xf4, 0x46, 0x83,
0x9a, 0x64, 0x34, 0xb9, 0xcb, 0xde, 0x83, 0xf2, 0xfe, 0x28, 0x8c, 0x9b, 0xea, 0x37, 0xa0, 0x82,
0x6d, 0x4d, 0xb8, 0x39, 0x3e, 0x50, 0xd3, 0x7b, 0x66, 0x6d, 0xde, 0x70, 0x46, 0x41, 0xdd, 0x12,
0x14, 0x94, 0x39, 0xa1, 0xef, 0xd1, 0x24, 0xb9, 0xfd, 0x7b, 0x0b, 0xaa, 0x82, 0x04, 0x8b, 0x9a,
0xb6, 0xe4, 0x4b, 0x51, 0xa7, 0x2e, 0x2c, 0x5f, 0xde, 0x7c, 0x4a, 0x4c, 0xda, 0xff, 0xfc, 0xec,
0x5a, 0x65, 0x3f, 0x60, 0x4e, 0xbf, 0xef, 0x77, 0x24, 0xb5, 0x6e, 0xd1, 0xff, 0x1f, 0x32, 0x6e,
0x57, 0xb6, 0x3e, 0x67, 0xd2, 0x0a, 0x0a, 0x72, 0x0b, 0x40, 0xe6, 0x9f, 0x2d, 0x87, 0x3b, 0xb5,
0xec, 0x79, 0xf4, 0x06, 0xa1, 0xbd, 0x2b, 0x45, 0x94, 0xfa, 0x50, 0x22, 0xde, 0x86, 0xb9, 0x63,
0x6c, 0xc0, 0xbe, 0xb4, 0x22, 0x35, 0xbd, 0x7d, 0x1d, 0x40, 0xbd, 0x46, 0x88, 0x3a, 0xbe, 0x9c,
0x98, 0x4a, 0xca, 0xfa, 0x52, 0xf6, 0x1b, 0x50, 0xdc, 0x71, 0xbd, 0x93, 0x76, 0xdf, 0xed, 0x88,
0xa1, 0x29, 0xd7, 0x77, 0xbd, 0x13, 0x7d, 0xd6, 0x95, 0xe9, 0xb3, 0xc4, 0x19, 0x0d, 0xb1, 0x81,
0x4a, 0x4a, 0xfb, 0x67, 0x16, 0x10, 0x81, 0xd4, 0xe3, 0x49, 0x5c, 0xf9, 0x65, 0x28, 0x58, 0x66,
0x28, 0xd4, 0x60, 0xae, 0x17, 0xf8, 0xa3, 0xe1, 0xa6, 0x0e, 0x11, 0x0d, 0x0a, 0xfa, 0x3e, 0x3e,
0x46, 0xc8, 0xfe, 0x4e, 0x02, 0x5f, 0x3a, 0x74, 0x7e, 0x69, 0xc1, 0x65, 0x43, 0x88, 0xf6, 0x68,
@@ -3500,69 +3486,67 @@ var fileDescriptor_f22805646f4f62b6 = []byte{
0x46, 0xbf, 0xb1, 0xa0, 0x64, 0x88, 0x48, 0x36, 0x61, 0xb1, 0xef, 0x70, 0xe6, 0x75, 0xc6, 0x47,
0x0f, 0xb5, 0x78, 0xca, 0x2b, 0xe3, 0x19, 0xca, 0x94, 0x9d, 0x56, 0x15, 0x7d, 0x7c, 0x9b, 0xef,
0x40, 0x3e, 0x64, 0x81, 0xab, 0xc2, 0xdb, 0xcc, 0x60, 0x51, 0x77, 0xaf, 0x08, 0xc4, 0xc5, 0x65,
0xbe, 0x50, 0x8a, 0x55, 0x90, 0xfd, 0xb7, 0xa4, 0x77, 0x2b, 0xc7, 0x9a, 0x1e, 0xca, 0x2e, 0xb0,
0xbe, 0x50, 0x8a, 0x55, 0x90, 0xfd, 0xf7, 0xa4, 0x77, 0x2b, 0xc7, 0x9a, 0x1e, 0xca, 0x2e, 0xb0,
0xd6, 0x6c, 0xaa, 0xb5, 0x62, 0xf9, 0x32, 0x17, 0xc9, 0x57, 0x85, 0xcc, 0xf0, 0xf6, 0x6d, 0x35,
0xd2, 0x88, 0xa5, 0xc4, 0xdc, 0x42, 0xc7, 0x43, 0xcc, 0x2d, 0x89, 0xd9, 0x50, 0x7d, 0xbc, 0x58,
0x22, 0xe6, 0xd6, 0x86, 0x6a, 0xd8, 0xc5, 0xd2, 0x7e, 0x0f, 0xea, 0x69, 0x71, 0xa2, 0x5c, 0xf4,
0x36, 0x14, 0x43, 0x44, 0xb9, 0x6c, 0x3a, 0x05, 0xa4, 0x7c, 0x17, 0x53, 0xdb, 0xbf, 0xb5, 0xa0,
0x92, 0x30, 0x6c, 0xa2, 0x12, 0xe5, 0x54, 0x25, 0x2a, 0x83, 0xe5, 0xa1, 0x32, 0x32, 0xd4, 0xf2,
0x04, 0xf4, 0x00, 0xf5, 0x6d, 0x51, 0xeb, 0x81, 0x80, 0xe4, 0x28, 0x53, 0xa4, 0x56, 0x28, 0xa0,
0x63, 0xbc, 0x5c, 0x81, 0x5a, 0xc7, 0x02, 0xea, 0xaa, 0x8b, 0x59, 0x5d, 0x9c, 0x21, 0xb9, 0xc3,
0x47, 0xb2, 0x57, 0xca, 0x51, 0x05, 0x89, 0x13, 0x4f, 0x5c, 0xaf, 0x8b, 0xdd, 0x51, 0x8e, 0xe2,
0xda, 0x66, 0xf2, 0x41, 0x52, 0x09, 0x2e, 0xd2, 0xac, 0x68, 0x7d, 0x02, 0x16, 0x8e, 0xfa, 0xfc,
0x20, 0x2e, 0x94, 0x06, 0x46, 0xb4, 0x1a, 0x12, 0x52, 0x6e, 0x53, 0x4f, 0x8d, 0x21, 0xa4, 0xa0,
0x8a, 0x52, 0x64, 0xc1, 0xc5, 0xa9, 0x5d, 0xe1, 0x26, 0x7d, 0xe7, 0x98, 0xf5, 0x8d, 0x5e, 0x21,
0x46, 0x08, 0x39, 0x10, 0x38, 0x34, 0x6a, 0xb3, 0x81, 0x21, 0xeb, 0x30, 0xcb, 0xb5, 0x6b, 0x5c,
0x3b, 0x5b, 0x86, 0x7d, 0xdf, 0xf5, 0x38, 0x9d, 0xe5, 0xa1, 0x88, 0xa1, 0xe5, 0xf4, 0x6d, 0x34,
0x86, 0xab, 0x84, 0xa8, 0x50, 0x5c, 0x0b, 0xef, 0x38, 0x75, 0xfa, 0x78, 0xb0, 0x45, 0xc5, 0x52,
0x4c, 0x85, 0xec, 0x11, 0x1b, 0x0c, 0xfb, 0x4e, 0x70, 0xa0, 0x5e, 0x90, 0x32, 0xf8, 0x56, 0x3f,
0x89, 0x26, 0xcf, 0x43, 0x55, 0xa3, 0xf4, 0x8b, 0xb2, 0x72, 0xce, 0x29, 0xbc, 0xdd, 0x86, 0x4b,
0xf8, 0x38, 0xbc, 0xed, 0x85, 0xdc, 0xf1, 0xf8, 0xf9, 0x59, 0x39, 0xca, 0xb2, 0x2a, 0xd3, 0x24,
0xb2, 0xac, 0x8c, 0x4d, 0xcc, 0xb2, 0x8f, 0x60, 0x29, 0xc9, 0x54, 0xb9, 0x70, 0x23, 0x8a, 0x29,
0xe9, 0xbf, 0x71, 0xda, 0x51, 0x94, 0x6d, 0xdc, 0x8d, 0x02, 0xeb, 0xf1, 0x9f, 0xdd, 0x7e, 0x6a,
0x41, 0x25, 0xc1, 0x8b, 0xdc, 0x86, 0x3c, 0x9a, 0x6d, 0x3a, 0x66, 0xa6, 0xdf, 0x13, 0xd4, 0x6b,
0xbe, 0xfa, 0x20, 0xd9, 0x9a, 0x59, 0x2a, 0x19, 0x92, 0x6b, 0x50, 0x1a, 0x06, 0xfe, 0xe0, 0x48,
0x71, 0x95, 0x6f, 0x6f, 0x20, 0x50, 0x3b, 0x88, 0xb1, 0xff, 0x9d, 0x81, 0x45, 0xbc, 0x3e, 0x75,
0xbc, 0x1e, 0x7b, 0x22, 0x1a, 0xc5, 0x29, 0x81, 0xb3, 0xa1, 0x32, 0x23, 0xae, 0x45, 0x8d, 0x0c,
0x1f, 0x3a, 0x41, 0x77, 0x7b, 0x4b, 0x55, 0x38, 0x0d, 0x0a, 0xe7, 0xc5, 0xa5, 0xcc, 0x6f, 0x72,
0xb0, 0x31, 0x30, 0xc9, 0x1f, 0x66, 0xe6, 0x26, 0x7e, 0x98, 0x31, 0x67, 0xb2, 0xc2, 0x39, 0x33,
0x59, 0xf1, 0xc2, 0x99, 0x0c, 0xd2, 0x66, 0x32, 0x63, 0x12, 0x2a, 0x25, 0x27, 0x21, 0x73, 0x5a,
0x2b, 0x4f, 0x4c, 0x6b, 0x7a, 0x4a, 0xaa, 0x9c, 0x39, 0x25, 0xcd, 0x7f, 0xa9, 0x29, 0x69, 0xe1,
0x71, 0xa7, 0x24, 0xec, 0x0c, 0x54, 0xd0, 0x84, 0xb5, 0xaa, 0xbc, 0x73, 0x84, 0xb0, 0x43, 0x20,
0xa6, 0xa9, 0x95, 0x9f, 0xbf, 0x30, 0xe1, 0xe7, 0x97, 0xe2, 0xf2, 0xea, 0x0e, 0xd8, 0x57, 0x76,
0xf2, 0x9f, 0x40, 0xa1, 0xa5, 0x24, 0x78, 0xf2, 0xee, 0xfd, 0x7f, 0x50, 0x16, 0x09, 0x28, 0xe4,
0xce, 0x60, 0x78, 0x34, 0x90, 0xfe, 0x9d, 0xa1, 0xa5, 0x08, 0xb7, 0x1b, 0xda, 0x77, 0x20, 0xdf,
0x76, 0x06, 0xc3, 0xfe, 0x34, 0xf1, 0xec, 0x14, 0x71, 0x7c, 0x8a, 0x65, 0x9c, 0x62, 0x7f, 0x62,
0x01, 0xc4, 0xba, 0xf8, 0x2a, 0xb7, 0x58, 0x87, 0xb9, 0x10, 0x85, 0xd1, 0x8d, 0xc4, 0x42, 0xac,
0x3e, 0xc4, 0x2b, 0x7a, 0x4d, 0x75, 0x61, 0xfc, 0x92, 0x5b, 0xa6, 0xc5, 0xb3, 0x13, 0xc5, 0x5f,
0x2b, 0x5e, 0x71, 0x8d, 0x29, 0x9f, 0xff, 0x00, 0x16, 0x26, 0xc6, 0x1c, 0x52, 0x86, 0xc2, 0xde,
0xfd, 0xa3, 0x16, 0xa5, 0xf7, 0x69, 0x75, 0x86, 0x5c, 0x82, 0x85, 0xdd, 0x3b, 0xef, 0x1f, 0xed,
0x6c, 0x1f, 0xb6, 0x8e, 0x0e, 0xe8, 0x9d, 0xbb, 0xad, 0x76, 0xd5, 0x12, 0x48, 0x5c, 0x1f, 0x1d,
0xdc, 0xbf, 0x7f, 0xb4, 0x73, 0x87, 0xde, 0x6b, 0x55, 0x67, 0xc9, 0x22, 0x54, 0xde, 0xdd, 0x7b,
0x7b, 0xef, 0xfe, 0x7b, 0x7b, 0xea, 0xe3, 0x4c, 0xf3, 0x57, 0x16, 0xe4, 0x05, 0x7b, 0x16, 0x90,
0xef, 0x43, 0x31, 0x1a, 0x96, 0xc8, 0xe5, 0xc4, 0x8c, 0x65, 0x0e, 0x50, 0xf5, 0xa7, 0x12, 0x5b,
0xda, 0x39, 0xed, 0x19, 0x72, 0x07, 0x4a, 0x11, 0xf1, 0x61, 0xf3, 0xbf, 0x61, 0xd1, 0xfc, 0x97,
0x05, 0x55, 0xe5, 0x97, 0xf7, 0x98, 0xc7, 0x02, 0x87, 0xfb, 0x91, 0x60, 0x38, 0xe9, 0x4c, 0x70,
0x35, 0xc7, 0xa6, 0xb3, 0x05, 0xdb, 0x06, 0xb8, 0xc7, 0xb8, 0xee, 0x32, 0xaf, 0xa4, 0x97, 0x55,
0xc9, 0xe3, 0xea, 0x19, 0x35, 0x57, 0xb3, 0xba, 0x07, 0x10, 0x07, 0x26, 0x89, 0xbb, 0x84, 0xa9,
0xc4, 0x5c, 0xbf, 0x92, 0xba, 0x17, 0xdd, 0xf4, 0x0f, 0x59, 0x98, 0x13, 0x1b, 0x2e, 0x0b, 0xc8,
0x9b, 0x50, 0xf9, 0x81, 0xeb, 0x75, 0xa3, 0x9f, 0x91, 0x49, 0xca, 0xef, 0xce, 0x9a, 0x6d, 0x3d,
0x6d, 0xcb, 0x30, 0x41, 0x59, 0xff, 0x30, 0xd5, 0x61, 0x1e, 0x27, 0x67, 0xfc, 0x1a, 0x5a, 0x7f,
0x7a, 0x0a, 0x1f, 0xb1, 0x68, 0x41, 0xc9, 0xf8, 0xa5, 0xd5, 0xd4, 0xd6, 0xd4, 0xef, 0xaf, 0xe7,
0xb1, 0xb9, 0x07, 0x10, 0xbf, 0xc3, 0x90, 0x73, 0x5e, 0x64, 0xeb, 0x57, 0x52, 0xf7, 0x22, 0x46,
0x6f, 0xeb, 0x2b, 0xc9, 0x07, 0x9d, 0x73, 0x59, 0x3d, 0x93, 0xfa, 0x40, 0x64, 0x30, 0x3b, 0x84,
0x85, 0x89, 0xf7, 0x0f, 0x72, 0xd1, 0xb3, 0x62, 0x7d, 0xf5, 0x6c, 0x82, 0x88, 0xef, 0x0f, 0x8d,
0x57, 0x27, 0xfd, 0xae, 0x72, 0x31, 0x67, 0xfb, 0x2c, 0x02, 0x53, 0xe6, 0xe6, 0x5f, 0xb3, 0x50,
0x6d, 0xf3, 0x80, 0x39, 0x03, 0xd7, 0xeb, 0x69, 0x97, 0x79, 0x1d, 0xf2, 0xaa, 0xf0, 0x3d, 0xae,
0x89, 0x37, 0x2c, 0x11, 0x0f, 0x4f, 0xc4, 0x36, 0x1b, 0x16, 0xd9, 0x7d, 0x82, 0xd6, 0xd9, 0xb0,
0xc8, 0xfb, 0x5f, 0x8f, 0x7d, 0x36, 0x2c, 0xf2, 0xc1, 0xd7, 0x67, 0xa1, 0x0d, 0x8b, 0xec, 0xc3,
0xa2, 0xca, 0x15, 0x4f, 0x24, 0x3b, 0x6c, 0x58, 0xe4, 0x10, 0x2e, 0x99, 0x1c, 0x55, 0xf3, 0x49,
0xae, 0x26, 0xbf, 0x4b, 0xb6, 0xd7, 0x86, 0x86, 0xd3, 0xfa, 0x64, 0xc1, 0xb7, 0xf9, 0x27, 0x0b,
0xe6, 0x74, 0x26, 0x3c, 0x4a, 0x9d, 0x73, 0xed, 0xf3, 0xa6, 0x3f, 0x75, 0xd0, 0xb3, 0xe7, 0xd2,
0x3c, 0xf1, 0x6c, 0xb9, 0x59, 0xfb, 0xe8, 0xf3, 0x15, 0xeb, 0x93, 0xcf, 0x57, 0xac, 0x7f, 0x7e,
0xbe, 0x62, 0xfd, 0xfa, 0x8b, 0x95, 0x99, 0x4f, 0xbe, 0x58, 0x99, 0xf9, 0xf4, 0x8b, 0x95, 0x99,
0xe3, 0x3c, 0xfe, 0xfb, 0xd1, 0xcb, 0xff, 0x09, 0x00, 0x00, 0xff, 0xff, 0x09, 0xb5, 0xc0, 0xda,
0xff, 0x24, 0x00, 0x00,
0xd2, 0x88, 0x4f, 0x89, 0xb9, 0x85, 0x8e, 0x87, 0x98, 0x5b, 0x12, 0xb3, 0xa1, 0xfa, 0x78, 0xf1,
0x89, 0x98, 0x5b, 0x1b, 0xaa, 0x61, 0x17, 0x9f, 0xf6, 0x7b, 0x50, 0x4f, 0x8b, 0x13, 0xe5, 0xa2,
0xb7, 0xa1, 0x18, 0x22, 0xca, 0x65, 0xd3, 0x29, 0x20, 0x65, 0x5f, 0x4c, 0x6d, 0xff, 0xd6, 0x82,
0x4a, 0xc2, 0xb0, 0x89, 0x4a, 0x94, 0x53, 0x95, 0xa8, 0x0c, 0x96, 0x87, 0xca, 0xc8, 0x50, 0xcb,
0x13, 0xd0, 0x03, 0xd4, 0xb7, 0x45, 0xad, 0x07, 0x02, 0x92, 0xa3, 0x4c, 0x91, 0x5a, 0xa1, 0x80,
0x8e, 0xf1, 0x72, 0x05, 0x6a, 0x1d, 0x0b, 0xa8, 0xab, 0x2e, 0x66, 0x75, 0x71, 0x86, 0xe4, 0x0e,
0x1f, 0xc9, 0x5e, 0x29, 0x47, 0x15, 0x24, 0x4e, 0x3c, 0x71, 0xbd, 0x2e, 0x76, 0x47, 0x39, 0x8a,
0xdf, 0x36, 0x93, 0x0f, 0x92, 0x4a, 0x70, 0x91, 0x66, 0x45, 0xeb, 0x13, 0xb0, 0x70, 0xd4, 0xe7,
0x07, 0x71, 0xa1, 0x34, 0x30, 0xa2, 0xd5, 0x90, 0x90, 0x72, 0x9b, 0x7a, 0x6a, 0x0c, 0x21, 0x05,
0x55, 0x94, 0x22, 0x0b, 0x2e, 0x4e, 0xad, 0x0a, 0x37, 0xe9, 0x3b, 0xc7, 0xac, 0x6f, 0xf4, 0x0a,
0x31, 0x42, 0xc8, 0x81, 0xc0, 0xa1, 0x51, 0x9b, 0x0d, 0x0c, 0x59, 0x87, 0x59, 0xae, 0x5d, 0xe3,
0xda, 0xd9, 0x32, 0xec, 0xfb, 0xae, 0xc7, 0xe9, 0x2c, 0x0f, 0x45, 0x0c, 0x2d, 0xa7, 0x2f, 0xa3,
0x31, 0x5c, 0x25, 0x44, 0x85, 0xe2, 0xb7, 0xf0, 0x8e, 0x53, 0xa7, 0x8f, 0x07, 0x5b, 0x54, 0x7c,
0x8a, 0xa9, 0x90, 0x3d, 0x62, 0x83, 0x61, 0xdf, 0x09, 0x0e, 0xd4, 0x0b, 0x52, 0x06, 0xdf, 0xea,
0x27, 0xd1, 0xe4, 0x79, 0xa8, 0x6a, 0x94, 0x7e, 0x51, 0x56, 0xce, 0x39, 0x85, 0xb7, 0xdb, 0x70,
0x09, 0x1f, 0x87, 0xb7, 0xbd, 0x90, 0x3b, 0x1e, 0x3f, 0x3f, 0x2b, 0x47, 0x59, 0x56, 0x65, 0x9a,
0x44, 0x96, 0x95, 0xb1, 0x89, 0x59, 0xf6, 0x11, 0x2c, 0x25, 0x99, 0x2a, 0x17, 0x6e, 0x44, 0x31,
0x25, 0xfd, 0x37, 0x4e, 0x3b, 0x8a, 0xb2, 0x8d, 0xab, 0x51, 0x60, 0x3d, 0xfe, 0xb3, 0xdb, 0x4f,
0x2d, 0xa8, 0x24, 0x78, 0x91, 0xdb, 0x90, 0x47, 0xb3, 0x4d, 0xc7, 0xcc, 0xf4, 0x7b, 0x82, 0x7a,
0xcd, 0x57, 0x1b, 0x92, 0xad, 0x99, 0xa5, 0x92, 0x21, 0xb9, 0x06, 0xa5, 0x61, 0xe0, 0x0f, 0x8e,
0x14, 0x57, 0xf9, 0xf6, 0x06, 0x02, 0xb5, 0x83, 0x18, 0xfb, 0x4f, 0x19, 0x58, 0xc4, 0xeb, 0x53,
0xc7, 0xeb, 0xb1, 0x27, 0xa2, 0x51, 0x9c, 0x12, 0x38, 0x1b, 0x2a, 0x33, 0xe2, 0x77, 0xf2, 0xe7,
0x95, 0xb9, 0x89, 0x9f, 0x57, 0xcc, 0xc9, 0xaa, 0x70, 0xce, 0x64, 0x55, 0xbc, 0x70, 0xb2, 0x82,
0xb4, 0xc9, 0xca, 0x98, 0x67, 0x4a, 0xc9, 0x79, 0xc6, 0x9c, 0xb9, 0xca, 0x13, 0x33, 0x97, 0x9e,
0x75, 0x2a, 0x67, 0xce, 0x3a, 0xf3, 0x5f, 0x6a, 0xd6, 0x59, 0x78, 0xdc, 0x59, 0x07, 0xeb, 0xbb,
0x72, 0xfd, 0xb0, 0x56, 0x95, 0x77, 0x8e, 0x10, 0x76, 0x08, 0xc4, 0x34, 0x98, 0xf2, 0xd6, 0x17,
0x26, 0xbc, 0xf5, 0x52, 0x5c, 0x24, 0xdd, 0x01, 0xfb, 0xca, 0xae, 0xfa, 0x13, 0x28, 0xb4, 0x94,
0x04, 0x4f, 0xde, 0x49, 0xff, 0x0f, 0xca, 0x22, 0x8d, 0x84, 0xdc, 0x19, 0x0c, 0x8f, 0x06, 0xd2,
0x4b, 0x33, 0xb4, 0x14, 0xe1, 0x76, 0x43, 0xfb, 0x0e, 0xe4, 0xdb, 0xce, 0x60, 0xd8, 0x9f, 0x26,
0x9e, 0x9d, 0x22, 0x8e, 0x4f, 0xb1, 0x8c, 0x53, 0xec, 0x4f, 0x2c, 0x80, 0x58, 0x17, 0x5f, 0xe5,
0x16, 0xeb, 0x30, 0x17, 0xa2, 0x30, 0xba, 0x1d, 0x58, 0x88, 0xd5, 0x87, 0x78, 0x45, 0xaf, 0xa9,
0x2e, 0x8c, 0x42, 0x72, 0xcb, 0xb4, 0x78, 0x76, 0xa2, 0x84, 0x6b, 0xc5, 0x2b, 0xae, 0x31, 0xe5,
0xf3, 0x1f, 0xc0, 0xc2, 0xc4, 0xb0, 0x42, 0xca, 0x50, 0xd8, 0xbb, 0x7f, 0xd4, 0xa2, 0xf4, 0x3e,
0xad, 0xce, 0x90, 0x4b, 0xb0, 0xb0, 0x7b, 0xe7, 0xfd, 0xa3, 0x9d, 0xed, 0xc3, 0xd6, 0xd1, 0x01,
0xbd, 0x73, 0xb7, 0xd5, 0xae, 0x5a, 0x02, 0x89, 0xdf, 0x47, 0x07, 0xf7, 0xef, 0x1f, 0xed, 0xdc,
0xa1, 0xf7, 0x5a, 0xd5, 0x59, 0xb2, 0x08, 0x95, 0x77, 0xf7, 0xde, 0xde, 0xbb, 0xff, 0xde, 0x9e,
0xda, 0x9c, 0x69, 0xfe, 0xca, 0x82, 0xbc, 0x60, 0xcf, 0x02, 0xf2, 0x7d, 0x28, 0x46, 0x23, 0x0f,
0xb9, 0x9c, 0x98, 0x94, 0xcc, 0x31, 0xa8, 0xfe, 0x54, 0x62, 0x49, 0x3b, 0xa7, 0x3d, 0x43, 0xee,
0x40, 0x29, 0x22, 0x3e, 0x6c, 0xfe, 0x37, 0x2c, 0x9a, 0xff, 0xb6, 0xa0, 0xaa, 0xfc, 0xf2, 0x1e,
0xf3, 0x58, 0xe0, 0x70, 0x3f, 0x12, 0x0c, 0xe7, 0x95, 0x09, 0xae, 0xe6, 0xf0, 0x73, 0xb6, 0x60,
0xdb, 0x00, 0xf7, 0x18, 0xd7, 0xbd, 0xe2, 0x95, 0xf4, 0xe2, 0x28, 0x79, 0x5c, 0x3d, 0xa3, 0x72,
0x6a, 0x56, 0xf7, 0x00, 0xe2, 0xc0, 0x24, 0x71, 0xad, 0x9f, 0x4a, 0xaf, 0xf5, 0x2b, 0xa9, 0x6b,
0xd1, 0x4d, 0xff, 0x90, 0x85, 0x39, 0xb1, 0xe0, 0xb2, 0x80, 0xbc, 0x09, 0x95, 0x1f, 0xb8, 0x5e,
0x37, 0xfa, 0x31, 0x98, 0xa4, 0xfc, 0x7a, 0xac, 0xd9, 0xd6, 0xd3, 0x96, 0x0c, 0x13, 0x94, 0xf5,
0xcf, 0x4b, 0x1d, 0xe6, 0x71, 0x72, 0xc6, 0x6f, 0x9a, 0xf5, 0xa7, 0xa7, 0xf0, 0x11, 0x8b, 0x16,
0x94, 0x8c, 0xdf, 0x4b, 0x4d, 0x6d, 0x4d, 0xfd, 0x8a, 0x7a, 0x1e, 0x9b, 0x7b, 0x00, 0xf1, 0x6b,
0x0a, 0x39, 0xe7, 0x5d, 0xb5, 0x7e, 0x25, 0x75, 0x2d, 0x62, 0xf4, 0xb6, 0xbe, 0x92, 0x7c, 0x96,
0x39, 0x97, 0xd5, 0x33, 0xa9, 0xcf, 0x3c, 0x06, 0xb3, 0x43, 0x58, 0x98, 0x78, 0xc5, 0x20, 0x17,
0x3d, 0x0e, 0xd6, 0x57, 0xcf, 0x26, 0x88, 0xf8, 0xfe, 0xd0, 0x78, 0x3b, 0xd2, 0xaf, 0x23, 0x17,
0x73, 0xb6, 0xcf, 0x22, 0x30, 0x65, 0x6e, 0xfe, 0x2d, 0x0b, 0xd5, 0x36, 0x0f, 0x98, 0x33, 0x70,
0xbd, 0x9e, 0x76, 0x99, 0xd7, 0x21, 0xaf, 0x0a, 0xdf, 0xe3, 0x9a, 0x78, 0xc3, 0x12, 0xf1, 0xf0,
0x44, 0x6c, 0xb3, 0x61, 0x91, 0xdd, 0x27, 0x68, 0x9d, 0x0d, 0x8b, 0xbc, 0xff, 0xf5, 0xd8, 0x67,
0xc3, 0x22, 0x1f, 0x7c, 0x7d, 0x16, 0xda, 0xb0, 0xc8, 0x3e, 0x2c, 0xaa, 0x5c, 0xf1, 0x44, 0xb2,
0xc3, 0x86, 0x45, 0x0e, 0xe1, 0x92, 0xc9, 0x51, 0xb5, 0x90, 0xe4, 0x6a, 0x72, 0x5f, 0xb2, 0x49,
0x36, 0x34, 0x9c, 0xd6, 0xed, 0x0a, 0xbe, 0xcd, 0x3f, 0x5b, 0x30, 0xa7, 0x33, 0xe1, 0x51, 0xea,
0xb4, 0x6a, 0x9f, 0x37, 0xc3, 0xa9, 0x83, 0x9e, 0x3d, 0x97, 0xe6, 0x89, 0x67, 0xcb, 0xcd, 0xda,
0x47, 0x9f, 0xaf, 0x58, 0x9f, 0x7c, 0xbe, 0x62, 0xfd, 0xeb, 0xf3, 0x15, 0xeb, 0xd7, 0x5f, 0xac,
0xcc, 0x7c, 0xf2, 0xc5, 0xca, 0xcc, 0xa7, 0x5f, 0xac, 0xcc, 0x1c, 0xe7, 0xf1, 0x9f, 0x88, 0x5e,
0xfe, 0x4f, 0x00, 0x00, 0x00, 0xff, 0xff, 0x64, 0x7a, 0x8f, 0x66, 0xc5, 0x24, 0x00, 0x00,
}
// Reference imports to suppress errors if they are not otherwise used.
@@ -7019,16 +7003,6 @@ func (m *QueryRangeRequest) MarshalToSizedBuffer(dAtA []byte) (int, error) {
i--
dAtA[i] = 0x3a
}
if m.ShardCount != 0 {
i = encodeVarintTempo(dAtA, i, uint64(m.ShardCount))
i--
dAtA[i] = 0x30
}
if m.ShardID != 0 {
i = encodeVarintTempo(dAtA, i, uint64(m.ShardID))
i--
dAtA[i] = 0x28
}
if m.Step != 0 {
i = encodeVarintTempo(dAtA, i, uint64(m.Step))
i--
@@ -8305,12 +8279,6 @@ func (m *QueryRangeRequest) Size() (n int) {
if m.Step != 0 {
n += 1 + sovTempo(uint64(m.Step))
}
if m.ShardID != 0 {
n += 1 + sovTempo(uint64(m.ShardID))
}
if m.ShardCount != 0 {
n += 1 + sovTempo(uint64(m.ShardCount))
}
l = len(m.QueryMode)
if l > 0 {
n += 1 + l + sovTempo(uint64(l))
@@ -15092,44 +15060,6 @@ func (m *QueryRangeRequest) Unmarshal(dAtA []byte) error {
break
}
}
case 5:
if wireType != 0 {
return fmt.Errorf("proto: wrong wireType = %d for field ShardID", wireType)
}
m.ShardID = 0
for shift := uint(0); ; shift += 7 {
if shift >= 64 {
return ErrIntOverflowTempo
}
if iNdEx >= l {
return io.ErrUnexpectedEOF
}
b := dAtA[iNdEx]
iNdEx++
m.ShardID |= uint32(b&0x7F) << shift
if b < 0x80 {
break
}
}
case 6:
if wireType != 0 {
return fmt.Errorf("proto: wrong wireType = %d for field ShardCount", wireType)
}
m.ShardCount = 0
for shift := uint(0); ; shift += 7 {
if shift >= 64 {
return ErrIntOverflowTempo
}
if iNdEx >= l {
return io.ErrUnexpectedEOF
}
b := dAtA[iNdEx]
iNdEx++
m.ShardCount |= uint32(b&0x7F) << shift
if b < 0x80 {
break
}
}
case 7:
if wireType != 2 {
return fmt.Errorf("proto: wrong wireType = %d for field QueryMode", wireType)
+2 -2
View File
@@ -388,8 +388,8 @@ message QueryRangeRequest {
uint64 start = 2;
uint64 end = 3;
uint64 step = 4;
uint32 shardID = 5;
uint32 shardCount = 6;
//uint32 shardID = 5; // removed
//uint32 shardCount = 6; // removed
string queryMode = 7;
// New RF1 fields
string blockID = 8;
+1 -89
View File
@@ -2,11 +2,8 @@ package traceql
import (
"context"
"encoding/binary"
"errors"
"fmt"
"hash"
"hash/fnv"
"math"
"sort"
"sync"
@@ -728,7 +725,7 @@ func (e *Engine) CompileMetricsQueryRangeNonRaw(req *tempopb.QueryRangeRequest,
// Dedupe spans parameter is an indicator of whether to expect duplicates in the datasource. For
// example if the datasource is replication factor=1 or only a single block then we know there
// aren't duplicates, and we can make some optimizations.
func (e *Engine) CompileMetricsQueryRange(req *tempopb.QueryRangeRequest, dedupeSpans bool, exemplars int, timeOverlapCutoff float64, allowUnsafeQueryHints bool) (*MetricsEvalulator, error) {
func (e *Engine) CompileMetricsQueryRange(req *tempopb.QueryRangeRequest, exemplars int, timeOverlapCutoff float64, allowUnsafeQueryHints bool) (*MetricsEvalulator, error) {
if req.Start <= 0 {
return nil, fmt.Errorf("start required")
}
@@ -751,10 +748,6 @@ func (e *Engine) CompileMetricsQueryRange(req *tempopb.QueryRangeRequest, dedupe
return nil, fmt.Errorf("not a metrics query")
}
if v, ok := expr.Hints.GetBool(HintDedupe, allowUnsafeQueryHints); ok {
dedupeSpans = v
}
if v, ok := expr.Hints.GetInt(HintExemplars, allowUnsafeQueryHints); ok {
exemplars = v
}
@@ -765,31 +758,11 @@ func (e *Engine) CompileMetricsQueryRange(req *tempopb.QueryRangeRequest, dedupe
me := &MetricsEvalulator{
storageReq: storageReq,
metricsPipeline: metricsPipeline,
dedupeSpans: dedupeSpans,
timeOverlapCutoff: timeOverlapCutoff,
maxExemplars: exemplars,
exemplarMap: make(map[string]struct{}, exemplars), // TODO: Lazy, use bloom filter, CM sketch or something
}
// TraceID (optional)
if req.ShardCount > 1 {
// For sharding it must be in the first pass so that we only evalulate our traces.
storageReq.ShardID = req.ShardID
storageReq.ShardCount = req.ShardCount
if !storageReq.HasAttribute(IntrinsicTraceIDAttribute) {
storageReq.Conditions = append(storageReq.Conditions, Condition{Attribute: IntrinsicTraceIDAttribute})
}
}
if dedupeSpans {
// For dedupe we only need the trace ID on matching spans, so it can go in the second pass.
// This is a no-op if we are already sharding and it's in the first pass.
// Finally, this is often optimized back to the first pass when it lets us avoid a second pass altogether.
if !storageReq.HasAttribute(IntrinsicTraceIDAttribute) {
storageReq.SecondPassConditions = append(storageReq.SecondPassConditions, Condition{Attribute: IntrinsicTraceIDAttribute})
}
}
// Span start time (always required)
if !storageReq.HasAttribute(IntrinsicSpanStartTimeAttribute) {
// Technically we only need the start time of matching spans, so we add it to the second pass.
@@ -905,8 +878,6 @@ func lookup(needles []Attribute, haystack Span) Static {
type MetricsEvalulator struct {
start, end uint64
checkTime bool
dedupeSpans bool
deduper *SpanDeduper2
maxExemplars, exemplarCount int
exemplarMap map[string]struct{}
timeOverlapCutoff float64
@@ -962,10 +933,6 @@ func (e *MetricsEvalulator) Do(ctx context.Context, f SpansetFetcher, fetcherSta
return err
}
if e.dedupeSpans && e.deduper == nil {
e.deduper = NewSpanDeduper2()
}
defer fetch.Results.Close()
for {
@@ -987,11 +954,6 @@ func (e *MetricsEvalulator) Do(ctx context.Context, f SpansetFetcher, fetcherSta
}
}
if e.dedupeSpans && e.deduper.Skip(ss.TraceID, s.StartTimeUnixNanos()) {
e.spansDeduped++
continue
}
e.spansTotal++
e.metricsPipeline.observe(s)
@@ -1042,56 +1004,6 @@ func (e *MetricsEvalulator) sampleExemplar(id []byte) bool {
return true
}
// SpanDeduper2 is EXTREMELY LAZY. It attempts to dedupe spans for metrics
// without requiring any new data fields. It uses trace ID and span start time
// which are already loaded. This of course terrible, but did I mention that
// this is extremely lazy? Additionally it uses sharded maps by the lowest byte
// of the trace ID to reduce the pressure on any single map. Maybe it's good enough. Let's find out!
type SpanDeduper2 struct {
m []map[uint32]struct{}
h hash.Hash32
buf []byte
traceID Attribute
}
func NewSpanDeduper2() *SpanDeduper2 {
maps := make([]map[uint32]struct{}, 256)
for i := range maps {
maps[i] = make(map[uint32]struct{}, 1000)
}
return &SpanDeduper2{
m: maps,
h: fnv.New32a(),
buf: make([]byte, 8),
traceID: NewIntrinsic(IntrinsicTraceID),
}
}
func (d *SpanDeduper2) Skip(tid []byte, startTime uint64) bool {
d.h.Reset()
d.h.Write(tid)
binary.BigEndian.PutUint64(d.buf, startTime)
d.h.Write(d.buf)
v := d.h.Sum32()
// Use last byte of the trace to choose the submap.
// Empty ID uses submap 0.
mapIdx := byte(0)
if len(tid) > 0 {
mapIdx = tid[len(tid)-1]
}
m := d.m[mapIdx]
if _, ok := m[v]; ok {
return true
}
m[v] = struct{}{}
return false
}
// MetricsFrontendEvaluator pipes the sharded job results back into the engine for the rest
// of the pipeline. i.e. This evaluator is for the query-frontend.
type MetricsFrontendEvaluator struct {
+11 -57
View File
@@ -219,7 +219,7 @@ func TestCompileMetricsQueryRange(t *testing.T) {
Start: c.start,
End: c.end,
Step: c.step,
}, false, 0, 0, false)
}, 0, 0, false)
if c.expectedErr != nil {
require.EqualError(t, err, c.expectedErr.Error())
@@ -231,9 +231,6 @@ func TestCompileMetricsQueryRange(t *testing.T) {
func TestCompileMetricsQueryRangeFetchSpansRequest(t *testing.T) {
tc := map[string]struct {
q string
shardID uint32
shardCount uint32
dedupe bool
expectedReq FetchSpansRequest
}{
"minimal": {
@@ -249,37 +246,26 @@ func TestCompileMetricsQueryRangeFetchSpansRequest(t *testing.T) {
},
},
"dedupe": {
q: "{} | rate()",
dedupe: true,
q: "{} | rate()",
expectedReq: FetchSpansRequest{
AllConditions: true,
Conditions: []Condition{
{
Attribute: IntrinsicSpanStartTimeAttribute,
},
{
Attribute: IntrinsicTraceIDAttribute, // Required for dedupe
},
},
},
},
"secondPass": {
q: "{duration > 10s} | rate() by (resource.cluster)",
shardID: 123,
shardCount: 456,
q: "{duration > 10s} | rate() by (resource.cluster)",
expectedReq: FetchSpansRequest{
AllConditions: true,
ShardID: 123,
ShardCount: 456,
Conditions: []Condition{
{
Attribute: IntrinsicDurationAttribute,
Op: OpGreater,
Operands: Operands{NewStaticDuration(10 * time.Second)},
},
{
Attribute: IntrinsicTraceIDAttribute, // Required for sharding
},
},
SecondPassConditions: []Condition{
{
@@ -294,22 +280,15 @@ func TestCompileMetricsQueryRangeFetchSpansRequest(t *testing.T) {
},
},
"optimizations": {
q: "{duration > 10s} | rate() by (name, resource.service.name)",
shardID: 123,
shardCount: 456,
q: "{duration > 10s} | rate() by (name, resource.service.name)",
expectedReq: FetchSpansRequest{
AllConditions: true,
ShardID: 123,
ShardCount: 456,
Conditions: []Condition{
{
Attribute: IntrinsicDurationAttribute,
Op: OpGreater,
Operands: Operands{NewStaticDuration(10 * time.Second)},
},
{
Attribute: IntrinsicTraceIDAttribute, // Required for sharding
},
{
// Intrinsic moved to first pass
Attribute: IntrinsicNameAttribute,
@@ -329,13 +308,11 @@ func TestCompileMetricsQueryRangeFetchSpansRequest(t *testing.T) {
for n, tc := range tc {
t.Run(n, func(t *testing.T) {
eval, err := NewEngine().CompileMetricsQueryRange(&tempopb.QueryRangeRequest{
Query: tc.q,
ShardID: tc.shardID,
ShardCount: tc.shardCount,
Start: 1,
End: 2,
Step: 3,
}, tc.dedupe, 0, 0, false)
Query: tc.q,
Start: 1,
End: 2,
Step: 3,
}, 0, 0, false)
require.NoError(t, err)
// Nil out func to Equal works
@@ -438,7 +415,7 @@ func TestQuantileOverTime(t *testing.T) {
}
// 3 layers of processing matches: query-frontend -> queriers -> generators -> blocks
layer1, err := e.CompileMetricsQueryRange(req, false, 0, 0, false)
layer1, err := e.CompileMetricsQueryRange(req, 0, 0, false)
require.NoError(t, err)
layer2, err := e.CompileMetricsQueryRangeNonRaw(req, AggregateModeSum)
@@ -545,7 +522,7 @@ func TestHistogramOverTime(t *testing.T) {
}
// 3 layers of processing matches: query-frontend -> queriers -> generators -> blocks
layer1, err := e.CompileMetricsQueryRange(req, false, 0, 0, false)
layer1, err := e.CompileMetricsQueryRange(req, 0, 0, false)
require.NoError(t, err)
layer2, err := e.CompileMetricsQueryRangeNonRaw(req, AggregateModeSum)
@@ -574,26 +551,3 @@ func TestHistogramOverTime(t *testing.T) {
final := layer3.Results()
require.Equal(t, out, final)
}
func TestSpanDeduper(t *testing.T) {
d := NewSpanDeduper2()
in := []struct {
tid []byte
ts uint64
}{
{nil, 0},
{[]byte{1}, 1},
{[]byte{1, 1}, 1},
{[]byte{1, 2}, 2},
}
for _, tc := range in {
// First call is always false
require.False(t, d.Skip(tc.tid, tc.ts))
// Second call is always true
require.True(t, d.Skip(tc.tid, tc.ts))
}
d.Skip(nil, 0)
}
-2
View File
@@ -8,8 +8,6 @@ import (
// and not part of the language or engine, we organize them here in one place.
const (
HintSample = "sample"
HintDedupe = "dedupe"
HintJobInterval = "job_interval"
HintJobSize = "job_size"
HintTimeOverlapCutoff = "time_overlap_cutoff"
HintConcurrentBlocks = "concurrent_blocks"
-4
View File
@@ -86,10 +86,6 @@ type FetchSpansRequest struct {
EndTimeUnixNanos uint64
Conditions []Condition
// mdisibio - Better to push trace by ID filtering into Conditions with a new between op?
ShardID uint32
ShardCount uint32
// Hints
// By default, the storage layer fetches spans meeting any of the criteria.
-136
View File
@@ -1,136 +0,0 @@
package traceidboundary
import (
"bytes"
"encoding/binary"
"math"
)
type Boundary struct {
Min, Max []byte
}
// Pairs returns the boundaries that match trace IDs in that shard. Internally this is
// similar to how queriers divide the block ID-space, but here it's trace IDs instead.
// The inputs are 1-based because it seems more readable: shard 1 of 10. Most boundaries
// are [,) lower inclusive, upper exclusive. However the last boundary that ends in the
// max value 0xFFFF... is [,] inclusive/inclusive and indicated when the return value
// upperInclusive is set.
// Of course there are some caveats:
// - Trace IDs can be 16 or 8 bytes. If we naively sharded only in 16-byte space it would
// be unbalanced because all 8-byte IDs would land in the first shard. Therefore we
// divide in both 16- and 8-byte spaces and a single shard covers a range in each.
// - There are different regimes of 8-byte ID generation and they are not uniformly
// distributed. So there are several sub-shards within 8-byte space to compensate.
func Pairs(shard, of uint32) (boundaries []Boundary, upperInclusive bool) {
boundaries = append(boundaries, complicatedShardingFor8ByteIDs(shard, of)...)
// Final pair is the normal full precision 16-byte IDs.
b := bounds(of, 0, math.MaxUint64, 0)
// Avoid overlap with the 64-bit precision ones. I.e. a minimum of 0x0000.... would
// unintentionally include all 64-bit IDs. The first 65-bit ID starts here:
b[0] = []byte{0, 0, 0, 0, 0, 0, 0, 0x01, 0, 0, 0, 0, 0, 0, 0, 0}
// Adjust max to be full 16-byte max
b[of] = []byte{0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF}
boundaries = append(boundaries, Boundary{
Min: b[shard-1],
Max: b[shard],
})
// Top most 0xFFFFF... boundary is inclusive
upperInclusive = shard == of
return
}
// Funcs returns helper functions that match trace IDs in the given shard.
func Funcs(shard, of uint32) (testSingle func([]byte) bool, testRange func([]byte, []byte) bool) {
pairs, upperInclusive := Pairs(shard, of)
upper := -1
if upperInclusive {
upper = 0
}
isMatch := func(id []byte) bool {
for _, p := range pairs {
if bytes.Compare(p.Min, id) <= 0 && bytes.Compare(id, p.Max) <= upper {
return true
}
}
return false
}
withinRange := func(min []byte, max []byte) bool {
for _, p := range pairs {
if bytes.Compare(p.Min, max) <= 0 && bytes.Compare(min, p.Max) <= upper {
return true
}
}
return false
}
return isMatch, withinRange
}
// complicatedShardingFor8ByteIDs generates a list of trace ID boundaries that is subdividing
// the 64-bit space. Not optimal or universal rules. This seems like overkill but in practice
// 8-byte IDs are unevenly weighted towards lower values starting with zeros. The benefit of
// this approach is better fairness across shards, and also *invariance* across workloads,
// no matter if your instrumentation is generating 8-byte or 16-byte trace IDs.
func complicatedShardingFor8ByteIDs(shard, of uint32) []Boundary {
var results []Boundary
regions := []struct {
min, max uint64
}{
{0x0000000000000000, 0x00FFFFFFFFFFFFFF}, // Region with upper byte = 0
{0x0100000000000000, 0x0FFFFFFFFFFFFFFF}, // Region with upper nibble = 0
{0x1000000000000000, 0x7FFFFFFFFFFFFFFF}, // Region for 63-bit IDs (upper bit = 0)
{0x8000000000000000, 0xFFFFFFFFFFFFFFFF}, // Region for true 64-bit IDs
}
for _, r := range regions {
b := bounds(of, r.min, r.max, 8)
results = append(results, Boundary{
Min: b[shard-1],
Max: b[shard],
})
}
return results
}
func bounds(shards uint32, min, max uint64, dest int) [][]byte {
if shards == 0 {
return nil
}
bounds := make([][]byte, shards+1)
for i := uint32(0); i < shards+1; i++ {
bounds[i] = make([]byte, 16)
}
bucketSz := (max - min) / uint64(shards)
// numLarger is the number of buckets that have to be bumped by 1
numLarger := max % bucketSz
boundary := min
for i := uint32(0); i < shards; i++ {
binary.BigEndian.PutUint64(bounds[i][dest:], boundary)
boundary += bucketSz
if numLarger != 0 {
numLarger--
boundary++
}
}
binary.BigEndian.PutUint64(bounds[shards][dest:], max)
return bounds
}
@@ -1,82 +0,0 @@
package traceidboundary
import (
"fmt"
"testing"
"github.com/stretchr/testify/require"
)
func TestPairs(t *testing.T) {
testCases := []struct {
shard, of uint32
expectedPairs []Boundary
expectedUpper bool
}{
//---------------------------------------------
// Simplest case, 2 shards
//---------------------------------------------
{1, 2, []Boundary{
{
// First half of upper-byte = 0 (between 0x0000 and 0x00FF)
[]byte{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0},
[]byte{0, 0, 0, 0, 0, 0, 0, 0, 0, 0x80, 0, 0, 0, 0, 0, 0},
},
{
// First half of upper-nibble = 0 (between 0x01 and 0x0F)
[]byte{0, 0, 0, 0, 0, 0, 0, 0, 0x01, 0, 0, 0, 0, 0, 0, 0},
[]byte{0, 0, 0, 0, 0, 0, 0, 0, 0x08, 0x80, 0, 0, 0, 0, 0, 0},
},
{
// First half of upper-bit = 0 (between 0x10 and 0x7F)
[]byte{0, 0, 0, 0, 0, 0, 0, 0, 0x10, 0, 0, 0, 0, 0, 0, 0},
[]byte{0, 0, 0, 0, 0, 0, 0, 0, 0x48, 0, 0, 0, 0, 0, 0, 0},
},
{
// First half of full 8-byte ids (between 0x80 and 0xFF)
[]byte{0, 0, 0, 0, 0, 0, 0, 0, 0x80, 0, 0, 0, 0, 0, 0, 0},
[]byte{0, 0, 0, 0, 0, 0, 0, 0, 0xC0, 0, 0, 0, 0, 0, 0, 0},
},
{
// First half of full 16-byte ids (between 0x00 and 0x80)
[]byte{0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0},
[]byte{0x80, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0},
},
}, false},
{2, 2, []Boundary{
{
// Second half of upper-byte = 0 (between 0x0000 and 0x00FF)
[]byte{0, 0, 0, 0, 0, 0, 0, 0, 0, 0x80, 0, 0, 0, 0, 0, 0},
[]byte{0, 0, 0, 0, 0, 0, 0, 0, 0, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF},
},
{
// Second half of upper-nibble = 0 (between 0x01 and 0x0F)
[]byte{0, 0, 0, 0, 0, 0, 0, 0, 0x08, 0x80, 0, 0, 0, 0, 0, 0},
[]byte{0, 0, 0, 0, 0, 0, 0, 0, 0x0F, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF},
},
{
// Second half of upper-bit = 0 (between 0x10 and 0x7F)
[]byte{0, 0, 0, 0, 0, 0, 0, 0, 0x48, 0, 0, 0, 0, 0, 0, 0},
[]byte{0, 0, 0, 0, 0, 0, 0, 0, 0x7F, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF},
},
{
// Second half of full 8-byte ids (between 0x80 and 0xFF)
[]byte{0, 0, 0, 0, 0, 0, 0, 0, 0xC0, 0, 0, 0, 0, 0, 0, 0},
[]byte{0, 0, 0, 0, 0, 0, 0, 0, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF},
},
{
// Second half of full 16-byte ids (between 0x80 and 0xFF)
[]byte{0x80, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0},
[]byte{0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF},
},
}, true},
}
for _, tc := range testCases {
t.Run(fmt.Sprintf("%d of %d", tc.shard, tc.of), func(t *testing.T) {
pairs, upper := Pairs(tc.shard, tc.of)
require.Equal(t, tc.expectedPairs, pairs)
require.Equal(t, tc.expectedUpper, upper)
})
}
}
+5 -19
View File
@@ -19,7 +19,6 @@ import (
v1 "github.com/grafana/tempo/pkg/tempopb/trace/v1"
"github.com/grafana/tempo/pkg/traceql"
"github.com/grafana/tempo/pkg/util"
"github.com/grafana/tempo/pkg/util/traceidboundary"
"github.com/grafana/tempo/tempodb/encoding/common"
)
@@ -1200,7 +1199,7 @@ func (i *mergeSpansetIterator) Close() {
// V
func fetch(ctx context.Context, req traceql.FetchSpansRequest, pf *parquet.File, rowGroups []parquet.RowGroup) (*spansetIterator, error) {
iter, err := createAllIterator(ctx, nil, req.Conditions, req.AllConditions, req.StartTimeUnixNanos, req.EndTimeUnixNanos, req.ShardID, req.ShardCount, rowGroups, pf)
iter, err := createAllIterator(ctx, nil, req.Conditions, req.AllConditions, req.StartTimeUnixNanos, req.EndTimeUnixNanos, rowGroups, pf)
if err != nil {
return nil, fmt.Errorf("error creating iterator: %w", err)
}
@@ -1208,7 +1207,7 @@ func fetch(ctx context.Context, req traceql.FetchSpansRequest, pf *parquet.File,
if req.SecondPass != nil {
iter = newBridgeIterator(newRebatchIterator(iter), req.SecondPass)
iter, err = createAllIterator(ctx, iter, req.SecondPassConditions, false, 0, 0, req.ShardID, req.ShardCount, rowGroups, pf)
iter, err = createAllIterator(ctx, iter, req.SecondPassConditions, false, 0, 0, rowGroups, pf)
if err != nil {
return nil, fmt.Errorf("error creating second pass iterator: %w", err)
}
@@ -1218,7 +1217,7 @@ func fetch(ctx context.Context, req traceql.FetchSpansRequest, pf *parquet.File,
}
func createAllIterator(ctx context.Context, primaryIter parquetquery.Iterator, conds []traceql.Condition, allConditions bool, start, end uint64,
shardID, shardCount uint32, rgs []parquet.RowGroup, pf *parquet.File,
rgs []parquet.RowGroup, pf *parquet.File,
) (parquetquery.Iterator, error) {
// Categorize conditions into span-level or resource-level
var (
@@ -1292,7 +1291,7 @@ func createAllIterator(ctx context.Context, primaryIter parquetquery.Iterator, c
return nil, fmt.Errorf("creating resource iterator: %w", err)
}
return createTraceIterator(makeIter, resourceIter, traceConditions, start, end, shardID, shardCount, allConditions)
return createTraceIterator(makeIter, resourceIter, traceConditions, start, end, allConditions)
}
// createSpanIterator iterates through all span-level columns, groups them into rows representing
@@ -1560,7 +1559,7 @@ func createResourceIterator(makeIter makeIterFn, spanIterator parquetquery.Itera
required, iters, batchCol)
}
func createTraceIterator(makeIter makeIterFn, resourceIter parquetquery.Iterator, conds []traceql.Condition, start, end uint64, shardID, shardCount uint32, allConditions bool) (parquetquery.Iterator, error) {
func createTraceIterator(makeIter makeIterFn, resourceIter parquetquery.Iterator, conds []traceql.Condition, start, end uint64, allConditions bool) (parquetquery.Iterator, error) {
iters := make([]parquetquery.Iterator, 0, 3)
// add conditional iterators first. this way if someone searches for { traceDuration > 1s && span.foo = "bar"} the query will
@@ -2276,16 +2275,3 @@ func orIfNeeded(preds []parquetquery.Predicate) parquetquery.Predicate {
return parquetquery.NewOrPredicate(preds...)
}
}
// NewTraceIDShardingPredicate creates a predicate for the TraceID column to match only IDs
// within the shard. If sharding isn't present, returns nil meaning no predicate.
func NewTraceIDShardingPredicate(shardID, shardCount uint32) parquetquery.Predicate {
if shardCount <= 1 || shardID <= 0 {
return nil
}
isMatch, withinRange := traceidboundary.Funcs(shardID, shardCount)
extract := func(v parquet.Value) []byte { return v.ByteArray() }
return parquetquery.NewGenericPredicate(isMatch, withinRange, extract)
}
+6 -79
View File
@@ -13,15 +13,12 @@ import (
"time"
"unsafe"
"github.com/opentracing/opentracing-go"
"github.com/parquet-go/parquet-go"
"github.com/grafana/tempo/pkg/cache"
"github.com/grafana/tempo/pkg/parquetquery"
v1 "github.com/grafana/tempo/pkg/tempopb/trace/v1"
"github.com/grafana/tempo/pkg/traceql"
"github.com/grafana/tempo/pkg/util"
"github.com/grafana/tempo/pkg/util/traceidboundary"
"github.com/grafana/tempo/tempodb/backend"
"github.com/grafana/tempo/tempodb/encoding/common"
)
@@ -893,15 +890,7 @@ func (b *backendBlock) Fetch(ctx context.Context, req traceql.FetchSpansRequest,
return traceql.FetchSpansResponse{}, err
}
var rgs []parquet.RowGroup
if req.ShardCount > 0 {
rgs, err = b.rowGroupsForShard(ctx, pf, *b.meta, req.ShardID, req.ShardCount)
if err != nil {
return traceql.FetchSpansResponse{}, err
}
} else {
rgs = rowGroupsFromFile(pf, opts)
}
rgs := rowGroupsFromFile(pf, opts)
iter, err := fetch(ctx, req, pf, rgs, b.meta.DedicatedColumns)
if err != nil {
@@ -1350,7 +1339,7 @@ func (i *mergeSpansetIterator) Close() {
// V
func fetch(ctx context.Context, req traceql.FetchSpansRequest, pf *parquet.File, rowGroups []parquet.RowGroup, dc backend.DedicatedColumns) (*spansetIterator, error) {
iter, err := createAllIterator(ctx, nil, req.Conditions, req.AllConditions, req.StartTimeUnixNanos, req.EndTimeUnixNanos, req.ShardID, req.ShardCount, rowGroups, pf, dc, false)
iter, err := createAllIterator(ctx, nil, req.Conditions, req.AllConditions, req.StartTimeUnixNanos, req.EndTimeUnixNanos, rowGroups, pf, dc, false)
if err != nil {
return nil, fmt.Errorf("error creating iterator: %w", err)
}
@@ -1358,7 +1347,7 @@ func fetch(ctx context.Context, req traceql.FetchSpansRequest, pf *parquet.File,
if req.SecondPass != nil {
iter = newBridgeIterator(newRebatchIterator(iter), req.SecondPass)
iter, err = createAllIterator(ctx, iter, req.SecondPassConditions, false, 0, 0, req.ShardID, req.ShardCount, rowGroups, pf, dc, req.SecondPassSelectAll)
iter, err = createAllIterator(ctx, iter, req.SecondPassConditions, false, 0, 0, rowGroups, pf, dc, req.SecondPassSelectAll)
if err != nil {
return nil, fmt.Errorf("error creating second pass iterator: %w", err)
}
@@ -1406,7 +1395,7 @@ func categorizeConditions(conditions []traceql.Condition) (mingled bool, spanCon
}
func createAllIterator(ctx context.Context, primaryIter parquetquery.Iterator, conds []traceql.Condition, allConditions bool, start, end uint64,
shardID, shardCount uint32, rgs []parquet.RowGroup, pf *parquet.File, dc backend.DedicatedColumns, selectAll bool,
rgs []parquet.RowGroup, pf *parquet.File, dc backend.DedicatedColumns, selectAll bool,
) (parquetquery.Iterator, error) {
// categorizeConditions conditions into span-level or resource-level
mingledConditions, spanConditions, resourceConditions, traceConditions, err := categorizeConditions(conds)
@@ -1445,7 +1434,7 @@ func createAllIterator(ctx context.Context, primaryIter parquetquery.Iterator, c
return nil, fmt.Errorf("creating resource iterator: %w", err)
}
return createTraceIterator(makeIter, resourceIter, traceConditions, start, end, shardID, shardCount, allConditions, selectAll)
return createTraceIterator(makeIter, resourceIter, traceConditions, start, end, allConditions, selectAll)
}
// createSpanIterator iterates through all span-level columns, groups them into rows representing
@@ -1858,7 +1847,7 @@ func createResourceIterator(makeIter makeIterFn, spanIterator parquetquery.Itera
return parquetquery.NewLeftJoinIterator(DefinitionLevelResourceSpans, required, iters, batchCol, parquetquery.WithPool(pqSpansetPool))
}
func createTraceIterator(makeIter makeIterFn, resourceIter parquetquery.Iterator, conds []traceql.Condition, start, end uint64, _, _ uint32, allConditions bool, selectAll bool) (parquetquery.Iterator, error) {
func createTraceIterator(makeIter makeIterFn, resourceIter parquetquery.Iterator, conds []traceql.Condition, start, end uint64, allConditions bool, selectAll bool) (parquetquery.Iterator, error) {
iters := make([]parquetquery.Iterator, 0, 3)
// add conditional iterators first. this way if someone searches for { traceDuration > 1s && span.foo = "bar"} the query will
@@ -2610,65 +2599,3 @@ func unsafeToString(b []byte) string {
}
return unsafe.String(unsafe.SliceData(b), len(b))
}
// NewTraceIDShardingPredicate creates a predicate for the TraceID column to match only IDs
// within the shard. If sharding isn't present, returns nil meaning no predicate.
func NewTraceIDShardingPredicate(shardID, shardCount uint32) parquetquery.Predicate {
if shardCount <= 1 || shardID <= 0 {
return nil
}
isMatch, withinRange := traceidboundary.Funcs(shardID, shardCount)
extract := func(v parquet.Value) []byte { return v.ByteArray() }
return parquetquery.NewGenericPredicate(isMatch, withinRange, extract)
}
// rowGroupsForShard uses the block trace ID index to more efficiently find the row
// groups that contain trace IDs within the given shard. Reading the trace ID index
// is a single read and typically comes from cache. Without this we have to test every
// row group in the file which would be N reads.
func (b *backendBlock) rowGroupsForShard(ctx context.Context, pf *parquet.File, m backend.BlockMeta, shardID, shardCount uint32) ([]parquet.RowGroup, error) {
span, _ := opentracing.StartSpanFromContext(ctx, "parquet.rowGroupsForShard")
defer span.Finish()
cacheInfo := &backend.CacheInfo{
Meta: &m,
Role: cache.RoleTraceIDIdx,
}
indexBytes, err := b.r.Read(ctx, common.NameIndex, b.meta.BlockID, b.meta.TenantID, cacheInfo)
if errors.Is(err, backend.ErrDoesNotExist) {
// No index, check all groups
return pf.RowGroups(), nil
}
if err != nil {
return nil, err
}
index, err := unmarshalIndex(indexBytes)
if err != nil {
return nil, fmt.Errorf("error parsing index (%s, %s): %w", b.meta.TenantID, b.meta.BlockID, err)
}
_, testRange := traceidboundary.Funcs(shardID, shardCount)
rgs := pf.RowGroups()
matches := []parquet.RowGroup{}
for i := 0; i < len(index.RowGroups); i++ {
if i == 0 {
if testRange([]byte{}, index.RowGroups[i]) {
matches = append(matches, rgs[i])
}
} else {
if testRange(index.RowGroups[i-1], index.RowGroups[i]) {
matches = append(matches, rgs[i])
}
}
}
span.SetTag("totalRowGroups", len(rgs))
span.SetTag("matchedRowGroups", len(matches))
return matches, nil
}
@@ -4,9 +4,7 @@ import (
"bytes"
"context"
"fmt"
"math"
"math/rand"
"os"
"path"
"strconv"
"testing"
@@ -21,7 +19,6 @@ import (
"github.com/grafana/tempo/pkg/traceql"
"github.com/grafana/tempo/pkg/traceqlmetrics"
"github.com/grafana/tempo/pkg/util/test"
"github.com/grafana/tempo/pkg/util/traceidboundary"
"github.com/grafana/tempo/tempodb/backend"
"github.com/grafana/tempo/tempodb/backend/local"
"github.com/grafana/tempo/tempodb/encoding/common"
@@ -756,15 +753,13 @@ func BenchmarkBackendBlockQueryRange(b *testing.B) {
}
req := &tempopb.QueryRangeRequest{
Query: tc,
Step: uint64(time.Minute),
Start: uint64(st.UnixNano()),
End: uint64(end.UnixNano()),
ShardID: 30,
ShardCount: 65,
Query: tc,
Step: uint64(time.Minute),
Start: uint64(st.UnixNano()),
End: uint64(end.UnixNano()),
}
eval, err := e.CompileMetricsQueryRange(req, false, 4, 0, false)
eval, err := e.CompileMetricsQueryRange(req, 4, 0, false)
require.NoError(b, err)
b.ResetTimer()
@@ -783,124 +778,6 @@ func BenchmarkBackendBlockQueryRange(b *testing.B) {
}
}
func TestTraceIDShardingQuality(t *testing.T) {
// Use debug=1 go test -v -run=TestTraceIDShardingQuality
if os.Getenv("debug") != "1" {
t.Skip()
}
var (
ctx = context.TODO()
opts = common.DefaultSearchOptions()
tenantID = "1"
// blockID = uuid.MustParse("06ebd383-8d4e-4289-b0e9-cf2197d611d5")
blockID = uuid.MustParse("18364616-f80d-45a6-b2a3-cb63e203edff")
path = "/Users/marty/src/tmp/"
)
r, _, _, err := local.New(&local.Config{
Path: path,
})
require.NoError(t, err)
rr := backend.NewReader(r)
meta, err := rr.BlockMeta(ctx, blockID, tenantID)
require.NoError(t, err)
require.Equal(t, VersionString, meta.Version)
block := newBackendBlock(meta, rr)
pf, _, err := block.openForSearch(ctx, opts)
require.NoError(t, err)
fetchReq := traceql.FetchSpansRequest{
AllConditions: true,
Conditions: []traceql.Condition{
{Attribute: traceql.IntrinsicTraceIDAttribute},
},
}
f := traceql.NewSpansetFetcherWrapper(func(ctx context.Context, req traceql.FetchSpansRequest) (traceql.FetchSpansResponse, error) {
return block.Fetch(ctx, req, opts)
})
summarizeCounts := func(prefix string, cs []int) {
count := len(cs)
sum := 0
l := math.MaxInt
h := math.MinInt
for _, v := range cs {
sum += v
l = min(l, v)
h = max(h, v)
}
fmt.Printf("Shards:%d %s: Min:%d Max:%d Total:%d Avg:%.1f Quality:%.1f %%\n",
count, prefix, l, h, sum, float64(sum)/float64(count), float64(l)/float64(h)*100.0)
}
shardsToTest := []int{10, 100, 1000}
for _, shards := range shardsToTest {
t.Run(strconv.Itoa(shards), func(t *testing.T) {
var (
rgCounts = make([]int, shards)
trCounts = make([]int, shards)
pairs = make([][]traceidboundary.Boundary, shards)
funcs = make([]func([]byte) bool, shards)
)
for s := 1; s <= shards; s++ {
pairs[s-1], _ = traceidboundary.Pairs(uint32(s), uint32(shards))
funcs[s-1], _ = traceidboundary.Funcs(uint32(s), uint32(shards))
rgs, err := block.rowGroupsForShard(ctx, pf, *meta, uint32(s), uint32(shards))
require.NoError(t, err)
rgCounts[s-1] = len(rgs)
}
resp, err := f.Fetch(ctx, fetchReq)
require.NoError(t, err)
defer resp.Results.Close()
for {
ss, err := resp.Results.Next(ctx)
require.NoError(t, err)
if ss == nil {
break
}
// Match the trace ID against every shard
matched := []int{}
for i := 0; i < shards; i++ {
if funcs[i](ss.TraceID) {
trCounts[i]++
matched = append(matched, i)
}
}
// Check for missing or overlapping ranges
if len(matched) > 1 {
fmt.Printf("TraceID %X matched %d shards\n", ss.TraceID, len(matched))
for _, s := range matched {
for i, b := range pairs[s] {
fmt.Printf(" Bucket %d %d: %X %X\n", s, i, b.Min, b.Max)
}
}
panic("trace matched multiple shards")
} else if len(matched) == 0 {
fmt.Println("TraceID not matched by any shard:", ss.TraceID)
panic("trace id not matched by any shard")
}
ss.Release()
}
summarizeCounts("Traces", trCounts)
summarizeCounts("RowGroups", rgCounts)
})
}
}
func TestDescendantOf(t *testing.T) {
ancestor1 := &span{nestedSetLeft: 3, nestedSetRight: 8}
descendant1a := &span{nestedSetLeft: 4, nestedSetRight: 5}
+1 -74
View File
@@ -13,15 +13,12 @@ import (
"time"
"unsafe"
"github.com/opentracing/opentracing-go"
"github.com/parquet-go/parquet-go"
"github.com/grafana/tempo/pkg/cache"
"github.com/grafana/tempo/pkg/parquetquery"
v1 "github.com/grafana/tempo/pkg/tempopb/trace/v1"
"github.com/grafana/tempo/pkg/traceql"
"github.com/grafana/tempo/pkg/util"
"github.com/grafana/tempo/pkg/util/traceidboundary"
"github.com/grafana/tempo/tempodb/backend"
"github.com/grafana/tempo/tempodb/encoding/common"
)
@@ -979,15 +976,7 @@ func (b *backendBlock) Fetch(ctx context.Context, req traceql.FetchSpansRequest,
return traceql.FetchSpansResponse{}, err
}
var rgs []parquet.RowGroup
if req.ShardCount > 0 {
rgs, err = b.rowGroupsForShard(ctx, pf, *b.meta, req.ShardID, req.ShardCount)
if err != nil {
return traceql.FetchSpansResponse{}, err
}
} else {
rgs = rowGroupsFromFile(pf, opts)
}
rgs := rowGroupsFromFile(pf, opts)
iter, err := fetch(ctx, req, pf, rgs, b.meta.DedicatedColumns)
if err != nil {
@@ -3118,68 +3107,6 @@ func unsafeToString(b []byte) string {
return unsafe.String(unsafe.SliceData(b), len(b))
}
// NewTraceIDShardingPredicate creates a predicate for the TraceID column to match only IDs
// within the shard. If sharding isn't present, returns nil meaning no predicate.
func NewTraceIDShardingPredicate(shardID, shardCount uint32) parquetquery.Predicate {
if shardCount <= 1 || shardID <= 0 {
return nil
}
isMatch, withinRange := traceidboundary.Funcs(shardID, shardCount)
extract := func(v parquet.Value) []byte { return v.ByteArray() }
return parquetquery.NewGenericPredicate(isMatch, withinRange, extract)
}
// rowGroupsForShard uses the block trace ID index to more efficiently find the row
// groups that contain trace IDs within the given shard. Reading the trace ID index
// is a single read and typically comes from cache. Without this we have to test every
// row group in the file which would be N reads.
func (b *backendBlock) rowGroupsForShard(ctx context.Context, pf *parquet.File, m backend.BlockMeta, shardID, shardCount uint32) ([]parquet.RowGroup, error) {
span, _ := opentracing.StartSpanFromContext(ctx, "parquet.rowGroupsForShard")
defer span.Finish()
cacheInfo := &backend.CacheInfo{
Meta: &m,
Role: cache.RoleTraceIDIdx,
}
indexBytes, err := b.r.Read(ctx, common.NameIndex, b.meta.BlockID, b.meta.TenantID, cacheInfo)
if errors.Is(err, backend.ErrDoesNotExist) {
// No index, check all groups
return pf.RowGroups(), nil
}
if err != nil {
return nil, err
}
index, err := unmarshalIndex(indexBytes)
if err != nil {
return nil, fmt.Errorf("error parsing index (%s, %s): %w", b.meta.TenantID, b.meta.BlockID, err)
}
_, testRange := traceidboundary.Funcs(shardID, shardCount)
rgs := pf.RowGroups()
matches := []parquet.RowGroup{}
for i := 0; i < len(index.RowGroups); i++ {
if i == 0 {
if testRange([]byte{}, index.RowGroups[i]) {
matches = append(matches, rgs[i])
}
} else {
if testRange(index.RowGroups[i-1], index.RowGroups[i]) {
matches = append(matches, rgs[i])
}
}
}
span.SetTag("totalRowGroups", len(rgs))
span.SetTag("matchedRowGroups", len(matches))
return matches, nil
}
func otlpStatusToTraceqlStatus(v uint64) traceql.Status {
// Map OTLP status code back to TraceQL enum.
// For other values, use the raw integer.
@@ -4,7 +4,6 @@ import (
"bytes"
"context"
"fmt"
"math"
"math/rand"
"os"
"path"
@@ -24,7 +23,6 @@ import (
"github.com/grafana/tempo/pkg/traceqlmetrics"
"github.com/grafana/tempo/pkg/util"
"github.com/grafana/tempo/pkg/util/test"
"github.com/grafana/tempo/pkg/util/traceidboundary"
"github.com/grafana/tempo/tempodb/backend"
"github.com/grafana/tempo/tempodb/backend/local"
"github.com/grafana/tempo/tempodb/encoding/common"
@@ -1075,7 +1073,7 @@ func BenchmarkBackendBlockQueryRange(b *testing.B) {
End: uint64(end.UnixNano()),
}
eval, err := e.CompileMetricsQueryRange(req, false, 2, 0, false)
eval, err := e.CompileMetricsQueryRange(req, 2, 0, false)
require.NoError(b, err)
b.ResetTimer()
@@ -1162,7 +1160,7 @@ func TestBackendBlockQueryRange(t *testing.T) {
End: uint64(end.UnixNano()),
}
eval, err := e.CompileMetricsQueryRange(req, false, 1, 0, false)
eval, err := e.CompileMetricsQueryRange(req, 1, 0, false)
require.NoError(t, err)
require.NoError(t, eval.Do(ctx, f, uint64(block.meta.StartTime.UnixNano()), uint64(block.meta.EndTime.UnixNano())))
@@ -1180,124 +1178,6 @@ func TestBackendBlockQueryRange(t *testing.T) {
}
}
func TestTraceIDShardingQuality(t *testing.T) {
// Use debug=1 go test -v -run=TestTraceIDShardingQuality
if os.Getenv("debug") != "1" {
t.Skip()
}
var (
ctx = context.TODO()
opts = common.DefaultSearchOptions()
tenantID = "1"
// blockID = uuid.MustParse("06ebd383-8d4e-4289-b0e9-cf2197d611d5")
blockID = uuid.MustParse("18364616-f80d-45a6-b2a3-cb63e203edff")
path = "/Users/marty/src/tmp/"
)
r, _, _, err := local.New(&local.Config{
Path: path,
})
require.NoError(t, err)
rr := backend.NewReader(r)
meta, err := rr.BlockMeta(ctx, blockID, tenantID)
require.NoError(t, err)
require.Equal(t, VersionString, meta.Version)
block := newBackendBlock(meta, rr)
pf, _, err := block.openForSearch(ctx, opts)
require.NoError(t, err)
fetchReq := traceql.FetchSpansRequest{
AllConditions: true,
Conditions: []traceql.Condition{
{Attribute: traceql.IntrinsicTraceIDAttribute},
},
}
f := traceql.NewSpansetFetcherWrapper(func(ctx context.Context, req traceql.FetchSpansRequest) (traceql.FetchSpansResponse, error) {
return block.Fetch(ctx, req, opts)
})
summarizeCounts := func(prefix string, cs []int) {
count := len(cs)
sum := 0
l := math.MaxInt
h := math.MinInt
for _, v := range cs {
sum += v
l = min(l, v)
h = max(h, v)
}
fmt.Printf("Shards:%d %s: Min:%d Max:%d Total:%d Avg:%.1f Quality:%.1f %%\n",
count, prefix, l, h, sum, float64(sum)/float64(count), float64(l)/float64(h)*100.0)
}
shardsToTest := []int{10, 100, 1000}
for _, shards := range shardsToTest {
t.Run(strconv.Itoa(shards), func(t *testing.T) {
var (
rgCounts = make([]int, shards)
trCounts = make([]int, shards)
pairs = make([][]traceidboundary.Boundary, shards)
funcs = make([]func([]byte) bool, shards)
)
for s := 1; s <= shards; s++ {
pairs[s-1], _ = traceidboundary.Pairs(uint32(s), uint32(shards))
funcs[s-1], _ = traceidboundary.Funcs(uint32(s), uint32(shards))
rgs, err := block.rowGroupsForShard(ctx, pf, *meta, uint32(s), uint32(shards))
require.NoError(t, err)
rgCounts[s-1] = len(rgs)
}
resp, err := f.Fetch(ctx, fetchReq)
require.NoError(t, err)
defer resp.Results.Close()
for {
ss, err := resp.Results.Next(ctx)
require.NoError(t, err)
if ss == nil {
break
}
// Match the trace ID against every shard
matched := []int{}
for i := 0; i < shards; i++ {
if funcs[i](ss.TraceID) {
trCounts[i]++
matched = append(matched, i)
}
}
// Check for missing or overlapping ranges
if len(matched) > 1 {
fmt.Printf("TraceID %X matched %d shards\n", ss.TraceID, len(matched))
for _, s := range matched {
for i, b := range pairs[s] {
fmt.Printf(" Bucket %d %d: %X %X\n", s, i, b.Min, b.Max)
}
}
panic("trace matched multiple shards")
} else if len(matched) == 0 {
fmt.Println("TraceID not matched by any shard:", ss.TraceID)
panic("trace id not matched by any shard")
}
ss.Release()
}
summarizeCounts("Traces", trCounts)
summarizeCounts("RowGroups", rgCounts)
})
}
}
func ptr[T any](v T) *T {
return &v
}