Files
traces/tempodb/backend/encoding.go
T
Zach LeslieandGitHub 884bbea6a3 Replace TenantIndex, BlockMeta and CompactedBlockMeta with proto near-equivalents (#4072)
* Update proto for v1 backend

* Functioning blocklist with new package

* Functioning blocklist with existing package

* Generate proto to existing package

* Add wrapper around google UUID for proto marshaling

* Update blocklist for wrapper type

* Test WriteTenantIndex

* Implement json for uuid

* Almost functional backend tests

* Passing backend tests

* Improve UUID coverage and utility

* TempoDB updates for UUID and struct type changes

* Update method style for uuid.New()

* Update method style for uuid.MustParse()

* Add uuid.From helper

* Update modules for UUID and proto struct types

* Update poller integration test for UUID

* Update tempo-cli for UUID and proto struct type changes

* Lint and clean up

* Fix serverless

* Fix cli

* Fix cli

* Add fixture tests for roundtrip assurance between versions

* Clean up notes from proto definition

* Update backend/test

* Be more specific about unmarshal UUID

* Include event for compacted block

* Use better json tag for compacted meta

* Implement UnmarshalJson on CompcatedBlockMeta for embedded struct

* Test for CompactedMeta marshal/unmarshal

* Drop benchmark index

* Clean commented code

* Extend mocks to capture multiple writes

* Update backend/test fixtures

* Write both proto and json to the backend for all three files in all backends

* Fix lint

* Back UUID with google.UUID to avoid struct

* Relocate pkg/uuid to tempodb/backend

* Re-plumb UUID through tempodb

* Re-plumb UUID through cmd

* Re-plumb UUID through modules

* Re-plumb UUID through integration/

* Drop package spec from tempo.proto

* Improve encoding marshal performance

* Drop block proto and tenantindex json

* Drop additional proto include

* Implement zstd tenant index compression

* Add note about use of json in the dedicatedcolumns

* Fix ingester local block

* Spell

* Revert mock update

* Drop json fallback from tenantindex read

* Replace single-tenant test fixtures

* Update fixture replacement test

* Improve error handling and fix test

* Extend test for tenantindex round-trip

* Small fixes

* Revert "Drop json fallback from tenantindex read"

This reverts commit 834ff5b85d8a8a4d7eb560fbb67ba8f3a5a3959a.

* Add back mocks to capture multiple backend writes

* Add back proto & json writitng/reading for tenantindex

* Change TotalObjects type from int32 -> int64

The original change was int -> int32, which on 64bit machines is a
downgrade that I overlooked.  Raising here to int64 to maintain the
size.

* Store write buffer at object path in mocks to improve readability in tests

* Avoid allocation in interface validation

* Update note to include intern

* Pluralize metas and compacted_metas fields

* Fallback to json only on ErrDoesNotExist

* Lint

* Drop debug

* Revert "Pluralize metas and compacted_metas fields"

This reverts commit 46a8c36e37ac33e1ffc104287d898930e02cdd97.

* Import original tenantindex and test contents for backwards compatibility

* Update changelog
2024-10-02 18:40:59 +00:00

153 lines
2.9 KiB
Go

package backend
import (
"bytes"
"encoding/json"
"fmt"
"strings"
)
// Encoding is the identifier for a chunk encoding.
type Encoding byte
// The different available encodings.
// Make sure to preserve the order, as these numeric values are written to the chunks!
const (
EncNone Encoding = iota
EncGZIP
EncLZ4_64k
EncLZ4_256k
EncLZ4_1M
EncLZ4_4M
EncSnappy
EncZstd
EncS2
)
// SupportedEncoding is a slice of all supported encodings
var SupportedEncoding = []Encoding{
EncNone,
EncGZIP,
EncLZ4_64k,
EncLZ4_256k,
EncLZ4_1M,
EncLZ4_4M,
EncSnappy,
EncZstd,
EncS2,
}
func (e Encoding) String() string {
switch e {
case EncNone:
return "none"
case EncGZIP:
return "gzip"
case EncLZ4_64k:
return "lz4-64k"
case EncLZ4_256k:
return "lz4-256k"
case EncLZ4_1M:
return "lz4-1M"
case EncLZ4_4M:
return "lz4"
case EncSnappy:
return "snappy"
case EncZstd:
return "zstd"
case EncS2:
return "s2"
default:
return "unsupported"
}
}
// UnmarshalYAML implements the Unmarshaler interface of the yaml pkg.
func (e *Encoding) UnmarshalYAML(unmarshal func(interface{}) error) error {
var encString string
err := unmarshal(&encString)
if err != nil {
return err
}
*e, err = ParseEncoding(encString)
if err != nil {
return err
}
return nil
}
// MarshalYAML implements the Marshaler interface of the yaml pkg
func (e Encoding) MarshalYAML() (interface{}, error) {
return e.String(), nil
}
// UnmarshalJSON implements the Unmarshaler interface of the json pkg.
func (e *Encoding) UnmarshalJSON(b []byte) error {
var encString string
err := json.Unmarshal(b, &encString)
if err != nil {
return err
}
*e, err = ParseEncoding(encString)
if err != nil {
return err
}
return nil
}
// MarshalJSON implements the marshaler interface of the json pkg.
func (e Encoding) MarshalJSON() ([]byte, error) {
buffer := bytes.NewBufferString("\"" + e.String() + "\"")
return buffer.Bytes(), nil
}
func (e Encoding) Marshal() ([]byte, error) {
return []byte{byte(e)}, nil
}
func (e *Encoding) MarshalTo(data []byte) (n int, err error) {
data[0] = byte(*e)
return 1, nil
}
func (e *Encoding) Unmarshal(data []byte) error {
if len(data) != 1 {
return fmt.Errorf("invalid data: %s", data)
}
x := Encoding(data[0])
*e = x
return nil
}
func (e *Encoding) Size() int {
return 1
}
// ParseEncoding parses a chunk encoding (compression algorithm) by its name.
func ParseEncoding(enc string) (Encoding, error) {
for _, e := range SupportedEncoding {
if strings.EqualFold(e.String(), enc) {
return e, nil
}
}
return 0, fmt.Errorf("invalid encoding: %s, supported: %s", enc, SupportedEncodingString())
}
// SupportedEncodingString returns the list of supported Encoding.
func SupportedEncodingString() string {
var sb strings.Builder
for i := range SupportedEncoding {
sb.WriteString(SupportedEncoding[i].String())
if i != len(SupportedEncoding)-1 {
sb.WriteString(", ")
}
}
return sb.String()
}