mirror of
https://github.com/luxfi/zapdb.git
synced 2026-07-26 22:46:36 +00:00
cmd/replicate: fold zapdb-replicate native streaming-replication tool
ZapDB has native streaming Backup(w, since) + Load(r). The standalone luxfi/zapdb-replicate repo was duplicate scaffolding. Sidecar daemon + restore init-container now live here under cmd/replicate/. Pre-fold lineage: standalone repo at luxfi/zapdb-replicate; final commit there c13a738 (deps Go 1.26.4 bump). No history was merged; small repo, clean copy.
This commit is contained in:
Hanzo AI
@@ -12,6 +12,33 @@ go build ./...
|
||||
go test ./...
|
||||
```
|
||||
|
||||
## Replicate sidecar (`cmd/replicate/`)
|
||||
|
||||
Streams ZapDB chain state to S3 (or any `hanzoai/vfs`-backed store) via
|
||||
ZapDB's native `Backup(w, lastVersion)` + `Load(r)` API. Per-block age
|
||||
encryption. One binary, three subcommands:
|
||||
|
||||
- `zapdb-replicate replicate` — daemon, runs as a sidecar next to lqd. Ticks
|
||||
every `--interval`, publishes a manifest per tick, advances `HEAD.json`.
|
||||
- `zapdb-replicate restore` — one-shot, runs as an init-container. Reads
|
||||
HEAD, applies full + increments into an empty `--path`.
|
||||
- `zapdb-replicate prune` — GC manifests + blocks older than the Nth-most-
|
||||
recent full snapshot.
|
||||
|
||||
Build:
|
||||
```bash
|
||||
go build ./cmd/replicate
|
||||
docker build -t zapdb-replicate -f deploy/replicate/Dockerfile .
|
||||
```
|
||||
|
||||
K8s wiring lives in `deploy/replicate/`: `sidecar.yaml`, `init-container.yaml`,
|
||||
plus a helm chart under `deploy/replicate/chart/`. Operator wires these
|
||||
patches onto the lqd StatefulSet — no separate Deployment.
|
||||
|
||||
Folded in from the standalone `luxfi/zapdb-replicate` repo (2026-06-07):
|
||||
that repo was duplicate scaffolding around ZapDB's native streaming-
|
||||
replication API. Canonical home is here.
|
||||
|
||||
## Structure
|
||||
```
|
||||
zapdb/
|
||||
|
||||
@@ -0,0 +1,53 @@
|
||||
package main_test
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"testing"
|
||||
|
||||
badger "github.com/luxfi/zapdb"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
// TestBadgerSmoke confirms our understanding of badger's
|
||||
// Backup/Load is correct, independent of the chunkwriter/vfs stack.
|
||||
//
|
||||
// If this fails, the bug is in our use of badger, not in the chunk
|
||||
// layer.
|
||||
func TestBadgerSmoke(t *testing.T) {
|
||||
srcOpts := badger.DefaultOptions(t.TempDir())
|
||||
srcOpts.Logger = nil
|
||||
src, err := badger.Open(srcOpts)
|
||||
require.NoError(t, err)
|
||||
require.NoError(t, src.Update(func(txn *badger.Txn) error {
|
||||
for i := 0; i < 10; i++ {
|
||||
if err := txn.Set([]byte{byte('a' + i)}, []byte{byte('A' + i)}); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}))
|
||||
|
||||
var buf bytes.Buffer
|
||||
until, err := src.Backup(&buf, 0)
|
||||
require.NoError(t, err)
|
||||
require.Greater(t, until, uint64(0))
|
||||
require.NoError(t, src.Close())
|
||||
|
||||
dstOpts := badger.DefaultOptions(t.TempDir())
|
||||
dstOpts.Logger = nil
|
||||
dst, err := badger.Open(dstOpts)
|
||||
require.NoError(t, err)
|
||||
require.NoError(t, dst.Load(&buf, 16))
|
||||
|
||||
require.NoError(t, dst.View(func(txn *badger.Txn) error {
|
||||
for i := 0; i < 10; i++ {
|
||||
item, err := txn.Get([]byte{byte('a' + i)})
|
||||
require.NoError(t, err)
|
||||
val, err := item.ValueCopy(nil)
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, []byte{byte('A' + i)}, val, "key %c", 'a'+i)
|
||||
}
|
||||
return nil
|
||||
}))
|
||||
require.NoError(t, dst.Close())
|
||||
}
|
||||
@@ -0,0 +1,141 @@
|
||||
// Package manifest defines the JSON metadata blobs that describe each
|
||||
// snapshot of a ZapDB and the HEAD pointer that names the current chain
|
||||
// of full + incremental manifests.
|
||||
//
|
||||
// Layout in the backend:
|
||||
//
|
||||
// <prefix>/HEAD.json -- pointer
|
||||
// <prefix>/manifests/<ts>.json -- one per snapshot
|
||||
// <prefix>/blocks/<2-hex>/<blake3>.zap.age -- written by hanzoai/vfs
|
||||
//
|
||||
// HEAD names the most recent full snapshot plus the ordered list of
|
||||
// incrementals applied since. Restore walks HEAD: read the full, then
|
||||
// each incremental in order, calling db.Load(r) on each.
|
||||
package manifest
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"sort"
|
||||
"time"
|
||||
|
||||
"github.com/hanzoai/vfs"
|
||||
)
|
||||
|
||||
// Type names the snapshot category.
|
||||
type Type string
|
||||
|
||||
const (
|
||||
TypeFull Type = "full"
|
||||
TypeIncremental Type = "increment"
|
||||
)
|
||||
|
||||
// Manifest is the per-snapshot record written at <prefix>/manifests/<ts>.json.
|
||||
//
|
||||
// Blocks are listed in the order they were produced by vfs.PutBlock so
|
||||
// that Restore can stream them back into db.Load(r) without an index.
|
||||
//
|
||||
// PlainSize is the number of plaintext bytes the chunked writer
|
||||
// observed before padding+encryption. The last block is zero-padded to
|
||||
// vfs.BlockSize, so Restore truncates its final block read to
|
||||
// (PlainSize % BlockSize) when non-zero.
|
||||
type Manifest struct {
|
||||
Version int `json:"version"` // schema version
|
||||
Type Type `json:"type"` // full | increment
|
||||
Since uint64 `json:"since"` // 0 for full
|
||||
Until uint64 `json:"until"` // badger version watermark after this snap
|
||||
Network string `json:"network"` // mainnet|testnet|devnet|...
|
||||
Blocks []vfs.BlockID `json:"blocks"` // ordered block IDs
|
||||
PlainSize int64 `json:"plainSize"` // bytes of plaintext stream
|
||||
AgeScheme string `json:"ageScheme"` // "X25519+MLKEM768" etc.
|
||||
BlockSize int `json:"blockSize"` // vfs.BlockSize at write time
|
||||
CreatedAt time.Time `json:"createdAt"`
|
||||
Labels map[string]string `json:"labels,omitempty"`
|
||||
}
|
||||
|
||||
// Validate checks invariants we rely on at restore time.
|
||||
func (m *Manifest) Validate() error {
|
||||
if m.Version != 1 {
|
||||
return fmt.Errorf("manifest: unsupported version %d", m.Version)
|
||||
}
|
||||
switch m.Type {
|
||||
case TypeFull:
|
||||
if m.Since != 0 {
|
||||
return fmt.Errorf("manifest: full snapshot must have since=0, got %d", m.Since)
|
||||
}
|
||||
case TypeIncremental:
|
||||
if m.Since == 0 {
|
||||
return fmt.Errorf("manifest: incremental must have since>0")
|
||||
}
|
||||
if m.Until <= m.Since {
|
||||
return fmt.Errorf("manifest: until(%d) must be > since(%d)", m.Until, m.Since)
|
||||
}
|
||||
default:
|
||||
return fmt.Errorf("manifest: unknown type %q", m.Type)
|
||||
}
|
||||
if m.BlockSize != vfs.BlockSize {
|
||||
return fmt.Errorf("manifest: blockSize %d != current vfs.BlockSize %d (cross-version restore unsupported)", m.BlockSize, vfs.BlockSize)
|
||||
}
|
||||
if len(m.Blocks) == 0 && m.PlainSize > 0 {
|
||||
return fmt.Errorf("manifest: plainSize=%d but no blocks listed", m.PlainSize)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// MarshalJSON produces stable, deterministic JSON (sorted labels).
|
||||
func (m *Manifest) MarshalJSON() ([]byte, error) {
|
||||
type alias Manifest
|
||||
a := alias(*m)
|
||||
if len(a.Labels) > 0 {
|
||||
// json package already sorts map keys for marshaling, but be
|
||||
// explicit so the output is deterministic across Go versions.
|
||||
keys := make([]string, 0, len(a.Labels))
|
||||
for k := range a.Labels {
|
||||
keys = append(keys, k)
|
||||
}
|
||||
sort.Strings(keys)
|
||||
}
|
||||
return json.Marshal(a)
|
||||
}
|
||||
|
||||
// HEAD is the single pointer object the replicator updates after every
|
||||
// snapshot. Lives at <prefix>/HEAD.json.
|
||||
//
|
||||
// CurrentVersion is the badger watermark of the most recent snapshot.
|
||||
// Restore should land at exactly CurrentVersion after applying
|
||||
// LastFullManifest plus every entry of Increments in order.
|
||||
type HEAD struct {
|
||||
Version int `json:"version"` // schema version
|
||||
Network string `json:"network"`
|
||||
CurrentVersion uint64 `json:"currentVersion"` // badger watermark
|
||||
LastFullManifest string `json:"lastFullManifest"` // <ts>.json
|
||||
Increments []string `json:"increments"` // <ts>.json, in apply order
|
||||
UpdatedAt time.Time `json:"updatedAt"`
|
||||
}
|
||||
|
||||
// Key paths inside the backend prefix.
|
||||
const (
|
||||
HeadKey = "HEAD.json"
|
||||
ManifestPrefix = "manifests/"
|
||||
)
|
||||
|
||||
// ManifestKey returns the backend key for a manifest with the given
|
||||
// stable identifier. The identifier is the sortable manifest filename
|
||||
// (e.g. "20260606T040000.123456789Z-v0000000000000001").
|
||||
func ManifestKey(ts string) string { return ManifestPrefix + ts + ".json" }
|
||||
|
||||
// FormatTimestamp returns the canonical filename for a manifest at a
|
||||
// given (time, badger-watermark). UTC, sortable, with a 9-digit
|
||||
// nanosecond component AND the watermark suffix so two snapshots in
|
||||
// the same nanosecond at different versions still hash to distinct
|
||||
// keys.
|
||||
func FormatTimestamp(t time.Time) string {
|
||||
return t.UTC().Format("20060102T150405.000000000Z")
|
||||
}
|
||||
|
||||
// FormatManifestID is the canonical "<ts>-v<until>" filename ID for a
|
||||
// manifest. Used by PutManifest to guarantee uniqueness across
|
||||
// closely-spaced snapshots.
|
||||
func FormatManifestID(t time.Time, until uint64) string {
|
||||
return fmt.Sprintf("%s-v%016d", FormatTimestamp(t), until)
|
||||
}
|
||||
@@ -0,0 +1,114 @@
|
||||
package manifest
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/hanzoai/vfs"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func TestManifestRoundTrip(t *testing.T) {
|
||||
now := time.Date(2026, 6, 6, 4, 0, 0, 0, time.UTC)
|
||||
in := &Manifest{
|
||||
Version: 1,
|
||||
Type: TypeFull,
|
||||
Since: 0,
|
||||
Until: 12345,
|
||||
Network: "mainnet",
|
||||
Blocks: []vfs.BlockID{"a1b2", "c3d4"},
|
||||
PlainSize: 8192,
|
||||
AgeScheme: "X25519+MLKEM768",
|
||||
BlockSize: vfs.BlockSize,
|
||||
CreatedAt: now,
|
||||
Labels: map[string]string{"git-sha": "deadbeef"},
|
||||
}
|
||||
body, err := json.Marshal(in)
|
||||
require.NoError(t, err)
|
||||
|
||||
// Sanity check on size — keep manifests small. 1 KiB is the
|
||||
// stated budget for a typical full snapshot manifest with a
|
||||
// modest block count; we have 2 blocks here so the produced JSON
|
||||
// must be well under 1 KiB.
|
||||
require.Less(t, len(body), 1024, "manifest exceeded 1 KiB budget: %d bytes", len(body))
|
||||
t.Logf("manifest JSON size with 2 blocks: %d bytes", len(body))
|
||||
|
||||
var out Manifest
|
||||
require.NoError(t, json.Unmarshal(body, &out))
|
||||
require.Equal(t, in.Version, out.Version)
|
||||
require.Equal(t, in.Type, out.Type)
|
||||
require.Equal(t, in.Until, out.Until)
|
||||
require.Equal(t, in.Blocks, out.Blocks)
|
||||
require.Equal(t, in.PlainSize, out.PlainSize)
|
||||
require.Equal(t, in.Labels["git-sha"], out.Labels["git-sha"])
|
||||
require.True(t, in.CreatedAt.Equal(out.CreatedAt))
|
||||
require.NoError(t, out.Validate())
|
||||
}
|
||||
|
||||
func TestManifestValidateRejectsCrossBlockSize(t *testing.T) {
|
||||
m := &Manifest{
|
||||
Version: 1,
|
||||
Type: TypeFull,
|
||||
Until: 1,
|
||||
Blocks: []vfs.BlockID{"x"},
|
||||
PlainSize: 100,
|
||||
BlockSize: 8192, // wrong
|
||||
}
|
||||
require.Error(t, m.Validate())
|
||||
}
|
||||
|
||||
func TestManifestValidateRejectsIncrementSince0(t *testing.T) {
|
||||
m := &Manifest{
|
||||
Version: 1,
|
||||
Type: TypeIncremental,
|
||||
Since: 0,
|
||||
Until: 100,
|
||||
BlockSize: vfs.BlockSize,
|
||||
}
|
||||
require.Error(t, m.Validate())
|
||||
}
|
||||
|
||||
func TestFormatTimestamp(t *testing.T) {
|
||||
got := FormatTimestamp(time.Date(2026, 6, 6, 4, 0, 0, 0, time.UTC))
|
||||
require.Equal(t, "20260606T040000.000000000Z", got)
|
||||
}
|
||||
|
||||
func TestFormatManifestID(t *testing.T) {
|
||||
got := FormatManifestID(time.Date(2026, 6, 6, 4, 0, 0, 0, time.UTC), 12345)
|
||||
require.Equal(t, "20260606T040000.000000000Z-v0000000000012345", got)
|
||||
}
|
||||
|
||||
func TestManifestKey(t *testing.T) {
|
||||
require.Equal(t, "manifests/foo.json", ManifestKey("foo"))
|
||||
}
|
||||
|
||||
// TestManifestSizeSmallIncrement verifies the manifest stays small for
|
||||
// the typical 60-second incremental: a few hundred 4 KiB blocks of
|
||||
// fresh badger entries (~1 MiB of chain state). 1 KiB budget per
|
||||
// manifest is the stated goal.
|
||||
func TestManifestSizeSmallIncrement(t *testing.T) {
|
||||
now := time.Date(2026, 6, 6, 4, 0, 0, 0, time.UTC)
|
||||
// Simulate 12 blocks (~50 KiB) — typical 1-min incremental on
|
||||
// a quiet chain.
|
||||
blocks := make([]vfs.BlockID, 12)
|
||||
for i := range blocks {
|
||||
blocks[i] = vfs.BlockID("0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef")
|
||||
}
|
||||
m := &Manifest{
|
||||
Version: 1,
|
||||
Type: TypeIncremental,
|
||||
Since: 1000,
|
||||
Until: 1100,
|
||||
Network: "mainnet",
|
||||
Blocks: blocks,
|
||||
PlainSize: int64(len(blocks)) * int64(vfs.BlockSize),
|
||||
AgeScheme: "X25519+MLKEM768",
|
||||
BlockSize: vfs.BlockSize,
|
||||
CreatedAt: now,
|
||||
}
|
||||
body, err := json.Marshal(m)
|
||||
require.NoError(t, err)
|
||||
t.Logf("manifest JSON size with 12 blocks: %d bytes", len(body))
|
||||
require.Less(t, len(body), 1300, "manifest exceeded ~1 KiB budget for small increment: %d bytes", len(body))
|
||||
}
|
||||
@@ -0,0 +1,123 @@
|
||||
package manifest
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
"github.com/hanzoai/vfs/pkg/backend"
|
||||
)
|
||||
|
||||
// Store reads + writes manifests through a backend.Backend, using the
|
||||
// configured key prefix. The backend already roots its own prefix; Store
|
||||
// only adds the manifest/HEAD layout on top.
|
||||
type Store struct {
|
||||
be backend.Backend
|
||||
}
|
||||
|
||||
// NewStore wraps a backend with the manifest layout.
|
||||
func NewStore(be backend.Backend) *Store { return &Store{be: be} }
|
||||
|
||||
// PutManifest writes a manifest. Returns the chosen key.
|
||||
func (s *Store) PutManifest(ctx context.Context, m *Manifest) (string, error) {
|
||||
if err := m.Validate(); err != nil {
|
||||
return "", err
|
||||
}
|
||||
if m.CreatedAt.IsZero() {
|
||||
m.CreatedAt = time.Now().UTC()
|
||||
}
|
||||
key := ManifestKey(FormatManifestID(m.CreatedAt, m.Until))
|
||||
body, err := json.MarshalIndent(m, "", " ")
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("manifest: marshal: %w", err)
|
||||
}
|
||||
if err := s.be.Put(ctx, key, body); err != nil {
|
||||
return "", fmt.Errorf("manifest: put %s: %w", key, err)
|
||||
}
|
||||
return key, nil
|
||||
}
|
||||
|
||||
// GetManifest fetches and decodes a manifest by key (e.g.
|
||||
// "manifests/20260606T040000Z.json").
|
||||
func (s *Store) GetManifest(ctx context.Context, key string) (*Manifest, error) {
|
||||
body, err := s.be.Get(ctx, key)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("manifest: get %s: %w", key, err)
|
||||
}
|
||||
var m Manifest
|
||||
if err := json.Unmarshal(body, &m); err != nil {
|
||||
return nil, fmt.Errorf("manifest: decode %s: %w", key, err)
|
||||
}
|
||||
if err := m.Validate(); err != nil {
|
||||
return nil, fmt.Errorf("manifest: validate %s: %w", key, err)
|
||||
}
|
||||
return &m, nil
|
||||
}
|
||||
|
||||
// PutHead atomically updates the HEAD pointer. S3 has no compare-and-
|
||||
// swap so we just PUT; the daemon is the only writer for a given
|
||||
// prefix (enforced by the local state file).
|
||||
func (s *Store) PutHead(ctx context.Context, h *HEAD) error {
|
||||
h.Version = 1
|
||||
h.UpdatedAt = time.Now().UTC()
|
||||
body, err := json.MarshalIndent(h, "", " ")
|
||||
if err != nil {
|
||||
return fmt.Errorf("head: marshal: %w", err)
|
||||
}
|
||||
if err := s.be.Put(ctx, HeadKey, body); err != nil {
|
||||
return fmt.Errorf("head: put: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// GetHead reads HEAD. Returns (nil, nil) if HEAD does not exist yet
|
||||
// (fresh prefix). Any other error is wrapped.
|
||||
func (s *Store) GetHead(ctx context.Context) (*HEAD, error) {
|
||||
body, err := s.be.Get(ctx, HeadKey)
|
||||
if err != nil {
|
||||
if errors.Is(err, backend.ErrNotFound) {
|
||||
return nil, nil
|
||||
}
|
||||
return nil, fmt.Errorf("head: get: %w", err)
|
||||
}
|
||||
var h HEAD
|
||||
if err := json.Unmarshal(body, &h); err != nil {
|
||||
return nil, fmt.Errorf("head: decode: %w", err)
|
||||
}
|
||||
if h.Version != 1 {
|
||||
return nil, fmt.Errorf("head: unsupported schema version %d", h.Version)
|
||||
}
|
||||
return &h, nil
|
||||
}
|
||||
|
||||
// ListFulls returns all full-snapshot manifest keys under the prefix,
|
||||
// sorted ascending by timestamp. Used by prune and HEAD recovery.
|
||||
func (s *Store) ListFulls(ctx context.Context) ([]string, error) {
|
||||
return s.listManifestsByType(ctx, TypeFull)
|
||||
}
|
||||
|
||||
// ListIncrements returns all incremental-snapshot manifest keys under
|
||||
// the prefix, sorted ascending by timestamp.
|
||||
func (s *Store) ListIncrements(ctx context.Context) ([]string, error) {
|
||||
return s.listManifestsByType(ctx, TypeIncremental)
|
||||
}
|
||||
|
||||
func (s *Store) listManifestsByType(ctx context.Context, t Type) ([]string, error) {
|
||||
keys, errs := s.be.List(ctx, ManifestPrefix)
|
||||
var out []string
|
||||
for k := range keys {
|
||||
m, err := s.GetManifest(ctx, k)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if m.Type == t {
|
||||
out = append(out, k)
|
||||
}
|
||||
}
|
||||
if err := <-errs; err != nil {
|
||||
return nil, fmt.Errorf("manifest: list: %w", err)
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
@@ -0,0 +1,59 @@
|
||||
package replica
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"io"
|
||||
"testing"
|
||||
|
||||
badger "github.com/luxfi/zapdb"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
// TestChunkPreservesBackupStream tees the badger Backup stream into
|
||||
// BOTH a raw bytes.Buffer and our ChunkWriter in the same call, then
|
||||
// requires that the bytes read back through the ChunkReader equal the
|
||||
// raw buffer byte-for-byte. This catches any byte loss / reordering /
|
||||
// truncation in the chunk layer that round-trip tests with random
|
||||
// bytes would miss.
|
||||
func TestChunkPreservesBackupStream(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
v := newTestVFS(t)
|
||||
defer v.Close()
|
||||
|
||||
srcOpts := badger.DefaultOptions(t.TempDir())
|
||||
srcOpts.Logger = nil
|
||||
src, err := badger.Open(srcOpts)
|
||||
require.NoError(t, err)
|
||||
require.NoError(t, src.Update(func(txn *badger.Txn) error {
|
||||
for i := 0; i < 100; i++ {
|
||||
if err := txn.Set([]byte{byte(i)}, []byte{byte(i ^ 0xff)}); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}))
|
||||
|
||||
var raw bytes.Buffer
|
||||
cw := NewChunkWriter(ctx, v)
|
||||
tee := io.MultiWriter(&raw, cw)
|
||||
|
||||
_, err = src.Backup(tee, 0)
|
||||
require.NoError(t, err)
|
||||
require.NoError(t, cw.Close())
|
||||
require.NoError(t, src.Close())
|
||||
|
||||
cr := NewChunkReader(ctx, v, cw.Blocks(), cw.PlainSize())
|
||||
got, err := io.ReadAll(cr)
|
||||
require.NoError(t, err)
|
||||
|
||||
require.Equal(t, raw.Len(), len(got), "length mismatch")
|
||||
if !bytes.Equal(raw.Bytes(), got) {
|
||||
for i := 0; i < raw.Len(); i++ {
|
||||
if raw.Bytes()[i] != got[i] {
|
||||
t.Fatalf("first mismatch at byte %d (of %d, block %d offset %d): raw=%#x got=%#x",
|
||||
i, raw.Len(), i/4096, i%4096, raw.Bytes()[i], got[i])
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,171 @@
|
||||
// Package replica owns the backup-side wiring: open a ZapDB read-only,
|
||||
// take a snapshot, stream the badger Backup bytes through hanzoai/vfs
|
||||
// (4 KiB content-addressed age-encrypted blocks), and publish a
|
||||
// manifest naming the produced block list.
|
||||
package replica
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"io"
|
||||
|
||||
"github.com/hanzoai/vfs"
|
||||
)
|
||||
|
||||
// ChunkWriter buffers an io.Writer stream into vfs.BlockSize pages and
|
||||
// pushes each full page through vfs.PutBlock. The last page (Close)
|
||||
// flushes whatever bytes remain — vfs.Pad zero-pads short tails to
|
||||
// BlockSize before encryption.
|
||||
//
|
||||
// Concurrency: not safe for concurrent use. Caller serialises Write/Close.
|
||||
type ChunkWriter struct {
|
||||
ctx context.Context
|
||||
v *vfs.VFS
|
||||
buf []byte
|
||||
blocks []vfs.BlockID
|
||||
total int64 // plaintext bytes consumed
|
||||
closed bool
|
||||
}
|
||||
|
||||
// NewChunkWriter constructs a ChunkWriter against the given VFS. The
|
||||
// caller is responsible for Close() — it flushes any partial tail block.
|
||||
func NewChunkWriter(ctx context.Context, v *vfs.VFS) *ChunkWriter {
|
||||
return &ChunkWriter{
|
||||
ctx: ctx,
|
||||
v: v,
|
||||
buf: make([]byte, 0, vfs.BlockSize),
|
||||
}
|
||||
}
|
||||
|
||||
// Write implements io.Writer. Each completed vfs.BlockSize page is
|
||||
// encrypted + put to the backend; the produced BlockID is appended to
|
||||
// the in-memory ordered slice.
|
||||
func (c *ChunkWriter) Write(p []byte) (int, error) {
|
||||
if c.closed {
|
||||
return 0, fmt.Errorf("chunkwriter: write after close")
|
||||
}
|
||||
written := 0
|
||||
for len(p) > 0 {
|
||||
room := vfs.BlockSize - len(c.buf)
|
||||
if room > len(p) {
|
||||
room = len(p)
|
||||
}
|
||||
c.buf = append(c.buf, p[:room]...)
|
||||
p = p[room:]
|
||||
written += room
|
||||
if len(c.buf) == vfs.BlockSize {
|
||||
if err := c.flushFull(); err != nil {
|
||||
return written, err
|
||||
}
|
||||
}
|
||||
}
|
||||
c.total += int64(written)
|
||||
return written, nil
|
||||
}
|
||||
|
||||
func (c *ChunkWriter) flushFull() error {
|
||||
// vfs.PutBlock's cache aliases the input slice when len ==
|
||||
// BlockSize (vfs.Pad returns the input as-is). We reuse c.buf
|
||||
// for the next block, so we MUST hand PutBlock its own copy.
|
||||
block := make([]byte, len(c.buf))
|
||||
copy(block, c.buf)
|
||||
id, err := c.v.PutBlock(c.ctx, block)
|
||||
if err != nil {
|
||||
return fmt.Errorf("chunkwriter: put block %d: %w", len(c.blocks), err)
|
||||
}
|
||||
c.blocks = append(c.blocks, id)
|
||||
c.buf = c.buf[:0]
|
||||
return nil
|
||||
}
|
||||
|
||||
// Close flushes any partial tail block. Safe to call multiple times.
|
||||
// After Close, Blocks and PlainSize are stable.
|
||||
func (c *ChunkWriter) Close() error {
|
||||
if c.closed {
|
||||
return nil
|
||||
}
|
||||
c.closed = true
|
||||
if len(c.buf) == 0 {
|
||||
return nil
|
||||
}
|
||||
// Final short block — vfs.PutBlock will zero-pad to BlockSize.
|
||||
id, err := c.v.PutBlock(c.ctx, c.buf)
|
||||
if err != nil {
|
||||
return fmt.Errorf("chunkwriter: flush tail: %w", err)
|
||||
}
|
||||
c.blocks = append(c.blocks, id)
|
||||
c.buf = nil
|
||||
return nil
|
||||
}
|
||||
|
||||
// Blocks returns the ordered BlockIDs produced by this writer. Only
|
||||
// valid after Close.
|
||||
func (c *ChunkWriter) Blocks() []vfs.BlockID { return c.blocks }
|
||||
|
||||
// PlainSize returns the number of plaintext bytes the writer consumed.
|
||||
// Used by the manifest so Restore can truncate the final block.
|
||||
func (c *ChunkWriter) PlainSize() int64 { return c.total }
|
||||
|
||||
// ChunkReader reads back a stream the ChunkWriter produced. It fetches
|
||||
// each BlockID via vfs.GetBlock in order. Plaintext blocks are returned
|
||||
// padded to vfs.BlockSize; the reader truncates the final block to
|
||||
// (plainSize - (n-1)*BlockSize) when plainSize is not a multiple of
|
||||
// BlockSize.
|
||||
type ChunkReader struct {
|
||||
ctx context.Context
|
||||
v *vfs.VFS
|
||||
blocks []vfs.BlockID
|
||||
plainSize int64
|
||||
idx int
|
||||
cur []byte
|
||||
curOff int
|
||||
}
|
||||
|
||||
// NewChunkReader builds a reader. plainSize MUST match the manifest.
|
||||
func NewChunkReader(ctx context.Context, v *vfs.VFS, blocks []vfs.BlockID, plainSize int64) *ChunkReader {
|
||||
return &ChunkReader{
|
||||
ctx: ctx,
|
||||
v: v,
|
||||
blocks: blocks,
|
||||
plainSize: plainSize,
|
||||
}
|
||||
}
|
||||
|
||||
// Read implements io.Reader.
|
||||
func (r *ChunkReader) Read(p []byte) (int, error) {
|
||||
if len(p) == 0 {
|
||||
return 0, nil
|
||||
}
|
||||
n := 0
|
||||
for n < len(p) {
|
||||
if r.cur == nil || r.curOff >= len(r.cur) {
|
||||
if r.idx >= len(r.blocks) {
|
||||
if n == 0 {
|
||||
return 0, io.EOF
|
||||
}
|
||||
return n, nil
|
||||
}
|
||||
block, err := r.v.GetBlock(r.ctx, r.blocks[r.idx])
|
||||
if err != nil {
|
||||
return n, fmt.Errorf("chunkreader: get block %d: %w", r.idx, err)
|
||||
}
|
||||
// Truncate the final padded block to its logical size.
|
||||
if r.idx == len(r.blocks)-1 {
|
||||
logical := r.plainSize - int64(r.idx)*int64(vfs.BlockSize)
|
||||
if logical < 0 {
|
||||
return n, fmt.Errorf("chunkreader: negative logical size for tail block (plainSize=%d, idx=%d)", r.plainSize, r.idx)
|
||||
}
|
||||
if logical < int64(len(block)) {
|
||||
block = block[:logical]
|
||||
}
|
||||
}
|
||||
r.cur = block
|
||||
r.curOff = 0
|
||||
r.idx++
|
||||
}
|
||||
copied := copy(p[n:], r.cur[r.curOff:])
|
||||
r.curOff += copied
|
||||
n += copied
|
||||
}
|
||||
return n, nil
|
||||
}
|
||||
@@ -0,0 +1,98 @@
|
||||
package replica
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"crypto/rand"
|
||||
"io"
|
||||
"testing"
|
||||
|
||||
"github.com/hanzoai/vfs"
|
||||
"github.com/hanzoai/vfs/pkg/backend"
|
||||
_ "github.com/hanzoai/vfs/pkg/backend/file"
|
||||
"github.com/luxfi/age"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func newTestVFS(t *testing.T) *vfs.VFS {
|
||||
t.Helper()
|
||||
store := t.TempDir()
|
||||
be, err := backend.Open(context.Background(), "file://"+store)
|
||||
require.NoError(t, err)
|
||||
id, err := age.GenerateX25519Identity()
|
||||
require.NoError(t, err)
|
||||
crypto, err := vfs.NewCrypto([]age.Recipient{id.Recipient()}, []age.Identity{id})
|
||||
require.NoError(t, err)
|
||||
v, err := vfs.New(vfs.Config{Backend: be, Crypto: crypto, CacheMax: 1 << 20})
|
||||
require.NoError(t, err)
|
||||
return v
|
||||
}
|
||||
|
||||
// roundtrip is the simplest possible test: write N bytes through a
|
||||
// ChunkWriter, read them back through a ChunkReader, byte-compare.
|
||||
func TestChunkRoundTrip_ExactBlockMultiple(t *testing.T) {
|
||||
roundtrip(t, vfs.BlockSize*3) // exact multiple
|
||||
}
|
||||
|
||||
func TestChunkRoundTrip_OneByteOver(t *testing.T) {
|
||||
roundtrip(t, vfs.BlockSize*3+1)
|
||||
}
|
||||
|
||||
func TestChunkRoundTrip_OneByteUnder(t *testing.T) {
|
||||
roundtrip(t, vfs.BlockSize*3-1)
|
||||
}
|
||||
|
||||
func TestChunkRoundTrip_SubBlock(t *testing.T) {
|
||||
roundtrip(t, 17)
|
||||
}
|
||||
|
||||
func TestChunkRoundTrip_LargeRandom(t *testing.T) {
|
||||
roundtrip(t, 1_234_567)
|
||||
}
|
||||
|
||||
func TestChunkRoundTrip_Empty(t *testing.T) {
|
||||
v := newTestVFS(t)
|
||||
defer v.Close()
|
||||
ctx := context.Background()
|
||||
cw := NewChunkWriter(ctx, v)
|
||||
require.NoError(t, cw.Close())
|
||||
require.Equal(t, int64(0), cw.PlainSize())
|
||||
require.Empty(t, cw.Blocks())
|
||||
|
||||
r := NewChunkReader(ctx, v, nil, 0)
|
||||
out, err := io.ReadAll(r)
|
||||
require.NoError(t, err)
|
||||
require.Empty(t, out)
|
||||
}
|
||||
|
||||
func roundtrip(t *testing.T, n int) {
|
||||
t.Helper()
|
||||
v := newTestVFS(t)
|
||||
defer v.Close()
|
||||
ctx := context.Background()
|
||||
|
||||
payload := make([]byte, n)
|
||||
_, err := rand.Read(payload)
|
||||
require.NoError(t, err)
|
||||
|
||||
cw := NewChunkWriter(ctx, v)
|
||||
wrote, err := cw.Write(payload)
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, n, wrote)
|
||||
require.NoError(t, cw.Close())
|
||||
require.Equal(t, int64(n), cw.PlainSize())
|
||||
|
||||
cr := NewChunkReader(ctx, v, cw.Blocks(), cw.PlainSize())
|
||||
got, err := io.ReadAll(cr)
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, n, len(got), "length mismatch")
|
||||
if !bytes.Equal(payload, got) {
|
||||
// Find first mismatch position
|
||||
for i := 0; i < len(payload) && i < len(got); i++ {
|
||||
if payload[i] != got[i] {
|
||||
t.Fatalf("first byte mismatch at %d (block %d, offset %d): want %#x, got %#x; blocks=%v",
|
||||
i, i/vfs.BlockSize, i%vfs.BlockSize, payload[i], got[i], cw.Blocks())
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,219 @@
|
||||
package replica
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
"github.com/hanzoai/vfs"
|
||||
"github.com/hanzoai/vfs/pkg/backend"
|
||||
log "github.com/luxfi/log"
|
||||
badger "github.com/luxfi/zapdb"
|
||||
|
||||
"github.com/luxfi/zapdb/cmd/replicate/internal/manifest"
|
||||
"github.com/luxfi/zapdb/cmd/replicate/internal/state"
|
||||
)
|
||||
|
||||
// Replica is a single (database, backend) pair that the daemon ticks
|
||||
// against. Each tick: snapshot ZapDB, stream through vfs, publish a
|
||||
// manifest, advance HEAD, persist watermark.
|
||||
type Replica struct {
|
||||
cfg Config
|
||||
v *vfs.VFS
|
||||
backend backend.Backend
|
||||
store *manifest.Store
|
||||
}
|
||||
|
||||
// Config wires a single replica.
|
||||
type Config struct {
|
||||
DBPath string
|
||||
Network string // e.g. "mainnet"
|
||||
VFS *vfs.VFS // backend + crypto
|
||||
Backend backend.Backend
|
||||
StatePath string // override state file path; empty -> derived
|
||||
Interval time.Duration // between cycles, default 60s
|
||||
FullSnapshotEvery time.Duration // re-snapshot full at this cadence
|
||||
AgeSchemeLabel string // recorded in manifests for audit
|
||||
}
|
||||
|
||||
// New constructs a Replica. The VFS + Backend are taken in pre-built so
|
||||
// the daemon can share them across multiple databases.
|
||||
func New(cfg Config) (*Replica, error) {
|
||||
if cfg.DBPath == "" {
|
||||
return nil, errors.New("replica: DBPath required")
|
||||
}
|
||||
if cfg.Network == "" {
|
||||
return nil, errors.New("replica: Network required")
|
||||
}
|
||||
if cfg.VFS == nil || cfg.Backend == nil {
|
||||
return nil, errors.New("replica: VFS and Backend required")
|
||||
}
|
||||
if cfg.Interval <= 0 {
|
||||
cfg.Interval = 60 * time.Second
|
||||
}
|
||||
if cfg.FullSnapshotEvery <= 0 {
|
||||
cfg.FullSnapshotEvery = 168 * time.Hour
|
||||
}
|
||||
return &Replica{
|
||||
cfg: cfg,
|
||||
v: cfg.VFS,
|
||||
backend: cfg.Backend,
|
||||
store: manifest.NewStore(cfg.Backend),
|
||||
}, nil
|
||||
}
|
||||
|
||||
// Run loops the replicator until ctx is cancelled. Errors per cycle are
|
||||
// logged but do not terminate the loop — the next tick retries.
|
||||
func (r *Replica) Run(ctx context.Context) error {
|
||||
log.Info(fmt.Sprintf("[zapdb-replicate] starting replica path=%s net=%s interval=%s", r.cfg.DBPath, r.cfg.Network, r.cfg.Interval))
|
||||
|
||||
t := time.NewTicker(r.cfg.Interval)
|
||||
defer t.Stop()
|
||||
|
||||
// Run an initial cycle so we publish promptly on boot, before
|
||||
// the first ticker fire.
|
||||
if err := r.cycleSafe(ctx); err != nil {
|
||||
log.Error(fmt.Sprintf("[zapdb-replicate] cycle: %v", err))
|
||||
}
|
||||
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return ctx.Err()
|
||||
case <-t.C:
|
||||
if err := r.cycleSafe(ctx); err != nil {
|
||||
log.Error(fmt.Sprintf("[zapdb-replicate] cycle: %v", err))
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (r *Replica) cycleSafe(ctx context.Context) (err error) {
|
||||
defer func() {
|
||||
if rec := recover(); rec != nil {
|
||||
err = fmt.Errorf("zapdb-replicate: panic in cycle: %v", rec)
|
||||
}
|
||||
}()
|
||||
return r.cycle(ctx)
|
||||
}
|
||||
|
||||
func (r *Replica) cycle(ctx context.Context) error {
|
||||
statePath := r.cfg.StatePath
|
||||
if statePath == "" {
|
||||
statePath = state.Path(r.cfg.DBPath)
|
||||
}
|
||||
|
||||
st, err := state.Load(statePath)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Decide full vs incremental
|
||||
now := time.Now().UTC()
|
||||
isFull := false
|
||||
if st.LastVersion == 0 {
|
||||
isFull = true
|
||||
} else if st.LastFullTimestamp.IsZero() {
|
||||
isFull = true
|
||||
} else if now.Sub(st.LastFullTimestamp) >= r.cfg.FullSnapshotEvery {
|
||||
isFull = true
|
||||
}
|
||||
|
||||
since := st.LastVersion
|
||||
if isFull {
|
||||
since = 0
|
||||
}
|
||||
|
||||
// Open ZapDB read-only with lock-guard bypass so we coexist with luxd.
|
||||
db, err := openReadOnly(r.cfg.DBPath)
|
||||
if err != nil {
|
||||
return fmt.Errorf("zapdb-replicate: open %s: %w", r.cfg.DBPath, err)
|
||||
}
|
||||
defer db.Close()
|
||||
|
||||
cw := NewChunkWriter(ctx, r.v)
|
||||
until, err := db.Backup(cw, since)
|
||||
if err != nil {
|
||||
return fmt.Errorf("zapdb-replicate: badger.Backup(since=%d): %w", since, err)
|
||||
}
|
||||
if err := cw.Close(); err != nil {
|
||||
return fmt.Errorf("zapdb-replicate: chunkwriter close: %w", err)
|
||||
}
|
||||
|
||||
// Empty backup (no new entries since last watermark) — skip
|
||||
// manifest write so we don't spam the backend with no-op snapshots.
|
||||
if !isFull && len(cw.Blocks()) == 0 {
|
||||
log.Debug(fmt.Sprintf("[zapdb-replicate] no new entries since v=%d, skipping", since))
|
||||
return nil
|
||||
}
|
||||
|
||||
m := &manifest.Manifest{
|
||||
Version: 1,
|
||||
Type: manifest.TypeIncremental,
|
||||
Since: since,
|
||||
Until: until,
|
||||
Network: r.cfg.Network,
|
||||
Blocks: cw.Blocks(),
|
||||
PlainSize: cw.PlainSize(),
|
||||
AgeScheme: r.cfg.AgeSchemeLabel,
|
||||
BlockSize: vfs.BlockSize,
|
||||
CreatedAt: now,
|
||||
}
|
||||
if isFull {
|
||||
m.Type = manifest.TypeFull
|
||||
m.Since = 0
|
||||
}
|
||||
|
||||
key, err := r.store.PutManifest(ctx, m)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
log.Info(fmt.Sprintf("[zapdb-replicate] published manifest type=%s since=%d until=%d blocks=%d bytes=%d key=%s",
|
||||
m.Type, m.Since, m.Until, len(m.Blocks), m.PlainSize, key))
|
||||
|
||||
// Update HEAD
|
||||
head, err := r.store.GetHead(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if isFull || head == nil {
|
||||
head = &manifest.HEAD{
|
||||
Network: r.cfg.Network,
|
||||
CurrentVersion: until,
|
||||
LastFullManifest: key,
|
||||
Increments: nil,
|
||||
}
|
||||
} else {
|
||||
head.CurrentVersion = until
|
||||
head.Increments = append(head.Increments, key)
|
||||
}
|
||||
if err := r.store.PutHead(ctx, head); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Persist watermark
|
||||
st.Network = r.cfg.Network
|
||||
st.LastVersion = until
|
||||
st.LastSnapshotKind = string(m.Type)
|
||||
if isFull {
|
||||
st.LastFullVersion = until
|
||||
st.LastFullTimestamp = now
|
||||
}
|
||||
return state.Save(statePath, st)
|
||||
}
|
||||
|
||||
// openReadOnly opens the underlying BadgerDB in read-only mode with
|
||||
// BypassLockGuard so we coexist with a running luxd. We do not use the
|
||||
// high-level luxfi/database/zapdb wrapper because it forces SyncWrites
|
||||
// + GC goroutines that fight a primary writer.
|
||||
func openReadOnly(path string) (*badger.DB, error) {
|
||||
opts := badger.DefaultOptions(path)
|
||||
opts.ReadOnly = true
|
||||
opts.BypassLockGuard = true
|
||||
opts.Logger = nil
|
||||
// Smaller caches in the sidecar — luxd already has the big ones.
|
||||
opts.BlockCacheSize = 64 << 20
|
||||
opts.IndexCacheSize = 32 << 20
|
||||
return badger.Open(opts)
|
||||
}
|
||||
@@ -0,0 +1,125 @@
|
||||
package restore
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"sort"
|
||||
|
||||
"github.com/hanzoai/vfs"
|
||||
"github.com/hanzoai/vfs/pkg/backend"
|
||||
log "github.com/luxfi/log"
|
||||
|
||||
"github.com/luxfi/zapdb/cmd/replicate/internal/manifest"
|
||||
)
|
||||
|
||||
// PruneConfig wires a single prune run.
|
||||
type PruneConfig struct {
|
||||
VFS *vfs.VFS
|
||||
Backend backend.Backend
|
||||
KeepFulls int // keep the N most recent full snapshots; older fulls and their increments are deleted
|
||||
DryRun bool // log only, do not delete
|
||||
}
|
||||
|
||||
// Prune removes manifests older than the Nth-most-recent full
|
||||
// snapshot, plus the blocks that ONLY those manifests reference.
|
||||
//
|
||||
// Block GC: walk every surviving manifest, build a set of referenced
|
||||
// block IDs. Any block in the to-delete manifests that is NOT in the
|
||||
// survivor set can be deleted.
|
||||
func Prune(ctx context.Context, cfg PruneConfig) error {
|
||||
if cfg.KeepFulls < 1 {
|
||||
return fmt.Errorf("prune: --keep-fulls must be >= 1")
|
||||
}
|
||||
if cfg.VFS == nil || cfg.Backend == nil {
|
||||
return fmt.Errorf("prune: VFS and Backend required")
|
||||
}
|
||||
|
||||
store := manifest.NewStore(cfg.Backend)
|
||||
|
||||
allFulls, err := store.ListFulls(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
allIncs, err := store.ListIncrements(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
sort.Strings(allFulls)
|
||||
sort.Strings(allIncs)
|
||||
|
||||
if len(allFulls) <= cfg.KeepFulls {
|
||||
log.Info(fmt.Sprintf("[zapdb-prune] %d fulls present, keep=%d — nothing to prune", len(allFulls), cfg.KeepFulls))
|
||||
return nil
|
||||
}
|
||||
|
||||
// Split fulls into keep + drop sets.
|
||||
dropFulls := allFulls[:len(allFulls)-cfg.KeepFulls]
|
||||
keepFulls := allFulls[len(allFulls)-cfg.KeepFulls:]
|
||||
|
||||
// Increments are tied to the most recent preceding full's
|
||||
// timestamp. We drop any increment whose key sorts BEFORE the
|
||||
// oldest surviving full.
|
||||
oldestKept := keepFulls[0]
|
||||
|
||||
var dropIncs []string
|
||||
var keepIncs []string
|
||||
for _, k := range allIncs {
|
||||
if k < oldestKept {
|
||||
dropIncs = append(dropIncs, k)
|
||||
} else {
|
||||
keepIncs = append(keepIncs, k)
|
||||
}
|
||||
}
|
||||
|
||||
dropManifests := append([]string{}, dropFulls...)
|
||||
dropManifests = append(dropManifests, dropIncs...)
|
||||
keepManifests := append([]string{}, keepFulls...)
|
||||
keepManifests = append(keepManifests, keepIncs...)
|
||||
|
||||
// Build survivor block set.
|
||||
survivors := make(map[vfs.BlockID]struct{})
|
||||
for _, k := range keepManifests {
|
||||
m, err := store.GetManifest(ctx, k)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
for _, b := range m.Blocks {
|
||||
survivors[b] = struct{}{}
|
||||
}
|
||||
}
|
||||
|
||||
// Collect candidates for deletion from the drop manifests.
|
||||
deletable := make(map[vfs.BlockID]struct{})
|
||||
for _, k := range dropManifests {
|
||||
m, err := store.GetManifest(ctx, k)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
for _, b := range m.Blocks {
|
||||
if _, kept := survivors[b]; !kept {
|
||||
deletable[b] = struct{}{}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
log.Info(fmt.Sprintf("[zapdb-prune] dropping %d manifests (%d fulls, %d increments), %d blocks; dryRun=%v",
|
||||
len(dropManifests), len(dropFulls), len(dropIncs), len(deletable), cfg.DryRun))
|
||||
|
||||
if cfg.DryRun {
|
||||
return nil
|
||||
}
|
||||
|
||||
// Delete blocks first (orphan blocks with no manifest are
|
||||
// recoverable; a manifest pointing at deleted blocks is not).
|
||||
for bid := range deletable {
|
||||
if err := cfg.Backend.Delete(ctx, bid.Path()); err != nil {
|
||||
return fmt.Errorf("prune: delete block %s: %w", bid, err)
|
||||
}
|
||||
}
|
||||
for _, k := range dropManifests {
|
||||
if err := cfg.Backend.Delete(ctx, k); err != nil {
|
||||
return fmt.Errorf("prune: delete manifest %s: %w", k, err)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,160 @@
|
||||
// Package restore is the init-container side: read HEAD.json from the
|
||||
// backend, walk the manifest chain (full + increments in order), stream
|
||||
// each one through hanzoai/vfs into db.Load(r). After this completes,
|
||||
// the target ZapDB at the configured path holds a byte-equivalent copy
|
||||
// of whatever the replicator most recently published.
|
||||
package restore
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"time"
|
||||
|
||||
"github.com/hanzoai/vfs"
|
||||
"github.com/hanzoai/vfs/pkg/backend"
|
||||
log "github.com/luxfi/log"
|
||||
badger "github.com/luxfi/zapdb"
|
||||
|
||||
"github.com/luxfi/zapdb/cmd/replicate/internal/manifest"
|
||||
"github.com/luxfi/zapdb/cmd/replicate/internal/replica"
|
||||
"github.com/luxfi/zapdb/cmd/replicate/internal/state"
|
||||
)
|
||||
|
||||
// Config wires a single restore run.
|
||||
type Config struct {
|
||||
DBPath string // target path; created if it does not exist
|
||||
Network string // sanity check — must match HEAD/manifests
|
||||
VFS *vfs.VFS
|
||||
Backend backend.Backend
|
||||
// SkipIfNotEmpty: if true (default), restore aborts when the target
|
||||
// already contains data. Set false to force restore over an
|
||||
// existing dir.
|
||||
SkipIfNotEmpty bool
|
||||
}
|
||||
|
||||
// Run executes the restore. Returns nil if the target is already
|
||||
// populated and SkipIfNotEmpty is true.
|
||||
func Run(ctx context.Context, cfg Config) error {
|
||||
if cfg.DBPath == "" {
|
||||
return errors.New("restore: DBPath required")
|
||||
}
|
||||
if cfg.VFS == nil || cfg.Backend == nil {
|
||||
return errors.New("restore: VFS and Backend required")
|
||||
}
|
||||
if cfg.SkipIfNotEmpty {
|
||||
empty, err := isDirEmpty(cfg.DBPath)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if !empty {
|
||||
log.Info(fmt.Sprintf("[zapdb-restore] %s is not empty, skipping (use --force to override)", cfg.DBPath))
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
if err := os.MkdirAll(cfg.DBPath, 0o755); err != nil {
|
||||
return fmt.Errorf("restore: mkdir %s: %w", cfg.DBPath, err)
|
||||
}
|
||||
|
||||
store := manifest.NewStore(cfg.Backend)
|
||||
head, err := store.GetHead(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if head == nil {
|
||||
return errors.New("restore: HEAD.json not found in backend — nothing to restore")
|
||||
}
|
||||
if cfg.Network != "" && head.Network != "" && head.Network != cfg.Network {
|
||||
return fmt.Errorf("restore: network mismatch: requested %q, HEAD has %q", cfg.Network, head.Network)
|
||||
}
|
||||
if head.LastFullManifest == "" {
|
||||
return errors.New("restore: HEAD has no LastFullManifest")
|
||||
}
|
||||
|
||||
// Open the target DB writable. No BypassLockGuard — we want a
|
||||
// regular lock since luxd is NOT running.
|
||||
db, err := openWritable(cfg.DBPath)
|
||||
if err != nil {
|
||||
return fmt.Errorf("restore: open %s: %w", cfg.DBPath, err)
|
||||
}
|
||||
defer db.Close()
|
||||
|
||||
// Full snapshot
|
||||
if err := applyManifest(ctx, db, cfg.VFS, store, head.LastFullManifest, manifest.TypeFull); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Increments in order
|
||||
for _, key := range head.Increments {
|
||||
if err := applyManifest(ctx, db, cfg.VFS, store, key, manifest.TypeIncremental); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
// Persist a watermark so a follow-on replicator picks up here.
|
||||
st := &state.State{
|
||||
Network: head.Network,
|
||||
LastVersion: head.CurrentVersion,
|
||||
LastSnapshotKind: string(manifest.TypeIncremental),
|
||||
LastFullVersion: head.CurrentVersion,
|
||||
LastFullTimestamp: time.Now().UTC(),
|
||||
}
|
||||
if err := state.Save(state.Path(cfg.DBPath), st); err != nil {
|
||||
log.Warn(fmt.Sprintf("[zapdb-restore] could not write state file: %v (non-fatal)", err))
|
||||
}
|
||||
|
||||
log.Info(fmt.Sprintf("[zapdb-restore] complete: path=%s currentVersion=%d", cfg.DBPath, head.CurrentVersion))
|
||||
return nil
|
||||
}
|
||||
|
||||
func applyManifest(ctx context.Context, db *badger.DB, v *vfs.VFS, store *manifest.Store, key string, expectType manifest.Type) error {
|
||||
m, err := store.GetManifest(ctx, key)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if m.Type != expectType {
|
||||
return fmt.Errorf("restore: %s expected %s, got %s", key, expectType, m.Type)
|
||||
}
|
||||
log.Info(fmt.Sprintf("[zapdb-restore] applying %s type=%s blocks=%d bytes=%d (until=%d)",
|
||||
key, m.Type, len(m.Blocks), m.PlainSize, m.Until))
|
||||
|
||||
r := replica.NewChunkReader(ctx, v, m.Blocks, m.PlainSize)
|
||||
if err := db.Load(r, 16); err != nil {
|
||||
return fmt.Errorf("restore: apply %s: %w", key, err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func openWritable(path string) (*badger.DB, error) {
|
||||
opts := badger.DefaultOptions(path)
|
||||
opts.SyncWrites = true
|
||||
opts.Logger = nil
|
||||
return badger.Open(opts)
|
||||
}
|
||||
|
||||
func isDirEmpty(path string) (bool, error) {
|
||||
entries, err := os.ReadDir(path)
|
||||
if err != nil {
|
||||
if os.IsNotExist(err) {
|
||||
return true, nil
|
||||
}
|
||||
return false, fmt.Errorf("restore: stat %s: %w", path, err)
|
||||
}
|
||||
// Ignore lock + state side files when judging emptiness — they
|
||||
// may exist from a half-init.
|
||||
for _, e := range entries {
|
||||
name := e.Name()
|
||||
if name == "LOCK" || name == "MANIFEST" {
|
||||
return false, nil
|
||||
}
|
||||
// any other data file means populated
|
||||
if filepath.Ext(name) != "" {
|
||||
return false, nil
|
||||
}
|
||||
return false, nil
|
||||
}
|
||||
return true, nil
|
||||
}
|
||||
@@ -0,0 +1,78 @@
|
||||
// Package state owns the replicator's local watermark file.
|
||||
//
|
||||
// The file lives next to the database (default: /data/.zapdb-replicate-
|
||||
// state.json). It records the most recent badger version successfully
|
||||
// uploaded so a restart resumes the incremental stream where it left
|
||||
// off without re-uploading the whole DB.
|
||||
//
|
||||
// It is NOT the source of truth — that is the backend's HEAD.json. The
|
||||
// state file is just a cheap local cache that lets the daemon skip
|
||||
// re-listing the backend on each cycle.
|
||||
package state
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io/fs"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"time"
|
||||
)
|
||||
|
||||
// State is the persisted replicator watermark.
|
||||
type State struct {
|
||||
Version int `json:"version"` // schema version (1)
|
||||
Network string `json:"network"`
|
||||
LastVersion uint64 `json:"lastVersion"` // badger watermark we last uploaded through
|
||||
LastSnapshotKind string `json:"lastSnapshotKind"` // "full" | "increment"
|
||||
LastFullVersion uint64 `json:"lastFullVersion"` // watermark of the most recent full snapshot
|
||||
LastFullTimestamp time.Time `json:"lastFullTimestamp"` // when the last full snapshot ran
|
||||
UpdatedAt time.Time `json:"updatedAt"`
|
||||
}
|
||||
|
||||
// Path returns the canonical state file path for a database dir.
|
||||
func Path(dbPath string) string {
|
||||
dir := filepath.Dir(dbPath)
|
||||
base := filepath.Base(dbPath)
|
||||
return filepath.Join(dir, "."+base+".zapdb-replicate-state.json")
|
||||
}
|
||||
|
||||
// Load reads State from path. Returns a zero State (Version=1) if the
|
||||
// file does not exist; any other error is wrapped.
|
||||
func Load(path string) (*State, error) {
|
||||
body, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
if errors.Is(err, fs.ErrNotExist) {
|
||||
return &State{Version: 1}, nil
|
||||
}
|
||||
return nil, fmt.Errorf("state: read %s: %w", path, err)
|
||||
}
|
||||
var s State
|
||||
if err := json.Unmarshal(body, &s); err != nil {
|
||||
return nil, fmt.Errorf("state: decode %s: %w", path, err)
|
||||
}
|
||||
if s.Version != 1 {
|
||||
return nil, fmt.Errorf("state: unsupported version %d", s.Version)
|
||||
}
|
||||
return &s, nil
|
||||
}
|
||||
|
||||
// Save atomically writes State to path via rename. Caller is
|
||||
// responsible for ensuring dir exists.
|
||||
func Save(path string, s *State) error {
|
||||
s.Version = 1
|
||||
s.UpdatedAt = time.Now().UTC()
|
||||
body, err := json.MarshalIndent(s, "", " ")
|
||||
if err != nil {
|
||||
return fmt.Errorf("state: marshal: %w", err)
|
||||
}
|
||||
tmp := path + ".tmp"
|
||||
if err := os.WriteFile(tmp, body, 0o600); err != nil {
|
||||
return fmt.Errorf("state: write %s: %w", tmp, err)
|
||||
}
|
||||
if err := os.Rename(tmp, path); err != nil {
|
||||
return fmt.Errorf("state: rename %s -> %s: %w", tmp, path, err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,39 @@
|
||||
package state
|
||||
|
||||
import (
|
||||
"path/filepath"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func TestStateRoundTrip(t *testing.T) {
|
||||
path := filepath.Join(t.TempDir(), "state.json")
|
||||
in := &State{
|
||||
Network: "mainnet",
|
||||
LastVersion: 12345,
|
||||
LastSnapshotKind: "increment",
|
||||
LastFullVersion: 12000,
|
||||
LastFullTimestamp: time.Date(2026, 6, 6, 0, 0, 0, 0, time.UTC),
|
||||
}
|
||||
require.NoError(t, Save(path, in))
|
||||
out, err := Load(path)
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, in.Network, out.Network)
|
||||
require.Equal(t, in.LastVersion, out.LastVersion)
|
||||
require.Equal(t, in.LastFullVersion, out.LastFullVersion)
|
||||
require.True(t, in.LastFullTimestamp.Equal(out.LastFullTimestamp))
|
||||
}
|
||||
|
||||
func TestLoadMissingReturnsZero(t *testing.T) {
|
||||
s, err := Load(filepath.Join(t.TempDir(), "no-such-file.json"))
|
||||
require.NoError(t, err)
|
||||
require.NotNil(t, s)
|
||||
require.Equal(t, uint64(0), s.LastVersion)
|
||||
}
|
||||
|
||||
func TestPath(t *testing.T) {
|
||||
got := Path("/data/db/mainnet")
|
||||
require.Equal(t, "/data/db/.mainnet.zapdb-replicate-state.json", got)
|
||||
}
|
||||
@@ -0,0 +1,372 @@
|
||||
// Command zapdb-replicate is the sidecar + init-container binary for
|
||||
// streaming-replicating Lux ZapDB chain state to S3 (or any
|
||||
// hanzoai/vfs-supported backend) with per-block age encryption.
|
||||
//
|
||||
// Subcommands:
|
||||
//
|
||||
// replicate — daemon: ticks at --interval, publishes a manifest
|
||||
// per tick, advances HEAD.json.
|
||||
// restore — one-shot: reads HEAD, applies full + increments
|
||||
// into an empty --path. Intended for init containers.
|
||||
// prune — garbage-collect manifests + blocks older than the
|
||||
// Nth-most-recent full snapshot.
|
||||
// version — print build version.
|
||||
//
|
||||
// Both daemon and one-shot modes read the same YAML config file
|
||||
// (default: /etc/zapdb-replicate/config.yaml). Flags override.
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
"os/signal"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"syscall"
|
||||
"time"
|
||||
|
||||
"github.com/spf13/cobra"
|
||||
"gopkg.in/yaml.v3"
|
||||
|
||||
"github.com/hanzoai/vfs"
|
||||
"github.com/hanzoai/vfs/pkg/backend"
|
||||
_ "github.com/hanzoai/vfs/pkg/backend/file"
|
||||
_ "github.com/hanzoai/vfs/pkg/backend/s3"
|
||||
"github.com/luxfi/age"
|
||||
|
||||
"github.com/luxfi/zapdb/cmd/replicate/internal/replica"
|
||||
"github.com/luxfi/zapdb/cmd/replicate/internal/restore"
|
||||
)
|
||||
|
||||
var version = "dev"
|
||||
|
||||
// Config is the YAML on-disk shape. Single config file may declare
|
||||
// multiple databases, each with its own replica destination.
|
||||
type Config struct {
|
||||
Databases []DatabaseConfig `yaml:"databases"`
|
||||
}
|
||||
|
||||
// DatabaseConfig wires one ZapDB to one or more replicas. The replicate
|
||||
// daemon spawns one goroutine per (database, replica) pair.
|
||||
type DatabaseConfig struct {
|
||||
Path string `yaml:"path"` // /data/db/mainnet
|
||||
Network string `yaml:"network"` // "mainnet" | "testnet" | ...
|
||||
Replicas []ReplicaConfig `yaml:"replicas"`
|
||||
}
|
||||
|
||||
// ReplicaConfig is one destination. Only S3 + file are supported today.
|
||||
type ReplicaConfig struct {
|
||||
Type string `yaml:"type"` // "s3" | "file"
|
||||
Endpoint string `yaml:"endpoint"` // for s3-compatible
|
||||
Bucket string `yaml:"bucket"`
|
||||
Prefix string `yaml:"prefix"`
|
||||
Region string `yaml:"region"`
|
||||
URL string `yaml:"url"` // alternative to the above; e.g. "file:///tmp/x"
|
||||
AgeRecipientsPath string `yaml:"age-recipients"` // path to file containing one recipient per line
|
||||
AgeKeyPath string `yaml:"age-key"` // path to identity file
|
||||
Interval time.Duration `yaml:"interval"` // default 60s
|
||||
FullSnapshotEvery time.Duration `yaml:"full-snapshot-every"` // default 168h
|
||||
CacheBytes int64 `yaml:"cache-bytes"` // vfs LRU cap, default 64MiB
|
||||
}
|
||||
|
||||
func main() {
|
||||
ctx, cancel := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM)
|
||||
defer cancel()
|
||||
|
||||
root := &cobra.Command{
|
||||
Use: "zapdb-replicate",
|
||||
Short: "Encrypted streaming replication for Lux ZapDB",
|
||||
Version: version,
|
||||
}
|
||||
|
||||
root.AddCommand(replicateCmd(ctx))
|
||||
root.AddCommand(restoreCmd(ctx))
|
||||
root.AddCommand(pruneCmd(ctx))
|
||||
root.AddCommand(versionCmd())
|
||||
|
||||
if err := root.ExecuteContext(ctx); err != nil {
|
||||
fmt.Fprintln(os.Stderr, "zapdb-replicate:", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
}
|
||||
|
||||
func versionCmd() *cobra.Command {
|
||||
return &cobra.Command{
|
||||
Use: "version",
|
||||
Short: "Print the version",
|
||||
Run: func(_ *cobra.Command, _ []string) {
|
||||
fmt.Println("zapdb-replicate", version)
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
func replicateCmd(ctx context.Context) *cobra.Command {
|
||||
var configPath string
|
||||
cmd := &cobra.Command{
|
||||
Use: "replicate",
|
||||
Short: "Run the replicator daemon (sidecar mode)",
|
||||
RunE: func(_ *cobra.Command, _ []string) error {
|
||||
cfg, err := loadConfig(configPath)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return runReplicate(ctx, cfg)
|
||||
},
|
||||
}
|
||||
cmd.Flags().StringVarP(&configPath, "config", "c", "/etc/zapdb-replicate/config.yaml", "config file path")
|
||||
return cmd
|
||||
}
|
||||
|
||||
func restoreCmd(ctx context.Context) *cobra.Command {
|
||||
var (
|
||||
dbPath string
|
||||
network string
|
||||
replicaURL string
|
||||
ageRecipsPath string
|
||||
ageKeyPath string
|
||||
cacheBytes int64
|
||||
force bool
|
||||
)
|
||||
cmd := &cobra.Command{
|
||||
Use: "restore",
|
||||
Short: "Restore a ZapDB from S3 (init-container mode)",
|
||||
RunE: func(_ *cobra.Command, _ []string) error {
|
||||
if dbPath == "" {
|
||||
return errors.New("--path is required")
|
||||
}
|
||||
if replicaURL == "" {
|
||||
return errors.New("--url is required (e.g. s3://bucket/prefix?endpoint=...)")
|
||||
}
|
||||
be, v, err := openReplica(ctx, replicaURL, ageRecipsPath, ageKeyPath, cacheBytes)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer be.Close()
|
||||
defer v.Close()
|
||||
return restore.Run(ctx, restore.Config{
|
||||
DBPath: dbPath,
|
||||
Network: network,
|
||||
VFS: v,
|
||||
Backend: be,
|
||||
SkipIfNotEmpty: !force,
|
||||
})
|
||||
},
|
||||
}
|
||||
cmd.Flags().StringVar(&dbPath, "path", "", "target ZapDB directory (e.g. /data/db/mainnet)")
|
||||
cmd.Flags().StringVar(&network, "network", "", "sanity-check network name (mainnet|testnet|...)")
|
||||
cmd.Flags().StringVar(&replicaURL, "url", "", "backend URL (e.g. s3://lux-snapshots/mainnet/zapdb?endpoint=http://minio:9000)")
|
||||
cmd.Flags().StringVar(&ageRecipsPath, "age-recipients", "", "path to age recipients file")
|
||||
cmd.Flags().StringVar(&ageKeyPath, "age-key", "", "path to age identity file (required for restore)")
|
||||
cmd.Flags().Int64Var(&cacheBytes, "cache-bytes", 64<<20, "vfs in-memory cache cap")
|
||||
cmd.Flags().BoolVar(&force, "force", false, "restore even if target dir is non-empty")
|
||||
return cmd
|
||||
}
|
||||
|
||||
func pruneCmd(ctx context.Context) *cobra.Command {
|
||||
var (
|
||||
replicaURL string
|
||||
ageRecipsPath string
|
||||
ageKeyPath string
|
||||
cacheBytes int64
|
||||
keepFulls int
|
||||
dryRun bool
|
||||
)
|
||||
cmd := &cobra.Command{
|
||||
Use: "prune",
|
||||
Short: "GC manifests + blocks older than the Nth-most-recent full snapshot",
|
||||
RunE: func(_ *cobra.Command, _ []string) error {
|
||||
if replicaURL == "" {
|
||||
return errors.New("--url is required")
|
||||
}
|
||||
be, v, err := openReplica(ctx, replicaURL, ageRecipsPath, ageKeyPath, cacheBytes)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer be.Close()
|
||||
defer v.Close()
|
||||
return restore.Prune(ctx, restore.PruneConfig{
|
||||
VFS: v,
|
||||
Backend: be,
|
||||
KeepFulls: keepFulls,
|
||||
DryRun: dryRun,
|
||||
})
|
||||
},
|
||||
}
|
||||
cmd.Flags().StringVar(&replicaURL, "url", "", "backend URL")
|
||||
cmd.Flags().StringVar(&ageRecipsPath, "age-recipients", "", "path to age recipients file")
|
||||
cmd.Flags().StringVar(&ageKeyPath, "age-key", "", "path to age identity file")
|
||||
cmd.Flags().Int64Var(&cacheBytes, "cache-bytes", 16<<20, "vfs in-memory cache cap")
|
||||
cmd.Flags().IntVar(&keepFulls, "keep-fulls", 4, "keep the N most recent full snapshots")
|
||||
cmd.Flags().BoolVar(&dryRun, "dry-run", false, "log only, do not delete")
|
||||
return cmd
|
||||
}
|
||||
|
||||
func runReplicate(ctx context.Context, cfg *Config) error {
|
||||
if len(cfg.Databases) == 0 {
|
||||
return errors.New("config has no databases")
|
||||
}
|
||||
|
||||
errCh := make(chan error, len(cfg.Databases))
|
||||
for _, dbCfg := range cfg.Databases {
|
||||
if len(dbCfg.Replicas) == 0 {
|
||||
return fmt.Errorf("db %s: no replicas configured", dbCfg.Path)
|
||||
}
|
||||
for _, rep := range dbCfg.Replicas {
|
||||
rep := rep // capture
|
||||
dbCfg := dbCfg
|
||||
go func() {
|
||||
errCh <- runOneReplica(ctx, dbCfg, rep)
|
||||
}()
|
||||
}
|
||||
}
|
||||
|
||||
// Wait for ctx cancel or any goroutine to surface a hard error.
|
||||
// Per-cycle errors are logged + retried; this only triggers on
|
||||
// setup failure.
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return ctx.Err()
|
||||
case err := <-errCh:
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
func runOneReplica(ctx context.Context, dbCfg DatabaseConfig, rep ReplicaConfig) error {
|
||||
url := replicaBackendURL(rep)
|
||||
be, v, err := openReplica(ctx, url, rep.AgeRecipientsPath, rep.AgeKeyPath, rep.CacheBytes)
|
||||
if err != nil {
|
||||
return fmt.Errorf("db %s: open replica %s: %w", dbCfg.Path, url, err)
|
||||
}
|
||||
defer be.Close()
|
||||
defer v.Close()
|
||||
|
||||
r, err := replica.New(replica.Config{
|
||||
DBPath: dbCfg.Path,
|
||||
Network: dbCfg.Network,
|
||||
VFS: v,
|
||||
Backend: be,
|
||||
Interval: rep.Interval,
|
||||
FullSnapshotEvery: rep.FullSnapshotEvery,
|
||||
AgeSchemeLabel: ageSchemeLabel(rep),
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return r.Run(ctx)
|
||||
}
|
||||
|
||||
func replicaBackendURL(rep ReplicaConfig) string {
|
||||
if rep.URL != "" {
|
||||
return rep.URL
|
||||
}
|
||||
if rep.Type == "" || rep.Type == "s3" {
|
||||
u := "s3://" + rep.Bucket
|
||||
if rep.Prefix != "" {
|
||||
u += "/" + strings.TrimLeft(rep.Prefix, "/")
|
||||
}
|
||||
q := []string{}
|
||||
if rep.Endpoint != "" {
|
||||
q = append(q, "endpoint="+rep.Endpoint)
|
||||
}
|
||||
if rep.Region != "" {
|
||||
q = append(q, "region="+rep.Region)
|
||||
}
|
||||
if len(q) > 0 {
|
||||
u += "?" + strings.Join(q, "&")
|
||||
}
|
||||
return u
|
||||
}
|
||||
return rep.URL
|
||||
}
|
||||
|
||||
func ageSchemeLabel(rep ReplicaConfig) string {
|
||||
if rep.AgeRecipientsPath == "" {
|
||||
return "none"
|
||||
}
|
||||
return "age" // luxfi/age handles X25519+ML-KEM-768 transparently
|
||||
}
|
||||
|
||||
func openReplica(ctx context.Context, url, recipsPath, keyPath string, cacheBytes int64) (backend.Backend, *vfs.VFS, error) {
|
||||
be, err := backend.Open(ctx, url)
|
||||
if err != nil {
|
||||
return nil, nil, fmt.Errorf("backend.Open(%s): %w", url, err)
|
||||
}
|
||||
recs, ids, err := loadAge(recipsPath, keyPath)
|
||||
if err != nil {
|
||||
be.Close()
|
||||
return nil, nil, err
|
||||
}
|
||||
crypto, err := vfs.NewCrypto(recs, ids)
|
||||
if err != nil {
|
||||
be.Close()
|
||||
return nil, nil, err
|
||||
}
|
||||
if cacheBytes <= 0 {
|
||||
cacheBytes = 64 << 20
|
||||
}
|
||||
v, err := vfs.New(vfs.Config{
|
||||
Backend: be,
|
||||
Crypto: crypto,
|
||||
CacheMax: cacheBytes,
|
||||
})
|
||||
if err != nil {
|
||||
be.Close()
|
||||
return nil, nil, err
|
||||
}
|
||||
return be, v, nil
|
||||
}
|
||||
|
||||
func loadAge(recipientsPath, keyPath string) ([]age.Recipient, []age.Identity, error) {
|
||||
var recs []age.Recipient
|
||||
if recipientsPath != "" {
|
||||
body, err := os.ReadFile(recipientsPath)
|
||||
if err != nil {
|
||||
return nil, nil, fmt.Errorf("read --age-recipients %s: %w", recipientsPath, err)
|
||||
}
|
||||
parsed, err := age.ParseRecipients(strings.NewReader(string(body)))
|
||||
if err != nil {
|
||||
return nil, nil, fmt.Errorf("parse age recipients: %w", err)
|
||||
}
|
||||
recs = parsed
|
||||
}
|
||||
var ids []age.Identity
|
||||
if keyPath != "" {
|
||||
body, err := os.ReadFile(keyPath)
|
||||
if err != nil {
|
||||
return nil, nil, fmt.Errorf("read --age-key %s: %w", keyPath, err)
|
||||
}
|
||||
parsed, err := age.ParseIdentities(strings.NewReader(string(body)))
|
||||
if err != nil {
|
||||
return nil, nil, fmt.Errorf("parse age identities: %w", err)
|
||||
}
|
||||
ids = parsed
|
||||
}
|
||||
if len(recs) == 0 && len(ids) == 0 {
|
||||
return nil, nil, errors.New("at least one of --age-recipients or --age-key required")
|
||||
}
|
||||
return recs, ids, nil
|
||||
}
|
||||
|
||||
func loadConfig(path string) (*Config, error) {
|
||||
if path == "" {
|
||||
return nil, errors.New("config path required")
|
||||
}
|
||||
abs, err := filepath.Abs(path)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
body, err := os.ReadFile(abs)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("read config %s: %w", abs, err)
|
||||
}
|
||||
var cfg Config
|
||||
if err := yaml.Unmarshal(body, &cfg); err != nil {
|
||||
return nil, fmt.Errorf("decode config %s: %w", abs, err)
|
||||
}
|
||||
return &cfg, nil
|
||||
}
|
||||
|
||||
var _ = io.EOF // keep imports stable across edits
|
||||
@@ -0,0 +1,5 @@
|
||||
package main_test
|
||||
|
||||
import "os"
|
||||
|
||||
func makeDir(p string) error { return os.MkdirAll(p, 0o755) }
|
||||
@@ -0,0 +1,188 @@
|
||||
// End-to-end test of the replicate → restore cycle.
|
||||
//
|
||||
// Boots an in-memory hanzoai/vfs backed by a file:// backend, writes a
|
||||
// 1000-entry ZapDB, takes a full Backup() through the chunk writer,
|
||||
// adds 100 more entries, takes an incremental Backup() into a second
|
||||
// manifest, then restores both into a fresh ZapDB and verifies every
|
||||
// key is present.
|
||||
//
|
||||
// The test bypasses the replicator daemon's loop (and bypasses the
|
||||
// luxd-coexistence read-only open path) because we own both writes
|
||||
// and reads in-process.
|
||||
package main_test
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
|
||||
"github.com/hanzoai/vfs"
|
||||
"github.com/hanzoai/vfs/pkg/backend"
|
||||
_ "github.com/hanzoai/vfs/pkg/backend/file"
|
||||
"github.com/luxfi/age"
|
||||
badger "github.com/luxfi/zapdb"
|
||||
"github.com/stretchr/testify/require"
|
||||
|
||||
"github.com/luxfi/zapdb/cmd/replicate/internal/manifest"
|
||||
"github.com/luxfi/zapdb/cmd/replicate/internal/replica"
|
||||
)
|
||||
|
||||
func TestReplicateRestoreRoundTrip(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
|
||||
root := t.TempDir()
|
||||
srcPath := filepath.Join(root, "src")
|
||||
dstPath := filepath.Join(root, "dst")
|
||||
storePath := filepath.Join(root, "store")
|
||||
require.NoError(t, mkAll(srcPath, dstPath, storePath))
|
||||
|
||||
// vfs over file:// + age (classical X25519 is fine for tests)
|
||||
id, err := age.GenerateX25519Identity()
|
||||
require.NoError(t, err)
|
||||
rec := id.Recipient()
|
||||
|
||||
be, err := backend.Open(ctx, "file://"+storePath)
|
||||
require.NoError(t, err)
|
||||
defer be.Close()
|
||||
|
||||
crypto, err := vfs.NewCrypto([]age.Recipient{rec}, []age.Identity{id})
|
||||
require.NoError(t, err)
|
||||
v, err := vfs.New(vfs.Config{Backend: be, Crypto: crypto, CacheMax: 4 << 20})
|
||||
require.NoError(t, err)
|
||||
defer v.Close()
|
||||
|
||||
// Phase 1 — write 1000 keys into the source DB, full backup.
|
||||
src := openDB(t, srcPath)
|
||||
writeKeys(t, src, 0, 1000)
|
||||
full := streamBackup(t, ctx, src, v, 0)
|
||||
require.NoError(t, src.Close())
|
||||
|
||||
require.Equal(t, manifest.TypeFull, full.Type)
|
||||
require.Greater(t, full.Until, uint64(0), "full backup must advance the badger watermark")
|
||||
require.NotEmpty(t, full.Blocks, "full backup must emit at least one block")
|
||||
|
||||
store := manifest.NewStore(be)
|
||||
fullKey, err := store.PutManifest(ctx, full)
|
||||
require.NoError(t, err)
|
||||
head := &manifest.HEAD{
|
||||
Network: "test",
|
||||
CurrentVersion: full.Until,
|
||||
LastFullManifest: fullKey,
|
||||
}
|
||||
require.NoError(t, store.PutHead(ctx, head))
|
||||
|
||||
// Phase 2 — append 100 more keys, incremental backup since
|
||||
// full.Until.
|
||||
src = openDB(t, srcPath)
|
||||
writeKeys(t, src, 1000, 1100)
|
||||
inc := streamBackup(t, ctx, src, v, full.Until)
|
||||
require.NoError(t, src.Close())
|
||||
|
||||
require.Equal(t, manifest.TypeIncremental, inc.Type)
|
||||
require.Equal(t, full.Until, inc.Since)
|
||||
require.Greater(t, inc.Until, inc.Since)
|
||||
require.NotEmpty(t, inc.Blocks, "incremental backup with 100 new keys must emit blocks")
|
||||
|
||||
incKey, err := store.PutManifest(ctx, inc)
|
||||
require.NoError(t, err)
|
||||
head.CurrentVersion = inc.Until
|
||||
head.Increments = append(head.Increments, incKey)
|
||||
require.NoError(t, store.PutHead(ctx, head))
|
||||
|
||||
// Phase 3 — restore into a fresh DB and verify every key.
|
||||
dst := openDB(t, dstPath)
|
||||
|
||||
// Apply full
|
||||
fullM, err := store.GetManifest(ctx, fullKey)
|
||||
require.NoError(t, err)
|
||||
rFull := replica.NewChunkReader(ctx, v, fullM.Blocks, fullM.PlainSize)
|
||||
require.NoError(t, dst.Load(rFull, 16))
|
||||
|
||||
// Apply incremental
|
||||
incM, err := store.GetManifest(ctx, incKey)
|
||||
require.NoError(t, err)
|
||||
rInc := replica.NewChunkReader(ctx, v, incM.Blocks, incM.PlainSize)
|
||||
require.NoError(t, dst.Load(rInc, 16))
|
||||
|
||||
verifyKeys(t, dst, 0, 1100)
|
||||
require.NoError(t, dst.Close())
|
||||
}
|
||||
|
||||
func streamBackup(t *testing.T, ctx context.Context, db *badger.DB, v *vfs.VFS, since uint64) *manifest.Manifest {
|
||||
t.Helper()
|
||||
cw := replica.NewChunkWriter(ctx, v)
|
||||
until, err := db.Backup(cw, since)
|
||||
require.NoError(t, err)
|
||||
require.NoError(t, cw.Close())
|
||||
|
||||
mType := manifest.TypeFull
|
||||
if since > 0 {
|
||||
mType = manifest.TypeIncremental
|
||||
}
|
||||
return &manifest.Manifest{
|
||||
Version: 1,
|
||||
Type: mType,
|
||||
Since: since,
|
||||
Until: until,
|
||||
Network: "test",
|
||||
Blocks: cw.Blocks(),
|
||||
PlainSize: cw.PlainSize(),
|
||||
BlockSize: vfs.BlockSize,
|
||||
}
|
||||
}
|
||||
|
||||
func openDB(t *testing.T, path string) *badger.DB {
|
||||
t.Helper()
|
||||
opts := badger.DefaultOptions(path)
|
||||
opts.Logger = nil
|
||||
opts.SyncWrites = false // test perf
|
||||
db, err := badger.Open(opts)
|
||||
require.NoError(t, err)
|
||||
return db
|
||||
}
|
||||
|
||||
func writeKeys(t *testing.T, db *badger.DB, start, end int) {
|
||||
t.Helper()
|
||||
require.NoError(t, db.Update(func(txn *badger.Txn) error {
|
||||
for i := start; i < end; i++ {
|
||||
k := []byte(fmt.Sprintf("k%06d", i))
|
||||
v := []byte(fmt.Sprintf("v%06d-padding-to-make-the-stream-non-trivial-and-flush-multiple-pages", i))
|
||||
if err := txn.Set(k, v); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}))
|
||||
}
|
||||
|
||||
func verifyKeys(t *testing.T, db *badger.DB, start, end int) {
|
||||
t.Helper()
|
||||
require.NoError(t, db.View(func(txn *badger.Txn) error {
|
||||
for i := start; i < end; i++ {
|
||||
k := []byte(fmt.Sprintf("k%06d", i))
|
||||
item, err := txn.Get(k)
|
||||
if err != nil {
|
||||
return fmt.Errorf("get k%06d: %w", i, err)
|
||||
}
|
||||
val, err := item.ValueCopy(nil)
|
||||
if err != nil {
|
||||
return fmt.Errorf("valcopy k%06d: %w", i, err)
|
||||
}
|
||||
wantPrefix := fmt.Sprintf("v%06d", i)
|
||||
if got := string(val); got[:len(wantPrefix)] != wantPrefix {
|
||||
return fmt.Errorf("k%06d: want prefix %q, got %q", i, wantPrefix, got)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}))
|
||||
}
|
||||
|
||||
func mkAll(paths ...string) error {
|
||||
for _, p := range paths {
|
||||
if err := makeDir(p); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,26 @@
|
||||
# Build context must be the repo root (luxfi/zapdb).
|
||||
# docker build -t zapdb-replicate -f deploy/replicate/Dockerfile .
|
||||
FROM golang:1.26.4-bookworm AS build
|
||||
|
||||
WORKDIR /src
|
||||
COPY go.mod go.sum ./
|
||||
RUN --mount=type=cache,target=/go/pkg/mod \
|
||||
--mount=type=cache,target=/root/.cache/go-build \
|
||||
go mod download
|
||||
|
||||
COPY . .
|
||||
ARG VERSION=dev
|
||||
RUN --mount=type=cache,target=/go/pkg/mod \
|
||||
--mount=type=cache,target=/root/.cache/go-build \
|
||||
CGO_ENABLED=0 GOOS=linux GOARCH=amd64 \
|
||||
go build \
|
||||
-trimpath \
|
||||
-ldflags "-s -w -X main.version=${VERSION}" \
|
||||
-o /out/zapdb-replicate \
|
||||
./cmd/replicate
|
||||
|
||||
FROM gcr.io/distroless/static-debian12:nonroot
|
||||
COPY --from=build /out/zapdb-replicate /usr/local/bin/zapdb-replicate
|
||||
USER nonroot:nonroot
|
||||
ENTRYPOINT ["/usr/local/bin/zapdb-replicate"]
|
||||
CMD ["replicate", "--config", "/etc/zapdb-replicate/config.yaml"]
|
||||
@@ -0,0 +1,12 @@
|
||||
apiVersion: v2
|
||||
name: zapdb-replicate
|
||||
description: Encrypted streaming replication for Lux ZapDB chain state.
|
||||
type: application
|
||||
version: 0.1.0
|
||||
appVersion: "0.1.0"
|
||||
home: https://github.com/luxfi/zapdb/tree/main/cmd/replicate
|
||||
sources:
|
||||
- https://github.com/luxfi/zapdb
|
||||
maintainers:
|
||||
- name: luxfi
|
||||
email: ops@lux.network
|
||||
@@ -0,0 +1,20 @@
|
||||
{{- define "zapdb-replicate.name" -}}
|
||||
{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" -}}
|
||||
{{- end -}}
|
||||
|
||||
{{- define "zapdb-replicate.fullname" -}}
|
||||
{{- if .Values.fullnameOverride -}}
|
||||
{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" -}}
|
||||
{{- else -}}
|
||||
{{- $name := default .Chart.Name .Values.nameOverride -}}
|
||||
{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" -}}
|
||||
{{- end -}}
|
||||
{{- end -}}
|
||||
|
||||
{{- define "zapdb-replicate.labels" -}}
|
||||
helm.sh/chart: {{ printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" }}
|
||||
app.kubernetes.io/name: {{ include "zapdb-replicate.name" . }}
|
||||
app.kubernetes.io/instance: {{ .Release.Name }}
|
||||
app.kubernetes.io/version: {{ .Chart.AppVersion | quote }}
|
||||
app.kubernetes.io/managed-by: {{ .Release.Service }}
|
||||
{{- end -}}
|
||||
@@ -0,0 +1,22 @@
|
||||
apiVersion: v1
|
||||
kind: ConfigMap
|
||||
metadata:
|
||||
name: {{ include "zapdb-replicate.fullname" . }}
|
||||
labels:
|
||||
{{- include "zapdb-replicate.labels" . | nindent 4 }}
|
||||
data:
|
||||
config.yaml: |
|
||||
databases:
|
||||
- path: {{ .Values.dbPath }}
|
||||
network: {{ .Values.network }}
|
||||
replicas:
|
||||
- type: s3
|
||||
endpoint: {{ .Values.s3.endpoint }}
|
||||
bucket: {{ .Values.s3.bucket }}
|
||||
prefix: {{ .Values.s3.prefix }}
|
||||
region: {{ .Values.s3.region }}
|
||||
age-recipients: {{ .Values.age.recipientsPath }}
|
||||
age-key: {{ .Values.age.identityPath }}
|
||||
interval: {{ .Values.schedule.interval }}
|
||||
full-snapshot-every: {{ .Values.schedule.fullSnapshotEvery }}
|
||||
cache-bytes: {{ .Values.schedule.cacheBytes }}
|
||||
@@ -0,0 +1,41 @@
|
||||
image:
|
||||
repository: ghcr.io/luxfi/zapdb-replicate
|
||||
# Image is built from luxfi/zapdb (build context = repo root, Dockerfile = deploy/replicate/Dockerfile).
|
||||
# Tag tracks the luxfi/zapdb release (one binary cut from the parent module).
|
||||
tag: v0.1.0
|
||||
pullPolicy: IfNotPresent
|
||||
|
||||
# When true, this chart only renders a sidecar+initContainer to be
|
||||
# composed into another StatefulSet via helm dependency. The default
|
||||
# (false) renders a standalone test deployment.
|
||||
asInjector: false
|
||||
|
||||
# Source-of-truth ZapDB directory (mounted from the parent StatefulSet
|
||||
# via shared volume).
|
||||
dbPath: /data/db/mainnet
|
||||
network: mainnet
|
||||
|
||||
s3:
|
||||
endpoint: http://s3.lux-system.svc.cluster.local:9000
|
||||
bucket: lux-snapshots
|
||||
prefix: mainnet/zapdb
|
||||
region: us-east-1
|
||||
credentialsSecret: zapdb-replicate-s3-mainnet
|
||||
|
||||
age:
|
||||
recipientsPath: /etc/zapdb-replicate/age/recipients.txt
|
||||
identityPath: /etc/zapdb-replicate/age/identity.txt
|
||||
keysSecret: zapdb-replicate-age-mainnet
|
||||
|
||||
schedule:
|
||||
interval: 60s
|
||||
fullSnapshotEvery: 168h
|
||||
cacheBytes: 67108864
|
||||
|
||||
resources:
|
||||
requests:
|
||||
cpu: 100m
|
||||
memory: 256Mi
|
||||
limits:
|
||||
cpu: 500m
|
||||
memory: 512Mi
|
||||
@@ -0,0 +1,70 @@
|
||||
# Init container pattern: zapdb-replicate runs ONCE before luxd boots
|
||||
# to restore an empty /data/db/<network> from the most recent S3
|
||||
# snapshot chain. After it exits successfully, the main luxd container
|
||||
# starts and opens a populated DB.
|
||||
#
|
||||
# Pair this with the sidecar.yaml for ongoing replication; the init
|
||||
# container only matters on a cold boot (fresh PVC).
|
||||
apiVersion: apps/v1
|
||||
kind: StatefulSet
|
||||
metadata:
|
||||
name: luxd-mainnet
|
||||
namespace: lux
|
||||
spec:
|
||||
serviceName: luxd-mainnet
|
||||
replicas: 1
|
||||
selector:
|
||||
matchLabels:
|
||||
app: luxd-mainnet
|
||||
template:
|
||||
metadata:
|
||||
labels:
|
||||
app: luxd-mainnet
|
||||
spec:
|
||||
initContainers:
|
||||
- name: zapdb-restore
|
||||
image: ghcr.io/luxfi/zapdb-replicate:v0.1.0
|
||||
args:
|
||||
- restore
|
||||
- --path=/data/db/mainnet
|
||||
- --network=mainnet
|
||||
- --url=s3://lux-snapshots/mainnet/zapdb?endpoint=http://s3.lux-system.svc.cluster.local:9000®ion=us-east-1
|
||||
- --age-key=/etc/zapdb-replicate/age/identity.txt
|
||||
- --cache-bytes=134217728
|
||||
volumeMounts:
|
||||
- name: data
|
||||
mountPath: /data
|
||||
- name: zapdb-age-keys
|
||||
mountPath: /etc/zapdb-replicate/age
|
||||
readOnly: true
|
||||
- name: zapdb-s3-creds
|
||||
mountPath: /etc/zapdb-replicate/aws
|
||||
readOnly: true
|
||||
env:
|
||||
- name: AWS_SHARED_CREDENTIALS_FILE
|
||||
value: /etc/zapdb-replicate/aws/credentials
|
||||
- name: AWS_REGION
|
||||
value: us-east-1
|
||||
containers:
|
||||
- name: luxd
|
||||
image: ghcr.io/luxfi/node:v1.27.6
|
||||
args: ["--data-dir=/data", "--network-id=mainnet"]
|
||||
volumeMounts:
|
||||
- name: data
|
||||
mountPath: /data
|
||||
volumes:
|
||||
- name: zapdb-age-keys
|
||||
secret:
|
||||
secretName: zapdb-replicate-age-mainnet
|
||||
- name: zapdb-s3-creds
|
||||
secret:
|
||||
secretName: zapdb-replicate-s3-mainnet
|
||||
volumeClaimTemplates:
|
||||
- metadata:
|
||||
name: data
|
||||
spec:
|
||||
accessModes: [ReadWriteOnce]
|
||||
storageClassName: do-block-storage
|
||||
resources:
|
||||
requests:
|
||||
storage: 1000Gi
|
||||
@@ -0,0 +1,104 @@
|
||||
# Sidecar pattern: zapdb-replicate runs in the same Pod as luxd, opens
|
||||
# /data/db/<network> in read-only + bypass-lock-guard mode, streams
|
||||
# Backup() through hanzoai/vfs into S3.
|
||||
#
|
||||
# Tag MUST be a v* semver (semver-only policy 2026-04-30). Never :latest.
|
||||
apiVersion: apps/v1
|
||||
kind: StatefulSet
|
||||
metadata:
|
||||
name: luxd-mainnet
|
||||
namespace: lux
|
||||
spec:
|
||||
serviceName: luxd-mainnet
|
||||
replicas: 1
|
||||
selector:
|
||||
matchLabels:
|
||||
app: luxd-mainnet
|
||||
template:
|
||||
metadata:
|
||||
labels:
|
||||
app: luxd-mainnet
|
||||
spec:
|
||||
containers:
|
||||
- name: luxd
|
||||
image: ghcr.io/luxfi/node:v1.27.6
|
||||
args: ["--data-dir=/data", "--network-id=mainnet"]
|
||||
volumeMounts:
|
||||
- name: data
|
||||
mountPath: /data
|
||||
|
||||
- name: zapdb-replicate
|
||||
image: ghcr.io/luxfi/zapdb-replicate:v0.1.0
|
||||
imagePullPolicy: IfNotPresent
|
||||
args:
|
||||
- replicate
|
||||
- --config=/etc/zapdb-replicate/config.yaml
|
||||
volumeMounts:
|
||||
- name: data
|
||||
mountPath: /data
|
||||
readOnly: false # we write the local state file
|
||||
- name: zapdb-config
|
||||
mountPath: /etc/zapdb-replicate
|
||||
readOnly: true
|
||||
- name: zapdb-age-keys
|
||||
mountPath: /etc/zapdb-replicate/age
|
||||
readOnly: true
|
||||
- name: zapdb-s3-creds
|
||||
mountPath: /etc/zapdb-replicate/aws
|
||||
readOnly: true
|
||||
env:
|
||||
- name: AWS_SHARED_CREDENTIALS_FILE
|
||||
value: /etc/zapdb-replicate/aws/credentials
|
||||
- name: AWS_REGION
|
||||
value: us-east-1
|
||||
resources:
|
||||
requests:
|
||||
cpu: 100m
|
||||
memory: 256Mi
|
||||
limits:
|
||||
cpu: 500m
|
||||
memory: 512Mi
|
||||
|
||||
volumes:
|
||||
- name: zapdb-config
|
||||
configMap:
|
||||
name: zapdb-replicate-mainnet
|
||||
- name: zapdb-age-keys
|
||||
secret:
|
||||
secretName: zapdb-replicate-age-mainnet
|
||||
- name: zapdb-s3-creds
|
||||
secret:
|
||||
secretName: zapdb-replicate-s3-mainnet
|
||||
|
||||
volumeClaimTemplates:
|
||||
- metadata:
|
||||
name: data
|
||||
spec:
|
||||
accessModes: [ReadWriteOnce]
|
||||
storageClassName: do-block-storage
|
||||
resources:
|
||||
requests:
|
||||
storage: 1000Gi
|
||||
|
||||
---
|
||||
apiVersion: v1
|
||||
kind: ConfigMap
|
||||
metadata:
|
||||
name: zapdb-replicate-mainnet
|
||||
namespace: lux
|
||||
data:
|
||||
config.yaml: |
|
||||
databases:
|
||||
- path: /data/db/mainnet
|
||||
network: mainnet
|
||||
replicas:
|
||||
- type: s3
|
||||
endpoint: http://s3.lux-system.svc.cluster.local:9000
|
||||
bucket: lux-snapshots
|
||||
prefix: mainnet/zapdb
|
||||
region: us-east-1
|
||||
age-recipients: /etc/zapdb-replicate/age/recipients.txt
|
||||
age-key: /etc/zapdb-replicate/age/identity.txt
|
||||
interval: 60s
|
||||
full-snapshot-every: 168h
|
||||
cache-bytes: 67108864 # 64 MiB
|
||||
@@ -4,44 +4,94 @@ go 1.26.4
|
||||
|
||||
require (
|
||||
github.com/cespare/xxhash/v2 v2.3.0
|
||||
github.com/dgraph-io/ristretto/v2 v2.2.0
|
||||
github.com/dgraph-io/ristretto/v2 v2.4.0
|
||||
github.com/dustin/go-humanize v1.0.1
|
||||
github.com/google/flatbuffers v25.2.10+incompatible
|
||||
github.com/google/flatbuffers v25.12.19+incompatible
|
||||
github.com/hanzoai/vfs v0.3.2-0.20260519065328-c67215937fc2
|
||||
github.com/hanzos3/go-sdk v1.0.2
|
||||
github.com/klauspost/compress v1.18.2
|
||||
github.com/luxfi/age v1.4.0
|
||||
github.com/spf13/cobra v1.9.1
|
||||
github.com/stretchr/testify v1.10.0
|
||||
github.com/klauspost/compress v1.18.5
|
||||
github.com/luxfi/age v1.5.0
|
||||
github.com/luxfi/log v1.4.3
|
||||
github.com/spf13/cobra v1.10.2
|
||||
github.com/stretchr/testify v1.11.1
|
||||
go.opentelemetry.io/contrib/zpages v0.62.0
|
||||
go.opentelemetry.io/otel v1.37.0
|
||||
golang.org/x/sys v0.39.0
|
||||
google.golang.org/protobuf v1.36.7
|
||||
go.opentelemetry.io/otel v1.43.0
|
||||
golang.org/x/sys v0.43.0
|
||||
google.golang.org/protobuf v1.36.11
|
||||
gopkg.in/yaml.v3 v3.0.1
|
||||
)
|
||||
|
||||
require (
|
||||
filippo.io/hpke v0.4.0 // indirect
|
||||
github.com/davecgh/go-spew v1.1.1 // indirect
|
||||
github.com/andybalholm/brotli v1.2.1 // indirect
|
||||
github.com/aws/aws-sdk-go-v2 v1.41.2 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.5 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/config v1.32.8 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/credentials v1.19.8 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.17 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.18 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.18 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/internal/ini v1.8.4 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.18 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.5 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.10 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.18 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.18 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/s3 v1.96.2 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/signin v1.0.5 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/sso v1.30.9 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.14 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/sts v1.41.6 // indirect
|
||||
github.com/aws/smithy-go v1.24.1 // indirect
|
||||
github.com/cenkalti/backoff v2.2.1+incompatible // indirect
|
||||
github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect
|
||||
github.com/go-ini/ini v1.67.0 // indirect
|
||||
github.com/go-logr/logr v1.4.3 // indirect
|
||||
github.com/go-logr/stdr v1.2.2 // indirect
|
||||
github.com/gofiber/fiber/v3 v3.2.0 // indirect
|
||||
github.com/gofiber/schema v1.7.1 // indirect
|
||||
github.com/gofiber/utils/v2 v2.0.4 // indirect
|
||||
github.com/google/uuid v1.6.0 // indirect
|
||||
github.com/grandcat/zeroconf v1.0.0 // indirect
|
||||
github.com/hanzoai/cloud v0.1.0 // indirect
|
||||
github.com/hanzoai/zip v0.2.0 // indirect
|
||||
github.com/inconshreveable/mousetrap v1.1.0 // indirect
|
||||
github.com/klauspost/cpuid/v2 v2.2.11 // indirect
|
||||
github.com/klauspost/cpuid/v2 v2.3.0 // indirect
|
||||
github.com/klauspost/crc32 v1.3.0 // indirect
|
||||
github.com/luxfi/mdns v0.1.0 // indirect
|
||||
github.com/luxfi/zap v0.3.1 // indirect
|
||||
github.com/mattn/go-colorable v0.1.14 // indirect
|
||||
github.com/mattn/go-isatty v0.0.21 // indirect
|
||||
github.com/miekg/dns v1.1.72 // indirect
|
||||
github.com/minio/crc64nvme v1.1.1 // indirect
|
||||
github.com/minio/md5-simd v1.1.2 // indirect
|
||||
github.com/philhofer/fwd v1.2.0 // indirect
|
||||
github.com/pmezard/go-difflib v1.0.0 // indirect
|
||||
github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect
|
||||
github.com/rs/xid v1.6.0 // indirect
|
||||
github.com/spf13/pflag v1.0.6 // indirect
|
||||
github.com/tinylib/msgp v1.6.1 // indirect
|
||||
go.opentelemetry.io/auto/sdk v1.1.0 // indirect
|
||||
go.opentelemetry.io/otel/metric v1.37.0 // indirect
|
||||
go.opentelemetry.io/otel/sdk v1.37.0 // indirect
|
||||
go.opentelemetry.io/otel/trace v1.37.0 // indirect
|
||||
github.com/spf13/pflag v1.0.10 // indirect
|
||||
github.com/tinylib/msgp v1.6.4 // indirect
|
||||
github.com/valyala/bytebufferpool v1.0.0 // indirect
|
||||
github.com/valyala/fasthttp v1.70.0 // indirect
|
||||
github.com/zeebo/blake3 v0.2.4 // indirect
|
||||
go.opentelemetry.io/auto/sdk v1.2.1 // indirect
|
||||
go.opentelemetry.io/otel/metric v1.43.0 // indirect
|
||||
go.opentelemetry.io/otel/sdk v1.43.0 // indirect
|
||||
go.opentelemetry.io/otel/trace v1.43.0 // indirect
|
||||
go.yaml.in/yaml/v3 v3.0.4 // indirect
|
||||
golang.org/x/crypto v0.46.0 // indirect
|
||||
golang.org/x/net v0.48.0 // indirect
|
||||
golang.org/x/text v0.32.0 // indirect
|
||||
gopkg.in/yaml.v3 v3.0.1 // indirect
|
||||
golang.org/x/crypto v0.50.0 // indirect
|
||||
golang.org/x/mod v0.34.0 // indirect
|
||||
golang.org/x/net v0.53.0 // indirect
|
||||
golang.org/x/sync v0.20.0 // indirect
|
||||
golang.org/x/text v0.36.0 // indirect
|
||||
golang.org/x/tools v0.43.0 // indirect
|
||||
gopkg.in/natefinch/lumberjack.v2 v2.2.1 // indirect
|
||||
)
|
||||
|
||||
// Local source-of-truth replacements (forwarding to the working trees
|
||||
// so we build against unreleased fixes). Remove these for production
|
||||
// builds after the matching upstream tags ship.
|
||||
replace (
|
||||
github.com/hanzoai/cloud => ../../hanzo/cloud
|
||||
github.com/hanzoai/vfs => ../../hanzo/vfs
|
||||
github.com/luxfi/age => ../age
|
||||
)
|
||||
|
||||
@@ -2,17 +2,61 @@ c2sp.org/CCTV/age v0.0.0-20251208015420-e9274a7bdbfd h1:ZLsPO6WdZ5zatV4UfVpr7oAw
|
||||
c2sp.org/CCTV/age v0.0.0-20251208015420-e9274a7bdbfd/go.mod h1:SrHC2C7r5GkDk8R+NFVzYy/sdj0Ypg9htaPXQq5Cqeo=
|
||||
filippo.io/hpke v0.4.0 h1:p575VVQ6ted4pL+it6M00V/f2qTZITO0zgmdKCkd5+A=
|
||||
filippo.io/hpke v0.4.0/go.mod h1:EmAN849/P3qdeK+PCMkDpDm83vRHM5cDipBJ8xbQLVY=
|
||||
github.com/andybalholm/brotli v1.2.1 h1:R+f5xP285VArJDRgowrfb9DqL18yVK0gKAW/F+eTWro=
|
||||
github.com/andybalholm/brotli v1.2.1/go.mod h1:rzTDkvFWvIrjDXZHkuS16NPggd91W3kUSvPlQ1pLaKY=
|
||||
github.com/aws/aws-sdk-go-v2 v1.41.2 h1:LuT2rzqNQsauaGkPK/7813XxcZ3o3yePY0Iy891T2ls=
|
||||
github.com/aws/aws-sdk-go-v2 v1.41.2/go.mod h1:IvvlAZQXvTXznUPfRVfryiG1fbzE2NGK6m9u39YQ+S4=
|
||||
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.5 h1:zWFmPmgw4sveAYi1mRqG+E/g0461cJ5M4bJ8/nc6d3Q=
|
||||
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.5/go.mod h1:nVUlMLVV8ycXSb7mSkcNu9e3v/1TJq2RTlrPwhYWr5c=
|
||||
github.com/aws/aws-sdk-go-v2/config v1.32.8 h1:iu+64gwDKEoKnyTQskSku72dAwggKI5sV6rNvgSMpMs=
|
||||
github.com/aws/aws-sdk-go-v2/config v1.32.8/go.mod h1:MI2XvA+qDi3i9AJxX1E2fu730syEBzp/jnXrjxuHwgI=
|
||||
github.com/aws/aws-sdk-go-v2/credentials v1.19.8 h1:Jp2JYH1lRT3KhX4mshHPvVYsR5qqRec3hGvEarNYoR0=
|
||||
github.com/aws/aws-sdk-go-v2/credentials v1.19.8/go.mod h1:fZG9tuvyVfxknv1rKibIz3DobRaFw1Poe8IKtXB3XYY=
|
||||
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.17 h1:I0GyV8wiYrP8XpA70g1HBcQO1JlQxCMTW9npl5UbDHY=
|
||||
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.17/go.mod h1:tyw7BOl5bBe/oqvoIeECFJjMdzXoa/dfVz3QQ5lgHGA=
|
||||
github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.18 h1:F43zk1vemYIqPAwhjTjYIz0irU2EY7sOb/F5eJ3HuyM=
|
||||
github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.18/go.mod h1:w1jdlZXrGKaJcNoL+Nnrj+k5wlpGXqnNrKoP22HvAug=
|
||||
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.18 h1:xCeWVjj0ki0l3nruoyP2slHsGArMxeiiaoPN5QZH6YQ=
|
||||
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.18/go.mod h1:r/eLGuGCBw6l36ZRWiw6PaZwPXb6YOj+i/7MizNl5/k=
|
||||
github.com/aws/aws-sdk-go-v2/internal/ini v1.8.4 h1:WKuaxf++XKWlHWu9ECbMlha8WOEGm0OUEZqm4K/Gcfk=
|
||||
github.com/aws/aws-sdk-go-v2/internal/ini v1.8.4/go.mod h1:ZWy7j6v1vWGmPReu0iSGvRiise4YI5SkR3OHKTZ6Wuc=
|
||||
github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.18 h1:eZioDaZGJ0tMM4gzmkNIO2aAoQd+je7Ug7TkvAzlmkU=
|
||||
github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.18/go.mod h1:CCXwUKAJdoWr6/NcxZ+zsiPr6oH/Q5aTooRGYieAyj4=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.5 h1:CeY9LUdur+Dxoeldqoun6y4WtJ3RQtzk0JMP2gfUay0=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.5/go.mod h1:AZLZf2fMaahW5s/wMRciu1sYbdsikT/UHwbUjOdEVTc=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.10 h1:fJvQ5mIBVfKtiyx0AHY6HeWcRX5LGANLpq8SVR+Uazs=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.10/go.mod h1:Kzm5e6OmNH8VMkgK9t+ry5jEih4Y8whqs+1hrkxim1I=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.18 h1:LTRCYFlnnKFlKsyIQxKhJuDuA3ZkrDQMRYm6rXiHlLY=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.18/go.mod h1:XhwkgGG6bHSd00nO/mexWTcTjgd6PjuvWQMqSn2UaEk=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.18 h1:/A/xDuZAVD2BpsS2fftFRo/NoEKQJ8YTnJDEHBy2Gtg=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.18/go.mod h1:hWe9b4f+djUQGmyiGEeOnZv69dtMSgpDRIvNMvuvzvY=
|
||||
github.com/aws/aws-sdk-go-v2/service/s3 v1.96.2 h1:M1A9AjcFwlxTLuf0Faj88L8Iqw0n/AJHjpZTQzMMsSc=
|
||||
github.com/aws/aws-sdk-go-v2/service/s3 v1.96.2/go.mod h1:KsdTV6Q9WKUZm2mNJnUFmIoXfZux91M3sr/a4REX8e0=
|
||||
github.com/aws/aws-sdk-go-v2/service/signin v1.0.5 h1:VrhDvQib/i0lxvr3zqlUwLwJP4fpmpyD9wYG1vfSu+Y=
|
||||
github.com/aws/aws-sdk-go-v2/service/signin v1.0.5/go.mod h1:k029+U8SY30/3/ras4G/Fnv/b88N4mAfliNn08Dem4M=
|
||||
github.com/aws/aws-sdk-go-v2/service/sso v1.30.9 h1:v6EiMvhEYBoHABfbGB4alOYmCIrcgyPPiBE1wZAEbqk=
|
||||
github.com/aws/aws-sdk-go-v2/service/sso v1.30.9/go.mod h1:yifAsgBxgJWn3ggx70A3urX2AN49Y5sJTD1UQFlfqBw=
|
||||
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.14 h1:0jbJeuEHlwKJ9PfXtpSFc4MF+WIWORdhN1n30ITZGFM=
|
||||
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.14/go.mod h1:sTGThjphYE4Ohw8vJiRStAcu3rbjtXRsdNB0TvZ5wwo=
|
||||
github.com/aws/aws-sdk-go-v2/service/sts v1.41.6 h1:5fFjR/ToSOzB2OQ/XqWpZBmNvmP/pJ1jOWYlFDJTjRQ=
|
||||
github.com/aws/aws-sdk-go-v2/service/sts v1.41.6/go.mod h1:qgFDZQSD/Kys7nJnVqYlWKnh0SSdMjAi0uSwON4wgYQ=
|
||||
github.com/aws/smithy-go v1.24.1 h1:VbyeNfmYkWoxMVpGUAbQumkODcYmfMRfZ8yQiH30SK0=
|
||||
github.com/aws/smithy-go v1.24.1/go.mod h1:LEj2LM3rBRQJxPZTB4KuzZkaZYnZPnvgIhb4pu07mx0=
|
||||
github.com/cenkalti/backoff v2.2.1+incompatible h1:tNowT99t7UNflLxfYYSlKYsBpXdEet03Pg2g16Swow4=
|
||||
github.com/cenkalti/backoff v2.2.1+incompatible/go.mod h1:90ReRw6GdpyfrHakVjL/QHaoyV4aDUVVkXQJJJ3NXXM=
|
||||
github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs=
|
||||
github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
|
||||
github.com/cpuguy83/go-md2man/v2 v2.0.6/go.mod h1:oOW0eioCTA6cOiMLiUPZOpcVxMig6NIQQ7OS05n1F4g=
|
||||
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
|
||||
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||
github.com/dgraph-io/ristretto/v2 v2.2.0 h1:bkY3XzJcXoMuELV8F+vS8kzNgicwQFAaGINAEJdWGOM=
|
||||
github.com/dgraph-io/ristretto/v2 v2.2.0/go.mod h1:RZrm63UmcBAaYWC1DotLYBmTvgkrs0+XhBd7Npn7/zI=
|
||||
github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM=
|
||||
github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||
github.com/dgraph-io/ristretto/v2 v2.4.0 h1:I/w09yLjhdcVD2QV192UJcq8dPBaAJb9pOuMyNy0XlU=
|
||||
github.com/dgraph-io/ristretto/v2 v2.4.0/go.mod h1:0KsrXtXvnv0EqnzyowllbVJB8yBonswa2lTCK2gGo9E=
|
||||
github.com/dgryski/go-farm v0.0.0-20240924180020-3414d57e47da h1:aIftn67I1fkbMa512G+w+Pxci9hJPB8oMnkcP3iZF38=
|
||||
github.com/dgryski/go-farm v0.0.0-20240924180020-3414d57e47da/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw=
|
||||
github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY=
|
||||
github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto=
|
||||
github.com/fxamacker/cbor/v2 v2.9.1 h1:2rWm8B193Ll4VdjsJY28jxs70IdDsHRWgQYAI80+rMQ=
|
||||
github.com/fxamacker/cbor/v2 v2.9.1/go.mod h1:vM4b+DJCtHn+zz7h3FFp/hDAI9WNWCsZj23V5ytsSxQ=
|
||||
github.com/go-ini/ini v1.67.0 h1:z6ZrTEZqSWOTyH2FlglNbNgARyHG8oLW9gMELqKr06A=
|
||||
github.com/go-ini/ini v1.67.0/go.mod h1:ByCAeIL28uOIIG0E3PJtZPDL8WnHpFKFOtgjp+3Ies8=
|
||||
github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A=
|
||||
@@ -20,78 +64,156 @@ github.com/go-logr/logr v1.4.3 h1:CjnDlHq8ikf6E492q6eKboGOC0T8CDaOvkHCIg8idEI=
|
||||
github.com/go-logr/logr v1.4.3/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY=
|
||||
github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag=
|
||||
github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE=
|
||||
github.com/google/flatbuffers v25.2.10+incompatible h1:F3vclr7C3HpB1k9mxCGRMXq6FdUalZ6H/pNX4FP1v0Q=
|
||||
github.com/google/flatbuffers v25.2.10+incompatible/go.mod h1:1AeVuKshWv4vARoZatz6mlQ0JxURH0Kv5+zNeJKJCa8=
|
||||
github.com/gofiber/fiber/v3 v3.2.0 h1:g9+09D320foINPpCnR3ibQ5oBEFHjAWRRfDG1te54u8=
|
||||
github.com/gofiber/fiber/v3 v3.2.0/go.mod h1:FHOsc2Db7HhHpsE62QAaJlXVV1pNkbZEptZ4jtti7m4=
|
||||
github.com/gofiber/schema v1.7.1 h1:oSJBKdgP8JeIME4TQSAqlNKTU2iBB+2RNmKi8Nsc+TI=
|
||||
github.com/gofiber/schema v1.7.1/go.mod h1:A/X5Ffyru4p9eBdp99qu+nzviHzQiZ7odLT+TwxWhbk=
|
||||
github.com/gofiber/utils/v2 v2.0.4 h1:WwAxUA7L4MW2DjdEHF234lfqvBqd2vYYuBtA9TJq2ec=
|
||||
github.com/gofiber/utils/v2 v2.0.4/go.mod h1:GGERKU3Vhj5z6hS8YKvxL99A54DjOvTFZ0cjZnG4Lj4=
|
||||
github.com/google/flatbuffers v25.12.19+incompatible h1:haMV2JRRJCe1998HeW/p0X9UaMTK6SDo0ffLn2+DbLs=
|
||||
github.com/google/flatbuffers v25.12.19+incompatible/go.mod h1:1AeVuKshWv4vARoZatz6mlQ0JxURH0Kv5+zNeJKJCa8=
|
||||
github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8=
|
||||
github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU=
|
||||
github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
|
||||
github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
|
||||
github.com/grandcat/zeroconf v1.0.0 h1:uHhahLBKqwWBV6WZUDAT71044vwOTL+McW0mBJvo6kE=
|
||||
github.com/grandcat/zeroconf v1.0.0/go.mod h1:lTKmG1zh86XyCoUeIHSA4FJMBwCJiQmGfcP2PdzytEs=
|
||||
github.com/hanzoai/zip v0.2.0 h1:2GI1OoUv4QdCxmfiwoiyYsSHpiPAxrp0v+Gu4BLtTVI=
|
||||
github.com/hanzoai/zip v0.2.0/go.mod h1:3G+k2wy5bQ1wld66m0OPH8/LJ0kgIeNi1KPymMmlbf4=
|
||||
github.com/hanzos3/go-sdk v1.0.2 h1:EOJQGVnwclkzIyRJyWqtqmA2muyaSsF4y+7KYC4Vhdw=
|
||||
github.com/hanzos3/go-sdk v1.0.2/go.mod h1:oOp/rYVDpZIv2Mn3ZacAUiIahRNOi+V/vlGukvIcuI0=
|
||||
github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8=
|
||||
github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw=
|
||||
github.com/klauspost/compress v1.18.2 h1:iiPHWW0YrcFgpBYhsA6D1+fqHssJscY/Tm/y2Uqnapk=
|
||||
github.com/klauspost/compress v1.18.2/go.mod h1:R0h/fSBs8DE4ENlcrlib3PsXS61voFxhIs2DeRhCvJ4=
|
||||
github.com/klauspost/compress v1.18.5 h1:/h1gH5Ce+VWNLSWqPzOVn6XBO+vJbCNGvjoaGBFW2IE=
|
||||
github.com/klauspost/compress v1.18.5/go.mod h1:cwPg85FWrGar70rWktvGQj8/hthj3wpl0PGDogxkrSQ=
|
||||
github.com/klauspost/cpuid/v2 v2.0.1/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg=
|
||||
github.com/klauspost/cpuid/v2 v2.2.11 h1:0OwqZRYI2rFrjS4kvkDnqJkKHdHaRnCm68/DY4OxRzU=
|
||||
github.com/klauspost/cpuid/v2 v2.2.11/go.mod h1:hqwkgyIinND0mEev00jJYCxPNVRVXFQeu1XKlok6oO0=
|
||||
github.com/klauspost/cpuid/v2 v2.3.0 h1:S4CRMLnYUhGeDFDqkGriYKdfoFlDnMtqTiI/sFzhA9Y=
|
||||
github.com/klauspost/cpuid/v2 v2.3.0/go.mod h1:hqwkgyIinND0mEev00jJYCxPNVRVXFQeu1XKlok6oO0=
|
||||
github.com/klauspost/crc32 v1.3.0 h1:sSmTt3gUt81RP655XGZPElI0PelVTZ6YwCRnPSupoFM=
|
||||
github.com/klauspost/crc32 v1.3.0/go.mod h1:D7kQaZhnkX/Y0tstFGf8VUzv2UofNGqCjnC3zdHB0Hw=
|
||||
github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE=
|
||||
github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk=
|
||||
github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
|
||||
github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
|
||||
github.com/luxfi/age v1.4.0 h1:tU5q65RQSQdaVq64Z/DVUhiPQMoMPQT6pr41LsvG0AQ=
|
||||
github.com/luxfi/age v1.4.0/go.mod h1:iAYAxgvrXxcy746+Ovh/eWWDuF9teJLNcCSSOX9RYW0=
|
||||
github.com/luxfi/log v1.4.3 h1:xkUKRWvQ4ZwvlUC2e0/RTtHYZOYSMvSQ9W9lbjwBmiI=
|
||||
github.com/luxfi/log v1.4.3/go.mod h1:myIkufyiQomSQH34K981kbz6cG4WUoerRUh7F4XhlQI=
|
||||
github.com/luxfi/mdns v0.1.0 h1:VB3mQcETc9j5SY1S6lAgFtuGr/rjWuDgPYnxS+OKWMQ=
|
||||
github.com/luxfi/mdns v0.1.0/go.mod h1:/3dheKVjUk2yiS/ocH1IDzeLXOIe+kpVsErIGDOZdiQ=
|
||||
github.com/luxfi/pq v1.0.3 h1:pFlQm1+5FuKTDUh2y/23bXWkN4I2Rc5iuxJypwDFFMs=
|
||||
github.com/luxfi/pq v1.0.3/go.mod h1:8bppZcRElfrVt0n3nYCZW3iX1TvhvzNbdjNdK1irgIE=
|
||||
github.com/luxfi/zap v0.3.1 h1:RlOHthFTw3cikft3JmrdT/R8BFId2Ej7xeNiHDoTNs4=
|
||||
github.com/luxfi/zap v0.3.1/go.mod h1:+RVBXJPxTqY5RU148058qew9/3fyRWTg/eBIwOIMlCY=
|
||||
github.com/mattn/go-colorable v0.1.14 h1:9A9LHSqF/7dyVVX6g0U9cwm9pG3kP9gSzcuIPHPsaIE=
|
||||
github.com/mattn/go-colorable v0.1.14/go.mod h1:6LmQG8QLFO4G5z1gPvYEzlUgJ2wF+stgPZH1UqBm1s8=
|
||||
github.com/mattn/go-isatty v0.0.21 h1:xYae+lCNBP7QuW4PUnNG61ffM4hVIfm+zUzDuSzYLGs=
|
||||
github.com/mattn/go-isatty v0.0.21/go.mod h1:ZXfXG4SQHsB/w3ZeOYbR0PrPwLy+n6xiMrJlRFqopa4=
|
||||
github.com/miekg/dns v1.1.27/go.mod h1:KNUDUusw/aVsxyTYZM1oqvCicbwhgbNgztCETuNZ7xM=
|
||||
github.com/miekg/dns v1.1.72 h1:vhmr+TF2A3tuoGNkLDFK9zi36F2LS+hKTRW0Uf8kbzI=
|
||||
github.com/miekg/dns v1.1.72/go.mod h1:+EuEPhdHOsfk6Wk5TT2CzssZdqkmFhf8r+aVyDEToIs=
|
||||
github.com/minio/crc64nvme v1.1.1 h1:8dwx/Pz49suywbO+auHCBpCtlW1OfpcLN7wYgVR6wAI=
|
||||
github.com/minio/crc64nvme v1.1.1/go.mod h1:eVfm2fAzLlxMdUGc0EEBGSMmPwmXD5XiNRpnu9J3bvg=
|
||||
github.com/minio/md5-simd v1.1.2 h1:Gdi1DZK69+ZVMoNHRXJyNcxrMA4dSxoYHZSQbirFg34=
|
||||
github.com/minio/md5-simd v1.1.2/go.mod h1:MzdKDxYpY2BT9XQFocsiZf/NKVtR7nkE4RoEpN+20RM=
|
||||
github.com/ncruces/go-strftime v1.0.0 h1:HMFp8mLCTPp341M/ZnA4qaf7ZlsbTc+miZjCLOFAw7w=
|
||||
github.com/ncruces/go-strftime v1.0.0/go.mod h1:Fwc5htZGVVkseilnfgOVb9mKy6w1naJmn9CehxcKcls=
|
||||
github.com/philhofer/fwd v1.2.0 h1:e6DnBTl7vGY+Gz322/ASL4Gyp1FspeMvx1RNDoToZuM=
|
||||
github.com/philhofer/fwd v1.2.0/go.mod h1:RqIHx9QI14HlwKwm98g9Re5prTQ6LdeRQn+gXJFxsJM=
|
||||
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
|
||||
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
|
||||
github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
|
||||
github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
|
||||
github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U=
|
||||
github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
|
||||
github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94icq4NjY3clb7Lk8O1qJ8BdBEF8z0ibU0rE=
|
||||
github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo=
|
||||
github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0tI/otEQ=
|
||||
github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7so1lCWt35ZSgc=
|
||||
github.com/rs/xid v1.6.0 h1:fV591PaemRlL6JfRxGDEPl69wICngIQ3shQtzfy2gxU=
|
||||
github.com/rs/xid v1.6.0/go.mod h1:7XoLgs4eV+QndskICGsho+ADou8ySMSjJKDIan90Nz0=
|
||||
github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
|
||||
github.com/spf13/cobra v1.9.1 h1:CXSaggrXdbHK9CF+8ywj8Amf7PBRmPCOJugH954Nnlo=
|
||||
github.com/spf13/cobra v1.9.1/go.mod h1:nDyEzZ8ogv936Cinf6g1RU9MRY64Ir93oCnqb9wxYW0=
|
||||
github.com/spf13/pflag v1.0.6 h1:jFzHGLGAlb3ruxLB8MhbI6A8+AQX/2eW4qeyNZXNp2o=
|
||||
github.com/spf13/pflag v1.0.6/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
|
||||
github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA=
|
||||
github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
|
||||
github.com/tinylib/msgp v1.6.1 h1:ESRv8eL3u+DNHUoSAAQRE50Hm162zqAnBoGv9PzScPY=
|
||||
github.com/tinylib/msgp v1.6.1/go.mod h1:RSp0LW9oSxFut3KzESt5Voq4GVWyS+PSulT77roAqEA=
|
||||
go.opentelemetry.io/auto/sdk v1.1.0 h1:cH53jehLUN6UFLY71z+NDOiNJqDdPRaXzTel0sJySYA=
|
||||
go.opentelemetry.io/auto/sdk v1.1.0/go.mod h1:3wSPjt5PWp2RhlCcmmOial7AvC4DQqZb7a7wCow3W8A=
|
||||
github.com/shamaton/msgpack/v3 v3.1.0 h1:jsk0vEAqVvvS9+fTZ5/EcQ9tz860c9pWxJ4Iwecz8gU=
|
||||
github.com/shamaton/msgpack/v3 v3.1.0/go.mod h1:DcQG8jrdrQCIxr3HlMYkiXdMhK+KfN2CitkyzsQV4uc=
|
||||
github.com/spf13/cobra v1.10.2 h1:DMTTonx5m65Ic0GOoRY2c16WCbHxOOw6xxezuLaBpcU=
|
||||
github.com/spf13/cobra v1.10.2/go.mod h1:7C1pvHqHw5A4vrJfjNwvOdzYu0Gml16OCs2GRiTUUS4=
|
||||
github.com/spf13/pflag v1.0.9/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
|
||||
github.com/spf13/pflag v1.0.10 h1:4EBh2KAYBwaONj6b2Ye1GiHfwjqyROoF4RwYO+vPwFk=
|
||||
github.com/spf13/pflag v1.0.10/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
|
||||
github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U=
|
||||
github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U=
|
||||
github.com/tinylib/msgp v1.6.4 h1:mOwYbyYDLPj35mkA2BjjYejgJk9BuHxDdvRnb6v2ZcQ=
|
||||
github.com/tinylib/msgp v1.6.4/go.mod h1:RSp0LW9oSxFut3KzESt5Voq4GVWyS+PSulT77roAqEA=
|
||||
github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw=
|
||||
github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc=
|
||||
github.com/valyala/fasthttp v1.70.0 h1:LAhMGcWk13QZWm85+eg8ZBNbrq5mnkWFGbHMUJHIdXA=
|
||||
github.com/valyala/fasthttp v1.70.0/go.mod h1:oDZEHHkJ/Buyklg6uURmYs19442zFSnCIfX3j1FY3pE=
|
||||
github.com/x448/float16 v0.8.4 h1:qLwI1I70+NjRFUR3zs1JPUCgaCXSh3SW62uAKT1mSBM=
|
||||
github.com/x448/float16 v0.8.4/go.mod h1:14CWIYCyZA/cWjXOioeEpHeN/83MdbZDRQHoFcYsOfg=
|
||||
github.com/xyproto/randomstring v1.0.5 h1:YtlWPoRdgMu3NZtP45drfy1GKoojuR7hmRcnhZqKjWU=
|
||||
github.com/xyproto/randomstring v1.0.5/go.mod h1:rgmS5DeNXLivK7YprL0pY+lTuhNQW3iGxZ18UQApw/E=
|
||||
github.com/zeebo/assert v1.1.0 h1:hU1L1vLTHsnO8x8c9KAR5GmM5QscxHg5RNU5z5qbUWY=
|
||||
github.com/zeebo/assert v1.1.0/go.mod h1:Pq9JiuJQpG8JLJdtkwrJESF0Foym2/D9XMU5ciN/wJ0=
|
||||
github.com/zeebo/blake3 v0.2.4 h1:KYQPkhpRtcqh0ssGYcKLG1JYvddkEA8QwCM/yBqhaZI=
|
||||
github.com/zeebo/blake3 v0.2.4/go.mod h1:7eeQ6d2iXWRGF6npfaxl2CU+xy2Fjo2gxeyZGCRUjcE=
|
||||
github.com/zeebo/pcg v1.0.1 h1:lyqfGeWiv4ahac6ttHs+I5hwtH/+1mrhlCtVNQM2kHo=
|
||||
github.com/zeebo/pcg v1.0.1/go.mod h1:09F0S9iiKrwn9rlI5yjLkmrug154/YRW6KnnXVDM/l4=
|
||||
go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ64=
|
||||
go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y=
|
||||
go.opentelemetry.io/contrib/zpages v0.62.0 h1:9fUYTLmrK0x/lweM2uM+BOx069jLx8PxVqWhegGJ9Bo=
|
||||
go.opentelemetry.io/contrib/zpages v0.62.0/go.mod h1:C8kXoiC1Ytvereztus2R+kqdSa6W/MZ8FfS8Zwj+LiM=
|
||||
go.opentelemetry.io/otel v1.37.0 h1:9zhNfelUvx0KBfu/gb+ZgeAfAgtWrfHJZcAqFC228wQ=
|
||||
go.opentelemetry.io/otel v1.37.0/go.mod h1:ehE/umFRLnuLa/vSccNq9oS1ErUlkkK71gMcN34UG8I=
|
||||
go.opentelemetry.io/otel/metric v1.37.0 h1:mvwbQS5m0tbmqML4NqK+e3aDiO02vsf/WgbsdpcPoZE=
|
||||
go.opentelemetry.io/otel/metric v1.37.0/go.mod h1:04wGrZurHYKOc+RKeye86GwKiTb9FKm1WHtO+4EVr2E=
|
||||
go.opentelemetry.io/otel/sdk v1.37.0 h1:ItB0QUqnjesGRvNcmAcU0LyvkVyGJ2xftD29bWdDvKI=
|
||||
go.opentelemetry.io/otel/sdk v1.37.0/go.mod h1:VredYzxUvuo2q3WRcDnKDjbdvmO0sCzOvVAiY+yUkAg=
|
||||
go.opentelemetry.io/otel/trace v1.37.0 h1:HLdcFNbRQBE2imdSEgm/kwqmQj1Or1l/7bW6mxVK7z4=
|
||||
go.opentelemetry.io/otel/trace v1.37.0/go.mod h1:TlgrlQ+PtQO5XFerSPUYG0JSgGyryXewPGyayAWSBS0=
|
||||
go.opentelemetry.io/otel v1.43.0 h1:mYIM03dnh5zfN7HautFE4ieIig9amkNANT+xcVxAj9I=
|
||||
go.opentelemetry.io/otel v1.43.0/go.mod h1:JuG+u74mvjvcm8vj8pI5XiHy1zDeoCS2LB1spIq7Ay0=
|
||||
go.opentelemetry.io/otel/metric v1.43.0 h1:d7638QeInOnuwOONPp4JAOGfbCEpYb+K6DVWvdxGzgM=
|
||||
go.opentelemetry.io/otel/metric v1.43.0/go.mod h1:RDnPtIxvqlgO8GRW18W6Z/4P462ldprJtfxHxyKd2PY=
|
||||
go.opentelemetry.io/otel/sdk v1.43.0 h1:pi5mE86i5rTeLXqoF/hhiBtUNcrAGHLKQdhg4h4V9Dg=
|
||||
go.opentelemetry.io/otel/sdk v1.43.0/go.mod h1:P+IkVU3iWukmiit/Yf9AWvpyRDlUeBaRg6Y+C58QHzg=
|
||||
go.opentelemetry.io/otel/sdk/metric v1.43.0 h1:S88dyqXjJkuBNLeMcVPRFXpRw2fuwdvfCGLEo89fDkw=
|
||||
go.opentelemetry.io/otel/sdk/metric v1.43.0/go.mod h1:C/RJtwSEJ5hzTiUz5pXF1kILHStzb9zFlIEe85bhj6A=
|
||||
go.opentelemetry.io/otel/trace v1.43.0 h1:BkNrHpup+4k4w+ZZ86CZoHHEkohws8AY+WTX09nk+3A=
|
||||
go.opentelemetry.io/otel/trace v1.43.0/go.mod h1:/QJhyVBUUswCphDVxq+8mld+AvhXZLhe+8WVFxiFff0=
|
||||
go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto=
|
||||
go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE=
|
||||
go.yaml.in/yaml/v3 v3.0.4 h1:tfq32ie2Jv2UxXFdLJdh3jXuOzWiL1fo0bu/FbuKpbc=
|
||||
go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg=
|
||||
golang.org/x/crypto v0.46.0 h1:cKRW/pmt1pKAfetfu+RCEvjvZkA9RimPbh7bhFjGVBU=
|
||||
golang.org/x/crypto v0.46.0/go.mod h1:Evb/oLKmMraqjZ2iQTwDwvCtJkczlDuTmdJXoZVzqU0=
|
||||
golang.org/x/net v0.48.0 h1:zyQRTTrjc33Lhh0fBgT/H3oZq9WuvRR5gPC70xpDiQU=
|
||||
golang.org/x/net v0.48.0/go.mod h1:+ndRgGjkh8FGtu1w1FGbEC31if4VrNVMuKTgcAAnQRY=
|
||||
golang.org/x/sys v0.39.0 h1:CvCKL8MeisomCi6qNZ+wbb0DN9E5AATixKsvNtMoMFk=
|
||||
golang.org/x/sys v0.39.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks=
|
||||
golang.org/x/text v0.32.0 h1:ZD01bjUt1FQ9WJ0ClOL5vxgxOI/sVCNgX1YtKwcY0mU=
|
||||
golang.org/x/text v0.32.0/go.mod h1:o/rUWzghvpD5TXrTIBuJU77MTaN0ljMWE47kxGJQ7jY=
|
||||
google.golang.org/protobuf v1.36.7 h1:IgrO7UwFQGJdRNXH/sQux4R1Dj1WAKcLElzeeRaXV2A=
|
||||
google.golang.org/protobuf v1.36.7/go.mod h1:jduwjTPXsFjZGTmRluh+L6NjiWu7pchiJ2/5YcXBHnY=
|
||||
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
|
||||
golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
|
||||
golang.org/x/crypto v0.50.0 h1:zO47/JPrL6vsNkINmLoo/PH1gcxpls50DNogFvB5ZGI=
|
||||
golang.org/x/crypto v0.50.0/go.mod h1:3muZ7vA7PBCE6xgPX7nkzzjiUq87kRItoJQM1Yo8S+Q=
|
||||
golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg=
|
||||
golang.org/x/mod v0.34.0 h1:xIHgNUUnW6sYkcM5Jleh05DvLOtwc6RitGHbDk4akRI=
|
||||
golang.org/x/mod v0.34.0/go.mod h1:ykgH52iCZe79kzLLMhyCUzhMci+nQj+0XkbXpNYtVjY=
|
||||
golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
|
||||
golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
|
||||
golang.org/x/net v0.0.0-20190923162816-aa69164e4478/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
|
||||
golang.org/x/net v0.0.0-20200114155413-6afb5195e5aa/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
|
||||
golang.org/x/net v0.53.0 h1:d+qAbo5L0orcWAr0a9JweQpjXF19LMXJE8Ey7hwOdUA=
|
||||
golang.org/x/net v0.53.0/go.mod h1:JvMuJH7rrdiCfbeHoo3fCQU24Lf5JJwT9W3sJFulfgs=
|
||||
golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sync v0.20.0 h1:e0PTpb7pjO8GAtTs2dQ6jYa5BWYlMuX047Dco/pItO4=
|
||||
golang.org/x/sync v0.20.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0=
|
||||
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
||||
golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20190924154521-2837fb4f24fe/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.43.0 h1:Rlag2XtaFTxp19wS8MXlJwTvoh8ArU6ezoyFsMyCTNI=
|
||||
golang.org/x/sys v0.43.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw=
|
||||
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
|
||||
golang.org/x/text v0.36.0 h1:JfKh3XmcRPqZPKevfXVpI1wXPTqbkE5f7JA92a55Yxg=
|
||||
golang.org/x/text v0.36.0/go.mod h1:NIdBknypM8iqVmPiuco0Dh6P5Jcdk8lJL0CUebqK164=
|
||||
golang.org/x/tools v0.0.0-20191216052735-49a3e744a425/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
|
||||
golang.org/x/tools v0.43.0 h1:12BdW9CeB3Z+J/I/wj34VMl8X+fEXBxVR90JeMX5E7s=
|
||||
golang.org/x/tools v0.43.0/go.mod h1:uHkMso649BX2cZK6+RpuIPXS3ho2hZo4FVwfoy1vIk0=
|
||||
golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
|
||||
google.golang.org/protobuf v1.36.11 h1:fV6ZwhNocDyBLK0dj+fg8ektcVegBBuEolpbTQyBNVE=
|
||||
google.golang.org/protobuf v1.36.11/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco=
|
||||
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
|
||||
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk=
|
||||
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q=
|
||||
gopkg.in/natefinch/lumberjack.v2 v2.2.1 h1:bBRl1b0OH9s/DuPhuXpNl+VtCaJXFZ5/uEFST95x9zc=
|
||||
gopkg.in/natefinch/lumberjack.v2 v2.2.1/go.mod h1:YD8tP3GAjkrDg1eZH7EGmyESg/lsYskCTPBJVb9jqSc=
|
||||
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
|
||||
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
|
||||
modernc.org/libc v1.72.0 h1:IEu559v9a0XWjw0DPoVKtXpO2qt5NVLAnFaBbjq+n8c=
|
||||
modernc.org/libc v1.72.0/go.mod h1:tTU8DL8A+XLVkEY3x5E/tO7s2Q/q42EtnNWda/L5QhQ=
|
||||
modernc.org/mathutil v1.7.1 h1:GCZVGXdaN8gTqB1Mf/usp1Y/hSqgI2vAGGP4jZMCxOU=
|
||||
modernc.org/mathutil v1.7.1/go.mod h1:4p5IwJITfppl0G4sUEDtCr4DthTaT47/N3aT6MhfgJg=
|
||||
modernc.org/memory v1.11.0 h1:o4QC8aMQzmcwCK3t3Ux/ZHmwFPzE6hf2Y5LbkRs+hbI=
|
||||
modernc.org/memory v1.11.0/go.mod h1:/JP4VbVC+K5sU2wZi9bHoq2MAkCnrt2r98UGeSK7Mjw=
|
||||
modernc.org/sqlite v1.50.0 h1:eMowQSWLK0MeiQTdmz3lqoF5dqclujdlIKeJA11+7oM=
|
||||
modernc.org/sqlite v1.50.0/go.mod h1:m0w8xhwYUVY3H6pSDwc3gkJ/irZT/0YEXwBlhaxQEew=
|
||||
|
||||
Reference in New Issue
Block a user