feat(replica): BackendStore adapter — the ready Store over vfs backend
Completes the shared HA-SQLite lib: NewBackendStore(be) adapts any vfs backend.Backend (file locally, s3/SeaweedFS in prod) to the Replicator Store, with a cheap content-version sidecar (<key>.ver) so a reader answers Version() without fetching the whole DB. So a service wires HA in one line: NewReplicator(key, NewBackendStore(be), OpenSQLite(path)). Test: full owner->vfs-backend->reader path over the real file backend (SQLite rows survive push/pull, incl. updates + the version-skip).
This commit is contained in:
@@ -0,0 +1,66 @@
|
||||
// Copyright 2025 Hanzo AI Inc. All Rights Reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
|
||||
package replica
|
||||
|
||||
// BackendStore adapts a vfs backend.Backend (the raw object store — file locally,
|
||||
// s3/SeaweedFS in production) to the Replicator Store interface. This is the ready
|
||||
// Store every service wires: `replica.NewReplicator(key, replica.NewBackendStore(be), db)`.
|
||||
//
|
||||
// A tiny content-version sidecar (`<key>.ver` holding the snapshot's SHA-256) lets
|
||||
// a reader answer Version(key) cheaply — a small Get — instead of fetching the whole
|
||||
// database just to decide whether it changed. Get itself recomputes the version from
|
||||
// the fetched bytes (authoritative), so a missing/stale sidecar can never cause a
|
||||
// wrong restore — at worst a redundant pull.
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
|
||||
"github.com/hanzoai/vfs/pkg/backend"
|
||||
)
|
||||
|
||||
// BackendStore is a Store backed by a vfs backend.Backend.
|
||||
type BackendStore struct {
|
||||
be backend.Backend
|
||||
}
|
||||
|
||||
// NewBackendStore wraps be as a Replicator Store.
|
||||
func NewBackendStore(be backend.Backend) *BackendStore { return &BackendStore{be: be} }
|
||||
|
||||
func verKey(key string) string { return key + ".ver" }
|
||||
|
||||
// Put writes the snapshot then its version sidecar. If the sidecar write fails the
|
||||
// data is still durable; Version falls back to "" (unknown → the reader pulls), so
|
||||
// a partial Put degrades to a redundant pull, never data loss.
|
||||
func (s *BackendStore) Put(ctx context.Context, key string, data []byte) error {
|
||||
if err := s.be.Put(ctx, key, data); err != nil {
|
||||
return err
|
||||
}
|
||||
return s.be.Put(ctx, verKey(key), []byte(version(data)))
|
||||
}
|
||||
|
||||
// Get returns the snapshot bytes and their authoritative content version (recomputed
|
||||
// from the bytes, so it always matches what a writer stored).
|
||||
func (s *BackendStore) Get(ctx context.Context, key string) ([]byte, string, error) {
|
||||
data, err := s.be.Get(ctx, key)
|
||||
if err != nil {
|
||||
return nil, "", err
|
||||
}
|
||||
return data, version(data), nil
|
||||
}
|
||||
|
||||
// Version returns the cheap sidecar version, or "" when absent (a first-ever key or
|
||||
// a legacy object with no sidecar — the reader then pulls once).
|
||||
func (s *BackendStore) Version(ctx context.Context, key string) (string, error) {
|
||||
v, err := s.be.Get(ctx, verKey(key))
|
||||
if errors.Is(err, backend.ErrNotFound) {
|
||||
return "", nil
|
||||
}
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
return string(v), nil
|
||||
}
|
||||
@@ -0,0 +1,91 @@
|
||||
// Copyright 2025 Hanzo AI Inc. All Rights Reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
|
||||
package replica
|
||||
|
||||
import (
|
||||
"context"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
|
||||
"github.com/hanzoai/vfs/pkg/backend"
|
||||
_ "github.com/hanzoai/vfs/pkg/backend/file" // register file:// opener
|
||||
)
|
||||
|
||||
// TestBackendStoreEndToEnd is the full HA path a service runs: an OWNER writes a
|
||||
// real SQLite DB and Pushes it through a vfs backend; a READER on a different disk
|
||||
// Pulls it back and sees the committed rows. Proves the BackendStore adapter +
|
||||
// SQLiteDB + Replicator compose over the real object-store backend.
|
||||
func TestBackendStoreEndToEnd(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
objDir := t.TempDir() // stands in for SeaweedFS
|
||||
be, err := backend.Open(ctx, "file://"+objDir)
|
||||
if err != nil {
|
||||
t.Fatalf("backend.Open: %v", err)
|
||||
}
|
||||
defer be.Close()
|
||||
store := NewBackendStore(be)
|
||||
key := DBPath("acme", "", "kv")
|
||||
|
||||
// Owner: create + write + push.
|
||||
ownerDir := t.TempDir()
|
||||
owner, err := OpenSQLite(filepath.Join(ownerDir, "kv.db"))
|
||||
if err != nil {
|
||||
t.Fatalf("open owner: %v", err)
|
||||
}
|
||||
defer owner.Close()
|
||||
if _, err := owner.DB().ExecContext(ctx, `CREATE TABLE kv(k TEXT PRIMARY KEY, v TEXT)`); err != nil {
|
||||
t.Fatalf("create: %v", err)
|
||||
}
|
||||
if _, err := owner.DB().ExecContext(ctx, `INSERT INTO kv VALUES('hello','world')`); err != nil {
|
||||
t.Fatalf("insert: %v", err)
|
||||
}
|
||||
or := NewReplicator(key, store, owner)
|
||||
if err := or.Push(ctx); err != nil {
|
||||
t.Fatalf("push: %v", err)
|
||||
}
|
||||
|
||||
// Version sidecar is queryable cheaply.
|
||||
if v, err := store.Version(ctx, key); err != nil || v == "" {
|
||||
t.Fatalf("version after push = %q, err=%v; want non-empty", v, err)
|
||||
}
|
||||
|
||||
// Reader on a separate disk: pull + read.
|
||||
readerDir := t.TempDir()
|
||||
reader, err := OpenSQLite(filepath.Join(readerDir, "kv.db"))
|
||||
if err != nil {
|
||||
t.Fatalf("open reader: %v", err)
|
||||
}
|
||||
defer reader.Close()
|
||||
rr := NewReplicator(key, store, reader)
|
||||
if changed, err := rr.Pull(ctx); err != nil || !changed {
|
||||
t.Fatalf("pull: changed=%v err=%v; want changed=true", changed, err)
|
||||
}
|
||||
var got string
|
||||
if err := reader.DB().QueryRowContext(ctx, `SELECT v FROM kv WHERE k='hello'`).Scan(&got); err != nil {
|
||||
t.Fatalf("read pulled: %v", err)
|
||||
}
|
||||
if got != "world" {
|
||||
t.Fatalf("pulled kv[hello] = %q, want world", got)
|
||||
}
|
||||
|
||||
// Owner writes more + pushes; reader pulls the update.
|
||||
if _, err := owner.DB().ExecContext(ctx, `INSERT INTO kv VALUES('a','b')`); err != nil {
|
||||
t.Fatalf("insert 2: %v", err)
|
||||
}
|
||||
if err := or.Push(ctx); err != nil {
|
||||
t.Fatalf("push 2: %v", err)
|
||||
}
|
||||
if changed, err := rr.Pull(ctx); err != nil || !changed {
|
||||
t.Fatalf("pull 2: changed=%v err=%v; want changed=true", changed, err)
|
||||
}
|
||||
var n int
|
||||
if err := reader.DB().QueryRowContext(ctx, `SELECT COUNT(*) FROM kv`).Scan(&n); err != nil {
|
||||
t.Fatalf("count after update: %v", err)
|
||||
}
|
||||
if n != 2 {
|
||||
t.Fatalf("reader row count = %d, want 2", n)
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user