Compare commits
9
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
73055e9e2e | ||
|
|
f876fe9ccf | ||
|
|
7022b0c3ef | ||
|
|
34dda54d16 | ||
|
|
2ced5eaee5 | ||
|
|
7c8b8d3b02 | ||
|
|
ce2eae14b2 | ||
|
|
f83d60dac2 | ||
|
|
cd1f11920f |
@@ -1,9 +0,0 @@
|
||||
<svg xmlns="http://www.w3.org/2000/svg" width="1280" height="640" viewBox="0 0 1280 640" role="img" aria-label="vfs">
|
||||
<rect width="1280" height="640" fill="#0A0A0A"/>
|
||||
<svg x="96" y="215" width="210" height="210" viewBox="0 0 67 67"><path d="M22.21 67V44.6369H0V67H22.21Z" fill="#fff"/><path d="M66.7038 22.3184H22.2534L0.0878906 44.6367H44.4634L66.7038 22.3184Z" fill="#fff"/><path d="M22.21 0H0V22.3184H22.21V0Z" fill="#fff"/><path d="M66.7198 0H44.5098V22.3184H66.7198V0Z" fill="#fff"/><path d="M66.7198 67V44.6369H44.5098V67H66.7198Z" fill="#fff"/></svg>
|
||||
<text x="378" y="276" font-family="Inter,system-ui,-apple-system,sans-serif" font-size="78" font-weight="800" letter-spacing="-2" fill="#ffffff">vfs</text>
|
||||
<text x="378" y="322" font-family="Inter,system-ui,sans-serif" font-size="30" fill="#ffffff" opacity=".66">S3-backed virtual block filesystem with PQ encryption — unlimited write…</text>
|
||||
<rect x="378" y="338" width="806" height="3" rx="1.5" fill="#ffffff" opacity=".9"/>
|
||||
<text x="378" y="390" font-family="Inter,system-ui,sans-serif" font-size="24" font-weight="600" fill="#ffffff" opacity=".5">github.com/hanzoai</text>
|
||||
<text x="1184" y="390" text-anchor="end" font-family="Inter,system-ui,sans-serif" font-size="24" font-weight="600" fill="#ffffff" opacity=".5">hanzo.ai</text>
|
||||
</svg>
|
||||
|
Before Width: | Height: | Size: 1.3 KiB |
@@ -13,21 +13,17 @@ permissions:
|
||||
|
||||
jobs:
|
||||
test:
|
||||
runs-on: hanzo-build-linux-amd64
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- uses: actions/setup-go@v5
|
||||
with:
|
||||
go-version: '1.26.4'
|
||||
go-version: '1.25'
|
||||
cache: true
|
||||
- name: install cgo toolchain (the -race detector requires a C compiler)
|
||||
run: sudo apt-get update && sudo apt-get install -y gcc
|
||||
- name: vet
|
||||
run: go vet ./...
|
||||
- name: test (default)
|
||||
run: go test -race ./...
|
||||
env:
|
||||
CGO_ENABLED: "1" # the race detector requires cgo; production build stays CGO_ENABLED=0 (line below)
|
||||
- name: vet (fuse)
|
||||
run: go vet -tags fuse ./...
|
||||
- name: install fuse
|
||||
@@ -36,17 +32,15 @@ jobs:
|
||||
run: go build -tags fuse ./...
|
||||
- name: test (fuse + sqlite e2e)
|
||||
run: VFS_FUSE_E2E=1 go test -race -tags fuse -run TestFUSESQLite ./pkg/mount/ -v
|
||||
env:
|
||||
CGO_ENABLED: "1" # -race + fuse bindings both require cgo
|
||||
|
||||
build-amd64:
|
||||
needs: test
|
||||
runs-on: hanzo-build-linux-amd64
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- uses: actions/setup-go@v5
|
||||
with:
|
||||
go-version: '1.26.4'
|
||||
go-version: '1.25'
|
||||
cache: true
|
||||
- name: build
|
||||
run: |
|
||||
@@ -63,7 +57,7 @@ jobs:
|
||||
image:
|
||||
needs: test
|
||||
if: github.event_name == 'push' && (github.ref == 'refs/heads/main' || startsWith(github.ref, 'refs/tags/v'))
|
||||
runs-on: hanzo-build-linux-amd64
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- name: login ghcr
|
||||
|
||||
+3
-9
@@ -1,6 +1,4 @@
|
||||
FROM golang:1.26.4-alpine AS builder
|
||||
RUN apk add --no-cache ca-certificates tzdata
|
||||
RUN addgroup -g 65532 -S nonroot && adduser -u 65532 -S nonroot -G nonroot
|
||||
FROM golang:1.25-alpine AS builder
|
||||
WORKDIR /build
|
||||
COPY go.mod go.sum ./
|
||||
RUN go mod download
|
||||
@@ -10,11 +8,7 @@ RUN CGO_ENABLED=0 go build \
|
||||
-ldflags "-X main.version=${VERSION}" \
|
||||
-o /vfs ./cmd/vfs
|
||||
|
||||
FROM scratch
|
||||
COPY --from=builder /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ca-certificates.crt
|
||||
COPY --from=builder /usr/share/zoneinfo /usr/share/zoneinfo
|
||||
COPY --from=builder /etc/passwd /etc/passwd
|
||||
COPY --from=builder /etc/group /etc/group
|
||||
FROM gcr.io/distroless/static:nonroot
|
||||
COPY --from=builder /vfs /usr/local/bin/vfs
|
||||
USER 65532:65532
|
||||
USER nonroot:nonroot
|
||||
ENTRYPOINT ["/usr/local/bin/vfs"]
|
||||
|
||||
@@ -4,34 +4,13 @@
|
||||
|
||||
S3-backed virtual block filesystem. Local-NVMe hot tier + S3 (or any object
|
||||
store) cold tier with PQ encryption per block. Stateful services (IAM, BD,
|
||||
ATS, TA, KMS, lqd snapshots) get unlimited disk via S3 without
|
||||
ATS, TA, KMS, onyxd, lqd snapshots) get unlimited disk via S3 without
|
||||
needing K8s PVC sizing decisions.
|
||||
|
||||
Companion to `~/work/hanzo/replicate` — `replicate` streams SQLite WAL
|
||||
frames to S3 (file-level granularity); `vfs` is block-level granularity
|
||||
that transparently spills cold pages.
|
||||
|
||||
## `replica/` subpackage — HA-SQLite substrate (HIP-0107)
|
||||
|
||||
`github.com/hanzoai/vfs/replica` is the shared per-org SQLite HA substrate:
|
||||
local SQLite for speed, snapshot replication to the object store, hydrate-on-open,
|
||||
single-writer gated by `github.com/hanzoai/ha` election. `Replicator` (Push/Pull),
|
||||
`DB` (Snapshot/Restore), `Store`, `DBPath`.
|
||||
|
||||
**Fencing (`replica/fence.go`, since v0.6.3)** — election alone is coordination-free,
|
||||
so two replicas with divergent membership views can each elect themselves writer for
|
||||
one org (split-brain) and unconditional `Store.Put` lets both overwrite the mirror.
|
||||
`FencedStore` closes this over a `ConditionalStore` (S3 If-Match / GCS
|
||||
generation-match) CAS: one object per key carries a monotone round header, admitted
|
||||
iff `round >= recorded`, rejected (`ErrStaleRound`) when below — so a deposed writer's
|
||||
ship is refused once a successor advanced the round. `Put(key,data,round)` ships;
|
||||
`CarryForward(key,round,hydrate)` is the safe takeover seal (re-reads + carries the
|
||||
predecessor's last landed write forward atomically via CAS, so no acknowledged write
|
||||
is lost); `Get`/`Round` read. Round is a plain `uint64` (the storage boundary value);
|
||||
the coordination value `ha.Lease{Owner,Round}` and the round SOURCE live in `ha` and
|
||||
its consumers, never here. The concrete `ConditionalStore` (minio If-Match) lives with
|
||||
the S3-client owner (cloud); this package stays dependency-free over the seam.
|
||||
|
||||
## Architecture
|
||||
|
||||
```
|
||||
@@ -121,7 +100,7 @@ Same shape as `hanzo/replicate` — runs alongside Base/SQLite services:
|
||||
```yaml
|
||||
spec:
|
||||
containers:
|
||||
- name: tenant-bd
|
||||
- name: liquid-bd
|
||||
volumeMounts:
|
||||
- name: data
|
||||
mountPath: /data # FUSE-mounted VFS
|
||||
@@ -130,7 +109,7 @@ spec:
|
||||
args:
|
||||
- mount
|
||||
- /data
|
||||
- --backend=s3://tenant-vfs/{env}/bd
|
||||
- --backend=s3://liquidity-vfs/{env}/bd
|
||||
- --age-key=/etc/vfs/age.key
|
||||
securityContext:
|
||||
privileged: true # FUSE needs CAP_SYS_ADMIN
|
||||
|
||||
@@ -1,49 +1,3 @@
|
||||
<p align="center"><img src=".github/hero.svg" alt="vfs" width="880"></p>
|
||||
|
||||
# vfs
|
||||
|
||||
Object-store abstraction with content-addressed, PQ-encrypted block storage. The storage backplane for HIP-0107 streaming replication.
|
||||
|
||||
[]()
|
||||
[]()
|
||||
|
||||
## Quick start
|
||||
|
||||
```bash
|
||||
make build
|
||||
./bin/vfs put /tmp/in.txt --backend "s3://my-bucket/vfs-prefix?region=us-east-1" \
|
||||
--age-recipient "$RECIPIENT" --age-key /tmp/vfs.key
|
||||
```
|
||||
|
||||
## What this is
|
||||
|
||||
`vfs` is a 4 KiB block-level virtual filesystem that hashes every block with blake3-256, encrypts every block with `luxfi/age` (X25519, optionally hybrid PQ via ML-KEM-768), and stores them on a pluggable backend (`file://`, `s3://`, `gcs://`, `azureblob://`). In-memory LRU for the hot tier, object store for the cold tier — every Hanzo Go service that needs unlimited write capacity uses `vfs` instead of pre-sizing a PVC. Already exposes `func Mount()` for HIP-0106 inclusion.
|
||||
|
||||
## Specs
|
||||
|
||||
Implements:
|
||||
- HIP-0107 Streaming Replication over VFS
|
||||
- HIP-0106 Unified Cloud Binary (vfs subsystem — already exposes `Mount()`)
|
||||
|
||||
Companion to [hanzoai/replicate](https://github.com/hanzoai/replicate): `replicate` streams SQLite WAL frames file-level; `vfs` does block-level for any file.
|
||||
|
||||
## Architecture
|
||||
|
||||
```
|
||||
write() -> vfs.PutBlock -> blake3-256 hash -> luxfi/age encrypt
|
||||
|
|
||||
blocks/<2-hex-shard>/<full-hash>.zap.age
|
||||
|
|
||||
file:// | s3:// | gcs:// | azureblob://
|
||||
|
|
||||
in-memory LRU (hot)
|
||||
|
|
||||
FUSE mount (lands in 0.2.0)
|
||||
```
|
||||
|
||||
|
||||
---
|
||||
|
||||
# vfs
|
||||
|
||||
S3-backed virtual block filesystem with PQ encryption — unlimited write storage for stateful services.
|
||||
|
||||
-17
@@ -1,17 +0,0 @@
|
||||
# Security Policy
|
||||
|
||||
## Reporting a vulnerability
|
||||
|
||||
Email security@hanzo.ai with details. Encrypt with our PGP key (fingerprint TBD).
|
||||
|
||||
We respond within 48 hours. Critical issues receive same-day acknowledgment.
|
||||
|
||||
## Scope
|
||||
|
||||
This policy covers code in this repository. For the broader Hanzo platform threat model, see [hanzoai/HIPs](https://github.com/hanzoai/HIPs).
|
||||
|
||||
## Sandbox boundary
|
||||
|
||||
`vfs` encrypts every block client-side with `luxfi/age` (X25519, optionally hybrid PQ via ML-KEM-768) before any byte touches the configured backend, so the object store sees only ciphertext. Block integrity is enforced via blake3-256 content addressing — corruption or substitution by the backend is detected on read.
|
||||
|
||||
For runtime sandbox guarantees, see HIP-0105 (in-process extension runtimes).
|
||||
@@ -1,70 +0,0 @@
|
||||
// vfs-cost — print backend cost comparison for a workload.
|
||||
//
|
||||
// vfs-cost is intentionally workload-neutral. It takes a generic
|
||||
// Workload (or a benchmark projection) and reports the per-month cost
|
||||
// on every supported backend, sorted cheapest first.
|
||||
//
|
||||
// Application-specific projections (validator archives, sqlite query
|
||||
// DBs, snapshot pipelines) live in the calling repo, not here.
|
||||
//
|
||||
// Examples:
|
||||
//
|
||||
// # Manual workload: 100 GB stored, 5M PUTs/mo, 50M GETs/mo, 20 GB egress.
|
||||
// vfs-cost --storage 100 --puts 5e6 --gets 5e7 --egress 20
|
||||
//
|
||||
// # Take a measured ops/s rate from a benchmark and project forward.
|
||||
// vfs-cost --ops 6.4 --util 0.1 --avg 65536 --months 12 --read-amp 2
|
||||
package main
|
||||
|
||||
import (
|
||||
"flag"
|
||||
"fmt"
|
||||
"os"
|
||||
|
||||
"github.com/hanzoai/vfs/pkg/cost"
|
||||
)
|
||||
|
||||
func main() {
|
||||
storage := flag.Float64("storage", -1, "steady-state stored volume in GB (selects manual mode)")
|
||||
puts := flag.Float64("puts", 0, "PUT requests per month (manual mode)")
|
||||
gets := flag.Float64("gets", 0, "GET requests per month (manual mode)")
|
||||
egress := flag.Float64("egress", 0, "cross-region/cross-cloud egress per month, GB")
|
||||
|
||||
// benchmark mode
|
||||
ops := flag.Float64("ops", -1, "(benchmark mode) measured PUTs per second")
|
||||
util := flag.Float64("util", 0.1, "(benchmark mode) sustained-utilization fraction (0..1)")
|
||||
avg := flag.Int64("avg", 64*1024, "(benchmark mode) average file size in bytes")
|
||||
months := flag.Float64("months", 12, "(benchmark mode) retention in months")
|
||||
readAmp := flag.Float64("read-amp", 2.0, "(benchmark mode) GetsPerMonth / PutsPerMonth")
|
||||
|
||||
flag.Parse()
|
||||
|
||||
var (
|
||||
w cost.Workload
|
||||
name string
|
||||
)
|
||||
switch {
|
||||
case *storage > 0:
|
||||
w = cost.Workload{
|
||||
StorageGB: *storage,
|
||||
PutsPerMonth: *puts,
|
||||
GetsPerMonth: *gets,
|
||||
EgressGBPerMonth: *egress,
|
||||
Description: fmt.Sprintf("manual: %.0f GB stored, %.2e puts/mo, %.2e gets/mo, %.0f GB egress/mo",
|
||||
*storage, *puts, *gets, *egress),
|
||||
}
|
||||
name = "manual"
|
||||
case *ops > 0:
|
||||
w = cost.FromBenchmark(*ops, *util, *avg, *months, *readAmp)
|
||||
w.EgressGBPerMonth = *egress
|
||||
name = "benchmark-projection"
|
||||
default:
|
||||
fmt.Fprintln(os.Stderr, "vfs-cost: supply either --storage or --ops to describe a workload (see --help)")
|
||||
os.Exit(2)
|
||||
}
|
||||
|
||||
if _, err := os.Stdout.WriteString(cost.Report(name, w)); err != nil {
|
||||
fmt.Fprintln(os.Stderr, "write:", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,137 @@
|
||||
// vfsd — HIP-0106 thin shim.
|
||||
//
|
||||
// Mounts pkg/vfs into a zip.App via the same Mount() the unified cloud
|
||||
// binary calls. The CLI tooling for put/get/stats/mount lives in
|
||||
// cmd/vfs; this binary is the standalone HTTP server shape for
|
||||
// environments that need block CRUD over HTTP without the full cloud
|
||||
// surface.
|
||||
//
|
||||
// The data plane is only enabled when a VFS instance is attached via
|
||||
// vfs.SetInstance. Without one the binary still serves /v1/vfs/health
|
||||
// and a 503 /v1/vfs/readyz — useful for ConfigMap-driven k8s probes
|
||||
// while the operator wires the backend.
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"os"
|
||||
"os/signal"
|
||||
"strings"
|
||||
"syscall"
|
||||
"time"
|
||||
|
||||
"github.com/luxfi/age"
|
||||
"github.com/luxfi/log"
|
||||
|
||||
"github.com/hanzoai/cloud"
|
||||
"github.com/hanzoai/vfs/pkg/backend"
|
||||
_ "github.com/hanzoai/vfs/pkg/backend/file"
|
||||
_ "github.com/hanzoai/vfs/pkg/backend/s3"
|
||||
"github.com/hanzoai/vfs"
|
||||
"github.com/hanzoai/zip"
|
||||
"github.com/hanzoai/zip/middleware"
|
||||
)
|
||||
|
||||
// version is overridden at build time via -ldflags "-X main.version=...".
|
||||
var version = "dev"
|
||||
|
||||
func main() {
|
||||
vfs.Version = version
|
||||
|
||||
cfg := cloud.LoadConfig()
|
||||
if err := cfg.Validate(); err != nil {
|
||||
fmt.Fprintf(os.Stderr, "config: %v\n", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
|
||||
deps := cloud.BuildDeps(cfg)
|
||||
|
||||
// Optional backend wiring via env. VFSD_BACKEND=file:///tmp/v or
|
||||
// s3://bucket/prefix. When unset, the binary boots with no data
|
||||
// plane — /v1/vfs/blocks/* return 503 until SetInstance fires.
|
||||
if backendURL := strings.TrimSpace(os.Getenv("VFSD_BACKEND")); backendURL != "" {
|
||||
if v, err := buildVFS(backendURL); err != nil {
|
||||
log.Crit("vfsd: backend wiring failed", "err", err)
|
||||
} else {
|
||||
vfs.SetInstance(v)
|
||||
log.Info("vfsd: backend attached", "backend", backendURL)
|
||||
}
|
||||
} else {
|
||||
log.Info("vfsd: VFSD_BACKEND unset — data plane disabled (readyz returns 503)")
|
||||
}
|
||||
|
||||
app := zip.New(zip.Config{
|
||||
Logger: deps.Logger,
|
||||
AppName: "vfsd",
|
||||
})
|
||||
app.Use(middleware.Recover())
|
||||
app.Use(middleware.RequestID())
|
||||
app.Use(middleware.Logger(deps.Logger))
|
||||
|
||||
if err := vfs.Mount(app, deps); err != nil {
|
||||
log.Crit("vfsd: mount", "err", err)
|
||||
}
|
||||
|
||||
listenErr := make(chan error, 1)
|
||||
go func() {
|
||||
log.Info("vfsd: listening", "addr", cfg.ListenAddr)
|
||||
listenErr <- app.Listen(cfg.ListenAddr)
|
||||
}()
|
||||
|
||||
sig := make(chan os.Signal, 1)
|
||||
signal.Notify(sig, syscall.SIGINT, syscall.SIGTERM)
|
||||
select {
|
||||
case s := <-sig:
|
||||
log.Info("vfsd: shutting down", "signal", s)
|
||||
case err := <-listenErr:
|
||||
log.Crit("vfsd: listen failed", "err", err)
|
||||
}
|
||||
|
||||
stopCtx, stopCancel := context.WithTimeout(context.Background(), 10*time.Second)
|
||||
defer stopCancel()
|
||||
_ = vfs.Shutdown(stopCtx)
|
||||
_ = app.ShutdownWithContext(stopCtx)
|
||||
}
|
||||
|
||||
// buildVFS wires a VFS instance from env. Mirrors the cmd/vfs CLI's
|
||||
// openVFS but reads from env instead of flags. age keys come from
|
||||
// VFSD_AGE_KEY (path to key file) — required for any meaningful
|
||||
// roundtrip. Without it the instance is write-only.
|
||||
func buildVFS(backendURL string) (*vfs.VFS, error) {
|
||||
ctx := context.Background()
|
||||
be, err := backend.Open(ctx, backendURL)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("backend.Open: %w", err)
|
||||
}
|
||||
|
||||
var recs []age.Recipient
|
||||
var ids []age.Identity
|
||||
if keyPath := strings.TrimSpace(os.Getenv("VFSD_AGE_KEY")); keyPath != "" {
|
||||
keyBytes, err := os.ReadFile(keyPath)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("read VFSD_AGE_KEY: %w", err)
|
||||
}
|
||||
parsed, err := age.ParseIdentities(strings.NewReader(string(keyBytes)))
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("parse age identities: %w", err)
|
||||
}
|
||||
ids = parsed
|
||||
// Recipients default to the identities' own public keys.
|
||||
for _, id := range parsed {
|
||||
if x25519, ok := id.(*age.X25519Identity); ok {
|
||||
recs = append(recs, x25519.Recipient())
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
crypto, err := vfs.NewCrypto(recs, ids)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("vfs.NewCrypto: %w", err)
|
||||
}
|
||||
return vfs.New(vfs.Config{
|
||||
Backend: be,
|
||||
Crypto: crypto,
|
||||
CacheMax: 256 << 20,
|
||||
})
|
||||
}
|
||||
@@ -1,481 +0,0 @@
|
||||
# Production readiness for VFS-backed stateful workloads
|
||||
|
||||
Status: **production-grade for finalized-block streaming; not for hot
|
||||
zapdb WAL writes** — see "Verdict" below.
|
||||
|
||||
This doc answers the seven concerns that gate using VFS as the durable
|
||||
backing store for stateful services like `luxd` (Lux primary network
|
||||
node, zapdb storage), liquid-EVM chain pods, indexer DBs, etc. The
|
||||
first four ("code-level concerns") have automated tests in
|
||||
[`production_test.go`](../production_test.go); the last three
|
||||
("operational concerns") are runbooks against a real cluster.
|
||||
|
||||
| # | Concern | Status | Evidence |
|
||||
|---|------------------------------------------|-------------|-------------------------------------------------------------------------|
|
||||
| 1 | crash / restart recovery | **PASS** | `TestCrashRecovery_Synced_Persists`, `TestCrashRecovery_UnsyncedWriteIsLost_NotCorrupted` |
|
||||
| 4 | compaction behavior under load | **PASS** | `TestCompaction_SustainedWrite` — finalized writes are non-fsync stream |
|
||||
| 5 | backend outage behavior | **PASS** | `TestBackendOutage_PutErrorBubblesUp`, `TestBackendOutage_GetErrorBubblesUp` |
|
||||
| 6 | KMS / age key rotation | **PASS** | `TestKeyRotation_MultiRecipientAllowsRollingRotation` |
|
||||
| 2 | empty-cache bootstrap timing | runbook §A | execute against staging |
|
||||
| 3 | warm-cache bootstrap timing | runbook §B | execute against staging |
|
||||
| 7 | rollback path to PVC-local state | runbook §C | doc'd procedure |
|
||||
|
||||
## Verdict
|
||||
|
||||
For blockchain state the right model is **hot-tier on local disk,
|
||||
finalized-tier on VFS**. Blocks past finality are immutable forever —
|
||||
they only need to be written ONCE and only ever read. This is
|
||||
content-addressed VFS's ideal workload:
|
||||
|
||||
- One-shot streaming write per finalized SST → never mutated
|
||||
- No `fsync` round-trip per write (batched commit at finality)
|
||||
- Content-hash dedup across all replicas (the same canonical block
|
||||
written by N validators = one VFS object on the backend)
|
||||
- Local disk only holds the unfinalized hot tier (WAL + memtable + last
|
||||
K SSTs that may still merge before finality)
|
||||
|
||||
This pattern means the §4 benchmark (6.4 fsync'd ops/s) is **not on the
|
||||
critical path** for chain state — it would only matter if we tried to
|
||||
serve `zapdb` WAL writes through VFS, which we don't. zapdb keeps its
|
||||
WAL on local SSD and streams finalized SSTs into VFS in batches.
|
||||
|
||||
✅ Production-grade now for:
|
||||
|
||||
- Finalized chain blocks (luxd archive tier — write once, immutable)
|
||||
- zapdb SST files past finality (stream + delete local copy)
|
||||
- Snapshots / pruned-state archives
|
||||
- Off-cluster backups of structured state
|
||||
- Analytics SQLite that batch-writes on schedule (e.g. indexer DBs)
|
||||
- Config + secret storage where audit trail + PQ encryption matter
|
||||
|
||||
❌ Not the right tool for:
|
||||
|
||||
- zapdb WAL on the hot path (use local SSD)
|
||||
- zapdb memtable spill on the hot path (use local SSD)
|
||||
- Anything that needs >10 sustained fsync'd ops/s in the steady state
|
||||
|
||||
The mitigation roadmap below (Sync coalescer, async upload pipeline,
|
||||
sharded metadata) only matters for cases outside the immutable
|
||||
finalized-block use case.
|
||||
|
||||
## The hot/cold split for zapdb
|
||||
|
||||
zapdb is luxd's LSM-tree storage engine (analogous to Pebble/Rocks, but the
|
||||
Lux fork). Its on-disk layout splits naturally along the finality line:
|
||||
|
||||
```
|
||||
Local PVC (SSD, bounded, ~50 GB)
|
||||
├─ wal/ ← hot writes, fsync per commit
|
||||
├─ memtable ← in-RAM, persisted via WAL
|
||||
├─ recent SST/ ← last K compactions; SSTs may still merge
|
||||
│ with newer data until they are >F blocks
|
||||
│ below current chain head ("finality depth")
|
||||
└─ manifest ← metadata index pointing at both tiers
|
||||
|
||||
VFS → S3 (object store, unbounded)
|
||||
├─ finalized SST/<hash> ← every SST that crossed finality depth
|
||||
│ gets streamed up, ONCE, then deleted from
|
||||
│ the local PVC; the manifest now points
|
||||
│ at the VFS-side block ID
|
||||
├─ snapshots/ ← periodic checkpoint exports
|
||||
└─ pruned-state-trie/ ← evicted Merkle trie nodes (history mode)
|
||||
```
|
||||
|
||||
The lifecycle of a finalized SST:
|
||||
|
||||
1. zapdb compaction emits a new SST file at level L on local SSD.
|
||||
2. The SST sits in `recent SST/` until its highest block height is
|
||||
`> chain_head - F` blocks (F = network finality depth; in Quasar/PQ
|
||||
strict mode this is ~30 blocks).
|
||||
3. Background "vfs-promoter" goroutine in zapdb streams the SST through
|
||||
VFS:
|
||||
- Read SST file → `vfs.PutBlock(chunk)` for each 4 KiB block
|
||||
- Receive back content-addressed `BlockID`s
|
||||
- Write a small metadata file recording the SST → [BlockID...] map
|
||||
- The metadata file lives on local PVC (small, ~1 KB per SST)
|
||||
- Delete the local SST file
|
||||
4. Next read of that SST height range: zapdb sees the SST is "VFS-only",
|
||||
fetches the metadata, reads the BlockIDs via `vfs.GetBlock`, caches
|
||||
the hot blocks in vfsd's LRU.
|
||||
|
||||
Key properties:
|
||||
|
||||
- Local PVC stays bounded at `WAL + memtable + recent_window`. For a
|
||||
Lux mainnet validator that's ~50 GB regardless of how long the chain
|
||||
has been running.
|
||||
- The S3 archive grows monotonically but ~$23/TB/month vs $0.10/GB/month
|
||||
for DOKS PVC = 4× cheaper at scale.
|
||||
- Cross-validator dedup: the same canonical SST emitted by N validators
|
||||
hashes to the same blocks → one S3 object backs N replicas.
|
||||
|
||||
## §1 — Crash / restart recovery
|
||||
|
||||
**Question:** if `vfsd` (or its host) crashes mid-write, is the FS view
|
||||
after restart consistent? Specifically:
|
||||
|
||||
- writes Sync()-ed before the crash must be visible byte-for-byte after
|
||||
the restart
|
||||
- writes not Sync()-ed before the crash must NOT resurface (no
|
||||
"phantom" data, no torn blocks)
|
||||
|
||||
**Verified by:**
|
||||
|
||||
```
|
||||
go test -run TestCrashRecovery -v
|
||||
```
|
||||
|
||||
The two test cases pin both halves of the durability contract. Both
|
||||
PASS on file backend; S3 inherits the same guarantee because the
|
||||
metadata blob is replaced atomically (S3 PUT is atomic at the object
|
||||
level).
|
||||
|
||||
**Caveats:**
|
||||
|
||||
- The metadata blob is a single object (`metadata/root.zap.age`) holding
|
||||
the entire inode tree. At >100K files this becomes too large to load
|
||||
on every reopen; sharded metadata (`metadata/<inode_id>.zap.age`) is
|
||||
on the roadmap. For the finalized-SST use case (~1 file per SST, with
|
||||
zapdb compaction emitting on the order of 1000s of SSTs per year of
|
||||
mainnet activity) we are within the headroom.
|
||||
- `Sync` is the durability boundary. POSIX consumers MUST call `fsync(2)`
|
||||
on every write they want durable across crash. The zapdb VFS-promoter
|
||||
fsyncs once at the end of each SST upload (after all its blocks are
|
||||
in), not once per block — matching the immutable-finalized model.
|
||||
|
||||
## §4 — Compaction behavior under load
|
||||
|
||||
**Question:** can VFS keep up with sustained write pressure from a
|
||||
zapdb-style LSM-tree workload?
|
||||
|
||||
**Answer for the hot-path workload (zapdb WAL + memtable spill):**
|
||||
No, VFS is not the right place. Use local SSD. See §4 details below
|
||||
for the measured numbers.
|
||||
|
||||
**Answer for the finalized-SST workload:**
|
||||
Yes, by a large margin. Finalized SSTs are streamed sequentially
|
||||
(no random writes), batched at the end (one fsync per SST), and never
|
||||
mutated again. The throughput bound is just network bandwidth to S3,
|
||||
which on a single DOKS node is ~1 GB/s — well above the SST emission
|
||||
rate (~10s of MB/s under sustained chain load).
|
||||
|
||||
**Measured by:**
|
||||
|
||||
```
|
||||
VFS_PROD_TEST=1 VFS_PROD_TEST_DURATION=20s go test -run TestCompaction_SustainedWrite -v
|
||||
```
|
||||
|
||||
The benchmark writes 64 KiB chunks to a 16-file rotation with `Sync`
|
||||
after every write — this models the worst case (heavy fsync from a
|
||||
naive consumer), not the finalized-SST stream pattern.
|
||||
|
||||
**Observed (Apple M-series, file backend, 2026-06-03):**
|
||||
|
||||
| Metric | Value |
|
||||
|--------|-----------------|
|
||||
| ops | 128 in 20s |
|
||||
| mean | 157 ms / op |
|
||||
| p100 | 321 ms / op |
|
||||
| thru | 6.4 ops/s |
|
||||
|
||||
**Interpretation:**
|
||||
|
||||
This is the "worst case" measurement — fsync-per-write through the
|
||||
metadata-blob rewrite path. The finalized-SST stream pattern looks
|
||||
nothing like this:
|
||||
|
||||
- One Sync per SST file at the end of upload (not per 64 KiB chunk)
|
||||
- Block uploads can be parallelized (currently serialized inside
|
||||
`File.Sync`, but trivial to parallelize)
|
||||
- The metadata blob is rewritten ONCE per SST, not per chunk
|
||||
|
||||
A representative measurement of the finalized-SST pattern (one Sync
|
||||
per 100 MB SST file) would be: `100 MB / 5 MB/s upload = 20s wall
|
||||
time, dominated by S3 PUT throughput`. That's well within the budget
|
||||
since SSTs are emitted at ~10 per minute under heavy load.
|
||||
|
||||
**Mitigations on the roadmap (only relevant if you push fsync-heavy
|
||||
workloads through VFS):**
|
||||
|
||||
1. **Sync coalescing.** Debounce metadata writes: flush at most once per
|
||||
200ms even if multiple `File.Sync` calls land in between. Reduces
|
||||
the metadata-blob churn by ~10×.
|
||||
2. **Async upload pipeline.** Pipeline block uploads in N parallel
|
||||
workers; Sync returns once blocks are queued + the metadata stub is
|
||||
written to a local WAL on the same backend. Recovery replays the
|
||||
WAL.
|
||||
3. **Sharded metadata.** Per-inode metadata blobs (one age-encrypted
|
||||
object per file) eliminate the whole-tree rewrite on every Sync.
|
||||
4. **Local cache spill-to-disk.** Hot blocks stay on local ephemeral
|
||||
SSD as well as object store; reads can be served from disk without
|
||||
round-tripping S3.
|
||||
|
||||
For zapdb-finalized-SST streaming, none of these are required.
|
||||
|
||||
## §1 — Crash / restart recovery
|
||||
|
||||
**Question:** if `vfsd` (or its host) crashes mid-write, is the FS view
|
||||
after restart consistent? Specifically:
|
||||
|
||||
- writes Sync()-ed before the crash must be visible byte-for-byte after
|
||||
the restart
|
||||
- writes not Sync()-ed before the crash must NOT resurface (no
|
||||
"phantom" data, no torn blocks)
|
||||
|
||||
**Verified by:**
|
||||
|
||||
```
|
||||
go test -run TestCrashRecovery -v
|
||||
```
|
||||
|
||||
The two test cases pin both halves of the durability contract. Both
|
||||
PASS on file backend; S3 inherits the same guarantee because the
|
||||
metadata blob is replaced atomically (S3 PUT is atomic at the object
|
||||
level).
|
||||
|
||||
**Caveats:**
|
||||
|
||||
- The metadata blob is a single object (`metadata/root.zap.age`) holding
|
||||
the entire inode tree. At >100K files this becomes too large to load
|
||||
on every reopen; sharded metadata (`metadata/<inode_id>.zap.age`) is
|
||||
on the roadmap. For luxd zapdb (~10-100 files in a typical RocksDB
|
||||
layout) we are well below this limit.
|
||||
- `Sync` is the durability boundary. POSIX consumers MUST call `fsync(2)`
|
||||
on every write they want durable across crash — exactly what zapdb
|
||||
/ SQLite already do.
|
||||
|
||||
## §4 — Compaction behavior under load
|
||||
|
||||
**Question:** can VFS keep up with sustained write pressure from a
|
||||
zapdb-style LSM-tree workload (many small files, frequent fsync,
|
||||
occasional bulk rewrite during compaction)?
|
||||
|
||||
**Measured by:**
|
||||
|
||||
```
|
||||
VFS_PROD_TEST=1 VFS_PROD_TEST_DURATION=20s go test -run TestCompaction_SustainedWrite -v
|
||||
```
|
||||
|
||||
The benchmark writes 64 KiB chunks to a 16-file rotation with `Sync`
|
||||
after every write — a coarse approximation of LSM compaction's IO
|
||||
pattern. The test runs locally against the `file://` backend (no
|
||||
network); S3/GCS results will be slower.
|
||||
|
||||
**Observed (Apple M-series, file backend, 2026-06-03):**
|
||||
|
||||
| Metric | Value |
|
||||
|--------|-----------------|
|
||||
| ops | 128 in 20s |
|
||||
| mean | 157 ms / op |
|
||||
| p100 | 321 ms / op |
|
||||
| thru | 6.4 ops/s |
|
||||
|
||||
**Interpretation:**
|
||||
|
||||
Each Sync re-encrypts + re-uploads every dirty block AND rewrites the
|
||||
entire metadata blob (whole inode tree as one age-encrypted object).
|
||||
This makes Sync linear in the number of dirty blocks + the metadata
|
||||
size. For an LSM workload with thousands of fsync'd writes per second
|
||||
on local SSD, VFS is ~150× too slow to be the hot-tier store.
|
||||
|
||||
**Mitigations on the roadmap (none of which are required for the
|
||||
cold/archive use case):**
|
||||
|
||||
1. **Sync coalescing.** Debounce metadata writes: flush at most once per
|
||||
200ms even if multiple `File.Sync` calls land in between. Reduces
|
||||
the metadata-blob churn by ~10×.
|
||||
2. **Async upload pipeline.** Pipeline block uploads in N parallel
|
||||
workers; Sync returns once blocks are queued + the metadata stub is
|
||||
written to a local WAL on the same backend. Recovery replays the
|
||||
WAL.
|
||||
3. **Sharded metadata.** Per-inode metadata blobs (one age-encrypted
|
||||
object per file) eliminate the whole-tree rewrite on every Sync.
|
||||
4. **Local cache spill-to-disk.** Hot blocks stay on local ephemeral
|
||||
SSD as well as object store; reads can be served from disk without
|
||||
round-tripping S3.
|
||||
|
||||
Until at least (1)+(2) land, do not target hot luxd state.
|
||||
|
||||
## §5 — Backend outage behavior
|
||||
|
||||
**Question:** when the backend (S3 / GCS / DO Spaces) is unreachable,
|
||||
does VFS hang, retry forever, lose data silently, or surface the error
|
||||
to the caller?
|
||||
|
||||
**Verified by:**
|
||||
|
||||
```
|
||||
go test -run TestBackendOutage -v
|
||||
```
|
||||
|
||||
Two tests inject backend errors and confirm:
|
||||
|
||||
- `TestBackendOutage_PutErrorBubblesUp`: a backend Put failure surfaces
|
||||
as a `Sync` error. The POSIX consumer sees fsync return non-nil and
|
||||
can treat the write as failed.
|
||||
- `TestBackendOutage_GetErrorBubblesUp`: a backend Get failure surfaces
|
||||
as a ReadAt error. The consumer never sees zero bytes silently.
|
||||
|
||||
Both PASS.
|
||||
|
||||
**Operational implication:** when S3 is partitioned, luxd's zapdb
|
||||
will get `fsync: EIO` and crash-loop. That's the correct behavior —
|
||||
better to fail hard than to acknowledge a write that's lost.
|
||||
|
||||
**What does NOT exist yet:**
|
||||
|
||||
- Per-request retry with exponential backoff in the backend. The
|
||||
current behavior is fail-fast on first error. Adding bounded retries
|
||||
(e.g. 3 tries with 1s/2s/4s backoff) would let the FS ride out
|
||||
transient 5xx responses without bubbling.
|
||||
- A circuit breaker / health endpoint that goes "degraded" before going
|
||||
"failed", letting orchestration page on warning rather than on
|
||||
crash-loop.
|
||||
|
||||
## §6 — KMS / age key rotation
|
||||
|
||||
**Question:** can we rotate the age key without rewriting every
|
||||
encrypted block in the store?
|
||||
|
||||
**Verified by:**
|
||||
|
||||
```
|
||||
go test -run TestKeyRotation -v
|
||||
```
|
||||
|
||||
The test walks through the rolling rotation procedure:
|
||||
|
||||
1. Write blocks under `Crypto{recipients: [old], identities: [old]}`.
|
||||
2. Enter rotation window: switch to
|
||||
`Crypto{recipients: [old, new], identities: [old, new]}`. New writes
|
||||
include stanzas for both recipients; reads work via either identity.
|
||||
3. Exit rotation window: switch to
|
||||
`Crypto{recipients: [new], identities: [new]}`. Blocks written
|
||||
during the rotation window remain readable (newID is in the
|
||||
envelope); blocks written before the rotation are NO LONGER readable
|
||||
under just `[new]`.
|
||||
|
||||
This pins the contract that rotation is **real** (dropping the old key
|
||||
loses access to pre-rotation blocks) **and** non-destructive (no bulk
|
||||
re-encrypt required).
|
||||
|
||||
**Operational procedure:**
|
||||
|
||||
1. Generate a new age identity (`luxfi/age` CLI or
|
||||
`age.GenerateX25519Identity()`). Store the private key in Hanzo KMS
|
||||
at `secret/vfs/<service>/<env>/keys/<key-id>`.
|
||||
2. Update the `vfsd` config to list BOTH the old and the new key.
|
||||
Restart vfsd (rolling). Length of rotation window = how long you
|
||||
need old blocks readable; typically 30-90 days.
|
||||
3. (Optional) Run the migration tool (TBD — not yet written) to
|
||||
re-encrypt pre-rotation blocks under the new key. Necessary only if
|
||||
you want to drop the old key from KMS before the rotation window
|
||||
ends.
|
||||
4. After the window: update vfsd config to drop the old key. Restart.
|
||||
5. Delete the old key from KMS (or mark it `historical`).
|
||||
|
||||
## §2 — Empty-cache bootstrap timing (runbook)
|
||||
|
||||
**Procedure:**
|
||||
|
||||
1. Provision a fresh luxd pod with vfsd sidecar. Mount VFS at
|
||||
`/data/db-vfs`. No pre-populated cache.
|
||||
2. Configure backend = S3 bucket holding a previously-snapshotted luxd
|
||||
state directory.
|
||||
3. Start luxd with `--data-dir=/data/db-vfs`.
|
||||
4. Measure wall-clock from luxd start to `info.isBootstrapped == true`
|
||||
on the P-chain.
|
||||
|
||||
**Expected:** because zapdb on cold-start reads thousands of SST
|
||||
files to rebuild the manifest, expect **hours** on an empty cache with
|
||||
S3 latencies (50-100ms × thousands of reads). Operators who care about
|
||||
bootstrap-from-cold should use a snapshot warm path (see §3).
|
||||
|
||||
**Status:** untested — execute against staging before any production
|
||||
deployment.
|
||||
|
||||
## §3 — Warm-cache bootstrap timing (runbook)
|
||||
|
||||
**Procedure:**
|
||||
|
||||
1. Same setup as §2, but seed the vfsd LRU with a snapshot of the
|
||||
prior pod's cache. Two options:
|
||||
- **Tar copy** from a healthy pod's `/var/cache/vfsd` to the new
|
||||
pod before luxd starts.
|
||||
- **Persistent cache PVC**: vfsd's LRU stored on a small
|
||||
local-disk PVC (e.g. 64 GB SSD) so cache survives pod restart.
|
||||
2. Start luxd. Measure same metric as §2.
|
||||
|
||||
**Expected:** cache hit rate should track the working set's
|
||||
heat-distribution. For luxd mainnet the active working set is roughly
|
||||
the last N=2-5 epochs of state (~5-20 GB). With 32 GB warm cache the
|
||||
bootstrap should complete in ~5-10 minutes (limited by replay, not by
|
||||
backend reads).
|
||||
|
||||
**Status:** untested — same gate as §2.
|
||||
|
||||
## §7 — Rollback path to PVC-local state
|
||||
|
||||
If VFS proves unworkable (hot-path latency, backend SLA, anything),
|
||||
this is the documented rollback to plain PVC-local state. Tested by
|
||||
inversion: it's the same procedure that produced the original PVC
|
||||
layout we're migrating away from.
|
||||
|
||||
### Pre-rollback checklist
|
||||
|
||||
- Decide the cutoff height (e.g. "luxd block N at unix-time T").
|
||||
- Confirm a luxd snapshot at the cutoff exists on the VFS backend
|
||||
(so you can repopulate the new PVC from it).
|
||||
|
||||
### Procedure
|
||||
|
||||
1. **Drain.** Stop luxd cleanly (`kubectl delete pod luxd-0 --grace=600`).
|
||||
2. **Snapshot via vfsd.** From a temporary sidecar pod, run:
|
||||
```
|
||||
vfsd export --backend s3://lux-mainnet-state/luxd \
|
||||
--age-key <vfs-key> \
|
||||
--output /tmp/snapshot.tar
|
||||
```
|
||||
This walks the inode tree and writes a POSIX tar of the decrypted
|
||||
plaintext.
|
||||
3. **Provision a fresh PVC** of size = snapshot uncompressed +
|
||||
headroom. e.g. `kubectl create -f pvc-luxd-fresh.yaml`.
|
||||
4. **Restore.** Mount the new PVC at `/data/db` on a temporary pod,
|
||||
extract the snapshot, then chown to luxd uid:gid.
|
||||
5. **Switch the StatefulSet.** Update `luxd-startup` configmap:
|
||||
- Remove `vfsd` sidecar.
|
||||
- Repoint `--data-dir` from `/data/db-vfs` to `/data/db`.
|
||||
- Remove the `emptyDir` volume that held the FUSE mount.
|
||||
6. **Rolling restart** `luxd` pods one at a time, verifying P-chain
|
||||
bootstrap on each.
|
||||
7. **Decommission VFS.** Once all pods are PVC-backed and healthy,
|
||||
delete the `vfsd` deployment + backend bucket (or keep the bucket
|
||||
for offline forensics).
|
||||
|
||||
### What does NOT exist yet
|
||||
|
||||
- A `vfsd export` subcommand that walks the inode tree and emits a tar.
|
||||
Today, an operator would need to FUSE-mount the VFS and `tar -c .`
|
||||
from the mountpoint. Equivalent but less ergonomic.
|
||||
- A reverse direction (`vfsd import` from a tar) — needed if you want
|
||||
to re-mount VFS at a later date from an offline backup. Not on the
|
||||
critical path; you can always replay from chain.
|
||||
|
||||
## Running the tests
|
||||
|
||||
```bash
|
||||
# Default — quick coverage (skips the long sustained-write benchmark)
|
||||
GOWORK=off go test ./...
|
||||
|
||||
# Include the sustained-write benchmark (60s by default)
|
||||
VFS_PROD_TEST=1 GOWORK=off go test -run TestCompaction -v ./...
|
||||
|
||||
# Custom duration
|
||||
VFS_PROD_TEST=1 VFS_PROD_TEST_DURATION=5m GOWORK=off go test -run TestCompaction -v ./...
|
||||
```
|
||||
|
||||
For the FUSE end-to-end (TestFUSESQLite in `pkg/mount/`) you need
|
||||
Linux + kernel FUSE + privileges:
|
||||
|
||||
```bash
|
||||
VFS_FUSE_E2E=1 GOWORK=off go test -tags fuse -run TestFUSESQLite -v ./pkg/mount/
|
||||
```
|
||||
@@ -1,48 +1,70 @@
|
||||
module github.com/hanzoai/vfs
|
||||
|
||||
go 1.26.4
|
||||
go 1.26.3
|
||||
|
||||
require (
|
||||
bazil.org/fuse v0.0.0-20230120002735-62a210ff1fd5
|
||||
github.com/aws/aws-sdk-go-v2 v1.41.2
|
||||
github.com/aws/aws-sdk-go-v2/config v1.32.8
|
||||
github.com/aws/aws-sdk-go-v2/service/s3 v1.96.2
|
||||
github.com/hanzoai/sqlite v0.3.2
|
||||
github.com/aws/aws-sdk-go-v2 v1.30.0
|
||||
github.com/aws/aws-sdk-go-v2/config v1.27.0
|
||||
github.com/aws/aws-sdk-go-v2/service/s3 v1.55.0
|
||||
github.com/hanzoai/cloud v0.0.0
|
||||
github.com/hanzoai/zip v0.0.0
|
||||
github.com/jacobsa/fuse v0.0.0-20260302145937-f1ba38d60fdf
|
||||
github.com/luxfi/age v1.6.0
|
||||
github.com/spf13/cobra v1.10.2
|
||||
github.com/luxfi/age v1.5.0
|
||||
github.com/spf13/cobra v1.8.0
|
||||
github.com/zeebo/blake3 v0.2.4
|
||||
modernc.org/sqlite v1.50.0
|
||||
)
|
||||
|
||||
// HIP-0106 cloud Mount() — point at sibling cloud + zip checkouts.
|
||||
// CI overrides via GOPROXY once cloud is published.
|
||||
replace (
|
||||
github.com/hanzoai/cloud => ../cloud
|
||||
github.com/hanzoai/zip => ../zip
|
||||
)
|
||||
|
||||
require (
|
||||
filippo.io/hpke v0.4.0 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.5 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/credentials v1.19.8 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.17 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.18 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.18 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/internal/ini v1.8.4 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.18 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.5 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.10 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.18 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.18 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/signin v1.0.5 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/sso v1.30.9 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.14 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/sts v1.41.6 // indirect
|
||||
github.com/aws/smithy-go v1.24.1 // indirect
|
||||
github.com/andybalholm/brotli v1.2.1 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.2 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/credentials v1.17.0 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.15.0 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.8 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.8 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/internal/ini v1.8.0 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.8 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.11.2 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.3.10 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.11.10 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.17.8 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/sso v1.19.0 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.22.0 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/sts v1.27.0 // indirect
|
||||
github.com/aws/smithy-go v1.20.2 // indirect
|
||||
github.com/dustin/go-humanize v1.0.1 // indirect
|
||||
github.com/gofiber/fiber/v3 v3.2.0 // indirect
|
||||
github.com/gofiber/schema v1.7.1 // indirect
|
||||
github.com/gofiber/utils/v2 v2.0.4 // indirect
|
||||
github.com/google/uuid v1.6.0 // indirect
|
||||
github.com/hanzoai/csqlite v0.1.0 // indirect
|
||||
github.com/hanzoai/sqlcipher v0.1.0 // indirect
|
||||
github.com/inconshreveable/mousetrap v1.1.0 // indirect
|
||||
github.com/klauspost/cpuid/v2 v2.3.0 // indirect
|
||||
github.com/klauspost/compress v1.18.5 // indirect
|
||||
github.com/klauspost/cpuid/v2 v2.0.12 // indirect
|
||||
github.com/luxfi/log v1.4.3 // indirect
|
||||
github.com/mattn/go-colorable v0.1.14 // indirect
|
||||
github.com/mattn/go-isatty v0.0.21 // indirect
|
||||
github.com/ncruces/go-strftime v1.0.0 // indirect
|
||||
github.com/philhofer/fwd v1.2.0 // indirect
|
||||
github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect
|
||||
github.com/spf13/pflag v1.0.10 // indirect
|
||||
github.com/zeebo/assert v1.3.0 // indirect
|
||||
golang.org/x/crypto v0.52.0 // indirect
|
||||
golang.org/x/sys v0.46.0 // indirect
|
||||
github.com/spf13/pflag v1.0.5 // indirect
|
||||
github.com/tinylib/msgp v1.6.4 // indirect
|
||||
github.com/valyala/bytebufferpool v1.0.0 // indirect
|
||||
github.com/valyala/fasthttp v1.70.0 // indirect
|
||||
golang.org/x/crypto v0.50.0 // indirect
|
||||
golang.org/x/net v0.53.0 // indirect
|
||||
golang.org/x/sys v0.43.0 // indirect
|
||||
golang.org/x/text v0.36.0 // indirect
|
||||
gopkg.in/natefinch/lumberjack.v2 v2.2.1 // indirect
|
||||
modernc.org/libc v1.72.0 // indirect
|
||||
modernc.org/mathutil v1.7.1 // indirect
|
||||
modernc.org/memory v1.11.0 // indirect
|
||||
)
|
||||
|
||||
@@ -4,88 +4,162 @@ c2sp.org/CCTV/age v0.0.0-20251208015420-e9274a7bdbfd h1:ZLsPO6WdZ5zatV4UfVpr7oAw
|
||||
c2sp.org/CCTV/age v0.0.0-20251208015420-e9274a7bdbfd/go.mod h1:SrHC2C7r5GkDk8R+NFVzYy/sdj0Ypg9htaPXQq5Cqeo=
|
||||
filippo.io/hpke v0.4.0 h1:p575VVQ6ted4pL+it6M00V/f2qTZITO0zgmdKCkd5+A=
|
||||
filippo.io/hpke v0.4.0/go.mod h1:EmAN849/P3qdeK+PCMkDpDm83vRHM5cDipBJ8xbQLVY=
|
||||
github.com/aws/aws-sdk-go-v2 v1.41.2 h1:LuT2rzqNQsauaGkPK/7813XxcZ3o3yePY0Iy891T2ls=
|
||||
github.com/aws/aws-sdk-go-v2 v1.41.2/go.mod h1:IvvlAZQXvTXznUPfRVfryiG1fbzE2NGK6m9u39YQ+S4=
|
||||
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.5 h1:zWFmPmgw4sveAYi1mRqG+E/g0461cJ5M4bJ8/nc6d3Q=
|
||||
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.5/go.mod h1:nVUlMLVV8ycXSb7mSkcNu9e3v/1TJq2RTlrPwhYWr5c=
|
||||
github.com/aws/aws-sdk-go-v2/config v1.32.8 h1:iu+64gwDKEoKnyTQskSku72dAwggKI5sV6rNvgSMpMs=
|
||||
github.com/aws/aws-sdk-go-v2/config v1.32.8/go.mod h1:MI2XvA+qDi3i9AJxX1E2fu730syEBzp/jnXrjxuHwgI=
|
||||
github.com/aws/aws-sdk-go-v2/credentials v1.19.8 h1:Jp2JYH1lRT3KhX4mshHPvVYsR5qqRec3hGvEarNYoR0=
|
||||
github.com/aws/aws-sdk-go-v2/credentials v1.19.8/go.mod h1:fZG9tuvyVfxknv1rKibIz3DobRaFw1Poe8IKtXB3XYY=
|
||||
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.17 h1:I0GyV8wiYrP8XpA70g1HBcQO1JlQxCMTW9npl5UbDHY=
|
||||
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.17/go.mod h1:tyw7BOl5bBe/oqvoIeECFJjMdzXoa/dfVz3QQ5lgHGA=
|
||||
github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.18 h1:F43zk1vemYIqPAwhjTjYIz0irU2EY7sOb/F5eJ3HuyM=
|
||||
github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.18/go.mod h1:w1jdlZXrGKaJcNoL+Nnrj+k5wlpGXqnNrKoP22HvAug=
|
||||
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.18 h1:xCeWVjj0ki0l3nruoyP2slHsGArMxeiiaoPN5QZH6YQ=
|
||||
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.18/go.mod h1:r/eLGuGCBw6l36ZRWiw6PaZwPXb6YOj+i/7MizNl5/k=
|
||||
github.com/aws/aws-sdk-go-v2/internal/ini v1.8.4 h1:WKuaxf++XKWlHWu9ECbMlha8WOEGm0OUEZqm4K/Gcfk=
|
||||
github.com/aws/aws-sdk-go-v2/internal/ini v1.8.4/go.mod h1:ZWy7j6v1vWGmPReu0iSGvRiise4YI5SkR3OHKTZ6Wuc=
|
||||
github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.18 h1:eZioDaZGJ0tMM4gzmkNIO2aAoQd+je7Ug7TkvAzlmkU=
|
||||
github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.18/go.mod h1:CCXwUKAJdoWr6/NcxZ+zsiPr6oH/Q5aTooRGYieAyj4=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.5 h1:CeY9LUdur+Dxoeldqoun6y4WtJ3RQtzk0JMP2gfUay0=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.5/go.mod h1:AZLZf2fMaahW5s/wMRciu1sYbdsikT/UHwbUjOdEVTc=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.10 h1:fJvQ5mIBVfKtiyx0AHY6HeWcRX5LGANLpq8SVR+Uazs=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.10/go.mod h1:Kzm5e6OmNH8VMkgK9t+ry5jEih4Y8whqs+1hrkxim1I=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.18 h1:LTRCYFlnnKFlKsyIQxKhJuDuA3ZkrDQMRYm6rXiHlLY=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.18/go.mod h1:XhwkgGG6bHSd00nO/mexWTcTjgd6PjuvWQMqSn2UaEk=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.18 h1:/A/xDuZAVD2BpsS2fftFRo/NoEKQJ8YTnJDEHBy2Gtg=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.18/go.mod h1:hWe9b4f+djUQGmyiGEeOnZv69dtMSgpDRIvNMvuvzvY=
|
||||
github.com/aws/aws-sdk-go-v2/service/s3 v1.96.2 h1:M1A9AjcFwlxTLuf0Faj88L8Iqw0n/AJHjpZTQzMMsSc=
|
||||
github.com/aws/aws-sdk-go-v2/service/s3 v1.96.2/go.mod h1:KsdTV6Q9WKUZm2mNJnUFmIoXfZux91M3sr/a4REX8e0=
|
||||
github.com/aws/aws-sdk-go-v2/service/signin v1.0.5 h1:VrhDvQib/i0lxvr3zqlUwLwJP4fpmpyD9wYG1vfSu+Y=
|
||||
github.com/aws/aws-sdk-go-v2/service/signin v1.0.5/go.mod h1:k029+U8SY30/3/ras4G/Fnv/b88N4mAfliNn08Dem4M=
|
||||
github.com/aws/aws-sdk-go-v2/service/sso v1.30.9 h1:v6EiMvhEYBoHABfbGB4alOYmCIrcgyPPiBE1wZAEbqk=
|
||||
github.com/aws/aws-sdk-go-v2/service/sso v1.30.9/go.mod h1:yifAsgBxgJWn3ggx70A3urX2AN49Y5sJTD1UQFlfqBw=
|
||||
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.14 h1:0jbJeuEHlwKJ9PfXtpSFc4MF+WIWORdhN1n30ITZGFM=
|
||||
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.14/go.mod h1:sTGThjphYE4Ohw8vJiRStAcu3rbjtXRsdNB0TvZ5wwo=
|
||||
github.com/aws/aws-sdk-go-v2/service/sts v1.41.6 h1:5fFjR/ToSOzB2OQ/XqWpZBmNvmP/pJ1jOWYlFDJTjRQ=
|
||||
github.com/aws/aws-sdk-go-v2/service/sts v1.41.6/go.mod h1:qgFDZQSD/Kys7nJnVqYlWKnh0SSdMjAi0uSwON4wgYQ=
|
||||
github.com/aws/smithy-go v1.24.1 h1:VbyeNfmYkWoxMVpGUAbQumkODcYmfMRfZ8yQiH30SK0=
|
||||
github.com/aws/smithy-go v1.24.1/go.mod h1:LEj2LM3rBRQJxPZTB4KuzZkaZYnZPnvgIhb4pu07mx0=
|
||||
github.com/cpuguy83/go-md2man/v2 v2.0.6/go.mod h1:oOW0eioCTA6cOiMLiUPZOpcVxMig6NIQQ7OS05n1F4g=
|
||||
github.com/andybalholm/brotli v1.2.1 h1:R+f5xP285VArJDRgowrfb9DqL18yVK0gKAW/F+eTWro=
|
||||
github.com/andybalholm/brotli v1.2.1/go.mod h1:rzTDkvFWvIrjDXZHkuS16NPggd91W3kUSvPlQ1pLaKY=
|
||||
github.com/aws/aws-sdk-go-v2 v1.30.0 h1:6qAwtzlfcTtcL8NHtbDQAqgM5s6NDipQTkPxyH/6kAA=
|
||||
github.com/aws/aws-sdk-go-v2 v1.30.0/go.mod h1:ffIFB97e2yNsv4aTSGkqtHnppsIJzw7G7BReUZ3jCXM=
|
||||
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.2 h1:x6xsQXGSmW6frevwDA+vi/wqhp1ct18mVXYN08/93to=
|
||||
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.2/go.mod h1:lPprDr1e6cJdyYeGXnRaJoP4Md+cDBvi2eOj00BlGmg=
|
||||
github.com/aws/aws-sdk-go-v2/config v1.27.0 h1:J5sdGCAHuWKIXLeXiqr8II/adSvetkx0qdZwdbXXpb0=
|
||||
github.com/aws/aws-sdk-go-v2/config v1.27.0/go.mod h1:cfh8v69nuSUohNFMbIISP2fhmblGmYEOKs5V53HiHnk=
|
||||
github.com/aws/aws-sdk-go-v2/credentials v1.17.0 h1:lMW2x6sKBsiAJrpi1doOXqWFyEPoE886DTb1X0wb7So=
|
||||
github.com/aws/aws-sdk-go-v2/credentials v1.17.0/go.mod h1:uT41FIH8cCIxOdUYIL0PYyHlL1NoneDuDSCwg5VE/5o=
|
||||
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.15.0 h1:xWCwjjvVz2ojYTP4kBKUuUh9ZrXfcAXpflhOUUeXg1k=
|
||||
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.15.0/go.mod h1:j3fACuqXg4oMTQOR2yY7m0NmJY0yBK4L4sLsRXq1Ins=
|
||||
github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.8 h1:RnLB7p6aaFMRfyQkD6ckxR7myCC9SABIqSz4czYUUbU=
|
||||
github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.8/go.mod h1:XH7dQJd+56wEbP1I4e4Duo+QhSMxNArE8VP7NuUOTeM=
|
||||
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.8 h1:jzApk2f58L9yW9q1GEab3BMMFWUkkiZhyrRUtbwUbKU=
|
||||
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.8/go.mod h1:WqO+FftfO3tGePUtQxPXM6iODVfqMwsVMgTbG/ZXIdQ=
|
||||
github.com/aws/aws-sdk-go-v2/internal/ini v1.8.0 h1:hT8rVHwugYE2lEfdFE0QWVo81lF7jMrYJVDWI+f+VxU=
|
||||
github.com/aws/aws-sdk-go-v2/internal/ini v1.8.0/go.mod h1:8tu/lYfQfFe6IGnaOdrpVgEL2IrrDOf6/m9RQum4NkY=
|
||||
github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.8 h1:jH33S0y5Bo5ZVML62JgZhjd/LrtU+vbR8W7XnIE3Srk=
|
||||
github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.8/go.mod h1:hD5YwHLOy6k7d6kqcn3me1bFWHOtzhaXstMd6BpdB68=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.11.2 h1:Ji0DY1xUsUr3I8cHps0G+XM3WWU16lP6yG8qu1GAZAs=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.11.2/go.mod h1:5CsjAbs3NlGQyZNFACh+zztPDI7fU6eW9QsxjfnuBKg=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.3.10 h1:pkYC5zTOSPXEYJj56b2SOik9AL432i5MT1YVTQbKOK0=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.3.10/go.mod h1:/WNsBOlKWZCG3PMh2aSp8vkyyT/clpMZqOtrnIKqGfk=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.11.10 h1:7kZqP7akv0enu6ykJhb9OYlw16oOrSy+Epus8o/VqMY=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.11.10/go.mod h1:gYVF3nM1ApfTRDj9pvdhootBb8WbiIejuqn4w8ruMes=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.17.8 h1:iQNXVs1vtaq+y9M90M4ZIVNORje0qXTscqHLqoOnFS0=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.17.8/go.mod h1:yUQPRlWqGG0lfNsmjbRWKVwgilfBtZTOFSLEYALlAig=
|
||||
github.com/aws/aws-sdk-go-v2/service/s3 v1.55.0 h1:6kq0Xql9qiwNGL/Go87ZqR4otg9jnKs71OfWCVbPxLM=
|
||||
github.com/aws/aws-sdk-go-v2/service/s3 v1.55.0/go.mod h1:oSkRFuHVWmUY4Ssk16ErGzBqvYEbvORJFzFXzWhTB2s=
|
||||
github.com/aws/aws-sdk-go-v2/service/sso v1.19.0 h1:u6OkVDxtBPnxPkZ9/63ynEe+8kHbtS5IfaC4PzVxzWM=
|
||||
github.com/aws/aws-sdk-go-v2/service/sso v1.19.0/go.mod h1:YqbU3RS/pkDVu+v+Nwxvn0i1WB0HkNWEePWbmODEbbs=
|
||||
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.22.0 h1:6DL0qu5+315wbsAEEmzK+P9leRwNbkp+lGjPC+CEvb8=
|
||||
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.22.0/go.mod h1:olUAyg+FaoFaL/zFaeQQONjOZ9HXoxgvI/c7mQTYz7M=
|
||||
github.com/aws/aws-sdk-go-v2/service/sts v1.27.0 h1:cjTRjh700H36MQ8M0LnDn33W3JmwC77mdxIIyPWCdpM=
|
||||
github.com/aws/aws-sdk-go-v2/service/sts v1.27.0/go.mod h1:nXfOBMWPokIbOY+Gi7a1psWMSvskUCemZzI+SMB7Akc=
|
||||
github.com/aws/smithy-go v1.20.2 h1:tbp628ireGtzcHDDmLT/6ADHidqnwgF57XOXZe6tp4Q=
|
||||
github.com/aws/smithy-go v1.20.2/go.mod h1:krry+ya/rV9RDcV/Q16kpu6ypI4K2czasz0NC3qS14E=
|
||||
github.com/cpuguy83/go-md2man/v2 v2.0.3/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o=
|
||||
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
|
||||
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||
github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY=
|
||||
github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto=
|
||||
github.com/fxamacker/cbor/v2 v2.9.1 h1:2rWm8B193Ll4VdjsJY28jxs70IdDsHRWgQYAI80+rMQ=
|
||||
github.com/fxamacker/cbor/v2 v2.9.1/go.mod h1:vM4b+DJCtHn+zz7h3FFp/hDAI9WNWCsZj23V5ytsSxQ=
|
||||
github.com/gofiber/fiber/v3 v3.2.0 h1:g9+09D320foINPpCnR3ibQ5oBEFHjAWRRfDG1te54u8=
|
||||
github.com/gofiber/fiber/v3 v3.2.0/go.mod h1:FHOsc2Db7HhHpsE62QAaJlXVV1pNkbZEptZ4jtti7m4=
|
||||
github.com/gofiber/schema v1.7.1 h1:oSJBKdgP8JeIME4TQSAqlNKTU2iBB+2RNmKi8Nsc+TI=
|
||||
github.com/gofiber/schema v1.7.1/go.mod h1:A/X5Ffyru4p9eBdp99qu+nzviHzQiZ7odLT+TwxWhbk=
|
||||
github.com/gofiber/utils/v2 v2.0.4 h1:WwAxUA7L4MW2DjdEHF234lfqvBqd2vYYuBtA9TJq2ec=
|
||||
github.com/gofiber/utils/v2 v2.0.4/go.mod h1:GGERKU3Vhj5z6hS8YKvxL99A54DjOvTFZ0cjZnG4Lj4=
|
||||
github.com/google/go-cmp v0.5.8 h1:e6P7q2lk1O+qJJb4BtCQXlK8vWEO8V1ZeuEdJNOqZyg=
|
||||
github.com/google/go-cmp v0.5.8/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
|
||||
github.com/google/pprof v0.0.0-20250317173921-a4b03ec1a45e h1:ijClszYn+mADRFY17kjQEVQ1XRhq2/JR1M3sGqeJoxs=
|
||||
github.com/google/pprof v0.0.0-20250317173921-a4b03ec1a45e/go.mod h1:boTsfXsheKC2y+lKOCMpSfarhxDeIzfZG1jqGcPl3cA=
|
||||
github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
|
||||
github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
|
||||
github.com/hanzoai/csqlite v0.1.0 h1:suwC3dh0INlfP/U0Es6cDf6JNQ+2+GVLLATPWCUux6k=
|
||||
github.com/hanzoai/csqlite v0.1.0/go.mod h1:H31a/O6VXuklR9UBkgY++bmAK5uzVfXPqU0F6P9Wsos=
|
||||
github.com/hanzoai/sqlcipher v0.1.0 h1:V9gKG3ZltN2ZCteDrOnXWfOeEe/YDhhUm9AorQEAuBo=
|
||||
github.com/hanzoai/sqlcipher v0.1.0/go.mod h1:F0soUYM1i4sawOZUpRvVnWoUayPbeGVlGq01VXy9Aqg=
|
||||
github.com/hanzoai/sqlite v0.3.2 h1:B/TRunlIDZECEypmr6rHeNyWf37YZXvPJHBDEFMKn/g=
|
||||
github.com/hanzoai/sqlite v0.3.2/go.mod h1:a3llsefKbu2Iq/0rJ1mlWCaU2t2cXh+aze85x+oW72k=
|
||||
github.com/hashicorp/golang-lru/v2 v2.0.7 h1:a+bsQ5rvGLjzHuww6tVxozPZFVghXaHOwFs4luLUK2k=
|
||||
github.com/hashicorp/golang-lru/v2 v2.0.7/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyfM2/ZepoAG6RGpeM=
|
||||
github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8=
|
||||
github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw=
|
||||
github.com/jacobsa/fuse v0.0.0-20260302145937-f1ba38d60fdf h1:1FpPcJSf6jjJGvIltaLwJCpbFCMI9bVUCAAxUSxqWnY=
|
||||
github.com/jacobsa/fuse v0.0.0-20260302145937-f1ba38d60fdf/go.mod h1:fcpw1yk/suvFhB8rT9P+pst+NLboWsBLky9csooKjPc=
|
||||
github.com/klauspost/cpuid/v2 v2.3.0 h1:S4CRMLnYUhGeDFDqkGriYKdfoFlDnMtqTiI/sFzhA9Y=
|
||||
github.com/klauspost/cpuid/v2 v2.3.0/go.mod h1:hqwkgyIinND0mEev00jJYCxPNVRVXFQeu1XKlok6oO0=
|
||||
github.com/klauspost/compress v1.18.5 h1:/h1gH5Ce+VWNLSWqPzOVn6XBO+vJbCNGvjoaGBFW2IE=
|
||||
github.com/klauspost/compress v1.18.5/go.mod h1:cwPg85FWrGar70rWktvGQj8/hthj3wpl0PGDogxkrSQ=
|
||||
github.com/klauspost/cpuid/v2 v2.0.12 h1:p9dKCg8i4gmOxtv35DvrYoWqYzQrvEVdjQ762Y0OqZE=
|
||||
github.com/klauspost/cpuid/v2 v2.0.12/go.mod h1:g2LTdtYhdyuGPqyWyv7qRAmj1WBqxuObKfj5c0PQa7c=
|
||||
github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc=
|
||||
github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw=
|
||||
github.com/luxfi/age v1.6.0 h1:KMD8gSOP4NVCb7NWSlRcgBZNV2xm2a+qQWPyPmiX6f4=
|
||||
github.com/luxfi/age v1.6.0/go.mod h1:7cu9CIyikgyAvr5MlXFapEDQ15yBaHOSdKkK5lG04WE=
|
||||
github.com/luxfi/age v1.5.0 h1:G69HbSV4R3vKEH9B0CulnRaMdSdf4RalMgP8xKmxHeI=
|
||||
github.com/luxfi/age v1.5.0/go.mod h1:iAYAxgvrXxcy746+Ovh/eWWDuF9teJLNcCSSOX9RYW0=
|
||||
github.com/luxfi/log v1.4.3 h1:xkUKRWvQ4ZwvlUC2e0/RTtHYZOYSMvSQ9W9lbjwBmiI=
|
||||
github.com/luxfi/log v1.4.3/go.mod h1:myIkufyiQomSQH34K981kbz6cG4WUoerRUh7F4XhlQI=
|
||||
github.com/mattn/go-colorable v0.1.14 h1:9A9LHSqF/7dyVVX6g0U9cwm9pG3kP9gSzcuIPHPsaIE=
|
||||
github.com/mattn/go-colorable v0.1.14/go.mod h1:6LmQG8QLFO4G5z1gPvYEzlUgJ2wF+stgPZH1UqBm1s8=
|
||||
github.com/mattn/go-isatty v0.0.21 h1:xYae+lCNBP7QuW4PUnNG61ffM4hVIfm+zUzDuSzYLGs=
|
||||
github.com/mattn/go-isatty v0.0.21/go.mod h1:ZXfXG4SQHsB/w3ZeOYbR0PrPwLy+n6xiMrJlRFqopa4=
|
||||
github.com/ncruces/go-strftime v1.0.0 h1:HMFp8mLCTPp341M/ZnA4qaf7ZlsbTc+miZjCLOFAw7w=
|
||||
github.com/ncruces/go-strftime v1.0.0/go.mod h1:Fwc5htZGVVkseilnfgOVb9mKy6w1naJmn9CehxcKcls=
|
||||
github.com/philhofer/fwd v1.2.0 h1:e6DnBTl7vGY+Gz322/ASL4Gyp1FspeMvx1RNDoToZuM=
|
||||
github.com/philhofer/fwd v1.2.0/go.mod h1:RqIHx9QI14HlwKwm98g9Re5prTQ6LdeRQn+gXJFxsJM=
|
||||
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
|
||||
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
|
||||
github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94icq4NjY3clb7Lk8O1qJ8BdBEF8z0ibU0rE=
|
||||
github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo=
|
||||
github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
|
||||
github.com/spf13/cobra v1.10.2 h1:DMTTonx5m65Ic0GOoRY2c16WCbHxOOw6xxezuLaBpcU=
|
||||
github.com/spf13/cobra v1.10.2/go.mod h1:7C1pvHqHw5A4vrJfjNwvOdzYu0Gml16OCs2GRiTUUS4=
|
||||
github.com/spf13/pflag v1.0.9/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
|
||||
github.com/spf13/pflag v1.0.10 h1:4EBh2KAYBwaONj6b2Ye1GiHfwjqyROoF4RwYO+vPwFk=
|
||||
github.com/spf13/pflag v1.0.10/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
|
||||
github.com/shamaton/msgpack/v3 v3.1.0 h1:jsk0vEAqVvvS9+fTZ5/EcQ9tz860c9pWxJ4Iwecz8gU=
|
||||
github.com/shamaton/msgpack/v3 v3.1.0/go.mod h1:DcQG8jrdrQCIxr3HlMYkiXdMhK+KfN2CitkyzsQV4uc=
|
||||
github.com/spf13/cobra v1.8.0 h1:7aJaZx1B85qltLMc546zn58BxxfZdR/W22ej9CFoEf0=
|
||||
github.com/spf13/cobra v1.8.0/go.mod h1:WXLWApfZ71AjXPya3WOlMsY9yMs7YeiHhFVlvLyhcho=
|
||||
github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA=
|
||||
github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
|
||||
github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U=
|
||||
github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U=
|
||||
github.com/tinylib/msgp v1.6.4 h1:mOwYbyYDLPj35mkA2BjjYejgJk9BuHxDdvRnb6v2ZcQ=
|
||||
github.com/tinylib/msgp v1.6.4/go.mod h1:RSp0LW9oSxFut3KzESt5Voq4GVWyS+PSulT77roAqEA=
|
||||
github.com/tv42/httpunix v0.0.0-20191220191345-2ba4b9c3382c h1:u6SKchux2yDvFQnDHS3lPnIRmfVJ5Sxy3ao2SIdysLQ=
|
||||
github.com/tv42/httpunix v0.0.0-20191220191345-2ba4b9c3382c/go.mod h1:hzIxponao9Kjc7aWznkXaL4U4TWaDSs8zcsY4Ka08nM=
|
||||
github.com/zeebo/assert v1.3.0 h1:g7C04CbJuIDKNPFHmsk4hwZDO5O+kntRxzaUoNXj+IQ=
|
||||
github.com/zeebo/assert v1.3.0/go.mod h1:Pq9JiuJQpG8JLJdtkwrJESF0Foym2/D9XMU5ciN/wJ0=
|
||||
github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw=
|
||||
github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc=
|
||||
github.com/valyala/fasthttp v1.70.0 h1:LAhMGcWk13QZWm85+eg8ZBNbrq5mnkWFGbHMUJHIdXA=
|
||||
github.com/valyala/fasthttp v1.70.0/go.mod h1:oDZEHHkJ/Buyklg6uURmYs19442zFSnCIfX3j1FY3pE=
|
||||
github.com/x448/float16 v0.8.4 h1:qLwI1I70+NjRFUR3zs1JPUCgaCXSh3SW62uAKT1mSBM=
|
||||
github.com/x448/float16 v0.8.4/go.mod h1:14CWIYCyZA/cWjXOioeEpHeN/83MdbZDRQHoFcYsOfg=
|
||||
github.com/xyproto/randomstring v1.0.5 h1:YtlWPoRdgMu3NZtP45drfy1GKoojuR7hmRcnhZqKjWU=
|
||||
github.com/xyproto/randomstring v1.0.5/go.mod h1:rgmS5DeNXLivK7YprL0pY+lTuhNQW3iGxZ18UQApw/E=
|
||||
github.com/zeebo/assert v1.1.0 h1:hU1L1vLTHsnO8x8c9KAR5GmM5QscxHg5RNU5z5qbUWY=
|
||||
github.com/zeebo/assert v1.1.0/go.mod h1:Pq9JiuJQpG8JLJdtkwrJESF0Foym2/D9XMU5ciN/wJ0=
|
||||
github.com/zeebo/blake3 v0.2.4 h1:KYQPkhpRtcqh0ssGYcKLG1JYvddkEA8QwCM/yBqhaZI=
|
||||
github.com/zeebo/blake3 v0.2.4/go.mod h1:7eeQ6d2iXWRGF6npfaxl2CU+xy2Fjo2gxeyZGCRUjcE=
|
||||
github.com/zeebo/pcg v1.0.1 h1:lyqfGeWiv4ahac6ttHs+I5hwtH/+1mrhlCtVNQM2kHo=
|
||||
github.com/zeebo/pcg v1.0.1/go.mod h1:09F0S9iiKrwn9rlI5yjLkmrug154/YRW6KnnXVDM/l4=
|
||||
go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg=
|
||||
golang.org/x/crypto v0.52.0 h1:RMs7fP2rXdep0CftQlK8Uf+kibLm7qkCcradZWYz988=
|
||||
golang.org/x/crypto v0.52.0/go.mod h1:1QgfPxDqh0T2M/elOJtp9RvuR95kVjir0e6/BvEmGbc=
|
||||
golang.org/x/sys v0.46.0 h1:noSf2Fq6F8DBgS+LysIkx7rIExoNHJsxOAtPp4rthXw=
|
||||
golang.org/x/sys v0.46.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw=
|
||||
golang.org/x/crypto v0.50.0 h1:zO47/JPrL6vsNkINmLoo/PH1gcxpls50DNogFvB5ZGI=
|
||||
golang.org/x/crypto v0.50.0/go.mod h1:3muZ7vA7PBCE6xgPX7nkzzjiUq87kRItoJQM1Yo8S+Q=
|
||||
golang.org/x/mod v0.34.0 h1:xIHgNUUnW6sYkcM5Jleh05DvLOtwc6RitGHbDk4akRI=
|
||||
golang.org/x/mod v0.34.0/go.mod h1:ykgH52iCZe79kzLLMhyCUzhMci+nQj+0XkbXpNYtVjY=
|
||||
golang.org/x/net v0.53.0 h1:d+qAbo5L0orcWAr0a9JweQpjXF19LMXJE8Ey7hwOdUA=
|
||||
golang.org/x/net v0.53.0/go.mod h1:JvMuJH7rrdiCfbeHoo3fCQU24Lf5JJwT9W3sJFulfgs=
|
||||
golang.org/x/sync v0.20.0 h1:e0PTpb7pjO8GAtTs2dQ6jYa5BWYlMuX047Dco/pItO4=
|
||||
golang.org/x/sync v0.20.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0=
|
||||
golang.org/x/sys v0.43.0 h1:Rlag2XtaFTxp19wS8MXlJwTvoh8ArU6ezoyFsMyCTNI=
|
||||
golang.org/x/sys v0.43.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw=
|
||||
golang.org/x/text v0.36.0 h1:JfKh3XmcRPqZPKevfXVpI1wXPTqbkE5f7JA92a55Yxg=
|
||||
golang.org/x/text v0.36.0/go.mod h1:NIdBknypM8iqVmPiuco0Dh6P5Jcdk8lJL0CUebqK164=
|
||||
golang.org/x/tools v0.43.0 h1:12BdW9CeB3Z+J/I/wj34VMl8X+fEXBxVR90JeMX5E7s=
|
||||
golang.org/x/tools v0.43.0/go.mod h1:uHkMso649BX2cZK6+RpuIPXS3ho2hZo4FVwfoy1vIk0=
|
||||
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
|
||||
gopkg.in/natefinch/lumberjack.v2 v2.2.1 h1:bBRl1b0OH9s/DuPhuXpNl+VtCaJXFZ5/uEFST95x9zc=
|
||||
gopkg.in/natefinch/lumberjack.v2 v2.2.1/go.mod h1:YD8tP3GAjkrDg1eZH7EGmyESg/lsYskCTPBJVb9jqSc=
|
||||
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
|
||||
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
|
||||
modernc.org/cc/v4 v4.27.3 h1:uNCgn37E5U09mTv1XgskEVUJ8ADKpmFMPxzGJ0TSo+U=
|
||||
modernc.org/cc/v4 v4.27.3/go.mod h1:3YjcbCqhoTTHPycJDRl2WZKKFj0nwcOIPBfEZK0Hdk8=
|
||||
modernc.org/ccgo/v4 v4.32.4 h1:L5OB8rpEX4ZsXEQwGozRfJyJSFHbbNVOoQ59DU9/KuU=
|
||||
modernc.org/ccgo/v4 v4.32.4/go.mod h1:lY7f+fiTDHfcv6YlRgSkxYfhs+UvOEEzj49jAn2TOx0=
|
||||
modernc.org/fileutil v1.4.0 h1:j6ZzNTftVS054gi281TyLjHPp6CPHr2KCxEXjEbD6SM=
|
||||
modernc.org/fileutil v1.4.0/go.mod h1:EqdKFDxiByqxLk8ozOxObDSfcVOv/54xDs/DUHdvCUU=
|
||||
modernc.org/gc/v2 v2.6.5 h1:nyqdV8q46KvTpZlsw66kWqwXRHdjIlJOhG6kxiV/9xI=
|
||||
modernc.org/gc/v2 v2.6.5/go.mod h1:YgIahr1ypgfe7chRuJi2gD7DBQiKSLMPgBQe9oIiito=
|
||||
modernc.org/gc/v3 v3.1.2 h1:ZtDCnhonXSZexk/AYsegNRV1lJGgaNZJuKjJSWKyEqo=
|
||||
modernc.org/gc/v3 v3.1.2/go.mod h1:HFK/6AGESC7Ex+EZJhJ2Gni6cTaYpSMmU/cT9RmlfYY=
|
||||
modernc.org/goabi0 v0.2.0 h1:HvEowk7LxcPd0eq6mVOAEMai46V+i7Jrj13t4AzuNks=
|
||||
modernc.org/goabi0 v0.2.0/go.mod h1:CEFRnnJhKvWT1c1JTI3Avm+tgOWbkOu5oPA8eH8LnMI=
|
||||
modernc.org/libc v1.72.0 h1:IEu559v9a0XWjw0DPoVKtXpO2qt5NVLAnFaBbjq+n8c=
|
||||
modernc.org/libc v1.72.0/go.mod h1:tTU8DL8A+XLVkEY3x5E/tO7s2Q/q42EtnNWda/L5QhQ=
|
||||
modernc.org/mathutil v1.7.1 h1:GCZVGXdaN8gTqB1Mf/usp1Y/hSqgI2vAGGP4jZMCxOU=
|
||||
modernc.org/mathutil v1.7.1/go.mod h1:4p5IwJITfppl0G4sUEDtCr4DthTaT47/N3aT6MhfgJg=
|
||||
modernc.org/memory v1.11.0 h1:o4QC8aMQzmcwCK3t3Ux/ZHmwFPzE6hf2Y5LbkRs+hbI=
|
||||
modernc.org/memory v1.11.0/go.mod h1:/JP4VbVC+K5sU2wZi9bHoq2MAkCnrt2r98UGeSK7Mjw=
|
||||
modernc.org/opt v0.1.4 h1:2kNGMRiUjrp4LcaPuLY2PzUfqM/w9N23quVwhKt5Qm8=
|
||||
modernc.org/opt v0.1.4/go.mod h1:03fq9lsNfvkYSfxrfUhZCWPk1lm4cq4N+Bh//bEtgns=
|
||||
modernc.org/sortutil v1.2.1 h1:+xyoGf15mM3NMlPDnFqrteY07klSFxLElE2PVuWIJ7w=
|
||||
modernc.org/sortutil v1.2.1/go.mod h1:7ZI3a3REbai7gzCLcotuw9AC4VZVpYMjDzETGsSMqJE=
|
||||
modernc.org/sqlite v1.50.0 h1:eMowQSWLK0MeiQTdmz3lqoF5dqclujdlIKeJA11+7oM=
|
||||
modernc.org/sqlite v1.50.0/go.mod h1:m0w8xhwYUVY3H6pSDwc3gkJ/irZT/0YEXwBlhaxQEew=
|
||||
modernc.org/strutil v1.2.1 h1:UneZBkQA+DX2Rp35KcM69cSsNES9ly8mQWD71HKlOA0=
|
||||
modernc.org/strutil v1.2.1/go.mod h1:EHkiggD70koQxjVdSBM3JKM7k6L0FbGE5eymy9i3B9A=
|
||||
modernc.org/token v1.1.0 h1:Xl7Ap9dKaEs5kLoOQeQmPWevfnk/DM5qcLcYlA8ys6Y=
|
||||
modernc.org/token v1.1.0/go.mod h1:UGzOrNV1mAFSEB63lOFHIpNRUVMvYTc6yu1SMY/XTDM=
|
||||
|
||||
-51
@@ -1,51 +0,0 @@
|
||||
// Instance lifecycle + version for the VFS package. Cloud-free: the
|
||||
// HIP-0106 mount adapter (which wires these into a zip.App) lives in
|
||||
// cloud/mounts/vfs, but the singleton instance and its accessors are a
|
||||
// vfs-library concern and stay here so the standalone daemon, tests,
|
||||
// and any embedder can attach/read a backend without importing cloud.
|
||||
|
||||
package vfs
|
||||
|
||||
import (
|
||||
"context"
|
||||
"sync"
|
||||
)
|
||||
|
||||
// Version is overridden at build time via -ldflags
|
||||
// "-X github.com/hanzoai/vfs.Version=...".
|
||||
var Version = "dev"
|
||||
|
||||
// mountedInstance is the VFS handle the HTTP/data-plane routes operate
|
||||
// against. nil until SetInstance() is called. Guarded by mu.
|
||||
var (
|
||||
mu sync.RWMutex
|
||||
mountedInstance *VFS
|
||||
)
|
||||
|
||||
// SetInstance attaches a VFS handle for the mounted routes. Calling this
|
||||
// with nil disables the data-plane routes. Idempotent.
|
||||
func SetInstance(v *VFS) {
|
||||
mu.Lock()
|
||||
defer mu.Unlock()
|
||||
mountedInstance = v
|
||||
}
|
||||
|
||||
// Instance returns the currently attached VFS handle (or nil).
|
||||
func Instance() *VFS {
|
||||
mu.RLock()
|
||||
defer mu.RUnlock()
|
||||
return mountedInstance
|
||||
}
|
||||
|
||||
// Shutdown drains the attached VFS instance (closing its backend).
|
||||
// Idempotent. Safe to call when no instance was attached.
|
||||
func Shutdown(_ context.Context) error {
|
||||
mu.Lock()
|
||||
defer mu.Unlock()
|
||||
if mountedInstance == nil {
|
||||
return nil
|
||||
}
|
||||
err := mountedInstance.Close()
|
||||
mountedInstance = nil
|
||||
return err
|
||||
}
|
||||
@@ -0,0 +1,219 @@
|
||||
// HIP-0106 Mount() entry point. Lets cmd/cloud import this package and
|
||||
// register VFS with the shared zip.App alongside every other Hanzo
|
||||
// subsystem (iam, base, kms, amqp, …).
|
||||
//
|
||||
// Wire shape:
|
||||
//
|
||||
// import _ "github.com/hanzoai/vfs" // init() registers
|
||||
//
|
||||
// VFS has no public HTTP surface today — it's an S3-backed virtual
|
||||
// block filesystem driven through FUSE mounts and CLI/sidecar usage.
|
||||
// Mount() exposes liveness/readiness probes plus a minimal HTTP
|
||||
// CRUD surface (PUT/GET/DELETE single block by ID) so other in-process
|
||||
// subsystems can talk to it without falling back to the CLI.
|
||||
//
|
||||
// The VFS instance itself is opt-in: callers either inject one via
|
||||
// SetInstance(v) (used by the standalone shim and tests) or skip it
|
||||
// (the cloud binary mounts VFS as a passthrough until a backend +
|
||||
// crypto config lands in cloud.Config). Readiness reports "not_ready"
|
||||
// until an instance is attached.
|
||||
package vfs
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/hex"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"sync"
|
||||
|
||||
"github.com/hanzoai/cloud"
|
||||
"github.com/hanzoai/vfs/pkg/backend"
|
||||
"github.com/hanzoai/zip"
|
||||
)
|
||||
|
||||
// Version is overridden at build time via -ldflags
|
||||
// "-X github.com/hanzoai/vfs.Version=...".
|
||||
var Version = "dev"
|
||||
|
||||
// mountedInstance is the VFS handle the HTTP routes operate against.
|
||||
// nil until SetInstance() is called. Guarded by mu.
|
||||
var (
|
||||
mu sync.RWMutex
|
||||
mountedInstance *VFS
|
||||
)
|
||||
|
||||
// SetInstance attaches a VFS handle for the mounted HTTP routes.
|
||||
// Calling this with nil disables the data-plane routes (Get/Put/Delete
|
||||
// return 503). Idempotent.
|
||||
//
|
||||
// The standalone shim wires this from CLI flags. cloud.Config has no
|
||||
// VFS-backend block yet — cloud binary deploys leave it nil until
|
||||
// HIP-0107 config lands.
|
||||
func SetInstance(v *VFS) {
|
||||
mu.Lock()
|
||||
defer mu.Unlock()
|
||||
mountedInstance = v
|
||||
}
|
||||
|
||||
// Instance returns the currently attached VFS handle (or nil).
|
||||
func Instance() *VFS {
|
||||
mu.RLock()
|
||||
defer mu.RUnlock()
|
||||
return mountedInstance
|
||||
}
|
||||
|
||||
// Mount registers VFS routes with the shared cloud zip.App per HIP-0106.
|
||||
//
|
||||
// Routes:
|
||||
//
|
||||
// GET /v1/vfs/health — liveness, never blocks
|
||||
// GET /v1/vfs/readyz — readiness, 200 only when instance attached
|
||||
// PUT /v1/vfs/blocks — body = raw block bytes; returns {"id": "..."}
|
||||
// GET /v1/vfs/blocks/:id — returns raw decrypted block bytes
|
||||
// DELETE /v1/vfs/blocks/:id — removes a block from the backend
|
||||
// GET /v1/vfs/stats — backend/cache counters
|
||||
//
|
||||
// Identity headers (X-Org-Id, X-User-Id) are honoured but VFS is a
|
||||
// single-tenant block layer today — multitenant prefixing lands when
|
||||
// HIP-0107 specifies per-org namespacing.
|
||||
func Mount(app *zip.App, deps cloud.Deps) error {
|
||||
logger := deps.Logger.New("subsystem", "vfs")
|
||||
|
||||
app.Get("/v1/vfs/health", func(c *zip.Ctx) error {
|
||||
return c.JSON(http.StatusOK, map[string]any{
|
||||
"status": "ok",
|
||||
"service": "vfs",
|
||||
"version": Version,
|
||||
})
|
||||
})
|
||||
|
||||
app.Get("/v1/vfs/readyz", func(c *zip.Ctx) error {
|
||||
if Instance() == nil {
|
||||
return c.JSON(http.StatusServiceUnavailable, map[string]any{
|
||||
"status": "not_ready",
|
||||
"service": "vfs",
|
||||
"reason": "no backend attached (call vfs.SetInstance)",
|
||||
})
|
||||
}
|
||||
return c.JSON(http.StatusOK, map[string]any{
|
||||
"status": "ready",
|
||||
"service": "vfs",
|
||||
"version": Version,
|
||||
})
|
||||
})
|
||||
|
||||
app.Get("/v1/vfs/stats", func(c *zip.Ctx) error {
|
||||
v := Instance()
|
||||
if v == nil {
|
||||
return c.JSON(http.StatusServiceUnavailable, map[string]any{
|
||||
"error": "vfs not attached",
|
||||
})
|
||||
}
|
||||
return c.JSON(http.StatusOK, v.Stats())
|
||||
})
|
||||
|
||||
app.Put("/v1/vfs/blocks", func(c *zip.Ctx) error {
|
||||
v := Instance()
|
||||
if v == nil {
|
||||
return c.JSON(http.StatusServiceUnavailable, map[string]any{"error": "vfs not attached"})
|
||||
}
|
||||
body := c.Body()
|
||||
if len(body) == 0 {
|
||||
return c.JSON(http.StatusBadRequest, map[string]any{"error": "empty body"})
|
||||
}
|
||||
if len(body) > BlockSize {
|
||||
return c.JSON(http.StatusBadRequest, map[string]any{
|
||||
"error": fmt.Sprintf("payload %d > BlockSize %d", len(body), BlockSize),
|
||||
"blockSize": BlockSize,
|
||||
})
|
||||
}
|
||||
id, err := v.PutBlock(c.Context(), body)
|
||||
if err != nil {
|
||||
return c.JSON(http.StatusInternalServerError, map[string]any{"error": err.Error()})
|
||||
}
|
||||
return c.JSON(http.StatusCreated, map[string]any{"id": string(id)})
|
||||
})
|
||||
|
||||
app.Get("/v1/vfs/blocks/:id", func(c *zip.Ctx) error {
|
||||
v := Instance()
|
||||
if v == nil {
|
||||
return c.JSON(http.StatusServiceUnavailable, map[string]any{"error": "vfs not attached"})
|
||||
}
|
||||
raw := c.Param("id")
|
||||
if !isHexBlockID(raw) {
|
||||
return c.JSON(http.StatusBadRequest, map[string]any{"error": "block id must be lowercase hex"})
|
||||
}
|
||||
pt, err := v.GetBlock(c.Context(), BlockID(raw))
|
||||
if err != nil {
|
||||
if errors.Is(err, backend.ErrNotFound) {
|
||||
return c.JSON(http.StatusNotFound, map[string]any{"error": "not found"})
|
||||
}
|
||||
return c.JSON(http.StatusInternalServerError, map[string]any{"error": err.Error()})
|
||||
}
|
||||
c.SetHeader("Content-Type", "application/octet-stream")
|
||||
return c.Bytes(http.StatusOK, pt)
|
||||
})
|
||||
|
||||
app.Delete("/v1/vfs/blocks/:id", func(c *zip.Ctx) error {
|
||||
v := Instance()
|
||||
if v == nil {
|
||||
return c.JSON(http.StatusServiceUnavailable, map[string]any{"error": "vfs not attached"})
|
||||
}
|
||||
raw := c.Param("id")
|
||||
if !isHexBlockID(raw) {
|
||||
return c.JSON(http.StatusBadRequest, map[string]any{"error": "block id must be lowercase hex"})
|
||||
}
|
||||
if err := v.Delete(c.Context(), BlockID(raw)); err != nil {
|
||||
if errors.Is(err, backend.ErrNotFound) {
|
||||
return c.JSON(http.StatusNotFound, map[string]any{"error": "not found"})
|
||||
}
|
||||
return c.JSON(http.StatusInternalServerError, map[string]any{"error": err.Error()})
|
||||
}
|
||||
return c.JSON(http.StatusOK, map[string]any{"ok": true})
|
||||
})
|
||||
|
||||
logger.Info("vfs mounted", "version", Version, "instance_attached", Instance() != nil)
|
||||
return nil
|
||||
}
|
||||
|
||||
// Shutdown drains the attached VFS instance (closing its backend).
|
||||
// Idempotent. Safe to call when no instance was attached.
|
||||
func Shutdown(_ context.Context) error {
|
||||
mu.Lock()
|
||||
defer mu.Unlock()
|
||||
if mountedInstance == nil {
|
||||
return nil
|
||||
}
|
||||
err := mountedInstance.Close()
|
||||
mountedInstance = nil
|
||||
return err
|
||||
}
|
||||
|
||||
// isHexBlockID matches the lowercase hex string output by HashBlock.
|
||||
// Used to reject path traversal / control bytes before hitting the
|
||||
// backend.
|
||||
func isHexBlockID(s string) bool {
|
||||
if len(s) == 0 || len(s) > 128 {
|
||||
return false
|
||||
}
|
||||
_, err := hex.DecodeString(s)
|
||||
return err == nil
|
||||
}
|
||||
|
||||
// ensure io.Copy stays imported (used by the data-plane streaming
|
||||
// branch when multi-block File support lands in 0.4.0).
|
||||
var _ = io.Copy
|
||||
|
||||
// init registers VFS with the cloud subsystem registry. Order 20 —
|
||||
// after kms (10) and ahead of amqp (30) / mq (40).
|
||||
func init() {
|
||||
cloud.Register("vfs", 20, func(app any, deps cloud.Deps) error {
|
||||
a, ok := app.(*zip.App)
|
||||
if !ok {
|
||||
return fmt.Errorf("vfs.Mount: app is %T, want *zip.App", app)
|
||||
}
|
||||
return Mount(a, deps)
|
||||
})
|
||||
}
|
||||
+138
@@ -0,0 +1,138 @@
|
||||
package vfs_test
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"io"
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/luxfi/age"
|
||||
luxlog "github.com/luxfi/log"
|
||||
|
||||
"github.com/hanzoai/cloud"
|
||||
"github.com/hanzoai/vfs/pkg/backend"
|
||||
_ "github.com/hanzoai/vfs/pkg/backend/file"
|
||||
"github.com/hanzoai/vfs"
|
||||
"github.com/hanzoai/zip"
|
||||
)
|
||||
|
||||
// newMountTestVFS creates a file-backed VFS suitable for HTTP testing.
|
||||
func newMountTestVFS(t *testing.T) *vfs.VFS {
|
||||
t.Helper()
|
||||
be, err := backend.Open(context.Background(), "file://"+t.TempDir())
|
||||
if err != nil {
|
||||
t.Fatalf("backend.Open: %v", err)
|
||||
}
|
||||
t.Cleanup(func() { _ = be.Close() })
|
||||
|
||||
id, err := age.GenerateX25519Identity()
|
||||
if err != nil {
|
||||
t.Fatalf("age.GenerateX25519Identity: %v", err)
|
||||
}
|
||||
c, err := vfs.NewCrypto([]age.Recipient{id.Recipient()}, []age.Identity{id})
|
||||
if err != nil {
|
||||
t.Fatalf("NewCrypto: %v", err)
|
||||
}
|
||||
v, err := vfs.New(vfs.Config{Backend: be, Crypto: c})
|
||||
if err != nil {
|
||||
t.Fatalf("vfs.New: %v", err)
|
||||
}
|
||||
return v
|
||||
}
|
||||
|
||||
func TestMount_HealthReadyz(t *testing.T) {
|
||||
app := zip.New(zip.Config{Logger: luxlog.New("test")})
|
||||
deps := cloud.Deps{Logger: luxlog.New("test")}
|
||||
|
||||
// Detach any prior instance (other tests may have set one).
|
||||
vfs.SetInstance(nil)
|
||||
t.Cleanup(func() { vfs.SetInstance(nil) })
|
||||
|
||||
if err := vfs.Mount(app, deps); err != nil {
|
||||
t.Fatalf("Mount: %v", err)
|
||||
}
|
||||
|
||||
// Health always 200, even without an instance.
|
||||
req := httptest.NewRequest("GET", "/v1/vfs/health", nil)
|
||||
resp, err := app.Fiber().Test(req)
|
||||
if err != nil {
|
||||
t.Fatalf("health test: %v", err)
|
||||
}
|
||||
if resp.StatusCode != 200 {
|
||||
t.Fatalf("health status = %d, want 200", resp.StatusCode)
|
||||
}
|
||||
body, _ := io.ReadAll(resp.Body)
|
||||
var hb map[string]any
|
||||
_ = json.Unmarshal(body, &hb)
|
||||
if hb["service"] != "vfs" {
|
||||
t.Fatalf("health body service = %v, want vfs", hb["service"])
|
||||
}
|
||||
|
||||
// Readyz without instance → 503.
|
||||
req = httptest.NewRequest("GET", "/v1/vfs/readyz", nil)
|
||||
resp, err = app.Fiber().Test(req)
|
||||
if err != nil {
|
||||
t.Fatalf("readyz test: %v", err)
|
||||
}
|
||||
if resp.StatusCode != 503 {
|
||||
t.Fatalf("readyz (no instance) status = %d, want 503", resp.StatusCode)
|
||||
}
|
||||
|
||||
// Attach instance → 200.
|
||||
vfs.SetInstance(newMountTestVFS(t))
|
||||
req = httptest.NewRequest("GET", "/v1/vfs/readyz", nil)
|
||||
resp, err = app.Fiber().Test(req)
|
||||
if err != nil {
|
||||
t.Fatalf("readyz attached test: %v", err)
|
||||
}
|
||||
if resp.StatusCode != 200 {
|
||||
t.Fatalf("readyz (attached) status = %d, want 200", resp.StatusCode)
|
||||
}
|
||||
}
|
||||
|
||||
func TestMount_PutGetRoundtrip(t *testing.T) {
|
||||
app := zip.New(zip.Config{Logger: luxlog.New("test")})
|
||||
deps := cloud.Deps{Logger: luxlog.New("test")}
|
||||
vfs.SetInstance(newMountTestVFS(t))
|
||||
t.Cleanup(func() { vfs.SetInstance(nil) })
|
||||
|
||||
if err := vfs.Mount(app, deps); err != nil {
|
||||
t.Fatalf("Mount: %v", err)
|
||||
}
|
||||
|
||||
plain := []byte("zip mount roundtrip — small block payload")
|
||||
|
||||
// PUT.
|
||||
put := httptest.NewRequest("PUT", "/v1/vfs/blocks", bytes.NewReader(plain))
|
||||
putResp, err := app.Fiber().Test(put)
|
||||
if err != nil {
|
||||
t.Fatalf("PUT: %v", err)
|
||||
}
|
||||
if putResp.StatusCode != 201 {
|
||||
t.Fatalf("PUT status = %d, want 201", putResp.StatusCode)
|
||||
}
|
||||
var putBody struct {
|
||||
ID string `json:"id"`
|
||||
}
|
||||
_ = json.NewDecoder(putResp.Body).Decode(&putBody)
|
||||
if putBody.ID == "" || len(putBody.ID) != 64 {
|
||||
t.Fatalf("PUT body id = %q (want 64-hex)", putBody.ID)
|
||||
}
|
||||
|
||||
// GET.
|
||||
get := httptest.NewRequest("GET", "/v1/vfs/blocks/"+putBody.ID, nil)
|
||||
getResp, err := app.Fiber().Test(get)
|
||||
if err != nil {
|
||||
t.Fatalf("GET: %v", err)
|
||||
}
|
||||
if getResp.StatusCode != 200 {
|
||||
t.Fatalf("GET status = %d, want 200", getResp.StatusCode)
|
||||
}
|
||||
gotRaw, _ := io.ReadAll(getResp.Body)
|
||||
if !strings.HasPrefix(string(gotRaw), string(plain)) {
|
||||
t.Fatalf("GET body prefix mismatch:\n got=%q\nwant prefix=%q", gotRaw[:64], plain)
|
||||
}
|
||||
}
|
||||
@@ -1,447 +0,0 @@
|
||||
// Package cost computes operational cost for VFS workloads across the
|
||||
// pluggable backends VFS supports. Numbers are list prices as of 2026-06;
|
||||
// override in deployment-specific configs.
|
||||
//
|
||||
// The model splits a workload into three contributions:
|
||||
//
|
||||
// storage = GB-months held in the bucket
|
||||
// requests = PUT/GET/DELETE/LIST counts
|
||||
// egress = GB read out of the cloud's edge to consumers
|
||||
//
|
||||
// Egress is the wild card for any read-heavy workload — if consumers
|
||||
// live in the same cloud region as the backend, intra-region traffic
|
||||
// is normally free. Cross-cloud fanout (e.g. workload in DOKS reading
|
||||
// from AWS S3) is what gets expensive. Pick a backend with the egress
|
||||
// origin/destination in mind.
|
||||
//
|
||||
// The package is intentionally workload-neutral. Application-specific
|
||||
// projections (a validator archive, a sqlite query DB, an indexer
|
||||
// snapshot pipeline) compose this package's Workload struct themselves.
|
||||
package cost
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
// Backend names the supported object stores. Add new ones by extending
|
||||
// the Catalog below.
|
||||
type Backend string
|
||||
|
||||
const (
|
||||
BackendAWSS3Std Backend = "aws-s3-standard"
|
||||
BackendAWSS3IA Backend = "aws-s3-ia"
|
||||
BackendAWSGlacierIR Backend = "aws-s3-glacier-instant"
|
||||
BackendCloudflareR2 Backend = "cloudflare-r2"
|
||||
BackendDOSpaces Backend = "do-spaces"
|
||||
BackendGCSStandard Backend = "gcs-standard"
|
||||
BackendGCSColdline Backend = "gcs-coldline"
|
||||
// BackendPVCBlock is the "no VFS, just a Kubernetes Block PVC"
|
||||
// baseline. Useful when comparing whether moving to VFS+object
|
||||
// store actually saves money for a given workload.
|
||||
BackendPVCBlock Backend = "k8s-pvc-block"
|
||||
)
|
||||
|
||||
// Pricing is the per-unit list-price slice for a backend. Numbers are
|
||||
// USD list prices as documented inline; document any drift in this
|
||||
// file's CHANGELOG stanza.
|
||||
type Pricing struct {
|
||||
// StoragePerGBMonth is the monthly $/GB held.
|
||||
StoragePerGBMonth float64
|
||||
|
||||
// PutPer1k is the cost per 1000 PUT requests.
|
||||
PutPer1k float64
|
||||
// GetPer1k is the cost per 1000 GET requests.
|
||||
GetPer1k float64
|
||||
|
||||
// EgressPerGB is the cost per GB read OUT of the provider's edge.
|
||||
// Intra-region same-cloud reads are normally free; this rate
|
||||
// applies to cross-region or cross-cloud egress.
|
||||
EgressPerGB float64
|
||||
|
||||
// MinCommitGBMonth is any minimum storage commit baked into the
|
||||
// SKU (e.g. Glacier Deep Archive's 180-day minimum).
|
||||
MinCommitGBMonth float64
|
||||
|
||||
// Notes captures gotchas (e.g. R2's $0 egress is the headline).
|
||||
Notes string
|
||||
}
|
||||
|
||||
// Catalog is the source-of-truth price list. Keep alphabetically-ish
|
||||
// sorted within a provider.
|
||||
var Catalog = map[Backend]Pricing{
|
||||
BackendAWSS3Std: {
|
||||
StoragePerGBMonth: 0.023, PutPer1k: 0.005, GetPer1k: 0.0004,
|
||||
EgressPerGB: 0.09,
|
||||
Notes: "AWS S3 Standard us-east-1; egress free intra-region, $0.09/GB to internet after 100 GB free tier.",
|
||||
},
|
||||
BackendAWSS3IA: {
|
||||
StoragePerGBMonth: 0.0125, PutPer1k: 0.01, GetPer1k: 0.001,
|
||||
EgressPerGB: 0.09,
|
||||
Notes: "AWS S3 Infrequent Access; 30-day min storage duration; retrieval fee $0.01/GB.",
|
||||
},
|
||||
BackendAWSGlacierIR: {
|
||||
StoragePerGBMonth: 0.004, PutPer1k: 0.02, GetPer1k: 0.01,
|
||||
EgressPerGB: 0.09,
|
||||
Notes: "AWS S3 Glacier Instant Retrieval; 90-day min storage duration; reads are millisecond.",
|
||||
},
|
||||
BackendCloudflareR2: {
|
||||
StoragePerGBMonth: 0.015, PutPer1k: 0.0045, GetPer1k: 0.00036,
|
||||
EgressPerGB: 0,
|
||||
Notes: "Cloudflare R2; $0 egress is the headline — best for cross-cloud reads.",
|
||||
},
|
||||
BackendDOSpaces: {
|
||||
StoragePerGBMonth: 0.02, PutPer1k: 0.005, GetPer1k: 0,
|
||||
EgressPerGB: 0.01,
|
||||
Notes: "DigitalOcean Spaces; $5/mo baseline includes 250 GB + 1 TB egress, then $0.02/GB + $0.01/GB.",
|
||||
},
|
||||
BackendGCSStandard: {
|
||||
StoragePerGBMonth: 0.020, PutPer1k: 0.005, GetPer1k: 0.0004,
|
||||
EgressPerGB: 0.12,
|
||||
Notes: "GCS Standard us-central1; egress to internet $0.12/GB.",
|
||||
},
|
||||
BackendGCSColdline: {
|
||||
StoragePerGBMonth: 0.004, PutPer1k: 0.01, GetPer1k: 0.05,
|
||||
EgressPerGB: 0.12,
|
||||
Notes: "GCS Coldline; 90-day min storage; expensive reads at $0.05/GB retrieval.",
|
||||
},
|
||||
BackendPVCBlock: {
|
||||
StoragePerGBMonth: 0.10, PutPer1k: 0, GetPer1k: 0,
|
||||
EgressPerGB: 0,
|
||||
Notes: "Kubernetes block-storage PVC baseline (DOKS / EKS gp3 / GKE pd-balanced range); $0.10/GB-mo, no per-op costs; in-region only.",
|
||||
},
|
||||
}
|
||||
|
||||
// Workload describes a measured (or projected) usage pattern. Convert
|
||||
// from a benchmark's reported counts via the helpers below.
|
||||
//
|
||||
// Workload is intentionally application-neutral: a validator-archive
|
||||
// study, a sqlite query DB, or a snapshot pipeline all populate the
|
||||
// same fields. Domain-specific knowledge (e.g. "chain X emits SST
|
||||
// files of size Y") belongs in the caller, not in this package.
|
||||
type Workload struct {
|
||||
// StorageGB is the steady-state stored volume.
|
||||
StorageGB float64
|
||||
|
||||
// PutsPerMonth + GetsPerMonth are the request counts at steady
|
||||
// state.
|
||||
PutsPerMonth float64
|
||||
GetsPerMonth float64
|
||||
|
||||
// EgressGBPerMonth is what leaves the provider. Set to 0 if both
|
||||
// vfsd and consumers run in the same region as the backend.
|
||||
EgressGBPerMonth float64
|
||||
|
||||
// Description is purely cosmetic; surfaces in reports.
|
||||
Description string
|
||||
}
|
||||
|
||||
// Estimate is the per-month cost breakdown for a workload on a single
|
||||
// backend.
|
||||
type Estimate struct {
|
||||
Backend Backend
|
||||
Workload string
|
||||
StorageUSD float64
|
||||
RequestsUSD float64
|
||||
EgressUSD float64
|
||||
TotalUSD float64
|
||||
Notes string
|
||||
}
|
||||
|
||||
// EstimateFor returns the per-month cost breakdown for a workload on a
|
||||
// given backend.
|
||||
func EstimateFor(w Workload, be Backend) Estimate {
|
||||
p, ok := Catalog[be]
|
||||
if !ok {
|
||||
return Estimate{Backend: be, Notes: "unknown backend"}
|
||||
}
|
||||
storage := w.StorageGB * p.StoragePerGBMonth
|
||||
requests := (w.PutsPerMonth/1000.0)*p.PutPer1k + (w.GetsPerMonth/1000.0)*p.GetPer1k
|
||||
egress := w.EgressGBPerMonth * p.EgressPerGB
|
||||
|
||||
return Estimate{
|
||||
Backend: be,
|
||||
Workload: w.Description,
|
||||
StorageUSD: storage,
|
||||
RequestsUSD: requests,
|
||||
EgressUSD: egress,
|
||||
TotalUSD: storage + requests + egress,
|
||||
Notes: p.Notes,
|
||||
}
|
||||
}
|
||||
|
||||
// String formats the estimate as a single-line summary.
|
||||
func (e Estimate) String() string {
|
||||
return fmt.Sprintf(
|
||||
"%-22s storage=$%7.2f requests=$%7.2f egress=$%7.2f total=$%7.2f/mo",
|
||||
e.Backend, e.StorageUSD, e.RequestsUSD, e.EgressUSD, e.TotalUSD,
|
||||
)
|
||||
}
|
||||
|
||||
// CompareBackends runs the workload across every backend and returns
|
||||
// the estimates sorted by total monthly cost (cheapest first).
|
||||
func CompareBackends(w Workload) []Estimate {
|
||||
out := make([]Estimate, 0, len(Catalog))
|
||||
for be := range Catalog {
|
||||
out = append(out, EstimateFor(w, be))
|
||||
}
|
||||
// Bubble sort is fine at this size + saves the import.
|
||||
for i := 0; i < len(out)-1; i++ {
|
||||
for j := i + 1; j < len(out); j++ {
|
||||
if out[j].TotalUSD < out[i].TotalUSD {
|
||||
out[i], out[j] = out[j], out[i]
|
||||
}
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// FromBenchmark converts a sustained-write benchmark's measured per-op
|
||||
// rate into a steady-state Workload projection.
|
||||
//
|
||||
// inputs:
|
||||
//
|
||||
// measuredOpsPerSec: throughput observed in a benchmark
|
||||
// sustainedFraction: fraction of time the workload runs at that rate
|
||||
// (e.g. 0.1 if the workload is mostly idle and
|
||||
// only bursts during heavy activity windows)
|
||||
// avgFileSizeBytes: the average size of files written
|
||||
// monthsRetained: how long data is held before being deleted
|
||||
//
|
||||
// Returns a Workload describing the steady state — including the
|
||||
// computed StorageGB growth based on write rate × retention.
|
||||
//
|
||||
// readAmplification = GetsPerMonth / PutsPerMonth at steady state.
|
||||
// A value of 0 disables reads entirely (write-only archive); 2 is
|
||||
// reasonable for an archive read by API consumers; 100+ is "actively
|
||||
// queried" workloads.
|
||||
func FromBenchmark(measuredOpsPerSec, sustainedFraction float64, avgFileSizeBytes int64, monthsRetained, readAmplification float64) Workload {
|
||||
secondsPerMonth := float64(time.Hour*24*30) / float64(time.Second)
|
||||
effectiveOpsPerSec := measuredOpsPerSec * sustainedFraction
|
||||
opsPerMonth := effectiveOpsPerSec * secondsPerMonth
|
||||
|
||||
bytesPerMonth := opsPerMonth * float64(avgFileSizeBytes)
|
||||
storageBytes := bytesPerMonth * monthsRetained
|
||||
|
||||
return Workload{
|
||||
StorageGB: storageBytes / (1 << 30),
|
||||
PutsPerMonth: opsPerMonth,
|
||||
GetsPerMonth: opsPerMonth * readAmplification,
|
||||
Description: fmt.Sprintf("%.1f ops/s × %.0f%% util × %d B avg × %.1f mo retention × %.1f read amp",
|
||||
measuredOpsPerSec, sustainedFraction*100, avgFileSizeBytes, monthsRetained, readAmplification),
|
||||
}
|
||||
}
|
||||
|
||||
// Report renders the comparison as a readable multi-line table.
|
||||
// Pass the workload's name to disambiguate when reporting multiple.
|
||||
func Report(name string, w Workload) string {
|
||||
var b strings.Builder
|
||||
fmt.Fprintf(&b, "Workload: %s\n", name)
|
||||
fmt.Fprintf(&b, " %s\n", w.Description)
|
||||
fmt.Fprintf(&b, " storage_gb=%.1f puts/mo=%.2e gets/mo=%.2e egress_gb/mo=%.1f\n\n",
|
||||
w.StorageGB, w.PutsPerMonth, w.GetsPerMonth, w.EgressGBPerMonth)
|
||||
for _, e := range CompareBackends(w) {
|
||||
fmt.Fprintf(&b, " %s\n", e.String())
|
||||
}
|
||||
return b.String()
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------
|
||||
// Tiered storage: hot SSD + cold object-store archive
|
||||
// ---------------------------------------------------------------------
|
||||
|
||||
// Tiered describes a workload split between a hot tier (local block
|
||||
// storage / SSD, sized for "recent N" data) and a cold tier (VFS into
|
||||
// an object-store backend, holding everything older than the hot
|
||||
// horizon).
|
||||
//
|
||||
// The model is brand-neutral: blocks/SSTs/snapshots/files all reduce
|
||||
// to a steady-state byte rate (RatePerSecond), a hot retention window
|
||||
// (HotWindow), and a cold retention horizon (ColdRetention).
|
||||
type Tiered struct {
|
||||
// RatePerSecond is the steady-state byte production rate (e.g. a
|
||||
// validator emitting finalized state at X MB/s).
|
||||
RatePerSecond float64
|
||||
|
||||
// HotWindow is how long the most-recent data stays on local SSD.
|
||||
// Typical values: 24h, 7 days, 30 days.
|
||||
HotWindow time.Duration
|
||||
|
||||
// ColdRetention is how long data is held in the cold archive
|
||||
// before tier-down or deletion. Use a large value (years) to
|
||||
// model "keep forever".
|
||||
ColdRetention time.Duration
|
||||
|
||||
// HotBackend selects the local-disk pricing entry (typically
|
||||
// BackendPVCBlock).
|
||||
HotBackend Backend
|
||||
|
||||
// ColdBackend selects the object-store entry.
|
||||
ColdBackend Backend
|
||||
|
||||
// PutsPerByte / GetsPerByte map raw bytes to backend request
|
||||
// counts. A pure-stream workload writing 64 KiB objects gives
|
||||
// PutsPerByte = 1/65536. A 4 KiB-block content-addressed VFS
|
||||
// gives PutsPerByte = 1/4096.
|
||||
PutsPerByte float64
|
||||
GetsPerByte float64
|
||||
|
||||
// EgressGBPerMonth is what leaves the cold backend's edge per
|
||||
// month (intra-region reads are free for most providers).
|
||||
EgressGBPerMonth float64
|
||||
}
|
||||
|
||||
// TieredEstimate is the combined cost of a Tiered workload.
|
||||
type TieredEstimate struct {
|
||||
Hot Estimate
|
||||
Cold Estimate
|
||||
TotalUSD float64
|
||||
HotStorageGB float64
|
||||
ColdStorageGB float64
|
||||
}
|
||||
|
||||
// String prints both tiers + the combined total on one short summary.
|
||||
func (e TieredEstimate) String() string {
|
||||
return fmt.Sprintf(
|
||||
"hot[%s %.1f GB]=$%.2f cold[%s %.1f GB]=$%.2f total=$%.2f/mo",
|
||||
e.Hot.Backend, e.HotStorageGB, e.Hot.TotalUSD,
|
||||
e.Cold.Backend, e.ColdStorageGB, e.Cold.TotalUSD,
|
||||
e.TotalUSD,
|
||||
)
|
||||
}
|
||||
|
||||
// Estimate computes the per-month cost for a Tiered workload.
|
||||
//
|
||||
// Hot tier holds RatePerSecond * HotWindow bytes; no per-op cost (PVC
|
||||
// has no PUT/GET fees), no egress (in-region only).
|
||||
//
|
||||
// Cold tier holds RatePerSecond * ColdRetention bytes, with PUTs/GETs
|
||||
// derived from PutsPerByte * (RatePerSecond * seconds/month).
|
||||
func (t Tiered) Estimate() TieredEstimate {
|
||||
const secondsPerMonth = float64(time.Hour*24*30) / float64(time.Second)
|
||||
|
||||
hotBytes := t.RatePerSecond * t.HotWindow.Seconds()
|
||||
coldBytes := t.RatePerSecond * t.ColdRetention.Seconds()
|
||||
bytesPerMonth := t.RatePerSecond * secondsPerMonth
|
||||
|
||||
hot := Workload{
|
||||
StorageGB: hotBytes / (1 << 30),
|
||||
PutsPerMonth: 0, // local SSD has no per-op cost
|
||||
GetsPerMonth: 0,
|
||||
EgressGBPerMonth: 0,
|
||||
Description: "hot tier (local SSD)",
|
||||
}
|
||||
cold := Workload{
|
||||
StorageGB: coldBytes / (1 << 30),
|
||||
PutsPerMonth: bytesPerMonth * t.PutsPerByte,
|
||||
GetsPerMonth: bytesPerMonth * t.GetsPerByte,
|
||||
EgressGBPerMonth: t.EgressGBPerMonth,
|
||||
Description: fmt.Sprintf("cold archive (%s retention)", t.ColdRetention),
|
||||
}
|
||||
|
||||
he := EstimateFor(hot, t.HotBackend)
|
||||
ce := EstimateFor(cold, t.ColdBackend)
|
||||
return TieredEstimate{
|
||||
Hot: he,
|
||||
Cold: ce,
|
||||
TotalUSD: he.TotalUSD + ce.TotalUSD,
|
||||
HotStorageGB: hot.StorageGB,
|
||||
ColdStorageGB: cold.StorageGB,
|
||||
}
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------
|
||||
// Cluster archive sharing: 1 writer + N readers, one shared bucket
|
||||
// ---------------------------------------------------------------------
|
||||
|
||||
// Cluster models a fleet of nodes that share a single content-addressed
|
||||
// cold archive. This is the architectural win that makes object-store
|
||||
// archival cost-effective at scale:
|
||||
//
|
||||
// - Writers (typically validators) PUT new data into the archive.
|
||||
// Because content addressing dedupes identical bytes across all
|
||||
// writers, the effective storage + PUT cost stays linear in
|
||||
// CHAIN GROWTH, not in writer count.
|
||||
// - Readers (read-replicas, explorers, indexers, archive nodes,
|
||||
// follow nodes) GET data from the same archive bucket — they
|
||||
// never PUT, and they don't need their own copy of historical
|
||||
// state on disk.
|
||||
//
|
||||
// The cluster pays:
|
||||
//
|
||||
// storage ← single bucket, sized for cumulative archive
|
||||
// PUTs ← deduped to ONE writer's share regardless of fleet size
|
||||
// GETs ← N readers each pulling their share of archived data
|
||||
// egress ← only if readers live outside the backend's region
|
||||
type Cluster struct {
|
||||
// Writers is the number of nodes producing new state. Used only
|
||||
// for noting in the report; the PUT cost itself is independent
|
||||
// of this once dedup is in place.
|
||||
Writers int
|
||||
// Readers is the number of read-only nodes pulling from the
|
||||
// shared archive. Each contributes its own GET volume.
|
||||
Readers int
|
||||
|
||||
// Underlying single-tier workload (per writer × dedup factor,
|
||||
// or equivalently the chain-growth rate times the writer-share).
|
||||
// In a deduped cluster this is just the chain's intrinsic growth.
|
||||
Base Workload
|
||||
|
||||
// ReadsPerReaderRatio is each reader's fraction of the total GET
|
||||
// volume relative to one full archive scan per month. 0 = no
|
||||
// reads; 1 = one full re-read of the archive per reader per
|
||||
// month; 12 = monthly re-reads per reader (very heavy).
|
||||
ReadsPerReaderRatio float64
|
||||
|
||||
// EgressPerReaderGBPerMonth models the cross-region egress per
|
||||
// reader. Set to 0 if all readers are in the backend's region.
|
||||
EgressPerReaderGBPerMonth float64
|
||||
}
|
||||
|
||||
// Workload returns the cluster's effective archive workload (single
|
||||
// backend bucket, all writers deduped, summed reader GETs + egress).
|
||||
func (c Cluster) Workload() Workload {
|
||||
w := c.Base
|
||||
w.GetsPerMonth = c.Base.PutsPerMonth * c.ReadsPerReaderRatio * float64(c.Readers)
|
||||
w.EgressGBPerMonth = c.EgressPerReaderGBPerMonth * float64(c.Readers)
|
||||
w.Description = fmt.Sprintf(
|
||||
"cluster: %d writers (deduped to 1) + %d readers × %.2f read amp; %s",
|
||||
c.Writers, c.Readers, c.ReadsPerReaderRatio, c.Base.Description,
|
||||
)
|
||||
return w
|
||||
}
|
||||
|
||||
// BlockTimeWorkload constructs a Workload from a chain's block-rate
|
||||
// parameters. This is the generic primitive a chain-specific paper or
|
||||
// CLI composes; the package itself stays brand-neutral.
|
||||
//
|
||||
// blockTime: target time between blocks (e.g. 1s, 100ms, 1ms)
|
||||
// avgBlockBytes: average serialized block size on disk
|
||||
// monthsRetained: how long blocks are held before deletion
|
||||
// readAmplification: GET/PUT ratio at steady state
|
||||
// objectBytes: the granularity each block is split into when
|
||||
// written to the archive (e.g. 4096 for VFS 4 KiB
|
||||
// blocks; equal to avgBlockBytes for one-PUT-per-
|
||||
// block writers)
|
||||
//
|
||||
// Returns a workload that callers can feed to EstimateFor /
|
||||
// CompareBackends / Report, or wrap with Tiered / Cluster.
|
||||
func BlockTimeWorkload(blockTime time.Duration, avgBlockBytes int64, monthsRetained, readAmplification float64, objectBytes int64) Workload {
|
||||
const secondsPerMonth = float64(time.Hour*24*30) / float64(time.Second)
|
||||
blocksPerSec := 1.0 / blockTime.Seconds()
|
||||
bytesPerSec := blocksPerSec * float64(avgBlockBytes)
|
||||
bytesPerMonth := bytesPerSec * secondsPerMonth
|
||||
storageBytes := bytesPerMonth * monthsRetained
|
||||
objectsPerByte := 1.0 / float64(objectBytes)
|
||||
|
||||
return Workload{
|
||||
StorageGB: storageBytes / (1 << 30),
|
||||
PutsPerMonth: bytesPerMonth * objectsPerByte,
|
||||
GetsPerMonth: bytesPerMonth * objectsPerByte * readAmplification,
|
||||
Description: fmt.Sprintf(
|
||||
"block-time=%s, %d B/block, %d B/object, %.1f mo retention, %.1f read amp",
|
||||
blockTime, avgBlockBytes, objectBytes, monthsRetained, readAmplification,
|
||||
),
|
||||
}
|
||||
}
|
||||
@@ -1,190 +0,0 @@
|
||||
package cost_test
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/hanzoai/vfs/pkg/cost"
|
||||
)
|
||||
|
||||
func TestFromBenchmark_ProducesPositiveCosts(t *testing.T) {
|
||||
w := cost.FromBenchmark(6.4, 0.1, 64*1024, 12, 2.0)
|
||||
if w.PutsPerMonth <= 0 {
|
||||
t.Fatalf("PutsPerMonth must be positive; got %f", w.PutsPerMonth)
|
||||
}
|
||||
if w.StorageGB <= 0 {
|
||||
t.Fatalf("StorageGB must be positive; got %f", w.StorageGB)
|
||||
}
|
||||
for _, e := range cost.CompareBackends(w) {
|
||||
if e.TotalUSD < 0 {
|
||||
t.Errorf("%s: negative cost %f", e.Backend, e.TotalUSD)
|
||||
}
|
||||
}
|
||||
t.Log("\n" + cost.Report("benchmark-projection-6.4ops", w))
|
||||
}
|
||||
|
||||
func TestReport_FormatsAllBackends(t *testing.T) {
|
||||
w := cost.Workload{
|
||||
StorageGB: 100,
|
||||
PutsPerMonth: 1_000_000,
|
||||
GetsPerMonth: 1_000_000,
|
||||
EgressGBPerMonth: 10,
|
||||
Description: "synthetic 100 GB / 1M+1M req / 10 GB egress",
|
||||
}
|
||||
r := cost.Report("synthetic", w)
|
||||
for be := range cost.Catalog {
|
||||
if !strings.Contains(r, string(be)) {
|
||||
t.Errorf("report missing backend %q in:\n%s", be, r)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestPVCBaseline_IsExpensiveAtScale(t *testing.T) {
|
||||
// At 1 TB stored, the PVC baseline ($0.10/GB-mo = $102.40/mo)
|
||||
// should be substantially more expensive than R2 ($0.015/GB-mo =
|
||||
// $15.36/mo). This pins the "PVC is the expensive baseline"
|
||||
// invariant — flip detection if pricing changes.
|
||||
w := cost.Workload{
|
||||
StorageGB: 1024,
|
||||
PutsPerMonth: 1_000,
|
||||
GetsPerMonth: 1_000,
|
||||
Description: "1 TB cold archive",
|
||||
}
|
||||
pvc := cost.EstimateFor(w, cost.BackendPVCBlock)
|
||||
r2 := cost.EstimateFor(w, cost.BackendCloudflareR2)
|
||||
if pvc.TotalUSD < r2.TotalUSD*3 {
|
||||
t.Errorf("PVC baseline ($%f) is meant to be substantially more expensive than R2 ($%f); ratio is only %.2fx",
|
||||
pvc.TotalUSD, r2.TotalUSD, pvc.TotalUSD/r2.TotalUSD)
|
||||
}
|
||||
}
|
||||
|
||||
func TestR2_EgressIsZero(t *testing.T) {
|
||||
// R2's headline is $0 egress; pin that.
|
||||
w := cost.Workload{
|
||||
StorageGB: 100,
|
||||
PutsPerMonth: 1_000_000,
|
||||
GetsPerMonth: 1_000_000,
|
||||
EgressGBPerMonth: 10_000, // 10 TB egress
|
||||
Description: "egress-heavy",
|
||||
}
|
||||
e := cost.EstimateFor(w, cost.BackendCloudflareR2)
|
||||
if e.EgressUSD != 0 {
|
||||
t.Errorf("R2 egress cost expected 0; got $%f", e.EgressUSD)
|
||||
}
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------
|
||||
// Block-time sweep: 1 ms → 1 s, on a representative chain workload.
|
||||
// ---------------------------------------------------------------------
|
||||
|
||||
// TestBlockTimeSweep_PrintsCostCurve walks block-time from 1ms to 1s,
|
||||
// printing the cold-archive monthly cost on R2 + the hot SSD size
|
||||
// needed for a 24h / 7d / 30d retention window. Pure smoke test; the
|
||||
// real interpretation lives in the lux-papers companion study.
|
||||
func TestBlockTimeSweep_PrintsCostCurve(t *testing.T) {
|
||||
// Representative single-chain block size; chains with heavier
|
||||
// activity should rerun with their own values.
|
||||
const avgBlockBytes int64 = 20 * 1024 // 20 KB / block
|
||||
const objectBytes int64 = 4 * 1024 // VFS 4 KiB blocks
|
||||
const monthsRetention = 12.0
|
||||
const readAmp = 2.0
|
||||
|
||||
blockTimes := []time.Duration{
|
||||
1 * time.Millisecond,
|
||||
10 * time.Millisecond,
|
||||
100 * time.Millisecond,
|
||||
250 * time.Millisecond,
|
||||
500 * time.Millisecond,
|
||||
1 * time.Second,
|
||||
}
|
||||
hotWindows := []time.Duration{
|
||||
24 * time.Hour,
|
||||
7 * 24 * time.Hour,
|
||||
30 * 24 * time.Hour,
|
||||
}
|
||||
|
||||
var b strings.Builder
|
||||
fmt.Fprintf(&b, "\nBlock-time sweep (avgBlock=%d B, archive=R2, retention=%.0f mo)\n",
|
||||
avgBlockBytes, monthsRetention)
|
||||
fmt.Fprintf(&b, "%-10s %-12s %-12s %-12s %s\n",
|
||||
"block-time", "hot24h-GB", "hot7d-GB", "hot30d-GB", "cold-R2-$/mo")
|
||||
|
||||
for _, bt := range blockTimes {
|
||||
w := cost.BlockTimeWorkload(bt, avgBlockBytes, monthsRetention, readAmp, objectBytes)
|
||||
coldR2 := cost.EstimateFor(w, cost.BackendCloudflareR2)
|
||||
|
||||
// Hot-tier size for each window.
|
||||
bytesPerSec := float64(avgBlockBytes) / bt.Seconds()
|
||||
var hot [3]float64
|
||||
for i, hw := range hotWindows {
|
||||
hot[i] = bytesPerSec * hw.Seconds() / (1 << 30)
|
||||
}
|
||||
fmt.Fprintf(&b, "%-10s %-12.2f %-12.2f %-12.2f $%.2f\n",
|
||||
bt, hot[0], hot[1], hot[2], coldR2.TotalUSD,
|
||||
)
|
||||
}
|
||||
t.Log(b.String())
|
||||
}
|
||||
|
||||
// TestTiered_24h_Hot_R2_Cold validates the Tiered cost split: 24h SSD
|
||||
// hot tier + R2 cold archive at 1s block time.
|
||||
func TestTiered_24h_Hot_R2_Cold(t *testing.T) {
|
||||
tiered := cost.Tiered{
|
||||
RatePerSecond: 20 * 1024, // 20 KB/s
|
||||
HotWindow: 24 * time.Hour,
|
||||
ColdRetention: 12 * 30 * 24 * time.Hour, // 12 months
|
||||
HotBackend: cost.BackendPVCBlock,
|
||||
ColdBackend: cost.BackendCloudflareR2,
|
||||
PutsPerByte: 1.0 / 4096,
|
||||
GetsPerByte: 2.0 / 4096,
|
||||
}
|
||||
est := tiered.Estimate()
|
||||
if est.HotStorageGB <= 0 || est.ColdStorageGB <= 0 {
|
||||
t.Fatalf("storage sizes must be positive: hot=%.2f cold=%.2f",
|
||||
est.HotStorageGB, est.ColdStorageGB)
|
||||
}
|
||||
if est.ColdStorageGB <= est.HotStorageGB {
|
||||
t.Errorf("cold archive should hold > hot tier at 12 mo retention; got hot=%.2f cold=%.2f",
|
||||
est.HotStorageGB, est.ColdStorageGB)
|
||||
}
|
||||
t.Logf("\n%s\n", est)
|
||||
}
|
||||
|
||||
// TestCluster_DedupKeepsWriterCostFlat validates the "one writer
|
||||
// effectively, regardless of fleet size" invariant: a deduped cluster's
|
||||
// PUT cost is independent of writer count.
|
||||
func TestCluster_DedupKeepsWriterCostFlat(t *testing.T) {
|
||||
base := cost.BlockTimeWorkload(1*time.Second, 20*1024, 12, 0, 4096)
|
||||
|
||||
cluster1 := cost.Cluster{Writers: 1, Readers: 0, Base: base, ReadsPerReaderRatio: 0}
|
||||
cluster11 := cost.Cluster{Writers: 11, Readers: 0, Base: base, ReadsPerReaderRatio: 0}
|
||||
|
||||
e1 := cost.EstimateFor(cluster1.Workload(), cost.BackendCloudflareR2)
|
||||
e11 := cost.EstimateFor(cluster11.Workload(), cost.BackendCloudflareR2)
|
||||
|
||||
if e1.TotalUSD != e11.TotalUSD {
|
||||
t.Errorf("deduped cluster cost must be flat across writers; 1w=$%.2f 11w=$%.2f",
|
||||
e1.TotalUSD, e11.TotalUSD)
|
||||
}
|
||||
t.Logf("dedup invariant pinned: 1 writer = 11 writers = $%.2f/mo", e1.TotalUSD)
|
||||
}
|
||||
|
||||
// TestCluster_ReadersScaleGetCost validates the read-replica scaling
|
||||
// model: GET cost is linear in reader count.
|
||||
func TestCluster_ReadersScaleGetCost(t *testing.T) {
|
||||
base := cost.BlockTimeWorkload(1*time.Second, 20*1024, 12, 0, 4096)
|
||||
|
||||
for _, n := range []int{1, 11, 100, 1000} {
|
||||
c := cost.Cluster{
|
||||
Writers: 1,
|
||||
Readers: n,
|
||||
Base: base,
|
||||
ReadsPerReaderRatio: 0.1, // each reader pulls 10% of writes
|
||||
}
|
||||
w := c.Workload()
|
||||
e := cost.EstimateFor(w, cost.BackendCloudflareR2)
|
||||
t.Logf("readers=%-4d total=$%6.2f/mo (gets/mo=%.2e)", n, e.TotalUSD, w.GetsPerMonth)
|
||||
}
|
||||
}
|
||||
@@ -6,8 +6,7 @@
|
||||
// the database survived the round trip.
|
||||
//
|
||||
// Run with:
|
||||
//
|
||||
// VFS_FUSE_E2E=1 go test -tags fuse -run TestFUSESQLite ./pkg/mount/
|
||||
// VFS_FUSE_E2E=1 go test -tags fuse -run TestFUSESQLite ./pkg/mount/
|
||||
//
|
||||
// Linux: requires kernel ≥ 3.10 with FUSE support (default on every
|
||||
// modern distro).
|
||||
@@ -28,13 +27,13 @@ import (
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
_ "github.com/hanzoai/sqlite"
|
||||
"github.com/luxfi/age"
|
||||
_ "modernc.org/sqlite"
|
||||
|
||||
"github.com/hanzoai/vfs"
|
||||
"github.com/hanzoai/vfs/pkg/backend"
|
||||
_ "github.com/hanzoai/vfs/pkg/backend/file"
|
||||
"github.com/hanzoai/vfs/pkg/mount"
|
||||
"github.com/hanzoai/vfs"
|
||||
)
|
||||
|
||||
func TestFUSESQLite(t *testing.T) {
|
||||
@@ -128,7 +127,7 @@ func TestFUSESQLite(t *testing.T) {
|
||||
t.Fatalf("Prepare: %v", err)
|
||||
}
|
||||
for i := 0; i < 100; i++ {
|
||||
if _, err := stmt.Exec(fmt.Sprintf("USDC_%d", i), "USD Coin", 6); err != nil {
|
||||
if _, err := stmt.Exec(fmt.Sprintf("LQDTY_%d", i), "Liquidity", 18); err != nil {
|
||||
t.Fatalf("INSERT %d: %v", i, err)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,611 +0,0 @@
|
||||
// Production-readiness tests for using VFS to back persistent stateful
|
||||
// workloads (luxd chain DB, SQLite, etc.).
|
||||
//
|
||||
// Covers the four code-level concerns that gate "can we mount VFS at
|
||||
// /data/db and walk away":
|
||||
//
|
||||
// #1 crash/restart recovery — mid-write process death + reopen,
|
||||
// verify FS view is consistent.
|
||||
// #4 compaction under load — sustained write pattern matching
|
||||
// zapdb compaction (many small files,
|
||||
// frequent fsync, occasional bulk rewrite).
|
||||
// #5 backend outage — inject Put/Get errors and verify the
|
||||
// FS surfaces them rather than hanging or
|
||||
// silently losing data.
|
||||
// #6 key rotation — confirm Crypto accepts a multi-recipient
|
||||
// list (rolling rotation viable) and that
|
||||
// a single Get round-trip still works after
|
||||
// a recipient is added or removed.
|
||||
//
|
||||
// The three operational concerns — empty-cache bootstrap timing,
|
||||
// warm-cache bootstrap timing, and rollback path to PVC-local state —
|
||||
// are runbooks against a real cluster, not Go tests; see
|
||||
// docs/PRODUCTION-READINESS.md.
|
||||
package vfs_test
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"os"
|
||||
"strings"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/luxfi/age"
|
||||
|
||||
"github.com/hanzoai/vfs"
|
||||
"github.com/hanzoai/vfs/pkg/backend"
|
||||
)
|
||||
|
||||
// ---------------------------------------------------------------------
|
||||
// shared helpers
|
||||
// ---------------------------------------------------------------------
|
||||
|
||||
// buildFS mounts a VFS rooted in dir with the given identity. Returns
|
||||
// the *FS and a teardown func (Sync only — VFS exposes no Close; the
|
||||
// Sync is the durability boundary).
|
||||
func buildFS(t *testing.T, dir string, id *age.X25519Identity) (*vfs.FS, func()) {
|
||||
t.Helper()
|
||||
be, err := backend.Open(context.Background(), "file://"+dir)
|
||||
if err != nil {
|
||||
t.Fatalf("backend.Open: %v", err)
|
||||
}
|
||||
c, err := vfs.NewCrypto([]age.Recipient{id.Recipient()}, []age.Identity{id})
|
||||
if err != nil {
|
||||
t.Fatalf("NewCrypto: %v", err)
|
||||
}
|
||||
v, err := vfs.New(vfs.Config{Backend: be, Crypto: c})
|
||||
if err != nil {
|
||||
t.Fatalf("vfs.New: %v", err)
|
||||
}
|
||||
fs, err := vfs.NewFS(context.Background(), v)
|
||||
if err != nil {
|
||||
t.Fatalf("NewFS: %v", err)
|
||||
}
|
||||
teardown := func() {
|
||||
if err := fs.Sync(context.Background()); err != nil {
|
||||
t.Errorf("teardown Sync: %v", err)
|
||||
}
|
||||
}
|
||||
return fs, teardown
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------
|
||||
// #1 crash / restart recovery
|
||||
// ---------------------------------------------------------------------
|
||||
|
||||
// TestCrashRecovery_Synced_Persists confirms that any file that was
|
||||
// Sync()-ed before the crash is observable identically after reopen,
|
||||
// and that the inode tree (sizes, mode, mtimes) round-trips.
|
||||
//
|
||||
// This is the durability contract a POSIX consumer (zapdb, SQLite)
|
||||
// relies on: `fsync(2)` returning nil must mean the data is durable
|
||||
// across an arbitrary process kill.
|
||||
func TestCrashRecovery_Synced_Persists(t *testing.T) {
|
||||
t.Parallel()
|
||||
dir := t.TempDir()
|
||||
id, err := age.GenerateX25519Identity()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// Round 1: write three files, Sync, then drop FS without unmount.
|
||||
const N = 3
|
||||
payloads := make([][]byte, N)
|
||||
{
|
||||
fs, _ := buildFS(t, dir, id)
|
||||
for i := 0; i < N; i++ {
|
||||
name := fmt.Sprintf("/file-%d.db", i)
|
||||
if _, err := fs.Create(name, 0o644); err != nil {
|
||||
t.Fatalf("Create %s: %v", name, err)
|
||||
}
|
||||
f, err := fs.Open(context.Background(), name)
|
||||
if err != nil {
|
||||
t.Fatalf("Open %s: %v", name, err)
|
||||
}
|
||||
// Cross-block payload to exercise the block-splitting path.
|
||||
payloads[i] = make([]byte, 3*vfs.BlockSize+1234)
|
||||
for j := range payloads[i] {
|
||||
payloads[i][j] = byte(i*7 + j%251)
|
||||
}
|
||||
if _, err := f.WriteAt(payloads[i], 0); err != nil {
|
||||
t.Fatalf("WriteAt %s: %v", name, err)
|
||||
}
|
||||
if err := f.Sync(); err != nil {
|
||||
t.Fatalf("Sync %s: %v", name, err)
|
||||
}
|
||||
if err := f.Close(); err != nil {
|
||||
t.Fatalf("Close %s: %v", name, err)
|
||||
}
|
||||
}
|
||||
// Simulated crash: we drop `fs` on the floor without
|
||||
// any further Sync. The File.Sync calls already pushed the
|
||||
// metadata blob; this is the durability contract under test.
|
||||
}
|
||||
|
||||
// Round 2: reopen + verify byte-equal reads + correct sizes.
|
||||
fs2, _ := buildFS(t, dir, id)
|
||||
for i := 0; i < N; i++ {
|
||||
name := fmt.Sprintf("/file-%d.db", i)
|
||||
f, err := fs2.Open(context.Background(), name)
|
||||
if err != nil {
|
||||
t.Fatalf("Open after crash %s: %v", name, err)
|
||||
}
|
||||
stat, err := f.Stat()
|
||||
if err != nil {
|
||||
t.Fatalf("Stat after crash %s: %v", name, err)
|
||||
}
|
||||
if int64(stat.Size) != int64(len(payloads[i])) {
|
||||
t.Fatalf("size drift on %s: got %d want %d", name, stat.Size, len(payloads[i]))
|
||||
}
|
||||
got := make([]byte, len(payloads[i]))
|
||||
n, err := f.ReadAt(got, 0)
|
||||
// io.ReaderAt contract: hitting end-of-file returns (n, io.EOF)
|
||||
// even when the read was complete. Only fail on other errors.
|
||||
if err != nil && err.Error() != "EOF" {
|
||||
t.Fatalf("ReadAt %s: %v", name, err)
|
||||
}
|
||||
if n != len(payloads[i]) {
|
||||
t.Fatalf("short read on %s: got %d want %d", name, n, len(payloads[i]))
|
||||
}
|
||||
for j := range got {
|
||||
if got[j] != payloads[i][j] {
|
||||
t.Fatalf("byte drift on %s at offset %d: got %02x want %02x",
|
||||
name, j, got[j], payloads[i][j])
|
||||
}
|
||||
}
|
||||
_ = f.Close()
|
||||
}
|
||||
}
|
||||
|
||||
// TestCrashRecovery_UnsyncedWriteIsLost_NotCorrupted is the negative
|
||||
// half of the durability story: a write that was NOT Sync()-ed before
|
||||
// crash must not appear in the post-restart view (no torn-write
|
||||
// resurrection from orphaned blocks), and pre-existing files must
|
||||
// remain readable byte-for-byte.
|
||||
//
|
||||
// This pins the policy: POSIX consumers MUST call fsync to guarantee
|
||||
// durability — exactly the contract zapdb/SQLite already expect.
|
||||
func TestCrashRecovery_UnsyncedWriteIsLost_NotCorrupted(t *testing.T) {
|
||||
t.Parallel()
|
||||
dir := t.TempDir()
|
||||
id, err := age.GenerateX25519Identity()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
stable := []byte("durable-bytes-from-first-mount")
|
||||
{
|
||||
fs, _ := buildFS(t, dir, id)
|
||||
if _, err := fs.Create("/stable.db", 0o644); err != nil {
|
||||
t.Fatalf("Create stable: %v", err)
|
||||
}
|
||||
f, _ := fs.Open(context.Background(), "/stable.db")
|
||||
if _, err := f.WriteAt(stable, 0); err != nil {
|
||||
t.Fatalf("WriteAt stable: %v", err)
|
||||
}
|
||||
if err := f.Sync(); err != nil {
|
||||
t.Fatalf("Sync stable: %v", err)
|
||||
}
|
||||
_ = f.Close()
|
||||
// Now write a second file but DO NOT Sync — simulates a write
|
||||
// in flight when the host crashes.
|
||||
if _, err := fs.Create("/in-flight.db", 0o644); err != nil {
|
||||
t.Fatalf("Create in-flight: %v", err)
|
||||
}
|
||||
f2, _ := fs.Open(context.Background(), "/in-flight.db")
|
||||
if _, err := f2.WriteAt([]byte("never-fsynced"), 0); err != nil {
|
||||
t.Fatalf("WriteAt in-flight: %v", err)
|
||||
}
|
||||
// Deliberately skip Sync + Close — simulating crash.
|
||||
}
|
||||
|
||||
fs2, _ := buildFS(t, dir, id)
|
||||
|
||||
// stable.db must round-trip exactly.
|
||||
f, err := fs2.Open(context.Background(), "/stable.db")
|
||||
if err != nil {
|
||||
t.Fatalf("Open stable after crash: %v", err)
|
||||
}
|
||||
got := make([]byte, len(stable))
|
||||
if _, err := f.ReadAt(got, 0); err != nil && err.Error() != "EOF" {
|
||||
t.Fatalf("ReadAt stable: %v", err)
|
||||
}
|
||||
for i := range got {
|
||||
if got[i] != stable[i] {
|
||||
t.Fatalf("corruption on stable.db at byte %d", i)
|
||||
}
|
||||
}
|
||||
_ = f.Close()
|
||||
|
||||
// in-flight.db must be ABSENT (the metadata root that registers
|
||||
// it was never written). If we see the file, VFS persisted
|
||||
// metadata without a Sync — durability contract broken.
|
||||
if _, err := fs2.Open(context.Background(), "/in-flight.db"); err == nil {
|
||||
t.Fatalf("in-flight.db visible after crash without fsync — durability policy violated")
|
||||
}
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------
|
||||
// #4 compaction-style sustained write
|
||||
// ---------------------------------------------------------------------
|
||||
|
||||
// TestCompaction_SustainedWrite emulates zapdb's compaction pattern:
|
||||
// many small files, frequent fsync, and an occasional large rewrite.
|
||||
// Reports per-op latency + throughput so the operator can decide if the
|
||||
// observed numbers fit luxd's write budget under chain load.
|
||||
//
|
||||
// Skipped by default — set VFS_PROD_TEST=1 to opt in. The test runs
|
||||
// 60 seconds of writes; CI runs should pass shorter durations via
|
||||
// VFS_PROD_TEST_DURATION=30s.
|
||||
func TestCompaction_SustainedWrite(t *testing.T) {
|
||||
if v := os.Getenv("VFS_PROD_TEST"); v == "" || v == "0" || v == "false" {
|
||||
t.Skip("set VFS_PROD_TEST=1 to run production-readiness benchmarks")
|
||||
}
|
||||
dir := t.TempDir()
|
||||
id, err := age.GenerateX25519Identity()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
fs, teardown := buildFS(t, dir, id)
|
||||
defer teardown()
|
||||
|
||||
dur := 60 * time.Second
|
||||
if v := os.Getenv("VFS_PROD_TEST_DURATION"); v != "" {
|
||||
d, err := time.ParseDuration(v)
|
||||
if err != nil {
|
||||
t.Fatalf("VFS_PROD_TEST_DURATION: %v", err)
|
||||
}
|
||||
dur = d
|
||||
}
|
||||
deadline := time.Now().Add(dur)
|
||||
|
||||
// Pattern: 16 "SST" files, each grown by 64KB and Sync()'ed in a
|
||||
// loop. Every 8 iterations we drop one file (compaction merge) and
|
||||
// create a new one. This is a coarse-grained approximation; what
|
||||
// we want is throughput and tail latency numbers, not a faithful
|
||||
// LSM simulation.
|
||||
const numFiles = 16
|
||||
files := make([]*vfs.File, 0, numFiles)
|
||||
for i := 0; i < numFiles; i++ {
|
||||
name := fmt.Sprintf("/sst-%03d.dat", i)
|
||||
if _, err := fs.Create(name, 0o644); err != nil {
|
||||
t.Fatalf("Create %s: %v", name, err)
|
||||
}
|
||||
f, err := fs.Open(context.Background(), name)
|
||||
if err != nil {
|
||||
t.Fatalf("Open %s: %v", name, err)
|
||||
}
|
||||
files = append(files, f)
|
||||
}
|
||||
|
||||
chunk := make([]byte, 64*1024)
|
||||
for j := range chunk {
|
||||
chunk[j] = byte(j)
|
||||
}
|
||||
|
||||
var ops int64
|
||||
var totalNs int64
|
||||
var maxNs int64
|
||||
iter := 0
|
||||
for time.Now().Before(deadline) {
|
||||
f := files[iter%numFiles]
|
||||
offset := int64((iter / numFiles) * len(chunk))
|
||||
|
||||
start := time.Now()
|
||||
if _, err := f.WriteAt(chunk, offset); err != nil {
|
||||
t.Fatalf("WriteAt iter=%d: %v", iter, err)
|
||||
}
|
||||
if err := f.Sync(); err != nil {
|
||||
t.Fatalf("Sync iter=%d: %v", iter, err)
|
||||
}
|
||||
elapsed := time.Since(start).Nanoseconds()
|
||||
ops++
|
||||
totalNs += elapsed
|
||||
if elapsed > maxNs {
|
||||
maxNs = elapsed
|
||||
}
|
||||
iter++
|
||||
}
|
||||
|
||||
for _, f := range files {
|
||||
_ = f.Close()
|
||||
}
|
||||
|
||||
mean := time.Duration(totalNs / ops)
|
||||
t.Logf("compaction sustained-write: ops=%d duration=%s mean=%s max=%s thru=%.2f ops/s",
|
||||
ops, dur, mean, time.Duration(maxNs), float64(ops)/dur.Seconds())
|
||||
|
||||
// The acceptance bar is operator-policy, not a hard test fail.
|
||||
// We just record the numbers so the runbook has empirical data.
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------
|
||||
// #5 backend outage behavior
|
||||
// ---------------------------------------------------------------------
|
||||
|
||||
// erroringBackend wraps a real backend and injects errors on demand.
|
||||
// Used to simulate S3 503s / network partitions / timeouts without
|
||||
// requiring a fault-injection proxy.
|
||||
type erroringBackend struct {
|
||||
inner backend.Backend
|
||||
err atomic.Pointer[error]
|
||||
}
|
||||
|
||||
func newErroringBackend(inner backend.Backend) *erroringBackend {
|
||||
return &erroringBackend{inner: inner}
|
||||
}
|
||||
|
||||
func (e *erroringBackend) setErr(err error) {
|
||||
if err == nil {
|
||||
e.err.Store(nil)
|
||||
return
|
||||
}
|
||||
e.err.Store(&err)
|
||||
}
|
||||
|
||||
func (e *erroringBackend) Get(ctx context.Context, key string) ([]byte, error) {
|
||||
if err := e.err.Load(); err != nil {
|
||||
return nil, *err
|
||||
}
|
||||
return e.inner.Get(ctx, key)
|
||||
}
|
||||
|
||||
func (e *erroringBackend) Put(ctx context.Context, key string, b []byte) error {
|
||||
if err := e.err.Load(); err != nil {
|
||||
return *err
|
||||
}
|
||||
return e.inner.Put(ctx, key, b)
|
||||
}
|
||||
|
||||
func (e *erroringBackend) Delete(ctx context.Context, key string) error {
|
||||
if err := e.err.Load(); err != nil {
|
||||
return *err
|
||||
}
|
||||
return e.inner.Delete(ctx, key)
|
||||
}
|
||||
|
||||
func (e *erroringBackend) List(ctx context.Context, prefix string) (<-chan string, <-chan error) {
|
||||
if err := e.err.Load(); err != nil {
|
||||
keys := make(chan string)
|
||||
errs := make(chan error, 1)
|
||||
close(keys)
|
||||
errs <- *err
|
||||
close(errs)
|
||||
return keys, errs
|
||||
}
|
||||
return e.inner.List(ctx, prefix)
|
||||
}
|
||||
|
||||
func (e *erroringBackend) Stat(ctx context.Context, key string) (int64, error) {
|
||||
if err := e.err.Load(); err != nil {
|
||||
return 0, *err
|
||||
}
|
||||
return e.inner.Stat(ctx, key)
|
||||
}
|
||||
|
||||
func (e *erroringBackend) Close() error {
|
||||
return e.inner.Close()
|
||||
}
|
||||
|
||||
func (e *erroringBackend) String() string {
|
||||
return "erroring(" + e.inner.String() + ")"
|
||||
}
|
||||
|
||||
// TestBackendOutage_PutErrorBubblesUp confirms that a backend Put
|
||||
// failure surfaces as a Sync error rather than being swallowed or
|
||||
// silently retried forever. A POSIX consumer seeing fsync return
|
||||
// non-nil is the signal it needs to treat the write as failed.
|
||||
func TestBackendOutage_PutErrorBubblesUp(t *testing.T) {
|
||||
t.Parallel()
|
||||
dir := t.TempDir()
|
||||
id, err := age.GenerateX25519Identity()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
innerBE, err := backend.Open(context.Background(), "file://"+dir)
|
||||
if err != nil {
|
||||
t.Fatalf("backend.Open: %v", err)
|
||||
}
|
||||
eb := newErroringBackend(innerBE)
|
||||
c, _ := vfs.NewCrypto([]age.Recipient{id.Recipient()}, []age.Identity{id})
|
||||
v, err := vfs.New(vfs.Config{Backend: eb, Crypto: c})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
fs, err := vfs.NewFS(context.Background(), v)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
if _, err := fs.Create("/erred.db", 0o644); err != nil {
|
||||
t.Fatalf("Create: %v", err)
|
||||
}
|
||||
f, _ := fs.Open(context.Background(), "/erred.db")
|
||||
|
||||
// Inject backend Put failures, then attempt a write+Sync. Sync
|
||||
// must fail (the block PutBlock cannot reach the backend).
|
||||
injected := errors.New("simulated S3 503")
|
||||
eb.setErr(injected)
|
||||
if _, err := f.WriteAt([]byte("never lands"), 0); err != nil {
|
||||
t.Fatalf("WriteAt should buffer locally, got: %v", err)
|
||||
}
|
||||
syncErr := f.Sync()
|
||||
if syncErr == nil {
|
||||
t.Fatalf("Sync must return error when backend is down; instead returned nil — durability policy violated")
|
||||
}
|
||||
if !errors.Is(syncErr, injected) && !strings.Contains(syncErr.Error(), "simulated S3 503") {
|
||||
t.Fatalf("Sync error must wrap underlying backend error; got %v", syncErr)
|
||||
}
|
||||
|
||||
// Recovery: clear the error, retry Sync, expect success.
|
||||
eb.setErr(nil)
|
||||
if err := f.Sync(); err != nil {
|
||||
t.Fatalf("Sync after backend recovery: %v", err)
|
||||
}
|
||||
_ = f.Close()
|
||||
}
|
||||
|
||||
// TestBackendOutage_GetErrorBubblesUp confirms Get failures surface to
|
||||
// the caller as read errors rather than zero bytes (silent data loss).
|
||||
func TestBackendOutage_GetErrorBubblesUp(t *testing.T) {
|
||||
t.Parallel()
|
||||
dir := t.TempDir()
|
||||
id, err := age.GenerateX25519Identity()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
innerBE, err := backend.Open(context.Background(), "file://"+dir)
|
||||
if err != nil {
|
||||
t.Fatalf("backend.Open: %v", err)
|
||||
}
|
||||
eb := newErroringBackend(innerBE)
|
||||
c, _ := vfs.NewCrypto([]age.Recipient{id.Recipient()}, []age.Identity{id})
|
||||
v, err := vfs.New(vfs.Config{Backend: eb, Crypto: c})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
fs, err := vfs.NewFS(context.Background(), v)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// Write a file successfully first (so we have a durable block on
|
||||
// the inner backend).
|
||||
if _, err := fs.Create("/payload.db", 0o644); err != nil {
|
||||
t.Fatalf("Create: %v", err)
|
||||
}
|
||||
f, _ := fs.Open(context.Background(), "/payload.db")
|
||||
want := []byte("durable-payload-but-backend-will-flake-on-read-and-this-spans-multiple-blocks-easily")
|
||||
for len(want) < 3*vfs.BlockSize {
|
||||
want = append(want, want...)
|
||||
}
|
||||
if _, err := f.WriteAt(want, 0); err != nil {
|
||||
t.Fatalf("WriteAt: %v", err)
|
||||
}
|
||||
if err := f.Sync(); err != nil {
|
||||
t.Fatalf("Sync: %v", err)
|
||||
}
|
||||
_ = f.Close()
|
||||
|
||||
// Now flake the backend and reopen — read must fail explicitly.
|
||||
eb.setErr(errors.New("simulated network partition"))
|
||||
f2, err := fs.Open(context.Background(), "/payload.db")
|
||||
if err != nil {
|
||||
t.Fatalf("Open: %v", err)
|
||||
}
|
||||
got := make([]byte, len(want))
|
||||
_, readErr := f2.ReadAt(got, 0)
|
||||
if readErr == nil {
|
||||
t.Fatalf("ReadAt must surface backend error; got nil err — VFS swallowed the outage")
|
||||
}
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------
|
||||
// #6 key rotation
|
||||
// ---------------------------------------------------------------------
|
||||
|
||||
// TestKeyRotation_MultiRecipientAllowsRollingRotation confirms that
|
||||
// the Crypto layer accepts BOTH the old and the new recipient at write
|
||||
// time so that during a rotation window:
|
||||
//
|
||||
// - new writes go to <oldRcpt, newRcpt> (decryptable by either ID)
|
||||
// - existing blocks stay decryptable by the old ID
|
||||
// - after the rotation window, drop the old ID from Decrypt; new
|
||||
// writes flip to <newRcpt> only.
|
||||
//
|
||||
// This is the design contract that makes rolling key rotation possible
|
||||
// without a stop-the-world re-encrypt.
|
||||
func TestKeyRotation_MultiRecipientAllowsRollingRotation(t *testing.T) {
|
||||
t.Parallel()
|
||||
dir := t.TempDir()
|
||||
oldID, err := age.GenerateX25519Identity()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
newID, err := age.GenerateX25519Identity()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
be, err := backend.Open(context.Background(), "file://"+dir)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// Stage 1: write under old key only.
|
||||
cOld, err := vfs.NewCrypto(
|
||||
[]age.Recipient{oldID.Recipient()},
|
||||
[]age.Identity{oldID},
|
||||
)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
vOld, err := vfs.New(vfs.Config{Backend: be, Crypto: cOld})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
oldBlockID, err := vOld.PutBlock(context.Background(), []byte("written-under-old-key"))
|
||||
if err != nil {
|
||||
t.Fatalf("PutBlock under old: %v", err)
|
||||
}
|
||||
|
||||
// Stage 2: rotation window — write under {old,new} recipients;
|
||||
// read under {old,new} identities.
|
||||
cBoth, err := vfs.NewCrypto(
|
||||
[]age.Recipient{oldID.Recipient(), newID.Recipient()},
|
||||
[]age.Identity{oldID, newID},
|
||||
)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
vBoth, err := vfs.New(vfs.Config{Backend: be, Crypto: cBoth})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
newBlockID, err := vBoth.PutBlock(context.Background(), []byte("written-during-rotation"))
|
||||
if err != nil {
|
||||
t.Fatalf("PutBlock under both: %v", err)
|
||||
}
|
||||
|
||||
// Both blocks readable under rotation Crypto.
|
||||
for label, id := range map[string]vfs.BlockID{
|
||||
"oldBlock-via-both": oldBlockID,
|
||||
"newBlock-via-both": newBlockID,
|
||||
} {
|
||||
_, err := vBoth.GetBlock(context.Background(), id)
|
||||
if err != nil {
|
||||
t.Fatalf("%s decrypt failed under {old+new} identities: %v", label, err)
|
||||
}
|
||||
}
|
||||
|
||||
// Stage 3: post-window — drop the old recipient/identity.
|
||||
cNew, err := vfs.NewCrypto(
|
||||
[]age.Recipient{newID.Recipient()},
|
||||
[]age.Identity{newID},
|
||||
)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
vNew, err := vfs.New(vfs.Config{Backend: be, Crypto: cNew})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// newBlockID was written under {old, new} so its age envelope
|
||||
// includes a stanza for newID — it must still decrypt.
|
||||
if _, err := vNew.GetBlock(context.Background(), newBlockID); err != nil {
|
||||
t.Fatalf("newBlock should still decrypt under {new} only: %v", err)
|
||||
}
|
||||
|
||||
// oldBlockID was written under {old} only — newID is not in the
|
||||
// envelope, so decryption must fail. This is the proof that
|
||||
// rotation actually rotates (rather than the old key being a
|
||||
// no-op).
|
||||
if _, err := vNew.GetBlock(context.Background(), oldBlockID); err == nil {
|
||||
t.Fatalf("oldBlock decrypted under {new} only — proof that key rotation isn't real")
|
||||
}
|
||||
}
|
||||
@@ -1,102 +0,0 @@
|
||||
// Copyright 2025 Hanzo AI Inc. All Rights Reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
|
||||
package replica
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
|
||||
"github.com/hanzoai/vfs/pkg/backend"
|
||||
_ "github.com/hanzoai/vfs/pkg/backend/file"
|
||||
)
|
||||
|
||||
// seedDB creates a SQLite DB at path with n rows (~a real per-org store's size).
|
||||
func seedDB(b *testing.B, path string, rows int) *SQLiteDB {
|
||||
b.Helper()
|
||||
db, err := OpenSQLite(path)
|
||||
if err != nil {
|
||||
b.Fatalf("open: %v", err)
|
||||
}
|
||||
ctx := context.Background()
|
||||
if _, err := db.DB().ExecContext(ctx, `CREATE TABLE kv(k INTEGER PRIMARY KEY, v TEXT)`); err != nil {
|
||||
b.Fatalf("create: %v", err)
|
||||
}
|
||||
tx, _ := db.DB().Begin()
|
||||
for i := 0; i < rows; i++ {
|
||||
if _, err := tx.Exec(`INSERT INTO kv(k,v) VALUES(?,?)`, i, fmt.Sprintf("row-%d-payload-data-padding", i)); err != nil {
|
||||
b.Fatalf("insert: %v", err)
|
||||
}
|
||||
}
|
||||
_ = tx.Commit()
|
||||
return db
|
||||
}
|
||||
|
||||
// BenchmarkSnapshot measures the WAL-checkpointed VACUUM-INTO snapshot (the backup
|
||||
// hot path — runs every PushLoop tick per org).
|
||||
func BenchmarkSnapshot(b *testing.B) {
|
||||
for _, rows := range []int{1_000, 100_000} {
|
||||
b.Run(fmt.Sprintf("rows=%d", rows), func(b *testing.B) {
|
||||
db := seedDB(b, filepath.Join(b.TempDir(), "s.db"), rows)
|
||||
defer db.Close()
|
||||
ctx := context.Background()
|
||||
var bytes int
|
||||
b.ResetTimer()
|
||||
for i := 0; i < b.N; i++ {
|
||||
data, err := db.Snapshot(ctx)
|
||||
if err != nil {
|
||||
b.Fatalf("snapshot: %v", err)
|
||||
}
|
||||
bytes = len(data)
|
||||
}
|
||||
b.StopTimer()
|
||||
b.ReportMetric(float64(bytes)/1024, "KiB/snap")
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// BenchmarkPushPull measures the full replication round trip (snapshot → seal-skipped
|
||||
// → object-store Put → Version → Get → restore) over the real file backend.
|
||||
func BenchmarkPushPull(b *testing.B) {
|
||||
for _, rows := range []int{1_000, 100_000} {
|
||||
b.Run(fmt.Sprintf("rows=%d", rows), func(b *testing.B) {
|
||||
ctx := context.Background()
|
||||
objDir := b.TempDir()
|
||||
be, err := backend.Open(ctx, "file://"+objDir)
|
||||
if err != nil {
|
||||
b.Fatalf("backend: %v", err)
|
||||
}
|
||||
defer be.Close()
|
||||
store := NewBackendStore(be)
|
||||
key := DBPath("acme", "", "kv")
|
||||
|
||||
owner := seedDB(b, filepath.Join(b.TempDir(), "o.db"), rows)
|
||||
defer owner.Close()
|
||||
or := NewReplicator(key, store, owner)
|
||||
reader := seedDB(b, filepath.Join(b.TempDir(), "r.db"), 0)
|
||||
defer reader.Close()
|
||||
rr := NewReplicator(key, store, reader)
|
||||
|
||||
b.ResetTimer()
|
||||
for i := 0; i < b.N; i++ {
|
||||
if err := or.Push(ctx); err != nil {
|
||||
b.Fatalf("push: %v", err)
|
||||
}
|
||||
// Force a real pull each iter (bump content so version differs).
|
||||
if _, err := owner.DB().ExecContext(ctx, `INSERT INTO kv(k,v) VALUES(?,?)`, rows+i, "x"); err != nil {
|
||||
b.Fatalf("bump: %v", err)
|
||||
}
|
||||
if err := or.Push(ctx); err != nil {
|
||||
b.Fatalf("push2: %v", err)
|
||||
}
|
||||
if _, err := rr.Pull(ctx); err != nil {
|
||||
b.Fatalf("pull: %v", err)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -1,271 +0,0 @@
|
||||
// Copyright 2025 Hanzo AI Inc. All Rights Reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
|
||||
package replica
|
||||
|
||||
// Fencing turns the unconditional Store.Put — whose object-store contract is
|
||||
// literally "overwriting is allowed" — into a round-fenced write, so a deposed or
|
||||
// partitioned writer can no longer clobber a newer owner's snapshot. It is the
|
||||
// STORAGE half of single-writer safety: coordination (github.com/hanzoai/ha)
|
||||
// decides who writes and at what monotone round; FencedStore makes the object
|
||||
// store REJECT a write stamped with a superseded round, no matter what a stale
|
||||
// local view believes.
|
||||
//
|
||||
// # The admission rule
|
||||
//
|
||||
// One object per key carries a small round header in front of the payload. The
|
||||
// store — never this process — enforces, atomically:
|
||||
//
|
||||
// admit a Put at `round` iff round >= the round recorded in the object;
|
||||
// reject with ErrStaleRound when round < recorded.
|
||||
//
|
||||
// `>=` (not the one-shot fence's strict `>`) is deliberate and is the whole
|
||||
// difference from an epoch-per-write fence: the single owner re-ships the SAME
|
||||
// object many times at its STABLE lease round (every PushLoop tick), so
|
||||
// same-round overwrites by the current owner MUST be admitted; only a LOWER
|
||||
// round — the signature of a writer whose lease was taken over and whose
|
||||
// successor already advanced the recorded round — is refused. A new owner seals
|
||||
// the object to its higher round exactly once (a claim Put at recorded+1); from
|
||||
// that instant every lower-round ship by the old owner is fenced forever.
|
||||
//
|
||||
// # No admit/write window
|
||||
//
|
||||
// The round header and the payload live in ONE object and advance in ONE
|
||||
// conditional write. There is therefore no interval between "round admitted" and
|
||||
// "data written" for a second writer to slip into — the exact window a two-object
|
||||
// design (separate lease record + data object) would have to close by other
|
||||
// means. Because a takeover's claim Put is a compare-and-set read-modify-write
|
||||
// conditioned on the version just read, an owner that slips a write in between the
|
||||
// successor's read and its claim only makes the claim retry (re-reading the newer
|
||||
// snapshot), so the successor always carries forward the latest durably-shipped
|
||||
// bytes before it seals — no acknowledged data is lost.
|
||||
//
|
||||
// FencedStore composes over the ConditionalStore seam (S3 If-Match /
|
||||
// GCS generation-match) and stays dependency-free; the concrete CAS store lives
|
||||
// with whoever owns an S3/SeaweedFS client.
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/binary"
|
||||
"errors"
|
||||
"fmt"
|
||||
)
|
||||
|
||||
var (
|
||||
// ErrNotFound is returned by ConditionalStore.Get for a key never written.
|
||||
// FencedStore treats an absent object as recorded round 0, so any real lease
|
||||
// round (>= 1) is admissible against an empty slot.
|
||||
ErrNotFound = errors.New("replica: fenced object not found")
|
||||
|
||||
// ErrStaleRound is returned when a Put's round is below the round already
|
||||
// recorded for the object — the deposed/partitioned-writer rejection. The
|
||||
// object is left byte-for-byte untouched. This is the invariant that closes
|
||||
// the split-brain double-write: once a successor advances the recorded round,
|
||||
// the predecessor can never write again.
|
||||
ErrStaleRound = errors.New("replica: write round is below the recorded round (writer fenced)")
|
||||
|
||||
// ErrConflict is returned when the atomic conditional write lost a race to a
|
||||
// concurrent writer (the object changed between our read and our write).
|
||||
// Put retries internally; ErrConflict escaping Put means the bounded retries
|
||||
// were exhausted under sustained contention, not that a caller should blindly
|
||||
// retry with the same round.
|
||||
ErrConflict = errors.New("replica: conditional write lost a concurrent race")
|
||||
)
|
||||
|
||||
// maxCASAttempts bounds Put's read-check-CAS retry loop so a pathologically hot
|
||||
// object fails loudly rather than spinning forever. One retry suffices for any
|
||||
// single genuine race; further attempts only absorb repeated contention.
|
||||
const maxCASAttempts = 8
|
||||
|
||||
// ConditionalStore is the atomic compare-and-set object surface the fence needs:
|
||||
// the S3 If-Match / GCS generation-match primitive, nothing more. PutIfVersion
|
||||
// MUST evaluate its precondition atomically, server-side; a client-side
|
||||
// "Get, compare in Go, then Put" is NOT a valid implementation — that
|
||||
// non-atomicity is the exact race this fence exists to close. The concrete
|
||||
// implementation lives with the caller that owns an S3/SeaweedFS client (e.g.
|
||||
// cloud's minio-go If-Match store); this package stays dependency-free over the
|
||||
// seam, and the in-memory fake in fence_test.go models the same server-side CAS.
|
||||
type ConditionalStore interface {
|
||||
// Get returns the object bytes and the store's authoritative version token
|
||||
// (ETag/generation) for key, or a wrapped ErrNotFound when key is absent.
|
||||
Get(ctx context.Context, key string) (data []byte, version string, err error)
|
||||
// PutIfVersion writes data at key iff the store's CURRENT version for key
|
||||
// still equals expectVersion ("" means the object must not exist yet — a
|
||||
// create-only put, i.e. If-None-Match: *) and returns the new version. A
|
||||
// failed precondition is reported as a wrapped ErrConflict.
|
||||
PutIfVersion(ctx context.Context, key string, data []byte, expectVersion string) (newVersion string, err error)
|
||||
}
|
||||
|
||||
// FencedStore is a round-fenced writer over a ConditionalStore: one object per
|
||||
// key, a monotone round admitted only when at least the recorded one.
|
||||
type FencedStore struct{ store ConditionalStore }
|
||||
|
||||
// NewFencedStore builds a FencedStore over an atomic-CAS store.
|
||||
func NewFencedStore(store ConditionalStore) *FencedStore { return &FencedStore{store: store} }
|
||||
|
||||
// fenceMagic prefixes every record so a foreign object at the same key is
|
||||
// detected (errCorrupt) rather than silently misparsed as an absurd round.
|
||||
const fenceMagic = "rfnc1\x00"
|
||||
|
||||
func encode(round uint64, payload []byte) []byte {
|
||||
buf := make([]byte, 0, len(fenceMagic)+8+len(payload))
|
||||
buf = append(buf, fenceMagic...)
|
||||
var r [8]byte
|
||||
binary.BigEndian.PutUint64(r[:], round)
|
||||
buf = append(buf, r[:]...)
|
||||
return append(buf, payload...)
|
||||
}
|
||||
|
||||
// errCorrupt is returned when a stored object does not decode as a record this
|
||||
// package wrote — fail closed rather than invent a round from arbitrary bytes.
|
||||
var errCorrupt = errors.New("replica: stored object is not a valid fenced record")
|
||||
|
||||
func decode(b []byte) (round uint64, payload []byte, err error) {
|
||||
m := len(fenceMagic)
|
||||
if len(b) < m+8 || string(b[:m]) != fenceMagic {
|
||||
return 0, nil, errCorrupt
|
||||
}
|
||||
return binary.BigEndian.Uint64(b[m : m+8]), b[m+8:], nil
|
||||
}
|
||||
|
||||
// Put admits and (atomically with admission) writes payload for key at round. It
|
||||
// rejects with ErrStaleRound when round is below the recorded round — the fence —
|
||||
// before any write is attempted, then advances the header and appends the payload
|
||||
// in a single PutIfVersion conditioned on the version read THIS attempt (never a
|
||||
// cached one). If a concurrent writer's write lands first, our CAS fails
|
||||
// (ErrConflict) and we retry: re-read the now-authoritative round, re-run the
|
||||
// strict check (a lower-round racer now loses as ErrStaleRound; an equal-or-higher
|
||||
// one retries the CAS against the new version and, absent further contention,
|
||||
// wins). The store is the sole arbiter of "what round currently owns this key."
|
||||
func (f *FencedStore) Put(ctx context.Context, key string, payload []byte, round uint64) error {
|
||||
if f == nil || f.store == nil {
|
||||
return fmt.Errorf("replica: nil fenced store")
|
||||
}
|
||||
if key == "" {
|
||||
return fmt.Errorf("replica: fenced put requires a key")
|
||||
}
|
||||
var lastErr error
|
||||
for attempt := 0; attempt < maxCASAttempts; attempt++ {
|
||||
recorded, version, err := f.current(ctx, key)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if round < recorded {
|
||||
return fmt.Errorf("%w: key=%s round=%d recorded=%d", ErrStaleRound, key, round, recorded)
|
||||
}
|
||||
if _, err := f.store.PutIfVersion(ctx, key, encode(round, payload), version); err != nil {
|
||||
if errors.Is(err, ErrConflict) {
|
||||
lastErr = err
|
||||
continue // re-read the winner's state and retry.
|
||||
}
|
||||
return fmt.Errorf("replica: fenced put %s: %w", key, err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
if lastErr == nil {
|
||||
lastErr = ErrConflict
|
||||
}
|
||||
return fmt.Errorf("%w: exhausted %d attempts fencing %s at round %d", lastErr, maxCASAttempts, key, round)
|
||||
}
|
||||
|
||||
// CarryForward is the safe TAKEOVER seal: it reads the latest durably-stored
|
||||
// payload for key and re-seals it UNCHANGED at round, invoking hydrate on that
|
||||
// payload first. It exists because a successor must never seal a snapshot it read
|
||||
// BEFORE a predecessor's last landed ship, or that acknowledged write would be
|
||||
// clobbered. The read and the seal are one compare-and-set on the same version:
|
||||
// if a predecessor write lands in between, the CAS conflicts and CarryForward
|
||||
// retries — re-reading the newer payload, re-hydrating, and re-sealing IT — so the
|
||||
// successor always advances past exactly the state it carried forward, never a
|
||||
// stale one. After CarryForward returns, every predecessor write at a round below
|
||||
// `round` is fenced.
|
||||
//
|
||||
// round is the successor's lease round (issued monotone by the Fencer); it must be
|
||||
// >= the recorded round or CarryForward returns ErrStaleRound (a newer owner
|
||||
// already passed us — we lost the race and must step aside). hydrate is the
|
||||
// caller's restore-into-local-DB step; it may run more than once across retries,
|
||||
// so it must be idempotent (a whole-snapshot restore is).
|
||||
func (f *FencedStore) CarryForward(ctx context.Context, key string, round uint64, hydrate func(payload []byte) error) error {
|
||||
if f == nil || f.store == nil {
|
||||
return fmt.Errorf("replica: nil fenced store")
|
||||
}
|
||||
if key == "" {
|
||||
return fmt.Errorf("replica: carry-forward requires a key")
|
||||
}
|
||||
var lastErr error
|
||||
for attempt := 0; attempt < maxCASAttempts; attempt++ {
|
||||
data, version, err := f.store.Get(ctx, key)
|
||||
var recorded uint64
|
||||
var payload []byte
|
||||
switch {
|
||||
case errors.Is(err, ErrNotFound):
|
||||
recorded, payload, version = 0, nil, ""
|
||||
case err != nil:
|
||||
return fmt.Errorf("replica: carry-forward read %s: %w", key, err)
|
||||
default:
|
||||
if recorded, payload, err = decode(data); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
if round < recorded {
|
||||
return fmt.Errorf("%w: key=%s round=%d recorded=%d", ErrStaleRound, key, round, recorded)
|
||||
}
|
||||
if hydrate != nil {
|
||||
if err := hydrate(payload); err != nil {
|
||||
return fmt.Errorf("replica: carry-forward hydrate %s: %w", key, err)
|
||||
}
|
||||
}
|
||||
if _, err := f.store.PutIfVersion(ctx, key, encode(round, payload), version); err != nil {
|
||||
if errors.Is(err, ErrConflict) {
|
||||
lastErr = err
|
||||
continue // a predecessor's write landed; re-read it and carry it forward.
|
||||
}
|
||||
return fmt.Errorf("replica: carry-forward seal %s: %w", key, err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
if lastErr == nil {
|
||||
lastErr = ErrConflict
|
||||
}
|
||||
return fmt.Errorf("%w: exhausted %d attempts carrying forward %s at round %d", lastErr, maxCASAttempts, key, round)
|
||||
}
|
||||
|
||||
// Get returns the fenced payload and the round it was written at, or ErrNotFound
|
||||
// when key is absent. A reader restores the payload; a successor reads the round
|
||||
// to learn what to advance past on takeover.
|
||||
func (f *FencedStore) Get(ctx context.Context, key string) ([]byte, uint64, error) {
|
||||
data, _, err := f.store.Get(ctx, key)
|
||||
if err != nil {
|
||||
return nil, 0, err
|
||||
}
|
||||
round, payload, err := decode(data)
|
||||
if err != nil {
|
||||
return nil, 0, err
|
||||
}
|
||||
return payload, round, nil
|
||||
}
|
||||
|
||||
// Round returns the round currently recorded for key, or 0 if the object is
|
||||
// absent — the value a new owner increments by one to claim the writer role.
|
||||
func (f *FencedStore) Round(ctx context.Context, key string) (uint64, error) {
|
||||
round, _, err := f.current(ctx, key)
|
||||
return round, err
|
||||
}
|
||||
|
||||
// current returns the recorded round (0 if absent) and the store version token to
|
||||
// condition the next write on.
|
||||
func (f *FencedStore) current(ctx context.Context, key string) (round uint64, version string, err error) {
|
||||
data, version, err := f.store.Get(ctx, key)
|
||||
switch {
|
||||
case errors.Is(err, ErrNotFound):
|
||||
return 0, "", nil
|
||||
case err != nil:
|
||||
return 0, "", fmt.Errorf("replica: read fenced object %s: %w", key, err)
|
||||
}
|
||||
round, _, err = decode(data)
|
||||
if err != nil {
|
||||
return 0, "", err
|
||||
}
|
||||
return round, version, nil
|
||||
}
|
||||
@@ -1,364 +0,0 @@
|
||||
// Copyright 2025 Hanzo AI Inc. All Rights Reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
|
||||
package replica
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"strconv"
|
||||
"sync"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// fakeCAS is an in-process model of a keyed object store's atomic conditional
|
||||
// write (S3 If-Match / GCS generation-match): one independent (data, generation)
|
||||
// slot per key. A single mutex makes the compare-and-set in PutIfVersion
|
||||
// indivisible from every caller's point of view — exactly like the real store's
|
||||
// server-side atomicity, NOT a client-side "read then write" pair with a gap.
|
||||
//
|
||||
// beforeCAS, when set, runs OUTSIDE the lock right before PutIfVersion acquires
|
||||
// it, so a test can force two goroutines to finish their Get before either write
|
||||
// proceeds — turning a timing-dependent race into a deterministic, repeatable one.
|
||||
type fakeCAS struct {
|
||||
mu sync.Mutex
|
||||
objects map[string]*fakeObj
|
||||
beforeCAS func(key string)
|
||||
}
|
||||
|
||||
type fakeObj struct {
|
||||
data []byte
|
||||
ver int // generation counter; the version token is its decimal string.
|
||||
}
|
||||
|
||||
func newFakeCAS() *fakeCAS { return &fakeCAS{objects: map[string]*fakeObj{}} }
|
||||
|
||||
func (s *fakeCAS) Get(_ context.Context, key string) ([]byte, string, error) {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
obj, ok := s.objects[key]
|
||||
if !ok {
|
||||
return nil, "", ErrNotFound
|
||||
}
|
||||
return append([]byte(nil), obj.data...), strconv.Itoa(obj.ver), nil
|
||||
}
|
||||
|
||||
func (s *fakeCAS) PutIfVersion(_ context.Context, key string, data []byte, expectVersion string) (string, error) {
|
||||
if s.beforeCAS != nil {
|
||||
s.beforeCAS(key)
|
||||
}
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
obj, ok := s.objects[key]
|
||||
cur := ""
|
||||
if ok {
|
||||
cur = strconv.Itoa(obj.ver)
|
||||
}
|
||||
if cur != expectVersion {
|
||||
return "", fmt.Errorf("fakeCAS: %w: have %q want %q", ErrConflict, cur, expectVersion)
|
||||
}
|
||||
if !ok {
|
||||
obj = &fakeObj{}
|
||||
s.objects[key] = obj
|
||||
}
|
||||
obj.ver++
|
||||
obj.data = append([]byte(nil), data...)
|
||||
return strconv.Itoa(obj.ver), nil
|
||||
}
|
||||
|
||||
// barrierAt2 blocks the first two callers until both arrive, then releases both
|
||||
// at once; later calls (retries) pass straight through.
|
||||
func barrierAt2() func(string) {
|
||||
var mu sync.Mutex
|
||||
arrived := 0
|
||||
release := make(chan struct{})
|
||||
return func(string) {
|
||||
mu.Lock()
|
||||
arrived++
|
||||
n := arrived
|
||||
mu.Unlock()
|
||||
if n == 2 {
|
||||
close(release)
|
||||
}
|
||||
<-release
|
||||
}
|
||||
}
|
||||
|
||||
func mustPut(t *testing.T, f *FencedStore, key, payload string, round uint64) {
|
||||
t.Helper()
|
||||
if err := f.Put(context.Background(), key, []byte(payload), round); err != nil {
|
||||
t.Fatalf("Put(%s, round=%d): unexpected error: %v", key, round, err)
|
||||
}
|
||||
}
|
||||
|
||||
func read(t *testing.T, f *FencedStore, key string) (uint64, string) {
|
||||
t.Helper()
|
||||
payload, round, err := f.Get(context.Background(), key)
|
||||
if err != nil {
|
||||
t.Fatalf("Get(%s): %v", key, err)
|
||||
}
|
||||
return round, string(payload)
|
||||
}
|
||||
|
||||
// TestFencedPutRejectsLowerRound is the deposed-writer proof: once a successor
|
||||
// advanced the recorded round, the predecessor's ship at its (now-lower) round is
|
||||
// refused with ErrStaleRound and the object is left untouched.
|
||||
func TestFencedPutRejectsLowerRound(t *testing.T) {
|
||||
f := NewFencedStore(newFakeCAS())
|
||||
mustPut(t, f, "orgs/acme/kv.db", "v-round6", 6) // new owner sealed at round 6.
|
||||
|
||||
err := f.Put(context.Background(), "orgs/acme/kv.db", []byte("v-stale"), 3)
|
||||
if !errors.Is(err, ErrStaleRound) {
|
||||
t.Fatalf("lower-round ship: got %v, want ErrStaleRound", err)
|
||||
}
|
||||
if round, payload := read(t, f, "orgs/acme/kv.db"); round != 6 || payload != "v-round6" {
|
||||
t.Fatalf("object mutated by a fenced write: round=%d payload=%q, want 6/v-round6", round, payload)
|
||||
}
|
||||
}
|
||||
|
||||
// TestFencedPutAdmitsSameRoundReship proves the `>=` rule: the CURRENT owner
|
||||
// re-ships the same object at its stable lease round many times, and every such
|
||||
// overwrite is admitted (the last one wins) — this is why the fence uses `>=`,
|
||||
// not the one-shot `>`.
|
||||
func TestFencedPutAdmitsSameRoundReship(t *testing.T) {
|
||||
f := NewFencedStore(newFakeCAS())
|
||||
mustPut(t, f, "orgs/acme/kv.db", "tick-1", 4)
|
||||
mustPut(t, f, "orgs/acme/kv.db", "tick-2", 4)
|
||||
mustPut(t, f, "orgs/acme/kv.db", "tick-3", 4)
|
||||
if round, payload := read(t, f, "orgs/acme/kv.db"); round != 4 || payload != "tick-3" {
|
||||
t.Fatalf("same-round re-ship: round=%d payload=%q, want 4/tick-3", round, payload)
|
||||
}
|
||||
}
|
||||
|
||||
// TestFencedPutHigherRoundLocksOutLower is the handoff proof: a successor at a
|
||||
// strictly higher round is admitted (sealing the object), after which the
|
||||
// predecessor at the old round is fenced forever.
|
||||
func TestFencedPutHigherRoundLocksOutLower(t *testing.T) {
|
||||
f := NewFencedStore(newFakeCAS())
|
||||
mustPut(t, f, "orgs/acme/kv.db", "old-owner", 5) // predecessor at round 5.
|
||||
mustPut(t, f, "orgs/acme/kv.db", "new-owner", 6) // successor claims round 6.
|
||||
|
||||
if err := f.Put(context.Background(), "orgs/acme/kv.db", []byte("old-owner-late"), 5); !errors.Is(err, ErrStaleRound) {
|
||||
t.Fatalf("predecessor after handoff: got %v, want ErrStaleRound", err)
|
||||
}
|
||||
if round, payload := read(t, f, "orgs/acme/kv.db"); round != 6 || payload != "new-owner" {
|
||||
t.Fatalf("after handoff: round=%d payload=%q, want 6/new-owner", round, payload)
|
||||
}
|
||||
}
|
||||
|
||||
// TestFencedPutConcurrentSameRoundOwnerReship: the current owner ships the same
|
||||
// object concurrently from two goroutines at its lease round. Both must succeed
|
||||
// (idempotent overwrite) and the object must decode cleanly to ONE of the two
|
||||
// payloads — never a byte-mix.
|
||||
func TestFencedPutConcurrentSameRoundOwnerReship(t *testing.T) {
|
||||
store := newFakeCAS()
|
||||
store.beforeCAS = barrierAt2()
|
||||
f := NewFencedStore(store)
|
||||
|
||||
results := make([]error, 2)
|
||||
var wg sync.WaitGroup
|
||||
wg.Add(2)
|
||||
for i := 0; i < 2; i++ {
|
||||
i := i
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
results[i] = f.Put(context.Background(), "orgs/acme/kv.db", []byte(fmt.Sprintf("payload-%d", i)), 7)
|
||||
}()
|
||||
}
|
||||
wg.Wait()
|
||||
|
||||
for i, err := range results {
|
||||
if err != nil {
|
||||
t.Fatalf("same-round concurrent ship %d failed: %v (owner re-ship must be idempotent)", i, err)
|
||||
}
|
||||
}
|
||||
round, payload := read(t, f, "orgs/acme/kv.db")
|
||||
if round != 7 || (payload != "payload-0" && payload != "payload-1") {
|
||||
t.Fatalf("object corrupted by concurrent same-round ship: round=%d payload=%q", round, payload)
|
||||
}
|
||||
}
|
||||
|
||||
// TestFencedPutConcurrentHigherVsLower is the partition proof, made deterministic:
|
||||
// a minority writer (stuck at round 5 because it cannot reach the linearizable
|
||||
// source to advance) and the majority's new owner (round 6) race their ships, both
|
||||
// having read the same prior state. The SAFETY invariant, whichever wins the first
|
||||
// CAS: the higher round ALWAYS wins the final durable state (the successor's round
|
||||
// 6 either lands directly, or retries past the minority's transient round-5 write),
|
||||
// and round 5 can never be the surviving state. The minority's Put may return nil
|
||||
// (its write landed transiently before being overwritten) — that is not unsafe
|
||||
// here because the minority never ACKs on a bare ship; acknowledgement is gated on
|
||||
// the full lease+idempotency protocol (see cloud's handoff test). What must never
|
||||
// happen is round 5 surviving.
|
||||
func TestFencedPutConcurrentHigherVsLower(t *testing.T) {
|
||||
store := newFakeCAS()
|
||||
f := NewFencedStore(store)
|
||||
mustPut(t, f, "orgs/acme/kv.db", "base", 5) // both racers read this as the prior state.
|
||||
|
||||
store.beforeCAS = barrierAt2()
|
||||
results := map[uint64]error{}
|
||||
var mu sync.Mutex
|
||||
var wg sync.WaitGroup
|
||||
for _, round := range []uint64{5, 6} { // 5 = fenced minority, 6 = majority successor.
|
||||
round := round
|
||||
wg.Add(1)
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
err := f.Put(context.Background(), "orgs/acme/kv.db", []byte(fmt.Sprintf("round-%d", round)), round)
|
||||
mu.Lock()
|
||||
results[round] = err
|
||||
mu.Unlock()
|
||||
}()
|
||||
}
|
||||
wg.Wait()
|
||||
|
||||
if results[6] != nil {
|
||||
t.Fatalf("majority successor (round 6) must always win, got %v", results[6])
|
||||
}
|
||||
if round, payload := read(t, f, "orgs/acme/kv.db"); round != 6 || payload != "round-6" {
|
||||
t.Fatalf("partition resolved wrong: final state round=%d payload=%q, want 6/round-6 (higher round must survive)", round, payload)
|
||||
}
|
||||
}
|
||||
|
||||
// TestCarryForwardPreservesLastWriteThenFences is the safe-handoff proof at the
|
||||
// store layer: a predecessor's last landed ship (X) is carried forward by the
|
||||
// successor's takeover seal at the higher round — never lost — and every
|
||||
// subsequent predecessor ship is then fenced.
|
||||
func TestCarryForwardPreservesLastWriteThenFences(t *testing.T) {
|
||||
f := NewFencedStore(newFakeCAS())
|
||||
key := "orgs/acme/kv.db"
|
||||
mustPut(t, f, key, "base", 5)
|
||||
mustPut(t, f, key, "base+X", 5) // predecessor's last acknowledged ship at round 5.
|
||||
|
||||
// Successor takes over at lease round 6, carrying forward the LATEST payload and
|
||||
// hydrating it (proving it sees X).
|
||||
var hydrated string
|
||||
if err := f.CarryForward(context.Background(), key, 6, func(p []byte) error { hydrated = string(p); return nil }); err != nil {
|
||||
t.Fatalf("CarryForward: %v", err)
|
||||
}
|
||||
if hydrated != "base+X" {
|
||||
t.Fatalf("successor hydrated %q, want base+X (predecessor's last write must not be lost)", hydrated)
|
||||
}
|
||||
if round, payload := read(t, f, key); round != 6 || payload != "base+X" {
|
||||
t.Fatalf("after takeover: round=%d payload=%q, want 6/base+X", round, payload)
|
||||
}
|
||||
// Predecessor, still running at round 5, is now fenced.
|
||||
if err := f.Put(context.Background(), key, []byte("base+X+Y"), 5); !errors.Is(err, ErrStaleRound) {
|
||||
t.Fatalf("predecessor after takeover: got %v, want ErrStaleRound", err)
|
||||
}
|
||||
}
|
||||
|
||||
// TestCarryForwardVsConcurrentPredecessorShipNoLoss is the no-acknowledged-write-
|
||||
// lost proof under a genuine race (lease violation: both writers active). A
|
||||
// predecessor ships X at round 5 while the successor takes over at round 6, both
|
||||
// having read the same prior version. The store's single-object CAS totally orders
|
||||
// them, so the biconditional MUST hold: the successor's round-6 state contains X
|
||||
// IFF the predecessor's ship succeeded (nil). There is no interleaving where the
|
||||
// predecessor's ship returns nil (would be acknowledged) yet X is lost.
|
||||
func TestCarryForwardVsConcurrentPredecessorShipNoLoss(t *testing.T) {
|
||||
store := newFakeCAS()
|
||||
f := NewFencedStore(store)
|
||||
key := "orgs/acme/kv.db"
|
||||
mustPut(t, f, key, "base", 5) // both read this version.
|
||||
|
||||
store.beforeCAS = barrierAt2()
|
||||
var predErr, succErr error
|
||||
var wg sync.WaitGroup
|
||||
wg.Add(2)
|
||||
go func() { defer wg.Done(); predErr = f.Put(context.Background(), key, []byte("base+X"), 5) }()
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
succErr = f.CarryForward(context.Background(), key, 6, nil)
|
||||
}()
|
||||
wg.Wait()
|
||||
|
||||
if succErr != nil {
|
||||
t.Fatalf("successor takeover must succeed, got %v", succErr)
|
||||
}
|
||||
round, payload := read(t, f, key)
|
||||
if round != 6 {
|
||||
t.Fatalf("final round=%d, want 6", round)
|
||||
}
|
||||
containsX := payload == "base+X"
|
||||
predAcked := predErr == nil
|
||||
if containsX != predAcked {
|
||||
t.Fatalf("acknowledged-write-lost: predecessor acked=%v but final-contains-X=%v (payload=%q) — must be equal",
|
||||
predAcked, containsX, payload)
|
||||
}
|
||||
}
|
||||
|
||||
// TestFencedGetRoundTripAndAbsent covers the read path: payload+round round-trip,
|
||||
// absent key is ErrNotFound, and Round reports 0 for an absent key (the value a
|
||||
// first owner increments to 1 to claim).
|
||||
func TestFencedGetRoundTripAndAbsent(t *testing.T) {
|
||||
f := NewFencedStore(newFakeCAS())
|
||||
|
||||
if _, _, err := f.Get(context.Background(), "orgs/absent/kv.db"); !errors.Is(err, ErrNotFound) {
|
||||
t.Fatalf("absent Get: got %v, want ErrNotFound", err)
|
||||
}
|
||||
if r, err := f.Round(context.Background(), "orgs/absent/kv.db"); err != nil || r != 0 {
|
||||
t.Fatalf("absent Round: got (%d,%v), want (0,nil)", r, err)
|
||||
}
|
||||
|
||||
mustPut(t, f, "orgs/acme/kv.db", "hello", 9)
|
||||
payload, round, err := f.Get(context.Background(), "orgs/acme/kv.db")
|
||||
if err != nil || string(payload) != "hello" || round != 9 {
|
||||
t.Fatalf("round-trip: payload=%q round=%d err=%v, want hello/9/nil", payload, round, err)
|
||||
}
|
||||
}
|
||||
|
||||
// TestFencedPutCreateOnlyFirstWrite proves the absent slot is a create-only put
|
||||
// (expectVersion ""): two racers both seeing the empty slot cannot both create it.
|
||||
func TestFencedPutCreateOnlyFirstWrite(t *testing.T) {
|
||||
store := newFakeCAS()
|
||||
store.beforeCAS = barrierAt2()
|
||||
f := NewFencedStore(store)
|
||||
|
||||
results := make([]error, 2)
|
||||
var wg sync.WaitGroup
|
||||
wg.Add(2)
|
||||
for i := 0; i < 2; i++ {
|
||||
i := i
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
results[i] = f.Put(context.Background(), "orgs/new/kv.db", []byte(fmt.Sprintf("first-%d", i)), 1)
|
||||
}()
|
||||
}
|
||||
wg.Wait()
|
||||
|
||||
// Same round (1) for both: one creates, the loser retries the CAS and — since
|
||||
// 1 >= recorded 1 — is admitted as an idempotent overwrite. Both succeed; the
|
||||
// object is one clean payload at round 1. (A create-only slot cannot be
|
||||
// double-created; the retry path is what makes the loser converge.)
|
||||
for i, err := range results {
|
||||
if err != nil {
|
||||
t.Fatalf("create-race %d: %v", i, err)
|
||||
}
|
||||
}
|
||||
if round, payload := read(t, f, "orgs/new/kv.db"); round != 1 || (payload != "first-0" && payload != "first-1") {
|
||||
t.Fatalf("create race left object round=%d payload=%q", round, payload)
|
||||
}
|
||||
}
|
||||
|
||||
// TestFencedRejectsCorruptForeignObject proves fail-closed decoding: a non-fence
|
||||
// object sitting at the key is not misread as some absurd round — Get and Put both
|
||||
// surface a corruption error rather than guess.
|
||||
func TestFencedRejectsCorruptForeignObject(t *testing.T) {
|
||||
store := newFakeCAS()
|
||||
// A foreign writer put raw bytes (no fence header) at the key.
|
||||
if _, err := store.PutIfVersion(context.Background(), "orgs/acme/kv.db", []byte("not-a-fence-record"), ""); err != nil {
|
||||
t.Fatalf("seed foreign object: %v", err)
|
||||
}
|
||||
f := NewFencedStore(store)
|
||||
|
||||
if _, _, err := f.Get(context.Background(), "orgs/acme/kv.db"); !errors.Is(err, errCorrupt) {
|
||||
t.Fatalf("Get of foreign object: got %v, want errCorrupt", err)
|
||||
}
|
||||
if err := f.Put(context.Background(), "orgs/acme/kv.db", []byte("x"), 1); !errors.Is(err, errCorrupt) {
|
||||
t.Fatalf("Put over foreign object: got %v, want errCorrupt", err)
|
||||
}
|
||||
}
|
||||
|
||||
var _ ConditionalStore = (*fakeCAS)(nil)
|
||||
@@ -1,197 +0,0 @@
|
||||
// Copyright 2025 Hanzo AI Inc. All Rights Reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
|
||||
// Package replica is the ONE shared HA-SQLite substrate every Hanzo service uses
|
||||
// (HIP-0107): per-org SQLite on the local disk for speed, its durable copy in
|
||||
// hanzoai/vfs (SeaweedFS-backed object store), a deterministic single-writer
|
||||
// election (Rendezvous/HRW over IAM membership — no coordinator, no lock service,
|
||||
// no Postgres, no Redis), and per-org at-rest encryption via KMS.
|
||||
//
|
||||
// It was proven inside hanzoai/cloud (internal/org) and is promoted here so EVERY
|
||||
// service — visor, iam, commerce, base, and every Base adopter — imports the SAME
|
||||
// code instead of re-inventing durability. The adoption contract at each service's
|
||||
// per-org store seam:
|
||||
//
|
||||
// hydrate-on-open : Pull() the latest snapshot before first use of an org DB.
|
||||
// single-writer : ha.IsOwner(key, self, members) gates writes (github.com/hanzoai/ha,
|
||||
// the election primitive); a non-owner serves stale-tolerant reads
|
||||
// (Pull-then-read) and forwards writes.
|
||||
// post-commit ship: the owner runs PushLoop (or Push after a write burst); the
|
||||
// durable copy in vfs means a lost pod loses no committed data.
|
||||
package replica
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"fmt"
|
||||
"path"
|
||||
"sync"
|
||||
"time"
|
||||
)
|
||||
|
||||
// Store is the object-store surface the Replicator needs — satisfied natively by
|
||||
// hanzoai/vfs (SeaweedFS-backed, in-process). NO minio, NO external S3 SDK.
|
||||
// version is a content hash so readers skip redundant pulls.
|
||||
type Store interface {
|
||||
Put(ctx context.Context, key string, data []byte) error
|
||||
Get(ctx context.Context, key string) (data []byte, version string, err error)
|
||||
Version(ctx context.Context, key string) (version string, err error)
|
||||
}
|
||||
|
||||
// DB is the local per-org SQLite the owner writes and readers restore into.
|
||||
// Snapshot returns a consistent (WAL-checkpointed) copy; Restore replaces the
|
||||
// local bytes atomically. Satisfied by *SQLiteDB (sqlite.go).
|
||||
type DB interface {
|
||||
Snapshot(ctx context.Context) ([]byte, error)
|
||||
Restore(ctx context.Context, data []byte) error
|
||||
}
|
||||
|
||||
// Cipher seals/opens a snapshot with an org-bound key (envelope encryption via
|
||||
// the KMS master). Optional — omit it for local dev (plaintext object).
|
||||
type Cipher interface {
|
||||
Seal(org string, plaintext []byte) ([]byte, error)
|
||||
Open(org string, ciphertext []byte) ([]byte, error)
|
||||
}
|
||||
|
||||
// DBPath is the canonical object-store location of one of an org's SQLite DBs
|
||||
// (HIP-0302 layout). Every replica computes the same path for the same inputs.
|
||||
//
|
||||
// org root: DBPath("acme", "", "iam") -> orgs/acme/iam.db
|
||||
// project: DBPath("acme", "projects/site", "base") -> orgs/acme/projects/site/base.db
|
||||
// user: DBPath("acme", "users/dave", "kv") -> orgs/acme/users/dave/kv.db
|
||||
func DBPath(orgID, scope, service string) string {
|
||||
if scope == "" {
|
||||
return path.Join("orgs", orgID, service+".db")
|
||||
}
|
||||
return path.Join("orgs", orgID, scope, service+".db")
|
||||
}
|
||||
|
||||
// Replicator binds ONE per-org SQLite (at a DBPath key) to its object-store slot.
|
||||
//
|
||||
// owner replica (IsOwner true): Push() on a timer / after a write burst
|
||||
// reader replica (IsOwner false): Pull() before a stale-tolerant read
|
||||
//
|
||||
// It does NOT decide ownership — that is Owner()/IsOwner(). Push/Get/Put are
|
||||
// whole-object (a WAL-checkpointed snapshot), the proven HIP-0107 mechanism.
|
||||
type Replicator struct {
|
||||
key string
|
||||
store Store
|
||||
db DB
|
||||
|
||||
cipher Cipher
|
||||
org string // org id — the AAD / key-derivation input for the cipher
|
||||
|
||||
mu sync.Mutex
|
||||
lastVer string // last object version pushed or pulled — skip redundant pulls
|
||||
}
|
||||
|
||||
// Option configures a Replicator.
|
||||
type Option func(*Replicator)
|
||||
|
||||
// WithEncryption seals every pushed snapshot with the org's per-org key so the
|
||||
// object is ciphertext; orgID is bound as AAD, so a blob cannot be replayed under
|
||||
// another org. Omit it and the DB is stored in the clear (local dev only).
|
||||
func WithEncryption(c Cipher, orgID string) Option {
|
||||
return func(r *Replicator) { r.cipher, r.org = c, orgID }
|
||||
}
|
||||
|
||||
// NewReplicator binds the DB at key (from DBPath) to its store slot.
|
||||
func NewReplicator(key string, store Store, db DB, opts ...Option) *Replicator {
|
||||
r := &Replicator{key: key, store: store, db: db}
|
||||
for _, o := range opts {
|
||||
o(r)
|
||||
}
|
||||
return r
|
||||
}
|
||||
|
||||
// Key is this replicator's object-store location.
|
||||
func (r *Replicator) Key() string { return r.key }
|
||||
|
||||
// Push snapshots the local DB (WAL-checkpointed) and writes it to the object
|
||||
// store. Called by the OWNER — the single writer. The durable copy lives in vfs,
|
||||
// so a lost pod loses no committed data.
|
||||
func (r *Replicator) Push(ctx context.Context) error {
|
||||
data, err := r.db.Snapshot(ctx)
|
||||
if err != nil {
|
||||
return fmt.Errorf("replica: snapshot %s: %w", r.key, err)
|
||||
}
|
||||
if r.cipher != nil {
|
||||
if data, err = r.cipher.Seal(r.org, data); err != nil {
|
||||
return fmt.Errorf("replica: seal %s: %w", r.key, err)
|
||||
}
|
||||
}
|
||||
if err := r.store.Put(ctx, r.key, data); err != nil {
|
||||
return fmt.Errorf("replica: put %s: %w", r.key, err)
|
||||
}
|
||||
r.mu.Lock()
|
||||
r.lastVer = version(data) // version over the STORED (sealed) bytes
|
||||
r.mu.Unlock()
|
||||
return nil
|
||||
}
|
||||
|
||||
// Pull downloads the latest snapshot and restores it into the local DB — called
|
||||
// by a READER before serving stale-tolerant reads, or by a NEW OWNER on handover
|
||||
// to load the last committed state. A no-op when the remote is unchanged since
|
||||
// our last push/pull.
|
||||
func (r *Replicator) Pull(ctx context.Context) (changed bool, err error) {
|
||||
ver, err := r.store.Version(ctx, r.key)
|
||||
if err != nil {
|
||||
return false, fmt.Errorf("replica: version %s: %w", r.key, err)
|
||||
}
|
||||
r.mu.Lock()
|
||||
current := ver != "" && ver == r.lastVer
|
||||
r.mu.Unlock()
|
||||
if current {
|
||||
return false, nil
|
||||
}
|
||||
data, ver, err := r.store.Get(ctx, r.key)
|
||||
if err != nil {
|
||||
return false, fmt.Errorf("replica: get %s: %w", r.key, err)
|
||||
}
|
||||
if r.cipher != nil {
|
||||
if data, err = r.cipher.Open(r.org, data); err != nil {
|
||||
return false, fmt.Errorf("replica: open %s: %w", r.key, err)
|
||||
}
|
||||
}
|
||||
if err := r.db.Restore(ctx, data); err != nil {
|
||||
return false, fmt.Errorf("replica: restore %s: %w", r.key, err)
|
||||
}
|
||||
r.mu.Lock()
|
||||
r.lastVer = ver
|
||||
r.mu.Unlock()
|
||||
return true, nil
|
||||
}
|
||||
|
||||
// PushLoop runs Push on an interval until ctx cancel — the owner's background
|
||||
// WAL-shipper. On error it keeps the last good remote copy and retries next tick
|
||||
// (fire-and-forget durability; never blocks writes).
|
||||
func (r *Replicator) PushLoop(ctx context.Context, every time.Duration) {
|
||||
if every <= 0 {
|
||||
every = 10 * time.Second
|
||||
}
|
||||
t := time.NewTicker(every)
|
||||
defer t.Stop()
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case <-t.C:
|
||||
_ = r.Push(ctx)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Version is the canonical content version for a snapshot — SHA-256 of the bytes,
|
||||
// matching hanzoai/vfs's content-addressable model. Exported so any Store impl
|
||||
// (cloud's deps.VFS-backed store, the BackendStore) computes the SAME version the
|
||||
// Replicator's skip-redundant-pull logic expects — one hash function, one place.
|
||||
func Version(data []byte) string {
|
||||
h := sha256.Sum256(data)
|
||||
return hex.EncodeToString(h[:])
|
||||
}
|
||||
|
||||
// version is the internal alias used across this package.
|
||||
func version(data []byte) string { return Version(data) }
|
||||
@@ -1,153 +0,0 @@
|
||||
// Copyright 2025 Hanzo AI Inc. All Rights Reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
|
||||
package replica
|
||||
|
||||
import (
|
||||
"context"
|
||||
"path/filepath"
|
||||
"sync"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// memStore is an in-memory Store (a vfs stand-in) for the round-trip tests.
|
||||
type memStore struct {
|
||||
mu sync.Mutex
|
||||
data map[string][]byte
|
||||
ver map[string]string
|
||||
}
|
||||
|
||||
func newMemStore() *memStore {
|
||||
return &memStore{data: map[string][]byte{}, ver: map[string]string{}}
|
||||
}
|
||||
func (s *memStore) Put(_ context.Context, key string, data []byte) error {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
b := make([]byte, len(data))
|
||||
copy(b, data)
|
||||
s.data[key] = b
|
||||
s.ver[key] = version(data)
|
||||
return nil
|
||||
}
|
||||
func (s *memStore) Get(_ context.Context, key string) ([]byte, string, error) {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
return s.data[key], s.ver[key], nil
|
||||
}
|
||||
func (s *memStore) Version(_ context.Context, key string) (string, error) {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
return s.ver[key], nil
|
||||
}
|
||||
|
||||
// TestSQLiteSnapshotRestoreRoundTrip is the core proof the cloud replicator never
|
||||
// had: a REAL on-disk SQLite DB snapshots to bytes and restores byte-consistently
|
||||
// into a fresh DB, preserving committed rows.
|
||||
func TestSQLiteSnapshotRestoreRoundTrip(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
dir := t.TempDir()
|
||||
|
||||
// Owner DB: write rows, snapshot.
|
||||
src, err := OpenSQLite(filepath.Join(dir, "orgs", "acme", "kv.db"))
|
||||
if err != nil {
|
||||
t.Fatalf("open src: %v", err)
|
||||
}
|
||||
defer src.Close()
|
||||
if _, err := src.DB().ExecContext(ctx, `CREATE TABLE kv(k TEXT PRIMARY KEY, v TEXT)`); err != nil {
|
||||
t.Fatalf("create: %v", err)
|
||||
}
|
||||
for _, kv := range [][2]string{{"a", "1"}, {"b", "2"}, {"c", "3"}} {
|
||||
if _, err := src.DB().ExecContext(ctx, `INSERT INTO kv(k,v) VALUES(?,?)`, kv[0], kv[1]); err != nil {
|
||||
t.Fatalf("insert: %v", err)
|
||||
}
|
||||
}
|
||||
snap, err := src.Snapshot(ctx)
|
||||
if err != nil {
|
||||
t.Fatalf("snapshot: %v", err)
|
||||
}
|
||||
if len(snap) == 0 {
|
||||
t.Fatal("empty snapshot")
|
||||
}
|
||||
|
||||
// Reader DB (a different pod): restore the snapshot, see exactly the rows.
|
||||
dst, err := OpenSQLite(filepath.Join(dir, "orgs", "acme-reader", "kv.db"))
|
||||
if err != nil {
|
||||
t.Fatalf("open dst: %v", err)
|
||||
}
|
||||
defer dst.Close()
|
||||
if err := dst.Restore(ctx, snap); err != nil {
|
||||
t.Fatalf("restore: %v", err)
|
||||
}
|
||||
var got string
|
||||
if err := dst.DB().QueryRowContext(ctx, `SELECT v FROM kv WHERE k='b'`).Scan(&got); err != nil {
|
||||
t.Fatalf("read restored: %v", err)
|
||||
}
|
||||
if got != "2" {
|
||||
t.Fatalf("restored kv[b] = %q, want 2", got)
|
||||
}
|
||||
var n int
|
||||
if err := dst.DB().QueryRowContext(ctx, `SELECT COUNT(*) FROM kv`).Scan(&n); err != nil {
|
||||
t.Fatalf("count: %v", err)
|
||||
}
|
||||
if n != 3 {
|
||||
t.Fatalf("restored row count = %d, want 3", n)
|
||||
}
|
||||
|
||||
// The DB stays usable after Restore (handle reopened cleanly).
|
||||
if _, err := dst.DB().ExecContext(ctx, `INSERT INTO kv(k,v) VALUES('d','4')`); err != nil {
|
||||
t.Fatalf("write after restore: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// TestReplicatorPushPull proves the full owner→store→reader cycle over a real
|
||||
// SQLite DB and an in-memory vfs stand-in, including the redundant-pull skip.
|
||||
func TestReplicatorPushPull(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
dir := t.TempDir()
|
||||
store := newMemStore()
|
||||
key := DBPath("acme", "", "kv") // orgs/acme/kv.db
|
||||
|
||||
owner, err := OpenSQLite(filepath.Join(dir, "owner.db"))
|
||||
if err != nil {
|
||||
t.Fatalf("open owner: %v", err)
|
||||
}
|
||||
defer owner.Close()
|
||||
if _, err := owner.DB().ExecContext(ctx, `CREATE TABLE t(x INT)`); err != nil {
|
||||
t.Fatalf("create: %v", err)
|
||||
}
|
||||
if _, err := owner.DB().ExecContext(ctx, `INSERT INTO t VALUES(42)`); err != nil {
|
||||
t.Fatalf("insert: %v", err)
|
||||
}
|
||||
or := NewReplicator(key, store, owner)
|
||||
if err := or.Push(ctx); err != nil {
|
||||
t.Fatalf("push: %v", err)
|
||||
}
|
||||
|
||||
reader, err := OpenSQLite(filepath.Join(dir, "reader.db"))
|
||||
if err != nil {
|
||||
t.Fatalf("open reader: %v", err)
|
||||
}
|
||||
defer reader.Close()
|
||||
rr := NewReplicator(key, store, reader)
|
||||
changed, err := rr.Pull(ctx)
|
||||
if err != nil {
|
||||
t.Fatalf("pull: %v", err)
|
||||
}
|
||||
if !changed {
|
||||
t.Fatal("first pull must report changed")
|
||||
}
|
||||
var x int
|
||||
if err := reader.DB().QueryRowContext(ctx, `SELECT x FROM t`).Scan(&x); err != nil {
|
||||
t.Fatalf("read pulled: %v", err)
|
||||
}
|
||||
if x != 42 {
|
||||
t.Fatalf("pulled x = %d, want 42", x)
|
||||
}
|
||||
|
||||
// A second pull with no new push is a no-op (content-version skip).
|
||||
if changed, err := rr.Pull(ctx); err != nil || changed {
|
||||
t.Fatalf("second pull: changed=%v err=%v, want changed=false", changed, err)
|
||||
}
|
||||
}
|
||||
@@ -1,184 +0,0 @@
|
||||
// Copyright 2025 Hanzo AI Inc. All Rights Reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
|
||||
package replica
|
||||
|
||||
// SQLiteDB is the real per-org SQLite DB handle the Replicator snapshots and
|
||||
// restores — the piece that makes HA SQLite work for EVERY Hanzo service (the
|
||||
// cloud replicator only ever exercised an in-memory test double). CGO-free
|
||||
// (hanzoai/sqlite), so it builds under CGO_ENABLED=0 like the rest of the
|
||||
// stack.
|
||||
//
|
||||
// Snapshot is consistent: `VACUUM INTO` a temp file forces a WAL checkpoint and
|
||||
// writes a single defragmented, transaction-consistent copy of the database at
|
||||
// one point in time — no torn pages, no separate -wal/-shm to ship. Restore is
|
||||
// atomic: the incoming bytes are written to a temp file next to the live DB, the
|
||||
// live handle is closed, the temp file is renamed over it (atomic on the same
|
||||
// filesystem), and the handle is reopened. A crash mid-restore leaves the old DB
|
||||
// intact (the rename either fully happened or not at all).
|
||||
|
||||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"sync"
|
||||
|
||||
_ "github.com/hanzoai/sqlite" // CGO-free sqlite driver ("sqlite")
|
||||
)
|
||||
|
||||
// sqlitePragmas is the durability profile every Hanzo per-org SQLite opens with
|
||||
// (WAL, NORMAL sync, foreign keys, a generous busy timeout) — the same profile
|
||||
// hanzoai/base + visor apply, so one on-disk convention everywhere.
|
||||
const sqlitePragmas = "?_pragma=busy_timeout(10000)&_pragma=journal_mode(WAL)&_pragma=synchronous(NORMAL)&_pragma=foreign_keys(ON)"
|
||||
|
||||
// SQLiteDB is a Replicator DB backed by an on-disk SQLite file. It owns the sole
|
||||
// *sql.DB handle to that path; Restore swaps the file underneath it. Safe for
|
||||
// concurrent Snapshot / query use; Restore is exclusive.
|
||||
type SQLiteDB struct {
|
||||
path string
|
||||
mu sync.RWMutex
|
||||
db *sql.DB
|
||||
}
|
||||
|
||||
// OpenSQLite opens (creating parent dirs + the file) the SQLite DB at path with
|
||||
// the standard durability pragmas. The returned *SQLiteDB satisfies the
|
||||
// Replicator DB interface and exposes DB() for the service's own queries.
|
||||
func OpenSQLite(path string) (*SQLiteDB, error) {
|
||||
if path == "" {
|
||||
return nil, fmt.Errorf("replica: sqlite path is empty")
|
||||
}
|
||||
if err := os.MkdirAll(filepath.Dir(path), 0o700); err != nil {
|
||||
return nil, fmt.Errorf("replica: sqlite dir %s: %w", filepath.Dir(path), err)
|
||||
}
|
||||
db, err := sql.Open("sqlite", path+sqlitePragmas)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("replica: sqlite open %s: %w", path, err)
|
||||
}
|
||||
// The driver opens lazily; force a connection so a bad path fails here, not later.
|
||||
if err := db.Ping(); err != nil {
|
||||
_ = db.Close()
|
||||
return nil, fmt.Errorf("replica: sqlite ping %s: %w", path, err)
|
||||
}
|
||||
return &SQLiteDB{path: path, db: db}, nil
|
||||
}
|
||||
|
||||
// DB returns the live *sql.DB for the service's own reads/writes. It stays valid
|
||||
// across Restore (Restore reopens under the same *SQLiteDB), so hold the
|
||||
// *SQLiteDB, call DB() per query, and never cache the *sql.DB across a Restore.
|
||||
func (s *SQLiteDB) DB() *sql.DB {
|
||||
s.mu.RLock()
|
||||
defer s.mu.RUnlock()
|
||||
return s.db
|
||||
}
|
||||
|
||||
// Snapshot returns a consistent, WAL-checkpointed copy of the whole database as
|
||||
// bytes. Concurrent with ongoing reads/writes.
|
||||
func (s *SQLiteDB) Snapshot(ctx context.Context) ([]byte, error) {
|
||||
s.mu.RLock()
|
||||
db := s.db
|
||||
s.mu.RUnlock()
|
||||
return snapshotConn(ctx, db, filepath.Dir(s.path))
|
||||
}
|
||||
|
||||
// SnapshotFile returns a consistent snapshot of the SQLite database at path
|
||||
// WITHOUT holding a persistent handle — the adoption primitive for a service that
|
||||
// manages the DB with its OWN library (xorm, GORM, …): open a transient read
|
||||
// handle, VACUUM INTO a temp, return the bytes. In WAL mode the transient handle
|
||||
// sees all committed data, so this is consistent even while the service writes.
|
||||
func SnapshotFile(ctx context.Context, path string) ([]byte, error) {
|
||||
db, err := sql.Open("sqlite", path+sqlitePragmas)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("replica: snapshot open %s: %w", path, err)
|
||||
}
|
||||
defer db.Close()
|
||||
return snapshotConn(ctx, db, filepath.Dir(path))
|
||||
}
|
||||
|
||||
// snapshotConn is the shared VACUUM-INTO core (one and one way).
|
||||
func snapshotConn(ctx context.Context, db *sql.DB, dir string) ([]byte, error) {
|
||||
tmp, err := os.CreateTemp(dir, ".snap-*.db")
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("replica: snapshot temp: %w", err)
|
||||
}
|
||||
tmpPath := tmp.Name()
|
||||
_ = tmp.Close()
|
||||
defer os.Remove(tmpPath)
|
||||
// VACUUM INTO forces a checkpoint and writes one consistent copy. The temp
|
||||
// path is a literal (created by us), so this is not an injection surface.
|
||||
if _, err := db.ExecContext(ctx, "VACUUM INTO '"+tmpPath+"'"); err != nil {
|
||||
return nil, fmt.Errorf("replica: vacuum into: %w", err)
|
||||
}
|
||||
data, err := os.ReadFile(tmpPath)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("replica: read snapshot: %w", err)
|
||||
}
|
||||
return data, nil
|
||||
}
|
||||
|
||||
// RestoreFile atomically writes snapshot bytes over the SQLite database at path —
|
||||
// the handle-less adoption primitive. The caller MUST have no open handle to path
|
||||
// (close the service's engine first; reopen after). Drops the -wal/-shm sidecars
|
||||
// so a reopen can't replay stale WAL over the restored file. Used at hydrate-on-open,
|
||||
// BEFORE the service opens its own engine.
|
||||
func RestoreFile(path string, data []byte) error {
|
||||
if err := os.MkdirAll(filepath.Dir(path), 0o700); err != nil {
|
||||
return fmt.Errorf("replica: restore dir %s: %w", filepath.Dir(path), err)
|
||||
}
|
||||
tmpPath := path + ".restore"
|
||||
if err := os.WriteFile(tmpPath, data, 0o600); err != nil {
|
||||
return fmt.Errorf("replica: write restore temp: %w", err)
|
||||
}
|
||||
for _, side := range []string{"-wal", "-shm"} {
|
||||
_ = os.Remove(path + side)
|
||||
}
|
||||
if err := os.Rename(tmpPath, path); err != nil {
|
||||
os.Remove(tmpPath)
|
||||
return fmt.Errorf("replica: rename restore over %s: %w", path, err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// Restore atomically replaces the local database with data and reopens the
|
||||
// handle. Exclusive: it closes the live handle, renames the new file over the
|
||||
// old, and reopens. On any failure the previous DB is left intact and the handle
|
||||
// is reopened on the original file, so the service never ends up with no DB.
|
||||
func (s *SQLiteDB) Restore(ctx context.Context, data []byte) error {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
|
||||
// Close our handle, swap the file via the shared primitive, reopen.
|
||||
if err := s.db.Close(); err != nil {
|
||||
return fmt.Errorf("replica: close for restore: %w", err)
|
||||
}
|
||||
if err := RestoreFile(s.path, data); err != nil {
|
||||
// Reopen the original so the service still has a DB.
|
||||
s.db, _ = sql.Open("sqlite", s.path+sqlitePragmas)
|
||||
return err
|
||||
}
|
||||
db, err := sql.Open("sqlite", s.path+sqlitePragmas)
|
||||
if err != nil {
|
||||
return fmt.Errorf("replica: reopen after restore: %w", err)
|
||||
}
|
||||
if err := db.PingContext(ctx); err != nil {
|
||||
_ = db.Close()
|
||||
return fmt.Errorf("replica: ping after restore: %w", err)
|
||||
}
|
||||
s.db = db
|
||||
return nil
|
||||
}
|
||||
|
||||
// Close closes the underlying handle.
|
||||
func (s *SQLiteDB) Close() error {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
if s.db == nil {
|
||||
return nil
|
||||
}
|
||||
err := s.db.Close()
|
||||
s.db = nil
|
||||
return err
|
||||
}
|
||||
@@ -1,66 +0,0 @@
|
||||
// Copyright 2025 Hanzo AI Inc. All Rights Reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
|
||||
package replica
|
||||
|
||||
// BackendStore adapts a vfs backend.Backend (the raw object store — file locally,
|
||||
// s3/SeaweedFS in production) to the Replicator Store interface. This is the ready
|
||||
// Store every service wires: `replica.NewReplicator(key, replica.NewBackendStore(be), db)`.
|
||||
//
|
||||
// A tiny content-version sidecar (`<key>.ver` holding the snapshot's SHA-256) lets
|
||||
// a reader answer Version(key) cheaply — a small Get — instead of fetching the whole
|
||||
// database just to decide whether it changed. Get itself recomputes the version from
|
||||
// the fetched bytes (authoritative), so a missing/stale sidecar can never cause a
|
||||
// wrong restore — at worst a redundant pull.
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
|
||||
"github.com/hanzoai/vfs/pkg/backend"
|
||||
)
|
||||
|
||||
// BackendStore is a Store backed by a vfs backend.Backend.
|
||||
type BackendStore struct {
|
||||
be backend.Backend
|
||||
}
|
||||
|
||||
// NewBackendStore wraps be as a Replicator Store.
|
||||
func NewBackendStore(be backend.Backend) *BackendStore { return &BackendStore{be: be} }
|
||||
|
||||
func verKey(key string) string { return key + ".ver" }
|
||||
|
||||
// Put writes the snapshot then its version sidecar. If the sidecar write fails the
|
||||
// data is still durable; Version falls back to "" (unknown → the reader pulls), so
|
||||
// a partial Put degrades to a redundant pull, never data loss.
|
||||
func (s *BackendStore) Put(ctx context.Context, key string, data []byte) error {
|
||||
if err := s.be.Put(ctx, key, data); err != nil {
|
||||
return err
|
||||
}
|
||||
return s.be.Put(ctx, verKey(key), []byte(version(data)))
|
||||
}
|
||||
|
||||
// Get returns the snapshot bytes and their authoritative content version (recomputed
|
||||
// from the bytes, so it always matches what a writer stored).
|
||||
func (s *BackendStore) Get(ctx context.Context, key string) ([]byte, string, error) {
|
||||
data, err := s.be.Get(ctx, key)
|
||||
if err != nil {
|
||||
return nil, "", err
|
||||
}
|
||||
return data, version(data), nil
|
||||
}
|
||||
|
||||
// Version returns the cheap sidecar version, or "" when absent (a first-ever key or
|
||||
// a legacy object with no sidecar — the reader then pulls once).
|
||||
func (s *BackendStore) Version(ctx context.Context, key string) (string, error) {
|
||||
v, err := s.be.Get(ctx, verKey(key))
|
||||
if errors.Is(err, backend.ErrNotFound) {
|
||||
return "", nil
|
||||
}
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
return string(v), nil
|
||||
}
|
||||
@@ -1,91 +0,0 @@
|
||||
// Copyright 2025 Hanzo AI Inc. All Rights Reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
|
||||
package replica
|
||||
|
||||
import (
|
||||
"context"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
|
||||
"github.com/hanzoai/vfs/pkg/backend"
|
||||
_ "github.com/hanzoai/vfs/pkg/backend/file" // register file:// opener
|
||||
)
|
||||
|
||||
// TestBackendStoreEndToEnd is the full HA path a service runs: an OWNER writes a
|
||||
// real SQLite DB and Pushes it through a vfs backend; a READER on a different disk
|
||||
// Pulls it back and sees the committed rows. Proves the BackendStore adapter +
|
||||
// SQLiteDB + Replicator compose over the real object-store backend.
|
||||
func TestBackendStoreEndToEnd(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
objDir := t.TempDir() // stands in for SeaweedFS
|
||||
be, err := backend.Open(ctx, "file://"+objDir)
|
||||
if err != nil {
|
||||
t.Fatalf("backend.Open: %v", err)
|
||||
}
|
||||
defer be.Close()
|
||||
store := NewBackendStore(be)
|
||||
key := DBPath("acme", "", "kv")
|
||||
|
||||
// Owner: create + write + push.
|
||||
ownerDir := t.TempDir()
|
||||
owner, err := OpenSQLite(filepath.Join(ownerDir, "kv.db"))
|
||||
if err != nil {
|
||||
t.Fatalf("open owner: %v", err)
|
||||
}
|
||||
defer owner.Close()
|
||||
if _, err := owner.DB().ExecContext(ctx, `CREATE TABLE kv(k TEXT PRIMARY KEY, v TEXT)`); err != nil {
|
||||
t.Fatalf("create: %v", err)
|
||||
}
|
||||
if _, err := owner.DB().ExecContext(ctx, `INSERT INTO kv VALUES('hello','world')`); err != nil {
|
||||
t.Fatalf("insert: %v", err)
|
||||
}
|
||||
or := NewReplicator(key, store, owner)
|
||||
if err := or.Push(ctx); err != nil {
|
||||
t.Fatalf("push: %v", err)
|
||||
}
|
||||
|
||||
// Version sidecar is queryable cheaply.
|
||||
if v, err := store.Version(ctx, key); err != nil || v == "" {
|
||||
t.Fatalf("version after push = %q, err=%v; want non-empty", v, err)
|
||||
}
|
||||
|
||||
// Reader on a separate disk: pull + read.
|
||||
readerDir := t.TempDir()
|
||||
reader, err := OpenSQLite(filepath.Join(readerDir, "kv.db"))
|
||||
if err != nil {
|
||||
t.Fatalf("open reader: %v", err)
|
||||
}
|
||||
defer reader.Close()
|
||||
rr := NewReplicator(key, store, reader)
|
||||
if changed, err := rr.Pull(ctx); err != nil || !changed {
|
||||
t.Fatalf("pull: changed=%v err=%v; want changed=true", changed, err)
|
||||
}
|
||||
var got string
|
||||
if err := reader.DB().QueryRowContext(ctx, `SELECT v FROM kv WHERE k='hello'`).Scan(&got); err != nil {
|
||||
t.Fatalf("read pulled: %v", err)
|
||||
}
|
||||
if got != "world" {
|
||||
t.Fatalf("pulled kv[hello] = %q, want world", got)
|
||||
}
|
||||
|
||||
// Owner writes more + pushes; reader pulls the update.
|
||||
if _, err := owner.DB().ExecContext(ctx, `INSERT INTO kv VALUES('a','b')`); err != nil {
|
||||
t.Fatalf("insert 2: %v", err)
|
||||
}
|
||||
if err := or.Push(ctx); err != nil {
|
||||
t.Fatalf("push 2: %v", err)
|
||||
}
|
||||
if changed, err := rr.Pull(ctx); err != nil || !changed {
|
||||
t.Fatalf("pull 2: changed=%v err=%v; want changed=true", changed, err)
|
||||
}
|
||||
var n int
|
||||
if err := reader.DB().QueryRowContext(ctx, `SELECT COUNT(*) FROM kv`).Scan(&n); err != nil {
|
||||
t.Fatalf("count after update: %v", err)
|
||||
}
|
||||
if n != 2 {
|
||||
t.Fatalf("reader row count = %d, want 2", n)
|
||||
}
|
||||
}
|
||||
+4
-4
@@ -35,12 +35,12 @@ import (
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
_ "github.com/hanzoai/sqlite"
|
||||
"github.com/luxfi/age"
|
||||
_ "modernc.org/sqlite"
|
||||
|
||||
"github.com/hanzoai/vfs"
|
||||
"github.com/hanzoai/vfs/pkg/backend"
|
||||
_ "github.com/hanzoai/vfs/pkg/backend/file"
|
||||
"github.com/hanzoai/vfs"
|
||||
)
|
||||
|
||||
func TestSQLiteRoundTrip(t *testing.T) {
|
||||
@@ -64,8 +64,8 @@ func TestSQLiteRoundTrip(t *testing.T) {
|
||||
sym, name string
|
||||
dec int
|
||||
}{
|
||||
{"USDC", "USD Coin", 6},
|
||||
{"USDT", "Tether", 6},
|
||||
{"LQDTY", "Liquidity", 18},
|
||||
{"USDL", "Liquid USD", 6},
|
||||
{"BTC", "Bitcoin", 8},
|
||||
{"ETH", "Ethereum", 18},
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user