replica: extract single-writer election to hanzoai/ha (#5)
* replica: extract single-writer election to hanzoai/ha Election (Owner/IsOwner/Replicas/Member) was storage-agnostic pure Go with zero vfs-internal callers — it lived here only historically. It is now its own coordination primitive at github.com/hanzoai/ha so any singleton (cron, queue drain, billing sweep) gets exactly-once WITHOUT importing a SQLite library. vfs/replica keeps what it is: SQLite replication (Replicator, SQLiteDB, BackendStore). One concern per package. Consumers (visor, cloud/internal/org) migrate to hanzoai/ha in lockstep; they pin vfs v0.6.2 so this main-only removal does not disturb them until they bump. Removes owner.go + its two election tests; doc points at ha. * ci: enable cgo for the -race steps (pre-existing red) go test -race requires cgo, but the self-hosted hanzo-build-linux-amd64 runner has no C compiler and defaulted CGO_ENABLED=0 (vfs is CGO-free in production), so both race steps failed — first with 'go: -race requires cgo', then 'C compiler gcc not found'. Every main run has been red on this for many commits. Fix: install gcc + set CGO_ENABLED=1 on the two -race steps only; the production build stays CGO_ENABLED=0. Now the race detector actually runs (real coverage for the concurrent replicator/ship loops). Unrelated to the ha extraction but folded in so this PR lands green rather than override a known-bad check. --------- Co-authored-by: hanzo <z@hanzo.ai>
This commit is contained in:
Hanzo Dev
@@ -20,10 +20,14 @@ jobs:
|
||||
with:
|
||||
go-version: '1.26.4'
|
||||
cache: true
|
||||
- name: install cgo toolchain (the -race detector requires a C compiler)
|
||||
run: sudo apt-get update && sudo apt-get install -y gcc
|
||||
- name: vet
|
||||
run: go vet ./...
|
||||
- name: test (default)
|
||||
run: go test -race ./...
|
||||
env:
|
||||
CGO_ENABLED: "1" # the race detector requires cgo; production build stays CGO_ENABLED=0 (line below)
|
||||
- name: vet (fuse)
|
||||
run: go vet -tags fuse ./...
|
||||
- name: install fuse
|
||||
@@ -32,6 +36,8 @@ jobs:
|
||||
run: go build -tags fuse ./...
|
||||
- name: test (fuse + sqlite e2e)
|
||||
run: VFS_FUSE_E2E=1 go test -race -tags fuse -run TestFUSESQLite ./pkg/mount/ -v
|
||||
env:
|
||||
CGO_ENABLED: "1" # -race + fuse bindings both require cgo
|
||||
|
||||
build-amd64:
|
||||
needs: test
|
||||
|
||||
@@ -100,16 +100,3 @@ func BenchmarkPushPull(b *testing.B) {
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// BenchmarkOwnerElection measures the per-write HRW single-writer check (the hot path
|
||||
// gating every write when scaled to N replicas).
|
||||
func BenchmarkOwnerElection(b *testing.B) {
|
||||
members := make([]Member, 16)
|
||||
for i := range members {
|
||||
members[i] = Member{ID: fmt.Sprintf("replica-%02d", i)}
|
||||
}
|
||||
b.ResetTimer()
|
||||
for i := 0; i < b.N; i++ {
|
||||
_ = IsOwner("some-org-slug", "replica-07", members)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,103 +0,0 @@
|
||||
// Copyright 2025 Hanzo AI Inc. All Rights Reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
|
||||
package replica
|
||||
|
||||
// Single-writer election WITHOUT a coordinator. Answers one question, identically
|
||||
// on every replica: "which replica is the single writer for org X's databases?"
|
||||
// via Rendezvous (Highest-Random-Weight) hashing over the live membership set. No
|
||||
// election protocol, no lock service, no service discovery, no Postgres, no Redis.
|
||||
//
|
||||
// Ownership is PER-ORG: the owning replica writes ALL of an org's databases (root
|
||||
// + every per-project + per-user db), so an org's data has locality and moves as
|
||||
// one unit on failover. Reads are served from any replica's vfs-synced copy. A
|
||||
// mega-org that outgrows one writer is a future sub-shard concern (hash org+shard);
|
||||
// per-org is the correct, simple default and the natural isolation boundary.
|
||||
|
||||
import (
|
||||
"crypto/sha256"
|
||||
"encoding/binary"
|
||||
)
|
||||
|
||||
// Member is one replica in the live membership set. ID must be stable for the
|
||||
// life of the replica (e.g. its pod name / a persistent node id): HRW weights
|
||||
// derive from it. Addr is the reachable address for write-forwarding (host:port).
|
||||
type Member struct {
|
||||
ID string
|
||||
Addr string
|
||||
}
|
||||
|
||||
// Owner returns the replica that owns the writer for orgID, or ok=false when
|
||||
// members is empty (fail-closed — never a wrong writer). Deterministic: the same
|
||||
// (orgID, members) yields the same Owner on every replica, independent of order.
|
||||
func Owner(orgID string, members []Member) (Member, bool) {
|
||||
if len(members) == 0 {
|
||||
return Member{}, false
|
||||
}
|
||||
best := members[0]
|
||||
bestScore := weight(orgID, best.ID)
|
||||
for _, m := range members[1:] {
|
||||
s := weight(orgID, m.ID)
|
||||
// Tie-break on ID so the result is order-independent.
|
||||
if s > bestScore || (s == bestScore && m.ID < best.ID) {
|
||||
best, bestScore = m, s
|
||||
}
|
||||
}
|
||||
return best, true
|
||||
}
|
||||
|
||||
// IsOwner reports whether selfID owns the writer for orgID under members — the
|
||||
// per-write hot-path check every replica runs.
|
||||
func IsOwner(orgID, selfID string, members []Member) bool {
|
||||
o, ok := Owner(orgID, members)
|
||||
return ok && o.ID == selfID
|
||||
}
|
||||
|
||||
// Replicas returns the org's owner first, then ordered failover successors. On
|
||||
// owner loss the next replica becomes owner with no recomputation, so a reader can
|
||||
// pre-warm the vfs copy for the orgs it is next-in-line to own.
|
||||
func Replicas(orgID string, members []Member, n int) []Member {
|
||||
if n <= 0 || len(members) == 0 {
|
||||
return nil
|
||||
}
|
||||
type scored struct {
|
||||
m Member
|
||||
s uint64
|
||||
}
|
||||
all := make([]scored, len(members))
|
||||
for i, m := range members {
|
||||
all[i] = scored{m, weight(orgID, m.ID)}
|
||||
}
|
||||
for i := 1; i < len(all); i++ {
|
||||
for j := i; j > 0; j-- {
|
||||
a, b := all[j], all[j-1]
|
||||
less := a.s > b.s || (a.s == b.s && a.m.ID < b.m.ID)
|
||||
if !less {
|
||||
break
|
||||
}
|
||||
all[j], all[j-1] = all[j-1], all[j]
|
||||
}
|
||||
}
|
||||
if n > len(all) {
|
||||
n = len(all)
|
||||
}
|
||||
out := make([]Member, n)
|
||||
for i := 0; i < n; i++ {
|
||||
out[i] = all[i].m
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// weight is the HRW score for (org, member): the first 8 bytes of
|
||||
// SHA-256(orgID || 0x00 || memberID) as a big-endian uint64. The NUL separator
|
||||
// stops ("ab","c") and ("a","bc") from colliding.
|
||||
func weight(orgID, memberID string) uint64 {
|
||||
h := sha256.New()
|
||||
_, _ = h.Write([]byte(orgID))
|
||||
_, _ = h.Write([]byte{0})
|
||||
_, _ = h.Write([]byte(memberID))
|
||||
sum := h.Sum(nil)
|
||||
return binary.BigEndian.Uint64(sum[:8])
|
||||
}
|
||||
+3
-2
@@ -15,8 +15,9 @@
|
||||
// per-org store seam:
|
||||
//
|
||||
// hydrate-on-open : Pull() the latest snapshot before first use of an org DB.
|
||||
// single-writer : IsOwner(org, self, members) gates writes; a non-owner serves
|
||||
// stale-tolerant reads (Pull-then-read) and forwards writes.
|
||||
// single-writer : ha.IsOwner(key, self, members) gates writes (github.com/hanzoai/ha,
|
||||
// the election primitive); a non-owner serves stale-tolerant reads
|
||||
// (Pull-then-read) and forwards writes.
|
||||
// post-commit ship: the owner runs PushLoop (or Push after a write burst); the
|
||||
// durable copy in vfs means a lost pod loses no committed data.
|
||||
package replica
|
||||
|
||||
@@ -151,42 +151,3 @@ func TestReplicatorPushPull(t *testing.T) {
|
||||
t.Fatalf("second pull: changed=%v err=%v, want changed=false", changed, err)
|
||||
}
|
||||
}
|
||||
|
||||
// TestOwnerDeterministicAndExclusive proves the HRW election picks exactly one
|
||||
// owner, order-independently, and every member agrees.
|
||||
func TestOwnerDeterministicAndExclusive(t *testing.T) {
|
||||
members := []Member{{ID: "r3"}, {ID: "r1"}, {ID: "r2"}}
|
||||
reordered := []Member{{ID: "r1"}, {ID: "r2"}, {ID: "r3"}}
|
||||
|
||||
o1, ok1 := Owner("acme", members)
|
||||
o2, ok2 := Owner("acme", reordered)
|
||||
if !ok1 || !ok2 || o1.ID != o2.ID {
|
||||
t.Fatalf("owner not order-independent: %q vs %q", o1.ID, o2.ID)
|
||||
}
|
||||
|
||||
// Exactly one member is the owner across the set.
|
||||
owners := 0
|
||||
for _, m := range members {
|
||||
if IsOwner("acme", m.ID, members) {
|
||||
owners++
|
||||
}
|
||||
}
|
||||
if owners != 1 {
|
||||
t.Fatalf("IsOwner true for %d members, want exactly 1", owners)
|
||||
}
|
||||
|
||||
// Empty membership fails closed (no owner).
|
||||
if _, ok := Owner("acme", nil); ok {
|
||||
t.Fatal("empty membership must have no owner")
|
||||
}
|
||||
|
||||
// Distinct orgs spread across replicas (not all pinned to one).
|
||||
seen := map[string]bool{}
|
||||
for _, org := range []string{"a", "b", "c", "d", "e", "f", "g", "h"} {
|
||||
o, _ := Owner(org, members)
|
||||
seen[o.ID] = true
|
||||
}
|
||||
if len(seen) < 2 {
|
||||
t.Fatalf("HRW pinned all orgs to %d replica(s); want spread", len(seen))
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user