Files
Hanzo DevandGitHub 6c3cfe9751 feat: cloud Mount() per HIP-0106 (#1)
* feat: cloud Mount() per HIP-0106

Adds pkg/vfs.Mount(*zip.App, cloud.Deps) error so the unified Hanzo
Cloud binary can blank-import and register VFS alongside every
other Hanzo subsystem.

VFS has no public HTTP surface today — it's an S3-backed virtual
block filesystem driven through FUSE/CLI/sidecar. Mount() exposes
liveness/readiness probes plus a minimal block-CRUD HTTP API so
co-resident subsystems can talk to VFS without falling back to the
CLI:

- GET    /v1/vfs/health       — always 200
- GET    /v1/vfs/readyz       — 200 only when an instance is
                                attached via vfs.SetInstance
- PUT    /v1/vfs/blocks       — payload upload
- GET    /v1/vfs/blocks/:id   — payload download
- DELETE /v1/vfs/blocks/:id   — backend delete
- GET    /v1/vfs/stats        — cache/backend counters

cmd/vfsd is a HIP-0106 thin shim that wires backend + age keys from
env (VFSD_BACKEND, VFSD_AGE_KEY) and listens on cloud.Config.ListenAddr.
The existing cmd/vfs CLI (put/get/stats/mount) is untouched.

init() registers with cloud.Register("vfs", 20, …). schema/vfs.zap
adds the minimal Health + Block ZAP interface; full multi-block
File surface lands in follow-up PRs.

Tests:
- pkg/vfs: TestMount_HealthReadyz + TestMount_PutGetRoundtrip green
- existing TestRoundTrip + TestStats unchanged

* refactor: collapse pkg/vfs/ → root for clean import path

Subsystem code lives in package vfs at repo root. Importers now use
`github.com/hanzoai/vfs` (no /pkg/vfs/ nesting). Standalone shim moved
to cmd/vfs/ where applicable.

Per Hanzo Go-stdlib pattern: repo IS the package.

* refactor: flatten import path — github.com/hanzoai/cloud (drop /pkg/cloud)
2026-05-18 23:37:45 -07:00

309 lines
7.8 KiB
Go

package vfs
import (
"context"
"errors"
"fmt"
"io"
"time"
"github.com/hanzoai/vfs/pkg/backend"
)
// File is a multi-block file handle. Reads/writes are byte-addressed;
// the implementation transparently fetches, modifies, and re-encrypts
// 4 KiB blocks. Dirty blocks are buffered in a per-handle write set
// until Sync flushes them to the backend (and updates the inode's
// block list in the FS metadata tree).
//
// Concurrency: a File is owned by a single goroutine. Concurrent FS
// access across multiple Files is supported by FS's RWMutex.
type File struct {
fs *FS
ctx context.Context
inodeID InodeID
// dirty maps block index → fresh plaintext (BlockSize-padded). Reads
// hit dirty first, then the inode's block list. Sync materialises
// dirty entries via vfs.PutBlock and atomically swaps inode.Blocks.
dirty map[int][]byte
}
// Stat returns a snapshot of the file's inode metadata.
func (f *File) Stat() (*Inode, error) {
f.fs.mu.RLock()
defer f.fs.mu.RUnlock()
in := f.fs.inodeByID(f.inodeID)
if in == nil {
return nil, fmt.Errorf("vfs.File.Stat: inode %d gone", f.inodeID)
}
return cloneInode(in), nil
}
// ReadAt reads len(p) bytes at the given offset. Returns io.EOF when
// reading past the end of the file (matching os.File semantics: when
// fewer bytes than requested are available, returns those bytes plus
// io.EOF).
func (f *File) ReadAt(p []byte, off int64) (int, error) {
if off < 0 {
return 0, fmt.Errorf("vfs.File.ReadAt: negative offset %d", off)
}
stat, err := f.Stat()
if err != nil {
return 0, err
}
if uint64(off) >= stat.Size {
return 0, io.EOF
}
want := len(p)
end := uint64(off) + uint64(want)
if end > stat.Size {
end = stat.Size
want = int(end - uint64(off))
}
written := 0
cur := uint64(off)
for cur < end {
blockIdx := int(cur / BlockSize)
blockOff := int(cur % BlockSize)
blockBytes, err := f.readBlock(blockIdx, stat)
if err != nil {
return written, err
}
copyEnd := BlockSize
remainingInFile := int(end - cur)
if remainingInFile < BlockSize-blockOff {
copyEnd = blockOff + remainingInFile
}
n := copy(p[written:want], blockBytes[blockOff:copyEnd])
written += n
cur += uint64(n)
}
if uint64(off)+uint64(written) >= stat.Size {
return written, io.EOF
}
return written, nil
}
// WriteAt writes len(p) bytes at the given offset, extending the file
// as needed. Partial-block writes do read-modify-write to avoid
// clobbering adjacent bytes.
//
// Sync MUST be called to flush dirty blocks to the backend; without
// Sync the writes live only in this File's in-memory dirty set and
// will be lost on close-without-sync.
func (f *File) WriteAt(p []byte, off int64) (int, error) {
if off < 0 {
return 0, fmt.Errorf("vfs.File.WriteAt: negative offset %d", off)
}
if len(p) == 0 {
return 0, nil
}
stat, err := f.Stat()
if err != nil {
return 0, err
}
if f.dirty == nil {
f.dirty = map[int][]byte{}
}
written := 0
cur := uint64(off)
end := cur + uint64(len(p))
for cur < end {
blockIdx := int(cur / BlockSize)
blockOff := int(cur % BlockSize)
var existing []byte
// Existing block content for read-modify-write (if any)
if buf, ok := f.dirty[blockIdx]; ok {
existing = append([]byte(nil), buf...)
} else if blockIdx < len(stat.Blocks) {
b, err := f.readBlock(blockIdx, stat)
if err != nil {
return written, err
}
existing = append([]byte(nil), b...)
} else {
existing = make([]byte, BlockSize)
}
copyLen := BlockSize - blockOff
remaining := len(p) - written
if copyLen > remaining {
copyLen = remaining
}
copy(existing[blockOff:blockOff+copyLen], p[written:written+copyLen])
f.dirty[blockIdx] = existing
written += copyLen
cur += uint64(copyLen)
}
// Extend size if we wrote past the old end. Inode update is staged
// here in memory; the canonical Size + Blocks land at Sync().
f.fs.mu.Lock()
in := f.fs.inodeByID(f.inodeID)
if end > in.Size {
in.Size = end
}
in.Mtime = time.Now()
f.fs.dirty = true
f.fs.mu.Unlock()
return written, nil
}
// Truncate sets the file size, dropping any blocks past the new end.
// Extending past the current size zero-pads (creates dirty zero blocks
// up to the new end).
func (f *File) Truncate(size uint64) error {
stat, err := f.Stat()
if err != nil {
return err
}
if size == stat.Size {
return nil
}
if f.dirty == nil {
f.dirty = map[int][]byte{}
}
if size < stat.Size {
// Shrink: drop dirty blocks past size, no read needed for backend.
newBlockCount := int((size + BlockSize - 1) / BlockSize)
for idx := range f.dirty {
if idx >= newBlockCount {
delete(f.dirty, idx)
}
}
} else {
// Grow: load (or zero-init) the last block, zero-pad in place.
// Subsequent blocks remain implicitly zero — they materialise
// on first read as a zero block.
oldEnd := stat.Size
newEnd := size
if oldEnd%BlockSize != 0 || oldEnd == 0 {
lastIdx := int(oldEnd / BlockSize)
var buf []byte
if cached, ok := f.dirty[lastIdx]; ok {
buf = append([]byte(nil), cached...)
} else if lastIdx < len(stat.Blocks) {
existing, err := f.readBlock(lastIdx, stat)
if err != nil {
return err
}
buf = append([]byte(nil), existing...)
} else {
buf = make([]byte, BlockSize)
}
startZero := int(oldEnd % BlockSize)
zeroEnd := BlockSize
if newEnd-uint64(lastIdx*BlockSize) < uint64(BlockSize) {
zeroEnd = int(newEnd - uint64(lastIdx*BlockSize))
}
for i := startZero; i < zeroEnd; i++ {
buf[i] = 0
}
f.dirty[lastIdx] = buf
}
}
f.fs.mu.Lock()
in := f.fs.inodeByID(f.inodeID)
in.Size = size
in.Mtime = time.Now()
f.fs.dirty = true
f.fs.mu.Unlock()
return nil
}
// Sync flushes all dirty blocks to the backend, updates the inode's
// block list, and persists the FS metadata tree. After a successful
// Sync the file's bytes are durable.
func (f *File) Sync() error {
stat, err := f.Stat()
if err != nil {
return err
}
// Re-encrypt + upload each dirty block.
newBlocks := make(map[int]BlockID, len(f.dirty))
for idx, buf := range f.dirty {
id, err := f.fs.v.PutBlock(f.ctx, buf)
if err != nil {
return fmt.Errorf("vfs.File.Sync: PutBlock idx=%d: %w", idx, err)
}
newBlocks[idx] = id
}
// Splice new blocks into the inode's block list.
f.fs.mu.Lock()
in := f.fs.inodeByID(f.inodeID)
totalBlocks := int((stat.Size + BlockSize - 1) / BlockSize)
if len(in.Blocks) < totalBlocks {
// Extend
grown := make([]BlockID, totalBlocks)
copy(grown, in.Blocks)
in.Blocks = grown
} else if len(in.Blocks) > totalBlocks {
in.Blocks = in.Blocks[:totalBlocks]
}
for idx, id := range newBlocks {
if idx < len(in.Blocks) {
in.Blocks[idx] = id
}
}
f.fs.dirty = true
f.fs.mu.Unlock()
// Clear local dirty cache once committed.
f.dirty = nil
// Persist the metadata tree.
return f.fs.Sync(f.ctx)
}
// Close flushes pending writes and releases the handle. Mirrors
// os.File.Close — must be called even on read-only handles to allow
// the FS to release any resources.
func (f *File) Close() error {
if len(f.dirty) > 0 {
if err := f.Sync(); err != nil {
return err
}
}
f.dirty = nil
return nil
}
// readBlock returns the plaintext bytes for the given block index.
// Blocks past the inode's block list (but inside the file size, e.g.
// after a Truncate-grow that hasn't been Synced) read as zero.
func (f *File) readBlock(idx int, stat *Inode) ([]byte, error) {
if buf, ok := f.dirty[idx]; ok {
return buf, nil
}
if idx >= len(stat.Blocks) {
return make([]byte, BlockSize), nil
}
id := stat.Blocks[idx]
if id == "" {
return make([]byte, BlockSize), nil
}
b, err := f.fs.v.GetBlock(f.ctx, id)
if err != nil {
if errors.Is(err, backend.ErrNotFound) {
return nil, fmt.Errorf("vfs.File.readBlock: dangling block %s at idx %d", id, idx)
}
return nil, err
}
return b, nil
}
// Compile-time interface checks.
var (
_ io.ReaderAt = (*File)(nil)
_ io.WriterAt = (*File)(nil)
)