luxfi/node
1
0
Fork 0
mirror of https://github.com/luxfi/node.git synced 2026-07-27 03:39:39 +00:00
Code Issues Packages Projects Releases Wiki Activity

codec: rip github.com/luxfi/codec from leaf-misc (Wave 1D)

Browse Source 
Wave 1D of the luxfi/codec final rip: leaf-misc tier under node/.
This batch removes the direct codec/wrappers, codec/linearcodec, and
codec.Manager imports from p2p network protocol, IP/port encoding,
indexer/keystore on-disk storage, warp socket IPC, x/archivedb +
merkledb internals, and the primary-network wallet UTXO loader.

Migration shape:

(a) codec/wrappers.Packer + length constants
    → node/utils/wrappers (same shape, already in tree as the local
      canonical home for binary IO helpers). 14 files: indexer/,
      network/, utils/ips/, utils/metric/, warp/, x/.

(b) codec.Manager + linearcodec for on-disk container storage
    → hand-rolled big-endian binary marshal/unmarshal. Hard cut;
      no codec-version prefix; payload size bounded explicitly:
      - indexer/codec.go: marshalContainer/unmarshalContainer
      - service/keystore/codec.go: marshalHash/unmarshalHash and
        marshalUser/unmarshalUser, 16 MiB blob cap retained.

(c) codec.Manager for cross-chain UTXO parsing (wallet/network/primary)
    → utxo.ParseUTXO via the ZAP wire dispatcher already registered
      by node/vms/components/lux. Drops the per-chain codec.Manager
      slot from FetchState; AddAllUTXOs no longer takes a codec.
      Underscore-import of vms/components/lux ensures the dispatcher
      is wired even for callers that don't already pull platformvm.

No snow/snowman/snowball/avalanche/avm/subnet residue introduced.
This commit is contained in:
Hanzo AI
2026-06-05 11:57:14 -07:00
parent dcb4369847
commit 6de839a515
19 changed files with 224 additions and 83 deletions
Download Patch File Download Diff File Expand all files Collapse all files
indexer/codec.go
+49 -11
View File
@@ -1,24 +1,62 @@
// Copyright (C) 2019-2025, Lux Industries Inc. All rights reserved.
// Copyright (C) 2019-2026, Lux Industries Inc. All rights reserved.
// See the file LICENSE for licensing terms.
package indexer
import (
"math"
"encoding/binary"
"errors"
"github.com/luxfi/codec"
"github.com/luxfi/codec/linearcodec"
"github.com/luxfi/ids"
)
const CodecVersion = 0
// On-disk encoding of an indexed Container. Big-endian wire layout:
//
// bytes 0..31 : Container.ID (ids.IDLen)
// bytes 32..35 : len(Container.Bytes) (uint32 BE)
// bytes 36..N : Container.Bytes (variable)
// bytes N..N+8 : Container.Timestamp (int64 BE, two's complement)
//
// No codec version prefix — the indexer DB lives inside the node, so a
// forward-only schema is fine. Older versions stored a 2-byte codec
// version header; that format is no longer accepted (hard cut from
// luxfi/codec rip — Wave 1D).
var Codec codec.Manager
var (
errContainerTooShort = errors.New("indexer: encoded container shorter than header")
errContainerBytesLen = errors.New("indexer: encoded container length exceeds payload")
)
func init() {
lc := linearcodec.NewDefault()
Codec = codec.NewManager(math.MaxInt)
// marshalContainer encodes c to its on-disk representation.
func marshalContainer(c Container) ([]byte, error) {
out := make([]byte, 0, ids.IDLen+4+len(c.Bytes)+8)
out = append(out, c.ID[:]...)
var lenBuf [4]byte
binary.BigEndian.PutUint32(lenBuf[:], uint32(len(c.Bytes)))
out = append(out, lenBuf[:]...)
out = append(out, c.Bytes...)
var tsBuf [8]byte
binary.BigEndian.PutUint64(tsBuf[:], uint64(c.Timestamp))
out = append(out, tsBuf[:]...)
return out, nil
}
if err := Codec.RegisterCodec(CodecVersion, lc); err != nil {
panic(err)
// unmarshalContainer decodes b into a Container.
func unmarshalContainer(b []byte) (Container, error) {
const headerLen = ids.IDLen + 4
if len(b) < headerLen+8 {
return Container{}, errContainerTooShort
}
var c Container
copy(c.ID[:], b[:ids.IDLen])
bytesLen := binary.BigEndian.Uint32(b[ids.IDLen : ids.IDLen+4])
if uint64(len(b)) < uint64(headerLen)+uint64(bytesLen)+8 {
return Container{}, errContainerBytesLen
}
payloadStart := headerLen
payloadEnd := payloadStart + int(bytesLen)
c.Bytes = make([]byte, bytesLen)
copy(c.Bytes, b[payloadStart:payloadEnd])
c.Timestamp = int64(binary.BigEndian.Uint64(b[payloadEnd : payloadEnd+8]))
return c, nil
}
indexer/index.go
+3 -3
View File
@@ -131,7 +131,7 @@ func (i *index) Accept(rt *runtime.Runtime, containerID ids.ID, containerBytes [
)
// Persist index --> Container
nextAcceptedIndexBytes := database.PackUInt64(i.nextAcceptedIndex)
bytes, err := Codec.Marshal(CodecVersion, Container{
bytes, err := marshalContainer(Container{
ID: containerID,
Bytes: containerBytes,
Timestamp: i.clock.Time().UnixNano(),
@@ -189,8 +189,8 @@ func (i *index) getContainerByIndexBytes(indexBytes []byte) (Container, error) {
)
return Container{}, fmt.Errorf("couldn't read from database: %w", err)
}
var container Container
if _, err := Codec.Unmarshal(containerBytes, &container); err != nil {
container, err := unmarshalContainer(containerBytes)
if err != nil {
return Container{}, fmt.Errorf("couldn't unmarshal container: %w", err)
}
return container, nil
indexer/indexer.go
+1 -1
View File
@@ -10,7 +10,6 @@ import (
"github.com/gorilla/rpc/v2"
"github.com/luxfi/codec/wrappers"
"github.com/luxfi/database"
"github.com/luxfi/database/prefixdb"
"github.com/luxfi/ids"
@@ -19,6 +18,7 @@ import (
nodeconsensus "github.com/luxfi/node/consensus"
"github.com/luxfi/node/server/http"
"github.com/luxfi/node/utils/json"
"github.com/luxfi/node/utils/wrappers"
"github.com/luxfi/runtime"
"github.com/luxfi/timer/mockable"
"github.com/luxfi/vm"
network/network.go
+1 -1
View File
@@ -18,7 +18,6 @@ import (
"github.com/pires/go-proxyproto"
"go.uber.org/zap"
"github.com/luxfi/codec/wrappers"
consensusconfig "github.com/luxfi/consensus/config"
consensustracker "github.com/luxfi/consensus/networking/tracker"
"github.com/luxfi/constants"
@@ -38,6 +37,7 @@ import (
"github.com/luxfi/node/network/tracker"
"github.com/luxfi/node/service/health"
"github.com/luxfi/node/utils/bloom"
"github.com/luxfi/node/utils/wrappers"
"github.com/luxfi/node/version"
"github.com/luxfi/node/vms/platformvm/genesis"
"github.com/luxfi/node/vms/platformvm/txs"
network/peer/endpoint_signer.go
+1 -1
View File
@@ -11,11 +11,11 @@ import (
"net/netip"
"time"
"github.com/luxfi/codec/wrappers"
"github.com/luxfi/crypto/bls"
"github.com/luxfi/crypto/hash"
"github.com/luxfi/net/endpoints"
"github.com/luxfi/node/staking"
"github.com/luxfi/node/utils/wrappers"
"github.com/luxfi/utils"
)
network/peer/ip.go
+1 -1
View File
@@ -12,11 +12,11 @@ import (
"net/netip"
"time"
"github.com/luxfi/codec/wrappers"
"github.com/luxfi/crypto/bls"
"github.com/luxfi/crypto/hash"
"github.com/luxfi/crypto/mldsa"
"github.com/luxfi/node/staking"
"github.com/luxfi/node/utils/wrappers"
)
var (
network/peer/msg_length.go
+1 -1
View File
@@ -8,7 +8,7 @@ import (
"errors"
"fmt"
"github.com/luxfi/codec/wrappers"
"github.com/luxfi/node/utils/wrappers"
)
var (
network/peer/peer.go
+4 -5
View File
@@ -22,15 +22,15 @@ import (
"github.com/luxfi/crypto/bls"
"github.com/luxfi/ids"
"github.com/luxfi/math/set"
"github.com/luxfi/net/endpoints"
"github.com/luxfi/node/message"
"github.com/luxfi/node/proto/p2p"
"github.com/luxfi/node/staking"
"github.com/luxfi/node/utils/bloom"
"github.com/luxfi/node/utils/json"
"github.com/luxfi/node/utils/wrappers"
"github.com/luxfi/node/version"
"github.com/luxfi/utils"
"github.com/luxfi/node/utils/bloom"
"github.com/luxfi/net/endpoints"
"github.com/luxfi/node/utils/json"
"github.com/luxfi/codec/wrappers"
)
const (
@@ -596,7 +596,6 @@ func (p *peer) readMessages() {
continue
}
now := p.Clock.Time()
p.storeLastReceived(now)
p.Metrics.Received(msg, msgLen)
service/keystore/codec.go
+136 -17
View File
@@ -1,31 +1,150 @@
// Copyright (C) 2019-2025, Lux Industries Inc. All rights reserved.
// Copyright (C) 2019-2026, Lux Industries Inc. All rights reserved.
// See the file LICENSE for licensing terms.
package keystore
import (
"github.com/luxfi/codec"
"github.com/luxfi/codec/linearcodec"
"encoding/binary"
"errors"
"math"
"github.com/luxfi/node/utils/password"
)
// On-disk encoding of keystore user records. Big-endian wire layout.
//
// Hash bytes (48 bytes, fixed):
//
// bytes 0..31 : Hash.Password (argon2id digest)
// bytes 32..47 : Hash.Salt
//
// user bytes:
//
// bytes 0..47 : Hash (as above)
// bytes 48..51 : len(Data) (uint32 BE)
// then for each kvPair:
// uint32 BE: len(Key)
// len(Key) bytes: Key
// uint32 BE: len(Value)
// len(Value) bytes: Value
//
// No codec version prefix. maxKeystoreBlob bounds total decoded size to
// the historic 16 MiB cap (papers/oom-audit-2026-04-12.tex F-4).
const (
CodecVersion = 0
// maxKeystoreBlob caps the size of any single keystore payload.
// Real user keystores are well under 1 MiB; 16 MiB is generous
// while bounding server-side allocation to a non-pathological size.
maxKeystoreBlob = 16 * 1024 * 1024
// maxPackerSize caps the size of any single keystore codec payload.
// Real user keystores are well under 1 MiB; the previous 1 GiB ceiling
// was an OOM vector for authenticated RPC callers. 16 MiB is generous
// for any plausible future wallet/seed container while bounding
// server-side allocation to a non-pathological size.
// See papers/oom-audit-2026-04-12.tex F-4.
maxPackerSize = 16 * 1024 * 1024 // 16 MiB
hashLen = 32 + 16
)
var Codec codec.Manager
var (
errKeystoreBlobOversize = errors.New("keystore: blob exceeds 16 MiB cap")
errKeystoreHashShort = errors.New("keystore: hash bytes too short")
errKeystoreUserShort = errors.New("keystore: user bytes too short")
errKeystoreTrailing = errors.New("keystore: trailing bytes after decode")
errKeystoreEntryOversize = errors.New("keystore: kv entry exceeds blob")
)
func init() {
lc := linearcodec.NewDefault()
Codec = codec.NewManager(maxPackerSize)
if err := Codec.RegisterCodec(CodecVersion, lc); err != nil {
panic(err)
func marshalHash(h *password.Hash) ([]byte, error) {
out := make([]byte, hashLen)
copy(out[:32], h.Password[:])
copy(out[32:], h.Salt[:])
return out, nil
}
func unmarshalHash(b []byte, h *password.Hash) error {
if len(b) < hashLen {
return errKeystoreHashShort
}
if len(b) > maxKeystoreBlob {
return errKeystoreBlobOversize
}
copy(h.Password[:], b[:32])
copy(h.Salt[:], b[32:hashLen])
if len(b) > hashLen {
return errKeystoreTrailing
}
return nil
}
func marshalUser(u *user) ([]byte, error) {
// Size estimate.
total := uint64(hashLen + 4)
for _, kv := range u.Data {
total += 4 + uint64(len(kv.Key)) + 4 + uint64(len(kv.Value))
}
if total > maxKeystoreBlob {
return nil, errKeystoreBlobOversize
}
out := make([]byte, 0, total)
out = append(out, u.Hash.Password[:]...)
out = append(out, u.Hash.Salt[:]...)
var lenBuf [4]byte
binary.BigEndian.PutUint32(lenBuf[:], uint32(len(u.Data)))
out = append(out, lenBuf[:]...)
for _, kv := range u.Data {
if len(kv.Key) > math.MaxUint32 || len(kv.Value) > math.MaxUint32 {
return nil, errKeystoreEntryOversize
}
binary.BigEndian.PutUint32(lenBuf[:], uint32(len(kv.Key)))
out = append(out, lenBuf[:]...)
out = append(out, kv.Key...)
binary.BigEndian.PutUint32(lenBuf[:], uint32(len(kv.Value)))
out = append(out, lenBuf[:]...)
out = append(out, kv.Value...)
}
return out, nil
}
func unmarshalUser(b []byte, u *user) error {
if len(b) > maxKeystoreBlob {
return errKeystoreBlobOversize
}
if len(b) < hashLen+4 {
return errKeystoreUserShort
}
copy(u.Hash.Password[:], b[:32])
copy(u.Hash.Salt[:], b[32:hashLen])
off := hashLen
count := binary.BigEndian.Uint32(b[off : off+4])
off += 4
// Bound the slice growth by the remaining payload — each entry
// is at least 8 bytes (two length prefixes), so count cannot
// exceed (len(b)-off)/8 without underflowing.
maxEntries := uint64(len(b)-off) / 8
if uint64(count) > maxEntries {
return errKeystoreEntryOversize
}
u.Data = make([]kvPair, count)
for i := range u.Data {
if off+4 > len(b) {
return errKeystoreUserShort
}
klen := binary.BigEndian.Uint32(b[off : off+4])
off += 4
if uint64(off)+uint64(klen) > uint64(len(b)) {
return errKeystoreEntryOversize
}
u.Data[i].Key = make([]byte, klen)
copy(u.Data[i].Key, b[off:off+int(klen)])
off += int(klen)
if off+4 > len(b) {
return errKeystoreUserShort
}
vlen := binary.BigEndian.Uint32(b[off : off+4])
off += 4
if uint64(off)+uint64(vlen) > uint64(len(b)) {
return errKeystoreEntryOversize
}
u.Data[i].Value = make([]byte, vlen)
copy(u.Data[i].Value, b[off:off+int(vlen)])
off += int(vlen)
}
if off != len(b) {
return errKeystoreTrailing
}
return nil
}
service/keystore/keystore.go
+6 -7
View File
@@ -15,9 +15,9 @@ import (
"github.com/luxfi/database/prefixdb"
"github.com/luxfi/ids"
"github.com/luxfi/log"
"github.com/luxfi/vm/chains/atomic"
"github.com/luxfi/node/utils/json"
"github.com/luxfi/node/utils/password"
"github.com/luxfi/vm/chains/atomic"
)
const (
@@ -187,7 +187,7 @@ func (ks *keystore) CreateUser(username, pw string) error {
return err
}
passwordBytes, err := Codec.Marshal(CodecVersion, passwordHash)
passwordBytes, err := marshalHash(passwordHash)
if err != nil {
return err
}
@@ -287,14 +287,14 @@ func (ks *keystore) ImportUser(username, pw string, userBytes []byte) error {
}
userData := user{}
if _, err := Codec.Unmarshal(userBytes, &userData); err != nil {
if err := unmarshalUser(userBytes, &userData); err != nil {
return err
}
if !userData.Hash.Check(pw) {
return fmt.Errorf("%w: user %q", errIncorrectPassword, username)
}
usrBytes, err := Codec.Marshal(CodecVersion, &userData.Hash)
usrBytes, err := marshalHash(&userData.Hash)
if err != nil {
return err
}
@@ -354,7 +354,7 @@ func (ks *keystore) ExportUser(username, pw string) ([]byte, error) {
}
// Return the byte representation of the user
return Codec.Marshal(CodecVersion, &userData)
return marshalUser(&userData)
}
func (ks *keystore) getPassword(username string) (*password.Hash, error) {
@@ -376,6 +376,5 @@ func (ks *keystore) getPassword(username string) (*password.Hash, error) {
}
passwordHash = &password.Hash{}
_, err = Codec.Unmarshal(userBytes, passwordHash)
return passwordHash, err
return passwordHash, unmarshalHash(userBytes, passwordHash)
}
utils/ips/claimed_ip_port.go
+2 -2
View File
@@ -10,7 +10,7 @@ import (
"github.com/luxfi/crypto/hash"
"github.com/luxfi/ids"
"github.com/luxfi/node/staking"
"github.com/luxfi/codec/wrappers"
"github.com/luxfi/node/utils/wrappers"
)
const (
@@ -50,7 +50,7 @@ func NewClaimedIPPort(
Raw: cert.Raw,
PublicKey: cert.PublicKey,
}
ip := &ClaimedIPPort{
Cert: cert,
AddrPort: ipPort,
utils/ips/ip_port.go
+1 -2
View File
@@ -1,7 +1,6 @@
// Copyright (C) 2019-2025, Lux Industries Inc. All rights reserved.
// See the file LICENSE for licensing terms.
package ips
import (
@@ -10,7 +9,7 @@ import (
"net"
"strconv"
"github.com/luxfi/codec/wrappers"
"github.com/luxfi/node/utils/wrappers"
)
const nullStr = "null"
utils/metric/averager.go
+3 -3
View File
@@ -8,7 +8,7 @@ import (
metric "github.com/luxfi/metric"
"github.com/luxfi/codec/wrappers"
"github.com/luxfi/node/utils/wrappers"
)
var ErrFailedRegistering = errors.New("failed registering metric")
@@ -34,11 +34,11 @@ func NewAveragerWithErrs(name, desc string, registry metric.Registry, errs *wrap
a := averager{
count: metricsInstance.NewCounter(
AppendNamespace(name, "count"),
"Total # of observations of " + desc,
"Total # of observations of "+desc,
),
sum: metricsInstance.NewGauge(
AppendNamespace(name, "sum"),
"Sum of " + desc,
"Sum of "+desc,
),
}
wallet/network/primary/api.go
+8 -21
View File
@@ -12,19 +12,18 @@ import (
gethcommon "github.com/luxfi/geth/common"
"github.com/luxfi/geth/ethclient"
"github.com/luxfi/codec"
"github.com/luxfi/constants"
"github.com/luxfi/ids"
"github.com/luxfi/math/set"
lux "github.com/luxfi/utxo"
_ "github.com/luxfi/node/vms/components/lux" // registers utxo.ParseUTXO ZAP dispatcher
"github.com/luxfi/node/vms/platformvm"
"github.com/luxfi/node/wallet/chain/p"
"github.com/luxfi/node/wallet/chain/x"
"github.com/luxfi/rpc"
"github.com/luxfi/sdk/info"
lux "github.com/luxfi/utxo"
ethcommon "github.com/luxfi/geth/common"
ptxs "github.com/luxfi/node/vms/platformvm/txs"
pbuilder "github.com/luxfi/node/wallet/chain/p/builder"
xbuilder "github.com/luxfi/node/wallet/chain/x/builder"
walletcommon "github.com/luxfi/node/wallet/network/primary/common"
@@ -165,12 +164,10 @@ func FetchState(
chains := []struct {
id ids.ID
client UTXOClient
codec codec.Manager
}{
{
id: constants.PlatformChainID,
client: pClient,
codec: ptxs.Codec,
},
}
// Only include the X-chain entry when the network actually serves an
@@ -181,25 +178,17 @@ func FetchState(
chains = append(chains, struct {
id ids.ID
client UTXOClient
codec codec.Manager
}{
id: xCTX.BlockchainID,
client: xClient,
codec: codec.NewDefaultManager(),
})
}
// {
// id: cCTX.BlockchainID,
// client: cClient,
// codec: evm.Codec,
// },
for _, destinationChain := range chains {
for _, sourceChain := range chains {
err = AddAllUTXOs(
ctx,
utxos,
destinationChain.client,
destinationChain.codec,
sourceChain.id,
destinationChain.id,
addrList,
@@ -264,14 +253,14 @@ func FetchEthState(
}
// AddAllUTXOs fetches all the UTXOs referenced by [addresses] that were sent
// from [sourceChainID] to [destinationChainID] from the [client]. It then uses
// [codec] to parse the returned UTXOs and it adds them into [utxos]. If [ctx]
// expires, then the returned error will be immediately reported.
// from [sourceChainID] to [destinationChainID] from the [client]. It parses
// the returned UTXOs via the registered ZAP wire dispatcher
// (utxo.ParseUTXO, wired by node/vms/components/lux) and adds them into
// [utxos]. If [ctx] expires, the returned error is reported immediately.
func AddAllUTXOs(
ctx context.Context,
utxos walletcommon.UTXOs,
client UTXOClient,
codec codec.Manager,
sourceChainID ids.ID,
destinationChainID ids.ID,
addrs []ids.ShortID,
@@ -300,13 +289,11 @@ func AddAllUTXOs(
}
for _, utxoBytes := range utxosBytes {
var utxo lux.UTXO
_, err := codec.Unmarshal(utxoBytes, &utxo)
u, err := lux.ParseUTXO(utxoBytes)
if err != nil {
return err
}
if err := utxos.AddUTXO(ctx, sourceChainID, destinationChainID, &utxo); err != nil {
if err := utxos.AddUTXO(ctx, sourceChainID, destinationChainID, u); err != nil {
return err
}
}
warp/chainipc.go
+1 -1
View File
@@ -9,10 +9,10 @@ import (
"path/filepath"
"slices"
"github.com/luxfi/codec/wrappers"
"github.com/luxfi/ids"
"github.com/luxfi/log"
nodeconsensus "github.com/luxfi/node/consensus"
"github.com/luxfi/node/utils/wrappers"
)
const (
warp/eventsocket.go
+3 -3
View File
@@ -10,18 +10,18 @@ import (
"syscall"
consensuscore "github.com/luxfi/consensus/core"
"github.com/luxfi/codec/wrappers"
"github.com/luxfi/ids"
"github.com/luxfi/log"
nodeconsensus "github.com/luxfi/node/consensus"
"github.com/luxfi/node/utils/wrappers"
"github.com/luxfi/node/warp/socket"
"github.com/luxfi/runtime"
"github.com/luxfi/utils"
)
var (
_ consensuscore.Acceptor = (*EventSockets)(nil)
_ nodeconsensus.Acceptor = (*eventSocketAcceptor)(nil)
_ consensuscore.Acceptor = (*EventSockets)(nil)
_ nodeconsensus.Acceptor = (*eventSocketAcceptor)(nil)
)
// eventSocketAcceptor adapts an eventSocket to the nodeconsensus.Acceptor interface
warp/socket/socket.go
+1 -1
View File
@@ -15,7 +15,7 @@ import (
"syscall"
"github.com/luxfi/log"
"github.com/luxfi/codec/wrappers"
"github.com/luxfi/node/utils/wrappers"
)
var (
x/archivedb/key.go
+1 -1
View File
@@ -7,7 +7,7 @@ import (
"encoding/binary"
"errors"
"github.com/luxfi/codec/wrappers"
"github.com/luxfi/node/utils/wrappers"
)
var (
x/merkledb/cache.go
+1 -1
View File
@@ -8,7 +8,7 @@ import (
"sync"
"github.com/luxfi/container/linked"
"github.com/luxfi/codec/wrappers"
"github.com/luxfi/node/utils/wrappers"
)
var errEmptyCacheTooLarge = errors.New("cache is empty yet still too large")