Initial release

atomic.go

@@ -0,0 +1,63 @@
+// Copyright (C) 2019-2025, Lux Industries, Inc. All rights reserved.
+// See the file LICENSE for licensing terms.
+
+package utils
+
+import (
+	"encoding/json"
+	"sync"
+)
+
+var (
+	_ json.Marshaler   = (*Atomic[struct{}])(nil)
+	_ json.Unmarshaler = (*Atomic[struct{}])(nil)
+)
+
+type Atomic[T any] struct {
+	lock  sync.RWMutex
+	value T
+}
+
+func NewAtomic[T any](value T) *Atomic[T] {
+	return &Atomic[T]{
+		value: value,
+	}
+}
+
+func (a *Atomic[T]) Get() T {
+	a.lock.RLock()
+	defer a.lock.RUnlock()
+
+	return a.value
+}
+
+func (a *Atomic[T]) Set(value T) {
+	a.lock.Lock()
+	defer a.lock.Unlock()
+
+	a.value = value
+}
+
+func (a *Atomic[T]) Swap(value T) T {
+	a.lock.Lock()
+	defer a.lock.Unlock()
+
+	old := a.value
+	a.value = value
+
+	return old
+}
+
+func (a *Atomic[T]) MarshalJSON() ([]byte, error) {
+	a.lock.RLock()
+	defer a.lock.RUnlock()
+
+	return json.Marshal(a.value)
+}
+
+func (a *Atomic[T]) UnmarshalJSON(b []byte) error {
+	a.lock.Lock()
+	defer a.lock.Unlock()
+
+	return json.Unmarshal(b, &a.value)
+}

go.mod

@@ -0,0 +1,17 @@
+module github.com/luxfi/utils
+
+go 1.24.5
+
+require (
+	github.com/gorilla/rpc v1.2.1
+	github.com/luxfi/log v1.1.22
+	golang.org/x/sync v0.17.0
+)
+
+require (
+	github.com/holiman/uint256 v1.3.2 // indirect
+	go.uber.org/multierr v1.11.0 // indirect
+	go.uber.org/zap v1.27.0 // indirect
+	golang.org/x/exp v0.0.0-20250718183923-645b1fa84792 // indirect
+	gopkg.in/natefinch/lumberjack.v2 v2.2.1 // indirect
+)

go.sum

@@ -0,0 +1,26 @@
+github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM=
+github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/gorilla/rpc v1.2.1 h1:yC+LMV5esttgpVvNORL/xX4jvTTEUE30UZhZ5JF7K9k=
+github.com/gorilla/rpc v1.2.1/go.mod h1:uNpOihAlF5xRFLuTYhfR0yfCTm0WTQSQttkMSptRfGk=
+github.com/holiman/uint256 v1.3.2 h1:a9EgMPSC1AAaj1SZL5zIQD3WbwTuHrMGOerLjGmM/TA=
+github.com/holiman/uint256 v1.3.2/go.mod h1:EOMSn4q6Nyt9P6efbI3bueV4e1b3dGlUCXeiRV4ng7E=
+github.com/luxfi/log v1.1.22 h1:E+jX+ZZS4JX2HjuGdDTpLfrgFKEDD8byNYTa0m6HT1o=
+github.com/luxfi/log v1.1.22/go.mod h1:Q2eeOT4alCF+/B6+k/eWtVZQ2HncDlpd9vUvkTF0Suc=
+github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U=
+github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA=
+github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
+go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto=
+go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE=
+go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0=
+go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y=
+go.uber.org/zap v1.27.0 h1:aJMhYGrd5QSmlpLMr2MftRKl7t8J8PTZPA732ud/XR8=
+go.uber.org/zap v1.27.0/go.mod h1:GB2qFLM7cTU87MWRP2mPIjqfIDnGu+VIO4V/SdhGo2E=
+golang.org/x/exp v0.0.0-20250718183923-645b1fa84792 h1:R9PFI6EUdfVKgwKjZef7QIwGcBKu86OEFpJ9nUEP2l4=
+golang.org/x/exp v0.0.0-20250718183923-645b1fa84792/go.mod h1:A+z0yzpGtvnG90cToK5n2tu8UJVP2XUATh+r+sfOOOc=
+golang.org/x/sync v0.17.0 h1:l60nONMj9l5drqw6jlhIELNv9I0A4OFgRsG9k2oT9Ug=
+golang.org/x/sync v0.17.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI=
+gopkg.in/natefinch/lumberjack.v2 v2.2.1 h1:bBRl1b0OH9s/DuPhuXpNl+VtCaJXFZ5/uEFST95x9zc=
+gopkg.in/natefinch/lumberjack.v2 v2.2.1/go.mod h1:YD8tP3GAjkrDg1eZH7EGmyESg/lsYskCTPBJVb9jqSc=
+gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
+gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=

heap/map.go

@@ -0,0 +1,132 @@
+// Copyright (C) 2019-2025, Lux Industries, Inc. All rights reserved.
+// See the file LICENSE for licensing terms.
+
+package heap
+
+import (
+	"container/heap"
+
+	"github.com/luxfi/utils"
+)
+
+var _ heap.Interface = (*indexedQueue[int, int])(nil)
+
+func MapValues[K comparable, V any](m Map[K, V]) []V {
+	result := make([]V, 0, m.Len())
+	for _, e := range m.queue.entries {
+		result = append(result, e.v)
+	}
+	return result
+}
+
+// NewMap returns a heap without duplicates ordered by its values
+func NewMap[K comparable, V any](less func(a, b V) bool) Map[K, V] {
+	return Map[K, V]{
+		queue: &indexedQueue[K, V]{
+			queue: queue[entry[K, V]]{
+				less: func(a, b entry[K, V]) bool {
+					return less(a.v, b.v)
+				},
+			},
+			index: make(map[K]int),
+		},
+	}
+}
+
+type Map[K comparable, V any] struct {
+	queue *indexedQueue[K, V]
+}
+
+// Push returns the evicted previous value if present
+func (m *Map[K, V]) Push(k K, v V) (V, bool) {
+	if i, ok := m.queue.index[k]; ok {
+		prev := m.queue.entries[i]
+		m.queue.entries[i].v = v
+		heap.Fix(m.queue, i)
+		return prev.v, true
+	}
+
+	heap.Push(m.queue, entry[K, V]{k: k, v: v})
+	return utils.Zero[V](), false
+}
+
+func (m *Map[K, V]) Pop() (K, V, bool) {
+	if m.Len() == 0 {
+		return utils.Zero[K](), utils.Zero[V](), false
+	}
+
+	popped := heap.Pop(m.queue).(entry[K, V])
+	return popped.k, popped.v, true
+}
+
+func (m *Map[K, V]) Peek() (K, V, bool) {
+	if m.Len() == 0 {
+		return utils.Zero[K](), utils.Zero[V](), false
+	}
+
+	entry := m.queue.entries[0]
+	return entry.k, entry.v, true
+}
+
+func (m *Map[K, V]) Len() int {
+	return m.queue.Len()
+}
+
+func (m *Map[K, V]) Remove(k K) (V, bool) {
+	if i, ok := m.queue.index[k]; ok {
+		removed := heap.Remove(m.queue, i).(entry[K, V])
+		return removed.v, true
+	}
+	return utils.Zero[V](), false
+}
+
+func (m *Map[K, V]) Contains(k K) bool {
+	_, ok := m.queue.index[k]
+	return ok
+}
+
+func (m *Map[K, V]) Get(k K) (V, bool) {
+	if i, ok := m.queue.index[k]; ok {
+		got := m.queue.entries[i]
+		return got.v, true
+	}
+	return utils.Zero[V](), false
+}
+
+func (m *Map[K, V]) Fix(k K) {
+	if i, ok := m.queue.index[k]; ok {
+		heap.Fix(m.queue, i)
+	}
+}
+
+type indexedQueue[K comparable, V any] struct {
+	queue[entry[K, V]]
+	index map[K]int
+}
+
+func (h *indexedQueue[K, V]) Swap(i, j int) {
+	h.entries[i], h.entries[j] = h.entries[j], h.entries[i]
+	h.index[h.entries[i].k], h.index[h.entries[j].k] = i, j
+}
+
+func (h *indexedQueue[K, V]) Push(x any) {
+	entry := x.(entry[K, V])
+	h.entries = append(h.entries, entry)
+	h.index[entry.k] = len(h.index)
+}
+
+func (h *indexedQueue[K, V]) Pop() any {
+	end := len(h.entries) - 1
+
+	popped := h.entries[end]
+	h.entries[end] = entry[K, V]{}
+	h.entries = h.entries[:end]
+
+	delete(h.index, popped.k)
+	return popped
+}
+
+type entry[K any, V any] struct {
+	k K
+	v V
+}

heap/queue.go

@@ -0,0 +1,94 @@
+// Copyright (C) 2019-2025, Lux Industries, Inc. All rights reserved.
+// See the file LICENSE for licensing terms.
+
+package heap
+
+import (
+	"container/heap"
+
+	"github.com/luxfi/utils"
+)
+
+var _ heap.Interface = (*queue[int])(nil)
+
+// NewQueue returns an empty heap. See QueueOf for more.
+func NewQueue[T any](less func(a, b T) bool) Queue[T] {
+	return QueueOf(less)
+}
+
+// QueueOf returns a heap containing entries ordered by less.
+func QueueOf[T any](less func(a, b T) bool, entries ...T) Queue[T] {
+	q := Queue[T]{
+		queue: &queue[T]{
+			entries: make([]T, len(entries)),
+			less:    less,
+		},
+	}
+
+	copy(q.queue.entries, entries)
+	heap.Init(q.queue)
+	return q
+}
+
+type Queue[T any] struct {
+	queue *queue[T]
+}
+
+func (q *Queue[T]) Len() int {
+	return len(q.queue.entries)
+}
+
+func (q *Queue[T]) Push(t T) {
+	heap.Push(q.queue, t)
+}
+
+func (q *Queue[T]) Pop() (T, bool) {
+	if q.Len() == 0 {
+		return utils.Zero[T](), false
+	}
+
+	return heap.Pop(q.queue).(T), true
+}
+
+func (q *Queue[T]) Peek() (T, bool) {
+	if q.Len() == 0 {
+		return utils.Zero[T](), false
+	}
+
+	return q.queue.entries[0], true
+}
+
+func (q *Queue[T]) Fix(i int) {
+	heap.Fix(q.queue, i)
+}
+
+type queue[T any] struct {
+	entries []T
+	less    func(a, b T) bool
+}
+
+func (q *queue[T]) Len() int {
+	return len(q.entries)
+}
+
+func (q *queue[T]) Less(i, j int) bool {
+	return q.less(q.entries[i], q.entries[j])
+}
+
+func (q *queue[T]) Swap(i, j int) {
+	q.entries[i], q.entries[j] = q.entries[j], q.entries[i]
+}
+
+func (q *queue[T]) Push(e any) {
+	q.entries = append(q.entries, e.(T))
+}
+
+func (q *queue[T]) Pop() any {
+	end := len(q.entries) - 1
+
+	popped := q.entries[end]
+	q.entries[end] = utils.Zero[T]()
+	q.entries = q.entries[:end]
+
+	return popped
+}

json/codec.go

@@ -0,0 +1,60 @@
+// Copyright (C) 2019-2025, Lux Industries, Inc. All rights reserved.
+// See the file LICENSE for licensing terms.
+
+package json
+
+import (
+	"errors"
+	"fmt"
+	"net/http"
+	"strings"
+	"unicode"
+	"unicode/utf8"
+
+	"github.com/gorilla/rpc/v2"
+	"github.com/gorilla/rpc/v2/json2"
+)
+
+var (
+	errUppercaseMethod = errors.New("method must start with a non-uppercase letter")
+	errInvalidArg      = errors.New("couldn't unmarshal an argument. Ensure arguments are valid and properly formatted. See documentation for example calls")
+)
+
+// NewCodec returns a new json codec that will convert the first character of
+// the method to uppercase
+func NewCodec() rpc.Codec {
+	return lowercase{json2.NewCodec()}
+}
+
+type lowercase struct{ *json2.Codec }
+
+func (lc lowercase) NewRequest(r *http.Request) rpc.CodecRequest {
+	return &request{lc.Codec.NewRequest(r).(*json2.CodecRequest)}
+}
+
+type request struct{ *json2.CodecRequest }
+
+func (r *request) Method() (string, error) {
+	method, err := r.CodecRequest.Method()
+	methodSections := strings.SplitN(method, ".", 2)
+	if len(methodSections) != 2 || err != nil {
+		return method, err
+	}
+	class, function := methodSections[0], methodSections[1]
+	firstRune, runeLen := utf8.DecodeRuneInString(function)
+	if firstRune == utf8.RuneError {
+		return method, nil
+	}
+	if unicode.IsUpper(firstRune) {
+		return method, errUppercaseMethod
+	}
+	uppercaseRune := string(unicode.ToUpper(firstRune))
+	return fmt.Sprintf("%s.%s%s", class, uppercaseRune, function[runeLen:]), nil
+}
+
+func (r *request) ReadRequest(args interface{}) error {
+	if err := r.CodecRequest.ReadRequest(args); err != nil {
+		return errInvalidArg
+	}
+	return nil
+}

json/json.go

@@ -0,0 +1,75 @@
+// Copyright (C) 2019-2025, Lux Industries, Inc. All rights reserved.
+// See the file LICENSE for licensing terms.
+
+// Package json provides JSON serialization utilities for numeric types.
+package json
+
+import "strconv"
+
+const Null = "null"
+
+// Uint32 is a uint32 that can be JSON marshaled as a string.
+type Uint32 uint32
+
+func (u Uint32) MarshalJSON() ([]byte, error) {
+	return []byte(`"` + strconv.FormatUint(uint64(u), 10) + `"`), nil
+}
+
+func (u *Uint32) UnmarshalJSON(b []byte) error {
+	str := string(b)
+	if str == Null {
+		return nil
+	}
+	if len(str) >= 2 {
+		if lastIndex := len(str) - 1; str[0] == '"' && str[lastIndex] == '"' {
+			str = str[1:lastIndex]
+		}
+	}
+	val, err := strconv.ParseUint(str, 10, 32)
+	*u = Uint32(val)
+	return err
+}
+
+// Uint64 is a uint64 that can be JSON marshaled as a string.
+type Uint64 uint64
+
+func (u Uint64) MarshalJSON() ([]byte, error) {
+	return []byte(`"` + strconv.FormatUint(uint64(u), 10) + `"`), nil
+}
+
+func (u *Uint64) UnmarshalJSON(b []byte) error {
+	str := string(b)
+	if str == Null {
+		return nil
+	}
+	if len(str) >= 2 {
+		if lastIndex := len(str) - 1; str[0] == '"' && str[lastIndex] == '"' {
+			str = str[1:lastIndex]
+		}
+	}
+	val, err := strconv.ParseUint(str, 10, 64)
+	*u = Uint64(val)
+	return err
+}
+
+// Float64 is a float64 that can be JSON marshaled as a string.
+type Float64 float64
+
+func (f Float64) MarshalJSON() ([]byte, error) {
+	return []byte(`"` + strconv.FormatFloat(float64(f), 'f', -1, 64) + `"`), nil
+}
+
+func (f *Float64) UnmarshalJSON(b []byte) error {
+	str := string(b)
+	if str == Null {
+		return nil
+	}
+	if len(str) >= 2 {
+		if lastIndex := len(str) - 1; str[0] == '"' && str[lastIndex] == '"' {
+			str = str[1:lastIndex]
+		}
+	}
+	val, err := strconv.ParseFloat(str, 64)
+	*f = Float64(val)
+	return err
+}

perms/perms.go

@@ -0,0 +1,19 @@
+// Copyright (C) 2019-2025, Lux Industries, Inc. All rights reserved.
+// See the file LICENSE for licensing terms.
+
+// Package perms provides common file permission constants.
+package perms
+
+import "os"
+
+// File permissions
+const (
+	ReadOnly         = 0o400
+	ReadWrite        = 0o640
+	ReadWriteExecute = 0o750
+)
+
+// Create creates a file with the given permissions.
+func Create(name string, perm os.FileMode) (*os.File, error) {
+	return os.OpenFile(name, os.O_RDWR|os.O_CREATE|os.O_TRUNC, perm)
+}

profiler/profiler.go

@@ -0,0 +1,225 @@
+// Copyright (C) 2019-2025, Lux Industries, Inc. All rights reserved.
+// See the file LICENSE for licensing terms.
+
+// Package profiler provides CPU, memory, and lock profiling utilities.
+package profiler
+
+import (
+	"errors"
+	"fmt"
+	"os"
+	"path/filepath"
+	"runtime"
+	"runtime/pprof"
+	"time"
+
+	"golang.org/x/sync/errgroup"
+
+	"github.com/luxfi/utils/perms"
+)
+
+const (
+	cpuProfileFile  = "cpu.profile"
+	memProfileFile  = "mem.profile"
+	lockProfileFile = "lock.profile"
+)
+
+var (
+	_ Profiler = (*profiler)(nil)
+
+	errCPUProfilerRunning    = errors.New("cpu profiler already running")
+	errCPUProfilerNotRunning = errors.New("cpu profiler doesn't exist")
+)
+
+// Profiler provides methods for measuring process performance.
+type Profiler interface {
+	StartCPUProfiler() error
+	StopCPUProfiler() error
+	MemoryProfile() error
+	LockProfile() error
+}
+
+type profiler struct {
+	dir             string
+	cpuProfileName  string
+	memProfileName  string
+	lockProfileName string
+	cpuProfileFile  *os.File
+}
+
+// New returns a new Profiler that writes to the given directory.
+func New(dir string) Profiler {
+	return newProfiler(dir)
+}
+
+func newProfiler(dir string) *profiler {
+	return &profiler{
+		dir:             dir,
+		cpuProfileName:  filepath.Join(dir, cpuProfileFile),
+		memProfileName:  filepath.Join(dir, memProfileFile),
+		lockProfileName: filepath.Join(dir, lockProfileFile),
+	}
+}
+
+func (p *profiler) StartCPUProfiler() error {
+	if p.cpuProfileFile != nil {
+		return errCPUProfilerRunning
+	}
+
+	if err := os.MkdirAll(p.dir, perms.ReadWriteExecute); err != nil {
+		return err
+	}
+	file, err := perms.Create(p.cpuProfileName, perms.ReadWrite)
+	if err != nil {
+		return err
+	}
+	if err := pprof.StartCPUProfile(file); err != nil {
+		file.Close()
+		return err
+	}
+	p.cpuProfileFile = file
+	return nil
+}
+
+func (p *profiler) StopCPUProfiler() error {
+	if p.cpuProfileFile == nil {
+		return errCPUProfilerNotRunning
+	}
+
+	pprof.StopCPUProfile()
+	err := p.cpuProfileFile.Close()
+	p.cpuProfileFile = nil
+	return err
+}
+
+func (p *profiler) MemoryProfile() error {
+	if err := os.MkdirAll(p.dir, perms.ReadWriteExecute); err != nil {
+		return err
+	}
+
+	file, err := perms.Create(p.memProfileName, perms.ReadWrite)
+	if err != nil {
+		return err
+	}
+	defer file.Close()
+
+	runtime.GC()
+	return pprof.WriteHeapProfile(file)
+}
+
+func (p *profiler) LockProfile() error {
+	if err := os.MkdirAll(p.dir, perms.ReadWriteExecute); err != nil {
+		return err
+	}
+
+	file, err := perms.Create(p.lockProfileName, perms.ReadWrite)
+	if err != nil {
+		return err
+	}
+	defer file.Close()
+
+	profile := pprof.Lookup("mutex")
+	if profile == nil {
+		return errors.New("mutex profile not found")
+	}
+	return profile.WriteTo(file, 0)
+}
+
+// Config for continuous profiler.
+type Config struct {
+	Dir         string        `json:"dir"`
+	Enabled     bool          `json:"enabled"`
+	Freq        time.Duration `json:"freq"`
+	MaxNumFiles int           `json:"maxNumFiles"`
+}
+
+// ContinuousProfiler periodically captures profiles.
+type ContinuousProfiler interface {
+	Dispatch() error
+	Shutdown()
+}
+
+type continuousProfiler struct {
+	profiler    *profiler
+	freq        time.Duration
+	maxNumFiles int
+	closer      chan struct{}
+}
+
+// NewContinuous returns a new continuous profiler.
+func NewContinuous(dir string, freq time.Duration, maxNumFiles int) ContinuousProfiler {
+	return &continuousProfiler{
+		profiler:    newProfiler(dir),
+		freq:        freq,
+		maxNumFiles: maxNumFiles,
+		closer:      make(chan struct{}),
+	}
+}
+
+func (p *continuousProfiler) Dispatch() error {
+	t := time.NewTicker(p.freq)
+	defer t.Stop()
+
+	for {
+		if err := p.start(); err != nil {
+			return err
+		}
+
+		select {
+		case <-p.closer:
+			return p.stop()
+		case <-t.C:
+			if err := p.stop(); err != nil {
+				return err
+			}
+		}
+
+		if err := p.rotate(); err != nil {
+			return err
+		}
+	}
+}
+
+func (p *continuousProfiler) start() error {
+	return p.profiler.StartCPUProfiler()
+}
+
+func (p *continuousProfiler) stop() error {
+	g := errgroup.Group{}
+	g.Go(p.profiler.StopCPUProfiler)
+	g.Go(p.profiler.MemoryProfile)
+	g.Go(p.profiler.LockProfile)
+	return g.Wait()
+}
+
+func (p *continuousProfiler) rotate() error {
+	g := errgroup.Group{}
+	g.Go(func() error { return rotate(p.profiler.cpuProfileName, p.maxNumFiles) })
+	g.Go(func() error { return rotate(p.profiler.memProfileName, p.maxNumFiles) })
+	g.Go(func() error { return rotate(p.profiler.lockProfileName, p.maxNumFiles) })
+	return g.Wait()
+}
+
+func (p *continuousProfiler) Shutdown() {
+	close(p.closer)
+}
+
+func rotate(name string, maxNumFiles int) error {
+	for i := maxNumFiles - 1; i > 0; i-- {
+		src := fmt.Sprintf("%s.%d", name, i)
+		dst := fmt.Sprintf("%s.%d", name, i+1)
+		if err := renameIfExists(src, dst); err != nil {
+			return err
+		}
+	}
+	return renameIfExists(name, name+".1")
+}
+
+func renameIfExists(src, dst string) error {
+	if _, err := os.Stat(src); os.IsNotExist(err) {
+		return nil
+	} else if err != nil {
+		return err
+	}
+	return os.Rename(src, dst)
+}

ulimit/ulimit_bsd.go

@@ -0,0 +1,58 @@
+// Copyright (C) 2019-2025, Lux Industries, Inc. All rights reserved.
+// See the file LICENSE for licensing terms.
+
+//go:build freebsd
+// +build freebsd
+
+package ulimit
+
+import (
+	"fmt"
+	"syscall"
+
+	"github.com/luxfi/log"
+)
+
+const DefaultFDLimit = 32 * 1024
+
+// Set attempts to bump the Rlimit which has a soft (Cur) and a hard (Max) value.
+// The soft limit is what is used by the kernel to report EMFILE errors. The hard
+// limit is a secondary limit which the process can be bumped to without additional
+// privileges. Bumping the Max limit further would require superuser privileges.
+// If the value is below the recommendation warn on start.
+// see: http://0pointer.net/blog/file-descriptor-limits.html
+func Set(limit uint64, log log.Logger) error {
+	// Note: BSD Rlimit is type int64
+	// ref: https://cs.opensource.google/go/x/sys/+/b874c991:unix/ztypes_freebsd_amd64.go
+	bsdMax := int64(limit)
+	var rLimit syscall.Rlimit
+	err := syscall.Getrlimit(syscall.RLIMIT_NOFILE, &rLimit)
+	if err != nil {
+		return fmt.Errorf("error getting rlimit: %w", err)
+	}
+
+	if bsdMax > rLimit.Max {
+		return fmt.Errorf("error fd-limit: (%d) greater than max: (%d)", limit, rLimit.Max)
+	}
+
+	rLimit.Cur = bsdMax
+
+	// set new limit
+	if err := syscall.Setrlimit(syscall.RLIMIT_NOFILE, &rLimit); err != nil {
+		return fmt.Errorf("error setting fd-limit: %w", err)
+	}
+
+	// verify limit
+	if err := syscall.Getrlimit(syscall.RLIMIT_NOFILE, &rLimit); err != nil {
+		return fmt.Errorf("error getting rlimit: %w", err)
+	}
+
+	if rLimit.Cur < DefaultFDLimit {
+		log.Warn("fd-limit is less than recommended and could result in reduced performance",
+			"currentLimit", rLimit.Cur,
+			"recommendedLimit", DefaultFDLimit,
+		)
+	}
+
+	return nil
+}

ulimit/ulimit_darwin.go

@@ -0,0 +1,59 @@
+// Copyright (C) 2019-2025, Lux Industries, Inc. All rights reserved.
+// See the file LICENSE for licensing terms.
+
+//go:build darwin
+// +build darwin
+
+package ulimit
+
+import (
+	"fmt"
+	"syscall"
+
+	"github.com/luxfi/log"
+)
+
+const DefaultFDLimit = 10 * 1024
+
+// Set attempts to bump the Rlimit which has a soft (Cur) and a hard (Max) value.
+// The soft limit is what is used by the kernel to report EMFILE errors. The hard
+// limit is a secondary limit which the process can be bumped to without additional
+// privileges. Bumping the Max limit further would require superuser privileges.
+// If the value is below the recommendation warn on start.
+// see: http://0pointer.net/blog/file-descriptor-limits.html
+func Set(limit uint64, log log.Logger) error {
+	var rLimit syscall.Rlimit
+	err := syscall.Getrlimit(syscall.RLIMIT_NOFILE, &rLimit)
+	if err != nil {
+		return fmt.Errorf("error getting rlimit: %w", err)
+	}
+
+	// Darwin max number of FDs to allocatable for darwin systems.
+	// The max file limit is 10240, even though the max returned by
+	// Getrlimit is 1<<63-1. This is OPEN_MAX in sys/syslimits.h.
+	// See https://github.com/golang/go/issues/30401
+	if limit > DefaultFDLimit {
+		return fmt.Errorf("error fd-limit: (%d) greater than max: (%d)", limit, DefaultFDLimit)
+	}
+
+	rLimit.Cur = limit
+
+	// set new limit
+	if err := syscall.Setrlimit(syscall.RLIMIT_NOFILE, &rLimit); err != nil {
+		return fmt.Errorf("error setting fd-limit: %w", err)
+	}
+
+	// verify limit
+	if err := syscall.Getrlimit(syscall.RLIMIT_NOFILE, &rLimit); err != nil {
+		return fmt.Errorf("error getting rlimit: %w", err)
+	}
+
+	if rLimit.Cur < DefaultFDLimit {
+		log.Warn("fd-limit is less than recommended and could result in reduced performance",
+			"currentLimit", rLimit.Cur,
+			"recommendedLimit", DefaultFDLimit,
+		)
+	}
+
+	return nil
+}

ulimit/ulimit_unix.go

@@ -0,0 +1,55 @@
+// Copyright (C) 2019-2025, Lux Industries, Inc. All rights reserved.
+// See the file LICENSE for licensing terms.
+
+//go:build linux || netbsd || openbsd
+// +build linux netbsd openbsd
+
+package ulimit
+
+import (
+	"fmt"
+	"syscall"
+
+	"github.com/luxfi/log"
+)
+
+const DefaultFDLimit = 32 * 1024
+
+// Set attempts to bump the Rlimit which has a soft (Cur) and a hard (Max) value.
+// The soft limit is what is used by the kernel to report EMFILE errors. The hard
+// limit is a secondary limit which the process can be bumped to without additional
+// privileges. Bumping the Max limit further would require superuser privileges.
+// If the current Max is below our recommendation we will warn on start.
+// see: http://0pointer.net/blog/file-descriptor-limits.html
+func Set(limit uint64, log log.Logger) error {
+	var rLimit syscall.Rlimit
+	err := syscall.Getrlimit(syscall.RLIMIT_NOFILE, &rLimit)
+	if err != nil {
+		return fmt.Errorf("error getting rlimit: %w", err)
+	}
+
+	if limit > rLimit.Max {
+		return fmt.Errorf("error fd-limit: (%d) greater than max: (%d)", limit, rLimit.Max)
+	}
+
+	rLimit.Cur = limit
+
+	// set new limit
+	if err := syscall.Setrlimit(syscall.RLIMIT_NOFILE, &rLimit); err != nil {
+		return fmt.Errorf("error setting fd-limit: %w", err)
+	}
+
+	// verify limit
+	if err := syscall.Getrlimit(syscall.RLIMIT_NOFILE, &rLimit); err != nil {
+		return fmt.Errorf("error getting rlimit: %w", err)
+	}
+
+	if rLimit.Cur < DefaultFDLimit {
+		log.Warn("fd-limit is less than recommended and could result in reduced performance",
+			"currentLimit", rLimit.Cur,
+			"recommendedLimit", DefaultFDLimit,
+		)
+	}
+
+	return nil
+}

ulimit/ulimit_windows.go

@@ -0,0 +1,19 @@
+// Copyright (C) 2019-2025, Lux Industries, Inc. All rights reserved.
+// See the file LICENSE for licensing terms.
+
+//go:build windows
+// +build windows
+
+package ulimit
+
+import "github.com/luxfi/log"
+
+const DefaultFDLimit = 16384
+
+// Set is a no-op for windows and will warn if the default is not used.
+func Set(limit uint64, log log.Logger) error {
+	if limit != DefaultFDLimit {
+		log.Warn("fd-limit is not supported for windows")
+	}
+	return nil
+}

wrappers/errors.go

@@ -0,0 +1,42 @@
+// Copyright (C) 2019-2025, Lux Industries, Inc. All rights reserved.
+// See the file LICENSE for licensing terms.
+
+// Package wrappers provides common wrapper types and utilities.
+package wrappers
+
+const (
+	// ByteLen is the number of bytes per byte
+	ByteLen = 1
+	// ShortLen is the number of bytes per short
+	ShortLen = 2
+	// IntLen is the number of bytes per int
+	IntLen = 4
+	// LongLen is the number of bytes per long
+	LongLen = 8
+	// BoolLen is the number of bytes per bool
+	BoolLen = 1
+	// IPLen is the number of bytes per IP
+	IPLen = 16 + ShortLen
+)
+
+// Errs collects errors during a series of operations.
+type Errs struct {
+	Err error
+}
+
+// Errored returns true if an error has been recorded.
+func (errs *Errs) Errored() bool {
+	return errs.Err != nil
+}
+
+// Add records the first non-nil error.
+func (errs *Errs) Add(errors ...error) {
+	if errs.Err == nil {
+		for _, err := range errors {
+			if err != nil {
+				errs.Err = err
+				break
+			}
+		}
+	}
+}

wrappers/packer.go

@@ -0,0 +1,259 @@
+// Copyright (C) 2019-2025, Lux Industries, Inc. All rights reserved.
+// See the file LICENSE for licensing terms.
+
+package wrappers
+
+import (
+	"encoding/binary"
+	"errors"
+	"math"
+)
+
+const (
+	MaxStringLen = math.MaxUint16
+)
+
+var (
+	ErrInsufficientLength = errors.New("packer has insufficient length for input")
+	errNegativeOffset     = errors.New("negative offset")
+	errInvalidInput       = errors.New("input does not match expected format")
+	errBadBool            = errors.New("unexpected value when unpacking bool")
+	errOversized          = errors.New("size is larger than limit")
+)
+
+// StringLen returns the packed length of a string
+func StringLen(str string) int {
+	return ShortLen + len(str)
+}
+
+// Packer packs and unpacks a byte array from/to standard values
+type Packer struct {
+	Errs
+
+	// The largest allowed size of expanding the byte array
+	MaxSize int
+	// The current byte array
+	Bytes []byte
+	// The offset that is being written to in the byte array
+	Offset int
+}
+
+// PackByte appends a byte to the byte array
+func (p *Packer) PackByte(val byte) {
+	p.expand(ByteLen)
+	if p.Errored() {
+		return
+	}
+
+	p.Bytes[p.Offset] = val
+	p.Offset++
+}
+
+// UnpackByte unpacks a byte from the byte array
+func (p *Packer) UnpackByte() byte {
+	p.checkSpace(ByteLen)
+	if p.Errored() {
+		return 0
+	}
+
+	val := p.Bytes[p.Offset]
+	p.Offset += ByteLen
+	return val
+}
+
+// PackShort appends a short to the byte array
+func (p *Packer) PackShort(val uint16) {
+	p.expand(ShortLen)
+	if p.Errored() {
+		return
+	}
+
+	binary.BigEndian.PutUint16(p.Bytes[p.Offset:], val)
+	p.Offset += ShortLen
+}
+
+// UnpackShort unpacks a short from the byte array
+func (p *Packer) UnpackShort() uint16 {
+	p.checkSpace(ShortLen)
+	if p.Errored() {
+		return 0
+	}
+
+	val := binary.BigEndian.Uint16(p.Bytes[p.Offset:])
+	p.Offset += ShortLen
+	return val
+}
+
+// PackInt appends an int to the byte array
+func (p *Packer) PackInt(val uint32) {
+	p.expand(IntLen)
+	if p.Errored() {
+		return
+	}
+
+	binary.BigEndian.PutUint32(p.Bytes[p.Offset:], val)
+	p.Offset += IntLen
+}
+
+// UnpackInt unpacks an int from the byte array
+func (p *Packer) UnpackInt() uint32 {
+	p.checkSpace(IntLen)
+	if p.Errored() {
+		return 0
+	}
+
+	val := binary.BigEndian.Uint32(p.Bytes[p.Offset:])
+	p.Offset += IntLen
+	return val
+}
+
+// PackLong appends a long to the byte array
+func (p *Packer) PackLong(val uint64) {
+	p.expand(LongLen)
+	if p.Errored() {
+		return
+	}
+
+	binary.BigEndian.PutUint64(p.Bytes[p.Offset:], val)
+	p.Offset += LongLen
+}
+
+// UnpackLong unpacks a long from the byte array
+func (p *Packer) UnpackLong() uint64 {
+	p.checkSpace(LongLen)
+	if p.Errored() {
+		return 0
+	}
+
+	val := binary.BigEndian.Uint64(p.Bytes[p.Offset:])
+	p.Offset += LongLen
+	return val
+}
+
+// PackBool packs a bool into the byte array
+func (p *Packer) PackBool(b bool) {
+	if b {
+		p.PackByte(1)
+	} else {
+		p.PackByte(0)
+	}
+}
+
+// UnpackBool unpacks a bool from the byte array
+func (p *Packer) UnpackBool() bool {
+	b := p.UnpackByte()
+	switch b {
+	case 0:
+		return false
+	case 1:
+		return true
+	default:
+		p.Add(errBadBool)
+		return false
+	}
+}
+
+// PackFixedBytes appends a byte slice with no length descriptor to the byte array
+func (p *Packer) PackFixedBytes(bytes []byte) {
+	p.expand(len(bytes))
+	if p.Errored() {
+		return
+	}
+
+	copy(p.Bytes[p.Offset:], bytes)
+	p.Offset += len(bytes)
+}
+
+// UnpackFixedBytes unpacks a byte slice with no length descriptor from the byte array
+func (p *Packer) UnpackFixedBytes(size int) []byte {
+	p.checkSpace(size)
+	if p.Errored() {
+		return nil
+	}
+
+	bytes := p.Bytes[p.Offset : p.Offset+size]
+	p.Offset += size
+	return bytes
+}
+
+// PackBytes appends a byte slice to the byte array
+func (p *Packer) PackBytes(bytes []byte) {
+	p.PackInt(uint32(len(bytes)))
+	p.PackFixedBytes(bytes)
+}
+
+// UnpackBytes unpacks a byte slice from the byte array
+func (p *Packer) UnpackBytes() []byte {
+	size := p.UnpackInt()
+	return p.UnpackFixedBytes(int(size))
+}
+
+// UnpackLimitedBytes unpacks a byte slice. If the size of the slice is greater
+// than limit, adds errOversized to the packer and returns nil.
+func (p *Packer) UnpackLimitedBytes(limit uint32) []byte {
+	size := p.UnpackInt()
+	if size > limit {
+		p.Add(errOversized)
+		return nil
+	}
+	return p.UnpackFixedBytes(int(size))
+}
+
+// PackStr appends a string to the byte array
+func (p *Packer) PackStr(str string) {
+	strSize := len(str)
+	if strSize > MaxStringLen {
+		p.Add(errInvalidInput)
+		return
+	}
+	p.PackShort(uint16(strSize))
+	p.PackFixedBytes([]byte(str))
+}
+
+// UnpackStr unpacks a string from the byte array
+func (p *Packer) UnpackStr() string {
+	strSize := p.UnpackShort()
+	return string(p.UnpackFixedBytes(int(strSize)))
+}
+
+// UnpackLimitedStr unpacks a string. If the size of the string is greater than
+// limit, adds errOversized to the packer and returns the empty string.
+func (p *Packer) UnpackLimitedStr(limit uint16) string {
+	strSize := p.UnpackShort()
+	if strSize > limit {
+		p.Add(errOversized)
+		return ""
+	}
+	return string(p.UnpackFixedBytes(int(strSize)))
+}
+
+// checkSpace requires that there is at least bytes of write space left in the
+// byte array. If this is not true, an error is added to the packer.
+func (p *Packer) checkSpace(bytes int) {
+	switch {
+	case p.Offset < 0:
+		p.Add(errNegativeOffset)
+	case bytes < 0:
+		p.Add(errInvalidInput)
+	case len(p.Bytes)-p.Offset < bytes:
+		p.Add(ErrInsufficientLength)
+	}
+}
+
+// expand ensures that there is bytes bytes left of space in the byte slice.
+// If this is not allowed due to the maximum size, an error is added to the packer.
+func (p *Packer) expand(bytes int) {
+	neededSize := bytes + p.Offset
+	switch {
+	case neededSize <= len(p.Bytes):
+		return
+	case neededSize > p.MaxSize:
+		p.Err = ErrInsufficientLength
+		return
+	case neededSize <= cap(p.Bytes):
+		p.Bytes = p.Bytes[:neededSize]
+		return
+	default:
+		p.Bytes = append(p.Bytes[:cap(p.Bytes)], make([]byte, neededSize-cap(p.Bytes))...)
+	}
+}

zero.go

@@ -0,0 +1,10 @@
+// Copyright (C) 2019-2025, Lux Industries, Inc. All rights reserved.
+// See the file LICENSE for licensing terms.
+
+package utils
+
+// Zero returns the zero value of any type T.
+func Zero[T any]() T {
+	var zero T
+	return zero
+}