commit dc9b148ef8e41d6673f63f6872638b822d7c92bc Author: Hanzo Dev Date: Thu Dec 4 08:38:37 2025 -0800 Initial release diff --git a/atomic.go b/atomic.go new file mode 100644 index 0000000..92a8a21 --- /dev/null +++ b/atomic.go @@ -0,0 +1,63 @@ +// Copyright (C) 2019-2025, Lux Industries, Inc. All rights reserved. +// See the file LICENSE for licensing terms. + +package utils + +import ( + "encoding/json" + "sync" +) + +var ( + _ json.Marshaler = (*Atomic[struct{}])(nil) + _ json.Unmarshaler = (*Atomic[struct{}])(nil) +) + +type Atomic[T any] struct { + lock sync.RWMutex + value T +} + +func NewAtomic[T any](value T) *Atomic[T] { + return &Atomic[T]{ + value: value, + } +} + +func (a *Atomic[T]) Get() T { + a.lock.RLock() + defer a.lock.RUnlock() + + return a.value +} + +func (a *Atomic[T]) Set(value T) { + a.lock.Lock() + defer a.lock.Unlock() + + a.value = value +} + +func (a *Atomic[T]) Swap(value T) T { + a.lock.Lock() + defer a.lock.Unlock() + + old := a.value + a.value = value + + return old +} + +func (a *Atomic[T]) MarshalJSON() ([]byte, error) { + a.lock.RLock() + defer a.lock.RUnlock() + + return json.Marshal(a.value) +} + +func (a *Atomic[T]) UnmarshalJSON(b []byte) error { + a.lock.Lock() + defer a.lock.Unlock() + + return json.Unmarshal(b, &a.value) +} diff --git a/go.mod b/go.mod new file mode 100644 index 0000000..44061d7 --- /dev/null +++ b/go.mod @@ -0,0 +1,17 @@ +module github.com/luxfi/utils + +go 1.24.5 + +require ( + github.com/gorilla/rpc v1.2.1 + github.com/luxfi/log v1.1.22 + golang.org/x/sync v0.17.0 +) + +require ( + github.com/holiman/uint256 v1.3.2 // indirect + go.uber.org/multierr v1.11.0 // indirect + go.uber.org/zap v1.27.0 // indirect + golang.org/x/exp v0.0.0-20250718183923-645b1fa84792 // indirect + gopkg.in/natefinch/lumberjack.v2 v2.2.1 // indirect +) diff --git a/go.sum b/go.sum new file mode 100644 index 0000000..ea5c907 --- /dev/null +++ b/go.sum @@ -0,0 +1,26 @@ +github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= +github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/gorilla/rpc v1.2.1 h1:yC+LMV5esttgpVvNORL/xX4jvTTEUE30UZhZ5JF7K9k= +github.com/gorilla/rpc v1.2.1/go.mod h1:uNpOihAlF5xRFLuTYhfR0yfCTm0WTQSQttkMSptRfGk= +github.com/holiman/uint256 v1.3.2 h1:a9EgMPSC1AAaj1SZL5zIQD3WbwTuHrMGOerLjGmM/TA= +github.com/holiman/uint256 v1.3.2/go.mod h1:EOMSn4q6Nyt9P6efbI3bueV4e1b3dGlUCXeiRV4ng7E= +github.com/luxfi/log v1.1.22 h1:E+jX+ZZS4JX2HjuGdDTpLfrgFKEDD8byNYTa0m6HT1o= +github.com/luxfi/log v1.1.22/go.mod h1:Q2eeOT4alCF+/B6+k/eWtVZQ2HncDlpd9vUvkTF0Suc= +github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= +github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA= +github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= +go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= +go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= +go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= +go.uber.org/zap v1.27.0 h1:aJMhYGrd5QSmlpLMr2MftRKl7t8J8PTZPA732ud/XR8= +go.uber.org/zap v1.27.0/go.mod h1:GB2qFLM7cTU87MWRP2mPIjqfIDnGu+VIO4V/SdhGo2E= +golang.org/x/exp v0.0.0-20250718183923-645b1fa84792 h1:R9PFI6EUdfVKgwKjZef7QIwGcBKu86OEFpJ9nUEP2l4= +golang.org/x/exp v0.0.0-20250718183923-645b1fa84792/go.mod h1:A+z0yzpGtvnG90cToK5n2tu8UJVP2XUATh+r+sfOOOc= +golang.org/x/sync v0.17.0 h1:l60nONMj9l5drqw6jlhIELNv9I0A4OFgRsG9k2oT9Ug= +golang.org/x/sync v0.17.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI= +gopkg.in/natefinch/lumberjack.v2 v2.2.1 h1:bBRl1b0OH9s/DuPhuXpNl+VtCaJXFZ5/uEFST95x9zc= +gopkg.in/natefinch/lumberjack.v2 v2.2.1/go.mod h1:YD8tP3GAjkrDg1eZH7EGmyESg/lsYskCTPBJVb9jqSc= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/heap/map.go b/heap/map.go new file mode 100644 index 0000000..a10fda8 --- /dev/null +++ b/heap/map.go @@ -0,0 +1,132 @@ +// Copyright (C) 2019-2025, Lux Industries, Inc. All rights reserved. +// See the file LICENSE for licensing terms. + +package heap + +import ( + "container/heap" + + "github.com/luxfi/utils" +) + +var _ heap.Interface = (*indexedQueue[int, int])(nil) + +func MapValues[K comparable, V any](m Map[K, V]) []V { + result := make([]V, 0, m.Len()) + for _, e := range m.queue.entries { + result = append(result, e.v) + } + return result +} + +// NewMap returns a heap without duplicates ordered by its values +func NewMap[K comparable, V any](less func(a, b V) bool) Map[K, V] { + return Map[K, V]{ + queue: &indexedQueue[K, V]{ + queue: queue[entry[K, V]]{ + less: func(a, b entry[K, V]) bool { + return less(a.v, b.v) + }, + }, + index: make(map[K]int), + }, + } +} + +type Map[K comparable, V any] struct { + queue *indexedQueue[K, V] +} + +// Push returns the evicted previous value if present +func (m *Map[K, V]) Push(k K, v V) (V, bool) { + if i, ok := m.queue.index[k]; ok { + prev := m.queue.entries[i] + m.queue.entries[i].v = v + heap.Fix(m.queue, i) + return prev.v, true + } + + heap.Push(m.queue, entry[K, V]{k: k, v: v}) + return utils.Zero[V](), false +} + +func (m *Map[K, V]) Pop() (K, V, bool) { + if m.Len() == 0 { + return utils.Zero[K](), utils.Zero[V](), false + } + + popped := heap.Pop(m.queue).(entry[K, V]) + return popped.k, popped.v, true +} + +func (m *Map[K, V]) Peek() (K, V, bool) { + if m.Len() == 0 { + return utils.Zero[K](), utils.Zero[V](), false + } + + entry := m.queue.entries[0] + return entry.k, entry.v, true +} + +func (m *Map[K, V]) Len() int { + return m.queue.Len() +} + +func (m *Map[K, V]) Remove(k K) (V, bool) { + if i, ok := m.queue.index[k]; ok { + removed := heap.Remove(m.queue, i).(entry[K, V]) + return removed.v, true + } + return utils.Zero[V](), false +} + +func (m *Map[K, V]) Contains(k K) bool { + _, ok := m.queue.index[k] + return ok +} + +func (m *Map[K, V]) Get(k K) (V, bool) { + if i, ok := m.queue.index[k]; ok { + got := m.queue.entries[i] + return got.v, true + } + return utils.Zero[V](), false +} + +func (m *Map[K, V]) Fix(k K) { + if i, ok := m.queue.index[k]; ok { + heap.Fix(m.queue, i) + } +} + +type indexedQueue[K comparable, V any] struct { + queue[entry[K, V]] + index map[K]int +} + +func (h *indexedQueue[K, V]) Swap(i, j int) { + h.entries[i], h.entries[j] = h.entries[j], h.entries[i] + h.index[h.entries[i].k], h.index[h.entries[j].k] = i, j +} + +func (h *indexedQueue[K, V]) Push(x any) { + entry := x.(entry[K, V]) + h.entries = append(h.entries, entry) + h.index[entry.k] = len(h.index) +} + +func (h *indexedQueue[K, V]) Pop() any { + end := len(h.entries) - 1 + + popped := h.entries[end] + h.entries[end] = entry[K, V]{} + h.entries = h.entries[:end] + + delete(h.index, popped.k) + return popped +} + +type entry[K any, V any] struct { + k K + v V +} diff --git a/heap/queue.go b/heap/queue.go new file mode 100644 index 0000000..3544fe9 --- /dev/null +++ b/heap/queue.go @@ -0,0 +1,94 @@ +// Copyright (C) 2019-2025, Lux Industries, Inc. All rights reserved. +// See the file LICENSE for licensing terms. + +package heap + +import ( + "container/heap" + + "github.com/luxfi/utils" +) + +var _ heap.Interface = (*queue[int])(nil) + +// NewQueue returns an empty heap. See QueueOf for more. +func NewQueue[T any](less func(a, b T) bool) Queue[T] { + return QueueOf(less) +} + +// QueueOf returns a heap containing entries ordered by less. +func QueueOf[T any](less func(a, b T) bool, entries ...T) Queue[T] { + q := Queue[T]{ + queue: &queue[T]{ + entries: make([]T, len(entries)), + less: less, + }, + } + + copy(q.queue.entries, entries) + heap.Init(q.queue) + return q +} + +type Queue[T any] struct { + queue *queue[T] +} + +func (q *Queue[T]) Len() int { + return len(q.queue.entries) +} + +func (q *Queue[T]) Push(t T) { + heap.Push(q.queue, t) +} + +func (q *Queue[T]) Pop() (T, bool) { + if q.Len() == 0 { + return utils.Zero[T](), false + } + + return heap.Pop(q.queue).(T), true +} + +func (q *Queue[T]) Peek() (T, bool) { + if q.Len() == 0 { + return utils.Zero[T](), false + } + + return q.queue.entries[0], true +} + +func (q *Queue[T]) Fix(i int) { + heap.Fix(q.queue, i) +} + +type queue[T any] struct { + entries []T + less func(a, b T) bool +} + +func (q *queue[T]) Len() int { + return len(q.entries) +} + +func (q *queue[T]) Less(i, j int) bool { + return q.less(q.entries[i], q.entries[j]) +} + +func (q *queue[T]) Swap(i, j int) { + q.entries[i], q.entries[j] = q.entries[j], q.entries[i] +} + +func (q *queue[T]) Push(e any) { + q.entries = append(q.entries, e.(T)) +} + +func (q *queue[T]) Pop() any { + end := len(q.entries) - 1 + + popped := q.entries[end] + q.entries[end] = utils.Zero[T]() + q.entries = q.entries[:end] + + return popped +} diff --git a/json/codec.go b/json/codec.go new file mode 100644 index 0000000..150913a --- /dev/null +++ b/json/codec.go @@ -0,0 +1,60 @@ +// Copyright (C) 2019-2025, Lux Industries, Inc. All rights reserved. +// See the file LICENSE for licensing terms. + +package json + +import ( + "errors" + "fmt" + "net/http" + "strings" + "unicode" + "unicode/utf8" + + "github.com/gorilla/rpc/v2" + "github.com/gorilla/rpc/v2/json2" +) + +var ( + errUppercaseMethod = errors.New("method must start with a non-uppercase letter") + errInvalidArg = errors.New("couldn't unmarshal an argument. Ensure arguments are valid and properly formatted. See documentation for example calls") +) + +// NewCodec returns a new json codec that will convert the first character of +// the method to uppercase +func NewCodec() rpc.Codec { + return lowercase{json2.NewCodec()} +} + +type lowercase struct{ *json2.Codec } + +func (lc lowercase) NewRequest(r *http.Request) rpc.CodecRequest { + return &request{lc.Codec.NewRequest(r).(*json2.CodecRequest)} +} + +type request struct{ *json2.CodecRequest } + +func (r *request) Method() (string, error) { + method, err := r.CodecRequest.Method() + methodSections := strings.SplitN(method, ".", 2) + if len(methodSections) != 2 || err != nil { + return method, err + } + class, function := methodSections[0], methodSections[1] + firstRune, runeLen := utf8.DecodeRuneInString(function) + if firstRune == utf8.RuneError { + return method, nil + } + if unicode.IsUpper(firstRune) { + return method, errUppercaseMethod + } + uppercaseRune := string(unicode.ToUpper(firstRune)) + return fmt.Sprintf("%s.%s%s", class, uppercaseRune, function[runeLen:]), nil +} + +func (r *request) ReadRequest(args interface{}) error { + if err := r.CodecRequest.ReadRequest(args); err != nil { + return errInvalidArg + } + return nil +} diff --git a/json/json.go b/json/json.go new file mode 100644 index 0000000..8e9e4c1 --- /dev/null +++ b/json/json.go @@ -0,0 +1,75 @@ +// Copyright (C) 2019-2025, Lux Industries, Inc. All rights reserved. +// See the file LICENSE for licensing terms. + +// Package json provides JSON serialization utilities for numeric types. +package json + +import "strconv" + +const Null = "null" + +// Uint32 is a uint32 that can be JSON marshaled as a string. +type Uint32 uint32 + +func (u Uint32) MarshalJSON() ([]byte, error) { + return []byte(`"` + strconv.FormatUint(uint64(u), 10) + `"`), nil +} + +func (u *Uint32) UnmarshalJSON(b []byte) error { + str := string(b) + if str == Null { + return nil + } + if len(str) >= 2 { + if lastIndex := len(str) - 1; str[0] == '"' && str[lastIndex] == '"' { + str = str[1:lastIndex] + } + } + val, err := strconv.ParseUint(str, 10, 32) + *u = Uint32(val) + return err +} + +// Uint64 is a uint64 that can be JSON marshaled as a string. +type Uint64 uint64 + +func (u Uint64) MarshalJSON() ([]byte, error) { + return []byte(`"` + strconv.FormatUint(uint64(u), 10) + `"`), nil +} + +func (u *Uint64) UnmarshalJSON(b []byte) error { + str := string(b) + if str == Null { + return nil + } + if len(str) >= 2 { + if lastIndex := len(str) - 1; str[0] == '"' && str[lastIndex] == '"' { + str = str[1:lastIndex] + } + } + val, err := strconv.ParseUint(str, 10, 64) + *u = Uint64(val) + return err +} + +// Float64 is a float64 that can be JSON marshaled as a string. +type Float64 float64 + +func (f Float64) MarshalJSON() ([]byte, error) { + return []byte(`"` + strconv.FormatFloat(float64(f), 'f', -1, 64) + `"`), nil +} + +func (f *Float64) UnmarshalJSON(b []byte) error { + str := string(b) + if str == Null { + return nil + } + if len(str) >= 2 { + if lastIndex := len(str) - 1; str[0] == '"' && str[lastIndex] == '"' { + str = str[1:lastIndex] + } + } + val, err := strconv.ParseFloat(str, 64) + *f = Float64(val) + return err +} diff --git a/perms/perms.go b/perms/perms.go new file mode 100644 index 0000000..2aae2f9 --- /dev/null +++ b/perms/perms.go @@ -0,0 +1,19 @@ +// Copyright (C) 2019-2025, Lux Industries, Inc. All rights reserved. +// See the file LICENSE for licensing terms. + +// Package perms provides common file permission constants. +package perms + +import "os" + +// File permissions +const ( + ReadOnly = 0o400 + ReadWrite = 0o640 + ReadWriteExecute = 0o750 +) + +// Create creates a file with the given permissions. +func Create(name string, perm os.FileMode) (*os.File, error) { + return os.OpenFile(name, os.O_RDWR|os.O_CREATE|os.O_TRUNC, perm) +} diff --git a/profiler/profiler.go b/profiler/profiler.go new file mode 100644 index 0000000..4300f37 --- /dev/null +++ b/profiler/profiler.go @@ -0,0 +1,225 @@ +// Copyright (C) 2019-2025, Lux Industries, Inc. All rights reserved. +// See the file LICENSE for licensing terms. + +// Package profiler provides CPU, memory, and lock profiling utilities. +package profiler + +import ( + "errors" + "fmt" + "os" + "path/filepath" + "runtime" + "runtime/pprof" + "time" + + "golang.org/x/sync/errgroup" + + "github.com/luxfi/utils/perms" +) + +const ( + cpuProfileFile = "cpu.profile" + memProfileFile = "mem.profile" + lockProfileFile = "lock.profile" +) + +var ( + _ Profiler = (*profiler)(nil) + + errCPUProfilerRunning = errors.New("cpu profiler already running") + errCPUProfilerNotRunning = errors.New("cpu profiler doesn't exist") +) + +// Profiler provides methods for measuring process performance. +type Profiler interface { + StartCPUProfiler() error + StopCPUProfiler() error + MemoryProfile() error + LockProfile() error +} + +type profiler struct { + dir string + cpuProfileName string + memProfileName string + lockProfileName string + cpuProfileFile *os.File +} + +// New returns a new Profiler that writes to the given directory. +func New(dir string) Profiler { + return newProfiler(dir) +} + +func newProfiler(dir string) *profiler { + return &profiler{ + dir: dir, + cpuProfileName: filepath.Join(dir, cpuProfileFile), + memProfileName: filepath.Join(dir, memProfileFile), + lockProfileName: filepath.Join(dir, lockProfileFile), + } +} + +func (p *profiler) StartCPUProfiler() error { + if p.cpuProfileFile != nil { + return errCPUProfilerRunning + } + + if err := os.MkdirAll(p.dir, perms.ReadWriteExecute); err != nil { + return err + } + file, err := perms.Create(p.cpuProfileName, perms.ReadWrite) + if err != nil { + return err + } + if err := pprof.StartCPUProfile(file); err != nil { + file.Close() + return err + } + p.cpuProfileFile = file + return nil +} + +func (p *profiler) StopCPUProfiler() error { + if p.cpuProfileFile == nil { + return errCPUProfilerNotRunning + } + + pprof.StopCPUProfile() + err := p.cpuProfileFile.Close() + p.cpuProfileFile = nil + return err +} + +func (p *profiler) MemoryProfile() error { + if err := os.MkdirAll(p.dir, perms.ReadWriteExecute); err != nil { + return err + } + + file, err := perms.Create(p.memProfileName, perms.ReadWrite) + if err != nil { + return err + } + defer file.Close() + + runtime.GC() + return pprof.WriteHeapProfile(file) +} + +func (p *profiler) LockProfile() error { + if err := os.MkdirAll(p.dir, perms.ReadWriteExecute); err != nil { + return err + } + + file, err := perms.Create(p.lockProfileName, perms.ReadWrite) + if err != nil { + return err + } + defer file.Close() + + profile := pprof.Lookup("mutex") + if profile == nil { + return errors.New("mutex profile not found") + } + return profile.WriteTo(file, 0) +} + +// Config for continuous profiler. +type Config struct { + Dir string `json:"dir"` + Enabled bool `json:"enabled"` + Freq time.Duration `json:"freq"` + MaxNumFiles int `json:"maxNumFiles"` +} + +// ContinuousProfiler periodically captures profiles. +type ContinuousProfiler interface { + Dispatch() error + Shutdown() +} + +type continuousProfiler struct { + profiler *profiler + freq time.Duration + maxNumFiles int + closer chan struct{} +} + +// NewContinuous returns a new continuous profiler. +func NewContinuous(dir string, freq time.Duration, maxNumFiles int) ContinuousProfiler { + return &continuousProfiler{ + profiler: newProfiler(dir), + freq: freq, + maxNumFiles: maxNumFiles, + closer: make(chan struct{}), + } +} + +func (p *continuousProfiler) Dispatch() error { + t := time.NewTicker(p.freq) + defer t.Stop() + + for { + if err := p.start(); err != nil { + return err + } + + select { + case <-p.closer: + return p.stop() + case <-t.C: + if err := p.stop(); err != nil { + return err + } + } + + if err := p.rotate(); err != nil { + return err + } + } +} + +func (p *continuousProfiler) start() error { + return p.profiler.StartCPUProfiler() +} + +func (p *continuousProfiler) stop() error { + g := errgroup.Group{} + g.Go(p.profiler.StopCPUProfiler) + g.Go(p.profiler.MemoryProfile) + g.Go(p.profiler.LockProfile) + return g.Wait() +} + +func (p *continuousProfiler) rotate() error { + g := errgroup.Group{} + g.Go(func() error { return rotate(p.profiler.cpuProfileName, p.maxNumFiles) }) + g.Go(func() error { return rotate(p.profiler.memProfileName, p.maxNumFiles) }) + g.Go(func() error { return rotate(p.profiler.lockProfileName, p.maxNumFiles) }) + return g.Wait() +} + +func (p *continuousProfiler) Shutdown() { + close(p.closer) +} + +func rotate(name string, maxNumFiles int) error { + for i := maxNumFiles - 1; i > 0; i-- { + src := fmt.Sprintf("%s.%d", name, i) + dst := fmt.Sprintf("%s.%d", name, i+1) + if err := renameIfExists(src, dst); err != nil { + return err + } + } + return renameIfExists(name, name+".1") +} + +func renameIfExists(src, dst string) error { + if _, err := os.Stat(src); os.IsNotExist(err) { + return nil + } else if err != nil { + return err + } + return os.Rename(src, dst) +} diff --git a/ulimit/ulimit_bsd.go b/ulimit/ulimit_bsd.go new file mode 100644 index 0000000..8826f68 --- /dev/null +++ b/ulimit/ulimit_bsd.go @@ -0,0 +1,58 @@ +// Copyright (C) 2019-2025, Lux Industries, Inc. All rights reserved. +// See the file LICENSE for licensing terms. + +//go:build freebsd +// +build freebsd + +package ulimit + +import ( + "fmt" + "syscall" + + "github.com/luxfi/log" +) + +const DefaultFDLimit = 32 * 1024 + +// Set attempts to bump the Rlimit which has a soft (Cur) and a hard (Max) value. +// The soft limit is what is used by the kernel to report EMFILE errors. The hard +// limit is a secondary limit which the process can be bumped to without additional +// privileges. Bumping the Max limit further would require superuser privileges. +// If the value is below the recommendation warn on start. +// see: http://0pointer.net/blog/file-descriptor-limits.html +func Set(limit uint64, log log.Logger) error { + // Note: BSD Rlimit is type int64 + // ref: https://cs.opensource.google/go/x/sys/+/b874c991:unix/ztypes_freebsd_amd64.go + bsdMax := int64(limit) + var rLimit syscall.Rlimit + err := syscall.Getrlimit(syscall.RLIMIT_NOFILE, &rLimit) + if err != nil { + return fmt.Errorf("error getting rlimit: %w", err) + } + + if bsdMax > rLimit.Max { + return fmt.Errorf("error fd-limit: (%d) greater than max: (%d)", limit, rLimit.Max) + } + + rLimit.Cur = bsdMax + + // set new limit + if err := syscall.Setrlimit(syscall.RLIMIT_NOFILE, &rLimit); err != nil { + return fmt.Errorf("error setting fd-limit: %w", err) + } + + // verify limit + if err := syscall.Getrlimit(syscall.RLIMIT_NOFILE, &rLimit); err != nil { + return fmt.Errorf("error getting rlimit: %w", err) + } + + if rLimit.Cur < DefaultFDLimit { + log.Warn("fd-limit is less than recommended and could result in reduced performance", + "currentLimit", rLimit.Cur, + "recommendedLimit", DefaultFDLimit, + ) + } + + return nil +} diff --git a/ulimit/ulimit_darwin.go b/ulimit/ulimit_darwin.go new file mode 100644 index 0000000..1decb3a --- /dev/null +++ b/ulimit/ulimit_darwin.go @@ -0,0 +1,59 @@ +// Copyright (C) 2019-2025, Lux Industries, Inc. All rights reserved. +// See the file LICENSE for licensing terms. + +//go:build darwin +// +build darwin + +package ulimit + +import ( + "fmt" + "syscall" + + "github.com/luxfi/log" +) + +const DefaultFDLimit = 10 * 1024 + +// Set attempts to bump the Rlimit which has a soft (Cur) and a hard (Max) value. +// The soft limit is what is used by the kernel to report EMFILE errors. The hard +// limit is a secondary limit which the process can be bumped to without additional +// privileges. Bumping the Max limit further would require superuser privileges. +// If the value is below the recommendation warn on start. +// see: http://0pointer.net/blog/file-descriptor-limits.html +func Set(limit uint64, log log.Logger) error { + var rLimit syscall.Rlimit + err := syscall.Getrlimit(syscall.RLIMIT_NOFILE, &rLimit) + if err != nil { + return fmt.Errorf("error getting rlimit: %w", err) + } + + // Darwin max number of FDs to allocatable for darwin systems. + // The max file limit is 10240, even though the max returned by + // Getrlimit is 1<<63-1. This is OPEN_MAX in sys/syslimits.h. + // See https://github.com/golang/go/issues/30401 + if limit > DefaultFDLimit { + return fmt.Errorf("error fd-limit: (%d) greater than max: (%d)", limit, DefaultFDLimit) + } + + rLimit.Cur = limit + + // set new limit + if err := syscall.Setrlimit(syscall.RLIMIT_NOFILE, &rLimit); err != nil { + return fmt.Errorf("error setting fd-limit: %w", err) + } + + // verify limit + if err := syscall.Getrlimit(syscall.RLIMIT_NOFILE, &rLimit); err != nil { + return fmt.Errorf("error getting rlimit: %w", err) + } + + if rLimit.Cur < DefaultFDLimit { + log.Warn("fd-limit is less than recommended and could result in reduced performance", + "currentLimit", rLimit.Cur, + "recommendedLimit", DefaultFDLimit, + ) + } + + return nil +} diff --git a/ulimit/ulimit_unix.go b/ulimit/ulimit_unix.go new file mode 100644 index 0000000..d8aa3be --- /dev/null +++ b/ulimit/ulimit_unix.go @@ -0,0 +1,55 @@ +// Copyright (C) 2019-2025, Lux Industries, Inc. All rights reserved. +// See the file LICENSE for licensing terms. + +//go:build linux || netbsd || openbsd +// +build linux netbsd openbsd + +package ulimit + +import ( + "fmt" + "syscall" + + "github.com/luxfi/log" +) + +const DefaultFDLimit = 32 * 1024 + +// Set attempts to bump the Rlimit which has a soft (Cur) and a hard (Max) value. +// The soft limit is what is used by the kernel to report EMFILE errors. The hard +// limit is a secondary limit which the process can be bumped to without additional +// privileges. Bumping the Max limit further would require superuser privileges. +// If the current Max is below our recommendation we will warn on start. +// see: http://0pointer.net/blog/file-descriptor-limits.html +func Set(limit uint64, log log.Logger) error { + var rLimit syscall.Rlimit + err := syscall.Getrlimit(syscall.RLIMIT_NOFILE, &rLimit) + if err != nil { + return fmt.Errorf("error getting rlimit: %w", err) + } + + if limit > rLimit.Max { + return fmt.Errorf("error fd-limit: (%d) greater than max: (%d)", limit, rLimit.Max) + } + + rLimit.Cur = limit + + // set new limit + if err := syscall.Setrlimit(syscall.RLIMIT_NOFILE, &rLimit); err != nil { + return fmt.Errorf("error setting fd-limit: %w", err) + } + + // verify limit + if err := syscall.Getrlimit(syscall.RLIMIT_NOFILE, &rLimit); err != nil { + return fmt.Errorf("error getting rlimit: %w", err) + } + + if rLimit.Cur < DefaultFDLimit { + log.Warn("fd-limit is less than recommended and could result in reduced performance", + "currentLimit", rLimit.Cur, + "recommendedLimit", DefaultFDLimit, + ) + } + + return nil +} diff --git a/ulimit/ulimit_windows.go b/ulimit/ulimit_windows.go new file mode 100644 index 0000000..c879c9d --- /dev/null +++ b/ulimit/ulimit_windows.go @@ -0,0 +1,19 @@ +// Copyright (C) 2019-2025, Lux Industries, Inc. All rights reserved. +// See the file LICENSE for licensing terms. + +//go:build windows +// +build windows + +package ulimit + +import "github.com/luxfi/log" + +const DefaultFDLimit = 16384 + +// Set is a no-op for windows and will warn if the default is not used. +func Set(limit uint64, log log.Logger) error { + if limit != DefaultFDLimit { + log.Warn("fd-limit is not supported for windows") + } + return nil +} diff --git a/wrappers/errors.go b/wrappers/errors.go new file mode 100644 index 0000000..879cf01 --- /dev/null +++ b/wrappers/errors.go @@ -0,0 +1,42 @@ +// Copyright (C) 2019-2025, Lux Industries, Inc. All rights reserved. +// See the file LICENSE for licensing terms. + +// Package wrappers provides common wrapper types and utilities. +package wrappers + +const ( + // ByteLen is the number of bytes per byte + ByteLen = 1 + // ShortLen is the number of bytes per short + ShortLen = 2 + // IntLen is the number of bytes per int + IntLen = 4 + // LongLen is the number of bytes per long + LongLen = 8 + // BoolLen is the number of bytes per bool + BoolLen = 1 + // IPLen is the number of bytes per IP + IPLen = 16 + ShortLen +) + +// Errs collects errors during a series of operations. +type Errs struct { + Err error +} + +// Errored returns true if an error has been recorded. +func (errs *Errs) Errored() bool { + return errs.Err != nil +} + +// Add records the first non-nil error. +func (errs *Errs) Add(errors ...error) { + if errs.Err == nil { + for _, err := range errors { + if err != nil { + errs.Err = err + break + } + } + } +} diff --git a/wrappers/packer.go b/wrappers/packer.go new file mode 100644 index 0000000..9a2edaa --- /dev/null +++ b/wrappers/packer.go @@ -0,0 +1,259 @@ +// Copyright (C) 2019-2025, Lux Industries, Inc. All rights reserved. +// See the file LICENSE for licensing terms. + +package wrappers + +import ( + "encoding/binary" + "errors" + "math" +) + +const ( + MaxStringLen = math.MaxUint16 +) + +var ( + ErrInsufficientLength = errors.New("packer has insufficient length for input") + errNegativeOffset = errors.New("negative offset") + errInvalidInput = errors.New("input does not match expected format") + errBadBool = errors.New("unexpected value when unpacking bool") + errOversized = errors.New("size is larger than limit") +) + +// StringLen returns the packed length of a string +func StringLen(str string) int { + return ShortLen + len(str) +} + +// Packer packs and unpacks a byte array from/to standard values +type Packer struct { + Errs + + // The largest allowed size of expanding the byte array + MaxSize int + // The current byte array + Bytes []byte + // The offset that is being written to in the byte array + Offset int +} + +// PackByte appends a byte to the byte array +func (p *Packer) PackByte(val byte) { + p.expand(ByteLen) + if p.Errored() { + return + } + + p.Bytes[p.Offset] = val + p.Offset++ +} + +// UnpackByte unpacks a byte from the byte array +func (p *Packer) UnpackByte() byte { + p.checkSpace(ByteLen) + if p.Errored() { + return 0 + } + + val := p.Bytes[p.Offset] + p.Offset += ByteLen + return val +} + +// PackShort appends a short to the byte array +func (p *Packer) PackShort(val uint16) { + p.expand(ShortLen) + if p.Errored() { + return + } + + binary.BigEndian.PutUint16(p.Bytes[p.Offset:], val) + p.Offset += ShortLen +} + +// UnpackShort unpacks a short from the byte array +func (p *Packer) UnpackShort() uint16 { + p.checkSpace(ShortLen) + if p.Errored() { + return 0 + } + + val := binary.BigEndian.Uint16(p.Bytes[p.Offset:]) + p.Offset += ShortLen + return val +} + +// PackInt appends an int to the byte array +func (p *Packer) PackInt(val uint32) { + p.expand(IntLen) + if p.Errored() { + return + } + + binary.BigEndian.PutUint32(p.Bytes[p.Offset:], val) + p.Offset += IntLen +} + +// UnpackInt unpacks an int from the byte array +func (p *Packer) UnpackInt() uint32 { + p.checkSpace(IntLen) + if p.Errored() { + return 0 + } + + val := binary.BigEndian.Uint32(p.Bytes[p.Offset:]) + p.Offset += IntLen + return val +} + +// PackLong appends a long to the byte array +func (p *Packer) PackLong(val uint64) { + p.expand(LongLen) + if p.Errored() { + return + } + + binary.BigEndian.PutUint64(p.Bytes[p.Offset:], val) + p.Offset += LongLen +} + +// UnpackLong unpacks a long from the byte array +func (p *Packer) UnpackLong() uint64 { + p.checkSpace(LongLen) + if p.Errored() { + return 0 + } + + val := binary.BigEndian.Uint64(p.Bytes[p.Offset:]) + p.Offset += LongLen + return val +} + +// PackBool packs a bool into the byte array +func (p *Packer) PackBool(b bool) { + if b { + p.PackByte(1) + } else { + p.PackByte(0) + } +} + +// UnpackBool unpacks a bool from the byte array +func (p *Packer) UnpackBool() bool { + b := p.UnpackByte() + switch b { + case 0: + return false + case 1: + return true + default: + p.Add(errBadBool) + return false + } +} + +// PackFixedBytes appends a byte slice with no length descriptor to the byte array +func (p *Packer) PackFixedBytes(bytes []byte) { + p.expand(len(bytes)) + if p.Errored() { + return + } + + copy(p.Bytes[p.Offset:], bytes) + p.Offset += len(bytes) +} + +// UnpackFixedBytes unpacks a byte slice with no length descriptor from the byte array +func (p *Packer) UnpackFixedBytes(size int) []byte { + p.checkSpace(size) + if p.Errored() { + return nil + } + + bytes := p.Bytes[p.Offset : p.Offset+size] + p.Offset += size + return bytes +} + +// PackBytes appends a byte slice to the byte array +func (p *Packer) PackBytes(bytes []byte) { + p.PackInt(uint32(len(bytes))) + p.PackFixedBytes(bytes) +} + +// UnpackBytes unpacks a byte slice from the byte array +func (p *Packer) UnpackBytes() []byte { + size := p.UnpackInt() + return p.UnpackFixedBytes(int(size)) +} + +// UnpackLimitedBytes unpacks a byte slice. If the size of the slice is greater +// than limit, adds errOversized to the packer and returns nil. +func (p *Packer) UnpackLimitedBytes(limit uint32) []byte { + size := p.UnpackInt() + if size > limit { + p.Add(errOversized) + return nil + } + return p.UnpackFixedBytes(int(size)) +} + +// PackStr appends a string to the byte array +func (p *Packer) PackStr(str string) { + strSize := len(str) + if strSize > MaxStringLen { + p.Add(errInvalidInput) + return + } + p.PackShort(uint16(strSize)) + p.PackFixedBytes([]byte(str)) +} + +// UnpackStr unpacks a string from the byte array +func (p *Packer) UnpackStr() string { + strSize := p.UnpackShort() + return string(p.UnpackFixedBytes(int(strSize))) +} + +// UnpackLimitedStr unpacks a string. If the size of the string is greater than +// limit, adds errOversized to the packer and returns the empty string. +func (p *Packer) UnpackLimitedStr(limit uint16) string { + strSize := p.UnpackShort() + if strSize > limit { + p.Add(errOversized) + return "" + } + return string(p.UnpackFixedBytes(int(strSize))) +} + +// checkSpace requires that there is at least bytes of write space left in the +// byte array. If this is not true, an error is added to the packer. +func (p *Packer) checkSpace(bytes int) { + switch { + case p.Offset < 0: + p.Add(errNegativeOffset) + case bytes < 0: + p.Add(errInvalidInput) + case len(p.Bytes)-p.Offset < bytes: + p.Add(ErrInsufficientLength) + } +} + +// expand ensures that there is bytes bytes left of space in the byte slice. +// If this is not allowed due to the maximum size, an error is added to the packer. +func (p *Packer) expand(bytes int) { + neededSize := bytes + p.Offset + switch { + case neededSize <= len(p.Bytes): + return + case neededSize > p.MaxSize: + p.Err = ErrInsufficientLength + return + case neededSize <= cap(p.Bytes): + p.Bytes = p.Bytes[:neededSize] + return + default: + p.Bytes = append(p.Bytes[:cap(p.Bytes)], make([]byte, neededSize-cap(p.Bytes))...) + } +} diff --git a/zero.go b/zero.go new file mode 100644 index 0000000..6ae5a55 --- /dev/null +++ b/zero.go @@ -0,0 +1,10 @@ +// Copyright (C) 2019-2025, Lux Industries, Inc. All rights reserved. +// See the file LICENSE for licensing terms. + +package utils + +// Zero returns the zero value of any type T. +func Zero[T any]() T { + var zero T + return zero +}