feat: ZAP MsgMetricBatch=2 receiver (v0.4.0)
Hosts a luxfi/zap node (opt-in via O11Y_ZAP_PORT) that decodes luxfi/metric.MetricBatch from MsgMetricBatch messages (Root().Bytes(0), the o11y wire convention) and ingests into the addressed tenant's store. Matches luxfi/metric.NewZAPExporter exactly, so any in-cluster luxfi/metric exporter pushes here over ZAP without the HTTP hop. Tenant = batch Resource[org] else brand. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
Antje Worring
co-authored by
Claude Opus 4.8
Claude Opus 4.8
parent
a5a86ce17a
commit
b7f36804e9
@@ -5,12 +5,15 @@ go 1.26.4
|
||||
require (
|
||||
github.com/hanzoai/cloud v0.1.1
|
||||
github.com/hanzoai/zip v0.2.0
|
||||
github.com/luxfi/log v1.4.3
|
||||
github.com/luxfi/metric v1.5.8
|
||||
github.com/luxfi/zap v0.7.2
|
||||
)
|
||||
|
||||
require (
|
||||
github.com/andybalholm/brotli v1.2.1 // indirect
|
||||
github.com/cenkalti/backoff v2.2.1+incompatible // indirect
|
||||
github.com/cloudflare/circl v1.6.3 // indirect
|
||||
github.com/gofiber/fiber/v3 v3.2.0 // indirect
|
||||
github.com/gofiber/schema v1.7.1 // indirect
|
||||
github.com/gofiber/utils/v2 v2.0.4 // indirect
|
||||
@@ -18,9 +21,9 @@ require (
|
||||
github.com/gorilla/rpc v1.2.1 // indirect
|
||||
github.com/grandcat/zeroconf v1.0.0 // indirect
|
||||
github.com/klauspost/compress v1.18.5 // indirect
|
||||
github.com/luxfi/log v1.4.3 // indirect
|
||||
github.com/luxfi/mdns v0.1.0 // indirect
|
||||
github.com/luxfi/zap v0.6.0 // indirect
|
||||
github.com/luxfi/accel v1.1.9 // indirect
|
||||
github.com/luxfi/crypto v1.19.17 // indirect
|
||||
github.com/luxfi/mdns v0.1.1 // indirect
|
||||
github.com/mattn/go-colorable v0.1.14 // indirect
|
||||
github.com/mattn/go-isatty v0.0.21 // indirect
|
||||
github.com/miekg/dns v1.1.72 // indirect
|
||||
@@ -28,13 +31,13 @@ require (
|
||||
github.com/tinylib/msgp v1.6.4 // indirect
|
||||
github.com/valyala/bytebufferpool v1.0.0 // indirect
|
||||
github.com/valyala/fasthttp v1.70.0 // indirect
|
||||
golang.org/x/crypto v0.50.0 // indirect
|
||||
golang.org/x/mod v0.34.0 // indirect
|
||||
golang.org/x/net v0.53.0 // indirect
|
||||
golang.org/x/crypto v0.52.0 // indirect
|
||||
golang.org/x/mod v0.36.0 // indirect
|
||||
golang.org/x/net v0.55.0 // indirect
|
||||
golang.org/x/sync v0.20.0 // indirect
|
||||
golang.org/x/sys v0.43.0 // indirect
|
||||
golang.org/x/text v0.36.0 // indirect
|
||||
golang.org/x/tools v0.43.0 // indirect
|
||||
golang.org/x/sys v0.45.0 // indirect
|
||||
golang.org/x/text v0.37.0 // indirect
|
||||
golang.org/x/tools v0.45.0 // indirect
|
||||
google.golang.org/protobuf v1.36.11 // indirect
|
||||
gopkg.in/natefinch/lumberjack.v2 v2.2.1 // indirect
|
||||
)
|
||||
|
||||
@@ -2,6 +2,8 @@ github.com/andybalholm/brotli v1.2.1 h1:R+f5xP285VArJDRgowrfb9DqL18yVK0gKAW/F+eT
|
||||
github.com/andybalholm/brotli v1.2.1/go.mod h1:rzTDkvFWvIrjDXZHkuS16NPggd91W3kUSvPlQ1pLaKY=
|
||||
github.com/cenkalti/backoff v2.2.1+incompatible h1:tNowT99t7UNflLxfYYSlKYsBpXdEet03Pg2g16Swow4=
|
||||
github.com/cenkalti/backoff v2.2.1+incompatible/go.mod h1:90ReRw6GdpyfrHakVjL/QHaoyV4aDUVVkXQJJJ3NXXM=
|
||||
github.com/cloudflare/circl v1.6.3 h1:9GPOhQGF9MCYUeXyMYlqTR6a5gTrgR/fBLXvUgtVcg8=
|
||||
github.com/cloudflare/circl v1.6.3/go.mod h1:2eXP6Qfat4O/Yhh8BznvKnJ+uzEoTQ6jVKJRn81BiS4=
|
||||
github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM=
|
||||
github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||
github.com/fxamacker/cbor/v2 v2.9.1 h1:2rWm8B193Ll4VdjsJY28jxs70IdDsHRWgQYAI80+rMQ=
|
||||
@@ -26,16 +28,20 @@ github.com/hanzoai/zip v0.2.0 h1:2GI1OoUv4QdCxmfiwoiyYsSHpiPAxrp0v+Gu4BLtTVI=
|
||||
github.com/hanzoai/zip v0.2.0/go.mod h1:3G+k2wy5bQ1wld66m0OPH8/LJ0kgIeNi1KPymMmlbf4=
|
||||
github.com/klauspost/compress v1.18.5 h1:/h1gH5Ce+VWNLSWqPzOVn6XBO+vJbCNGvjoaGBFW2IE=
|
||||
github.com/klauspost/compress v1.18.5/go.mod h1:cwPg85FWrGar70rWktvGQj8/hthj3wpl0PGDogxkrSQ=
|
||||
github.com/luxfi/accel v1.1.9 h1:Tsk6gXj2uKE19501bD0ajRYdeCHIlTGb6jYyLc+F8hc=
|
||||
github.com/luxfi/accel v1.1.9/go.mod h1:K00BcnLzEYMPHwCFq8Tf/450ApmTs9xBVvYOnofJCkc=
|
||||
github.com/luxfi/crypto v1.19.17 h1:l2LLu7UFyICtJVfraLDLRi+lFGiDXKHSL18M9/m1gsQ=
|
||||
github.com/luxfi/crypto v1.19.17/go.mod h1:INjdZtke85k8hX/QAmTMAY8bbZ4gzGZQLqURg3xf6Gk=
|
||||
github.com/luxfi/log v1.4.3 h1:xkUKRWvQ4ZwvlUC2e0/RTtHYZOYSMvSQ9W9lbjwBmiI=
|
||||
github.com/luxfi/log v1.4.3/go.mod h1:myIkufyiQomSQH34K981kbz6cG4WUoerRUh7F4XhlQI=
|
||||
github.com/luxfi/mdns v0.1.0 h1:VB3mQcETc9j5SY1S6lAgFtuGr/rjWuDgPYnxS+OKWMQ=
|
||||
github.com/luxfi/mdns v0.1.0/go.mod h1:/3dheKVjUk2yiS/ocH1IDzeLXOIe+kpVsErIGDOZdiQ=
|
||||
github.com/luxfi/mdns v0.1.1 h1:g2eRr9AXcziPkkcd24M+Qu9ApEpoKKjfI79QSNqv0rQ=
|
||||
github.com/luxfi/mdns v0.1.1/go.mod h1:dbp5f3h3aE7CGzwbaWzBM9cwdcekhmSrWhQevgYhhNA=
|
||||
github.com/luxfi/metric v1.5.8 h1:axPwfq+erOlIue7IJp5g+hMcMtVhYHja9gJAaT3+KNA=
|
||||
github.com/luxfi/metric v1.5.8/go.mod h1:fO2giazkg4NDtr72JM/QXJBYebplAMeWC1JoZyNDvKw=
|
||||
github.com/luxfi/pq v1.0.3 h1:pFlQm1+5FuKTDUh2y/23bXWkN4I2Rc5iuxJypwDFFMs=
|
||||
github.com/luxfi/pq v1.0.3/go.mod h1:8bppZcRElfrVt0n3nYCZW3iX1TvhvzNbdjNdK1irgIE=
|
||||
github.com/luxfi/zap v0.6.0 h1:baVcYKeqaNemGisYFtYtfIvVLW4hRXZo3wlXtOkk7NM=
|
||||
github.com/luxfi/zap v0.6.0/go.mod h1:1SDWtb4B3kxkd3VKLDae0x7lMtlvAsWLQxhhNIYsePI=
|
||||
github.com/luxfi/zap v0.7.2 h1:YecWTWNE5PPJXL56sLIkzS8b23bprUwZ5lPAQuLUtTE=
|
||||
github.com/luxfi/zap v0.7.2/go.mod h1:1k+nwT+JW802YzuPAuf7CxMSGr/qxvbGgGwi5k6X9Ok=
|
||||
github.com/mattn/go-colorable v0.1.14 h1:9A9LHSqF/7dyVVX6g0U9cwm9pG3kP9gSzcuIPHPsaIE=
|
||||
github.com/mattn/go-colorable v0.1.14/go.mod h1:6LmQG8QLFO4G5z1gPvYEzlUgJ2wF+stgPZH1UqBm1s8=
|
||||
github.com/mattn/go-isatty v0.0.21 h1:xYae+lCNBP7QuW4PUnNG61ffM4hVIfm+zUzDuSzYLGs=
|
||||
@@ -65,31 +71,31 @@ github.com/xyproto/randomstring v1.0.5 h1:YtlWPoRdgMu3NZtP45drfy1GKoojuR7hmRcnhZ
|
||||
github.com/xyproto/randomstring v1.0.5/go.mod h1:rgmS5DeNXLivK7YprL0pY+lTuhNQW3iGxZ18UQApw/E=
|
||||
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
|
||||
golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
|
||||
golang.org/x/crypto v0.50.0 h1:zO47/JPrL6vsNkINmLoo/PH1gcxpls50DNogFvB5ZGI=
|
||||
golang.org/x/crypto v0.50.0/go.mod h1:3muZ7vA7PBCE6xgPX7nkzzjiUq87kRItoJQM1Yo8S+Q=
|
||||
golang.org/x/crypto v0.52.0 h1:RMs7fP2rXdep0CftQlK8Uf+kibLm7qkCcradZWYz988=
|
||||
golang.org/x/crypto v0.52.0/go.mod h1:1QgfPxDqh0T2M/elOJtp9RvuR95kVjir0e6/BvEmGbc=
|
||||
golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg=
|
||||
golang.org/x/mod v0.34.0 h1:xIHgNUUnW6sYkcM5Jleh05DvLOtwc6RitGHbDk4akRI=
|
||||
golang.org/x/mod v0.34.0/go.mod h1:ykgH52iCZe79kzLLMhyCUzhMci+nQj+0XkbXpNYtVjY=
|
||||
golang.org/x/mod v0.36.0 h1:JJjpVx6myfUsUdAzZuOSTTmRE0PfZeNWzzvKrP7amb4=
|
||||
golang.org/x/mod v0.36.0/go.mod h1:moc6ELqsWcOw5Ef3xVprK5ul/MvtVvkIXLziUOICjUQ=
|
||||
golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
|
||||
golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
|
||||
golang.org/x/net v0.0.0-20190923162816-aa69164e4478/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
|
||||
golang.org/x/net v0.0.0-20200114155413-6afb5195e5aa/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
|
||||
golang.org/x/net v0.53.0 h1:d+qAbo5L0orcWAr0a9JweQpjXF19LMXJE8Ey7hwOdUA=
|
||||
golang.org/x/net v0.53.0/go.mod h1:JvMuJH7rrdiCfbeHoo3fCQU24Lf5JJwT9W3sJFulfgs=
|
||||
golang.org/x/net v0.55.0 h1:bcvxaJn3e1U6InsFWt1JUq1aSjnRxLzT2rtD2KfkDF8=
|
||||
golang.org/x/net v0.55.0/go.mod h1:L5U2KuzuOe1lY7Z+aWVIKK6qEeJXnXV9yzGA+WCHJww=
|
||||
golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sync v0.20.0 h1:e0PTpb7pjO8GAtTs2dQ6jYa5BWYlMuX047Dco/pItO4=
|
||||
golang.org/x/sync v0.20.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0=
|
||||
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
||||
golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20190924154521-2837fb4f24fe/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.43.0 h1:Rlag2XtaFTxp19wS8MXlJwTvoh8ArU6ezoyFsMyCTNI=
|
||||
golang.org/x/sys v0.43.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw=
|
||||
golang.org/x/sys v0.45.0 h1:dO4czNzziLiiXplLQgBCEpCvXQ3dnkn0SdaZSYdQ+FY=
|
||||
golang.org/x/sys v0.45.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw=
|
||||
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
|
||||
golang.org/x/text v0.36.0 h1:JfKh3XmcRPqZPKevfXVpI1wXPTqbkE5f7JA92a55Yxg=
|
||||
golang.org/x/text v0.36.0/go.mod h1:NIdBknypM8iqVmPiuco0Dh6P5Jcdk8lJL0CUebqK164=
|
||||
golang.org/x/text v0.37.0 h1:Cqjiwd9eSg8e0QAkyCaQTNHFIIzWtidPahFWR83rTrc=
|
||||
golang.org/x/text v0.37.0/go.mod h1:a5sjxXGs9hsn/AJVwuElvCAo9v8QYLzvavO5z2PiM38=
|
||||
golang.org/x/tools v0.0.0-20191216052735-49a3e744a425/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
|
||||
golang.org/x/tools v0.43.0 h1:12BdW9CeB3Z+J/I/wj34VMl8X+fEXBxVR90JeMX5E7s=
|
||||
golang.org/x/tools v0.43.0/go.mod h1:uHkMso649BX2cZK6+RpuIPXS3ho2hZo4FVwfoy1vIk0=
|
||||
golang.org/x/tools v0.45.0 h1:18qN3FAooORvApf5XjCXgsuayZOEtXf6JK18I3+ONa8=
|
||||
golang.org/x/tools v0.45.0/go.mod h1:LuUGqqaXcXMEFEruIVJVm5mgDD8vww/z/SR1gQ4uE/0=
|
||||
golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
|
||||
google.golang.org/protobuf v1.36.11 h1:fV6ZwhNocDyBLK0dj+fg8ektcVegBBuEolpbTQyBNVE=
|
||||
google.golang.org/protobuf v1.36.11/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco=
|
||||
|
||||
@@ -14,6 +14,7 @@ package metrics
|
||||
import (
|
||||
"fmt"
|
||||
"net/http"
|
||||
"os"
|
||||
"strconv"
|
||||
"strings"
|
||||
|
||||
@@ -23,7 +24,7 @@ import (
|
||||
)
|
||||
|
||||
// Version is surfaced on the /health routes.
|
||||
const Version = "0.3.0"
|
||||
const Version = "0.4.0"
|
||||
|
||||
// reg is the process-global per-tenant store registry, initialised in Mount.
|
||||
var reg *Registry
|
||||
@@ -62,6 +63,17 @@ func Mount(app *zip.App, deps cloud.Deps) error {
|
||||
reg = NewRegistry(deps.DataDir)
|
||||
brand = deps.Brand
|
||||
|
||||
// Optional ZAP push ingest — bind a luxfi/zap node accepting MsgMetricBatch
|
||||
// when O11Y_ZAP_PORT is set. HTTP /v1/metrics/batch carries the same wire
|
||||
// shape, so this is a transport optimisation, not a requirement.
|
||||
if p := os.Getenv("O11Y_ZAP_PORT"); p != "" {
|
||||
if port, err := strconv.Atoi(p); err == nil {
|
||||
if _, err := startZAPReceiver(port, "o11y-metrics-"+brand, log); err != nil {
|
||||
log.Warn("zap metric receiver disabled", "err", err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// --- Metrics ---
|
||||
app.Get("/v1/metrics/health", func(c *zip.Ctx) error {
|
||||
return c.JSON(http.StatusOK, map[string]any{
|
||||
|
||||
@@ -0,0 +1,50 @@
|
||||
package metrics
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
|
||||
luxlog "github.com/luxfi/log"
|
||||
metric "github.com/luxfi/metric"
|
||||
zap "github.com/luxfi/zap"
|
||||
)
|
||||
|
||||
// MsgMetricBatch is the canonical ZAP MsgType for metric batches — it matches
|
||||
// luxfi/metric.MsgMetricBatch and o11y/pkg/zapmetricreceiver so any luxfi/metric
|
||||
// ZAP exporter ingests here unchanged.
|
||||
const MsgMetricBatch uint16 = 2
|
||||
|
||||
// zapReceiver hosts a luxfi/zap node that accepts MsgMetricBatch messages and
|
||||
// ingests them into the addressed tenant's metric store.
|
||||
type zapReceiver struct{ node *zap.Node }
|
||||
|
||||
// startZAPReceiver binds a ZAP node on port that decodes a luxfi/metric.MetricBatch
|
||||
// (JSON at message field 0, the o11y wire convention) from each MsgMetricBatch
|
||||
// message and writes it into reg.For(org).Metrics — org is the batch's
|
||||
// Resource["org"], else the deployment brand. port<=0 disables it (HTTP
|
||||
// /v1/metrics/batch stays the ingest). The ZAP transport avoids the HTTP hop and
|
||||
// is the push path for in-cluster luxfi/metric exporters.
|
||||
func startZAPReceiver(port int, nodeID string, log luxlog.Logger) (*zapReceiver, error) {
|
||||
if port <= 0 {
|
||||
return nil, nil
|
||||
}
|
||||
node := zap.NewNode(zap.NodeConfig{NodeID: nodeID, Port: port, NoDiscovery: true})
|
||||
node.Handle(MsgMetricBatch, func(ctx context.Context, peerID string, msg *zap.Message) (*zap.Message, error) {
|
||||
var b metric.MetricBatch
|
||||
if err := json.Unmarshal(msg.Root().Bytes(0), &b); err != nil {
|
||||
log.Warn("zap metric batch decode", "peer", peerID, "err", err)
|
||||
return nil, nil
|
||||
}
|
||||
org := b.Resource["org"]
|
||||
if org == "" {
|
||||
org = brand
|
||||
}
|
||||
reg.For(org).Metrics.IngestBatch(&b)
|
||||
return nil, nil
|
||||
})
|
||||
if err := node.Start(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
log.Info("zap metric receiver listening", "port", port, "nodeID", nodeID, "msgType", MsgMetricBatch)
|
||||
return &zapReceiver{node: node}, nil
|
||||
}
|
||||
Reference in New Issue
Block a user