Revert "chore: gitignore artifacts + storage updates"

This reverts commit d3c5bbdbd0.

go.mod

@@ -48,7 +48,7 @@ require (
 	github.com/klauspost/reedsolomon v1.12.4
 	github.com/lib/pq v1.10.9
 	github.com/lithammer/shortuuid/v4 v4.2.0
-	github.com/luxfi/zap v0.3.1
+	github.com/zap-proto/go v0.2.0
 	github.com/miekg/dns v1.1.65
 	github.com/minio/cli v1.24.2
 	github.com/minio/csvparser v1.0.0
@@ -90,7 +90,6 @@ require (
 	github.com/tinylib/msgp v1.4.0
 	github.com/valyala/bytebufferpool v1.0.0
 	github.com/xdg/scram v1.0.5
-	github.com/zap-proto/go v0.2.0
 	github.com/zeebo/xxh3 v1.0.2
 	go.etcd.io/etcd/api/v3 v3.5.21
 	go.etcd.io/etcd/client/v3 v3.5.21

go.sum

@@ -400,8 +400,6 @@ github.com/luxfi/mdns v0.1.0 h1:VB3mQcETc9j5SY1S6lAgFtuGr/rjWuDgPYnxS+OKWMQ=
 github.com/luxfi/mdns v0.1.0/go.mod h1:/3dheKVjUk2yiS/ocH1IDzeLXOIe+kpVsErIGDOZdiQ=
 github.com/luxfi/zap v0.2.0 h1:RzvOkp3EoN5UCkpnqfObHLM1sEHy7YcxXKuermhE/VA=
 github.com/luxfi/zap v0.2.0/go.mod h1:2hydPSa/XMCMtfW6/DC9M5Bt03N5h75QwCV5Vypsqr4=
-github.com/luxfi/zap v0.3.1 h1:RlOHthFTw3cikft3JmrdT/R8BFId2Ej7xeNiHDoTNs4=
-github.com/luxfi/zap v0.3.1/go.mod h1:+RVBXJPxTqY5RU148058qew9/3fyRWTg/eBIwOIMlCY=
 github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ=
 github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc=
 github.com/mailru/easyjson v0.9.0 h1:PrnmzHw7262yW8sTBwxi1PdJA3Iw/EKBa8psRf7d9a4=
@@ -660,7 +658,6 @@ github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9dec
 github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY=
 github.com/yusufpapurcu/wmi v1.2.4 h1:zFUKzehAFReQwLys1b/iSMl+JQGSCSjtVqQn9bBrPo0=
 github.com/yusufpapurcu/wmi v1.2.4/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0=
-github.com/zap-proto/go v0.2.0/go.mod h1:OITYSXRd0v93MWZuGl9XnPS77a09DYzN/Uy9/VwhuCs=
 github.com/zeebo/assert v1.3.0 h1:g7C04CbJuIDKNPFHmsk4hwZDO5O+kntRxzaUoNXj+IQ=
 github.com/zeebo/assert v1.3.0/go.mod h1:Pq9JiuJQpG8JLJdtkwrJESF0Foym2/D9XMU5ciN/wJ0=
 github.com/zeebo/errs v1.4.0 h1:XNdoD/RRMKP7HD0UhJnIzUy74ISdGGxURlYG8HSWSfM=

internal/config/lambda/target/lazyinit.go

@@ -0,0 +1,51 @@
+// Copyright (c) 2015-2023 Hanzo AI, Inc.
+//
+// This file is part of Hanzo S3 Object Storage stack
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package target
+
+import (
+	"sync"
+	"sync/atomic"
+)
+
+// Inspired from Golang sync.Once but it is only marked
+// initialized when the provided function returns nil.
+
+type lazyInit struct {
+	done uint32
+	m    sync.Mutex
+}
+
+func (l *lazyInit) Do(f func() error) error {
+	if atomic.LoadUint32(&l.done) == 0 {
+		return l.doSlow(f)
+	}
+	return nil
+}
+
+func (l *lazyInit) doSlow(f func() error) error {
+	l.m.Lock()
+	defer l.m.Unlock()
+	if atomic.LoadUint32(&l.done) == 0 {
+		if err := f(); err != nil {
+			return err
+		}
+		// Mark as done only when f() is successful
+		atomic.StoreUint32(&l.done, 1)
+	}
+	return nil
+}

internal/config/lambda/target/webhook.go

@@ -0,0 +1,246 @@
+// Copyright (c) 2015-2023 Hanzo AI, Inc.
+//
+// This file is part of Hanzo S3 Object Storage stack
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package target
+
+import (
+	"bytes"
+	"context"
+	"crypto/tls"
+	"encoding/json"
+	"errors"
+	"net/http"
+	"strings"
+	"sync/atomic"
+	"syscall"
+	"time"
+
+	"github.com/hanzoai/s3/internal/config/lambda/event"
+	xhttp "github.com/hanzoai/s3/internal/http"
+	"github.com/hanzoai/s3/internal/logger"
+	"github.com/minio/pkg/v3/certs"
+	xnet "github.com/minio/pkg/v3/net"
+)
+
+// Webhook constants
+const (
+	WebhookEndpoint   = "endpoint"
+	WebhookAuthToken  = "auth_token"
+	WebhookClientCert = "client_cert"
+	WebhookClientKey  = "client_key"
+
+	EnvWebhookEnable     = "S3_LAMBDA_WEBHOOK_ENABLE"
+	EnvWebhookEndpoint   = "S3_LAMBDA_WEBHOOK_ENDPOINT"
+	EnvWebhookAuthToken  = "S3_LAMBDA_WEBHOOK_AUTH_TOKEN"
+	EnvWebhookClientCert = "S3_LAMBDA_WEBHOOK_CLIENT_CERT"
+	EnvWebhookClientKey  = "S3_LAMBDA_WEBHOOK_CLIENT_KEY"
+)
+
+// WebhookArgs - Webhook target arguments.
+type WebhookArgs struct {
+	Enable     bool            `json:"enable"`
+	Endpoint   xnet.URL        `json:"endpoint"`
+	AuthToken  string          `json:"authToken"`
+	Transport  *http.Transport `json:"-"`
+	ClientCert string          `json:"clientCert"`
+	ClientKey  string          `json:"clientKey"`
+}
+
+// Validate WebhookArgs fields
+func (w WebhookArgs) Validate() error {
+	if !w.Enable {
+		return nil
+	}
+	if w.Endpoint.IsEmpty() {
+		return errors.New("endpoint empty")
+	}
+	if w.ClientCert != "" && w.ClientKey == "" || w.ClientCert == "" && w.ClientKey != "" {
+		return errors.New("cert and key must be specified as a pair")
+	}
+	return nil
+}
+
+// WebhookTarget - Webhook target.
+type WebhookTarget struct {
+	activeRequests int64
+	totalRequests  int64
+	failedRequests int64
+
+	lazyInit lazyInit
+
+	id         event.TargetID
+	args       WebhookArgs
+	transport  *http.Transport
+	httpClient *http.Client
+	loggerOnce logger.LogOnce
+	cancel     context.CancelFunc
+	cancelCh   <-chan struct{}
+}
+
+// ID - returns target ID.
+func (target *WebhookTarget) ID() event.TargetID {
+	return target.id
+}
+
+// IsActive - Return true if target is up and active
+func (target *WebhookTarget) IsActive() (bool, error) {
+	if err := target.init(); err != nil {
+		return false, err
+	}
+	return target.isActive()
+}
+
+// errNotConnected - indicates that the target connection is not active.
+var errNotConnected = errors.New("not connected to target server/service")
+
+func (target *WebhookTarget) isActive() (bool, error) {
+	ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
+	defer cancel()
+
+	req, err := http.NewRequestWithContext(ctx, http.MethodHead, target.args.Endpoint.String(), nil)
+	if err != nil {
+		if xnet.IsNetworkOrHostDown(err, false) {
+			return false, errNotConnected
+		}
+		return false, err
+	}
+	tokens := strings.Fields(target.args.AuthToken)
+	switch len(tokens) {
+	case 2:
+		req.Header.Set("Authorization", target.args.AuthToken)
+	case 1:
+		req.Header.Set("Authorization", "Bearer "+target.args.AuthToken)
+	}
+
+	resp, err := target.httpClient.Do(req)
+	if err != nil {
+		if xnet.IsNetworkOrHostDown(err, true) {
+			return false, errNotConnected
+		}
+		return false, err
+	}
+	xhttp.DrainBody(resp.Body)
+	// No network failure i.e response from the target means its up
+	return true, nil
+}
+
+// Stat - returns lambda webhook target statistics such as
+// current calls in progress, successfully completed functions
+// failed functions.
+func (target *WebhookTarget) Stat() event.TargetStat {
+	return event.TargetStat{
+		ID:             target.id,
+		ActiveRequests: atomic.LoadInt64(&target.activeRequests),
+		TotalRequests:  atomic.LoadInt64(&target.totalRequests),
+		FailedRequests: atomic.LoadInt64(&target.failedRequests),
+	}
+}
+
+// Send - sends an event to the webhook.
+func (target *WebhookTarget) Send(eventData event.Event) (resp *http.Response, err error) {
+	atomic.AddInt64(&target.activeRequests, 1)
+	defer atomic.AddInt64(&target.activeRequests, -1)
+
+	atomic.AddInt64(&target.totalRequests, 1)
+	defer func() {
+		if err != nil {
+			atomic.AddInt64(&target.failedRequests, 1)
+		}
+	}()
+
+	if err = target.init(); err != nil {
+		return nil, err
+	}
+
+	data, err := json.Marshal(eventData)
+	if err != nil {
+		return nil, err
+	}
+
+	req, err := http.NewRequest(http.MethodPost, target.args.Endpoint.String(), bytes.NewReader(data))
+	if err != nil {
+		return nil, err
+	}
+
+	// Verify if the authToken already contains
+	// <Key> <Token> like format, if this is
+	// already present we can blindly use the
+	// authToken as is instead of adding 'Bearer'
+	tokens := strings.Fields(target.args.AuthToken)
+	switch len(tokens) {
+	case 2:
+		req.Header.Set("Authorization", target.args.AuthToken)
+	case 1:
+		req.Header.Set("Authorization", "Bearer "+target.args.AuthToken)
+	}
+
+	req.Header.Set("Content-Type", "application/json")
+
+	return target.httpClient.Do(req)
+}
+
+// Close the target. Will cancel all active requests.
+func (target *WebhookTarget) Close() error {
+	target.cancel()
+	return nil
+}
+
+func (target *WebhookTarget) init() error {
+	return target.lazyInit.Do(target.initWebhook)
+}
+
+// Only called from init()
+func (target *WebhookTarget) initWebhook() error {
+	args := target.args
+	transport := target.transport
+
+	if args.ClientCert != "" && args.ClientKey != "" {
+		manager, err := certs.NewManager(context.Background(), args.ClientCert, args.ClientKey, tls.LoadX509KeyPair)
+		if err != nil {
+			return err
+		}
+		manager.ReloadOnSignal(syscall.SIGHUP) // allow reloads upon SIGHUP
+		transport.TLSClientConfig.GetClientCertificate = manager.GetClientCertificate
+	}
+	target.httpClient = &http.Client{Transport: transport}
+
+	yes, err := target.isActive()
+	if err != nil {
+		return err
+	}
+	if !yes {
+		return errNotConnected
+	}
+
+	return nil
+}
+
+// NewWebhookTarget - creates new Webhook target.
+func NewWebhookTarget(ctx context.Context, id string, args WebhookArgs, loggerOnce logger.LogOnce, transport *http.Transport) (*WebhookTarget, error) {
+	ctx, cancel := context.WithCancel(ctx)
+
+	target := &WebhookTarget{
+		id:         event.TargetID{ID: id, Name: "webhook"},
+		args:       args,
+		loggerOnce: loggerOnce,
+		transport:  transport,
+		cancel:     cancel,
+		cancelCh:   ctx.Done(),
+	}
+
+	return target, nil
+}

internal/event/target/amqp.go

@@ -0,0 +1,376 @@
+// Copyright (c) 2015-2023 Hanzo AI, Inc.
+//
+// This file is part of Hanzo S3 Object Storage stack
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package target
+
+import (
+	"context"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"net"
+	"net/url"
+	"os"
+	"path/filepath"
+	"sync"
+
+	"github.com/hanzoai/s3/internal/event"
+	"github.com/hanzoai/s3/internal/logger"
+	"github.com/hanzoai/s3/internal/once"
+	"github.com/hanzoai/s3/internal/store"
+	xnet "github.com/minio/pkg/v3/net"
+	"github.com/rabbitmq/amqp091-go"
+)
+
+// AMQPArgs - AMQP target arguments.
+type AMQPArgs struct {
+	Enable            bool        `json:"enable"`
+	URL               amqp091.URI `json:"url"`
+	Exchange          string      `json:"exchange"`
+	RoutingKey        string      `json:"routingKey"`
+	ExchangeType      string      `json:"exchangeType"`
+	DeliveryMode      uint8       `json:"deliveryMode"`
+	Mandatory         bool        `json:"mandatory"`
+	Immediate         bool        `json:"immediate"`
+	Durable           bool        `json:"durable"`
+	Internal          bool        `json:"internal"`
+	NoWait            bool        `json:"noWait"`
+	AutoDeleted       bool        `json:"autoDeleted"`
+	PublisherConfirms bool        `json:"publisherConfirms"`
+	QueueDir          string      `json:"queueDir"`
+	QueueLimit        uint64      `json:"queueLimit"`
+}
+
+// AMQP input constants.
+//
+// ST1003 We cannot change these exported names.
+//
+//nolint:staticcheck
+const (
+	AmqpQueueDir   = "queue_dir"
+	AmqpQueueLimit = "queue_limit"
+
+	AmqpURL               = "url"
+	AmqpExchange          = "exchange"
+	AmqpRoutingKey        = "routing_key"
+	AmqpExchangeType      = "exchange_type"
+	AmqpDeliveryMode      = "delivery_mode"
+	AmqpMandatory         = "mandatory"
+	AmqpImmediate         = "immediate"
+	AmqpDurable           = "durable"
+	AmqpInternal          = "internal"
+	AmqpNoWait            = "no_wait"
+	AmqpAutoDeleted       = "auto_deleted"
+	AmqpArguments         = "arguments"
+	AmqpPublisherConfirms = "publisher_confirms"
+
+	EnvAMQPEnable            = "S3_NOTIFY_AMQP_ENABLE"
+	EnvAMQPURL               = "S3_NOTIFY_AMQP_URL"
+	EnvAMQPExchange          = "S3_NOTIFY_AMQP_EXCHANGE"
+	EnvAMQPRoutingKey        = "S3_NOTIFY_AMQP_ROUTING_KEY"
+	EnvAMQPExchangeType      = "S3_NOTIFY_AMQP_EXCHANGE_TYPE"
+	EnvAMQPDeliveryMode      = "S3_NOTIFY_AMQP_DELIVERY_MODE"
+	EnvAMQPMandatory         = "S3_NOTIFY_AMQP_MANDATORY"
+	EnvAMQPImmediate         = "S3_NOTIFY_AMQP_IMMEDIATE"
+	EnvAMQPDurable           = "S3_NOTIFY_AMQP_DURABLE"
+	EnvAMQPInternal          = "S3_NOTIFY_AMQP_INTERNAL"
+	EnvAMQPNoWait            = "S3_NOTIFY_AMQP_NO_WAIT"
+	EnvAMQPAutoDeleted       = "S3_NOTIFY_AMQP_AUTO_DELETED"
+	EnvAMQPArguments         = "S3_NOTIFY_AMQP_ARGUMENTS"
+	EnvAMQPPublisherConfirms = "S3_NOTIFY_AMQP_PUBLISHING_CONFIRMS"
+	EnvAMQPQueueDir          = "S3_NOTIFY_AMQP_QUEUE_DIR"
+	EnvAMQPQueueLimit        = "S3_NOTIFY_AMQP_QUEUE_LIMIT"
+)
+
+// Validate AMQP arguments
+func (a *AMQPArgs) Validate() error {
+	if !a.Enable {
+		return nil
+	}
+	if _, err := amqp091.ParseURI(a.URL.String()); err != nil {
+		return err
+	}
+	if a.QueueDir != "" {
+		if !filepath.IsAbs(a.QueueDir) {
+			return errors.New("queueDir path should be absolute")
+		}
+	}
+
+	return nil
+}
+
+// AMQPTarget - AMQP target
+type AMQPTarget struct {
+	initOnce once.Init
+
+	id         event.TargetID
+	args       AMQPArgs
+	conn       *amqp091.Connection
+	connMutex  sync.Mutex
+	store      store.Store[event.Event]
+	loggerOnce logger.LogOnce
+
+	quitCh chan struct{}
+}
+
+// ID - returns TargetID.
+func (target *AMQPTarget) ID() event.TargetID {
+	return target.id
+}
+
+// Name - returns the Name of the target.
+func (target *AMQPTarget) Name() string {
+	return target.ID().String()
+}
+
+// Store returns any underlying store if set.
+func (target *AMQPTarget) Store() event.TargetStore {
+	return target.store
+}
+
+// IsActive - Return true if target is up and active
+func (target *AMQPTarget) IsActive() (bool, error) {
+	if err := target.init(); err != nil {
+		return false, err
+	}
+
+	return target.isActive()
+}
+
+func (target *AMQPTarget) isActive() (bool, error) {
+	ch, _, err := target.channel()
+	if err != nil {
+		return false, err
+	}
+	defer func() {
+		ch.Close()
+	}()
+	return true, nil
+}
+
+func (target *AMQPTarget) channel() (*amqp091.Channel, chan amqp091.Confirmation, error) {
+	var err error
+	var conn *amqp091.Connection
+	var ch *amqp091.Channel
+
+	isAMQPClosedErr := func(err error) bool {
+		if err == amqp091.ErrClosed {
+			return true
+		}
+
+		if nerr, ok := err.(*net.OpError); ok {
+			return (nerr.Err.Error() == "use of closed network connection")
+		}
+
+		return false
+	}
+
+	target.connMutex.Lock()
+	defer target.connMutex.Unlock()
+
+	if target.conn != nil {
+		ch, err = target.conn.Channel()
+		if err == nil {
+			if target.args.PublisherConfirms {
+				confirms := ch.NotifyPublish(make(chan amqp091.Confirmation, 1))
+				if err := ch.Confirm(false); err != nil {
+					ch.Close()
+					return nil, nil, err
+				}
+				return ch, confirms, nil
+			}
+			return ch, nil, nil
+		}
+
+		if !isAMQPClosedErr(err) {
+			return nil, nil, err
+		}
+
+		// close when we know this is a network error.
+		target.conn.Close()
+	}
+
+	conn, err = amqp091.Dial(target.args.URL.String())
+	if err != nil {
+		if xnet.IsConnRefusedErr(err) {
+			return nil, nil, store.ErrNotConnected
+		}
+		return nil, nil, err
+	}
+
+	ch, err = conn.Channel()
+	if err != nil {
+		return nil, nil, err
+	}
+
+	target.conn = conn
+
+	if target.args.PublisherConfirms {
+		confirms := ch.NotifyPublish(make(chan amqp091.Confirmation, 1))
+		if err := ch.Confirm(false); err != nil {
+			ch.Close()
+			return nil, nil, err
+		}
+		return ch, confirms, nil
+	}
+
+	return ch, nil, nil
+}
+
+// send - sends an event to the AMQP091.
+func (target *AMQPTarget) send(eventData event.Event, ch *amqp091.Channel, confirms chan amqp091.Confirmation) error {
+	objectName, err := url.QueryUnescape(eventData.S3.Object.Key)
+	if err != nil {
+		return err
+	}
+	key := eventData.S3.Bucket.Name + "/" + objectName
+
+	data, err := json.Marshal(event.Log{EventName: eventData.EventName, Key: key, Records: []event.Event{eventData}})
+	if err != nil {
+		return err
+	}
+
+	headers := make(amqp091.Table)
+	// Add more information here as required, but be aware to not overload headers
+	headers["minio-bucket"] = eventData.S3.Bucket.Name
+	headers["minio-event"] = eventData.EventName.String()
+
+	if err = ch.ExchangeDeclare(target.args.Exchange, target.args.ExchangeType, target.args.Durable,
+		target.args.AutoDeleted, target.args.Internal, target.args.NoWait, nil); err != nil {
+		return err
+	}
+
+	if err = ch.Publish(target.args.Exchange, target.args.RoutingKey, target.args.Mandatory,
+		target.args.Immediate, amqp091.Publishing{
+			Headers:      headers,
+			ContentType:  "application/json",
+			DeliveryMode: target.args.DeliveryMode,
+			Body:         data,
+		}); err != nil {
+		return err
+	}
+
+	// check for publisher confirms only if its enabled
+	if target.args.PublisherConfirms {
+		confirmed := <-confirms
+		if !confirmed.Ack {
+			return fmt.Errorf("failed delivery of delivery tag: %d", confirmed.DeliveryTag)
+		}
+	}
+
+	return nil
+}
+
+// Save - saves the events to the store which will be replayed when the amqp connection is active.
+func (target *AMQPTarget) Save(eventData event.Event) error {
+	if target.store != nil {
+		_, err := target.store.Put(eventData)
+		return err
+	}
+	if err := target.init(); err != nil {
+		return err
+	}
+	ch, confirms, err := target.channel()
+	if err != nil {
+		return err
+	}
+	defer ch.Close()
+
+	return target.send(eventData, ch, confirms)
+}
+
+// SendFromStore - reads an event from store and sends it to AMQP091.
+func (target *AMQPTarget) SendFromStore(key store.Key) error {
+	if err := target.init(); err != nil {
+		return err
+	}
+
+	ch, confirms, err := target.channel()
+	if err != nil {
+		return err
+	}
+	defer ch.Close()
+
+	eventData, eErr := target.store.Get(key)
+	if eErr != nil {
+		// The last event key in a successful batch will be sent in the channel atmost once by the replayEvents()
+		// Such events will not exist and wouldve been already been sent successfully.
+		if os.IsNotExist(eErr) {
+			return nil
+		}
+		return eErr
+	}
+
+	if err := target.send(eventData, ch, confirms); err != nil {
+		return err
+	}
+
+	// Delete the event from store.
+	return target.store.Del(key)
+}
+
+// Close - does nothing and available for interface compatibility.
+func (target *AMQPTarget) Close() error {
+	close(target.quitCh)
+	if target.conn != nil {
+		return target.conn.Close()
+	}
+	return nil
+}
+
+func (target *AMQPTarget) init() error {
+	return target.initOnce.Do(target.initAMQP)
+}
+
+func (target *AMQPTarget) initAMQP() error {
+	conn, err := amqp091.Dial(target.args.URL.String())
+	if err != nil {
+		if xnet.IsConnRefusedErr(err) || xnet.IsConnResetErr(err) {
+			target.loggerOnce(context.Background(), err, target.ID().String())
+		}
+		return err
+	}
+	target.conn = conn
+
+	return nil
+}
+
+// NewAMQPTarget - creates new AMQP target.
+func NewAMQPTarget(id string, args AMQPArgs, loggerOnce logger.LogOnce) (*AMQPTarget, error) {
+	var queueStore store.Store[event.Event]
+	if args.QueueDir != "" {
+		queueDir := filepath.Join(args.QueueDir, storePrefix+"-amqp-"+id)
+		queueStore = store.NewQueueStore[event.Event](queueDir, args.QueueLimit, event.StoreExtension)
+		if err := queueStore.Open(); err != nil {
+			return nil, fmt.Errorf("unable to initialize the queue store of AMQP `%s`: %w", id, err)
+		}
+	}
+
+	target := &AMQPTarget{
+		id:         event.TargetID{ID: id, Name: "amqp"},
+		args:       args,
+		loggerOnce: loggerOnce,
+		store:      queueStore,
+		quitCh:     make(chan struct{}),
+	}
+
+	if target.store != nil {
+		store.StreamItems(target.store, target, target.quitCh, target.loggerOnce)
+	}
+
+	return target, nil
+}

internal/event/target/elasticsearch.go

@@ -0,0 +1,584 @@
+// Copyright (c) 2015-2023 Hanzo AI, Inc.
+//
+// This file is part of Hanzo S3 Object Storage stack
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package target
+
+import (
+	"bytes"
+	"context"
+	"encoding/base64"
+	"encoding/json"
+	"fmt"
+	"net/http"
+	"net/url"
+	"os"
+	"path/filepath"
+	"strconv"
+	"strings"
+	"time"
+
+	elasticsearch7 "github.com/elastic/go-elasticsearch/v7"
+	"github.com/minio/highwayhash"
+	"github.com/hanzoai/s3/internal/event"
+	xhttp "github.com/hanzoai/s3/internal/http"
+	"github.com/hanzoai/s3/internal/logger"
+	"github.com/hanzoai/s3/internal/once"
+	"github.com/hanzoai/s3/internal/store"
+	xnet "github.com/minio/pkg/v3/net"
+	"github.com/pkg/errors"
+)
+
+// Elastic constants
+const (
+	ElasticFormat     = "format"
+	ElasticURL        = "url"
+	ElasticIndex      = "index"
+	ElasticQueueDir   = "queue_dir"
+	ElasticQueueLimit = "queue_limit"
+	ElasticUsername   = "username"
+	ElasticPassword   = "password"
+
+	EnvElasticEnable     = "S3_NOTIFY_ELASTICSEARCH_ENABLE"
+	EnvElasticFormat     = "S3_NOTIFY_ELASTICSEARCH_FORMAT"
+	EnvElasticURL        = "S3_NOTIFY_ELASTICSEARCH_URL"
+	EnvElasticIndex      = "S3_NOTIFY_ELASTICSEARCH_INDEX"
+	EnvElasticQueueDir   = "S3_NOTIFY_ELASTICSEARCH_QUEUE_DIR"
+	EnvElasticQueueLimit = "S3_NOTIFY_ELASTICSEARCH_QUEUE_LIMIT"
+	EnvElasticUsername   = "S3_NOTIFY_ELASTICSEARCH_USERNAME"
+	EnvElasticPassword   = "S3_NOTIFY_ELASTICSEARCH_PASSWORD"
+)
+
+// ESSupportStatus is a typed string representing the support status for
+// Elasticsearch
+type ESSupportStatus string
+
+const (
+	// ESSUnknown is default value
+	ESSUnknown ESSupportStatus = "ESSUnknown"
+	// ESSDeprecated -> support will be removed in future
+	ESSDeprecated ESSupportStatus = "ESSDeprecated"
+	// ESSUnsupported -> we won't work with this ES server
+	ESSUnsupported ESSupportStatus = "ESSUnsupported"
+	// ESSSupported -> all good!
+	ESSSupported ESSupportStatus = "ESSSupported"
+)
+
+func getESVersionSupportStatus(version string) (res ESSupportStatus, err error) {
+	parts := strings.Split(version, ".")
+	if len(parts) < 1 {
+		err = fmt.Errorf("bad ES version string: %s", version)
+		return res, err
+	}
+
+	majorVersion, err := strconv.Atoi(parts[0])
+	if err != nil {
+		err = fmt.Errorf("bad ES version string: %s", version)
+		return res, err
+	}
+
+	switch {
+	case majorVersion <= 6:
+		res = ESSUnsupported
+	default:
+		res = ESSSupported
+	}
+	return res, err
+}
+
+// magic HH-256 key as HH-256 hash of the first 100 decimals of π as utf-8 string with a zero key.
+var magicHighwayHash256Key = []byte("\x4b\xe7\x34\xfa\x8e\x23\x8a\xcd\x26\x3e\x83\xe6\xbb\x96\x85\x52\x04\x0f\x93\x5d\xa3\x9f\x44\x14\x97\xe0\x9d\x13\x22\xde\x36\xa0")
+
+// Interface for elasticsearch client objects
+type esClient interface {
+	isAtleastV7() bool
+	createIndex(ElasticsearchArgs) error
+	ping(context.Context, ElasticsearchArgs) (bool, error)
+	stop()
+	entryExists(context.Context, string, string) (bool, error)
+	removeEntry(context.Context, string, string) error
+	updateEntry(context.Context, string, string, event.Event) error
+	addEntry(context.Context, string, event.Event) error
+}
+
+// ElasticsearchArgs - Elasticsearch target arguments.
+type ElasticsearchArgs struct {
+	Enable     bool            `json:"enable"`
+	Format     string          `json:"format"`
+	URL        xnet.URL        `json:"url"`
+	Index      string          `json:"index"`
+	QueueDir   string          `json:"queueDir"`
+	QueueLimit uint64          `json:"queueLimit"`
+	Transport  *http.Transport `json:"-"`
+	Username   string          `json:"username"`
+	Password   string          `json:"password"`
+}
+
+// Validate ElasticsearchArgs fields
+func (a ElasticsearchArgs) Validate() error {
+	if !a.Enable {
+		return nil
+	}
+	if a.URL.IsEmpty() {
+		return errors.New("empty URL")
+	}
+	if a.Format != "" {
+		f := strings.ToLower(a.Format)
+		if f != event.NamespaceFormat && f != event.AccessFormat {
+			return errors.New("format value unrecognized")
+		}
+	}
+	if a.Index == "" {
+		return errors.New("empty index value")
+	}
+
+	if (a.Username == "" && a.Password != "") || (a.Username != "" && a.Password == "") {
+		return errors.New("username and password should be set in pairs")
+	}
+
+	return nil
+}
+
+// ElasticsearchTarget - Elasticsearch target.
+type ElasticsearchTarget struct {
+	initOnce once.Init
+
+	id         event.TargetID
+	args       ElasticsearchArgs
+	client     esClient
+	store      store.Store[event.Event]
+	loggerOnce logger.LogOnce
+	quitCh     chan struct{}
+}
+
+// ID - returns target ID.
+func (target *ElasticsearchTarget) ID() event.TargetID {
+	return target.id
+}
+
+// Name - returns the Name of the target.
+func (target *ElasticsearchTarget) Name() string {
+	return target.ID().String()
+}
+
+// Store returns any underlying store if set.
+func (target *ElasticsearchTarget) Store() event.TargetStore {
+	return target.store
+}
+
+// IsActive - Return true if target is up and active
+func (target *ElasticsearchTarget) IsActive() (bool, error) {
+	if err := target.init(); err != nil {
+		return false, err
+	}
+	return target.isActive()
+}
+
+func (target *ElasticsearchTarget) isActive() (bool, error) {
+	ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
+	defer cancel()
+
+	err := target.checkAndInitClient(ctx)
+	if err != nil {
+		return false, err
+	}
+
+	return target.client.ping(ctx, target.args)
+}
+
+// Save - saves the events to the store if queuestore is configured, which will be replayed when the elasticsearch connection is active.
+func (target *ElasticsearchTarget) Save(eventData event.Event) error {
+	if target.store != nil {
+		_, err := target.store.Put(eventData)
+		return err
+	}
+	if err := target.init(); err != nil {
+		return err
+	}
+
+	ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
+	defer cancel()
+
+	err := target.checkAndInitClient(ctx)
+	if err != nil {
+		return err
+	}
+
+	err = target.send(eventData)
+	if xnet.IsNetworkOrHostDown(err, false) {
+		return store.ErrNotConnected
+	}
+	return err
+}
+
+// send - sends the event to the target.
+func (target *ElasticsearchTarget) send(eventData event.Event) error {
+	ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
+	defer cancel()
+
+	if target.args.Format == event.NamespaceFormat {
+		objectName, err := url.QueryUnescape(eventData.S3.Object.Key)
+		if err != nil {
+			return err
+		}
+
+		// Calculate a hash of the key for the id of the ES document.
+		// Id's are limited to 512 bytes in V7+, so we need to do this.
+		var keyHash string
+		{
+			key := eventData.S3.Bucket.Name + "/" + objectName
+			if target.client.isAtleastV7() {
+				hh, _ := highwayhash.New(magicHighwayHash256Key) // New will never return error since key is 256 bit
+				hh.Write([]byte(key))
+				hashBytes := hh.Sum(nil)
+				keyHash = base64.URLEncoding.EncodeToString(hashBytes)
+			} else {
+				keyHash = key
+			}
+		}
+
+		if eventData.EventName == event.ObjectRemovedDelete {
+			err = target.client.removeEntry(ctx, target.args.Index, keyHash)
+		} else {
+			err = target.client.updateEntry(ctx, target.args.Index, keyHash, eventData)
+		}
+		return err
+	}
+
+	if target.args.Format == event.AccessFormat {
+		return target.client.addEntry(ctx, target.args.Index, eventData)
+	}
+
+	return nil
+}
+
+// SendFromStore - reads an event from store and sends it to Elasticsearch.
+func (target *ElasticsearchTarget) SendFromStore(key store.Key) error {
+	if err := target.init(); err != nil {
+		return err
+	}
+
+	ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
+	defer cancel()
+
+	err := target.checkAndInitClient(ctx)
+	if err != nil {
+		return err
+	}
+
+	eventData, eErr := target.store.Get(key)
+	if eErr != nil {
+		// The last event key in a successful batch will be sent in the channel atmost once by the replayEvents()
+		// Such events will not exist and wouldve been already been sent successfully.
+		if os.IsNotExist(eErr) {
+			return nil
+		}
+		return eErr
+	}
+
+	if err := target.send(eventData); err != nil {
+		if xnet.IsNetworkOrHostDown(err, false) {
+			return store.ErrNotConnected
+		}
+		return err
+	}
+
+	// Delete the event from store.
+	return target.store.Del(key)
+}
+
+// Close - does nothing and available for interface compatibility.
+func (target *ElasticsearchTarget) Close() error {
+	close(target.quitCh)
+	if target.client != nil {
+		// Stops the background processes that the client is running.
+		target.client.stop()
+	}
+	return nil
+}
+
+func (target *ElasticsearchTarget) checkAndInitClient(ctx context.Context) error {
+	if target.client != nil {
+		return nil
+	}
+
+	clientV7, err := newClientV7(target.args)
+	if err != nil {
+		return err
+	}
+
+	// Check es version to confirm if it is supported.
+	serverSupportStatus, version, err := clientV7.getServerSupportStatus(ctx)
+	if err != nil {
+		return err
+	}
+
+	switch serverSupportStatus {
+	case ESSUnknown:
+		return errors.New("unable to determine support status of ES (should not happen)")
+
+	case ESSDeprecated:
+		return errors.New("there is no currently deprecated version of ES in MinIO")
+
+	case ESSSupported:
+		target.client = clientV7
+
+	default:
+		// ESSUnsupported case
+		return fmt.Errorf("Elasticsearch version '%s' is not supported! Please use at least version 7.x.", version)
+	}
+
+	target.client.createIndex(target.args)
+	return nil
+}
+
+func (target *ElasticsearchTarget) init() error {
+	return target.initOnce.Do(target.initElasticsearch)
+}
+
+func (target *ElasticsearchTarget) initElasticsearch() error {
+	ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
+	defer cancel()
+
+	err := target.checkAndInitClient(ctx)
+	if err != nil {
+		if err != store.ErrNotConnected {
+			target.loggerOnce(context.Background(), err, target.ID().String())
+		}
+		return err
+	}
+
+	return nil
+}
+
+// NewElasticsearchTarget - creates new Elasticsearch target.
+func NewElasticsearchTarget(id string, args ElasticsearchArgs, loggerOnce logger.LogOnce) (*ElasticsearchTarget, error) {
+	var queueStore store.Store[event.Event]
+	if args.QueueDir != "" {
+		queueDir := filepath.Join(args.QueueDir, storePrefix+"-elasticsearch-"+id)
+		queueStore = store.NewQueueStore[event.Event](queueDir, args.QueueLimit, event.StoreExtension)
+		if err := queueStore.Open(); err != nil {
+			return nil, fmt.Errorf("unable to initialize the queue store of Elasticsearch `%s`: %w", id, err)
+		}
+	}
+
+	target := &ElasticsearchTarget{
+		id:         event.TargetID{ID: id, Name: "elasticsearch"},
+		args:       args,
+		store:      queueStore,
+		loggerOnce: loggerOnce,
+		quitCh:     make(chan struct{}),
+	}
+
+	if target.store != nil {
+		store.StreamItems(target.store, target, target.quitCh, target.loggerOnce)
+	}
+
+	return target, nil
+}
+
+// ES Client definitions and methods
+
+type esClientV7 struct {
+	*elasticsearch7.Client
+}
+
+func newClientV7(args ElasticsearchArgs) (*esClientV7, error) {
+	// Client options
+	elasticConfig := elasticsearch7.Config{
+		Addresses:  []string{args.URL.String()},
+		Transport:  args.Transport,
+		MaxRetries: 10,
+	}
+	// Set basic auth
+	if args.Username != "" && args.Password != "" {
+		elasticConfig.Username = args.Username
+		elasticConfig.Password = args.Password
+	}
+	// Create a client
+	client, err := elasticsearch7.NewClient(elasticConfig)
+	if err != nil {
+		return nil, err
+	}
+	clientV7 := &esClientV7{client}
+	return clientV7, nil
+}
+
+func (c *esClientV7) getServerSupportStatus(ctx context.Context) (ESSupportStatus, string, error) {
+	resp, err := c.Info(
+		c.Info.WithContext(ctx),
+	)
+	if err != nil {
+		return ESSUnknown, "", store.ErrNotConnected
+	}
+
+	defer resp.Body.Close()
+
+	m := make(map[string]any)
+	err = json.NewDecoder(resp.Body).Decode(&m)
+	if err != nil {
+		return ESSUnknown, "", fmt.Errorf("unable to get ES Server version - json parse error: %v", err)
+	}
+
+	if v, ok := m["version"].(map[string]any); ok {
+		if ver, ok := v["number"].(string); ok {
+			status, err := getESVersionSupportStatus(ver)
+			return status, ver, err
+		}
+	}
+	return ESSUnknown, "", fmt.Errorf("Unable to get ES Server Version - got INFO response: %v", m)
+}
+
+func (c *esClientV7) isAtleastV7() bool {
+	return true
+}
+
+// createIndex - creates the index if it does not exist.
+func (c *esClientV7) createIndex(args ElasticsearchArgs) error {
+	res, err := c.Indices.ResolveIndex([]string{args.Index})
+	if err != nil {
+		return err
+	}
+	defer res.Body.Close()
+
+	var v map[string]any
+	found := false
+	if err := json.NewDecoder(res.Body).Decode(&v); err != nil {
+		return fmt.Errorf("Error parsing response body: %v", err)
+	}
+
+	indices, ok := v["indices"].([]any)
+	if ok {
+		for _, index := range indices {
+			if name, ok := index.(map[string]any); ok && name["name"] == args.Index {
+				found = true
+				break
+			}
+		}
+	}
+
+	if !found {
+		resp, err := c.Indices.Create(args.Index)
+		if err != nil {
+			return err
+		}
+		defer xhttp.DrainBody(resp.Body)
+		if resp.IsError() {
+			return fmt.Errorf("Create index err: %v", res)
+		}
+		return nil
+	}
+	return nil
+}
+
+func (c *esClientV7) ping(ctx context.Context, _ ElasticsearchArgs) (bool, error) {
+	resp, err := c.Ping(
+		c.Ping.WithContext(ctx),
+	)
+	if err != nil {
+		return false, store.ErrNotConnected
+	}
+	xhttp.DrainBody(resp.Body)
+	return !resp.IsError(), nil
+}
+
+func (c *esClientV7) entryExists(ctx context.Context, index string, key string) (bool, error) {
+	res, err := c.Exists(
+		index,
+		key,
+		c.Exists.WithContext(ctx),
+	)
+	if err != nil {
+		return false, err
+	}
+	xhttp.DrainBody(res.Body)
+	return !res.IsError(), nil
+}
+
+func (c *esClientV7) removeEntry(ctx context.Context, index string, key string) error {
+	exists, err := c.entryExists(ctx, index, key)
+	if err == nil && exists {
+		res, err := c.Delete(
+			index,
+			key,
+			c.Delete.WithContext(ctx),
+		)
+		if err != nil {
+			return err
+		}
+		defer xhttp.DrainBody(res.Body)
+		if res.IsError() {
+			return fmt.Errorf("Delete err: %s", res.String())
+		}
+		return nil
+	}
+	return err
+}
+
+func (c *esClientV7) updateEntry(ctx context.Context, index string, key string, eventData event.Event) error {
+	doc := map[string]any{
+		"Records": []event.Event{eventData},
+	}
+	var buf bytes.Buffer
+	enc := json.NewEncoder(&buf)
+	err := enc.Encode(doc)
+	if err != nil {
+		return err
+	}
+	res, err := c.Index(
+		index,
+		&buf,
+		c.Index.WithDocumentID(key),
+		c.Index.WithContext(ctx),
+	)
+	if err != nil {
+		return err
+	}
+	defer xhttp.DrainBody(res.Body)
+	if res.IsError() {
+		return fmt.Errorf("Update err: %s", res.String())
+	}
+
+	return nil
+}
+
+func (c *esClientV7) addEntry(ctx context.Context, index string, eventData event.Event) error {
+	doc := map[string]any{
+		"Records": []event.Event{eventData},
+	}
+	var buf bytes.Buffer
+	enc := json.NewEncoder(&buf)
+	err := enc.Encode(doc)
+	if err != nil {
+		return err
+	}
+	res, err := c.Index(
+		index,
+		&buf,
+		c.Index.WithContext(ctx),
+	)
+	if err != nil {
+		return err
+	}
+	defer xhttp.DrainBody(res.Body)
+	if res.IsError() {
+		return fmt.Errorf("Add err: %s", res.String())
+	}
+	return nil
+}
+
+func (c *esClientV7) stop() {
+}

internal/event/target/kafka.go

@@ -0,0 +1,465 @@
+// Copyright (c) 2015-2023 Hanzo AI, Inc.
+//
+// This file is part of Hanzo S3 Object Storage stack
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package target
+
+import (
+	"context"
+	"crypto/tls"
+	"crypto/x509"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"log"
+	"net/url"
+	"os"
+	"path/filepath"
+	"strings"
+	"time"
+
+	"github.com/hanzoai/s3/internal/event"
+	"github.com/hanzoai/s3/internal/logger"
+	"github.com/hanzoai/s3/internal/once"
+	"github.com/hanzoai/s3/internal/store"
+	xnet "github.com/minio/pkg/v3/net"
+
+	"github.com/IBM/sarama"
+	saramatls "github.com/IBM/sarama/tools/tls"
+)
+
+// Kafka input constants
+const (
+	KafkaBrokers            = "brokers"
+	KafkaTopic              = "topic"
+	KafkaQueueDir           = "queue_dir"
+	KafkaQueueLimit         = "queue_limit"
+	KafkaTLS                = "tls"
+	KafkaTLSSkipVerify      = "tls_skip_verify"
+	KafkaTLSClientAuth      = "tls_client_auth"
+	KafkaSASL               = "sasl"
+	KafkaSASLUsername       = "sasl_username"
+	KafkaSASLPassword       = "sasl_password"
+	KafkaSASLMechanism      = "sasl_mechanism"
+	KafkaClientTLSCert      = "client_tls_cert"
+	KafkaClientTLSKey       = "client_tls_key"
+	KafkaVersion            = "version"
+	KafkaBatchSize          = "batch_size"
+	KafkaBatchCommitTimeout = "batch_commit_timeout"
+	KafkaCompressionCodec   = "compression_codec"
+	KafkaCompressionLevel   = "compression_level"
+
+	EnvKafkaEnable                   = "S3_NOTIFY_KAFKA_ENABLE"
+	EnvKafkaBrokers                  = "S3_NOTIFY_KAFKA_BROKERS"
+	EnvKafkaTopic                    = "S3_NOTIFY_KAFKA_TOPIC"
+	EnvKafkaQueueDir                 = "S3_NOTIFY_KAFKA_QUEUE_DIR"
+	EnvKafkaQueueLimit               = "S3_NOTIFY_KAFKA_QUEUE_LIMIT"
+	EnvKafkaTLS                      = "S3_NOTIFY_KAFKA_TLS"
+	EnvKafkaTLSSkipVerify            = "S3_NOTIFY_KAFKA_TLS_SKIP_VERIFY"
+	EnvKafkaTLSClientAuth            = "S3_NOTIFY_KAFKA_TLS_CLIENT_AUTH"
+	EnvKafkaSASLEnable               = "S3_NOTIFY_KAFKA_SASL"
+	EnvKafkaSASLUsername             = "S3_NOTIFY_KAFKA_SASL_USERNAME"
+	EnvKafkaSASLPassword             = "S3_NOTIFY_KAFKA_SASL_PASSWORD"
+	EnvKafkaSASLMechanism            = "S3_NOTIFY_KAFKA_SASL_MECHANISM"
+	EnvKafkaClientTLSCert            = "S3_NOTIFY_KAFKA_CLIENT_TLS_CERT"
+	EnvKafkaClientTLSKey             = "S3_NOTIFY_KAFKA_CLIENT_TLS_KEY"
+	EnvKafkaVersion                  = "S3_NOTIFY_KAFKA_VERSION"
+	EnvKafkaBatchSize                = "S3_NOTIFY_KAFKA_BATCH_SIZE"
+	EnvKafkaBatchCommitTimeout       = "S3_NOTIFY_KAFKA_BATCH_COMMIT_TIMEOUT"
+	EnvKafkaProducerCompressionCodec = "S3_NOTIFY_KAFKA_PRODUCER_COMPRESSION_CODEC"
+	EnvKafkaProducerCompressionLevel = "S3_NOTIFY_KAFKA_PRODUCER_COMPRESSION_LEVEL"
+)
+
+var codecs = map[string]sarama.CompressionCodec{
+	"none":   sarama.CompressionNone,
+	"gzip":   sarama.CompressionGZIP,
+	"snappy": sarama.CompressionSnappy,
+	"lz4":    sarama.CompressionLZ4,
+	"zstd":   sarama.CompressionZSTD,
+}
+
+// KafkaArgs - Kafka target arguments.
+type KafkaArgs struct {
+	Enable             bool          `json:"enable"`
+	Brokers            []xnet.Host   `json:"brokers"`
+	Topic              string        `json:"topic"`
+	QueueDir           string        `json:"queueDir"`
+	QueueLimit         uint64        `json:"queueLimit"`
+	Version            string        `json:"version"`
+	BatchSize          uint32        `json:"batchSize"`
+	BatchCommitTimeout time.Duration `json:"batchCommitTimeout"`
+	TLS                struct {
+		Enable        bool               `json:"enable"`
+		RootCAs       *x509.CertPool     `json:"-"`
+		SkipVerify    bool               `json:"skipVerify"`
+		ClientAuth    tls.ClientAuthType `json:"clientAuth"`
+		ClientTLSCert string             `json:"clientTLSCert"`
+		ClientTLSKey  string             `json:"clientTLSKey"`
+	} `json:"tls"`
+	SASL struct {
+		Enable    bool   `json:"enable"`
+		User      string `json:"username"`
+		Password  string `json:"password"`
+		Mechanism string `json:"mechanism"`
+	} `json:"sasl"`
+	Producer struct {
+		Compression      string `json:"compression"`
+		CompressionLevel int    `json:"compressionLevel"`
+	} `json:"producer"`
+}
+
+// Validate KafkaArgs fields
+func (k KafkaArgs) Validate() error {
+	if !k.Enable {
+		return nil
+	}
+	if len(k.Brokers) == 0 {
+		return errors.New("no broker address found")
+	}
+	for _, b := range k.Brokers {
+		if _, err := xnet.ParseHost(b.String()); err != nil {
+			return err
+		}
+	}
+	if k.QueueDir != "" {
+		if !filepath.IsAbs(k.QueueDir) {
+			return errors.New("queueDir path should be absolute")
+		}
+	}
+	if k.Version != "" {
+		if _, err := sarama.ParseKafkaVersion(k.Version); err != nil {
+			return err
+		}
+	}
+	if k.BatchSize > 1 {
+		if k.QueueDir == "" {
+			return errors.New("batch should be enabled only if queue dir is enabled")
+		}
+	}
+	if k.BatchCommitTimeout > 0 {
+		if k.QueueDir == "" || k.BatchSize <= 1 {
+			return errors.New("batch commit timeout should be set only if queue dir is enabled and batch size > 1")
+		}
+	}
+	return nil
+}
+
+// KafkaTarget - Kafka target.
+type KafkaTarget struct {
+	initOnce once.Init
+
+	id         event.TargetID
+	args       KafkaArgs
+	client     sarama.Client
+	producer   sarama.SyncProducer
+	config     *sarama.Config
+	store      store.Store[event.Event]
+	batch      *store.Batch[event.Event]
+	loggerOnce logger.LogOnce
+	quitCh     chan struct{}
+}
+
+// ID - returns target ID.
+func (target *KafkaTarget) ID() event.TargetID {
+	return target.id
+}
+
+// Name - returns the Name of the target.
+func (target *KafkaTarget) Name() string {
+	return target.ID().String()
+}
+
+// Store returns any underlying store if set.
+func (target *KafkaTarget) Store() event.TargetStore {
+	return target.store
+}
+
+// IsActive - Return true if target is up and active
+func (target *KafkaTarget) IsActive() (bool, error) {
+	if err := target.init(); err != nil {
+		return false, err
+	}
+	return target.isActive()
+}
+
+func (target *KafkaTarget) isActive() (bool, error) {
+	// Refer https://github.com/IBM/sarama/issues/1341
+	brokers := target.client.Brokers()
+	if len(brokers) == 0 {
+		return false, store.ErrNotConnected
+	}
+	return true, nil
+}
+
+// Save - saves the events to the store which will be replayed when the Kafka connection is active.
+func (target *KafkaTarget) Save(eventData event.Event) error {
+	if target.store != nil {
+		if target.batch != nil {
+			return target.batch.Add(eventData)
+		}
+		_, err := target.store.Put(eventData)
+		return err
+	}
+	if err := target.init(); err != nil {
+		return err
+	}
+	return target.send(eventData)
+}
+
+// send - sends an event to the kafka.
+func (target *KafkaTarget) send(eventData event.Event) error {
+	if target.producer == nil {
+		return store.ErrNotConnected
+	}
+	msg, err := target.toProducerMessage(eventData)
+	if err != nil {
+		return err
+	}
+	_, _, err = target.producer.SendMessage(msg)
+	return err
+}
+
+// sendMultiple sends multiple messages to the kafka.
+func (target *KafkaTarget) sendMultiple(events []event.Event) error {
+	if target.producer == nil {
+		return store.ErrNotConnected
+	}
+	var msgs []*sarama.ProducerMessage
+	for _, event := range events {
+		msg, err := target.toProducerMessage(event)
+		if err != nil {
+			return err
+		}
+		msgs = append(msgs, msg)
+	}
+	return target.producer.SendMessages(msgs)
+}
+
+// SendFromStore - reads an event from store and sends it to Kafka.
+func (target *KafkaTarget) SendFromStore(key store.Key) (err error) {
+	if err = target.init(); err != nil {
+		return err
+	}
+	switch {
+	case key.ItemCount == 1:
+		var event event.Event
+		event, err = target.store.Get(key)
+		if err != nil {
+			// The last event key in a successful batch will be sent in the channel atmost once by the replayEvents()
+			// Such events will not exist and wouldve been already been sent successfully.
+			if os.IsNotExist(err) {
+				return nil
+			}
+			return err
+		}
+		err = target.send(event)
+	case key.ItemCount > 1:
+		var events []event.Event
+		events, err = target.store.GetMultiple(key)
+		if err != nil {
+			if os.IsNotExist(err) {
+				return nil
+			}
+			return err
+		}
+		err = target.sendMultiple(events)
+	}
+	if err != nil {
+		if isKafkaConnErr(err) {
+			return store.ErrNotConnected
+		}
+		return err
+	}
+	// Delete the event from store.
+	return target.store.Del(key)
+}
+
+func (target *KafkaTarget) toProducerMessage(eventData event.Event) (*sarama.ProducerMessage, error) {
+	objectName, err := url.QueryUnescape(eventData.S3.Object.Key)
+	if err != nil {
+		return nil, err
+	}
+
+	key := eventData.S3.Bucket.Name + "/" + objectName
+	data, err := json.Marshal(event.Log{EventName: eventData.EventName, Key: key, Records: []event.Event{eventData}})
+	if err != nil {
+		return nil, err
+	}
+
+	return &sarama.ProducerMessage{
+		Topic: target.args.Topic,
+		Key:   sarama.StringEncoder(key),
+		Value: sarama.ByteEncoder(data),
+	}, nil
+}
+
+// Close - closes underneath kafka connection.
+func (target *KafkaTarget) Close() error {
+	close(target.quitCh)
+
+	if target.batch != nil {
+		target.batch.Close()
+	}
+
+	if target.producer != nil {
+		if target.store != nil {
+			// It is safe to abort the current transaction if
+			// queue_dir is configured
+			target.producer.AbortTxn()
+		} else {
+			target.producer.CommitTxn()
+		}
+		target.producer.Close()
+		return target.client.Close()
+	}
+
+	return nil
+}
+
+func (target *KafkaTarget) init() error {
+	return target.initOnce.Do(target.initKafka)
+}
+
+func (target *KafkaTarget) initKafka() error {
+	if os.Getenv("_S3_KAFKA_DEBUG") != "" {
+		sarama.DebugLogger = log.Default()
+	}
+
+	args := target.args
+
+	config := sarama.NewConfig()
+	if args.Version != "" {
+		kafkaVersion, err := sarama.ParseKafkaVersion(args.Version)
+		if err != nil {
+			target.loggerOnce(context.Background(), err, target.ID().String())
+			return err
+		}
+		config.Version = kafkaVersion
+	}
+
+	config.Net.KeepAlive = 60 * time.Second
+	config.Net.SASL.User = args.SASL.User
+	config.Net.SASL.Password = args.SASL.Password
+	initScramClient(args, config) // initializes configured scram client.
+	config.Net.SASL.Enable = args.SASL.Enable
+
+	tlsConfig, err := saramatls.NewConfig(args.TLS.ClientTLSCert, args.TLS.ClientTLSKey)
+	if err != nil {
+		target.loggerOnce(context.Background(), err, target.ID().String())
+		return err
+	}
+
+	config.Net.TLS.Enable = args.TLS.Enable
+	config.Net.TLS.Config = tlsConfig
+	config.Net.TLS.Config.InsecureSkipVerify = args.TLS.SkipVerify
+	config.Net.TLS.Config.ClientAuth = args.TLS.ClientAuth
+	config.Net.TLS.Config.RootCAs = args.TLS.RootCAs
+
+	// These settings are needed to ensure that kafka client doesn't hang on brokers
+	// refer https://github.com/IBM/sarama/issues/765#issuecomment-254333355
+	config.Producer.Retry.Max = 2
+	config.Producer.Retry.Backoff = (1 * time.Second)
+	config.Producer.Return.Successes = true
+	config.Producer.Return.Errors = true
+	config.Producer.RequiredAcks = 1
+	config.Producer.Timeout = (5 * time.Second)
+	// Set Producer Compression
+	cc, ok := codecs[strings.ToLower(args.Producer.Compression)]
+	if ok {
+		config.Producer.Compression = cc
+		config.Producer.CompressionLevel = args.Producer.CompressionLevel
+	}
+
+	config.Net.ReadTimeout = (5 * time.Second)
+	config.Net.DialTimeout = (5 * time.Second)
+	config.Net.WriteTimeout = (5 * time.Second)
+	config.Metadata.Retry.Max = 1
+	config.Metadata.Retry.Backoff = (1 * time.Second)
+	config.Metadata.RefreshFrequency = (15 * time.Minute)
+
+	target.config = config
+
+	brokers := []string{}
+	for _, broker := range args.Brokers {
+		brokers = append(brokers, broker.String())
+	}
+
+	client, err := sarama.NewClient(brokers, config)
+	if err != nil {
+		if !errors.Is(err, sarama.ErrOutOfBrokers) {
+			target.loggerOnce(context.Background(), err, target.ID().String())
+		}
+		return err
+	}
+
+	producer, err := sarama.NewSyncProducerFromClient(client)
+	if err != nil {
+		if !errors.Is(err, sarama.ErrOutOfBrokers) {
+			target.loggerOnce(context.Background(), err, target.ID().String())
+		}
+		return err
+	}
+	target.client = client
+	target.producer = producer
+
+	yes, err := target.isActive()
+	if err != nil {
+		return err
+	}
+	if !yes {
+		return store.ErrNotConnected
+	}
+
+	return nil
+}
+
+// NewKafkaTarget - creates new Kafka target with auth credentials.
+func NewKafkaTarget(id string, args KafkaArgs, loggerOnce logger.LogOnce) (*KafkaTarget, error) {
+	var queueStore store.Store[event.Event]
+	if args.QueueDir != "" {
+		queueDir := filepath.Join(args.QueueDir, storePrefix+"-kafka-"+id)
+		queueStore = store.NewQueueStore[event.Event](queueDir, args.QueueLimit, event.StoreExtension)
+		if err := queueStore.Open(); err != nil {
+			return nil, fmt.Errorf("unable to initialize the queue store of Kafka `%s`: %w", id, err)
+		}
+	}
+
+	target := &KafkaTarget{
+		id:         event.TargetID{ID: id, Name: "kafka"},
+		args:       args,
+		store:      queueStore,
+		loggerOnce: loggerOnce,
+		quitCh:     make(chan struct{}),
+	}
+	if target.store != nil {
+		if args.BatchSize > 1 {
+			target.batch = store.NewBatch[event.Event](store.BatchConfig[event.Event]{
+				Limit:         args.BatchSize,
+				Log:           loggerOnce,
+				Store:         queueStore,
+				CommitTimeout: args.BatchCommitTimeout,
+			})
+		}
+		store.StreamItems(target.store, target, target.quitCh, target.loggerOnce)
+	}
+
+	return target, nil
+}
+
+func isKafkaConnErr(err error) bool {
+	// Sarama opens the circuit breaker after 3 consecutive connection failures.
+	return err == sarama.ErrLeaderNotAvailable || err.Error() == "circuit breaker is open"
+}

internal/event/target/kafka_scram_client_contrib.go

@@ -0,0 +1,86 @@
+/*
+ * Hanzo S3 Object Storage (c) 2021-2023 Hanzo AI, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package target
+
+import (
+	"crypto/sha512"
+	"strings"
+
+	"github.com/IBM/sarama"
+	"github.com/xdg/scram"
+
+	"github.com/hanzoai/s3/internal/hash/sha256"
+)
+
+func initScramClient(args KafkaArgs, config *sarama.Config) {
+	switch strings.ToLower(args.SASL.Mechanism) {
+	case "sha512":
+		config.Net.SASL.SCRAMClientGeneratorFunc = func() sarama.SCRAMClient { return &XDGSCRAMClient{HashGeneratorFcn: KafkaSHA512} }
+		config.Net.SASL.Mechanism = sarama.SASLMechanism(sarama.SASLTypeSCRAMSHA512)
+	case "sha256":
+		config.Net.SASL.SCRAMClientGeneratorFunc = func() sarama.SCRAMClient { return &XDGSCRAMClient{HashGeneratorFcn: KafkaSHA256} }
+		config.Net.SASL.Mechanism = sarama.SASLMechanism(sarama.SASLTypeSCRAMSHA256)
+	default:
+		// default to PLAIN
+		config.Net.SASL.Mechanism = sarama.SASLMechanism(sarama.SASLTypePlaintext)
+	}
+}
+
+// KafkaSHA256 is a function that returns a crypto/sha256 hasher and should be used
+// to create Client objects configured for SHA-256 hashing.
+var KafkaSHA256 scram.HashGeneratorFcn = sha256.New
+
+// KafkaSHA512 is a function that returns a crypto/sha512 hasher and should be used
+// to create Client objects configured for SHA-512 hashing.
+var KafkaSHA512 scram.HashGeneratorFcn = sha512.New
+
+// XDGSCRAMClient implements the client-side of an authentication
+// conversation with a server.  A new conversation must be created for
+// each authentication attempt.
+type XDGSCRAMClient struct {
+	*scram.Client
+	*scram.ClientConversation
+	scram.HashGeneratorFcn
+}
+
+// Begin constructs a SCRAM client component based on a given hash.Hash
+// factory receiver.  This constructor will normalize the username, password
+// and authzID via the SASLprep algorithm, as recommended by RFC-5802.  If
+// SASLprep fails, the method returns an error.
+func (x *XDGSCRAMClient) Begin(userName, password, authzID string) (err error) {
+	x.Client, err = x.NewClient(userName, password, authzID)
+	if err != nil {
+		return err
+	}
+	x.ClientConversation = x.NewConversation()
+	return nil
+}
+
+// Step takes a string provided from a server (or just an empty string for the
+// very first conversation step) and attempts to move the authentication
+// conversation forward.  It returns a string to be sent to the server or an
+// error if the server message is invalid.  Calling Step after a conversation
+// completes is also an error.
+func (x *XDGSCRAMClient) Step(challenge string) (response string, err error) {
+	response, err = x.ClientConversation.Step(challenge)
+	return response, err
+}
+
+// Done returns true if the conversation is completed or has errored.
+func (x *XDGSCRAMClient) Done() bool {
+	return x.ClientConversation.Done()
+}

internal/event/target/mqtt.go

@@ -0,0 +1,308 @@
+// Copyright (c) 2015-2023 Hanzo AI, Inc.
+//
+// This file is part of Hanzo S3 Object Storage stack
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package target
+
+import (
+	"crypto/tls"
+	"crypto/x509"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"net/url"
+	"os"
+	"path/filepath"
+	"time"
+
+	mqtt "github.com/eclipse/paho.mqtt.golang"
+	"github.com/hanzoai/s3/internal/event"
+	"github.com/hanzoai/s3/internal/logger"
+	"github.com/hanzoai/s3/internal/once"
+	"github.com/hanzoai/s3/internal/store"
+	xnet "github.com/minio/pkg/v3/net"
+)
+
+const (
+	reconnectInterval = 5 * time.Second
+	storePrefix       = "minio"
+)
+
+// MQTT input constants
+const (
+	MqttBroker            = "broker"
+	MqttTopic             = "topic"
+	MqttQoS               = "qos"
+	MqttUsername          = "username"
+	MqttPassword          = "password"
+	MqttReconnectInterval = "reconnect_interval"
+	MqttKeepAliveInterval = "keep_alive_interval"
+	MqttQueueDir          = "queue_dir"
+	MqttQueueLimit        = "queue_limit"
+
+	EnvMQTTEnable            = "S3_NOTIFY_MQTT_ENABLE"
+	EnvMQTTBroker            = "S3_NOTIFY_MQTT_BROKER"
+	EnvMQTTTopic             = "S3_NOTIFY_MQTT_TOPIC"
+	EnvMQTTQoS               = "S3_NOTIFY_MQTT_QOS"
+	EnvMQTTUsername          = "S3_NOTIFY_MQTT_USERNAME"
+	EnvMQTTPassword          = "S3_NOTIFY_MQTT_PASSWORD"
+	EnvMQTTReconnectInterval = "S3_NOTIFY_MQTT_RECONNECT_INTERVAL"
+	EnvMQTTKeepAliveInterval = "S3_NOTIFY_MQTT_KEEP_ALIVE_INTERVAL"
+	EnvMQTTQueueDir          = "S3_NOTIFY_MQTT_QUEUE_DIR"
+	EnvMQTTQueueLimit        = "S3_NOTIFY_MQTT_QUEUE_LIMIT"
+)
+
+// MQTTArgs - MQTT target arguments.
+type MQTTArgs struct {
+	Enable               bool           `json:"enable"`
+	Broker               xnet.URL       `json:"broker"`
+	Topic                string         `json:"topic"`
+	QoS                  byte           `json:"qos"`
+	User                 string         `json:"username"`
+	Password             string         `json:"password"`
+	MaxReconnectInterval time.Duration  `json:"reconnectInterval"`
+	KeepAlive            time.Duration  `json:"keepAliveInterval"`
+	RootCAs              *x509.CertPool `json:"-"`
+	QueueDir             string         `json:"queueDir"`
+	QueueLimit           uint64         `json:"queueLimit"`
+}
+
+// Validate MQTTArgs fields
+func (m MQTTArgs) Validate() error {
+	if !m.Enable {
+		return nil
+	}
+	u, err := xnet.ParseURL(m.Broker.String())
+	if err != nil {
+		return err
+	}
+	switch u.Scheme {
+	case "ws", "wss", "tcp", "ssl", "tls", "tcps":
+	default:
+		return errors.New("unknown protocol in broker address")
+	}
+	if m.QueueDir != "" {
+		if !filepath.IsAbs(m.QueueDir) {
+			return errors.New("queueDir path should be absolute")
+		}
+		if m.QoS == 0 {
+			return errors.New("qos should be set to 1 or 2 if queueDir is set")
+		}
+	}
+
+	return nil
+}
+
+// MQTTTarget - MQTT target.
+type MQTTTarget struct {
+	initOnce once.Init
+
+	id         event.TargetID
+	args       MQTTArgs
+	client     mqtt.Client
+	store      store.Store[event.Event]
+	quitCh     chan struct{}
+	loggerOnce logger.LogOnce
+}
+
+// ID - returns target ID.
+func (target *MQTTTarget) ID() event.TargetID {
+	return target.id
+}
+
+// Name - returns the Name of the target.
+func (target *MQTTTarget) Name() string {
+	return target.ID().String()
+}
+
+// Store returns any underlying store if set.
+func (target *MQTTTarget) Store() event.TargetStore {
+	return target.store
+}
+
+// IsActive - Return true if target is up and active
+func (target *MQTTTarget) IsActive() (bool, error) {
+	if err := target.init(); err != nil {
+		return false, err
+	}
+	return target.isActive()
+}
+
+func (target *MQTTTarget) isActive() (bool, error) {
+	if !target.client.IsConnectionOpen() {
+		return false, store.ErrNotConnected
+	}
+	return true, nil
+}
+
+// send - sends an event to the mqtt.
+func (target *MQTTTarget) send(eventData event.Event) error {
+	objectName, err := url.QueryUnescape(eventData.S3.Object.Key)
+	if err != nil {
+		return err
+	}
+	key := eventData.S3.Bucket.Name + "/" + objectName
+
+	data, err := json.Marshal(event.Log{EventName: eventData.EventName, Key: key, Records: []event.Event{eventData}})
+	if err != nil {
+		return err
+	}
+
+	token := target.client.Publish(target.args.Topic, target.args.QoS, false, string(data))
+	if !token.WaitTimeout(reconnectInterval) {
+		return store.ErrNotConnected
+	}
+	return token.Error()
+}
+
+// SendFromStore - reads an event from store and sends it to MQTT.
+func (target *MQTTTarget) SendFromStore(key store.Key) error {
+	if err := target.init(); err != nil {
+		return err
+	}
+
+	// Do not send if the connection is not active.
+	_, err := target.isActive()
+	if err != nil {
+		return err
+	}
+
+	eventData, err := target.store.Get(key)
+	if err != nil {
+		// The last event key in a successful batch will be sent in the channel atmost once by the replayEvents()
+		// Such events will not exist and wouldve been already been sent successfully.
+		if os.IsNotExist(err) {
+			return nil
+		}
+		return err
+	}
+
+	if err = target.send(eventData); err != nil {
+		return err
+	}
+
+	// Delete the event from store.
+	return target.store.Del(key)
+}
+
+// Save - saves the events to the store if queuestore is configured, which will
+// be replayed when the mqtt connection is active.
+func (target *MQTTTarget) Save(eventData event.Event) error {
+	if target.store != nil {
+		_, err := target.store.Put(eventData)
+		return err
+	}
+	if err := target.init(); err != nil {
+		return err
+	}
+
+	// Do not send if the connection is not active.
+	_, err := target.isActive()
+	if err != nil {
+		return err
+	}
+
+	return target.send(eventData)
+}
+
+// Close - does nothing and available for interface compatibility.
+func (target *MQTTTarget) Close() error {
+	if target.client != nil {
+		target.client.Disconnect(100)
+	}
+	close(target.quitCh)
+	return nil
+}
+
+func (target *MQTTTarget) init() error {
+	return target.initOnce.Do(target.initMQTT)
+}
+
+func (target *MQTTTarget) initMQTT() error {
+	args := target.args
+
+	// Using hex here, to make sure we avoid 23
+	// character limit on client_id according to
+	// MQTT spec.
+	clientID := fmt.Sprintf("%x", time.Now().UnixNano())
+
+	options := mqtt.NewClientOptions().
+		SetClientID(clientID).
+		SetCleanSession(true).
+		SetUsername(args.User).
+		SetPassword(args.Password).
+		SetMaxReconnectInterval(args.MaxReconnectInterval).
+		SetKeepAlive(args.KeepAlive).
+		SetTLSConfig(&tls.Config{RootCAs: args.RootCAs}).
+		AddBroker(args.Broker.String())
+
+	target.client = mqtt.NewClient(options)
+
+	token := target.client.Connect()
+	ok := token.WaitTimeout(reconnectInterval)
+	if !ok {
+		return store.ErrNotConnected
+	}
+	if token.Error() != nil {
+		return token.Error()
+	}
+
+	yes, err := target.isActive()
+	if err != nil {
+		return err
+	}
+	if !yes {
+		return store.ErrNotConnected
+	}
+
+	return nil
+}
+
+// NewMQTTTarget - creates new MQTT target.
+func NewMQTTTarget(id string, args MQTTArgs, loggerOnce logger.LogOnce) (*MQTTTarget, error) {
+	if args.MaxReconnectInterval == 0 {
+		// Default interval
+		// https://github.com/eclipse/paho.mqtt.golang/blob/master/options.go#L115
+		args.MaxReconnectInterval = 10 * time.Minute
+	}
+
+	if args.KeepAlive == 0 {
+		args.KeepAlive = 10 * time.Second
+	}
+
+	var queueStore store.Store[event.Event]
+	if args.QueueDir != "" {
+		queueDir := filepath.Join(args.QueueDir, storePrefix+"-mqtt-"+id)
+		queueStore = store.NewQueueStore[event.Event](queueDir, args.QueueLimit, event.StoreExtension)
+		if err := queueStore.Open(); err != nil {
+			return nil, fmt.Errorf("unable to initialize the queue store of MQTT `%s`: %w", id, err)
+		}
+	}
+
+	target := &MQTTTarget{
+		id:         event.TargetID{ID: id, Name: "mqtt"},
+		args:       args,
+		store:      queueStore,
+		quitCh:     make(chan struct{}),
+		loggerOnce: loggerOnce,
+	}
+
+	if target.store != nil {
+		store.StreamItems(target.store, target, target.quitCh, target.loggerOnce)
+	}
+
+	return target, nil
+}

internal/event/target/mysql.go

@@ -0,0 +1,444 @@
+// Copyright (c) 2015-2023 Hanzo AI, Inc.
+//
+// This file is part of Hanzo S3 Object Storage stack
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package target
+
+import (
+	"context"
+	"database/sql"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"net/url"
+	"os"
+	"path/filepath"
+	"strconv"
+	"strings"
+	"time"
+
+	"github.com/go-sql-driver/mysql"
+	"github.com/hanzoai/s3/internal/event"
+	"github.com/hanzoai/s3/internal/logger"
+	"github.com/hanzoai/s3/internal/once"
+	"github.com/hanzoai/s3/internal/store"
+	xnet "github.com/minio/pkg/v3/net"
+)
+
+const (
+	mysqlTableExists = `SELECT 1 FROM %s;`
+	// Some MySQL has a 3072 byte limit on key sizes.
+	mysqlCreateNamespaceTable = `CREATE TABLE %s (
+             key_name VARCHAR(3072) NOT NULL,
+             key_hash CHAR(64) GENERATED ALWAYS AS (SHA2(key_name, 256)) STORED NOT NULL PRIMARY KEY,
+             value JSON)
+           CHARACTER SET = utf8mb4 COLLATE = utf8mb4_bin ROW_FORMAT = Dynamic;`
+	mysqlCreateAccessTable = `CREATE TABLE %s (event_time DATETIME NOT NULL, event_data JSON)
+                                    ROW_FORMAT = Dynamic;`
+
+	mysqlUpdateRow = `INSERT INTO %s (key_name, value) VALUES (?, ?) ON DUPLICATE KEY UPDATE value=VALUES(value);`
+	mysqlDeleteRow = `DELETE FROM %s WHERE key_hash = SHA2(?, 256);`
+	mysqlInsertRow = `INSERT INTO %s (event_time, event_data) VALUES (?, ?);`
+)
+
+// MySQL related constants
+const (
+	MySQLFormat             = "format"
+	MySQLDSNString          = "dsn_string"
+	MySQLTable              = "table"
+	MySQLHost               = "host"
+	MySQLPort               = "port"
+	MySQLUsername           = "username"
+	MySQLPassword           = "password"
+	MySQLDatabase           = "database"
+	MySQLQueueLimit         = "queue_limit"
+	MySQLQueueDir           = "queue_dir"
+	MySQLMaxOpenConnections = "max_open_connections"
+
+	EnvMySQLEnable             = "S3_NOTIFY_MYSQL_ENABLE"
+	EnvMySQLFormat             = "S3_NOTIFY_MYSQL_FORMAT"
+	EnvMySQLDSNString          = "S3_NOTIFY_MYSQL_DSN_STRING"
+	EnvMySQLTable              = "S3_NOTIFY_MYSQL_TABLE"
+	EnvMySQLHost               = "S3_NOTIFY_MYSQL_HOST"
+	EnvMySQLPort               = "S3_NOTIFY_MYSQL_PORT"
+	EnvMySQLUsername           = "S3_NOTIFY_MYSQL_USERNAME"
+	EnvMySQLPassword           = "S3_NOTIFY_MYSQL_PASSWORD"
+	EnvMySQLDatabase           = "S3_NOTIFY_MYSQL_DATABASE"
+	EnvMySQLQueueLimit         = "S3_NOTIFY_MYSQL_QUEUE_LIMIT"
+	EnvMySQLQueueDir           = "S3_NOTIFY_MYSQL_QUEUE_DIR"
+	EnvMySQLMaxOpenConnections = "S3_NOTIFY_MYSQL_MAX_OPEN_CONNECTIONS"
+)
+
+// MySQLArgs - MySQL target arguments.
+type MySQLArgs struct {
+	Enable             bool     `json:"enable"`
+	Format             string   `json:"format"`
+	DSN                string   `json:"dsnString"`
+	Table              string   `json:"table"`
+	Host               xnet.URL `json:"host"`
+	Port               string   `json:"port"`
+	User               string   `json:"user"`
+	Password           string   `json:"password"`
+	Database           string   `json:"database"`
+	QueueDir           string   `json:"queueDir"`
+	QueueLimit         uint64   `json:"queueLimit"`
+	MaxOpenConnections int      `json:"maxOpenConnections"`
+}
+
+// Validate MySQLArgs fields
+func (m MySQLArgs) Validate() error {
+	if !m.Enable {
+		return nil
+	}
+
+	if m.Format != "" {
+		f := strings.ToLower(m.Format)
+		if f != event.NamespaceFormat && f != event.AccessFormat {
+			return fmt.Errorf("unrecognized format")
+		}
+	}
+
+	if m.Table == "" {
+		return fmt.Errorf("table unspecified")
+	}
+
+	if m.DSN != "" {
+		if _, err := mysql.ParseDSN(m.DSN); err != nil {
+			return err
+		}
+	} else {
+		// Some fields need to be specified when DSN is unspecified
+		if m.Port == "" {
+			return fmt.Errorf("unspecified port")
+		}
+		if _, err := strconv.Atoi(m.Port); err != nil {
+			return fmt.Errorf("invalid port")
+		}
+		if m.Database == "" {
+			return fmt.Errorf("database unspecified")
+		}
+	}
+
+	if m.QueueDir != "" {
+		if !filepath.IsAbs(m.QueueDir) {
+			return errors.New("queueDir path should be absolute")
+		}
+	}
+
+	if m.MaxOpenConnections < 0 {
+		return errors.New("maxOpenConnections cannot be less than zero")
+	}
+
+	return nil
+}
+
+// MySQLTarget - MySQL target.
+type MySQLTarget struct {
+	initOnce once.Init
+
+	id         event.TargetID
+	args       MySQLArgs
+	updateStmt *sql.Stmt
+	deleteStmt *sql.Stmt
+	insertStmt *sql.Stmt
+	db         *sql.DB
+	store      store.Store[event.Event]
+	firstPing  bool
+	loggerOnce logger.LogOnce
+
+	quitCh chan struct{}
+}
+
+// ID - returns target ID.
+func (target *MySQLTarget) ID() event.TargetID {
+	return target.id
+}
+
+// Name - returns the Name of the target.
+func (target *MySQLTarget) Name() string {
+	return target.ID().String()
+}
+
+// Store returns any underlying store if set.
+func (target *MySQLTarget) Store() event.TargetStore {
+	return target.store
+}
+
+// IsActive - Return true if target is up and active
+func (target *MySQLTarget) IsActive() (bool, error) {
+	if err := target.init(); err != nil {
+		return false, err
+	}
+	return target.isActive()
+}
+
+func (target *MySQLTarget) isActive() (bool, error) {
+	if err := target.db.Ping(); err != nil {
+		if IsConnErr(err) {
+			return false, store.ErrNotConnected
+		}
+		return false, err
+	}
+	return true, nil
+}
+
+// Save - saves the events to the store which will be replayed when the SQL connection is active.
+func (target *MySQLTarget) Save(eventData event.Event) error {
+	if target.store != nil {
+		_, err := target.store.Put(eventData)
+		return err
+	}
+	if err := target.init(); err != nil {
+		return err
+	}
+
+	_, err := target.isActive()
+	if err != nil {
+		return err
+	}
+	return target.send(eventData)
+}
+
+// send - sends an event to the mysql.
+func (target *MySQLTarget) send(eventData event.Event) error {
+	if target.args.Format == event.NamespaceFormat {
+		objectName, err := url.QueryUnescape(eventData.S3.Object.Key)
+		if err != nil {
+			return err
+		}
+		key := eventData.S3.Bucket.Name + "/" + objectName
+
+		if eventData.EventName == event.ObjectRemovedDelete {
+			_, err = target.deleteStmt.Exec(key)
+		} else {
+			var data []byte
+			if data, err = json.Marshal(struct{ Records []event.Event }{[]event.Event{eventData}}); err != nil {
+				return err
+			}
+
+			_, err = target.updateStmt.Exec(key, data)
+		}
+
+		return err
+	}
+
+	if target.args.Format == event.AccessFormat {
+		eventTime, err := time.Parse(event.AMZTimeFormat, eventData.EventTime)
+		if err != nil {
+			return err
+		}
+
+		data, err := json.Marshal(struct{ Records []event.Event }{[]event.Event{eventData}})
+		if err != nil {
+			return err
+		}
+
+		_, err = target.insertStmt.Exec(eventTime, data)
+
+		return err
+	}
+
+	return nil
+}
+
+// SendFromStore - reads an event from store and sends it to MySQL.
+func (target *MySQLTarget) SendFromStore(key store.Key) error {
+	if err := target.init(); err != nil {
+		return err
+	}
+
+	_, err := target.isActive()
+	if err != nil {
+		return err
+	}
+
+	if !target.firstPing {
+		if err := target.executeStmts(); err != nil {
+			if IsConnErr(err) {
+				return store.ErrNotConnected
+			}
+			return err
+		}
+	}
+
+	eventData, eErr := target.store.Get(key)
+	if eErr != nil {
+		// The last event key in a successful batch will be sent in the channel atmost once by the replayEvents()
+		// Such events will not exist and wouldve been already been sent successfully.
+		if os.IsNotExist(eErr) {
+			return nil
+		}
+		return eErr
+	}
+
+	if err := target.send(eventData); err != nil {
+		if IsConnErr(err) {
+			return store.ErrNotConnected
+		}
+		return err
+	}
+
+	// Delete the event from store.
+	return target.store.Del(key)
+}
+
+// Close - closes underneath connections to MySQL database.
+func (target *MySQLTarget) Close() error {
+	close(target.quitCh)
+	if target.updateStmt != nil {
+		// FIXME: log returned error. ignore time being.
+		_ = target.updateStmt.Close()
+	}
+
+	if target.deleteStmt != nil {
+		// FIXME: log returned error. ignore time being.
+		_ = target.deleteStmt.Close()
+	}
+
+	if target.insertStmt != nil {
+		// FIXME: log returned error. ignore time being.
+		_ = target.insertStmt.Close()
+	}
+
+	if target.db != nil {
+		return target.db.Close()
+	}
+
+	return nil
+}
+
+// Executes the table creation statements.
+func (target *MySQLTarget) executeStmts() error {
+	_, err := target.db.Exec(fmt.Sprintf(mysqlTableExists, target.args.Table))
+	if err != nil {
+		createStmt := mysqlCreateNamespaceTable
+		if target.args.Format == event.AccessFormat {
+			createStmt = mysqlCreateAccessTable
+		}
+
+		if _, dbErr := target.db.Exec(fmt.Sprintf(createStmt, target.args.Table)); dbErr != nil {
+			return dbErr
+		}
+	}
+
+	switch target.args.Format {
+	case event.NamespaceFormat:
+		// insert or update statement
+		if target.updateStmt, err = target.db.Prepare(fmt.Sprintf(mysqlUpdateRow, target.args.Table)); err != nil {
+			return err
+		}
+		// delete statement
+		if target.deleteStmt, err = target.db.Prepare(fmt.Sprintf(mysqlDeleteRow, target.args.Table)); err != nil {
+			return err
+		}
+	case event.AccessFormat:
+		// insert statement
+		if target.insertStmt, err = target.db.Prepare(fmt.Sprintf(mysqlInsertRow, target.args.Table)); err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
+func (target *MySQLTarget) init() error {
+	return target.initOnce.Do(target.initMySQL)
+}
+
+func (target *MySQLTarget) initMySQL() error {
+	args := target.args
+
+	db, err := sql.Open("mysql", args.DSN)
+	if err != nil {
+		target.loggerOnce(context.Background(), err, target.ID().String())
+		return err
+	}
+	target.db = db
+
+	if args.MaxOpenConnections > 0 {
+		// Set the maximum connections limit
+		target.db.SetMaxOpenConns(args.MaxOpenConnections)
+	}
+
+	err = target.db.Ping()
+	if err != nil {
+		if !xnet.IsConnRefusedErr(err) && !xnet.IsConnResetErr(err) {
+			target.loggerOnce(context.Background(), err, target.ID().String())
+		}
+	} else {
+		if err = target.executeStmts(); err != nil {
+			target.loggerOnce(context.Background(), err, target.ID().String())
+		} else {
+			target.firstPing = true
+		}
+	}
+
+	if err != nil {
+		target.db.Close()
+		return err
+	}
+
+	yes, err := target.isActive()
+	if err != nil {
+		return err
+	}
+	if !yes {
+		return store.ErrNotConnected
+	}
+
+	return nil
+}
+
+// NewMySQLTarget - creates new MySQL target.
+func NewMySQLTarget(id string, args MySQLArgs, loggerOnce logger.LogOnce) (*MySQLTarget, error) {
+	var queueStore store.Store[event.Event]
+	if args.QueueDir != "" {
+		queueDir := filepath.Join(args.QueueDir, storePrefix+"-mysql-"+id)
+		queueStore = store.NewQueueStore[event.Event](queueDir, args.QueueLimit, event.StoreExtension)
+		if err := queueStore.Open(); err != nil {
+			return nil, fmt.Errorf("unable to initialize the queue store of MySQL `%s`: %w", id, err)
+		}
+	}
+
+	if args.DSN == "" {
+		config := mysql.Config{
+			User:                 args.User,
+			Passwd:               args.Password,
+			Net:                  "tcp",
+			Addr:                 args.Host.String() + ":" + args.Port,
+			DBName:               args.Database,
+			AllowNativePasswords: true,
+			CheckConnLiveness:    true,
+		}
+
+		args.DSN = config.FormatDSN()
+	}
+
+	target := &MySQLTarget{
+		id:         event.TargetID{ID: id, Name: "mysql"},
+		args:       args,
+		firstPing:  false,
+		store:      queueStore,
+		loggerOnce: loggerOnce,
+		quitCh:     make(chan struct{}),
+	}
+
+	if target.store != nil {
+		store.StreamItems(target.store, target, target.quitCh, target.loggerOnce)
+	}
+
+	return target, nil
+}

internal/event/target/mysql_test.go

@@ -0,0 +1,36 @@
+// Copyright (c) 2015-2023 Hanzo AI, Inc.
+//
+// This file is part of Hanzo S3 Object Storage stack
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package target
+
+import (
+	"database/sql"
+	"slices"
+	"testing"
+)
+
+// TestPostgreSQLRegistration checks if sql driver
+// is registered and fails otherwise.
+func TestMySQLRegistration(t *testing.T) {
+	var found bool
+	if slices.Contains(sql.Drivers(), "mysql") {
+		found = true
+	}
+	if !found {
+		t.Fatal("mysql driver not registered")
+	}
+}

internal/event/target/nats.go

@@ -0,0 +1,480 @@
+// Copyright (c) 2015-2023 Hanzo AI, Inc.
+//
+// This file is part of Hanzo S3 Object Storage stack
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package target
+
+import (
+	"context"
+	"crypto/tls"
+	"crypto/x509"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"net/url"
+	"os"
+	"path/filepath"
+
+	"github.com/google/uuid"
+	"github.com/hanzoai/s3/internal/event"
+	"github.com/hanzoai/s3/internal/logger"
+	"github.com/hanzoai/s3/internal/once"
+	"github.com/hanzoai/s3/internal/store"
+	xnet "github.com/minio/pkg/v3/net"
+	"github.com/nats-io/nats.go"
+	"github.com/nats-io/stan.go"
+)
+
+// NATS related constants
+const (
+	NATSAddress           = "address"
+	NATSSubject           = "subject"
+	NATSUsername          = "username"
+	NATSPassword          = "password"
+	NATSToken             = "token"
+	NATSNKeySeed          = "nkey_seed"
+	NATSTLS               = "tls"
+	NATSTLSSkipVerify     = "tls_skip_verify"
+	NATSTLSHandshakeFirst = "tls_handshake_first"
+	NATSPingInterval      = "ping_interval"
+	NATSQueueDir          = "queue_dir"
+	NATSQueueLimit        = "queue_limit"
+	NATSCertAuthority     = "cert_authority"
+	NATSClientCert        = "client_cert"
+	NATSClientKey         = "client_key"
+
+	// Streaming constants - deprecated
+	NATSStreaming                   = "streaming"
+	NATSStreamingClusterID          = "streaming_cluster_id"
+	NATSStreamingAsync              = "streaming_async"
+	NATSStreamingMaxPubAcksInFlight = "streaming_max_pub_acks_in_flight"
+
+	// JetStream constants
+	NATSJetStream = "jetstream"
+
+	EnvNATSEnable            = "S3_NOTIFY_NATS_ENABLE"
+	EnvNATSAddress           = "S3_NOTIFY_NATS_ADDRESS"
+	EnvNATSSubject           = "S3_NOTIFY_NATS_SUBJECT"
+	EnvNATSUsername          = "S3_NOTIFY_NATS_USERNAME"
+	NATSUserCredentials      = "S3_NOTIFY_NATS_USER_CREDENTIALS"
+	EnvNATSPassword          = "S3_NOTIFY_NATS_PASSWORD"
+	EnvNATSToken             = "S3_NOTIFY_NATS_TOKEN"
+	EnvNATSNKeySeed          = "S3_NOTIFY_NATS_NKEY_SEED"
+	EnvNATSTLS               = "S3_NOTIFY_NATS_TLS"
+	EnvNATSTLSSkipVerify     = "S3_NOTIFY_NATS_TLS_SKIP_VERIFY"
+	EnvNatsTLSHandshakeFirst = "S3_NOTIFY_NATS_TLS_HANDSHAKE_FIRST"
+	EnvNATSPingInterval      = "S3_NOTIFY_NATS_PING_INTERVAL"
+	EnvNATSQueueDir          = "S3_NOTIFY_NATS_QUEUE_DIR"
+	EnvNATSQueueLimit        = "S3_NOTIFY_NATS_QUEUE_LIMIT"
+	EnvNATSCertAuthority     = "S3_NOTIFY_NATS_CERT_AUTHORITY"
+	EnvNATSClientCert        = "S3_NOTIFY_NATS_CLIENT_CERT"
+	EnvNATSClientKey         = "S3_NOTIFY_NATS_CLIENT_KEY"
+
+	// Streaming constants - deprecated
+	EnvNATSStreaming                   = "S3_NOTIFY_NATS_STREAMING"
+	EnvNATSStreamingClusterID          = "S3_NOTIFY_NATS_STREAMING_CLUSTER_ID"
+	EnvNATSStreamingAsync              = "S3_NOTIFY_NATS_STREAMING_ASYNC"
+	EnvNATSStreamingMaxPubAcksInFlight = "S3_NOTIFY_NATS_STREAMING_MAX_PUB_ACKS_IN_FLIGHT"
+
+	// Jetstream constants
+	EnvNATSJetStream = "S3_NOTIFY_NATS_JETSTREAM"
+)
+
+// NATSArgs - NATS target arguments.
+type NATSArgs struct {
+	Enable            bool      `json:"enable"`
+	Address           xnet.Host `json:"address"`
+	Subject           string    `json:"subject"`
+	Username          string    `json:"username"`
+	UserCredentials   string    `json:"userCredentials"`
+	Password          string    `json:"password"`
+	Token             string    `json:"token"`
+	NKeySeed          string    `json:"nKeySeed"`
+	TLS               bool      `json:"tls"`
+	TLSSkipVerify     bool      `json:"tlsSkipVerify"`
+	TLSHandshakeFirst bool      `json:"tlsHandshakeFirst"`
+	Secure            bool      `json:"secure"`
+	CertAuthority     string    `json:"certAuthority"`
+	ClientCert        string    `json:"clientCert"`
+	ClientKey         string    `json:"clientKey"`
+	PingInterval      int64     `json:"pingInterval"`
+	QueueDir          string    `json:"queueDir"`
+	QueueLimit        uint64    `json:"queueLimit"`
+	JetStream         struct {
+		Enable bool `json:"enable"`
+	} `json:"jetStream"`
+	Streaming struct {
+		Enable             bool   `json:"enable"`
+		ClusterID          string `json:"clusterID"`
+		Async              bool   `json:"async"`
+		MaxPubAcksInflight int    `json:"maxPubAcksInflight"`
+	} `json:"streaming"`
+
+	RootCAs *x509.CertPool `json:"-"`
+}
+
+// Validate NATSArgs fields
+func (n NATSArgs) Validate() error {
+	if !n.Enable {
+		return nil
+	}
+
+	if n.Address.IsEmpty() {
+		return errors.New("empty address")
+	}
+
+	if n.Subject == "" {
+		return errors.New("empty subject")
+	}
+
+	if n.ClientCert != "" && n.ClientKey == "" || n.ClientCert == "" && n.ClientKey != "" {
+		return errors.New("cert and key must be specified as a pair")
+	}
+
+	if n.Username != "" && n.Password == "" || n.Username == "" && n.Password != "" {
+		return errors.New("username and password must be specified as a pair")
+	}
+
+	if n.Streaming.Enable {
+		if n.Streaming.ClusterID == "" {
+			return errors.New("empty cluster id")
+		}
+	}
+
+	if n.JetStream.Enable {
+		if n.Subject == "" {
+			return errors.New("empty subject")
+		}
+	}
+
+	if n.QueueDir != "" {
+		if !filepath.IsAbs(n.QueueDir) {
+			return errors.New("queueDir path should be absolute")
+		}
+	}
+
+	return nil
+}
+
+// To obtain a nats connection from args.
+func (n NATSArgs) connectNats() (*nats.Conn, error) {
+	connOpts := []nats.Option{nats.Name("Minio Notification"), nats.MaxReconnects(-1)}
+	if n.Username != "" && n.Password != "" {
+		connOpts = append(connOpts, nats.UserInfo(n.Username, n.Password))
+	}
+	if n.UserCredentials != "" {
+		connOpts = append(connOpts, nats.UserCredentials(n.UserCredentials))
+	}
+	if n.Token != "" {
+		connOpts = append(connOpts, nats.Token(n.Token))
+	}
+	if n.NKeySeed != "" {
+		nkeyOpt, err := nats.NkeyOptionFromSeed(n.NKeySeed)
+		if err != nil {
+			return nil, err
+		}
+		connOpts = append(connOpts, nkeyOpt)
+	}
+	if n.Secure || n.TLS && n.TLSSkipVerify {
+		connOpts = append(connOpts, nats.Secure(nil))
+	} else if n.TLS {
+		connOpts = append(connOpts, nats.Secure(&tls.Config{RootCAs: n.RootCAs}))
+	}
+	if n.TLSHandshakeFirst {
+		connOpts = append(connOpts, nats.TLSHandshakeFirst())
+	}
+	if n.CertAuthority != "" {
+		connOpts = append(connOpts, nats.RootCAs(n.CertAuthority))
+	}
+	if n.ClientCert != "" && n.ClientKey != "" {
+		connOpts = append(connOpts, nats.ClientCert(n.ClientCert, n.ClientKey))
+	}
+	return nats.Connect(n.Address.String(), connOpts...)
+}
+
+// To obtain a streaming connection from args.
+func (n NATSArgs) connectStan() (stan.Conn, error) {
+	scheme := "nats"
+	if n.Secure {
+		scheme = "tls"
+	}
+
+	var addressURL string
+	//nolint:gocritic
+	if n.Username != "" && n.Password != "" {
+		addressURL = scheme + "://" + n.Username + ":" + n.Password + "@" + n.Address.String()
+	} else if n.Token != "" {
+		addressURL = scheme + "://" + n.Token + "@" + n.Address.String()
+	} else {
+		addressURL = scheme + "://" + n.Address.String()
+	}
+
+	u, err := uuid.NewRandom()
+	if err != nil {
+		return nil, err
+	}
+	clientID := u.String()
+
+	connOpts := []stan.Option{stan.NatsURL(addressURL)}
+	if n.Streaming.MaxPubAcksInflight > 0 {
+		connOpts = append(connOpts, stan.MaxPubAcksInflight(n.Streaming.MaxPubAcksInflight))
+	}
+	if n.UserCredentials != "" {
+		connOpts = append(connOpts, stan.NatsOptions(nats.UserCredentials(n.UserCredentials)))
+	}
+
+	return stan.Connect(n.Streaming.ClusterID, clientID, connOpts...)
+}
+
+// NATSTarget - NATS target.
+type NATSTarget struct {
+	initOnce once.Init
+
+	id         event.TargetID
+	args       NATSArgs
+	natsConn   *nats.Conn
+	stanConn   stan.Conn
+	jstream    nats.JetStream
+	store      store.Store[event.Event]
+	loggerOnce logger.LogOnce
+	quitCh     chan struct{}
+}
+
+// ID - returns target ID.
+func (target *NATSTarget) ID() event.TargetID {
+	return target.id
+}
+
+// Name - returns the Name of the target.
+func (target *NATSTarget) Name() string {
+	return target.ID().String()
+}
+
+// Store returns any underlying store if set.
+func (target *NATSTarget) Store() event.TargetStore {
+	return target.store
+}
+
+// IsActive - Return true if target is up and active
+func (target *NATSTarget) IsActive() (bool, error) {
+	if err := target.init(); err != nil {
+		return false, err
+	}
+	return target.isActive()
+}
+
+func (target *NATSTarget) isActive() (bool, error) {
+	var connErr error
+	if target.args.Streaming.Enable {
+		if target.stanConn == nil || target.stanConn.NatsConn() == nil {
+			target.stanConn, connErr = target.args.connectStan()
+		} else if !target.stanConn.NatsConn().IsConnected() {
+			return false, store.ErrNotConnected
+		}
+	} else {
+		if target.natsConn == nil {
+			target.natsConn, connErr = target.args.connectNats()
+		} else if !target.natsConn.IsConnected() {
+			return false, store.ErrNotConnected
+		}
+	}
+
+	if connErr != nil {
+		if connErr.Error() == nats.ErrNoServers.Error() {
+			return false, store.ErrNotConnected
+		}
+		return false, connErr
+	}
+
+	if target.natsConn != nil && target.args.JetStream.Enable {
+		target.jstream, connErr = target.natsConn.JetStream()
+		if connErr != nil {
+			if connErr.Error() == nats.ErrNoServers.Error() {
+				return false, store.ErrNotConnected
+			}
+			return false, connErr
+		}
+	}
+
+	return true, nil
+}
+
+// Save - saves the events to the store which will be replayed when the Nats connection is active.
+func (target *NATSTarget) Save(eventData event.Event) error {
+	if target.store != nil {
+		_, err := target.store.Put(eventData)
+		return err
+	}
+
+	if err := target.init(); err != nil {
+		return err
+	}
+
+	_, err := target.isActive()
+	if err != nil {
+		return err
+	}
+	return target.send(eventData)
+}
+
+// send - sends an event to the Nats.
+func (target *NATSTarget) send(eventData event.Event) error {
+	objectName, err := url.QueryUnescape(eventData.S3.Object.Key)
+	if err != nil {
+		return err
+	}
+	key := eventData.S3.Bucket.Name + "/" + objectName
+
+	data, err := json.Marshal(event.Log{EventName: eventData.EventName, Key: key, Records: []event.Event{eventData}})
+	if err != nil {
+		return err
+	}
+
+	if target.stanConn != nil {
+		if target.args.Streaming.Async {
+			_, err = target.stanConn.PublishAsync(target.args.Subject, data, nil)
+		} else {
+			err = target.stanConn.Publish(target.args.Subject, data)
+		}
+	} else {
+		if target.jstream != nil {
+			_, err = target.jstream.Publish(target.args.Subject, data)
+		} else {
+			err = target.natsConn.Publish(target.args.Subject, data)
+		}
+	}
+	return err
+}
+
+// SendFromStore - reads an event from store and sends it to Nats.
+func (target *NATSTarget) SendFromStore(key store.Key) error {
+	if err := target.init(); err != nil {
+		return err
+	}
+
+	_, err := target.isActive()
+	if err != nil {
+		return err
+	}
+
+	eventData, eErr := target.store.Get(key)
+	if eErr != nil {
+		// The last event key in a successful batch will be sent in the channel atmost once by the replayEvents()
+		// Such events will not exist and wouldve been already been sent successfully.
+		if os.IsNotExist(eErr) {
+			return nil
+		}
+		return eErr
+	}
+
+	if err := target.send(eventData); err != nil {
+		return err
+	}
+
+	return target.store.Del(key)
+}
+
+// Close - closes underneath connections to NATS server.
+func (target *NATSTarget) Close() (err error) {
+	close(target.quitCh)
+	if target.stanConn != nil {
+		// closing the streaming connection does not close the provided NATS connection.
+		if target.stanConn.NatsConn() != nil {
+			target.stanConn.NatsConn().Close()
+		}
+		return target.stanConn.Close()
+	}
+
+	if target.natsConn != nil {
+		target.natsConn.Close()
+	}
+
+	return nil
+}
+
+func (target *NATSTarget) init() error {
+	return target.initOnce.Do(target.initNATS)
+}
+
+func (target *NATSTarget) initNATS() error {
+	args := target.args
+
+	var err error
+	if args.Streaming.Enable {
+		target.loggerOnce(context.Background(), errors.New("NATS Streaming is deprecated please migrate to JetStream"), target.ID().String())
+		var stanConn stan.Conn
+		stanConn, err = args.connectStan()
+		target.stanConn = stanConn
+	} else {
+		var natsConn *nats.Conn
+		natsConn, err = args.connectNats()
+		target.natsConn = natsConn
+	}
+	if err != nil {
+		if err.Error() != nats.ErrNoServers.Error() {
+			target.loggerOnce(context.Background(), err, target.ID().String())
+		}
+		return err
+	}
+
+	if target.natsConn != nil && args.JetStream.Enable {
+		var jstream nats.JetStream
+		jstream, err = target.natsConn.JetStream()
+		if err != nil {
+			if err.Error() != nats.ErrNoServers.Error() {
+				target.loggerOnce(context.Background(), err, target.ID().String())
+			}
+			return err
+		}
+		target.jstream = jstream
+	}
+
+	yes, err := target.isActive()
+	if err != nil {
+		return err
+	}
+	if !yes {
+		return store.ErrNotConnected
+	}
+
+	return nil
+}
+
+// NewNATSTarget - creates new NATS target.
+func NewNATSTarget(id string, args NATSArgs, loggerOnce logger.LogOnce) (*NATSTarget, error) {
+	var queueStore store.Store[event.Event]
+	if args.QueueDir != "" {
+		queueDir := filepath.Join(args.QueueDir, storePrefix+"-nats-"+id)
+		queueStore = store.NewQueueStore[event.Event](queueDir, args.QueueLimit, event.StoreExtension)
+		if err := queueStore.Open(); err != nil {
+			return nil, fmt.Errorf("unable to initialize the queue store of NATS `%s`: %w", id, err)
+		}
+	}
+
+	target := &NATSTarget{
+		id:         event.TargetID{ID: id, Name: "nats"},
+		args:       args,
+		loggerOnce: loggerOnce,
+		store:      queueStore,
+		quitCh:     make(chan struct{}),
+	}
+
+	if target.store != nil {
+		store.StreamItems(target.store, target, target.quitCh, target.loggerOnce)
+	}
+
+	return target, nil
+}

internal/event/target/nats_contrib_test.go

@@ -0,0 +1,131 @@
+/*
+ * Hanzo S3 Object Storage (c) 2021-2023 Hanzo AI, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package target
+
+import (
+	"testing"
+
+	"github.com/nats-io/nats-server/v2/server"
+
+	xnet "github.com/minio/pkg/v3/net"
+	natsserver "github.com/nats-io/nats-server/v2/test"
+)
+
+func TestNatsConnPlain(t *testing.T) {
+	opts := natsserver.DefaultTestOptions
+	opts.Port = 14222
+	s := natsserver.RunServer(&opts)
+	defer s.Shutdown()
+
+	clientConfig := &NATSArgs{
+		Enable: true,
+		Address: xnet.Host{
+			Name:      "localhost",
+			Port:      (xnet.Port(opts.Port)),
+			IsPortSet: true,
+		},
+		Subject: "test",
+	}
+	con, err := clientConfig.connectNats()
+	if err != nil {
+		t.Errorf("Could not connect to nats: %v", err)
+	}
+	defer con.Close()
+}
+
+func TestNatsConnUserPass(t *testing.T) {
+	opts := natsserver.DefaultTestOptions
+	opts.Port = 14223
+	opts.Username = "testminio"
+	opts.Password = "miniotest"
+	s := natsserver.RunServer(&opts)
+	defer s.Shutdown()
+
+	clientConfig := &NATSArgs{
+		Enable: true,
+		Address: xnet.Host{
+			Name:      "localhost",
+			Port:      (xnet.Port(opts.Port)),
+			IsPortSet: true,
+		},
+		Subject:  "test",
+		Username: opts.Username,
+		Password: opts.Password,
+	}
+
+	con, err := clientConfig.connectNats()
+	if err != nil {
+		t.Errorf("Could not connect to nats: %v", err)
+	}
+	defer con.Close()
+}
+
+func TestNatsConnToken(t *testing.T) {
+	opts := natsserver.DefaultTestOptions
+	opts.Port = 14223
+	opts.Authorization = "s3cr3t"
+	s := natsserver.RunServer(&opts)
+	defer s.Shutdown()
+
+	clientConfig := &NATSArgs{
+		Enable: true,
+		Address: xnet.Host{
+			Name:      "localhost",
+			Port:      (xnet.Port(opts.Port)),
+			IsPortSet: true,
+		},
+		Subject: "test",
+		Token:   opts.Authorization,
+	}
+
+	con, err := clientConfig.connectNats()
+	if err != nil {
+		t.Errorf("Could not connect to nats: %v", err)
+	}
+	defer con.Close()
+}
+
+func TestNatsConnNKeySeed(t *testing.T) {
+	opts := natsserver.DefaultTestOptions
+	opts.Port = 14223
+	opts.Nkeys = []*server.NkeyUser{
+		{
+			// Not a real NKey
+			// Taken from https://docs.nats.io/running-a-nats-service/configuration/securing_nats/auth_intro/nkey_auth
+			Nkey: "UDXU4RCSJNZOIQHZNWXHXORDPRTGNJAHAHFRGZNEEJCPQTT2M7NLCNF4",
+		},
+	}
+	s := natsserver.RunServer(&opts)
+	defer s.Shutdown()
+
+	clientConfig := &NATSArgs{
+		Enable: true,
+		Address: xnet.Host{
+			Name:      "localhost",
+			Port:      (xnet.Port(opts.Port)),
+			IsPortSet: true,
+		},
+		Subject:  "test",
+		NKeySeed: "testdata/contrib/test.nkey",
+	}
+
+	con, err := clientConfig.connectNats()
+	if err != nil {
+		t.Errorf("Could not connect to nats: %v", err)
+	}
+	defer con.Close()
+}

internal/event/target/nats_tls_contrib_test.go

@@ -0,0 +1,98 @@
+/*
+ * Hanzo S3 Object Storage (c) 2021-2023 Hanzo AI, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package target
+
+import (
+	"path"
+	"path/filepath"
+	"testing"
+
+	xnet "github.com/minio/pkg/v3/net"
+	natsserver "github.com/nats-io/nats-server/v2/test"
+)
+
+func TestNatsConnTLSCustomCA(t *testing.T) {
+	s, opts := natsserver.RunServerWithConfig(filepath.Join("testdata", "contrib", "nats_tls.conf"))
+	defer s.Shutdown()
+
+	clientConfig := &NATSArgs{
+		Enable: true,
+		Address: xnet.Host{
+			Name:      "localhost",
+			Port:      (xnet.Port(opts.Port)),
+			IsPortSet: true,
+		},
+		Subject:       "test",
+		Secure:        true,
+		CertAuthority: path.Join("testdata", "contrib", "certs", "root_ca_cert.pem"),
+	}
+
+	con, err := clientConfig.connectNats()
+	if err != nil {
+		t.Errorf("Could not connect to nats: %v", err)
+	}
+	defer con.Close()
+}
+
+func TestNatsConnTLSCustomCAHandshakeFirst(t *testing.T) {
+	s, opts := natsserver.RunServerWithConfig(filepath.Join("testdata", "contrib", "nats_tls_handshake_first.conf"))
+	defer s.Shutdown()
+
+	clientConfig := &NATSArgs{
+		Enable: true,
+		Address: xnet.Host{
+			Name:      "localhost",
+			Port:      (xnet.Port(opts.Port)),
+			IsPortSet: true,
+		},
+		Subject:           "test",
+		Secure:            true,
+		CertAuthority:     path.Join("testdata", "contrib", "certs", "root_ca_cert.pem"),
+		TLSHandshakeFirst: true,
+	}
+
+	con, err := clientConfig.connectNats()
+	if err != nil {
+		t.Errorf("Could not connect to nats: %v", err)
+	}
+	defer con.Close()
+}
+
+func TestNatsConnTLSClientAuthorization(t *testing.T) {
+	s, opts := natsserver.RunServerWithConfig(filepath.Join("testdata", "contrib", "nats_tls_client_cert.conf"))
+	defer s.Shutdown()
+
+	clientConfig := &NATSArgs{
+		Enable: true,
+		Address: xnet.Host{
+			Name:      "localhost",
+			Port:      (xnet.Port(opts.Port)),
+			IsPortSet: true,
+		},
+		Subject:       "test",
+		Secure:        true,
+		CertAuthority: path.Join("testdata", "contrib", "certs", "root_ca_cert.pem"),
+		ClientCert:    path.Join("testdata", "contrib", "certs", "nats_client_cert.pem"),
+		ClientKey:     path.Join("testdata", "contrib", "certs", "nats_client_key.pem"),
+	}
+
+	con, err := clientConfig.connectNats()
+	if err != nil {
+		t.Errorf("Could not connect to nats: %v", err)
+	}
+	defer con.Close()
+}

internal/event/target/nsq.go

@@ -0,0 +1,288 @@
+// Copyright (c) 2015-2023 Hanzo AI, Inc.
+//
+// This file is part of Hanzo S3 Object Storage stack
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package target
+
+import (
+	"context"
+	"crypto/tls"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"net/url"
+	"os"
+	"path/filepath"
+
+	"github.com/nsqio/go-nsq"
+
+	"github.com/hanzoai/s3/internal/event"
+	"github.com/hanzoai/s3/internal/logger"
+	"github.com/hanzoai/s3/internal/once"
+	"github.com/hanzoai/s3/internal/store"
+	xnet "github.com/minio/pkg/v3/net"
+)
+
+// NSQ constants
+const (
+	NSQAddress       = "nsqd_address"
+	NSQTopic         = "topic"
+	NSQTLS           = "tls"
+	NSQTLSSkipVerify = "tls_skip_verify"
+	NSQQueueDir      = "queue_dir"
+	NSQQueueLimit    = "queue_limit"
+
+	EnvNSQEnable        = "S3_NOTIFY_NSQ_ENABLE"
+	EnvNSQAddress       = "S3_NOTIFY_NSQ_NSQD_ADDRESS"
+	EnvNSQTopic         = "S3_NOTIFY_NSQ_TOPIC"
+	EnvNSQTLS           = "S3_NOTIFY_NSQ_TLS"
+	EnvNSQTLSSkipVerify = "S3_NOTIFY_NSQ_TLS_SKIP_VERIFY"
+	EnvNSQQueueDir      = "S3_NOTIFY_NSQ_QUEUE_DIR"
+	EnvNSQQueueLimit    = "S3_NOTIFY_NSQ_QUEUE_LIMIT"
+)
+
+// NSQArgs - NSQ target arguments.
+type NSQArgs struct {
+	Enable      bool      `json:"enable"`
+	NSQDAddress xnet.Host `json:"nsqdAddress"`
+	Topic       string    `json:"topic"`
+	TLS         struct {
+		Enable     bool `json:"enable"`
+		SkipVerify bool `json:"skipVerify"`
+	} `json:"tls"`
+	QueueDir   string `json:"queueDir"`
+	QueueLimit uint64 `json:"queueLimit"`
+}
+
+// Validate NSQArgs fields
+func (n NSQArgs) Validate() error {
+	if !n.Enable {
+		return nil
+	}
+
+	if n.NSQDAddress.IsEmpty() {
+		return errors.New("empty nsqdAddress")
+	}
+
+	if n.Topic == "" {
+		return errors.New("empty topic")
+	}
+	if n.QueueDir != "" {
+		if !filepath.IsAbs(n.QueueDir) {
+			return errors.New("queueDir path should be absolute")
+		}
+	}
+
+	return nil
+}
+
+// NSQTarget - NSQ target.
+type NSQTarget struct {
+	initOnce once.Init
+
+	id         event.TargetID
+	args       NSQArgs
+	producer   *nsq.Producer
+	store      store.Store[event.Event]
+	config     *nsq.Config
+	loggerOnce logger.LogOnce
+	quitCh     chan struct{}
+}
+
+// ID - returns target ID.
+func (target *NSQTarget) ID() event.TargetID {
+	return target.id
+}
+
+// Name - returns the Name of the target.
+func (target *NSQTarget) Name() string {
+	return target.ID().String()
+}
+
+// Store returns any underlying store if set.
+func (target *NSQTarget) Store() event.TargetStore {
+	return target.store
+}
+
+// IsActive - Return true if target is up and active
+func (target *NSQTarget) IsActive() (bool, error) {
+	if err := target.init(); err != nil {
+		return false, err
+	}
+	return target.isActive()
+}
+
+func (target *NSQTarget) isActive() (bool, error) {
+	if target.producer == nil {
+		producer, err := nsq.NewProducer(target.args.NSQDAddress.String(), target.config)
+		if err != nil {
+			return false, err
+		}
+		target.producer = producer
+	}
+
+	if err := target.producer.Ping(); err != nil {
+		// To treat "connection refused" errors as errNotConnected.
+		if xnet.IsConnRefusedErr(err) {
+			return false, store.ErrNotConnected
+		}
+		return false, err
+	}
+	return true, nil
+}
+
+// Save - saves the events to the store which will be replayed when the nsq connection is active.
+func (target *NSQTarget) Save(eventData event.Event) error {
+	if target.store != nil {
+		_, err := target.store.Put(eventData)
+		return err
+	}
+
+	if err := target.init(); err != nil {
+		return err
+	}
+
+	_, err := target.isActive()
+	if err != nil {
+		return err
+	}
+	return target.send(eventData)
+}
+
+// send - sends an event to the NSQ.
+func (target *NSQTarget) send(eventData event.Event) error {
+	objectName, err := url.QueryUnescape(eventData.S3.Object.Key)
+	if err != nil {
+		return err
+	}
+	key := eventData.S3.Bucket.Name + "/" + objectName
+
+	data, err := json.Marshal(event.Log{EventName: eventData.EventName, Key: key, Records: []event.Event{eventData}})
+	if err != nil {
+		return err
+	}
+
+	return target.producer.Publish(target.args.Topic, data)
+}
+
+// SendFromStore - reads an event from store and sends it to NSQ.
+func (target *NSQTarget) SendFromStore(key store.Key) error {
+	if err := target.init(); err != nil {
+		return err
+	}
+
+	_, err := target.isActive()
+	if err != nil {
+		return err
+	}
+
+	eventData, eErr := target.store.Get(key)
+	if eErr != nil {
+		// The last event key in a successful batch will be sent in the channel atmost once by the replayEvents()
+		// Such events will not exist and wouldve been already been sent successfully.
+		if os.IsNotExist(eErr) {
+			return nil
+		}
+		return eErr
+	}
+
+	if err := target.send(eventData); err != nil {
+		return err
+	}
+
+	// Delete the event from store.
+	return target.store.Del(key)
+}
+
+// Close - closes underneath connections to NSQD server.
+func (target *NSQTarget) Close() (err error) {
+	close(target.quitCh)
+	if target.producer != nil {
+		// this blocks until complete:
+		target.producer.Stop()
+	}
+	return nil
+}
+
+func (target *NSQTarget) init() error {
+	return target.initOnce.Do(target.initNSQ)
+}
+
+func (target *NSQTarget) initNSQ() error {
+	args := target.args
+
+	config := nsq.NewConfig()
+	if args.TLS.Enable {
+		config.TlsV1 = true
+		config.TlsConfig = &tls.Config{
+			InsecureSkipVerify: args.TLS.SkipVerify,
+		}
+	}
+	target.config = config
+
+	producer, err := nsq.NewProducer(args.NSQDAddress.String(), config)
+	if err != nil {
+		target.loggerOnce(context.Background(), err, target.ID().String())
+		return err
+	}
+	target.producer = producer
+
+	err = target.producer.Ping()
+	if err != nil {
+		// To treat "connection refused" errors as errNotConnected.
+		if !xnet.IsConnRefusedErr(err) && !xnet.IsConnResetErr(err) {
+			target.loggerOnce(context.Background(), err, target.ID().String())
+		}
+		target.producer.Stop()
+		return err
+	}
+
+	yes, err := target.isActive()
+	if err != nil {
+		return err
+	}
+	if !yes {
+		return store.ErrNotConnected
+	}
+
+	return nil
+}
+
+// NewNSQTarget - creates new NSQ target.
+func NewNSQTarget(id string, args NSQArgs, loggerOnce logger.LogOnce) (*NSQTarget, error) {
+	var queueStore store.Store[event.Event]
+	if args.QueueDir != "" {
+		queueDir := filepath.Join(args.QueueDir, storePrefix+"-nsq-"+id)
+		queueStore = store.NewQueueStore[event.Event](queueDir, args.QueueLimit, event.StoreExtension)
+		if err := queueStore.Open(); err != nil {
+			return nil, fmt.Errorf("unable to initialize the queue store of NSQ `%s`: %w", id, err)
+		}
+	}
+
+	target := &NSQTarget{
+		id:         event.TargetID{ID: id, Name: "nsq"},
+		args:       args,
+		loggerOnce: loggerOnce,
+		store:      queueStore,
+		quitCh:     make(chan struct{}),
+	}
+
+	if target.store != nil {
+		store.StreamItems(target.store, target, target.quitCh, target.loggerOnce)
+	}
+
+	return target, nil
+}

internal/event/target/nsq_test.go

@@ -0,0 +1,98 @@
+// Copyright (c) 2015-2023 Hanzo AI, Inc.
+//
+// This file is part of Hanzo S3 Object Storage stack
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package target
+
+import (
+	"testing"
+
+	xnet "github.com/minio/pkg/v3/net"
+)
+
+func TestNSQArgs_Validate(t *testing.T) {
+	type fields struct {
+		Enable      bool
+		NSQDAddress xnet.Host
+		Topic       string
+		TLS         struct {
+			Enable     bool
+			SkipVerify bool
+		}
+	}
+	tests := []struct {
+		name    string
+		fields  fields
+		wantErr bool
+	}{
+		{
+			name: "test1_missing_topic",
+			fields: fields{
+				Enable: true,
+				NSQDAddress: xnet.Host{
+					Name:      "127.0.0.1",
+					Port:      4150,
+					IsPortSet: true,
+				},
+				Topic: "",
+			},
+			wantErr: true,
+		},
+		{
+			name: "test2_disabled",
+			fields: fields{
+				Enable:      false,
+				NSQDAddress: xnet.Host{},
+				Topic:       "topic",
+			},
+			wantErr: false,
+		},
+		{
+			name: "test3_OK",
+			fields: fields{
+				Enable: true,
+				NSQDAddress: xnet.Host{
+					Name:      "127.0.0.1",
+					Port:      4150,
+					IsPortSet: true,
+				},
+				Topic: "topic",
+			},
+			wantErr: false,
+		},
+		{
+			name: "test4_emptynsqdaddr",
+			fields: fields{
+				Enable:      true,
+				NSQDAddress: xnet.Host{},
+				Topic:       "topic",
+			},
+			wantErr: true,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			n := NSQArgs{
+				Enable:      tt.fields.Enable,
+				NSQDAddress: tt.fields.NSQDAddress,
+				Topic:       tt.fields.Topic,
+			}
+			if err := n.Validate(); (err != nil) != tt.wantErr {
+				t.Errorf("NSQArgs.Validate() error = %v, wantErr %v", err, tt.wantErr)
+			}
+		})
+	}
+}

internal/event/target/postgresql.go

@@ -0,0 +1,493 @@
+// Copyright (c) 2015-2023 Hanzo AI, Inc.
+//
+// This file is part of Hanzo S3 Object Storage stack
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package target
+
+import (
+	"context"
+	"database/sql"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"net/url"
+	"os"
+	"path/filepath"
+	"regexp"
+	"strconv"
+	"strings"
+	"time"
+	"unicode"
+
+	_ "github.com/lib/pq" // Register postgres driver
+
+	"github.com/hanzoai/s3/internal/event"
+	"github.com/hanzoai/s3/internal/logger"
+	"github.com/hanzoai/s3/internal/once"
+	"github.com/hanzoai/s3/internal/store"
+	xnet "github.com/minio/pkg/v3/net"
+)
+
+const (
+	psqlTableExists          = `SELECT 1 FROM %s;`
+	psqlCreateNamespaceTable = `CREATE TABLE %s (key VARCHAR PRIMARY KEY, value JSONB);`
+	psqlCreateAccessTable    = `CREATE TABLE %s (event_time TIMESTAMP WITH TIME ZONE NOT NULL, event_data JSONB);`
+
+	psqlUpdateRow = `INSERT INTO %s (key, value) VALUES ($1, $2) ON CONFLICT (key) DO UPDATE SET value = EXCLUDED.value;`
+	psqlDeleteRow = `DELETE FROM %s WHERE key = $1;`
+	psqlInsertRow = `INSERT INTO %s (event_time, event_data) VALUES ($1, $2);`
+)
+
+// Postgres constants
+const (
+	PostgresFormat             = "format"
+	PostgresConnectionString   = "connection_string"
+	PostgresTable              = "table"
+	PostgresHost               = "host"
+	PostgresPort               = "port"
+	PostgresUsername           = "username"
+	PostgresPassword           = "password"
+	PostgresDatabase           = "database"
+	PostgresQueueDir           = "queue_dir"
+	PostgresQueueLimit         = "queue_limit"
+	PostgresMaxOpenConnections = "max_open_connections"
+
+	EnvPostgresEnable             = "S3_NOTIFY_POSTGRES_ENABLE"
+	EnvPostgresFormat             = "S3_NOTIFY_POSTGRES_FORMAT"
+	EnvPostgresConnectionString   = "S3_NOTIFY_POSTGRES_CONNECTION_STRING"
+	EnvPostgresTable              = "S3_NOTIFY_POSTGRES_TABLE"
+	EnvPostgresHost               = "S3_NOTIFY_POSTGRES_HOST"
+	EnvPostgresPort               = "S3_NOTIFY_POSTGRES_PORT"
+	EnvPostgresUsername           = "S3_NOTIFY_POSTGRES_USERNAME"
+	EnvPostgresPassword           = "S3_NOTIFY_POSTGRES_PASSWORD"
+	EnvPostgresDatabase           = "S3_NOTIFY_POSTGRES_DATABASE"
+	EnvPostgresQueueDir           = "S3_NOTIFY_POSTGRES_QUEUE_DIR"
+	EnvPostgresQueueLimit         = "S3_NOTIFY_POSTGRES_QUEUE_LIMIT"
+	EnvPostgresMaxOpenConnections = "S3_NOTIFY_POSTGRES_MAX_OPEN_CONNECTIONS"
+)
+
+// PostgreSQLArgs - PostgreSQL target arguments.
+type PostgreSQLArgs struct {
+	Enable             bool      `json:"enable"`
+	Format             string    `json:"format"`
+	ConnectionString   string    `json:"connectionString"`
+	Table              string    `json:"table"`
+	Host               xnet.Host `json:"host"`     // default: localhost
+	Port               string    `json:"port"`     // default: 5432
+	Username           string    `json:"username"` // default: user running minio
+	Password           string    `json:"password"` // default: no password
+	Database           string    `json:"database"` // default: same as user
+	QueueDir           string    `json:"queueDir"`
+	QueueLimit         uint64    `json:"queueLimit"`
+	MaxOpenConnections int       `json:"maxOpenConnections"`
+}
+
+// Validate PostgreSQLArgs fields
+func (p PostgreSQLArgs) Validate() error {
+	if !p.Enable {
+		return nil
+	}
+	if p.Table == "" {
+		return fmt.Errorf("empty table name")
+	}
+	if err := validatePsqlTableName(p.Table); err != nil {
+		return err
+	}
+
+	if p.Format != "" {
+		f := strings.ToLower(p.Format)
+		if f != event.NamespaceFormat && f != event.AccessFormat {
+			return fmt.Errorf("unrecognized format value")
+		}
+	}
+
+	if p.ConnectionString != "" {
+		// No pq API doesn't help to validate connection string
+		// prior connection, so no validation for now.
+	} else {
+		// Some fields need to be specified when ConnectionString is unspecified
+		if p.Port == "" {
+			return fmt.Errorf("unspecified port")
+		}
+		if _, err := strconv.Atoi(p.Port); err != nil {
+			return fmt.Errorf("invalid port")
+		}
+		if p.Database == "" {
+			return fmt.Errorf("database unspecified")
+		}
+	}
+
+	if p.QueueDir != "" {
+		if !filepath.IsAbs(p.QueueDir) {
+			return errors.New("queueDir path should be absolute")
+		}
+	}
+
+	if p.MaxOpenConnections < 0 {
+		return errors.New("maxOpenConnections cannot be less than zero")
+	}
+
+	return nil
+}
+
+// PostgreSQLTarget - PostgreSQL target.
+type PostgreSQLTarget struct {
+	initOnce once.Init
+
+	id         event.TargetID
+	args       PostgreSQLArgs
+	updateStmt *sql.Stmt
+	deleteStmt *sql.Stmt
+	insertStmt *sql.Stmt
+	db         *sql.DB
+	store      store.Store[event.Event]
+	firstPing  bool
+	connString string
+	loggerOnce logger.LogOnce
+	quitCh     chan struct{}
+}
+
+// ID - returns target ID.
+func (target *PostgreSQLTarget) ID() event.TargetID {
+	return target.id
+}
+
+// Name - returns the Name of the target.
+func (target *PostgreSQLTarget) Name() string {
+	return target.ID().String()
+}
+
+// Store returns any underlying store if set.
+func (target *PostgreSQLTarget) Store() event.TargetStore {
+	return target.store
+}
+
+// IsActive - Return true if target is up and active
+func (target *PostgreSQLTarget) IsActive() (bool, error) {
+	if err := target.init(); err != nil {
+		return false, err
+	}
+	return target.isActive()
+}
+
+func (target *PostgreSQLTarget) isActive() (bool, error) {
+	if err := target.db.Ping(); err != nil {
+		if IsConnErr(err) {
+			return false, store.ErrNotConnected
+		}
+		return false, err
+	}
+	return true, nil
+}
+
+// Save - saves the events to the store if questore is configured, which will be replayed when the PostgreSQL connection is active.
+func (target *PostgreSQLTarget) Save(eventData event.Event) error {
+	if target.store != nil {
+		_, err := target.store.Put(eventData)
+		return err
+	}
+
+	if err := target.init(); err != nil {
+		return err
+	}
+
+	_, err := target.isActive()
+	if err != nil {
+		return err
+	}
+	return target.send(eventData)
+}
+
+// IsConnErr - To detect a connection error.
+func IsConnErr(err error) bool {
+	return xnet.IsConnRefusedErr(err) || err.Error() == "sql: database is closed" || err.Error() == "sql: statement is closed" || err.Error() == "invalid connection"
+}
+
+// send - sends an event to the PostgreSQL.
+func (target *PostgreSQLTarget) send(eventData event.Event) error {
+	if target.args.Format == event.NamespaceFormat {
+		objectName, err := url.QueryUnescape(eventData.S3.Object.Key)
+		if err != nil {
+			return err
+		}
+		key := eventData.S3.Bucket.Name + "/" + objectName
+
+		if eventData.EventName == event.ObjectRemovedDelete {
+			_, err = target.deleteStmt.Exec(key)
+		} else {
+			var data []byte
+			if data, err = json.Marshal(struct{ Records []event.Event }{[]event.Event{eventData}}); err != nil {
+				return err
+			}
+
+			_, err = target.updateStmt.Exec(key, data)
+		}
+		return err
+	}
+
+	if target.args.Format == event.AccessFormat {
+		eventTime, err := time.Parse(event.AMZTimeFormat, eventData.EventTime)
+		if err != nil {
+			return err
+		}
+
+		data, err := json.Marshal(struct{ Records []event.Event }{[]event.Event{eventData}})
+		if err != nil {
+			return err
+		}
+
+		if _, err = target.insertStmt.Exec(eventTime, data); err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
+// SendFromStore - reads an event from store and sends it to PostgreSQL.
+func (target *PostgreSQLTarget) SendFromStore(key store.Key) error {
+	if err := target.init(); err != nil {
+		return err
+	}
+
+	_, err := target.isActive()
+	if err != nil {
+		return err
+	}
+	if !target.firstPing {
+		if err := target.executeStmts(); err != nil {
+			if IsConnErr(err) {
+				return store.ErrNotConnected
+			}
+			return err
+		}
+	}
+
+	eventData, eErr := target.store.Get(key)
+	if eErr != nil {
+		// The last event key in a successful batch will be sent in the channel atmost once by the replayEvents()
+		// Such events will not exist and wouldve been already been sent successfully.
+		if os.IsNotExist(eErr) {
+			return nil
+		}
+		return eErr
+	}
+
+	if err := target.send(eventData); err != nil {
+		if IsConnErr(err) {
+			return store.ErrNotConnected
+		}
+		return err
+	}
+
+	// Delete the event from store.
+	return target.store.Del(key)
+}
+
+// Close - closes underneath connections to PostgreSQL database.
+func (target *PostgreSQLTarget) Close() error {
+	close(target.quitCh)
+	if target.updateStmt != nil {
+		// FIXME: log returned error. ignore time being.
+		_ = target.updateStmt.Close()
+	}
+
+	if target.deleteStmt != nil {
+		// FIXME: log returned error. ignore time being.
+		_ = target.deleteStmt.Close()
+	}
+
+	if target.insertStmt != nil {
+		// FIXME: log returned error. ignore time being.
+		_ = target.insertStmt.Close()
+	}
+
+	if target.db != nil {
+		target.db.Close()
+	}
+
+	return nil
+}
+
+// Executes the table creation statements.
+func (target *PostgreSQLTarget) executeStmts() error {
+	_, err := target.db.Exec(fmt.Sprintf(psqlTableExists, target.args.Table))
+	if err != nil {
+		createStmt := psqlCreateNamespaceTable
+		if target.args.Format == event.AccessFormat {
+			createStmt = psqlCreateAccessTable
+		}
+
+		if _, dbErr := target.db.Exec(fmt.Sprintf(createStmt, target.args.Table)); dbErr != nil {
+			return dbErr
+		}
+	}
+
+	switch target.args.Format {
+	case event.NamespaceFormat:
+		// insert or update statement
+		if target.updateStmt, err = target.db.Prepare(fmt.Sprintf(psqlUpdateRow, target.args.Table)); err != nil {
+			return err
+		}
+		// delete statement
+		if target.deleteStmt, err = target.db.Prepare(fmt.Sprintf(psqlDeleteRow, target.args.Table)); err != nil {
+			return err
+		}
+	case event.AccessFormat:
+		// insert statement
+		if target.insertStmt, err = target.db.Prepare(fmt.Sprintf(psqlInsertRow, target.args.Table)); err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
+func (target *PostgreSQLTarget) init() error {
+	return target.initOnce.Do(target.initPostgreSQL)
+}
+
+func (target *PostgreSQLTarget) initPostgreSQL() error {
+	args := target.args
+
+	db, err := sql.Open("postgres", target.connString)
+	if err != nil {
+		return err
+	}
+	target.db = db
+
+	if args.MaxOpenConnections > 0 {
+		// Set the maximum connections limit
+		target.db.SetMaxOpenConns(args.MaxOpenConnections)
+	}
+
+	err = target.db.Ping()
+	if err != nil {
+		if !xnet.IsConnRefusedErr(err) && !xnet.IsConnResetErr(err) {
+			target.loggerOnce(context.Background(), err, target.ID().String())
+		}
+	} else {
+		if err = target.executeStmts(); err != nil {
+			target.loggerOnce(context.Background(), err, target.ID().String())
+		} else {
+			target.firstPing = true
+		}
+	}
+
+	if err != nil {
+		target.db.Close()
+		return err
+	}
+
+	yes, err := target.isActive()
+	if err != nil {
+		return err
+	}
+	if !yes {
+		return store.ErrNotConnected
+	}
+
+	return nil
+}
+
+// NewPostgreSQLTarget - creates new PostgreSQL target.
+func NewPostgreSQLTarget(id string, args PostgreSQLArgs, loggerOnce logger.LogOnce) (*PostgreSQLTarget, error) {
+	params := []string{args.ConnectionString}
+	if args.ConnectionString == "" {
+		params = []string{}
+		if !args.Host.IsEmpty() {
+			params = append(params, "host="+args.Host.String())
+		}
+		if args.Port != "" {
+			params = append(params, "port="+args.Port)
+		}
+		if args.Username != "" {
+			params = append(params, "username="+args.Username)
+		}
+		if args.Password != "" {
+			params = append(params, "password="+args.Password)
+		}
+		if args.Database != "" {
+			params = append(params, "dbname="+args.Database)
+		}
+	}
+	connStr := strings.Join(params, " ")
+
+	var queueStore store.Store[event.Event]
+	if args.QueueDir != "" {
+		queueDir := filepath.Join(args.QueueDir, storePrefix+"-postgresql-"+id)
+		queueStore = store.NewQueueStore[event.Event](queueDir, args.QueueLimit, event.StoreExtension)
+		if err := queueStore.Open(); err != nil {
+			return nil, fmt.Errorf("unable to initialize the queue store of PostgreSQL `%s`: %w", id, err)
+		}
+	}
+
+	target := &PostgreSQLTarget{
+		id:         event.TargetID{ID: id, Name: "postgresql"},
+		args:       args,
+		firstPing:  false,
+		store:      queueStore,
+		connString: connStr,
+		loggerOnce: loggerOnce,
+		quitCh:     make(chan struct{}),
+	}
+
+	if target.store != nil {
+		store.StreamItems(target.store, target, target.quitCh, target.loggerOnce)
+	}
+
+	return target, nil
+}
+
+var errInvalidPsqlTablename = errors.New("invalid PostgreSQL table")
+
+func validatePsqlTableName(name string) error {
+	// check for quoted string (string may not contain a quote)
+	if match, err := regexp.MatchString("^\"[^\"]+\"$", name); err != nil {
+		return err
+	} else if match {
+		return nil
+	}
+
+	// normalize the name to letters, digits, _ or $
+	valid := true
+	cleaned := strings.Map(func(r rune) rune {
+		switch {
+		case unicode.IsLetter(r):
+			return 'a'
+		case unicode.IsDigit(r):
+			return '0'
+		case r == '_', r == '$':
+			return r
+		default:
+			valid = false
+			return -1
+		}
+	}, name)
+
+	if valid {
+		// check for simple name or quoted name
+		// - letter/underscore followed by one or more letter/digit/underscore
+		// - any text between quotes (text cannot contain a quote itself)
+		if match, err := regexp.MatchString("^[a_][a0_$]*$", cleaned); err != nil {
+			return err
+		} else if match {
+			return nil
+		}
+	}
+
+	return errInvalidPsqlTablename
+}

internal/event/target/postgresql_test.go

@@ -0,0 +1,52 @@
+// Copyright (c) 2015-2023 Hanzo AI, Inc.
+//
+// This file is part of Hanzo S3 Object Storage stack
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package target
+
+import (
+	"database/sql"
+	"slices"
+	"testing"
+)
+
+// TestPostgreSQLRegistration checks if postgres driver
+// is registered and fails otherwise.
+func TestPostgreSQLRegistration(t *testing.T) {
+	var found bool
+	if slices.Contains(sql.Drivers(), "postgres") {
+		found = true
+	}
+	if !found {
+		t.Fatal("postgres driver not registered")
+	}
+}
+
+func TestPsqlTableNameValidation(t *testing.T) {
+	validTables := []string{"táblë", "table", "TableName", "\"Table name\"", "\"✅✅\"", "table$one", "\"táblë\""}
+	invalidTables := []string{"table name", "table \"name\"", "✅✅", "$table$"}
+
+	for _, name := range validTables {
+		if err := validatePsqlTableName(name); err != nil {
+			t.Errorf("Should be valid: %s - %s", name, err)
+		}
+	}
+	for _, name := range invalidTables {
+		if err := validatePsqlTableName(name); err != errInvalidPsqlTablename {
+			t.Errorf("Should be invalid: %s - %s", name, err)
+		}
+	}
+}

internal/event/target/redis.go

@@ -0,0 +1,382 @@
+// Copyright (c) 2015-2023 Hanzo AI, Inc.
+//
+// This file is part of Hanzo S3 Object Storage stack
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package target
+
+import (
+	"context"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"net/url"
+	"os"
+	"path/filepath"
+	"strings"
+	"time"
+
+	"github.com/gomodule/redigo/redis"
+	"github.com/hanzoai/s3/internal/event"
+	"github.com/hanzoai/s3/internal/logger"
+	"github.com/hanzoai/s3/internal/once"
+	"github.com/hanzoai/s3/internal/store"
+	xnet "github.com/minio/pkg/v3/net"
+)
+
+// Redis constants
+const (
+	RedisFormat     = "format"
+	RedisAddress    = "address"
+	RedisPassword   = "password"
+	RedisUser       = "user"
+	RedisKey        = "key"
+	RedisQueueDir   = "queue_dir"
+	RedisQueueLimit = "queue_limit"
+
+	EnvRedisEnable     = "S3_NOTIFY_REDIS_ENABLE"
+	EnvRedisFormat     = "S3_NOTIFY_REDIS_FORMAT"
+	EnvRedisAddress    = "S3_NOTIFY_REDIS_ADDRESS"
+	EnvRedisPassword   = "S3_NOTIFY_REDIS_PASSWORD"
+	EnvRedisUser       = "S3_NOTIFY_REDIS_USER"
+	EnvRedisKey        = "S3_NOTIFY_REDIS_KEY"
+	EnvRedisQueueDir   = "S3_NOTIFY_REDIS_QUEUE_DIR"
+	EnvRedisQueueLimit = "S3_NOTIFY_REDIS_QUEUE_LIMIT"
+)
+
+// RedisArgs - Redis target arguments.
+type RedisArgs struct {
+	Enable     bool      `json:"enable"`
+	Format     string    `json:"format"`
+	Addr       xnet.Host `json:"address"`
+	Password   string    `json:"password"`
+	User       string    `json:"user"`
+	Key        string    `json:"key"`
+	QueueDir   string    `json:"queueDir"`
+	QueueLimit uint64    `json:"queueLimit"`
+}
+
+// RedisAccessEvent holds event log data and timestamp
+type RedisAccessEvent struct {
+	Event     []event.Event
+	EventTime string
+}
+
+// Validate RedisArgs fields
+func (r RedisArgs) Validate() error {
+	if !r.Enable {
+		return nil
+	}
+
+	if r.Format != "" {
+		f := strings.ToLower(r.Format)
+		if f != event.NamespaceFormat && f != event.AccessFormat {
+			return fmt.Errorf("unrecognized format")
+		}
+	}
+
+	if r.Key == "" {
+		return fmt.Errorf("empty key")
+	}
+
+	if r.QueueDir != "" {
+		if !filepath.IsAbs(r.QueueDir) {
+			return errors.New("queueDir path should be absolute")
+		}
+	}
+
+	return nil
+}
+
+func (r RedisArgs) validateFormat(c redis.Conn) error {
+	typeAvailable, err := redis.String(c.Do("TYPE", r.Key))
+	if err != nil {
+		return err
+	}
+
+	if typeAvailable != "none" {
+		expectedType := "hash"
+		if r.Format == event.AccessFormat {
+			expectedType = "list"
+		}
+
+		if typeAvailable != expectedType {
+			return fmt.Errorf("expected type %v does not match with available type %v", expectedType, typeAvailable)
+		}
+	}
+
+	return nil
+}
+
+// RedisTarget - Redis target.
+type RedisTarget struct {
+	initOnce once.Init
+
+	id         event.TargetID
+	args       RedisArgs
+	pool       *redis.Pool
+	store      store.Store[event.Event]
+	firstPing  bool
+	loggerOnce logger.LogOnce
+	quitCh     chan struct{}
+}
+
+// ID - returns target ID.
+func (target *RedisTarget) ID() event.TargetID {
+	return target.id
+}
+
+// Name - returns the Name of the target.
+func (target *RedisTarget) Name() string {
+	return target.ID().String()
+}
+
+// Store returns any underlying store if set.
+func (target *RedisTarget) Store() event.TargetStore {
+	return target.store
+}
+
+// IsActive - Return true if target is up and active
+func (target *RedisTarget) IsActive() (bool, error) {
+	if err := target.init(); err != nil {
+		return false, err
+	}
+	return target.isActive()
+}
+
+func (target *RedisTarget) isActive() (bool, error) {
+	conn := target.pool.Get()
+	defer conn.Close()
+
+	_, pingErr := conn.Do("PING")
+	if pingErr != nil {
+		if xnet.IsConnRefusedErr(pingErr) {
+			return false, store.ErrNotConnected
+		}
+		return false, pingErr
+	}
+	return true, nil
+}
+
+// Save - saves the events to the store if questore is configured, which will be replayed when the redis connection is active.
+func (target *RedisTarget) Save(eventData event.Event) error {
+	if target.store != nil {
+		_, err := target.store.Put(eventData)
+		return err
+	}
+	if err := target.init(); err != nil {
+		return err
+	}
+	_, err := target.isActive()
+	if err != nil {
+		return err
+	}
+	return target.send(eventData)
+}
+
+// send - sends an event to the redis.
+func (target *RedisTarget) send(eventData event.Event) error {
+	conn := target.pool.Get()
+	defer conn.Close()
+
+	if target.args.Format == event.NamespaceFormat {
+		objectName, err := url.QueryUnescape(eventData.S3.Object.Key)
+		if err != nil {
+			return err
+		}
+		key := eventData.S3.Bucket.Name + "/" + objectName
+
+		if eventData.EventName == event.ObjectRemovedDelete {
+			_, err = conn.Do("HDEL", target.args.Key, key)
+		} else {
+			var data []byte
+			if data, err = json.Marshal(struct{ Records []event.Event }{[]event.Event{eventData}}); err != nil {
+				return err
+			}
+
+			_, err = conn.Do("HSET", target.args.Key, key, data)
+		}
+		if err != nil {
+			return err
+		}
+	}
+
+	if target.args.Format == event.AccessFormat {
+		data, err := json.Marshal([]RedisAccessEvent{{Event: []event.Event{eventData}, EventTime: eventData.EventTime}})
+		if err != nil {
+			return err
+		}
+		if _, err := conn.Do("RPUSH", target.args.Key, data); err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
+// SendFromStore - reads an event from store and sends it to redis.
+func (target *RedisTarget) SendFromStore(key store.Key) error {
+	if err := target.init(); err != nil {
+		return err
+	}
+
+	conn := target.pool.Get()
+	defer conn.Close()
+
+	_, pingErr := conn.Do("PING")
+	if pingErr != nil {
+		if xnet.IsConnRefusedErr(pingErr) {
+			return store.ErrNotConnected
+		}
+		return pingErr
+	}
+
+	if !target.firstPing {
+		if err := target.args.validateFormat(conn); err != nil {
+			if xnet.IsConnRefusedErr(err) {
+				return store.ErrNotConnected
+			}
+			return err
+		}
+		target.firstPing = true
+	}
+
+	eventData, eErr := target.store.Get(key)
+	if eErr != nil {
+		// The last event key in a successful batch will be sent in the channel atmost once by the replayEvents()
+		// Such events will not exist and would've been already been sent successfully.
+		if os.IsNotExist(eErr) {
+			return nil
+		}
+		return eErr
+	}
+
+	if err := target.send(eventData); err != nil {
+		if xnet.IsConnRefusedErr(err) {
+			return store.ErrNotConnected
+		}
+		return err
+	}
+
+	// Delete the event from store.
+	return target.store.Del(key)
+}
+
+// Close - releases the resources used by the pool.
+func (target *RedisTarget) Close() error {
+	close(target.quitCh)
+	if target.pool != nil {
+		return target.pool.Close()
+	}
+	return nil
+}
+
+func (target *RedisTarget) init() error {
+	return target.initOnce.Do(target.initRedis)
+}
+
+func (target *RedisTarget) initRedis() error {
+	conn := target.pool.Get()
+	defer conn.Close()
+
+	_, pingErr := conn.Do("PING")
+	if pingErr != nil {
+		if !xnet.IsConnRefusedErr(pingErr) && !xnet.IsConnResetErr(pingErr) {
+			target.loggerOnce(context.Background(), pingErr, target.ID().String())
+		}
+		return pingErr
+	}
+
+	if err := target.args.validateFormat(conn); err != nil {
+		target.loggerOnce(context.Background(), err, target.ID().String())
+		return err
+	}
+
+	target.firstPing = true
+
+	yes, err := target.isActive()
+	if err != nil {
+		return err
+	}
+	if !yes {
+		return store.ErrNotConnected
+	}
+
+	return nil
+}
+
+// NewRedisTarget - creates new Redis target.
+func NewRedisTarget(id string, args RedisArgs, loggerOnce logger.LogOnce) (*RedisTarget, error) {
+	var queueStore store.Store[event.Event]
+	if args.QueueDir != "" {
+		queueDir := filepath.Join(args.QueueDir, storePrefix+"-redis-"+id)
+		queueStore = store.NewQueueStore[event.Event](queueDir, args.QueueLimit, event.StoreExtension)
+		if err := queueStore.Open(); err != nil {
+			return nil, fmt.Errorf("unable to initialize the queue store of Redis `%s`: %w", id, err)
+		}
+	}
+
+	pool := &redis.Pool{
+		MaxIdle:     3,
+		IdleTimeout: 2 * 60 * time.Second,
+		Dial: func() (redis.Conn, error) {
+			conn, err := redis.Dial("tcp", args.Addr.String())
+			if err != nil {
+				return nil, err
+			}
+
+			if args.Password != "" {
+				if args.User != "" {
+					if _, err = conn.Do("AUTH", args.User, args.Password); err != nil {
+						conn.Close()
+						return nil, err
+					}
+				} else {
+					if _, err = conn.Do("AUTH", args.Password); err != nil {
+						conn.Close()
+						return nil, err
+					}
+				}
+			}
+
+			// Must be done after AUTH
+			if _, err = conn.Do("CLIENT", "SETNAME", "Hanzo S3"); err != nil {
+				conn.Close()
+				return nil, err
+			}
+
+			return conn, nil
+		},
+		TestOnBorrow: func(c redis.Conn, t time.Time) error {
+			_, err := c.Do("PING")
+			return err
+		},
+	}
+
+	target := &RedisTarget{
+		id:         event.TargetID{ID: id, Name: "redis"},
+		args:       args,
+		pool:       pool,
+		store:      queueStore,
+		loggerOnce: loggerOnce,
+		quitCh:     make(chan struct{}),
+	}
+
+	if target.store != nil {
+		store.StreamItems(target.store, target, target.quitCh, target.loggerOnce)
+	}
+
+	return target, nil
+}

internal/event/target/testdata/contrib/certs/nats_client_cert.pem

@@ -0,0 +1,13 @@
+-----BEGIN CERTIFICATE-----
+MIICCjCCAbGgAwIBAgIUKLFyLD0Ze9gR3A2aBxgEiT6MgZUwCgYIKoZIzj0EAwIw
+GDEWMBQGA1UEAwwNTWluaW8gUm9vdCBDQTAeFw0yMDA5MTQxMzI0MzNaFw0zMDA5
+MTIxMzI0MzNaMEIxCzAJBgNVBAYTAkNBMQ4wDAYDVQQKDAVNaW5JTzEPMA0GA1UE
+CwwGQ2xpZW50MRIwEAYDVQQDDAlsb2NhbGhvc3QwWTATBgcqhkjOPQIBBggqhkjO
+PQMBBwNCAARAhYrQXYbzeKyVSw8nf57gBphwFP1o5S7CjxoGKCfghzdhExKiEmbi
+sK+FSS2YtltU7cM7L7AduLIbuEnGHHYQo4GuMIGrMAkGA1UdEwQCMAAwUwYDVR0j
+BEwwSoAUWN6Fr30E5vvvNOBkuGGkqGzA3SihHKQaMBgxFjAUBgNVBAMMDU1pbmlv
+IFJvb3QgQ0GCFHiTsAON45VvwFb0MxHEdLPeWi95MA4GA1UdDwEB/wQEAwIFoDAd
+BgNVHSUEFjAUBggrBgEFBQcDAQYIKwYBBQUHAwIwGgYDVR0RBBMwEYcEfwAAAYIJ
+bG9jYWxob3N0MAoGCCqGSM49BAMCA0cAMEQCIC7MHOEf0C/zqw/ZOaCffeJIMeFm
+iT8ugBfhFbgGkd5YAiBz9FEfV4JMZQ4N29WLmvxxDSxkL8g5e3fnIK8Aa4excw==
+-----END CERTIFICATE-----

internal/event/target/testdata/contrib/certs/nats_client_key.pem

@@ -0,0 +1,5 @@
+-----BEGIN EC PRIVATE KEY-----
+MHcCAQEEIBluB2BuspJcz1e58rnXpQEx48/ZwNmygNw06NbdTZDroAoGCCqGSM49
+AwEHoUQDQgAEQIWK0F2G83islUsPJ3+e4AaYcBT9aOUuwo8aBign4Ic3YRMSohJm
+4rCvhUktmLZbVO3DOy+wHbiyG7hJxhx2EA==
+-----END EC PRIVATE KEY-----

internal/event/target/testdata/contrib/certs/nats_server_cert.pem

@@ -0,0 +1,12 @@
+-----BEGIN CERTIFICATE-----
+MIIByTCCAW+gAwIBAgIUdAg80BTm1El7s5ZZezgjsls9BwkwCgYIKoZIzj0EAwIw
+GDEWMBQGA1UEAwwNTWluaW8gUm9vdCBDQTAeFw0yMDA5MTQxMjQzMjNaFw0zMDA5
+MTIxMjQzMjNaMAAwWTATBgcqhkjOPQIBBggqhkjOPQMBBwNCAASolKUI7FVSA2Ts
++GSW/DHDKNczDNjfccI2GLETso6ie8buveOODj1JIL9ff5pRDN+U6QvwwlDmXEqh
+1a6XBI4Ho4GuMIGrMAkGA1UdEwQCMAAwUwYDVR0jBEwwSoAUWN6Fr30E5vvvNOBk
+uGGkqGzA3SihHKQaMBgxFjAUBgNVBAMMDU1pbmlvIFJvb3QgQ0GCFHiTsAON45Vv
+wFb0MxHEdLPeWi95MA4GA1UdDwEB/wQEAwIFoDAdBgNVHSUEFjAUBggrBgEFBQcD
+AQYIKwYBBQUHAwIwGgYDVR0RBBMwEYcEfwAAAYIJbG9jYWxob3N0MAoGCCqGSM49
+BAMCA0gAMEUCIB7WXnQAkmjw2QE6A3uOscOIctJnlVNREfm4V9CrF6UGAiEA734B
+vKlhMk8H459BRoIp8GpOuUWqLqocSmMM1febvcg=
+-----END CERTIFICATE-----

internal/event/target/testdata/contrib/certs/nats_server_key.pem

@@ -0,0 +1,5 @@
+-----BEGIN EC PRIVATE KEY-----
+MHcCAQEEILFuMS2xvsc/CsuqtSv3S2iSCcc28rZsg1wpR2kirXFloAoGCCqGSM49
+AwEHoUQDQgAEqJSlCOxVUgNk7PhklvwxwyjXMwzY33HCNhixE7KOonvG7r3jjg49
+SSC/X3+aUQzflOkL8MJQ5lxKodWulwSOBw==
+-----END EC PRIVATE KEY-----

internal/event/target/testdata/contrib/certs/root_ca_cert.pem

@@ -0,0 +1,11 @@
+-----BEGIN CERTIFICATE-----
+MIIBlTCCATygAwIBAgIUeJOwA43jlW/AVvQzEcR0s95aL3kwCgYIKoZIzj0EAwIw
+GDEWMBQGA1UEAwwNTWluaW8gUm9vdCBDQTAeFw0yMDA5MTQxMjMwMDJaFw0zMDA5
+MTIxMjMwMDJaMBgxFjAUBgNVBAMMDU1pbmlvIFJvb3QgQ0EwWTATBgcqhkjOPQIB
+BggqhkjOPQMBBwNCAARK9fVNGHc1h5B5fpOMyEdyhh18xNNcNUGQ5iGLO97Z0KtK
+5vRlDeeE1I0SaJgqppm9OEHw32JU0HMi4FBZi2Rso2QwYjAdBgNVHQ4EFgQUWN6F
+r30E5vvvNOBkuGGkqGzA3SgwHwYDVR0jBBgwFoAUWN6Fr30E5vvvNOBkuGGkqGzA
+3SgwDwYDVR0TAQH/BAUwAwEB/zAPBgNVHREECDAGhwR/AAABMAoGCCqGSM49BAMC
+A0cAMEQCIDPOiks2Vs3RmuJZl5HHjuqaFSOAp1g7pZpMb3Qrh9YDAiAtjO2xOpkS
+WynK8P7EfyQP/IUa7GxJIoHk6/H/TCsYvQ==
+-----END CERTIFICATE-----

internal/event/target/testdata/contrib/certs/root_ca_key.pem

@@ -0,0 +1,5 @@
+-----BEGIN EC PRIVATE KEY-----
+MHcCAQEEIB8tAGuc9FP4XbYqMP67TKgjL7OTrACGgEmTf+zMvYRhoAoGCCqGSM49
+AwEHoUQDQgAESvX1TRh3NYeQeX6TjMhHcoYdfMTTXDVBkOYhizve2dCrSub0ZQ3n
+hNSNEmiYKqaZvThB8N9iVNBzIuBQWYtkbA==
+-----END EC PRIVATE KEY-----

internal/event/target/testdata/contrib/nats_tls.conf

@@ -0,0 +1,7 @@
+port: 14225
+net: localhost
+
+tls {
+    cert_file:  "./testdata/contrib/certs/nats_server_cert.pem"
+    key_file:   "./testdata/contrib/certs/nats_server_key.pem"
+}

internal/event/target/testdata/contrib/nats_tls_client_cert.conf

@@ -0,0 +1,18 @@
+port: 14226
+net: localhost
+
+tls {
+    cert_file:  "./testdata/contrib/certs/nats_server_cert.pem"
+    key_file:   "./testdata/contrib/certs/nats_server_key.pem"
+    ca_file:   "./testdata/contrib/certs/root_ca_cert.pem"
+    verify_and_map: true
+}
+authorization {
+    ADMIN = {
+        publish = ">"
+        subscribe = ">"
+    }
+    users = [
+        {user: "CN=localhost,OU=Client,O=MinIO,C=CA", permissions: $ADMIN}
+    ]
+}

internal/event/target/testdata/contrib/nats_tls_handshake_first.conf

@@ -0,0 +1,8 @@
+port: 14227
+net: localhost
+
+tls {
+    cert_file:  "./testdata/contrib/certs/nats_server_cert.pem"
+    key_file:   "./testdata/contrib/certs/nats_server_key.pem"
+    handshake_first: true
+}

internal/event/target/testdata/contrib/test.nkey

@@ -0,0 +1 @@
+SUACSSL3UAHUDXKFSNVUZRF5UHPMWZ6BFDTJ7M6USDXIEDNPPQYYYCU3VY

internal/event/target/webhook.go

@@ -0,0 +1,316 @@
+// Copyright (c) 2015-2023 Hanzo AI, Inc.
+//
+// This file is part of Hanzo S3 Object Storage stack
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package target
+
+import (
+	"bytes"
+	"context"
+	"crypto/tls"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"net"
+	"net/http"
+	"net/url"
+	"os"
+	"path/filepath"
+	"strings"
+	"syscall"
+	"time"
+
+	"github.com/hanzoai/s3/internal/event"
+	xhttp "github.com/hanzoai/s3/internal/http"
+	"github.com/hanzoai/s3/internal/logger"
+	"github.com/hanzoai/s3/internal/once"
+	"github.com/hanzoai/s3/internal/store"
+	"github.com/minio/pkg/v3/certs"
+	xnet "github.com/minio/pkg/v3/net"
+)
+
+// Webhook constants
+const (
+	WebhookEndpoint   = "endpoint"
+	WebhookAuthToken  = "auth_token"
+	WebhookQueueDir   = "queue_dir"
+	WebhookQueueLimit = "queue_limit"
+	WebhookClientCert = "client_cert"
+	WebhookClientKey  = "client_key"
+
+	EnvWebhookEnable     = "S3_NOTIFY_WEBHOOK_ENABLE"
+	EnvWebhookEndpoint   = "S3_NOTIFY_WEBHOOK_ENDPOINT"
+	EnvWebhookAuthToken  = "S3_NOTIFY_WEBHOOK_AUTH_TOKEN"
+	EnvWebhookQueueDir   = "S3_NOTIFY_WEBHOOK_QUEUE_DIR"
+	EnvWebhookQueueLimit = "S3_NOTIFY_WEBHOOK_QUEUE_LIMIT"
+	EnvWebhookClientCert = "S3_NOTIFY_WEBHOOK_CLIENT_CERT"
+	EnvWebhookClientKey  = "S3_NOTIFY_WEBHOOK_CLIENT_KEY"
+)
+
+// WebhookArgs - Webhook target arguments.
+type WebhookArgs struct {
+	Enable     bool            `json:"enable"`
+	Endpoint   xnet.URL        `json:"endpoint"`
+	AuthToken  string          `json:"authToken"`
+	Transport  *http.Transport `json:"-"`
+	QueueDir   string          `json:"queueDir"`
+	QueueLimit uint64          `json:"queueLimit"`
+	ClientCert string          `json:"clientCert"`
+	ClientKey  string          `json:"clientKey"`
+}
+
+// Validate WebhookArgs fields
+func (w WebhookArgs) Validate() error {
+	if !w.Enable {
+		return nil
+	}
+	if w.Endpoint.IsEmpty() {
+		return errors.New("endpoint empty")
+	}
+	if w.QueueDir != "" {
+		if !filepath.IsAbs(w.QueueDir) {
+			return errors.New("queueDir path should be absolute")
+		}
+	}
+	if w.ClientCert != "" && w.ClientKey == "" || w.ClientCert == "" && w.ClientKey != "" {
+		return errors.New("cert and key must be specified as a pair")
+	}
+	return nil
+}
+
+// WebhookTarget - Webhook target.
+type WebhookTarget struct {
+	initOnce once.Init
+
+	id         event.TargetID
+	args       WebhookArgs
+	transport  *http.Transport
+	httpClient *http.Client
+	store      store.Store[event.Event]
+	loggerOnce logger.LogOnce
+	cancel     context.CancelFunc
+	cancelCh   <-chan struct{}
+
+	addr string // full address ip/dns with a port number, e.g.  x.x.x.x:8080
+}
+
+// ID - returns target ID.
+func (target *WebhookTarget) ID() event.TargetID {
+	return target.id
+}
+
+// Name - returns the Name of the target.
+func (target *WebhookTarget) Name() string {
+	return target.ID().String()
+}
+
+// IsActive - Return true if target is up and active
+func (target *WebhookTarget) IsActive() (bool, error) {
+	if err := target.init(); err != nil {
+		return false, err
+	}
+	return target.isActive()
+}
+
+// Store returns any underlying store if set.
+func (target *WebhookTarget) Store() event.TargetStore {
+	return target.store
+}
+
+func (target *WebhookTarget) isActive() (bool, error) {
+	conn, err := net.DialTimeout("tcp", target.addr, 5*time.Second)
+	if err != nil {
+		if xnet.IsNetworkOrHostDown(err, false) {
+			return false, store.ErrNotConnected
+		}
+		return false, err
+	}
+	defer conn.Close()
+	return true, nil
+}
+
+// Save - saves the events to the store if queuestore is configured,
+// which will be replayed when the webhook connection is active.
+func (target *WebhookTarget) Save(eventData event.Event) error {
+	if target.store != nil {
+		_, err := target.store.Put(eventData)
+		return err
+	}
+	if err := target.init(); err != nil {
+		return err
+	}
+	err := target.send(eventData)
+	if err != nil {
+		if xnet.IsNetworkOrHostDown(err, false) {
+			return store.ErrNotConnected
+		}
+	}
+	return err
+}
+
+// send - sends an event to the webhook.
+func (target *WebhookTarget) send(eventData event.Event) error {
+	objectName, err := url.QueryUnescape(eventData.S3.Object.Key)
+	if err != nil {
+		return err
+	}
+	key := eventData.S3.Bucket.Name + "/" + objectName
+
+	data, err := json.Marshal(event.Log{EventName: eventData.EventName, Key: key, Records: []event.Event{eventData}})
+	if err != nil {
+		return err
+	}
+
+	req, err := http.NewRequest(http.MethodPost, target.args.Endpoint.String(), bytes.NewReader(data))
+	if err != nil {
+		return err
+	}
+
+	// Verify if the authToken already contains
+	// <Key> <Token> like format, if this is
+	// already present we can blindly use the
+	// authToken as is instead of adding 'Bearer'
+	tokens := strings.Fields(target.args.AuthToken)
+	switch len(tokens) {
+	case 2:
+		req.Header.Set("Authorization", target.args.AuthToken)
+	case 1:
+		req.Header.Set("Authorization", "Bearer "+target.args.AuthToken)
+	}
+
+	req.Header.Set("Content-Type", "application/json")
+
+	resp, err := target.httpClient.Do(req)
+	if err != nil {
+		return err
+	}
+	xhttp.DrainBody(resp.Body)
+
+	if resp.StatusCode >= 200 && resp.StatusCode <= 299 {
+		// accepted HTTP status codes.
+		return nil
+	} else if resp.StatusCode == http.StatusForbidden {
+		return fmt.Errorf("%s returned '%s', please check if your auth token is correctly set", target.args.Endpoint, resp.Status)
+	}
+	return fmt.Errorf("%s returned '%s', please check your endpoint configuration", target.args.Endpoint, resp.Status)
+}
+
+// SendFromStore - reads an event from store and sends it to webhook.
+func (target *WebhookTarget) SendFromStore(key store.Key) error {
+	if err := target.init(); err != nil {
+		return err
+	}
+
+	eventData, eErr := target.store.Get(key)
+	if eErr != nil {
+		// The last event key in a successful batch will be sent in the channel atmost once by the replayEvents()
+		// Such events will not exist and would've been already been sent successfully.
+		if os.IsNotExist(eErr) {
+			return nil
+		}
+		return eErr
+	}
+
+	if err := target.send(eventData); err != nil {
+		if xnet.IsNetworkOrHostDown(err, false) {
+			return store.ErrNotConnected
+		}
+		return err
+	}
+
+	// Delete the event from store.
+	return target.store.Del(key)
+}
+
+// Close - does nothing and available for interface compatibility.
+func (target *WebhookTarget) Close() error {
+	target.cancel()
+	return nil
+}
+
+func (target *WebhookTarget) init() error {
+	return target.initOnce.Do(target.initWebhook)
+}
+
+// Only called from init()
+func (target *WebhookTarget) initWebhook() error {
+	args := target.args
+	transport := target.transport
+
+	if args.ClientCert != "" && args.ClientKey != "" {
+		manager, err := certs.NewManager(context.Background(), args.ClientCert, args.ClientKey, tls.LoadX509KeyPair)
+		if err != nil {
+			return err
+		}
+		manager.ReloadOnSignal(syscall.SIGHUP) // allow reloads upon SIGHUP
+		transport.TLSClientConfig.GetClientCertificate = manager.GetClientCertificate
+	}
+	target.httpClient = &http.Client{Transport: transport}
+
+	yes, err := target.isActive()
+	if err != nil {
+		return err
+	}
+	if !yes {
+		return store.ErrNotConnected
+	}
+
+	return nil
+}
+
+// NewWebhookTarget - creates new Webhook target.
+func NewWebhookTarget(ctx context.Context, id string, args WebhookArgs, loggerOnce logger.LogOnce, transport *http.Transport) (*WebhookTarget, error) {
+	ctx, cancel := context.WithCancel(ctx)
+
+	var queueStore store.Store[event.Event]
+	if args.QueueDir != "" {
+		queueDir := filepath.Join(args.QueueDir, storePrefix+"-webhook-"+id)
+		queueStore = store.NewQueueStore[event.Event](queueDir, args.QueueLimit, event.StoreExtension)
+		if err := queueStore.Open(); err != nil {
+			cancel()
+			return nil, fmt.Errorf("unable to initialize the queue store of Webhook `%s`: %w", id, err)
+		}
+	}
+
+	target := &WebhookTarget{
+		id:         event.TargetID{ID: id, Name: "webhook"},
+		args:       args,
+		loggerOnce: loggerOnce,
+		transport:  transport,
+		store:      queueStore,
+		cancel:     cancel,
+		cancelCh:   ctx.Done(),
+	}
+
+	// Calculate the webhook addr with the port number format
+	target.addr = args.Endpoint.Host
+	if _, _, err := net.SplitHostPort(args.Endpoint.Host); err != nil && strings.Contains(err.Error(), "missing port in address") {
+		switch strings.ToLower(args.Endpoint.Scheme) {
+		case "http":
+			target.addr += ":80"
+		case "https":
+			target.addr += ":443"
+		default:
+			return nil, errors.New("unsupported scheme")
+		}
+	}
+
+	if target.store != nil {
+		store.StreamItems(target.store, target, target.cancelCh, target.loggerOnce)
+	}
+
+	return target, nil
+}

internal/logger/target/console/console.go

@@ -0,0 +1,160 @@
+// Copyright (c) 2015-2021 Hanzo AI, Inc.
+//
+// This file is part of Hanzo S3 Object Storage stack
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package console
+
+import (
+	"encoding/json"
+	"fmt"
+	"io"
+	"strconv"
+	"strings"
+
+	"github.com/minio/madmin-go/v3/logger/log"
+	"github.com/hanzoai/s3/internal/color"
+	"github.com/hanzoai/s3/internal/logger"
+)
+
+// Target implements loggerTarget to send log
+// in plain or json format to the standard output.
+type Target struct {
+	output io.Writer
+}
+
+// Validate - validate if the tty can be written to
+func (c *Target) Validate() error {
+	return nil
+}
+
+// Endpoint returns the backend endpoint
+func (c *Target) Endpoint() string {
+	return ""
+}
+
+func (c *Target) String() string {
+	return "console"
+}
+
+// Send log message 'e' to console
+func (c *Target) Send(e any) error {
+	entry, ok := e.(log.Entry)
+	if !ok {
+		return fmt.Errorf("Uexpected log entry structure %#v", e)
+	}
+	if logger.IsJSON() {
+		logJSON, err := json.Marshal(&entry)
+		if err != nil {
+			return err
+		}
+		fmt.Fprintln(c.output, string(logJSON))
+		return nil
+	}
+
+	if entry.Level == logger.EventKind {
+		fmt.Fprintln(c.output, entry.Message)
+		return nil
+	}
+
+	traceLength := len(entry.Trace.Source)
+	trace := make([]string, traceLength)
+
+	// Add a sequence number and formatting for each stack trace
+	// No formatting is required for the first entry
+	for i, element := range entry.Trace.Source {
+		trace[i] = fmt.Sprintf("%8v: %s", traceLength-i, element)
+	}
+
+	tagString := ""
+	for key, value := range entry.Trace.Variables {
+		if value != "" {
+			if tagString != "" {
+				tagString += ", "
+			}
+			tagString += fmt.Sprintf("%s=%#v", key, value)
+		}
+	}
+
+	var apiString string
+	if entry.API != nil {
+		apiString = "API: " + entry.API.Name
+		if entry.API.Args != nil {
+			args := ""
+			if entry.API.Args.Bucket != "" {
+				args = args + "bucket=" + entry.API.Args.Bucket
+			}
+			if entry.API.Args.Object != "" {
+				args = args + ", object=" + entry.API.Args.Object
+			}
+			if entry.API.Args.VersionID != "" {
+				args = args + ", versionId=" + entry.API.Args.VersionID
+			}
+			if len(entry.API.Args.Objects) > 0 {
+				args = args + ", multiObject=true, numberOfObjects=" + strconv.Itoa(len(entry.API.Args.Objects))
+			}
+			if len(args) > 0 {
+				apiString += "(" + args + ")"
+			}
+		}
+	} else {
+		apiString = "INTERNAL"
+	}
+	timeString := "Time: " + entry.Time.Format(logger.TimeFormat)
+
+	var deploymentID string
+	if entry.DeploymentID != "" {
+		deploymentID = "\nDeploymentID: " + entry.DeploymentID
+	}
+
+	var requestID string
+	if entry.RequestID != "" {
+		requestID = "\nRequestID: " + entry.RequestID
+	}
+
+	var remoteHost string
+	if entry.RemoteHost != "" {
+		remoteHost = "\nRemoteHost: " + entry.RemoteHost
+	}
+
+	var host string
+	if entry.Host != "" {
+		host = "\nHost: " + entry.Host
+	}
+
+	var userAgent string
+	if entry.UserAgent != "" {
+		userAgent = "\nUserAgent: " + entry.UserAgent
+	}
+
+	if len(entry.Trace.Variables) > 0 {
+		tagString = "\n       " + tagString
+	}
+
+	msg := color.RedBold(entry.Trace.Message)
+	output := fmt.Sprintf("\n%s\n%s%s%s%s%s%s\nError: %s%s\n%s",
+		apiString, timeString, deploymentID, requestID, remoteHost, host, userAgent,
+		msg, tagString, strings.Join(trace, "\n"))
+
+	fmt.Fprintln(c.output, output)
+	return nil
+}
+
+// New initializes a new logger target
+// which prints log directly in the standard
+// output.
+func New(w io.Writer) *Target {
+	return &Target{output: w}
+}

internal/logger/target/http/http.go

@@ -0,0 +1,633 @@
+// Copyright (c) 2015-2024 Hanzo AI, Inc.
+//
+// This file is part of Hanzo S3 Object Storage stack
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package http
+
+import (
+	"bytes"
+	"context"
+	"errors"
+	"fmt"
+	"net/http"
+	"net/url"
+	"os"
+	"path/filepath"
+	"strconv"
+	"strings"
+	"sync"
+	"sync/atomic"
+	"time"
+
+	jsoniter "github.com/json-iterator/go"
+	xhttp "github.com/hanzoai/s3/internal/http"
+	xioutil "github.com/hanzoai/s3/internal/ioutil"
+	types "github.com/hanzoai/s3/internal/logger/target/loggertypes"
+	"github.com/hanzoai/s3/internal/once"
+	"github.com/hanzoai/s3/internal/store"
+	xnet "github.com/minio/pkg/v3/net"
+	"github.com/valyala/bytebufferpool"
+)
+
+const (
+
+	// maxWorkers is the maximum number of concurrent http loggers
+	maxWorkers = 16
+
+	// maxWorkers is the maximum number of concurrent batch http loggers
+	maxWorkersWithBatchEvents = 4
+
+	// the suffix for the configured queue dir where the logs will be persisted.
+	httpLoggerExtension = ".http.log"
+)
+
+const (
+	statusOffline = iota
+	statusOnline
+	statusClosed
+)
+
+var (
+	logChBuffers = make(map[string]chan any)
+	logChLock    = sync.Mutex{}
+)
+
+// Config http logger target
+type Config struct {
+	Enabled     bool              `json:"enabled"`
+	Name        string            `json:"name"`
+	UserAgent   string            `json:"userAgent"`
+	Endpoint    *xnet.URL         `json:"endpoint"`
+	AuthToken   string            `json:"authToken"`
+	ClientCert  string            `json:"clientCert"`
+	ClientKey   string            `json:"clientKey"`
+	BatchSize   int               `json:"batchSize"`
+	QueueSize   int               `json:"queueSize"`
+	QueueDir    string            `json:"queueDir"`
+	MaxRetry    int               `json:"maxRetry"`
+	RetryIntvl  time.Duration     `json:"retryInterval"`
+	Proxy       string            `json:"string"`
+	Transport   http.RoundTripper `json:"-"`
+	HTTPTimeout time.Duration     `json:"httpTimeout"`
+
+	// Custom logger
+	LogOnceIf func(ctx context.Context, err error, id string, errKind ...any) `json:"-"`
+}
+
+// Target implements logger.Target and sends the json
+// format of a log entry to the configured http endpoint.
+// An internal buffer of logs is maintained but when the
+// buffer is full, new logs are just ignored and an error
+// is returned to the caller.
+type Target struct {
+	totalMessages  atomic.Int64
+	failedMessages atomic.Int64
+	status         atomic.Int32
+
+	// Worker control
+	workers    atomic.Int64
+	maxWorkers int64
+
+	// workerStartMu sync.Mutex
+	lastStarted time.Time
+
+	wg sync.WaitGroup
+
+	// Channel of log entries.
+	// Reading logCh must hold read lock on logChMu (to avoid read race)
+	// Sending a value on logCh must hold read lock on logChMu (to avoid closing)
+	logCh   chan any
+	logChMu sync.RWMutex
+
+	// If this webhook is being re-configured we will
+	// assign the new webhook target to this field.
+	// The Send() method will then re-direct entries
+	// to the new target when the current one
+	// has been set to status "statusClosed".
+	// Once the glogal target slice has been migrated
+	// the current target will stop receiving entries.
+	migrateTarget *Target
+
+	// Number of events per HTTP send to webhook target
+	// this is ideally useful only if your endpoint can
+	// support reading multiple events on a stream for example
+	// like : Splunk HTTP Event collector, if you are unsure
+	// set this to '1'.
+	batchSize   int
+	payloadType string
+
+	// store to persist and replay the logs to the target
+	// to avoid missing events when the target is down.
+	store          store.Store[any]
+	storeCtxCancel context.CancelFunc
+
+	initQueueOnce once.Init
+
+	config      Config
+	client      *http.Client
+	httpTimeout time.Duration
+}
+
+// Name returns the name of the target
+func (h *Target) Name() string {
+	return "minio-http-" + h.config.Name
+}
+
+// Type - returns type of the target
+func (h *Target) Type() types.TargetType {
+	return types.TargetHTTP
+}
+
+// Endpoint returns the backend endpoint
+func (h *Target) Endpoint() string {
+	return h.config.Endpoint.String()
+}
+
+func (h *Target) String() string {
+	return h.config.Name
+}
+
+// IsOnline returns true if the target is reachable using a cached value
+func (h *Target) IsOnline(ctx context.Context) bool {
+	return h.status.Load() == statusOnline
+}
+
+// Stats returns the target statistics.
+func (h *Target) Stats() types.TargetStats {
+	h.logChMu.RLock()
+	queueLength := len(h.logCh)
+	h.logChMu.RUnlock()
+	stats := types.TargetStats{
+		TotalMessages:  h.totalMessages.Load(),
+		FailedMessages: h.failedMessages.Load(),
+		QueueLength:    queueLength,
+	}
+
+	return stats
+}
+
+// AssignMigrateTarget assigns a target
+// which will eventually replace the current target.
+func (h *Target) AssignMigrateTarget(migrateTgt *Target) {
+	h.migrateTarget = migrateTgt
+}
+
+// Init validate and initialize the http target
+func (h *Target) Init(ctx context.Context) (err error) {
+	if h.config.QueueDir != "" {
+		return h.initQueueOnce.DoWithContext(ctx, h.initDiskStore)
+	}
+	return h.initQueueOnce.DoWithContext(ctx, h.initMemoryStore)
+}
+
+func (h *Target) initDiskStore(ctx context.Context) (err error) {
+	ctx, cancel := context.WithCancel(ctx)
+	h.storeCtxCancel = cancel
+	h.lastStarted = time.Now()
+	go h.startQueueProcessor(ctx, true)
+
+	queueStore := store.NewQueueStore[any](
+		filepath.Join(h.config.QueueDir, h.Name()),
+		uint64(h.config.QueueSize),
+		httpLoggerExtension,
+	)
+
+	if err := queueStore.Open(); err != nil {
+		return fmt.Errorf("unable to initialize the queue store of %s webhook: %w", h.Name(), err)
+	}
+
+	h.store = queueStore
+	store.StreamItems(h.store, h, ctx.Done(), h.config.LogOnceIf)
+
+	return nil
+}
+
+func (h *Target) initMemoryStore(ctx context.Context) (err error) {
+	ctx, cancel := context.WithCancel(ctx)
+	h.storeCtxCancel = cancel
+	h.lastStarted = time.Now()
+	go h.startQueueProcessor(ctx, true)
+	return nil
+}
+
+func (h *Target) send(ctx context.Context, payload []byte, payloadCount int, payloadType string, timeout time.Duration) (err error) {
+	defer func() {
+		if err != nil {
+			if xnet.IsNetworkOrHostDown(err, false) {
+				h.status.Store(statusOffline)
+			}
+			h.failedMessages.Add(int64(payloadCount))
+		} else {
+			h.status.Store(statusOnline)
+		}
+	}()
+
+	ctx, cancel := context.WithTimeout(ctx, timeout)
+	defer cancel()
+
+	req, err := http.NewRequestWithContext(ctx, http.MethodPost,
+		h.Endpoint(), bytes.NewReader(payload))
+	if err != nil {
+		return fmt.Errorf("invalid configuration for '%s'; %v", h.Endpoint(), err)
+	}
+	if payloadType != "" {
+		req.Header.Set(xhttp.ContentType, payloadType)
+	}
+	req.Header.Set(xhttp.WebhookEventPayloadCount, strconv.Itoa(payloadCount))
+	req.Header.Set(xhttp.MinIOVersion, xhttp.GlobalMinIOVersion)
+	req.Header.Set(xhttp.MinioDeploymentID, xhttp.GlobalDeploymentID)
+
+	// Set user-agent to indicate MinIO release
+	// version to the configured log endpoint
+	req.Header.Set("User-Agent", h.config.UserAgent)
+
+	if h.config.AuthToken != "" {
+		req.Header.Set("Authorization", h.config.AuthToken)
+	}
+
+	resp, err := h.client.Do(req)
+	if err != nil {
+		return fmt.Errorf("%s returned '%w', please check your endpoint configuration", h.Endpoint(), err)
+	}
+
+	// Drain any response.
+	xhttp.DrainBody(resp.Body)
+
+	if resp.StatusCode >= 200 && resp.StatusCode <= 299 {
+		// accepted HTTP status codes.
+		return nil
+	} else if resp.StatusCode == http.StatusForbidden {
+		return fmt.Errorf("%s returned '%s', please check if your auth token is correctly set", h.Endpoint(), resp.Status)
+	}
+	return fmt.Errorf("%s returned '%s', please check your endpoint configuration", h.Endpoint(), resp.Status)
+}
+
+func (h *Target) startQueueProcessor(ctx context.Context, mainWorker bool) {
+	h.logChMu.RLock()
+	if h.logCh == nil {
+		h.logChMu.RUnlock()
+		return
+	}
+	h.logChMu.RUnlock()
+
+	h.workers.Add(1)
+	defer h.workers.Add(-1)
+
+	h.wg.Add(1)
+	defer h.wg.Done()
+
+	entries := make([]any, 0)
+	name := h.Name()
+
+	defer func() {
+		// re-load the global buffer pointer
+		// in case it was modified by a new target.
+		logChLock.Lock()
+		currentGlobalBuffer, ok := logChBuffers[name]
+		logChLock.Unlock()
+		if !ok {
+			return
+		}
+
+		for _, v := range entries {
+			select {
+			case currentGlobalBuffer <- v:
+			default:
+			}
+		}
+
+		if mainWorker {
+		drain:
+			for {
+				select {
+				case v, ok := <-h.logCh:
+					if !ok {
+						break drain
+					}
+
+					currentGlobalBuffer <- v
+				default:
+					break drain
+				}
+			}
+		}
+	}()
+
+	lastBatchProcess := time.Now()
+
+	buf := bytebufferpool.Get()
+	enc := jsoniter.ConfigCompatibleWithStandardLibrary.NewEncoder(buf)
+	defer bytebufferpool.Put(buf)
+
+	isDirQueue := h.config.QueueDir != ""
+
+	// globalBuffer is always created or adjusted
+	// before this method is launched.
+	logChLock.Lock()
+	globalBuffer := logChBuffers[name]
+	logChLock.Unlock()
+
+	ticker := time.NewTicker(time.Second)
+	defer ticker.Stop()
+
+	var count int
+	for {
+		var (
+			ok    bool
+			entry any
+		)
+
+		if count < h.batchSize {
+			tickered := false
+			select {
+			case <-ticker.C:
+				tickered = true
+			case entry = <-globalBuffer:
+			case entry, ok = <-h.logCh:
+				if !ok {
+					return
+				}
+			case <-ctx.Done():
+				return
+			}
+
+			if !tickered {
+				h.totalMessages.Add(1)
+				if !isDirQueue {
+					if err := enc.Encode(&entry); err != nil {
+						h.config.LogOnceIf(
+							ctx,
+							fmt.Errorf("unable to encode webhook log entry, err  '%w' entry: %v\n", err, entry),
+							h.Name(),
+						)
+						h.failedMessages.Add(1)
+						continue
+					}
+				} else {
+					entries = append(entries, entry)
+				}
+				count++
+			}
+
+			if len(h.logCh) > 0 || len(globalBuffer) > 0 || count == 0 {
+				// there is something in the log queue
+				// process it first, even if we tickered
+				// first, or we have not received any events
+				// yet, still wait on it.
+				continue
+			}
+
+			// If we are doing batching, we should wait
+			// at least for a second, before sending.
+			// Even if there is nothing in the queue.
+			if h.batchSize > 1 && time.Since(lastBatchProcess) < time.Second {
+				continue
+			}
+		}
+
+		// if we have reached the count send at once
+		// or we have crossed last second before batch was sent, send at once
+		lastBatchProcess = time.Now()
+
+		var retries int
+		retryIntvl := h.config.RetryIntvl
+		if retryIntvl <= 0 {
+			retryIntvl = 3 * time.Second
+		}
+
+		maxRetries := h.config.MaxRetry
+
+	retry:
+		// If the channel reaches above half capacity
+		// we spawn more workers. The workers spawned
+		// from this main worker routine will exit
+		// once the channel drops below half capacity
+		// and when it's been at least 30 seconds since
+		// we launched a new worker.
+		if mainWorker && len(h.logCh) > cap(h.logCh)/2 {
+			nWorkers := h.workers.Load()
+			if nWorkers < h.maxWorkers {
+				if time.Since(h.lastStarted).Milliseconds() > 10 {
+					h.lastStarted = time.Now()
+					go h.startQueueProcessor(ctx, false)
+				}
+			}
+		}
+
+		var err error
+		if !isDirQueue {
+			err = h.send(ctx, buf.Bytes(), count, h.payloadType, h.httpTimeout)
+		} else {
+			_, err = h.store.PutMultiple(entries)
+		}
+
+		if err != nil {
+			if errors.Is(err, context.Canceled) {
+				return
+			}
+
+			h.config.LogOnceIf(
+				context.Background(),
+				fmt.Errorf("unable to send audit/log entry(s) to '%s' err '%w': %d", name, err, count),
+				name,
+			)
+
+			time.Sleep(retryIntvl)
+			if maxRetries == 0 {
+				goto retry
+			}
+			retries++
+			if retries <= maxRetries {
+				goto retry
+			}
+		}
+
+		entries = make([]any, 0)
+		count = 0
+		if !isDirQueue {
+			buf.Reset()
+		}
+
+		if !mainWorker && len(h.logCh) < cap(h.logCh)/2 {
+			if time.Since(h.lastStarted).Seconds() > 30 {
+				return
+			}
+		}
+	}
+}
+
+// CreateOrAdjustGlobalBuffer will create or adjust the global log entry buffers
+// which are used to migrate log entries between old and new targets.
+func CreateOrAdjustGlobalBuffer(currentTgt *Target, newTgt *Target) {
+	logChLock.Lock()
+	defer logChLock.Unlock()
+
+	requiredCap := currentTgt.config.QueueSize + (currentTgt.config.BatchSize * int(currentTgt.maxWorkers))
+	currentCap := 0
+	name := newTgt.Name()
+
+	currentBuff, ok := logChBuffers[name]
+	if !ok {
+		logChBuffers[name] = make(chan any, requiredCap)
+		currentCap = requiredCap
+	} else {
+		currentCap = cap(currentBuff)
+		requiredCap += len(currentBuff)
+	}
+
+	if requiredCap > currentCap {
+		logChBuffers[name] = make(chan any, requiredCap)
+
+		if len(currentBuff) > 0 {
+		drain:
+			for {
+				select {
+				case v, ok := <-currentBuff:
+					if !ok {
+						break drain
+					}
+					logChBuffers[newTgt.Name()] <- v
+				default:
+					break drain
+				}
+			}
+		}
+	}
+}
+
+// New initializes a new logger target which
+// sends log over http to the specified endpoint
+func New(config Config) (*Target, error) {
+	maxWorkers := maxWorkers
+	if config.BatchSize > 100 {
+		maxWorkers = maxWorkersWithBatchEvents
+	} else if config.BatchSize <= 0 {
+		config.BatchSize = 1
+	}
+
+	h := &Target{
+		logCh:       make(chan any, config.QueueSize),
+		config:      config,
+		batchSize:   config.BatchSize,
+		maxWorkers:  int64(maxWorkers),
+		httpTimeout: config.HTTPTimeout,
+	}
+	h.status.Store(statusOffline)
+
+	if config.BatchSize > 1 {
+		h.payloadType = ""
+	} else {
+		h.payloadType = "application/json"
+	}
+
+	// If proxy available, set the same
+	if h.config.Proxy != "" {
+		proxyURL, _ := url.Parse(h.config.Proxy)
+		transport := h.config.Transport
+		if tr, ok := transport.(*http.Transport); ok {
+			ctransport := tr.Clone()
+			ctransport.Proxy = http.ProxyURL(proxyURL)
+			h.config.Transport = ctransport
+		}
+	}
+
+	h.client = &http.Client{Transport: h.config.Transport}
+	return h, nil
+}
+
+// SendFromStore - reads the log from store and sends it to webhook.
+func (h *Target) SendFromStore(key store.Key) (err error) {
+	var eventData []byte
+	eventData, err = h.store.GetRaw(key)
+	if err != nil {
+		if os.IsNotExist(err) {
+			return nil
+		}
+		return err
+	}
+
+	count := 1
+	v := strings.Split(key.Name, ":")
+	if len(v) == 2 {
+		count, err = strconv.Atoi(v[0])
+		if err != nil {
+			return err
+		}
+	}
+
+	if err := h.send(context.Background(), eventData, count, h.payloadType, h.httpTimeout); err != nil {
+		return err
+	}
+
+	// Delete the event from store.
+	return h.store.Del(key)
+}
+
+// Send the log message 'entry' to the http target.
+// Messages are queued in the disk if the store is enabled
+// If Cancel has been called the message is ignored.
+func (h *Target) Send(ctx context.Context, entry any) error {
+	if h.status.Load() == statusClosed {
+		if h.migrateTarget != nil {
+			return h.migrateTarget.Send(ctx, entry)
+		}
+		return nil
+	}
+
+	h.logChMu.RLock()
+	defer h.logChMu.RUnlock()
+	if h.logCh == nil {
+		// We are closing...
+		return nil
+	}
+
+	select {
+	case h.logCh <- entry:
+		h.totalMessages.Add(1)
+	case <-ctx.Done():
+		// return error only for context timedout.
+		if errors.Is(ctx.Err(), context.DeadlineExceeded) {
+			return ctx.Err()
+		}
+		return nil
+	default:
+		h.totalMessages.Add(1)
+		h.failedMessages.Add(1)
+		return errors.New("log buffer full")
+	}
+
+	return nil
+}
+
+// Cancel - cancels the target.
+// All queued messages are flushed and the function returns afterwards.
+// All messages sent to the target after this function has been called will be dropped.
+func (h *Target) Cancel() {
+	h.status.Store(statusClosed)
+	h.storeCtxCancel()
+
+	// Wait for messages to be sent...
+	h.wg.Wait()
+
+	// Set logch to nil and close it.
+	// This will block all Send operations,
+	// and finish the existing ones.
+	// All future ones will be discarded.
+	h.logChMu.Lock()
+	xioutil.SafeClose(h.logCh)
+	h.logCh = nil
+	h.logChMu.Unlock()
+}

internal/logger/target/kafka/kafka.go

@@ -0,0 +1,404 @@
+// Copyright (c) 2015-2023 Hanzo AI, Inc.
+//
+// This file is part of Hanzo S3 Object Storage stack
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package kafka
+
+import (
+	"context"
+	"crypto/tls"
+	"crypto/x509"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"log"
+	"os"
+	"path/filepath"
+	"sync"
+	"sync/atomic"
+	"time"
+
+	"github.com/IBM/sarama"
+	saramatls "github.com/IBM/sarama/tools/tls"
+
+	xioutil "github.com/hanzoai/s3/internal/ioutil"
+	types "github.com/hanzoai/s3/internal/logger/target/loggertypes"
+	"github.com/hanzoai/s3/internal/once"
+	"github.com/hanzoai/s3/internal/store"
+	xnet "github.com/minio/pkg/v3/net"
+)
+
+// the suffix for the configured queue dir where the logs will be persisted.
+const kafkaLoggerExtension = ".kafka.log"
+
+const (
+	statusClosed = iota
+	statusOffline
+	statusOnline
+)
+
+// Config - kafka target arguments.
+type Config struct {
+	Enabled bool        `json:"enable"`
+	Brokers []xnet.Host `json:"brokers"`
+	Topic   string      `json:"topic"`
+	Version string      `json:"version"`
+	TLS     struct {
+		Enable        bool               `json:"enable"`
+		RootCAs       *x509.CertPool     `json:"-"`
+		SkipVerify    bool               `json:"skipVerify"`
+		ClientAuth    tls.ClientAuthType `json:"clientAuth"`
+		ClientTLSCert string             `json:"clientTLSCert"`
+		ClientTLSKey  string             `json:"clientTLSKey"`
+	} `json:"tls"`
+	SASL struct {
+		Enable    bool   `json:"enable"`
+		User      string `json:"username"`
+		Password  string `json:"password"`
+		Mechanism string `json:"mechanism"`
+	} `json:"sasl"`
+	// Queue store
+	QueueSize int    `json:"queueSize"`
+	QueueDir  string `json:"queueDir"`
+
+	// Custom logger
+	LogOnce func(ctx context.Context, err error, id string, errKind ...any) `json:"-"`
+}
+
+// Target - Kafka target.
+type Target struct {
+	status int32
+
+	totalMessages  int64
+	failedMessages int64
+
+	wg sync.WaitGroup
+
+	// Channel of log entries.
+	// Reading logCh must hold read lock on logChMu (to avoid read race)
+	// Sending a value on logCh must hold read lock on logChMu (to avoid closing)
+	logCh   chan any
+	logChMu sync.RWMutex
+
+	// store to persist and replay the logs to the target
+	// to avoid missing events when the target is down.
+	store          store.Store[any]
+	storeCtxCancel context.CancelFunc
+
+	initKafkaOnce      once.Init
+	initQueueStoreOnce once.Init
+
+	client   sarama.Client
+	producer sarama.SyncProducer
+	kconfig  Config
+	config   *sarama.Config
+}
+
+func (h *Target) validate() error {
+	if len(h.kconfig.Brokers) == 0 {
+		return errors.New("no broker address found")
+	}
+	for _, b := range h.kconfig.Brokers {
+		if _, err := xnet.ParseHost(b.String()); err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+// Name returns the name of the target
+func (h *Target) Name() string {
+	return "minio-kafka-audit"
+}
+
+// Endpoint - return kafka target
+func (h *Target) Endpoint() string {
+	return "kafka"
+}
+
+// String - kafka string
+func (h *Target) String() string {
+	return "kafka"
+}
+
+// Stats returns the target statistics.
+func (h *Target) Stats() types.TargetStats {
+	h.logChMu.RLock()
+	queueLength := len(h.logCh)
+	h.logChMu.RUnlock()
+
+	return types.TargetStats{
+		TotalMessages:  atomic.LoadInt64(&h.totalMessages),
+		FailedMessages: atomic.LoadInt64(&h.failedMessages),
+		QueueLength:    queueLength,
+	}
+}
+
+// Init initialize kafka target
+func (h *Target) Init(ctx context.Context) error {
+	if !h.kconfig.Enabled {
+		return nil
+	}
+	if err := h.validate(); err != nil {
+		return err
+	}
+	if h.kconfig.QueueDir != "" {
+		if err := h.initQueueStoreOnce.DoWithContext(ctx, h.initQueueStore); err != nil {
+			return err
+		}
+		return h.initKafkaOnce.Do(h.init)
+	}
+	if err := h.init(); err != nil {
+		return err
+	}
+	go h.startKafkaLogger()
+	return nil
+}
+
+func (h *Target) initQueueStore(ctx context.Context) (err error) {
+	queueDir := filepath.Join(h.kconfig.QueueDir, h.Name())
+	queueStore := store.NewQueueStore[any](queueDir, uint64(h.kconfig.QueueSize), kafkaLoggerExtension)
+	if err = queueStore.Open(); err != nil {
+		return fmt.Errorf("unable to initialize the queue store of %s webhook: %w", h.Name(), err)
+	}
+	ctx, cancel := context.WithCancel(ctx)
+	h.store = queueStore
+	h.storeCtxCancel = cancel
+	store.StreamItems(h.store, h, ctx.Done(), h.kconfig.LogOnce)
+	return err
+}
+
+func (h *Target) startKafkaLogger() {
+	h.logChMu.RLock()
+	logCh := h.logCh
+	if logCh != nil {
+		// We are not allowed to add when logCh is nil
+		h.wg.Add(1)
+		defer h.wg.Done()
+	}
+	h.logChMu.RUnlock()
+
+	if logCh == nil {
+		return
+	}
+
+	// Create a routine which sends json logs received
+	// from an internal channel.
+	for entry := range logCh {
+		h.logEntry(entry)
+	}
+}
+
+func (h *Target) logEntry(entry any) {
+	atomic.AddInt64(&h.totalMessages, 1)
+	if err := h.send(entry); err != nil {
+		atomic.AddInt64(&h.failedMessages, 1)
+		h.kconfig.LogOnce(context.Background(), err, h.kconfig.Topic)
+	}
+}
+
+func (h *Target) send(entry any) error {
+	if err := h.initKafkaOnce.Do(h.init); err != nil {
+		return err
+	}
+	logJSON, err := json.Marshal(&entry)
+	if err != nil {
+		return err
+	}
+	msg := sarama.ProducerMessage{
+		Topic: h.kconfig.Topic,
+		Value: sarama.ByteEncoder(logJSON),
+	}
+	_, _, err = h.producer.SendMessage(&msg)
+	if err != nil {
+		atomic.StoreInt32(&h.status, statusOffline)
+	} else {
+		atomic.StoreInt32(&h.status, statusOnline)
+	}
+	return err
+}
+
+// Init initialize kafka target
+func (h *Target) init() error {
+	if os.Getenv("_S3_KAFKA_DEBUG") != "" {
+		sarama.DebugLogger = log.Default()
+	}
+
+	sconfig := sarama.NewConfig()
+	if h.kconfig.Version != "" {
+		kafkaVersion, err := sarama.ParseKafkaVersion(h.kconfig.Version)
+		if err != nil {
+			return err
+		}
+		sconfig.Version = kafkaVersion
+	}
+
+	sconfig.Net.KeepAlive = 60 * time.Second
+	sconfig.Net.SASL.User = h.kconfig.SASL.User
+	sconfig.Net.SASL.Password = h.kconfig.SASL.Password
+	initScramClient(h.kconfig, sconfig) // initializes configured scram client.
+	sconfig.Net.SASL.Enable = h.kconfig.SASL.Enable
+
+	tlsConfig, err := saramatls.NewConfig(h.kconfig.TLS.ClientTLSCert, h.kconfig.TLS.ClientTLSKey)
+	if err != nil {
+		return err
+	}
+
+	sconfig.Net.TLS.Enable = h.kconfig.TLS.Enable
+	sconfig.Net.TLS.Config = tlsConfig
+	sconfig.Net.TLS.Config.InsecureSkipVerify = h.kconfig.TLS.SkipVerify
+	sconfig.Net.TLS.Config.ClientAuth = h.kconfig.TLS.ClientAuth
+	sconfig.Net.TLS.Config.RootCAs = h.kconfig.TLS.RootCAs
+
+	// These settings are needed to ensure that kafka client doesn't hang on brokers
+	// refer https://github.com/IBM/sarama/issues/765#issuecomment-254333355
+	sconfig.Producer.Retry.Max = 2
+	sconfig.Producer.Retry.Backoff = (10 * time.Second)
+	sconfig.Producer.Return.Successes = true
+	sconfig.Producer.Return.Errors = true
+	sconfig.Producer.RequiredAcks = 1
+	sconfig.Producer.Timeout = (10 * time.Second)
+	sconfig.Net.ReadTimeout = (10 * time.Second)
+	sconfig.Net.DialTimeout = (10 * time.Second)
+	sconfig.Net.WriteTimeout = (10 * time.Second)
+	sconfig.Metadata.Retry.Max = 1
+	sconfig.Metadata.Retry.Backoff = (10 * time.Second)
+	sconfig.Metadata.RefreshFrequency = (15 * time.Minute)
+
+	h.config = sconfig
+
+	var brokers []string
+	for _, broker := range h.kconfig.Brokers {
+		brokers = append(brokers, broker.String())
+	}
+
+	client, err := sarama.NewClient(brokers, sconfig)
+	if err != nil {
+		return err
+	}
+
+	producer, err := sarama.NewSyncProducerFromClient(client)
+	if err != nil {
+		return err
+	}
+	h.client = client
+	h.producer = producer
+
+	if len(h.client.Brokers()) > 0 {
+		// Refer https://github.com/IBM/sarama/issues/1341
+		atomic.StoreInt32(&h.status, statusOnline)
+	}
+
+	return nil
+}
+
+// IsOnline returns true if the target is online.
+func (h *Target) IsOnline(_ context.Context) bool {
+	return atomic.LoadInt32(&h.status) == statusOnline
+}
+
+// Send log message 'e' to kafka target.
+func (h *Target) Send(ctx context.Context, entry any) error {
+	if h.store != nil {
+		// save the entry to the queue store which will be replayed to the target.
+		_, err := h.store.Put(entry)
+		return err
+	}
+	h.logChMu.RLock()
+	defer h.logChMu.RUnlock()
+	if h.logCh == nil {
+		// We are closing...
+		return nil
+	}
+
+	select {
+	case h.logCh <- entry:
+	case <-ctx.Done():
+		// return error only for context timedout.
+		if errors.Is(ctx.Err(), context.DeadlineExceeded) {
+			return ctx.Err()
+		}
+		return nil
+	default:
+		// log channel is full, do not wait and return
+		// an error immediately to the caller
+		atomic.AddInt64(&h.totalMessages, 1)
+		atomic.AddInt64(&h.failedMessages, 1)
+		return errors.New("log buffer full")
+	}
+	return nil
+}
+
+// SendFromStore - reads the log from store and sends it to kafka.
+func (h *Target) SendFromStore(key store.Key) (err error) {
+	auditEntry, err := h.store.Get(key)
+	if err != nil {
+		if os.IsNotExist(err) {
+			return nil
+		}
+		return err
+	}
+	atomic.AddInt64(&h.totalMessages, 1)
+	err = h.send(auditEntry)
+	if err != nil {
+		atomic.AddInt64(&h.failedMessages, 1)
+		return err
+	}
+	// Delete the event from store.
+	return h.store.Del(key)
+}
+
+// Cancel - cancels the target
+func (h *Target) Cancel() {
+	// If queuestore is configured, cancel it's context to
+	// stop the replay go-routine.
+	if h.store != nil {
+		h.storeCtxCancel()
+	}
+
+	// Set logch to nil and close it.
+	// This will block all Send operations,
+	// and finish the existing ones.
+	// All future ones will be discarded.
+	h.logChMu.Lock()
+	xioutil.SafeClose(h.logCh)
+	h.logCh = nil
+	h.logChMu.Unlock()
+
+	if h.producer != nil {
+		h.producer.Close()
+		h.client.Close()
+	}
+
+	// Wait for messages to be sent...
+	h.wg.Wait()
+}
+
+// New initializes a new logger target which
+// sends log over http to the specified endpoint
+func New(config Config) *Target {
+	target := &Target{
+		logCh:   make(chan any, config.QueueSize),
+		kconfig: config,
+		status:  statusOffline,
+	}
+	return target
+}
+
+// Type - returns type of the target
+func (h *Target) Type() types.TargetType {
+	return types.TargetKafka
+}

internal/logger/target/kafka/kafka_scram_client_contrib.go

@@ -0,0 +1,87 @@
+// Copyright (c) 2015-2021 Hanzo AI, Inc.
+//
+// This file is part of Hanzo S3 Object Storage stack
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package kafka
+
+import (
+	"crypto/sha512"
+	"strings"
+
+	"github.com/IBM/sarama"
+	"github.com/xdg/scram"
+
+	"github.com/hanzoai/s3/internal/hash/sha256"
+)
+
+func initScramClient(cfg Config, config *sarama.Config) {
+	switch strings.ToLower(cfg.SASL.Mechanism) {
+	case "sha512":
+		config.Net.SASL.SCRAMClientGeneratorFunc = func() sarama.SCRAMClient { return &XDGSCRAMClient{HashGeneratorFcn: KafkaSHA512} }
+		config.Net.SASL.Mechanism = sarama.SASLMechanism(sarama.SASLTypeSCRAMSHA512)
+	case "sha256":
+		config.Net.SASL.SCRAMClientGeneratorFunc = func() sarama.SCRAMClient { return &XDGSCRAMClient{HashGeneratorFcn: KafkaSHA256} }
+		config.Net.SASL.Mechanism = sarama.SASLMechanism(sarama.SASLTypeSCRAMSHA256)
+	default:
+		// default to PLAIN
+		config.Net.SASL.Mechanism = sarama.SASLMechanism(sarama.SASLTypePlaintext)
+	}
+}
+
+// KafkaSHA256 is a function that returns a crypto/sha256 hasher and should be used
+// to create Client objects configured for SHA-256 hashing.
+var KafkaSHA256 scram.HashGeneratorFcn = sha256.New
+
+// KafkaSHA512 is a function that returns a crypto/sha512 hasher and should be used
+// to create Client objects configured for SHA-512 hashing.
+var KafkaSHA512 scram.HashGeneratorFcn = sha512.New
+
+// XDGSCRAMClient implements the client-side of an authentication
+// conversation with a server.  A new conversation must be created for
+// each authentication attempt.
+type XDGSCRAMClient struct {
+	*scram.Client
+	*scram.ClientConversation
+	scram.HashGeneratorFcn
+}
+
+// Begin constructs a SCRAM client component based on a given hash.Hash
+// factory receiver.  This constructor will normalize the username, password
+// and authzID via the SASLprep algorithm, as recommended by RFC-5802.  If
+// SASLprep fails, the method returns an error.
+func (x *XDGSCRAMClient) Begin(userName, password, authzID string) (err error) {
+	x.Client, err = x.NewClient(userName, password, authzID)
+	if err != nil {
+		return err
+	}
+	x.ClientConversation = x.NewConversation()
+	return nil
+}
+
+// Step takes a string provided from a server (or just an empty string for the
+// very first conversation step) and attempts to move the authentication
+// conversation forward.  It returns a string to be sent to the server or an
+// error if the server message is invalid.  Calling Step after a conversation
+// completes is also an error.
+func (x *XDGSCRAMClient) Step(challenge string) (response string, err error) {
+	response, err = x.ClientConversation.Step(challenge)
+	return response, err
+}
+
+// Done returns true if the conversation is completed or has errored.
+func (x *XDGSCRAMClient) Done() bool {
+	return x.ClientConversation.Done()
+}

internal/logger/target/loggertypes/targettype_string.go

@@ -0,0 +1,26 @@
+// Code generated by "stringer -type=TargetType -trimprefix=Target types.go"; DO NOT EDIT.
+
+package loggertypes
+
+import "strconv"
+
+func _() {
+	// An "invalid array index" compiler error signifies that the constant values have changed.
+	// Re-run the stringer command to generate them again.
+	var x [1]struct{}
+	_ = x[TargetConsole-1]
+	_ = x[TargetHTTP-2]
+	_ = x[TargetKafka-3]
+}
+
+const _TargetType_name = "ConsoleHTTPKafka"
+
+var _TargetType_index = [...]uint8{0, 7, 11, 16}
+
+func (i TargetType) String() string {
+	i -= 1
+	if i >= TargetType(len(_TargetType_index)-1) {
+		return "TargetType(" + strconv.FormatInt(int64(i+1), 10) + ")"
+	}
+	return _TargetType_name[_TargetType_index[i]:_TargetType_index[i+1]]
+}

internal/logger/target/loggertypes/types.go

@@ -0,0 +1,43 @@
+// Copyright (c) 2015-2022 Hanzo AI, Inc.
+//
+// This file is part of Hanzo S3 Object Storage stack
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package loggertypes
+
+// TargetType indicates type of the target e.g. console, http, kafka
+type TargetType uint8
+
+//go:generate stringer -type=TargetType -trimprefix=Target $GOFILE
+
+// Constants for target types
+const (
+	_ TargetType = iota
+	TargetConsole
+	TargetHTTP
+	TargetKafka
+)
+
+// TargetStats contains statistics for a target.
+type TargetStats struct {
+	// QueueLength is the queue length if any messages are queued.
+	QueueLength int
+
+	// TotalMessages is the total number of messages sent in the lifetime of the target
+	TotalMessages int64
+
+	// FailedMessages should log message count that failed to send.
+	FailedMessages int64
+}

internal/logger/target/testlogger/testlogger.go

@@ -0,0 +1,170 @@
+// Copyright (c) 2015-2023 Hanzo AI, Inc.
+//
+// This file is part of Hanzo S3 Object Storage stack
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+// Package testlogger contains an autoregistering logger that can be used to capture logging events
+// for individual tests.
+// This package should only be included by test files.
+// To enable logging for a test, use:
+//
+//	func TestSomething(t *testing.T) {
+//		defer testlogger.T.SetLogTB(t)()
+//
+// This cannot be used for parallel tests.
+package testlogger
+
+import (
+	"context"
+	"fmt"
+	"os"
+	"strings"
+	"sync/atomic"
+	"testing"
+
+	"github.com/minio/madmin-go/v3/logger/log"
+	"github.com/hanzoai/s3/internal/logger"
+	types "github.com/hanzoai/s3/internal/logger/target/loggertypes"
+)
+
+const (
+	logMessage = iota
+	errorMessage
+	fatalMessage
+)
+
+// T is the test logger.
+var T = &testLogger{}
+
+func init() {
+	logger.AddSystemTarget(context.Background(), T)
+}
+
+type testLogger struct {
+	current atomic.Pointer[testing.TB]
+	action  atomic.Int32
+}
+
+// SetLogTB will set the logger to output to tb.
+// Call the returned function to disable logging.
+func (t *testLogger) SetLogTB(tb testing.TB) func() {
+	return t.setTB(tb, logMessage)
+}
+
+// SetErrorTB will set the logger to output to tb.Error.
+// Call the returned function to disable logging.
+func (t *testLogger) SetErrorTB(tb testing.TB) func() {
+	return t.setTB(tb, errorMessage)
+}
+
+// SetFatalTB will set the logger to output to tb.Panic.
+// Call the returned function to disable logging.
+func (t *testLogger) SetFatalTB(tb testing.TB) func() {
+	return t.setTB(tb, fatalMessage)
+}
+
+func (t *testLogger) setTB(tb testing.TB, action int32) func() {
+	old := t.action.Swap(action)
+	t.current.Store(&tb)
+	return func() {
+		t.current.Store(nil)
+		t.action.Store(old)
+	}
+}
+
+func (t *testLogger) String() string {
+	tb := t.current.Load()
+	if tb != nil {
+		tbb := *tb
+		return tbb.Name()
+	}
+	return ""
+}
+
+func (t *testLogger) Endpoint() string {
+	return ""
+}
+
+func (t *testLogger) Stats() types.TargetStats {
+	return types.TargetStats{}
+}
+
+func (t *testLogger) Init(ctx context.Context) error {
+	return nil
+}
+
+func (t *testLogger) IsOnline(ctx context.Context) bool {
+	return t.current.Load() != nil
+}
+
+func (t *testLogger) Cancel() {
+	t.current.Store(nil)
+}
+
+func (t *testLogger) Send(ctx context.Context, entry any) error {
+	tb := t.current.Load()
+	var logf func(format string, args ...any)
+	if tb != nil {
+		tbb := *tb
+		tbb.Helper()
+		switch t.action.Load() {
+		case errorMessage:
+			logf = tbb.Errorf
+		case fatalMessage:
+			logf = tbb.Fatalf
+		default:
+			logf = tbb.Logf
+		}
+	} else {
+		switch t.action.Load() {
+		case errorMessage:
+			logf = func(format string, args ...any) {
+				fmt.Fprintf(os.Stderr, format+"\n", args...)
+			}
+		case fatalMessage:
+			logf = func(format string, args ...any) {
+				fmt.Fprintf(os.Stderr, format+"\n", args...)
+			}
+			defer os.Exit(1)
+		default:
+			logf = func(format string, args ...any) {
+				fmt.Fprintf(os.Stdout, format+"\n", args...)
+			}
+		}
+	}
+
+	switch v := entry.(type) {
+	case log.Entry:
+		if v.Trace == nil {
+			logf("%s: %s", v.Level, v.Message)
+		} else {
+			msg := fmt.Sprintf("%s: %+v", v.Level, v.Trace.Message)
+			for i, m := range v.Trace.Source {
+				if i == 0 && strings.Contains(m, "logger.go:") {
+					continue
+				}
+				msg += fmt.Sprintf("\n%s", m)
+			}
+			logf("%s", msg)
+		}
+	default:
+		logf("%+v (%T)", v, v)
+	}
+	return nil
+}
+
+func (t *testLogger) Type() types.TargetType {
+	return types.TargetConsole
+}