hanzo/stream
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

remove reflection for Produce/Fetch (too slow) + minor tweaks

Browse Source 
Signed-off-by: moncef.abboud <moncef.abboud95@gmail.com>
This commit is contained in:
moncef.abboud
2025-04-09 17:50:13 -05:00
parent f104b7c14d
commit 089e9e66b7
9 changed files with 91 additions and 16 deletions
Download Patch File Download Diff File Expand all files Collapse all files
logging/logging.go
+6
View File
@@ -8,6 +8,7 @@ import (
// logging levels
const (
TRACE = "TRACE"
DEBUG = "DEBUG"
INFO = "INFO"
WARN = "WARN"
@@ -38,6 +39,11 @@ func Log(level, message string, a ...any) {
}
}
// Trace logs a message at TRACE level
func Trace(message string, a ...any) {
Log(TRACE, message, a...)
}
// Debug logs a message at DEBUG level
func Debug(message string, a ...any) {
Log(DEBUG, message, a...)
protocol/broker.go
+7 -2
View File
@@ -104,9 +104,10 @@ func (b *Broker) Startup() {
func (b *Broker) HandleConnection(conn net.Conn) {
defer conn.Close()
connectionAddr := conn.RemoteAddr().String()
log.Debug("Connection established with %s\n", connectionAddr)
log.Info("Connection established with %s\n", connectionAddr)
for {
startTime := time.Now()
// First, we read the length, then allocate a byte slice based on it.
// ReadFull (not Read) is used to ensure the entire request is read. Partial data would result in parsing errors
lengthBuffer := make([]byte, 4)
@@ -127,7 +128,7 @@ func (b *Broker) HandleConnection(conn net.Conn) {
}
req := serde.ParseHeader(buffer, connectionAddr)
apiKeyHandler := b.APIDispatcher(req.RequestAPIKey)
log.Debug("Received RequestAPIKey: %v | RequestAPIVersion: %v | CorrelationID: %v | Length: %v \n\n", apiKeyHandler.Name, req.RequestAPIVersion, req.CorrelationID, length)
log.Info("Received RequestAPIKey: %v | RequestAPIVersion: %v | CorrelationID: %v | Length: %v \n\n", apiKeyHandler.Name, req.RequestAPIVersion, req.CorrelationID, length)
response := apiKeyHandler.Handler(req)
_, err = conn.Write(response)
@@ -135,6 +136,10 @@ func (b *Broker) HandleConnection(conn net.Conn) {
log.Error("Error writing to connection: %v\n", err)
break
}
d := time.Now().Sub(startTime)
log.Trace("handleConnection Iteration took %v", d)
}
log.Debug("Connection with %s closed.\n", connectionAddr)
}
protocol/fetch.go
+35 -1
View File
@@ -78,9 +78,43 @@ type AbortedTransaction struct {
FirstOffset uint64
}
func decodeFetchRequest(d serde.Decoder, fetchRequest *FetchRequest) {
fetchRequest.ReplicaID = d.UInt32()
fetchRequest.MaxWaitMs = d.UInt32()
fetchRequest.MinBytes = d.UInt32()
fetchRequest.MaxBytes = d.UInt32()
fetchRequest.IsolationLevel = d.UInt8()
fetchRequest.SessionID = d.UInt32()
fetchRequest.SessionEpoch = d.UInt32()
lenTopic := int(d.CompactArrayLen())
for i := 0; i < lenTopic; i++ {
topic := FetchRequestTopic{Name: d.CompactString()}
lenPartitions := int(d.CompactArrayLen())
for j := 0; j < lenPartitions; j++ {
topic.Partitions = append(topic.Partitions, FetchRequestPartitionData{
PartitionIndex: d.UInt32(),
CurrentLeaderEpoch: d.UInt32(),
FetchOffset: d.UInt64(),
LastFetchedEpoch: d.UInt32(),
LogStartOffset: d.UInt64(),
PartitionMaxBytes: d.UInt32(),
})
d.EndStruct()
}
fetchRequest.Topics = append(fetchRequest.Topics, topic)
d.EndStruct()
}
return
}
func (b *Broker) getFetchResponse(req types.Request) []byte {
decoder := serde.NewDecoder(req.Body)
fetchRequest := decoder.Decode(&FetchRequest{}).(*FetchRequest)
fetchRequest := &FetchRequest{}
// for perf sensitive requests, we don't rely on reflection
decodeFetchRequest(decoder, fetchRequest)
log.Debug("fetchRequest %+v", fetchRequest)
numTotalRecordBytes := 0
protocol/produce.go
+23 -1
View File
@@ -58,9 +58,31 @@ type RecordError struct {
BatchIndexErrorMessage string // compact_nullable
}
func decodeProduceRequest(d serde.Decoder, produceRequest *ProduceRequest) {
produceRequest.TransactionalID = d.CompactString()
produceRequest.Acks = d.UInt16()
produceRequest.TimeoutMs = d.UInt32()
lenTopicData := int(d.CompactArrayLen())
for i := 0; i < lenTopicData; i++ {
topic := ProduceRequestTopicData{Name: d.CompactString()}
lenPartitionData := int(d.CompactArrayLen())
for j := 0; j < lenPartitionData; j++ {
topic.PartitionData = append(topic.PartitionData, ProduceRequestPartitionData{
Index: d.UInt32(), Records: d.CompactBytes(),
})
d.EndStruct()
}
produceRequest.TopicData = append(produceRequest.TopicData, topic)
d.EndStruct()
}
return
}
func (b *Broker) getProduceResponse(req types.Request) []byte {
decoder := serde.NewDecoder(req.Body)
produceRequest := decoder.Decode(&ProduceRequest{}).(*ProduceRequest)
produceRequest := &ProduceRequest{}
// for perf sensitive requests, we don't rely on reflection
decodeProduceRequest(decoder, produceRequest)
log.Debug("ProduceRequest %+v", produceRequest)
response := ProduceResponse{}
protocol/responses.go
+1 -1
View File
@@ -149,7 +149,7 @@ func (b *Broker) getListOffsetsResponse(req types.Request) []byte {
if p.Timestamp == uint64(ListOffsetsEarliestTimestamp) {
partition.Offset = state.GetPartition(t.Name, p.PartitionIndex).StartOffset()
} else if p.Timestamp == uint64(ListOffsetsLatestTimestamp) {
partition.Offset = state.GetPartition(t.Name, p.PartitionIndex).EndOffset()
partition.Offset = state.GetPartition(t.Name, p.PartitionIndex).EndOffset() + 1 // +1 to exclude the last msg
partition.Timestamp = state.GetPartition(t.Name, p.PartitionIndex).ActiveSegment().MaxTimestamp
} else {
// TODO: implement ListOffsetsMaxTimestamp
serde/serde.go
+1 -1
View File
@@ -462,7 +462,7 @@ func (d *Decoder) Uvarint() (uint64, int) {
func (d *Decoder) Varint() (int64, int) {
varint, n := binary.Varint(d.b[d.Offset:])
d.Offset += n
log.Info("Varint %v nb bytes %v", varint, n)
// log.Trace("Varint %v nb bytes %v", varint, n)
return varint, n
}
storage/log.go
+6 -2
View File
@@ -278,12 +278,16 @@ func GetPartitionDir(topic string, partition uint32) string {
// SyncPartition syncs the log and index files of the active segment for the given partition.
func SyncPartition(partition *types.Partition) error {
err := partition.ActiveSegment().LogFile.Sync()
activeSegment := partition.ActiveSegment()
if activeSegment == nil {
return fmt.Errorf("SyncPartition: partition %v-%v has no active segment", partition.TopicName, partition.Index)
}
err := activeSegment.LogFile.Sync()
if err != nil {
log.Error("Error syncing SegmentFile for %v: %v", partition, err)
return err
}
err = partition.ActiveSegment().IndexFile.Sync()
err = activeSegment.IndexFile.Sync()
if err != nil {
log.Error("Error syncing IndexFile for %v: %v", partition, err)
return err
storage/record_batch.go
+3 -3
View File
@@ -37,7 +37,7 @@ func ReadRecordBatch(b []byte) types.RecordBatch {
recordBatch.NumRecord = decoder.UInt32()
recordBatch.Records = decoder.GetRemainingBytes()
log.Debug("ReadRecordBatch recordBatch %+v", recordBatch)
log.Trace("ReadRecordBatch recordBatch %+v", recordBatch)
return recordBatch
}
@@ -106,12 +106,12 @@ func NewRecordBatch(recordKey []byte, recordValue []byte, attributes uint16) typ
rb.Records = encoder.Bytes()
if compressor := compress.GetCompressor(rb.Attributes); compressor != nil { // 1 == gzip
log.Info("NewRecordBatch recordBytes %v compressor %T", rb.Records, compressor)
log.Info("NewRecordBatch recordByte length s %v compressor %T", len(rb.Records), compressor)
compressed, err := compressor.Compress(rb.Records)
if err != nil {
log.Error("error while compressing RecordBatch %v. Error: %v", rb, err)
}
log.Info("NewRecordBatch After compression recordBytes %v", compressed)
log.Debug("NewRecordBatch After compression recordBytes length %v", len(compressed))
rb.Records = compressed
}
storage/segment.go
+9 -5
View File
@@ -165,12 +165,16 @@ func cleanupPartition(partition *types.Partition) error {
defer partition.Unlock()
activeSegment := partition.ActiveSegment()
if shouldRollSegment(activeSegment) {
log.Info("rolling segment for partition %v-%v \n", partition.TopicName, partition.Index)
err := rollPartitionSegment(partition)
if err != nil {
return fmt.Errorf("error while rolling partition %v-%v segment %v", partition.TopicName, partition.Index, err)
if activeSegment != nil {
if shouldRollSegment(activeSegment) {
log.Info("rolling segment for partition %v-%v \n", partition.TopicName, partition.Index)
err := rollPartitionSegment(partition)
if err != nil {
return fmt.Errorf("error while rolling partition %v-%v segment %v", partition.TopicName, partition.Index, err)
}
}
} else {
log.Error("cleanupPartition: partition %v-%v has no active segment", partition.TopicName, partition.Index)
}
if len(partition.Segments) > 1 {