Compare commits

...
Author SHA1 Message Date
hanzo-dev a9b45cafa7 fix(logs): un-stub GET /api/v1/logs — real ClickHouse read, not {"results":[]}
getLogs returned a hardcoded empty {"results":[]} — a lie: the datastore holds
38M+ real logs over the ZAP-fed pipeline. Now it reads the most recent logs in
the query window (limit/timestampStart/timestampEnd params; default last 15m,
100 rows, max 1000), newest first, from the CONFIGURED logs table
(r.logsDB.r.logsTableV2 — no hardcoded DB, so it tracks the live schema), and
maps the rows into the classic {results:[...]} shape. New reader method
GetRecentLogs owns the table name + query (separation of concerns) and reuses the
shared GetListResultV3 row scanner (one row-reading path); the handler only parses
params + maps rows. Selects the core log columns plus service.name and the k8s
namespace/pod/container the OTel collector tags each line with. Bounds are int64
nanos + int limit (never user strings) → injection-safe. Honest: a read error
surfaces as an error, never a fabricated empty list.
2026-07-04 01:37:31 -07:00
3 changed files with 95 additions and 22 deletions
+54 -19
View File
@@ -26,8 +26,8 @@ import (
"github.com/hanzoai/o11y/pkg/valuer"
"github.com/uptrace/bun"
errorsV2 "github.com/hanzoai/o11y/pkg/errors"
"github.com/google/uuid"
errorsV2 "github.com/hanzoai/o11y/pkg/errors"
"github.com/pkg/errors"
"github.com/prometheus/prometheus/promql"
@@ -57,19 +57,19 @@ import (
const (
primaryNamespace = "datastore"
archiveNamespace = "datastore-archive"
o11yTraceDBName = "observe_traces"
observeHistoryDBName = "signoz_analytics"
o11yTraceDBName = "observe_traces"
observeHistoryDBName = "signoz_analytics"
ruleStateHistoryTableName = "distributed_rule_state_history_v0"
o11yDurationMVTable = "distributed_durationSort"
o11yUsageExplorerTable = "distributed_usage_explorer"
o11ySpansTable = "distributed_signoz_spans"
o11yErrorIndexTable = "distributed_signoz_error_index_v2"
o11yTraceTableName = "distributed_observe_index_v2"
o11yTraceLocalTableName = "observe_index_v2"
observeMetricDBName = "observe_metrics"
o11yMetadataDbName = "observe_metadata"
o11yMeterDBName = "signoz_meter"
o11yMeterSamplesName = "samples_agg_1d"
o11yDurationMVTable = "distributed_durationSort"
o11yUsageExplorerTable = "distributed_usage_explorer"
o11ySpansTable = "distributed_signoz_spans"
o11yErrorIndexTable = "distributed_signoz_error_index_v2"
o11yTraceTableName = "distributed_observe_index_v2"
o11yTraceLocalTableName = "observe_index_v2"
observeMetricDBName = "observe_metrics"
o11yMetadataDbName = "observe_metadata"
o11yMeterDBName = "signoz_meter"
o11yMeterSamplesName = "samples_agg_1d"
o11ySampleLocalTableName = "samples_v4"
o11ySampleTableName = "distributed_samples_v4"
@@ -98,12 +98,12 @@ const (
o11yTableAttributesMetadata = "distributed_attributes_metadata"
o11yLocalTableAttributesMetadata = "attributes_metadata"
o11yUpdatedMetricsMetadataLocalTable = "updated_metadata"
o11yUpdatedMetricsMetadataTable = "distributed_updated_metadata"
minTimespanForProgressiveSearch = time.Hour
minTimespanForProgressiveSearchMargin = time.Minute
maxProgressiveSteps = 4
charset = "abcdefghijklmnopqrstuvwxyz" +
o11yUpdatedMetricsMetadataLocalTable = "updated_metadata"
o11yUpdatedMetricsMetadataTable = "distributed_updated_metadata"
minTimespanForProgressiveSearch = time.Hour
minTimespanForProgressiveSearchMargin = time.Minute
maxProgressiveSteps = 4
charset = "abcdefghijklmnopqrstuvwxyz" +
"ABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789"
NANOSECOND = 1000000000
)
@@ -4556,6 +4556,41 @@ func (r *DatastoreReader) GetListResultV3(ctx context.Context, query string) ([]
}
// GetRecentLogs reads the most recent logs in [startNano, endNano], newest first,
// capped at limit, from the configured logs table (r.logsDB.r.logsTableV2) — the
// real read behind the classic GET /api/v1/logs endpoint, replacing the empty
// {"results":[]} stub. The bounds are int64 nanosecond epochs and limit is an int
// (never user strings), so the fmt.Sprintf interpolation is injection-safe. It
// selects the core log columns plus service.name and the k8s namespace/pod/
// container the OTel collector tags each line with, and reuses the shared
// GetListResultV3 row scanner (one row-reading path).
func (r *DatastoreReader) GetRecentLogs(ctx context.Context, startNano, endNano int64, limit int) ([]*v3.Row, error) {
if limit <= 0 {
limit = 100
}
if limit > 1000 {
limit = 1000
}
if endNano <= 0 {
endNano = time.Now().UnixNano()
}
if startNano <= 0 || startNano >= endNano {
startNano = endNano - int64(15*time.Minute)
}
table := r.logsDB + "." + r.logsTableV2
query := fmt.Sprintf(
"SELECT timestamp, id, trace_id, span_id, severity_text, severity_number, body, "+
"resources_string['service.name'] AS service, "+
"resource.`service.name`::String AS service_name, "+
"resource.`k8s.namespace.name`::String AS k8s_namespace, "+
"resource.`k8s.pod.name`::String AS k8s_pod, "+
"resource.`k8s.container.name`::String AS k8s_container "+
"FROM %s WHERE timestamp >= %d AND timestamp <= %d "+
"ORDER BY timestamp DESC LIMIT %d",
table, startNano, endNano, limit)
return r.GetListResultV3(ctx, query)
}
// GetHostMetricsExistenceAndEarliestTime returns (count, minFirstReportedUnixMilli, error) for the given host metric names
// from distributed_metadata. When count is 0, minFirstReportedUnixMilli is 0.
func (r *DatastoreReader) GetMetricsExistenceAndEarliestTime(ctx context.Context, metricNames []string) (uint64, uint64, error) {
+37 -3
View File
@@ -24,6 +24,7 @@ import (
"text/template"
"time"
"github.com/hanzoai/o11y"
"github.com/hanzoai/o11y/pkg/alertmanager"
errorsV2 "github.com/hanzoai/o11y/pkg/errors"
"github.com/hanzoai/o11y/pkg/http/middleware"
@@ -32,7 +33,6 @@ import (
"github.com/hanzoai/o11y/pkg/query-service/app/cloudintegrations/services"
"github.com/hanzoai/o11y/pkg/query-service/app/integrations"
"github.com/hanzoai/o11y/pkg/query-service/app/metricsexplorer"
"github.com/hanzoai/o11y"
"github.com/hanzoai/o11y/pkg/valuer"
"github.com/prometheus/prometheus/promql"
@@ -234,7 +234,7 @@ func NewAPIHandler(opts APIHandlerOpts, config o11y.Config) (*APIHandler, error)
SummaryService: summaryService,
AlertmanagerAPI: opts.AlertmanagerAPI,
LicensingAPI: opts.LicensingAPI,
O11y: opts.O11y,
O11y: opts.O11y,
QueryParserAPI: opts.QueryParserAPI,
}
@@ -4096,8 +4096,42 @@ func (aH *APIHandler) logFieldUpdate(w http.ResponseWriter, r *http.Request) {
aH.WriteJSON(w, r, field)
}
// getLogs serves the classic GET /api/v1/logs — the most recent logs over the
// query window, newest first. Reads the configured logs table via the reader
// (real ClickHouse rows), replacing the former {"results":[]} stub. Params:
// limit (default 100, max 1000), timestampStart / timestampEnd (nanosecond
// epochs; default the last 15 minutes). Honest: a read error surfaces as an
// error, never a fabricated empty list.
func (aH *APIHandler) getLogs(w http.ResponseWriter, r *http.Request) {
aH.WriteJSON(w, r, map[string]interface{}{"results": []interface{}{}})
q := r.URL.Query()
limit := 100
if v, err := strconv.Atoi(strings.TrimSpace(q.Get("limit"))); err == nil && v > 0 {
limit = v
}
end := time.Now().UnixNano()
if v, err := strconv.ParseInt(strings.TrimSpace(q.Get("timestampEnd")), 10, 64); err == nil && v > 0 {
end = v
}
start := end - int64(15*time.Minute)
if v, err := strconv.ParseInt(strings.TrimSpace(q.Get("timestampStart")), 10, 64); err == nil && v > 0 {
start = v
}
rows, err := aH.reader.GetRecentLogs(r.Context(), start, end, limit)
if err != nil {
RespondError(w, &model.ApiError{Typ: model.ErrorInternal, Err: err}, "Failed to fetch logs from the DB")
return
}
results := make([]map[string]interface{}, 0, len(rows))
for _, row := range rows {
item := map[string]interface{}{"timestamp": row.Timestamp.UnixNano()}
for k, v := range row.Data {
item[k] = v
}
results = append(results, item)
}
aH.WriteJSON(w, r, map[string]interface{}{"results": results})
}
func (aH *APIHandler) logAggregate(w http.ResponseWriter, r *http.Request) {
@@ -66,6 +66,10 @@ type Reader interface {
GetTimeSeriesResultV3(ctx context.Context, query string) ([]*v3.Series, error)
GetListResultV3(ctx context.Context, query string) ([]*v3.Row, error)
// Logs
// GetRecentLogs reads the most recent logs in [startNano, endNano] (nanosecond
// epochs), newest first, capped at limit, from the configured logs table — the
// real read behind the classic GET /api/v1/logs endpoint (un-stubbed).
GetRecentLogs(ctx context.Context, startNano, endNano int64, limit int) ([]*v3.Row, error)
GetLogFields(ctx context.Context) (*model.GetFieldsResponse, *model.ApiError)
GetLogFieldsFromNames(ctx context.Context, fieldNames []string) (*model.GetFieldsResponse, *model.ApiError)
UpdateLogField(ctx context.Context, field *model.UpdateField) *model.ApiError