diff --git a/go.mod b/go.mod index 20ffdb22..03cc9ef6 100644 --- a/go.mod +++ b/go.mod @@ -68,6 +68,10 @@ require ( github.com/weaviate/weaviate-go-client/v5 v5.7.3 github.com/xuri/excelize/v2 v2.11.0 github.com/yanyiwu/gojieba v1.4.7 + go.opentelemetry.io/otel v1.43.0 + go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.43.0 + go.opentelemetry.io/otel/sdk v1.43.0 + go.opentelemetry.io/otel/trace v1.43.0 go.uber.org/dig v1.19.0 golang.org/x/crypto v0.53.0 golang.org/x/mod v0.36.0 @@ -312,12 +316,9 @@ require ( go.opentelemetry.io/auto/sdk v1.2.1 // indirect go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.67.0 // indirect go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.68.0 // indirect - go.opentelemetry.io/otel v1.43.0 // indirect go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.43.0 // indirect go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.43.0 // indirect go.opentelemetry.io/otel/metric v1.43.0 // indirect - go.opentelemetry.io/otel/sdk v1.43.0 // indirect - go.opentelemetry.io/otel/trace v1.43.0 // indirect go.opentelemetry.io/proto/otlp v1.10.0 // indirect go.uber.org/atomic v1.11.0 // indirect go.uber.org/automaxprocs v1.5.3 // indirect diff --git a/go.sum b/go.sum index 5d3b9d61..c29a8cf6 100644 --- a/go.sum +++ b/go.sum @@ -2685,6 +2685,8 @@ go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.43.0/go.mod h1:Vl1/iaggsuRl go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.0.1/go.mod h1:xOvWoTOrQjxjW61xtOmD/WKGRYb/P4NzRo3bs65U6Rk= go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.43.0 h1:RAE+JPfvEmvy+0LzyUA25/SGawPwIUbZ6u0Wug54sLc= go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.43.0/go.mod h1:AGmbycVGEsRx9mXMZ75CsOyhSP6MFIcj/6dnG+vhVjk= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.43.0 h1:3iZJKlCZufyRzPzlQhUIWVmfltrXuGyfjREgGP3UUjc= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.43.0/go.mod h1:/G+nUPfhq2e+qiXMGxMwumDrP5jtzU+mWN7/sjT2rak= go.opentelemetry.io/otel/metric v1.19.0/go.mod h1:L5rUsV9kM1IxCj1MmSdS+JQAcVm319EUrDVLrt7jqt8= go.opentelemetry.io/otel/metric v1.21.0/go.mod h1:o1p3CA8nNHW8j5yuQLdc1eeqEaPfzug24uvsyIEJRWM= go.opentelemetry.io/otel/metric v1.22.0/go.mod h1:evJGjVpZv0mQ5QBRJoBF64yMuOf4xCWdXjK8pzFvliY= diff --git a/internal/logger/logger.go b/internal/logger/logger.go index 636728b6..6cf9d1d9 100644 --- a/internal/logger/logger.go +++ b/internal/logger/logger.go @@ -15,6 +15,7 @@ import ( "github.com/Tencent/WeKnora/internal/types" "github.com/sirupsen/logrus" + "go.opentelemetry.io/otel/trace" "gopkg.in/natefinch/lumberjack.v2" ) @@ -538,5 +539,15 @@ func CloneContext(ctx context.Context) context.Context { } } + // Preserve the active OpenTelemetry span across the rebuild. The Langfuse + // *Trace handle above carries the trace id, but span PARENTING flows through + // the OTel span context (trace.SpanFromContext), which CloneContext would + // otherwise drop — orphaning child spans opened after a CloneContext (e.g. + // the agent engine's agent.execute becoming a separate trace from the HTTP + // root). Re-inject the recording span so children stitch to the same trace. + if sp := trace.SpanFromContext(ctx); sp.IsRecording() { + newCtx = trace.ContextWithSpan(newCtx, sp) + } + return newCtx } diff --git a/internal/tracing/langfuse/asynq.go b/internal/tracing/langfuse/asynq.go index ad52179d..5778d952 100644 --- a/internal/tracing/langfuse/asynq.go +++ b/internal/tracing/langfuse/asynq.go @@ -7,20 +7,24 @@ import ( "github.com/Tencent/WeKnora/internal/types" "github.com/hibiken/asynq" + "go.opentelemetry.io/otel/propagation" + oteltrace "go.opentelemetry.io/otel/trace" ) -// InjectTracing stamps the current trace/span ids (and a best-effort +// InjectTracing stamps the current W3C traceparent (plus a best-effort // user/session label) from ctx onto the given payload, provided the payload // embeds types.TracingContext (and thus implements LangfuseTracingCarrier). // -// Safe to call unconditionally: when Langfuse is disabled or no trace is -// present on ctx, it writes a zero-valued TracingContext — which round-trips -// through JSON as absent fields and therefore costs nothing. +// The traceparent is produced by propagator.Inject from the active OTel span +// context (the HTTP root span opened by GinMiddleware). The asynq worker +// re-extracts it so worker-side spans are children of the same trace — +// giving LiteFuse one stitched tree across the HTTP request and the async +// processing. This also makes a sop3 run's traceparent propagate through to +// any asynq jobs WeKnora enqueues while serving sop3's agent-chat call. // -// Call sites live at every asynq.NewTask creation point. We deliberately -// keep the API synchronous and mutating (rather than returning a new payload -// copy) so that existing enqueue code needs only a single added line just -// before json.Marshal, minimizing review surface and the risk of regressions. +// Safe to call unconditionally: when Langfuse is disabled or no span is +// present on ctx, it writes a zero-valued TracingContext — which round-trips +// through JSON as absent fields and costs nothing. func InjectTracing(ctx context.Context, carrier types.LangfuseTracingCarrier) { if carrier == nil { return @@ -30,12 +34,14 @@ func InjectTracing(ctx context.Context, carrier types.LangfuseTracingCarrier) { return } tc := types.TracingContext{} + c := propagation.MapCarrier{} + propagator.Inject(ctx, c) + tc.LangfuseTraceparent = c["traceparent"] + // Backward-compat: keep LangfuseTraceID = the W3C trace id for any legacy + // reader. LangfuseParentObservationID is no longer used by the OTLP path. if trace, ok := TraceFromContext(ctx); ok && trace != nil { tc.LangfuseTraceID = trace.ID } - if obs, ok := parentObservationFromCtx(ctx); ok { - tc.LangfuseParentObservationID = obs - } tc.LangfuseUserID = userIDFromCtx(ctx) tc.LangfuseSessionID = sessionIDFromCtx(ctx) carrier.SetLangfuseTracing(tc) @@ -59,23 +65,20 @@ func peekTracingContext(payload []byte) types.TracingContext { // AsynqMiddleware is the worker-side counterpart of GinMiddleware. It: // -// 1. Parses any tracing ids embedded in the task payload and either -// resumes the originating trace (so the Langfuse UI stitches the HTTP -// request and the async processing into one tree), or — for tasks that -// came from a scheduled job with no originating HTTP request — creates -// a standalone trace tagged with the task type. +// 1. Extracts the W3C traceparent stamped onto the task payload by +// InjectTracing and resumes the originating trace (so the Langfuse UI +// stitches the HTTP request and the async processing into one tree). For +// scheduled jobs with no upstream trace it opens a standalone trace named +// after the task type. // // 2. Opens a SPAN around the handler execution so every child generation -// (embedding / VLM / chat / rerank / ASR) auto-attaches to it via -// parentObservationId. +// (embedding / VLM / chat / rerank / ASR) auto-attaches to it. // // 3. Enriches the span with asynq's own metadata: task id, queue, retry // count, payload size. // -// When the manager is disabled it degrades to a pass-through; failure of -// the Langfuse path never blocks task execution. -// -// Register it once in router/task.go via mux.Use. +// When the manager is disabled it degrades to a pass-through; failure of the +// Langfuse path never blocks task execution. Register once via mux.Use. func AsynqMiddleware() asynq.MiddlewareFunc { return func(next asynq.Handler) asynq.Handler { return asynq.HandlerFunc(func(ctx context.Context, task *asynq.Task) error { @@ -99,15 +102,16 @@ func AsynqMiddleware() asynq.MiddlewareFunc { "payload_bytes": len(task.Payload()), } - // If the upstream enqueuer stamped a trace id onto the payload, - // we graft under that trace. Otherwise (scheduled jobs, legacy - // in-flight tasks that predate this code, tests, etc.) we start - // a standalone trace named after the task type so at least the - // worker-side work is observable. + // If the upstream enqueuer stamped a traceparent, resume that trace + // (worker spans become children of the HTTP trace). Otherwise start + // a standalone trace named after the task type. var trace *Trace shouldFinishTrace := false - if tc.LangfuseTraceID != "" { - ctx, trace = mgr.ResumeTrace(ctx, tc.LangfuseTraceID, tc.LangfuseParentObservationID) + if tc.LangfuseTraceparent != "" { + ctx = propagator.Extract(ctx, propagation.MapCarrier{"traceparent": tc.LangfuseTraceparent}) + if sc := oteltrace.SpanContextFromContext(ctx); sc.IsValid() { + ctx = withTrace(ctx, &Trace{ID: sc.TraceID().String(), manager: mgr}) + } } else { ctx, trace = mgr.StartTrace(ctx, TraceOptions{ Name: "asynq." + task.Type(), @@ -153,15 +157,9 @@ func AsynqMiddleware() asynq.MiddlewareFunc { // spanInputFromPayload surfaces a compact, human-readable summary of the // task payload for the Langfuse "Input" pane. We deliberately do NOT send -// the full JSON blob because: -// -// - Manual/text-ingest payloads can be many kilobytes of prose; -// - FAQ import payloads embed the full entry list; -// - Document-process payloads contain file URLs that, while small, may -// include presigned query strings that rotate (adding diff noise). -// -// Instead we preview the first ~1KB verbatim, which matches what Langfuse -// itself would display and keeps ingestion bandwidth predictable. +// the full JSON blob because manual/text-ingest payloads can be many +// kilobytes and FAQ import payloads embed the full entry list. Instead we +// preview the first ~1KB verbatim. func spanInputFromPayload(payload []byte) interface{} { const preview = 1024 if len(payload) == 0 { @@ -177,7 +175,7 @@ func spanInputFromPayload(payload []byte) interface{} { } // userIDFromCtx mirrors middleware.extractUserID but accepts a raw context -// (no gin.Context) so both HTTP and asynq paths can share the same fallback +// (no gin.Context) so both HTTP and asynq paths share the same fallback // logic: explicit UserID → tenant: → empty. func userIDFromCtx(ctx context.Context) string { if v, ok := ctx.Value(types.UserIDContextKey).(string); ok && v != "" { diff --git a/internal/tracing/langfuse/asynq_test.go b/internal/tracing/langfuse/asynq_test.go index 89f5507d..099a8eeb 100644 --- a/internal/tracing/langfuse/asynq_test.go +++ b/internal/tracing/langfuse/asynq_test.go @@ -3,12 +3,8 @@ package langfuse import ( "context" "encoding/json" - "io" - "net/http" - "net/http/httptest" - "sync" + "strings" "testing" - "time" "github.com/Tencent/WeKnora/internal/types" "github.com/hibiken/asynq" @@ -24,249 +20,104 @@ type dummyPayload struct { // TestInjectTracing_DisabledIsZero verifies InjectTracing is a no-op when // Langfuse is disabled: no panics, no trace fields written. func TestInjectTracing_DisabledIsZero(t *testing.T) { - // Explicitly install a disabled manager to shadow any globally-enabled - // one from a previous test in the same package. _, _ = Init(Config{Enabled: false}) p := &dummyPayload{KnowledgeID: "k1"} InjectTracing(context.Background(), p) - if p.LangfuseTraceID != "" || p.LangfuseParentObservationID != "" { + if p.LangfuseTraceparent != "" || p.LangfuseTraceID != "" { t.Fatalf("expected no tracing fields on disabled manager, got %+v", p.TracingContext) } } -// TestInjectTracing_PopulatesFromContext checks that when a trace is active -// on the context, its id is copied onto the payload and a subsequent -// peekTracingContext round-trips it correctly. -func TestInjectTracing_PopulatesFromContext(t *testing.T) { - // Stand up a fake ingestion endpoint — Init starts a real flush worker - // and we don't want it to panic on connection refused during the test. - srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - _, _ = io.Copy(io.Discard, r.Body) - w.WriteHeader(200) - _, _ = w.Write([]byte(`{}`)) - })) - defer srv.Close() - - m, err := Init(Config{ - Enabled: true, - Host: srv.URL, - PublicKey: "pk", - SecretKey: "sk", - FlushAt: 16, - FlushInterval: 50 * time.Millisecond, - QueueSize: 16, - RequestTimeout: 2 * time.Second, - SampleRate: 1.0, - }) - if err != nil { - t.Fatalf("init: %v", err) - } - defer m.Shutdown(context.Background()) +// TestInjectTracing_PopulatesTraceparent checks that when a trace is active +// on the context, a W3C traceparent is stamped onto the payload (so the +// asynq worker can resume the same trace). +func TestInjectTracing_PopulatesTraceparent(t *testing.T) { + m, _ := newTestManager(t) ctx, trace := m.StartTrace(context.Background(), TraceOptions{Name: "parent"}) - ctx, span := m.StartSpan(ctx, SpanOptions{Name: "wrap"}) - if span == nil || span.ID == "" { - t.Fatalf("expected a span with id, got %+v", span) - } - p := &dummyPayload{KnowledgeID: "k1"} InjectTracing(ctx, p) + if p.LangfuseTraceparent == "" { + t.Fatal("expected LangfuseTraceparent to be populated") + } + // The traceparent carries the trace id; trace.ID is the OTel trace id. + if !strings.HasPrefix(p.LangfuseTraceparent, "00-"+trace.ID) { + t.Errorf("traceparent %q does not carry trace id %s", p.LangfuseTraceparent, trace.ID) + } if p.LangfuseTraceID != trace.ID { - t.Errorf("trace id mismatch: got %q want %q", p.LangfuseTraceID, trace.ID) - } - if p.LangfuseParentObservationID != span.ID { - t.Errorf("parent observation id mismatch: got %q want %q", p.LangfuseParentObservationID, span.ID) - } - - // Round-trip the payload through JSON, which is what asynq does on the - // wire, and make sure peekTracingContext recovers the ids. - b, err := json.Marshal(p) - if err != nil { - t.Fatalf("marshal: %v", err) - } - got := peekTracingContext(b) - if got.LangfuseTraceID != trace.ID || got.LangfuseParentObservationID != span.ID { - t.Errorf("peek round-trip lost ids: %+v", got) + t.Errorf("LangfuseTraceID = %q, want %q", p.LangfuseTraceID, trace.ID) } } -// TestAsynqMiddleware_ResumeTrace asserts the middleware grafts a resumed -// trace onto the child handler's context, and that the ingestion endpoint -// receives span-create / span-update events for the wrapper — without -// emitting a duplicate trace-create (which would split the Langfuse UI -// tree into two disconnected roots). -func TestAsynqMiddleware_ResumeTrace(t *testing.T) { - var mu sync.Mutex - var batches []ingestionRequest - srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - body, _ := io.ReadAll(r.Body) - var req ingestionRequest - _ = json.Unmarshal(body, &req) - mu.Lock() - batches = append(batches, req) - mu.Unlock() - w.WriteHeader(200) - _, _ = w.Write([]byte(`{}`)) - })) - defer srv.Close() +// TestAsynqMiddleware_TraceparentPropagation is the cross-process correlation +// core test: InjectTracing stamps a traceparent onto the payload; the worker +// middleware re-extracts it, and the worker span inherits the upstream trace +// id — stitching the HTTP trace and the async job into one LiteFuse tree. +func TestAsynqMiddleware_TraceparentPropagation(t *testing.T) { + m, exp := newTestManager(t) - m, err := Init(Config{ - Enabled: true, - Host: srv.URL, - PublicKey: "pk", - SecretKey: "sk", - FlushAt: 1, - FlushInterval: 5 * time.Millisecond, - QueueSize: 32, - RequestTimeout: 2 * time.Second, - SampleRate: 1.0, - }) - if err != nil { - t.Fatalf("init: %v", err) - } - - // Build a payload that already carries a trace id (as if the HTTP layer - // injected one at enqueue time). + // Upstream caller opens a span and injects a traceparent onto the payload. + upstreamCtx, upstreamSpan := m.Tracer().Start(context.Background(), "upstream-http") + remoteTraceID := upstreamSpan.SpanContext().TraceID() payload := &dummyPayload{KnowledgeID: "k1"} - payload.LangfuseTraceID = "trace-xyz" - payload.LangfuseParentObservationID = "parent-span-abc" + InjectTracing(upstreamCtx, payload) + if payload.LangfuseTraceparent == "" { + t.Fatal("InjectTracing did not stamp a traceparent") + } raw, _ := json.Marshal(payload) - var receivedTraceID, receivedParentID string - var sawTrace bool - handler := asynq.HandlerFunc(func(ctx context.Context, task *asynq.Task) error { - if tr, ok := TraceFromContext(ctx); ok && tr != nil { - sawTrace = true - receivedTraceID = tr.ID - } - if pid, ok := parentObservationFromCtx(ctx); ok { - receivedParentID = pid - } - return nil - }) - - mw := AsynqMiddleware() - wrapped := mw(handler) - - task := asynq.NewTask("test:type", raw) - if err := wrapped.ProcessTask(context.Background(), task); err != nil { + mw := AsynqMiddleware()(asynq.HandlerFunc(func(context.Context, *asynq.Task) error { return nil })) + if err := mw.ProcessTask(context.Background(), asynq.NewTask("test:type", raw)); err != nil { t.Fatalf("handler err: %v", err) } - if !sawTrace { - t.Fatal("expected handler ctx to carry a resumed trace") - } - if receivedTraceID != "trace-xyz" { - t.Errorf("trace id mismatch: got %q want trace-xyz", receivedTraceID) - } - if receivedParentID == "" { - t.Error("expected parent observation to be set to the wrapper span id") - } - - if err := m.Shutdown(context.Background()); err != nil { - t.Fatalf("shutdown: %v", err) - } - - mu.Lock() - defer mu.Unlock() - var sawSpanCreate, sawSpanUpdate, sawTraceCreate bool - for _, b := range batches { - for _, ev := range b.Batch { - switch ev.Type { - case "span-create": - sawSpanCreate = true - // Resumed span should attach to the supplied trace id, not - // a fresh one we made up in the worker. - body, _ := json.Marshal(ev.Body) - var ob observationBody - _ = json.Unmarshal(body, &ob) - if ob.TraceID != "trace-xyz" { - t.Errorf("span-create trace id = %q, want trace-xyz", ob.TraceID) - } - if ob.ParentObservationID != "parent-span-abc" { - t.Errorf("span-create parentObservationId = %q, want parent-span-abc", ob.ParentObservationID) - } - case "span-update": - sawSpanUpdate = true - case "trace-create": - sawTraceCreate = true - } + for _, s := range exp.GetSpans() { + if s.Name != "asynq.test:type" { + continue } + if s.SpanContext.TraceID() != remoteTraceID { + t.Errorf("worker span trace id = %s, want upstream %s (traceparent not propagated)", + s.SpanContext.TraceID(), remoteTraceID) + } + return } - if !sawSpanCreate { - t.Error("missing span-create event") - } - if !sawSpanUpdate { - t.Error("missing span-update event") - } - if sawTraceCreate { - t.Error("resumed trace should not emit trace-create events (parent owns them)") - } + t.Fatal("asynq worker span not exported") } // TestAsynqMiddleware_StandaloneTrace asserts that when the payload carries -// NO upstream trace id (e.g. a scheduled job), the middleware opens a -// standalone trace tagged with the task type, so the worker-side work -// still shows up in Langfuse. +// NO upstream traceparent (e.g. a scheduled job), the middleware opens a +// standalone trace named after the task type. func TestAsynqMiddleware_StandaloneTrace(t *testing.T) { - var mu sync.Mutex - var batches []ingestionRequest - srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - body, _ := io.ReadAll(r.Body) - var req ingestionRequest - _ = json.Unmarshal(body, &req) - mu.Lock() - batches = append(batches, req) - mu.Unlock() - w.WriteHeader(200) - _, _ = w.Write([]byte(`{}`)) - })) - defer srv.Close() - - m, err := Init(Config{ - Enabled: true, Host: srv.URL, PublicKey: "pk", SecretKey: "sk", - FlushAt: 1, FlushInterval: 5 * time.Millisecond, QueueSize: 32, - RequestTimeout: 2 * time.Second, SampleRate: 1.0, - }) - if err != nil { - t.Fatalf("init: %v", err) - } + _, exp := newTestManager(t) payload := &dummyPayload{KnowledgeID: "kX"} raw, _ := json.Marshal(payload) - handler := asynq.HandlerFunc(func(ctx context.Context, task *asynq.Task) error { return nil }) - wrapped := AsynqMiddleware()(handler) - if err := wrapped.ProcessTask(context.Background(), asynq.NewTask("scheduled:ping", raw)); err != nil { + mw := AsynqMiddleware()(asynq.HandlerFunc(func(context.Context, *asynq.Task) error { return nil })) + if err := mw.ProcessTask(context.Background(), asynq.NewTask("scheduled:ping", raw)); err != nil { t.Fatalf("handler err: %v", err) } - if err := m.Shutdown(context.Background()); err != nil { - t.Fatalf("shutdown: %v", err) - } - - mu.Lock() - defer mu.Unlock() - // Langfuse uses trace-create events both on open and on update (the - // server merges by id), so we filter to the initial event that carries - // the Name — the finalizing update has no name. - var sawNamedTraceCreate bool - for _, b := range batches { - for _, ev := range b.Batch { - if ev.Type != "trace-create" { - continue - } - body, _ := json.Marshal(ev.Body) - var tb traceBody - _ = json.Unmarshal(body, &tb) - if tb.Name == "asynq.scheduled:ping" { - sawNamedTraceCreate = true - } + // The standalone run opens a root trace span ("asynq.scheduled:ping", + // type=trace) plus a worker span with the same name (type=span). + var sawRoot, sawSpan bool + for _, s := range exp.GetSpans() { + if s.Name != "asynq.scheduled:ping" { + continue + } + switch spanType(s) { + case obsTypeTrace: + sawRoot = true + case obsTypeSpan: + sawSpan = true } } - if !sawNamedTraceCreate { - t.Error("standalone run should emit a trace-create with name=asynq.scheduled:ping") + if !sawRoot { + t.Error("standalone run should open a root trace span named asynq.scheduled:ping") + } + if !sawSpan { + t.Error("standalone run should open a worker span") } } diff --git a/internal/tracing/langfuse/client.go b/internal/tracing/langfuse/client.go deleted file mode 100644 index d8c911cc..00000000 --- a/internal/tracing/langfuse/client.go +++ /dev/null @@ -1,85 +0,0 @@ -package langfuse - -import ( - "bytes" - "context" - "encoding/base64" - "encoding/json" - "fmt" - "io" - "net/http" - "strings" -) - -// client is a thin HTTP client for the Langfuse ingestion API. -type client struct { - host string - auth string // pre-computed "Basic " - httpClient *http.Client - debug bool -} - -func newClient(cfg Config) *client { - credentials := cfg.PublicKey + ":" + cfg.SecretKey - return &client{ - host: strings.TrimRight(cfg.Host, "/"), - auth: "Basic " + base64.StdEncoding.EncodeToString([]byte(credentials)), - httpClient: &http.Client{ - Timeout: cfg.RequestTimeout, - }, - debug: cfg.Debug, - } -} - -type ingestionRequest struct { - Batch []ingestionEvent `json:"batch"` -} - -// ingest posts a batch of events to Langfuse. The API accepts partial failure: -// individual event errors are returned in a 207 response, which we surface as -// a logged debug message rather than a hard error (the batch as a whole was -// accepted). -func (c *client) ingest(ctx context.Context, events []ingestionEvent) error { - if len(events) == 0 { - return nil - } - - body, err := json.Marshal(ingestionRequest{Batch: events}) - if err != nil { - return fmt.Errorf("langfuse: marshal batch: %w", err) - } - - endpoint := c.host + "/api/public/ingestion" - req, err := http.NewRequestWithContext(ctx, http.MethodPost, endpoint, bytes.NewReader(body)) - if err != nil { - return fmt.Errorf("langfuse: build request: %w", err) - } - req.Header.Set("Content-Type", "application/json") - req.Header.Set("Authorization", c.auth) - req.Header.Set("User-Agent", "weknora-langfuse/1.0") - - resp, err := c.httpClient.Do(req) - if err != nil { - return fmt.Errorf("langfuse: ingest request failed: %w", err) - } - defer resp.Body.Close() - - respBody, _ := io.ReadAll(resp.Body) - - // 207 = "multi-status": batch accepted, some events may have failed. - // 2xx = success. Anything else is a transport/auth error worth surfacing. - if resp.StatusCode >= 200 && resp.StatusCode < 300 { - if c.debug && resp.StatusCode == http.StatusMultiStatus { - return fmt.Errorf("langfuse: partial ingest success (207): %s", truncate(string(respBody), 512)) - } - return nil - } - return fmt.Errorf("langfuse: ingest failed with status %d: %s", resp.StatusCode, truncate(string(respBody), 512)) -} - -func truncate(s string, max int) string { - if len(s) <= max { - return s - } - return s[:max] + "..." -} diff --git a/internal/tracing/langfuse/config.go b/internal/tracing/langfuse/config.go index b831a799..3c470d11 100644 --- a/internal/tracing/langfuse/config.go +++ b/internal/tracing/langfuse/config.go @@ -1,9 +1,13 @@ -// Package langfuse implements a lightweight client for the Langfuse ingestion -// API (https://langfuse.com/docs/api). It lets WeKnora record LLM traces, -// generations and token usage in Langfuse without pulling in a heavy SDK. +// Package langfuse implements WeKnora's Langfuse/LiteFuse LLM tracing client on +// top of the OpenTelemetry Go SDK. It records chat/embedding/rerank/VLM/ASR +// generations plus the surrounding agent/HTTP/asynq spans as OTLP/HTTP spans +// to a Langfuse v3+ or LiteFuse backend (POST /api/public/otel/v1/traces). // -// The integration is fully opt-in: when disabled (the default), all public -// entry points are cheap no-ops, so callers can wire them unconditionally. +// The integration is fully opt-in: when disabled (the default), every public +// entry point is a cheap no-op, so callers can wire them unconditionally. A +// W3C traceparent propagated from upstream callers (e.g. sop3) is inherited so +// WeKnora's trace shares the upstream trace id — correlating runs across +// services in the LiteFuse UI. package langfuse import ( @@ -12,6 +16,8 @@ import ( "strconv" "strings" "time" + + sdktrace "go.opentelemetry.io/otel/sdk/trace" ) // Config holds the runtime configuration for the Langfuse client. @@ -45,6 +51,13 @@ type Config struct { SampleRate float64 // Debug enables verbose logging of batch send errors. Debug bool + + // testExporter, when non-nil, replaces the real OTLP exporter. Tests use + // it to inject an in-memory span exporter (tracetest.InMemoryExporter) so + // they can assert on exported spans deterministically without an HTTP + // server. Init registers it with a SimpleSpanProcessor (synchronous + // export on span End) when set. Unexported — production never sets it. + testExporter sdktrace.SpanExporter } // LoadConfigFromEnv builds a Config by reading the LANGFUSE_* environment diff --git a/internal/tracing/langfuse/context.go b/internal/tracing/langfuse/context.go index c0df66cb..236e2c2d 100644 --- a/internal/tracing/langfuse/context.go +++ b/internal/tracing/langfuse/context.go @@ -13,17 +13,13 @@ import ( // CloneContext call would drop the trace and downstream LLM wrappers would // each auto-create their own shallow trace, fragmenting a single HTTP request // into many unrelated traces in the Langfuse UI. +// +// Span parenting is NOT carried by this key: it flows through the standard +// OpenTelemetry context (trace.SpanFromContext), which tracer.Start wires up +// automatically. The *Trace on this key is only for handlers/middleware that +// need to set trace-level input/output. var traceCtxKey = types.LangfuseTraceContextKey -// parentObsCtxKey tracks the current "parent observation" id — i.e. the span -// that encloses any generation/span started under it. Unlike traceCtxKey, this -// key is private: spans are a pure tracer concern and logger.CloneContext -// should reset the parent across request boundaries (we don't want a leftover -// span id from one request leaking into another's LLM calls). -type parentObsCtxKeyType struct{} - -var parentObsCtxKey = parentObsCtxKeyType{} - // withTrace stores a *Trace on the context so downstream LLM wrappers can // attach their generations to it. func withTrace(ctx context.Context, t *Trace) context.Context { @@ -47,21 +43,3 @@ func traceFromCtx(ctx context.Context) (*Trace, bool) { func TraceFromContext(ctx context.Context) (*Trace, bool) { return traceFromCtx(ctx) } - -// withParentObservation stores the id of the enclosing span so children -// (sub-spans or generations) can attach via parentObservationId. -func withParentObservation(ctx context.Context, id string) context.Context { - if ctx == nil || id == "" { - return ctx - } - return context.WithValue(ctx, parentObsCtxKey, id) -} - -// parentObservationFromCtx returns the enclosing span id if any. -func parentObservationFromCtx(ctx context.Context) (string, bool) { - if ctx == nil { - return "", false - } - v, ok := ctx.Value(parentObsCtxKey).(string) - return v, ok && v != "" -} diff --git a/internal/tracing/langfuse/events.go b/internal/tracing/langfuse/events.go index 7c320115..52209a9a 100644 --- a/internal/tracing/langfuse/events.go +++ b/internal/tracing/langfuse/events.go @@ -4,17 +4,9 @@ import ( "time" ) -// Langfuse ingestion API event envelope. -// https://api.reference.langfuse.com/#tag/ingestion -type ingestionEvent struct { - ID string `json:"id"` - Timestamp string `json:"timestamp"` - Type string `json:"type"` - Body interface{} `json:"body"` -} - // TokenUsage captures the input/output/total token counts reported by the -// underlying model, in Langfuse's canonical schema. +// underlying model, in Langfuse's canonical schema. Set as the value of the +// langfuse.observation.usage_details span attribute (JSON-serialized). type TokenUsage struct { Input int `json:"input,omitempty"` Output int `json:"output,omitempty"` @@ -22,45 +14,48 @@ type TokenUsage struct { Unit string `json:"unit,omitempty"` } -// traceBody mirrors the /api/public/ingestion trace-create body. -type traceBody struct { - ID string `json:"id"` - Timestamp string `json:"timestamp,omitempty"` - Name string `json:"name,omitempty"` - UserID string `json:"userId,omitempty"` - SessionID string `json:"sessionId,omitempty"` - Release string `json:"release,omitempty"` - Environment string `json:"environment,omitempty"` - Input interface{} `json:"input,omitempty"` - Output interface{} `json:"output,omitempty"` - Metadata map[string]interface{} `json:"metadata,omitempty"` - Tags []string `json:"tags,omitempty"` - Public bool `json:"public,omitempty"` -} +// Langfuse / OpenTelemetry semantic-convention attribute keys mirrored from +// the official langfuse-python v4 SDK (_client/attributes.py). These are set +// as span attributes on the OTLP wire; LiteFuse (and Langfuse v3+) index +// traces/generations off them. +const ( + attrObsType = "langfuse.observation.type" + attrObsInput = "langfuse.observation.input" + attrObsOutput = "langfuse.observation.output" + attrObsMetadata = "langfuse.observation.metadata" + attrObsLevel = "langfuse.observation.level" + attrObsStatusMessage = "langfuse.observation.status_message" + attrObsModel = "langfuse.observation.model.name" + attrObsModelParams = "langfuse.observation.model.parameters" + attrObsUsageDetails = "langfuse.observation.usage_details" + attrObsCompletionStart = "langfuse.observation.completion_start_time" + attrTraceName = "langfuse.trace.name" + attrTraceInput = "langfuse.trace.input" + attrTraceOutput = "langfuse.trace.output" + attrTraceMetadata = "langfuse.trace.metadata" + attrTraceTags = "langfuse.trace.tags" + attrUserID = "user.id" + attrSessionID = "session.id" + attrEnvironment = "langfuse.environment" + attrRelease = "langfuse.release" + attrLangfusePubKey = "langfuse.public.key" -// observationBody is shared between span-create / generation-create / -// span-update / generation-update events (different fields are populated -// depending on the event type). -type observationBody struct { - ID string `json:"id,omitempty"` - TraceID string `json:"traceId,omitempty"` - ParentObservationID string `json:"parentObservationId,omitempty"` - Type string `json:"type,omitempty"` // SPAN, GENERATION, EVENT - Name string `json:"name,omitempty"` - StartTime string `json:"startTime,omitempty"` - EndTime string `json:"endTime,omitempty"` - Input interface{} `json:"input,omitempty"` - Output interface{} `json:"output,omitempty"` - Metadata map[string]interface{} `json:"metadata,omitempty"` - Level string `json:"level,omitempty"` // DEFAULT | ERROR | WARNING - StatusMessage string `json:"statusMessage,omitempty"` // free-form + // The LiteFuse/Langfuse v3 OTel gate keys the "events_full" + // direct-write path on the instrumentation scope name. langfuse-python + // v4 uses "langfuse-sdk"; LiteFuse's getSdkInfoFromResourceSpans only + // requires the scope name to contain "langfuse" for server-side SDK + // classification. The actual gate is the x-langfuse-ingestion-version:4 + // HTTP header (see exporter.go). + langfuseScopeName = "langfuse-sdk" + langfuseScopeVersion = "4.0.0" +) - // Generation-specific fields - Model string `json:"model,omitempty"` - ModelParameters map[string]interface{} `json:"modelParameters,omitempty"` - Usage *TokenUsage `json:"usage,omitempty"` - CompletionStart string `json:"completionStartTime,omitempty"` -} +// Observation types carried by the langfuse.observation.type attribute. +const ( + obsTypeTrace = "trace" + obsTypeSpan = "span" + obsTypeGeneration = "generation" +) func isoTime(t time.Time) string { return t.UTC().Format("2006-01-02T15:04:05.000Z") diff --git a/internal/tracing/langfuse/exporter.go b/internal/tracing/langfuse/exporter.go new file mode 100644 index 00000000..614b7001 --- /dev/null +++ b/internal/tracing/langfuse/exporter.go @@ -0,0 +1,41 @@ +package langfuse + +import ( + "context" + "encoding/base64" + "fmt" + "strings" + + "go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp" + sdktrace "go.opentelemetry.io/otel/sdk/trace" +) + +// newExporter builds an OTLP/HTTP trace exporter pointed at the +// Langfuse v3+ / LiteFuse OTel endpoint (POST /api/public/otel/v1/traces). +// +// Auth is HTTP Basic (public_key:secret_key). The x-langfuse-ingestion-version +// header is the gate opt-in that LiteFuse/Langfuse v3 require for the OTel +// direct-write path (verified against directWriteHelpers.ts — without it the +// server returns 400 "requires Python SDK >= 4.0.0"). The x-langfuse-sdk-* +// markers are sent for parity; WeKnora is a Go client, not the Python SDK. +func newExporter(ctx context.Context, cfg Config) (sdktrace.SpanExporter, error) { + endpoint := strings.TrimRight(cfg.Host, "/") + "/api/public/otel/v1/traces" + creds := cfg.PublicKey + ":" + cfg.SecretKey + opts := []otlptracehttp.Option{ + otlptracehttp.WithEndpointURL(endpoint), + otlptracehttp.WithHeaders(map[string]string{ + "Authorization": "Basic " + base64.StdEncoding.EncodeToString([]byte(creds)), + "x-langfuse-ingestion-version": "4", + "x-langfuse-sdk-name": "python", + "x-langfuse-sdk-version": langfuseScopeVersion, + }), + } + if cfg.RequestTimeout > 0 { + opts = append(opts, otlptracehttp.WithTimeout(cfg.RequestTimeout)) + } + exp, err := otlptracehttp.New(ctx, opts...) + if err != nil { + return nil, fmt.Errorf("langfuse: build otlp exporter: %w", err) + } + return exp, nil +} diff --git a/internal/tracing/langfuse/manager.go b/internal/tracing/langfuse/manager.go index 486be7d7..6af38769 100644 --- a/internal/tracing/langfuse/manager.go +++ b/internal/tracing/langfuse/manager.go @@ -2,31 +2,42 @@ package langfuse import ( "context" - "math/rand" "sync" "sync/atomic" - "time" "github.com/Tencent/WeKnora/internal/logger" - "github.com/google/uuid" + "go.opentelemetry.io/otel" + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/propagation" + "go.opentelemetry.io/otel/sdk/resource" + sdktrace "go.opentelemetry.io/otel/sdk/trace" + "go.opentelemetry.io/otel/trace" + "go.opentelemetry.io/otel/trace/noop" ) +// propagator is the W3C TraceContext propagator used to extract/inject +// traceparent across process boundaries (HTTP requests from sop3, asynq +// payloads). It is a package-level value rather than the global OTel +// propagator so tests remain isolated and Init never mutates global state. +var propagator = propagation.TraceContext{} + // Manager is the public façade of the langfuse package. A singleton is // installed via Init(); callers should treat a nil *Manager as "disabled" // and still invoke methods — every public method tolerates a nil receiver. +// +// Internally the manager owns an OpenTelemetry TracerProvider backed by an +// OTLP/HTTP exporter pointing at the Langfuse v3+ / LiteFuse OTel endpoint. +// The handles (*Trace / *Span / *Generation) wrap OTel spans; spans are +// buffered by the BatchSpanProcessor and exported complete on End, so there +// is no per-flush-batch duplication of root spans (the bug the legacy +// hand-rolled translator had on long traces spanning multiple flushes). type Manager struct { - cfg Config - client *client + cfg Config - queue chan ingestionEvent - done chan struct{} - workerWG sync.WaitGroup - closed atomic.Bool + tp *sdktrace.TracerProvider + tracer trace.Tracer - // rng is used for sampling decisions. Guarded by rngMu because - // math/rand.Source isn't goroutine-safe. - rngMu sync.Mutex - rng *rand.Rand + closed atomic.Bool } var ( @@ -43,12 +54,48 @@ func Init(cfg Config) (*Manager, error) { } m := &Manager{cfg: cfg} if cfg.Enabled { - m.client = newClient(cfg) - m.queue = make(chan ingestionEvent, cfg.QueueSize) - m.done = make(chan struct{}) - m.rng = rand.New(rand.NewSource(time.Now().UnixNano())) - m.workerWG.Add(1) - go m.runWorker() + resAttrs := []attribute.KeyValue{ + attribute.String("service.name", "weknora"), + attribute.String(attrLangfusePubKey, cfg.PublicKey), + } + if cfg.Environment != "" { + resAttrs = append(resAttrs, attribute.String(attrEnvironment, cfg.Environment)) + } + if cfg.Release != "" { + resAttrs = append(resAttrs, attribute.String(attrRelease, cfg.Release)) + } + res, err := resource.New(context.Background(), resource.WithAttributes(resAttrs...)) + if err != nil { + return nil, err + } + var sp sdktrace.SpanProcessor + if cfg.testExporter != nil { + // Test mode: synchronous export on span End (deterministic). + sp = sdktrace.NewSimpleSpanProcessor(cfg.testExporter) + } else { + exp, err := newExporter(context.Background(), cfg) + if err != nil { + return nil, err + } + sp = sdktrace.NewBatchSpanProcessor(exp, + sdktrace.WithBatchTimeout(cfg.FlushInterval), + sdktrace.WithMaxExportBatchSize(cfg.FlushAt), + sdktrace.WithMaxQueueSize(cfg.QueueSize), + ) + } + m.tp = sdktrace.NewTracerProvider( + sdktrace.WithResource(res), + sdktrace.WithSpanProcessor(sp), + sdktrace.WithSampler(sdktrace.ParentBased(sdktrace.TraceIDRatioBased(cfg.SampleRate))), + ) + m.tracer = m.tp.Tracer(langfuseScopeName, + trace.WithInstrumentationVersion(langfuseScopeVersion), + trace.WithInstrumentationAttributes(attribute.String("public_key", cfg.PublicKey)), + ) + // Make the W3C propagator available to any OTel-aware HTTP client + // in the process. Harmless to set globally; extraction/injection in + // this package use the package-level `propagator` directly. + otel.SetTextMapPropagator(propagator) } globalMu.Lock() @@ -57,7 +104,7 @@ func Init(cfg Config) (*Manager, error) { if cfg.Enabled { logger.Infof(context.Background(), - "[Langfuse] enabled host=%s flush_at=%d flush_interval=%s sample_rate=%.2f", + "[Langfuse] enabled host=%s flush_at=%d flush_interval=%s sample_rate=%.2f (OTLP/OTel SDK)", cfg.Host, cfg.FlushAt, cfg.FlushInterval, cfg.SampleRate, ) } @@ -72,114 +119,29 @@ func GetManager() *Manager { return global } -// Enabled reports whether the manager would actually emit events. +// Enabled reports whether the manager would actually emit spans. func (m *Manager) Enabled() bool { - return m != nil && m.cfg.Enabled && !m.closed.Load() + return m != nil && m.cfg.Enabled && !m.closed.Load() && m.tp != nil } -// Shutdown drains pending events, signals the worker to stop and waits for it. -// Safe to call multiple times. +// Tracer exposes the OTel tracer so middleware can create spans directly +// when needed (e.g. extracting a remote traceparent). Returns a no-op tracer +// when disabled. +func (m *Manager) Tracer() trace.Tracer { + if !m.Enabled() { + return noop.NewTracerProvider().Tracer(langfuseScopeName) + } + return m.tracer +} + +// Shutdown flushes pending spans and releases the exporter. Safe to call +// multiple times. func (m *Manager) Shutdown(ctx context.Context) error { - if m == nil || !m.cfg.Enabled { + if m == nil || !m.cfg.Enabled || m.tp == nil { return nil } if !m.closed.CompareAndSwap(false, true) { return nil } - close(m.done) - - doneCh := make(chan struct{}) - go func() { - m.workerWG.Wait() - close(doneCh) - }() - select { - case <-doneCh: - return nil - case <-ctx.Done(): - return ctx.Err() - } -} - -// enqueue drops silently when either disabled, full or closed. Langfuse is -// observability, not business logic — back-pressure or failures must never -// block the request path. -func (m *Manager) enqueue(ev ingestionEvent) { - if !m.Enabled() { - return - } - select { - case m.queue <- ev: - default: - if m.cfg.Debug { - logger.Warnf(context.Background(), "[Langfuse] queue full, dropping event type=%s", ev.Type) - } - } -} - -// sample decides whether to emit based on SampleRate. Sampling is applied once -// per trace; observations attached to an already-sampled trace are always kept -// (Langfuse itself would drop orphaned observations anyway). -func (m *Manager) sample() bool { - if !m.Enabled() { - return false - } - if m.cfg.SampleRate >= 1.0 { - return true - } - m.rngMu.Lock() - defer m.rngMu.Unlock() - return m.rng.Float64() < m.cfg.SampleRate -} - -// runWorker batches queued events and flushes them either when the batch -// reaches FlushAt, when FlushInterval elapses, or when the manager shuts down. -func (m *Manager) runWorker() { - defer m.workerWG.Done() - ticker := time.NewTicker(m.cfg.FlushInterval) - defer ticker.Stop() - - buf := make([]ingestionEvent, 0, m.cfg.FlushAt) - flush := func(reason string) { - if len(buf) == 0 { - return - } - ctx, cancel := context.WithTimeout(context.Background(), m.cfg.RequestTimeout) - defer cancel() - if err := m.client.ingest(ctx, buf); err != nil && m.cfg.Debug { - logger.Warnf(ctx, "[Langfuse] flush (%s) failed: %v", reason, err) - } - buf = buf[:0] - } - - for { - select { - case ev := <-m.queue: - buf = append(buf, ev) - if len(buf) >= m.cfg.FlushAt { - flush("batch-full") - } - case <-ticker.C: - flush("interval") - case <-m.done: - // Drain whatever remains in the queue before exiting. - for { - select { - case ev := <-m.queue: - buf = append(buf, ev) - if len(buf) >= m.cfg.FlushAt { - flush("batch-full-on-shutdown") - } - default: - flush("shutdown") - return - } - } - } - } -} - -// newID returns a Langfuse-compatible UUIDv4. -func newID() string { - return uuid.New().String() + return m.tp.Shutdown(ctx) } diff --git a/internal/tracing/langfuse/manager_test.go b/internal/tracing/langfuse/manager_test.go index 2d5999d0..1f485f94 100644 --- a/internal/tracing/langfuse/manager_test.go +++ b/internal/tracing/langfuse/manager_test.go @@ -2,17 +2,12 @@ package langfuse import ( "context" - "encoding/json" - "io" - "net/http" - "net/http/httptest" - "sync" "testing" - "time" ) // TestManager_DisabledIsNoop verifies that when the manager is disabled the -// public API is safe to call and produces no side effects. +// public API is safe to call and produces no side effects (no spans exported, +// no panics). func TestManager_DisabledIsNoop(t *testing.T) { m, err := Init(Config{Enabled: false}) if err != nil { @@ -31,100 +26,3 @@ func TestManager_DisabledIsNoop(t *testing.T) { t.Fatalf("shutdown: %v", err) } } - -// TestManager_FullRoundTrip boots a fake Langfuse server, runs a trace + -// generation through the manager, and asserts the ingested payload contains -// the expected ids, model name and usage. -func TestManager_FullRoundTrip(t *testing.T) { - var mu sync.Mutex - var batches []ingestionRequest - - srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - if r.URL.Path != "/api/public/ingestion" { - t.Errorf("unexpected path: %s", r.URL.Path) - w.WriteHeader(404) - return - } - if auth := r.Header.Get("Authorization"); auth == "" { - t.Errorf("missing Authorization header") - } - body, _ := io.ReadAll(r.Body) - var req ingestionRequest - if err := json.Unmarshal(body, &req); err != nil { - t.Errorf("decode body: %v", err) - w.WriteHeader(400) - return - } - mu.Lock() - batches = append(batches, req) - mu.Unlock() - w.WriteHeader(200) - w.Write([]byte(`{}`)) - })) - defer srv.Close() - - m, err := Init(Config{ - Enabled: true, - Host: srv.URL, - PublicKey: "pk", - SecretKey: "sk", - FlushAt: 1, - FlushInterval: 10 * time.Millisecond, - QueueSize: 16, - RequestTimeout: 2 * time.Second, - SampleRate: 1.0, - }) - if err != nil { - t.Fatalf("init: %v", err) - } - - ctx, trace := m.StartTrace(context.Background(), TraceOptions{ - Name: "test.trace", - UserID: "user-42", - }) - _, gen := m.StartGeneration(ctx, GenerationOptions{ - Name: "chat.completion", - Model: "gpt-test", - Input: []map[string]string{{"role": "user", "content": "hi"}}, - }) - gen.Finish("hello", &TokenUsage{Input: 10, Output: 20, Total: 30, Unit: "TOKENS"}, nil) - trace.Finish("hello", nil) - - if err := m.Shutdown(context.Background()); err != nil { - t.Fatalf("shutdown: %v", err) - } - - mu.Lock() - defer mu.Unlock() - - // We should have received at least: - // trace-create, generation-create, generation-update, trace-create(update) - // possibly split across multiple HTTP calls depending on batching. - var events []ingestionEvent - for _, b := range batches { - events = append(events, b.Batch...) - } - if len(events) < 4 { - t.Fatalf("expected >=4 events, got %d: %+v", len(events), events) - } - - var sawGenerationUpdate bool - for _, ev := range events { - if ev.Type != "generation-update" { - continue - } - b, _ := json.Marshal(ev.Body) - var body observationBody - _ = json.Unmarshal(b, &body) - if body.Usage == nil || body.Usage.Total != 30 { - t.Errorf("expected usage total=30, got %+v", body.Usage) - } - if body.TraceID != trace.ID { - t.Errorf("generation trace id mismatch: got %s want %s", body.TraceID, trace.ID) - } - sawGenerationUpdate = true - } - if !sawGenerationUpdate { - t.Fatalf("no generation-update event found") - } -} diff --git a/internal/tracing/langfuse/middleware.go b/internal/tracing/langfuse/middleware.go index ed2ec6df..72288e89 100644 --- a/internal/tracing/langfuse/middleware.go +++ b/internal/tracing/langfuse/middleware.go @@ -7,6 +7,7 @@ import ( "github.com/Tencent/WeKnora/internal/types" "github.com/gin-gonic/gin" + "go.opentelemetry.io/otel/propagation" ) // GinMiddleware returns a Gin handler that opens a Langfuse trace for each @@ -25,6 +26,13 @@ func GinMiddleware() gin.HandlerFunc { } ctx := c.Request.Context() + // Extract a W3C traceparent from the incoming request so the WeKnora + // trace inherits the upstream caller's trace id. This is what lets a + // sop3 pipeline run (identified by its W3C trace_id) and the WeKnora + // agent-chat call it triggers land under the same trace in LiteFuse. + // When no traceparent is present (human UI calls, other clients) the + // root span starts a fresh trace as before. + ctx = propagator.Extract(ctx, propagation.HeaderCarrier(c.Request.Header)) userID := extractUserID(ctx) sessionID := extractSessionID(c) @@ -49,11 +57,9 @@ func GinMiddleware() gin.HandlerFunc { c.Next() trace.Finish(map[string]interface{}{ - "status": c.Writer.Status(), - }, map[string]interface{}{ - "http.status_code": c.Writer.Status(), - "response.size": c.Writer.Size(), - }) + "status": c.Writer.Status(), + "response.size": c.Writer.Size(), + }, nil) } } diff --git a/internal/tracing/langfuse/tracer.go b/internal/tracing/langfuse/tracer.go index b0b9651f..207af729 100644 --- a/internal/tracing/langfuse/tracer.go +++ b/internal/tracing/langfuse/tracer.go @@ -2,41 +2,44 @@ package langfuse import ( "context" + "encoding/json" "time" + + "github.com/Tencent/WeKnora/internal/logger" + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/codes" + "go.opentelemetry.io/otel/trace" ) // Trace represents an active root observation. A Trace is conceptually one // "request" (e.g. a chat turn). Generations and spans attached to it roll up -// as children in the Langfuse UI. +// as children in the Langfuse UI. It wraps an OpenTelemetry root span; its +// ID is the OTel trace id (W3C 32-hex), which — when the request carried a +// traceparent header — is the upstream caller's trace id (sop3 correlation). type Trace struct { ID string + span trace.Span manager *Manager - sampled bool } -// Generation represents a single model invocation (LLM / embedding / VLM). +// Generation represents a single model invocation (LLM / embedding / VLM / ASR). type Generation struct { - ID string - TraceID string - ParentObservationID string - manager *Manager - sampled bool - startTime time.Time - model string - name string + ID string + span trace.Span + manager *Manager + model string + name string } // Span represents a logical unit of work that isn't itself an LLM call — for // example an asynq task execution, a pipeline stage, or a document-processing -// step. Generations and nested spans attach to it via parentObservationId. +// step. Generations and nested spans attach as children via the OTel span +// context (parenting is automatic through trace.SpanFromContext). type Span struct { - ID string - TraceID string - ParentObservationID string - manager *Manager - sampled bool - startTime time.Time - name string + ID string + span trace.Span + manager *Manager + name string } // TraceOptions configures a new trace. @@ -67,268 +70,223 @@ type SpanOptions struct { Metadata map[string]interface{} } -// StartTrace opens a new trace, stores its ID in the returned ctx, and returns -// a handle callers can finish with FinishTrace. When the manager is disabled -// or sampling excludes the trace, the returned *Trace is non-nil but all -// methods are no-ops so callers don't need nil checks. +// StartTrace opens a root span. When ctx carries a remote SpanContext (from a +// W3C traceparent extracted by GinMiddleware), the root span inherits the +// upstream trace id — this is what makes a sop3 run and its WeKnora call land +// under the same trace in LiteFuse. The returned *Trace is non-nil even when +// disabled (methods are no-ops), so callers don't need nil checks. func (m *Manager) StartTrace(ctx context.Context, opts TraceOptions) (context.Context, *Trace) { - if m == nil || !m.cfg.Enabled { - return ctx, &Trace{} + if !m.Enabled() { + return ctx, &Trace{manager: m} } - sampled := m.sample() - id := newID() - t := &Trace{ID: id, manager: m, sampled: sampled} - - if sampled { - env := opts.Environment - if env == "" { - env = m.cfg.Environment - } - release := opts.Release - if release == "" { - release = m.cfg.Release - } - body := traceBody{ - ID: id, - Timestamp: isoTime(time.Now()), - Name: opts.Name, - UserID: opts.UserID, - SessionID: opts.SessionID, - Input: opts.Input, - Metadata: opts.Metadata, - Tags: opts.Tags, - Environment: env, - Release: release, - } - m.enqueue(ingestionEvent{ - ID: newID(), - Timestamp: isoTime(time.Now()), - Type: "trace-create", - Body: body, - }) + name := opts.Name + attrs := []attribute.KeyValue{attribute.String(attrObsType, obsTypeTrace)} + if opts.Name != "" { + attrs = append(attrs, attribute.String(attrTraceName, opts.Name)) } + if opts.UserID != "" { + attrs = append(attrs, attribute.String(attrUserID, opts.UserID)) + } + if opts.SessionID != "" { + attrs = append(attrs, attribute.String(attrSessionID, opts.SessionID)) + } + env := opts.Environment + if env == "" { + env = m.cfg.Environment + } + if env != "" { + attrs = append(attrs, attribute.String(attrEnvironment, env)) + } + rel := opts.Release + if rel == "" { + rel = m.cfg.Release + } + if rel != "" { + attrs = append(attrs, attribute.String(attrRelease, rel)) + } + attrs = append(attrs, jsonAttr(attrTraceInput, opts.Input)) + attrs = append(attrs, jsonAttr(attrTraceMetadata, opts.Metadata)) + if len(opts.Tags) > 0 { + attrs = append(attrs, jsonAttr(attrTraceTags, opts.Tags)) + } + ctx, span := m.tracer.Start(ctx, name, trace.WithTimestamp(time.Now()), trace.WithAttributes(attrs...)) + t := &Trace{ID: span.SpanContext().TraceID().String(), span: span, manager: m} return withTrace(ctx, t), t } // Finish updates the trace with its final output. Safe to call on a disabled -// trace (no-op). +// trace (no-op). The trace's metadata attribute (request_id, http.method, +// etc.) is set once at StartTrace and left untouched here — overwriting it +// with the finish metadata would lose the open's correlation fields. func (t *Trace) Finish(output interface{}, metadata map[string]interface{}) { - if t == nil || t.manager == nil || !t.sampled { + _ = metadata + if t == nil || t.manager == nil || !t.manager.Enabled() || t.span == nil { return } - // "trace-create" events are also used to update traces in Langfuse — - // the server merges repeated events by ID. See the ingestion API docs. - body := traceBody{ - ID: t.ID, - Output: output, - Metadata: metadata, - } - t.manager.enqueue(ingestionEvent{ - ID: newID(), - Timestamp: isoTime(time.Now()), - Type: "trace-create", - Body: body, - }) + t.span.SetAttributes(jsonAttr(attrTraceOutput, output)) + t.span.End() } -// ResumeTrace reconstructs a *Trace handle from an upstream-provided trace id -// (and optional parent observation id), without emitting a new trace-create -// event. Used by the asynq middleware to graft async work onto the HTTP-level -// trace that originated it: the HTTP layer already issued trace-create, and -// the worker only needs to add child observations. -// -// When traceID is empty the returned *Trace is nil, signalling the caller -// should fall back to StartTrace if it wants a standalone root. -func (m *Manager) ResumeTrace(ctx context.Context, traceID, parentObservationID string) (context.Context, *Trace) { - if m == nil || !m.cfg.Enabled || traceID == "" { +// ResumeTrace reconstructs a *Trace handle from an externally-provided W3C +// trace id (and optional parent span id), without creating a new root span — +// the originating process (e.g. an HTTP request that already opened a trace) +// owns the root. Used to graft async work onto an existing trace: it sets a +// remote SpanContext on ctx so any child span/generation started under it +// inherits the upstream trace id. When traceID is empty the returned *Trace +// is nil, signalling the caller should fall back to StartTrace. +func (m *Manager) ResumeTrace(ctx context.Context, traceID, parentSpanID string) (context.Context, *Trace) { + if m == nil || !m.Enabled() || traceID == "" { return ctx, nil } - t := &Trace{ID: traceID, manager: m, sampled: true} - ctx = withTrace(ctx, t) - if parentObservationID != "" { - ctx = withParentObservation(ctx, parentObservationID) + tid, err := trace.TraceIDFromHex(traceID) + if err != nil { + // Not a W3C 32-hex trace id (legacy UUID, etc.); cannot resume. + return ctx, nil } - return ctx, t + var sid trace.SpanID + if parentSpanID != "" { + if s, err := trace.SpanIDFromHex(parentSpanID); err == nil { + sid = s + } + } + sc := trace.NewSpanContext(trace.SpanContextConfig{ + TraceID: tid, + SpanID: sid, + TraceFlags: trace.FlagsSampled, + Remote: true, + }) + ctx = trace.ContextWithRemoteSpanContext(ctx, sc) + t := &Trace{ID: traceID, manager: m} + return withTrace(ctx, t), t } -// StartSpan opens a SPAN observation under the trace (and optional parent -// observation) carried by ctx. When no trace is present, a shallow trace is -// auto-created, mirroring StartGeneration's behaviour. Returns a ctx whose -// parentObservationId is set to this span's id, so any nested span or -// generation will properly attach as a child. +// reestablishParentSpan re-injects the active trace's root span as the OTel +// parent when ctx carries a *Trace but no active OTel span. This happens when +// a context rebuild drops the OTel span while the *Trace handle survives on +// the exported key (e.g. a background goroutine derived from a non-request +// context, or a CloneContext that predates/missed the span fix). Without +// this, child spans (e.g. a summary generation) start a fresh root and orphan +// off the HTTP trace. +func (m *Manager) reestablishParentSpan(ctx context.Context) context.Context { + if !m.Enabled() { + return ctx + } + if sp := trace.SpanFromContext(ctx); sp.IsRecording() { + return ctx // already has an active span + } + if t, ok := traceFromCtx(ctx); ok && t != nil && t.span != nil { + return trace.ContextWithSpan(ctx, t.span) + } + return ctx +} + +// StartSpan opens a child span under the trace/span carried by ctx. When no +// trace is present, OTel creates a fresh root (mirroring StartGeneration's +// auto-trace behaviour). Returns a ctx whose active span is this span. func (m *Manager) StartSpan(ctx context.Context, opts SpanOptions) (context.Context, *Span) { - if m == nil || !m.cfg.Enabled { - return ctx, &Span{} + if !m.Enabled() { + return ctx, &Span{manager: m} } - trace, ok := traceFromCtx(ctx) - if !ok || trace == nil { - newCtx, t := m.StartTrace(ctx, TraceOptions{Name: opts.Name}) + ctx = m.reestablishParentSpan(ctx) + if _, ok := traceFromCtx(ctx); !ok { + // No active trace: open a shallow root so the span isn't orphaned. + newCtx, _ := m.StartTrace(ctx, TraceOptions{Name: opts.Name}) ctx = newCtx - trace = t } - if !trace.sampled { - return ctx, &Span{} + attrs := []attribute.KeyValue{ + attribute.String(attrObsType, obsTypeSpan), + jsonAttr(attrObsInput, opts.Input), + jsonAttr(attrObsMetadata, opts.Metadata), } - parent, _ := parentObservationFromCtx(ctx) - now := time.Now() - s := &Span{ - ID: newID(), - TraceID: trace.ID, - ParentObservationID: parent, - manager: m, - sampled: true, - startTime: now, - name: opts.Name, - } - body := observationBody{ - ID: s.ID, - TraceID: s.TraceID, - ParentObservationID: parent, - Type: "SPAN", - Name: opts.Name, - StartTime: isoTime(now), - Input: opts.Input, - Metadata: opts.Metadata, - } - m.enqueue(ingestionEvent{ - ID: newID(), - Timestamp: isoTime(now), - Type: "span-create", - Body: body, - }) - ctx = withParentObservation(ctx, s.ID) - return ctx, s + ctx, span := m.tracer.Start(ctx, opts.Name, trace.WithTimestamp(time.Now()), trace.WithAttributes(attrs...)) + return ctx, &Span{ID: span.SpanContext().SpanID().String(), span: span, manager: m, name: opts.Name} } // Finish updates a span with its final output, extra metadata and any error. -// A non-nil err marks the span as ERROR level in Langfuse. +// A non-nil err marks the span as ERROR. The span's metadata attribute is set +// once at StartSpan and left untouched here. func (s *Span) Finish(output interface{}, metadata map[string]interface{}, err error) { - if s == nil || s.manager == nil || !s.sampled { + _ = metadata + if s == nil || s.manager == nil || !s.manager.Enabled() || s.span == nil { return } - level := "DEFAULT" - var statusMsg string + s.span.SetAttributes(jsonAttr(attrObsOutput, output)) if err != nil { - level = "ERROR" - statusMsg = err.Error() + s.span.RecordError(err) + s.span.SetStatus(codes.Error, err.Error()) } - body := observationBody{ - ID: s.ID, - TraceID: s.TraceID, - Type: "SPAN", - EndTime: isoTime(time.Now()), - Output: output, - Metadata: metadata, - Level: level, - StatusMessage: statusMsg, - } - s.manager.enqueue(ingestionEvent{ - ID: newID(), - Timestamp: isoTime(time.Now()), - Type: "span-update", - Body: body, - }) + s.span.End() } // StartGeneration opens a generation observation under the trace carried by -// ctx (or a newly auto-created trace if none is present). If a parent span -// is present on ctx, the generation attaches under it via parentObservationId -// so the Langfuse tree shows: trace → span → generation. +// ctx (or a newly auto-created trace). If a parent span is present on ctx, +// the generation attaches under it via the OTel span context. func (m *Manager) StartGeneration(ctx context.Context, opts GenerationOptions) (context.Context, *Generation) { - if m == nil || !m.cfg.Enabled { - return ctx, &Generation{} + if !m.Enabled() { + return ctx, &Generation{manager: m, model: opts.Model, name: opts.Name} } - // If the caller hasn't opened a trace yet, create a shallow auto-trace so - // the generation has a parent. This keeps single-shot internal callers - // (e.g. test connections) observable. - trace, ok := traceFromCtx(ctx) - if !ok || trace == nil { - newCtx, t := m.StartTrace(ctx, TraceOptions{Name: opts.Name}) + ctx = m.reestablishParentSpan(ctx) + if _, ok := traceFromCtx(ctx); !ok { + newCtx, _ := m.StartTrace(ctx, TraceOptions{Name: opts.Name}) ctx = newCtx - trace = t } - if !trace.sampled { - return ctx, &Generation{} + attrs := []attribute.KeyValue{ + attribute.String(attrObsType, obsTypeGeneration), + attribute.String(attrObsModel, opts.Model), + jsonAttr(attrObsInput, opts.Input), + jsonAttr(attrObsMetadata, opts.Metadata), + jsonAttr(attrObsModelParams, opts.ModelParameters), } - parent, _ := parentObservationFromCtx(ctx) - now := time.Now() + ctx, span := m.tracer.Start(ctx, opts.Name, trace.WithTimestamp(time.Now()), trace.WithAttributes(attrs...)) g := &Generation{ - ID: newID(), - TraceID: trace.ID, - ParentObservationID: parent, - manager: m, - sampled: true, - startTime: now, - model: opts.Model, - name: opts.Name, + ID: span.SpanContext().SpanID().String(), + span: span, + manager: m, + model: opts.Model, + name: opts.Name, } - body := observationBody{ - ID: g.ID, - TraceID: g.TraceID, - ParentObservationID: parent, - Type: "GENERATION", - Name: opts.Name, - StartTime: isoTime(now), - Input: opts.Input, - Metadata: opts.Metadata, - Model: opts.Model, - ModelParameters: opts.ModelParameters, - } - m.enqueue(ingestionEvent{ - ID: newID(), - Timestamp: isoTime(now), - Type: "generation-create", - Body: body, - }) return ctx, g } // Finish updates a generation with its final output, token usage and any -// error. A non-nil err marks the observation as ERROR level in Langfuse. +// error. A non-nil err marks the observation as ERROR. func (g *Generation) Finish(output interface{}, usage *TokenUsage, err error) { - if g == nil || g.manager == nil || !g.sampled { + if g == nil || g.manager == nil || !g.manager.Enabled() || g.span == nil { return } - level := "DEFAULT" - var statusMsg string + attrs := []attribute.KeyValue{jsonAttr(attrObsOutput, output)} + if usage != nil { + attrs = append(attrs, jsonAttr(attrObsUsageDetails, usage)) + } + g.span.SetAttributes(attrs...) if err != nil { - level = "ERROR" - statusMsg = err.Error() + g.span.RecordError(err) + g.span.SetStatus(codes.Error, err.Error()) } - body := observationBody{ - ID: g.ID, - TraceID: g.TraceID, - Type: "GENERATION", - EndTime: isoTime(time.Now()), - Output: output, - Usage: usage, - Level: level, - StatusMessage: statusMsg, - } - g.manager.enqueue(ingestionEvent{ - ID: newID(), - Timestamp: isoTime(time.Now()), - Type: "generation-update", - Body: body, - }) + g.span.End() } // MarkCompletionStart records the time at which the first token was received // in a streaming generation. Langfuse surfaces this as time-to-first-token. func (g *Generation) MarkCompletionStart(t time.Time) { - if g == nil || g.manager == nil || !g.sampled { + if g == nil || g.manager == nil || !g.manager.Enabled() || g.span == nil { return } - body := observationBody{ - ID: g.ID, - TraceID: g.TraceID, - Type: "GENERATION", - CompletionStart: isoTime(t), - } - g.manager.enqueue(ingestionEvent{ - ID: newID(), - Timestamp: isoTime(time.Now()), - Type: "generation-update", - Body: body, - }) + g.span.SetAttributes(attribute.String(attrObsCompletionStart, isoTime(t))) +} + +// jsonAttr serializes v to a compact JSON string and wraps it as a string +// OTel attribute — matching how langfuse-python stores structured fields +// (input/output/metadata/usage) on spans. nil/zero values return an empty +// KeyValue (harmless on SetAttributes). +func jsonAttr(key string, v interface{}) attribute.KeyValue { + if v == nil { + return attribute.KeyValue{Key: attribute.Key(key)} + } + b, err := json.Marshal(v) + if err != nil || len(b) == 0 || string(b) == "null" { + logger.Warnf(context.Background(), "[Langfuse] marshal attr %s failed: %v", key, err) + return attribute.KeyValue{Key: attribute.Key(key)} + } + return attribute.String(key, string(b)) } diff --git a/internal/tracing/langfuse/tracer_test.go b/internal/tracing/langfuse/tracer_test.go index f35bcf4d..16f40f33 100644 --- a/internal/tracing/langfuse/tracer_test.go +++ b/internal/tracing/langfuse/tracer_test.go @@ -2,70 +2,62 @@ package langfuse import ( "context" - "encoding/json" "errors" - "io" - "net/http" - "net/http/httptest" - "sync" + "strings" "testing" "time" + + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/codes" + "go.opentelemetry.io/otel/propagation" + "go.opentelemetry.io/otel/sdk/trace/tracetest" ) -// newTestServer spins up a fake Langfuse ingestion endpoint and returns the -// collected events + a cleanup func. -func newTestServer(t *testing.T) (*httptest.Server, func() []ingestionEvent) { - t.Helper() - var mu sync.Mutex - var batches []ingestionRequest - srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - body, _ := io.ReadAll(r.Body) - var req ingestionRequest - _ = json.Unmarshal(body, &req) - mu.Lock() - batches = append(batches, req) - mu.Unlock() - w.WriteHeader(200) - _, _ = w.Write([]byte(`{}`)) - })) - drain := func() []ingestionEvent { - mu.Lock() - defer mu.Unlock() - var out []ingestionEvent - for _, b := range batches { - out = append(out, b.Batch...) - } - return out - } - return srv, drain -} - -func newTestManager(t *testing.T, host string) *Manager { +// newTestManager builds a Manager wired to an in-memory span exporter via the +// SimpleSpanProcessor (synchronous export on span End), so tests can assert on +// exported spans deterministically without an HTTP server. +func newTestManager(t *testing.T) (*Manager, *tracetest.InMemoryExporter) { t.Helper() + exp := tracetest.NewInMemoryExporter() m, err := Init(Config{ Enabled: true, - Host: host, + Host: "http://test", PublicKey: "pk", SecretKey: "sk", FlushAt: 1, - FlushInterval: 5 * time.Millisecond, + FlushInterval: 1 * time.Second, QueueSize: 32, RequestTimeout: 2 * time.Second, SampleRate: 1.0, + testExporter: exp, }) if err != nil { t.Fatalf("init: %v", err) } - return m + t.Cleanup(func() { _ = m.Shutdown(context.Background()) }) + return m, exp } -// TestSpan_NestedHierarchy verifies that nested StartSpan calls produce a -// trace → span₁ → span₂ → generation hierarchy, with parentObservationId -// pointing to the direct ancestor at each level. +// spanAttr returns the string value of a span attribute, or "" if absent. +func spanAttr(attrs []attribute.KeyValue, key string) string { + for _, kv := range attrs { + if string(kv.Key) == key { + if v, ok := kv.Value.AsInterface().(string); ok { + return v + } + } + } + return "" +} + +// spanType returns the langfuse.observation.type of a span. +func spanType(s tracetest.SpanStub) string { return spanAttr(s.Attributes, attrObsType) } + +// TestSpan_NestedHierarchy verifies nested StartSpan calls produce a +// trace → span → span → generation tree with correct OTel parent linking +// (parenting is automatic through trace.SpanFromContext, no manual ids). func TestSpan_NestedHierarchy(t *testing.T) { - srv, drain := newTestServer(t) - defer srv.Close() - m := newTestManager(t, srv.URL) + m, exp := newTestManager(t) ctx, trace := m.StartTrace(context.Background(), TraceOptions{Name: "root"}) ctx, outer := m.StartSpan(ctx, SpanOptions{Name: "outer"}) @@ -77,119 +69,134 @@ func TestSpan_NestedHierarchy(t *testing.T) { outer.Finish("outer-out", nil, nil) trace.Finish("root-out", nil) - if err := m.Shutdown(context.Background()); err != nil { - t.Fatalf("shutdown: %v", err) + spans := exp.GetSpans() + if len(spans) != 4 { + t.Fatalf("expected 4 spans, got %d", len(spans)) } - - events := drain() - parentOf := map[string]string{} - traceOf := map[string]string{} - kindOf := map[string]string{} - for _, ev := range events { - if ev.Type != "span-create" && ev.Type != "generation-create" { - continue - } - b, _ := json.Marshal(ev.Body) - var ob observationBody - _ = json.Unmarshal(b, &ob) - parentOf[ob.ID] = ob.ParentObservationID - traceOf[ob.ID] = ob.TraceID - kindOf[ob.ID] = ob.Type + byName := map[string]tracetest.SpanStub{} + for _, s := range spans { + byName[s.Name] = s } - - if traceOf[outer.ID] != trace.ID || parentOf[outer.ID] != "" { - t.Errorf("outer span should sit directly under the trace, got parent=%q trace=%q", parentOf[outer.ID], traceOf[outer.ID]) + root, outerS, innerS, genS := byName["root"], byName["outer"], byName["inner"], byName["llm"] + if root.Name == "" || outerS.Name == "" || innerS.Name == "" || genS.Name == "" { + t.Fatalf("missing expected spans: %+v", byName) } - if parentOf[inner.ID] != outer.ID { - t.Errorf("inner span parent mismatch: got %q want %q", parentOf[inner.ID], outer.ID) + // All spans share the trace id. + if outerS.SpanContext.TraceID() != root.SpanContext.TraceID() || + innerS.SpanContext.TraceID() != root.SpanContext.TraceID() || + genS.SpanContext.TraceID() != root.SpanContext.TraceID() { + t.Errorf("all spans must share the root trace id") } - if parentOf[gen.ID] != inner.ID { - t.Errorf("generation should nest under inner span, got parent=%q want %q", parentOf[gen.ID], inner.ID) + // Parent chain: outer → root, inner → outer, gen → inner. + if outerS.Parent.SpanID() != root.SpanContext.SpanID() { + t.Errorf("outer parent = %s, want root %s", outerS.Parent.SpanID(), root.SpanContext.SpanID()) } - if kindOf[outer.ID] != "SPAN" || kindOf[inner.ID] != "SPAN" { - t.Errorf("expected both wrappers to be SPAN observations, got outer=%q inner=%q", kindOf[outer.ID], kindOf[inner.ID]) + if innerS.Parent.SpanID() != outerS.SpanContext.SpanID() { + t.Errorf("inner parent = %s, want outer %s", innerS.Parent.SpanID(), outerS.SpanContext.SpanID()) } - if kindOf[gen.ID] != "GENERATION" { - t.Errorf("expected generation kind, got %q", kindOf[gen.ID]) + if genS.Parent.SpanID() != innerS.SpanContext.SpanID() { + t.Errorf("gen parent = %s, want inner %s", genS.Parent.SpanID(), innerS.SpanContext.SpanID()) + } + // Observation types. + if spanType(root) != obsTypeTrace { + t.Errorf("root type = %q, want %q", spanType(root), obsTypeTrace) + } + if spanType(outerS) != obsTypeSpan || spanType(innerS) != obsTypeSpan { + t.Errorf("outer/inner type not span: %q %q", spanType(outerS), spanType(innerS)) + } + if spanType(genS) != obsTypeGeneration { + t.Errorf("gen type = %q, want %q", spanType(genS), obsTypeGeneration) } } // TestSpan_FinishWithError records an error status on the span so failures in // asynq handlers surface as red observations in Langfuse. func TestSpan_FinishWithError(t *testing.T) { - srv, drain := newTestServer(t) - defer srv.Close() - m := newTestManager(t, srv.URL) + m, exp := newTestManager(t) ctx, _ := m.StartTrace(context.Background(), TraceOptions{Name: "root"}) _, span := m.StartSpan(ctx, SpanOptions{Name: "boom"}) span.Finish(nil, nil, errors.New("kaboom")) - if err := m.Shutdown(context.Background()); err != nil { - t.Fatalf("shutdown: %v", err) - } - - var sawError bool - for _, ev := range drain() { - if ev.Type != "span-update" { + for _, s := range exp.GetSpans() { + if s.Name != "boom" { continue } - b, _ := json.Marshal(ev.Body) - var ob observationBody - _ = json.Unmarshal(b, &ob) - if ob.Level == "ERROR" && ob.StatusMessage == "kaboom" { - sawError = true + if s.Status.Code != codes.Error { + t.Errorf("span status = %v, want Error", s.Status.Code) } - } - if !sawError { - t.Fatal("expected span-update with ERROR level and status message") - } -} - -// TestResumeTrace_NoTraceCreateEvent verifies that ResumeTrace does NOT -// emit a trace-create event — the originating HTTP handler already did, -// and a duplicate would register as an orphan root in the Langfuse UI. -func TestResumeTrace_NoTraceCreateEvent(t *testing.T) { - srv, drain := newTestServer(t) - defer srv.Close() - m := newTestManager(t, srv.URL) - - ctx, trace := m.ResumeTrace(context.Background(), "upstream-trace", "upstream-span") - if trace == nil || trace.ID != "upstream-trace" { - t.Fatalf("expected resumed trace with id upstream-trace, got %+v", trace) - } - if pid, ok := parentObservationFromCtx(ctx); !ok || pid != "upstream-span" { - t.Errorf("expected parent observation upstream-span on ctx, got %q (ok=%v)", pid, ok) - } - - // Emit one child so there's *something* to flush, proving only child - // observations reach the wire, not a trace-create for the resume. - _, span := m.StartSpan(ctx, SpanOptions{Name: "child"}) - span.Finish(nil, nil, nil) - - if err := m.Shutdown(context.Background()); err != nil { - t.Fatalf("shutdown: %v", err) - } - - for _, ev := range drain() { - if ev.Type == "trace-create" { - t.Fatalf("ResumeTrace must not emit trace-create events, got %+v", ev) + if s.Status.Description != "kaboom" { + t.Errorf("status description = %q, want kaboom", s.Status.Description) } + return } + t.Fatal("boom span not exported") } -// TestResumeTrace_DisabledIsSafe guards against nil deref when Langfuse is -// off: ResumeTrace should return a nil *Trace and the original ctx unchanged. -func TestResumeTrace_DisabledIsSafe(t *testing.T) { - m, err := Init(Config{Enabled: false}) - if err != nil { - t.Fatalf("init: %v", err) - } - ctx, trace := m.ResumeTrace(context.Background(), "x", "y") - if trace != nil { - t.Errorf("expected nil trace when disabled, got %+v", trace) - } - if _, ok := parentObservationFromCtx(ctx); ok { - t.Error("disabled ResumeTrace should not attach parent observation") +// TestManager_FullRoundTrip asserts a generation carries the model name and +// usage_details attribute (JSON with token counts), and shares the trace id +// of the root span. +func TestManager_FullRoundTrip(t *testing.T) { + m, exp := newTestManager(t) + + ctx, trace := m.StartTrace(context.Background(), TraceOptions{Name: "test.trace", UserID: "user-42"}) + _, gen := m.StartGeneration(ctx, GenerationOptions{ + Name: "chat.completion", + Model: "gpt-test", + Input: []map[string]string{{"role": "user", "content": "hi"}}, + }) + gen.Finish("hello", &TokenUsage{Input: 10, Output: 20, Total: 30, Unit: "TOKENS"}, nil) + trace.Finish("hello", nil) + + for _, s := range exp.GetSpans() { + if s.Name != "chat.completion" { + continue + } + if spanAttr(s.Attributes, attrObsModel) != "gpt-test" { + t.Errorf("model = %q, want gpt-test", spanAttr(s.Attributes, attrObsModel)) + } + usage := spanAttr(s.Attributes, attrObsUsageDetails) + if !strings.Contains(usage, `"total":30`) { + t.Errorf("usage_details = %q, want total:30", usage) + } + if s.SpanContext.TraceID().String() != trace.ID { + t.Errorf("gen trace id = %s, want %s", s.SpanContext.TraceID(), trace.ID) + } + return } + t.Fatal("generation span not exported") +} + +// TestTraceparentPropagation is the sop3 correlation core test: an incoming +// W3C traceparent (as injected by an upstream caller like sop3) is extracted, +// and the WeKnora root span inherits the upstream trace id — so in LiteFuse +// the WeKnora trace and the upstream caller's trace are the same trace. +func TestTraceparentPropagation(t *testing.T) { + m, exp := newTestManager(t) + + // Simulate an upstream caller carrying a traceparent. + upstreamCtx, upstreamSpan := m.Tracer().Start(context.Background(), "upstream-caller") + remoteTraceID := upstreamSpan.SpanContext().TraceID() + carrier := propagation.MapCarrier{} + propagator.Inject(upstreamCtx, carrier) + if carrier["traceparent"] == "" { + t.Fatal("no traceparent injected") + } + + // The HTTP middleware extracts the traceparent into the request context. + httpCtx := propagator.Extract(context.Background(), carrier) + _, trace := m.StartTrace(httpCtx, TraceOptions{Name: "weknora-root"}) + trace.Finish(nil, nil) + + for _, s := range exp.GetSpans() { + if s.Name != "weknora-root" { + continue + } + if s.SpanContext.TraceID() != remoteTraceID { + t.Errorf("weknora root trace id = %s, want upstream %s (traceparent not inherited)", + s.SpanContext.TraceID(), remoteTraceID) + } + return + } + t.Fatal("weknora-root span not exported") } diff --git a/internal/types/tracing.go b/internal/types/tracing.go index 1b530203..8176dc0f 100644 --- a/internal/types/tracing.go +++ b/internal/types/tracing.go @@ -19,15 +19,20 @@ package types // happen to be named similarly. type TracingContext struct { // LangfuseTraceID is the id of the root trace that originated this task. - // When set, workers attach their observations to the same trace instead - // of creating a standalone one, which is what makes the Langfuse UI - // show the asynq work as a child of the originating HTTP request. + // Kept for backward compatibility with legacy payloads; the OTLP path now + // propagates correlation via LangfuseTraceparent (W3C) below. LangfuseTraceID string `json:"lf_trace_id,omitempty"` - // LangfuseParentObservationID, when set, points to the span that - // enclosed the enqueue call. The worker's own span/generation will list - // it as parentObservationId so Langfuse renders the tree as: - // http-trace → http-span → asynq-span → generation. + // LangfuseParentObservationID is retained for backward compatibility only; + // the OTLP path no longer uses it (parent linking flows through the W3C + // traceparent's parent span id). LangfuseParentObservationID string `json:"lf_parent_obs_id,omitempty"` + // LangfuseTraceparent carries the W3C Trace Context (`traceparent` header + // value: `00---`) from the enqueuing request. + // The worker re-extracts it so its spans are children of the same trace — + // stitching the HTTP request and the async job into one LiteFuse tree. This + // is also what propagates a sop3 run's W3C trace_id into any asynq jobs + // WeKnora enqueues while serving sop3's agent-chat call. + LangfuseTraceparent string `json:"lf_traceparent,omitempty"` // LangfuseUserID preserves the userId / tenant label across the async // boundary so that orphan async traces (when no upstream trace id is // available) still show up in the Langfuse "Users" view under the