Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Fix span error tagging in GO YARPC/http #2285

Open
wants to merge 10 commits into
base: dev
Choose a base branch
from
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 5 additions & 0 deletions api/transport/propagation.go
Original file line number Diff line number Diff line change
Expand Up @@ -37,6 +37,11 @@ type CreateOpenTracingSpan struct {
ExtraTags opentracing.Tags
}

const (
//TracingTagStatusCode is the span tag key for the YAPRC status code.
TracingTagStatusCode = "rpc.yarpc.status_code"
)

// Do creates a new context that has a reference to the started span.
// This should be called before a Outbound makes a call
func (c *CreateOpenTracingSpan) Do(
Expand Down
6 changes: 6 additions & 0 deletions tracing.go
Original file line number Diff line number Diff line change
Expand Up @@ -26,8 +26,14 @@ import (
opentracing "github.com/opentracing/opentracing-go"
)

const (
//TracingComponentName helps determine the attribution of a span.
TracingComponentName = "yarpc"
)

// OpentracingTags are tags with YARPC metadata.
var OpentracingTags = opentracing.Tags{
"yarpc.version": Version,
"go.version": runtime.Version(),
"component": TracingComponentName,
}
35 changes: 25 additions & 10 deletions transport/http/handler.go
Original file line number Diff line number Diff line change
Expand Up @@ -122,7 +122,13 @@ func (h handler) callHandler(responseWriter *responseWriter, req *http.Request,
treq.Headers = treq.Headers.With(header, value)
}
}
ctx := req.Context()
ctx, cancel, parseTTLErr := parseTTL(ctx, treq, popHeader(req.Header, TTLMSHeader))
// parseTTLErr != nil is a problem only if the request is unary.
defer cancel()
ctx, span := h.createSpan(ctx, req, treq, start)
if err := transport.ValidateRequest(treq); err != nil {
UpdateSpanWithErrAndCode(span, err, yarpcerrors.FromError(err).Code())
return err
}
defer func() {
Expand All @@ -133,22 +139,18 @@ func (h handler) callHandler(responseWriter *responseWriter, req *http.Request,
}
}()

ctx := req.Context()
ctx, cancel, parseTTLErr := parseTTL(ctx, treq, popHeader(req.Header, TTLMSHeader))
// parseTTLErr != nil is a problem only if the request is unary.
defer cancel()
ctx, span := h.createSpan(ctx, req, treq, start)

spec, err := h.router.Choose(ctx, treq)
if err != nil {
updateSpanWithErr(span, err)
UpdateSpanWithErrAndCode(span, err, yarpcerrors.FromError(err).Code())
return err
}

if parseTTLErr != nil {
UpdateSpanWithErrAndCode(span, parseTTLErr, yarpcerrors.FromError(parseTTLErr).Code())
return parseTTLErr
}
if err := transport.ValidateRequestContext(ctx); err != nil {
UpdateSpanWithErrAndCode(span, err, yarpcerrors.FromError(err).Code())
return err
}
switch spec.Type() {
Expand All @@ -171,7 +173,7 @@ func (h handler) callHandler(responseWriter *responseWriter, req *http.Request,
err = yarpcerrors.Newf(yarpcerrors.CodeUnimplemented, "transport http does not handle %s handlers", spec.Type().String())
}

updateSpanWithErr(span, err)
UpdateSpanWithErrAndCode(span, err, yarpcerrors.FromError(err).Code())
return err
}

Expand All @@ -185,6 +187,7 @@ func handleOnewayRequest(
// returning from the request
var buff bytes.Buffer
if _, err := iopool.Copy(&buff, treq.Body); err != nil {
UpdateSpanWithErrAndCode(span, err, yarpcerrors.FromError(err).Code())
return err
}
treq.Body = &buff
Expand All @@ -203,16 +206,28 @@ func handleOnewayRequest(
Handler: onewayHandler,
Logger: logger,
})
updateSpanWithErr(span, err)
UpdateSpanWithErrAndCode(span, err, yarpcerrors.FromError(err).Code())
}()
return nil
}

func updateSpanWithErr(span opentracing.Span, err error) {
if err != nil {
span.SetTag("error", true)
span.LogFields(opentracinglog.String("event", err.Error()))
span.LogFields(
opentracinglog.String("event", "error"),
opentracinglog.String("message", err.Error()),
)
}
}

// UpdateSpanWithErrAndCode sets the error tag with errcode on a span, if an error is given.
// Returns the given error
func UpdateSpanWithErrAndCode(span opentracing.Span, err error, errCode yarpcerrors.Code) {
if err != nil {
span.SetTag(transport.TracingTagStatusCode, errCode)
}
updateSpanWithErr(span, err)
}

func (h handler) createSpan(ctx context.Context, req *http.Request, treq *transport.Request, start time.Time) (context.Context, opentracing.Span) {
Expand Down
18 changes: 11 additions & 7 deletions transport/http/outbound.go
Original file line number Diff line number Diff line change
Expand Up @@ -34,7 +34,6 @@ import (

"github.com/opentracing/opentracing-go"
"github.com/opentracing/opentracing-go/ext"
opentracinglog "github.com/opentracing/opentracing-go/log"
"go.uber.org/yarpc"
"go.uber.org/yarpc/api/peer"
"go.uber.org/yarpc/api/transport"
Expand Down Expand Up @@ -304,22 +303,27 @@ func (o *Outbound) call(ctx context.Context, treq *transport.Request) (*transpor
return nil, err
}
ctx, hreq, span, err := o.withOpentracingSpan(ctx, hreq, treq, start)
if span != nil {
defer span.Finish()
}
if err != nil {
UpdateSpanWithErrAndCode(span, err, yarpcerrors.FromError(err).Code())
return nil, err
}
defer span.Finish()

hreq = o.withCoreHeaders(hreq, treq, ttl)
hreq = hreq.WithContext(ctx)

response, err := o.roundTrip(hreq, treq, start, o.client)
if err != nil {
span.SetTag("error", true)
span.LogFields(opentracinglog.String("event", err.Error()))
return nil, err
if err != nil || (response.StatusCode >= 400 && response.StatusCode < 600) {
// If there's an error or the status code indicates an error, update the span
UpdateSpanWithErrAndCode(span, err, yarpcerrors.FromError(err).Code())
if err != nil {
return nil, err
}
}

span.SetTag("http.status_code", response.StatusCode)
kexiongliu123 marked this conversation as resolved.
Show resolved Hide resolved
span.SetTag("http.response.status_code", response.StatusCode)

// Service name match validation, return yarpcerrors.CodeInternal error if not match
if match, resSvcName := checkServiceMatch(treq.Service, response.Header); !match {
Expand Down