Olivine: Building a Redis-Compatible Service in Go. · 章節目錄

第 18 章

[Day 18] Open Telemetry

Open Telemetry is a standard of cloud native ecosystem for observability which includes:

  • Tracing: trace how request be processed, which functions are invoked.
  • Metrics: collect service metrics, such as CPU/memory usage, average response time.
  • Logging

Note: this topic is too early to Olivine, I’ll take some example, but I’ll not explain why — it’s out of scope.

Opentelemetry SDK

Opentelemetry provides a set of SDKs.

$ go get go.opentelemetry.io/otel go.opentelemetry.io/otel/sdk go.opentelemetry.io/otel/sdk/log go.opentelemetry.io/otel/sdk/metric go.opentelemetry.io/contrib/bridges/otelslog go.opentelemetry.io/contrib/exporters/autoexport go.opentelemetry.io/contrib/instrumentation/runtime

Initialize SDKs in internal/service/otel.go

package service

import (
	"context"
	"errors"
	"io"
	"log"
	"log/slog"
	"os"

	"go.opentelemetry.io/contrib/bridges/otelslog"
	"go.opentelemetry.io/contrib/exporters/autoexport"
	"go.opentelemetry.io/otel"
	otellog "go.opentelemetry.io/otel/log"
	"go.opentelemetry.io/otel/log/global"
	"go.opentelemetry.io/otel/propagation"
	sdklog "go.opentelemetry.io/otel/sdk/log"
	sdkmetric "go.opentelemetry.io/otel/sdk/metric"
	"go.opentelemetry.io/otel/sdk/resource"
	sdktrace "go.opentelemetry.io/otel/sdk/trace"
	semconv "go.opentelemetry.io/otel/semconv/v1.41.0"
)

const (
	otelEnabledEnv = "OTEL_ENABLED"
	serviceName    = "olivine"
)

func OTelEnabled() bool {
	return os.Getenv(otelEnabledEnv) == "1"
}

// SetupOTelSDK bootstraps the OpenTelemetry pipeline.
// If it does not return an error, make sure to call shutdown for proper cleanup.
func SetupOTelSDK(ctx context.Context) (func(context.Context) error, error) {
	var shutdownFuncs []func(context.Context) error
	var err error

	// shutdown calls cleanup functions registered via shutdownFuncs.
	// The errors from the calls are joined.
	// Each registered cleanup will be invoked once.
	shutdown := func(ctx context.Context) error {
		var err error
		for _, fn := range shutdownFuncs {
			err = errors.Join(err, fn(ctx))
		}
		shutdownFuncs = nil
		return err
	}

	// handleErr calls shutdown for cleanup and makes sure that all errors are returned.
	handleErr := func(inErr error) {
		err = errors.Join(inErr, shutdown(ctx))
	}

	// Set up propagator.
	prop := newPropagator()
	otel.SetTextMapPropagator(prop)

	res, err := newResource()
	if err != nil {
		handleErr(err)
		return shutdown, err
	}

	// Set up trace provider.
	tracerProvider, err := newTracerProvider(ctx, res)
	if err != nil {
		handleErr(err)
		return shutdown, err
	}
	shutdownFuncs = append(shutdownFuncs, tracerProvider.Shutdown)
	otel.SetTracerProvider(tracerProvider)

	// Set up meter provider.
	meterProvider, err := newMeterProvider(ctx, res)
	if err != nil {
		handleErr(err)
		return shutdown, err
	}
	shutdownFuncs = append(shutdownFuncs, meterProvider.Shutdown)
	otel.SetMeterProvider(meterProvider)

	// Set up logger provider.
	loggerProvider, err := newLoggerProvider(ctx, res)
	if err != nil {
		handleErr(err)
		return shutdown, err
	}
	shutdownFuncs = append(shutdownFuncs, loggerProvider.Shutdown)
	global.SetLoggerProvider(loggerProvider)

	// Bridge the standard slog logger to OpenTelemetry while preserving its
	// existing console output.
	defaultLogger := slog.Default()
	defaultLogWriter := log.Writer()
	defaultLogFlags := log.Flags()
	defaultLogPrefix := log.Prefix()
	slog.SetDefault(newSlogLogger(os.Stderr, loggerProvider))
	shutdownFuncs = append(shutdownFuncs, func(context.Context) error {
		slog.SetDefault(defaultLogger)
		log.SetOutput(defaultLogWriter)
		log.SetFlags(defaultLogFlags)
		log.SetPrefix(defaultLogPrefix)
		return nil
	})

	return shutdown, err
}

func newSlogLogger(consoleWriter io.Writer, loggerProvider otellog.LoggerProvider) *slog.Logger {
	consoleHandler := slog.NewTextHandler(consoleWriter, nil)
	otelHandler := otelslog.NewHandler(
		"olivine",
		otelslog.WithLoggerProvider(loggerProvider),
	)
	return slog.New(slog.NewMultiHandler(consoleHandler, otelHandler))
}

func newPropagator() propagation.TextMapPropagator {
	return propagation.NewCompositeTextMapPropagator(
		propagation.TraceContext{},
		propagation.Baggage{},
	)
}

func newResource() (*resource.Resource, error) {
	return resource.Merge(
		resource.Default(),
		resource.NewSchemaless(semconv.ServiceName(serviceName)),
	)
}

func newTracerProvider(ctx context.Context, res *resource.Resource) (*sdktrace.TracerProvider, error) {
	spanExporter, err := autoexport.NewSpanExporter(ctx)
	if err != nil {
		return nil, err
	}
	tracerProvider := sdktrace.NewTracerProvider(
		sdktrace.WithBatcher(spanExporter),
		sdktrace.WithResource(res),
	)
	return tracerProvider, nil
}

func newMeterProvider(ctx context.Context, res *resource.Resource) (*sdkmetric.MeterProvider, error) {
	metricReader, err := autoexport.NewMetricReader(ctx)
	if err != nil {
		return nil, err
	}

	meterProvider := sdkmetric.NewMeterProvider(
		sdkmetric.WithReader(metricReader),
		sdkmetric.WithResource(res),
	)
	return meterProvider, nil
}

func newLoggerProvider(ctx context.Context, res *resource.Resource) (*sdklog.LoggerProvider, error) {
	logExporter, err := autoexport.NewLogExporter(ctx)
	if err != nil {
		return nil, err
	}

	loggerProvider := sdklog.NewLoggerProvider(
		sdklog.WithProcessor(sdklog.NewBatchProcessor(logExporter)),
		sdklog.WithResource(res),
	)
	return loggerProvider, nil
}

This code is based on Getting Start | OpenTelemetry

Initialize SDKs in cmd/olivine/app.go:

// file: cmd/olivine/app.go
func (app *App) Run() error {
	ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM)
	defer stop()

+	if service.OTelEnabled() {
+		otelShutdown, err := service.SetupOTelSDK(ctx)
+		if err != nil {
+			return err
+		}
+		defer func() {
+			if err := otelShutdown(context.Background()); err != nil {
+				slog.Error("failed to shutdown otel", slog.Any("error", err))
+			}
+		}()
+	}

	slog.Info("restoring data from disk")
	// ...
}

Metrics

Opentelemetry SDK provides a metrics instrumentation of go runtime in go.opentelemetry.io/contrib/instrumentation/runtime

if service.OTelEnabled() {
	otelShutdown, err := service.SetupOTelSDK(ctx)
	if err != nil {
		return err
	}
	defer func() {
		if err := otelShutdown(context.Background()); err != nil {
			slog.Error("failed to shutdown otel", slog.Any("error", err))
		}
	}()
+	if err := runtime.Start(runtime.WithMinimumReadMemStatsInterval(time.Second)); err != nil {
+		return err
+	}
}

Try to start olivine:

$ OTEL_ENABLED=1 OTEL_METRICS_EXPORTER=console go run ./cmd/olivine

Note: OTEL_METRICS_EXPORTER is handled by go.opentelemetry.io/contrib/exporters/autoexport, read godoc for more information.

We can customize instrumentation of metrics:

  • How many keys in data.(*mapStorage)?
  • How’s the memory usage?
  • etc…

Tracing

It’s important to trace how request be handled for a client-server application, and it is easier by trace.Start() than before in open telemetry SDK.

// file: internal/server/handler.go
func (h *simpleHandler) ServeRESP(ctx context.Context, rd *resp.Reader) (ret resp.Value, err error) {
+	ctx, span := otel.Tracer("olivine").Start(ctx, "server.(*simpleHandler).ServeRESP")
+	defer span.End()

	command, err := resp.ReadCommand(rd)
// file: internal/service/cmd/ping.go
func (c *Ping) Exec(ctx context.Context, cmd *resp.Command) (ret resp.Value, err error) {
+	_, span := otel.Tracer("olivine").Start(ctx, "cmd.(*Ping).Exec")
+	defer span.End()

	if err := c.parse(cmd); err != nil {
		return nil, err
	}

Error Handling

Open telemetry SDK assume every requests are success by default, even if returns err != nil.

We can mark status as failed and record error by using:

if err != nil {
		span.RecordError(err)
		span.SetStatus(codes.Error, err.Error())
}

I created a internal/util/otel.go with EndSpan() to handle it.

func EndSpan(span trace.Span, err error) {
	if err != nil {
		span.RecordError(err)
		span.SetStatus(codes.Error, err.Error())
	}
	span.End()
}

Use defer func() { util.EndSpan(span, err) }() instead of defer span.End().

Logging

It’s no need to configure, becuase we initialized logger bridge for slog.

Jaeger, Proemetheus and Loki

Open telemetry in console is unusable: we cannot analyze it and it could be flush if too long.

Start Olivine with environments below:

$ OTEL_ENABLED=1 OTEL_SERVICE_NAME=olivine OTEL_TRACES_EXPORTER=otlp OTEL_EXPORTER_OTLP_TRACES_PROTOCOL=http/protobuf OTEL_EXPORTER_OTLP_TRACES_ENDPOINT=http://localhost:4318/v1/traces OTEL_METRICS_EXPORTER=otlp OTEL_EXPORTER_OTLP_METRICS_PROTOCOL=http/protobuf OTEL_EXPORTER_OTLP_METRICS_ENDPOINT=http://localhost:9090/api/v1/otlp/v1/metrics OTEL_LOGS_EXPORTER=otlp OTEL_EXPORTER_OTLP_LOGS_PROTOCOL=http/protobuf OTEL_EXPORTER_OTLP_LOGS_ENDPOINT=http://localhost:3100/otlp/v1/logs go run ./cmd/olivine

Open Telemetry Collector

Open Telemetry provides a collector to handle tracing, metrics and logging, it’s no need to setup endpoints by each service.

As production deployment, it suggests to use otel collector.

Grafana

Grafana is a dashboard to integrate a view or alert of open telemetry tracing, metrics and logging.

Note: I have implemented a docker-compose.dev.yml and deployments/ for open telemetry services integration, but they are out of scope.

Conclusion

I don’t think open telemetry integration (d18/otel) is good enough to merge into main branch, it misses a lot of production instrumentation or tracing.