Wire Railway OTLP env sync and proofing

Author: julianknutsen

.env.production.example

@@ -0,0 +1,10 @@
+# Railway root env template for hosted Wasteland OTLP export.
+#
+# Keep real secrets in Railway Variables, not in git. The service-scoped
+# variables below reference a shared Railway variable named OTLP_SHARED_TOKEN.
+
+OTEL_EXPORTER_OTLP_TRACES_ENDPOINT=https://otel.cloud.gascityhall.com/v1/traces
+OTEL_EXPORTER_OTLP_METRICS_ENDPOINT=https://otel.cloud.gascityhall.com/v1/metrics
+OTEL_EXPORTER_OTLP_HEADERS=X-OTLP-Shared-Token=${{shared.OTLP_SHARED_TOKEN}}
+WL_BROWSER_OTLP_TRACES_TARGET=https://otel.cloud.gascityhall.com/v1/traces
+WL_BROWSER_OTLP_HEADERS=X-OTLP-Shared-Token=${{shared.OTLP_SHARED_TOKEN}}

.gitignore

@@ -9,3 +9,4 @@ web/dist/*
 web/coverage/
 web/*.tsbuildinfo
 docs/production-checklist.md
+__pycache__/

Makefile

@@ -24,7 +24,7 @@ LDFLAGS := -X main.version=$(VERSION) \
            -X main.date=$(BUILD_TIME) \
            -X main.inferEnabled=$(INFER_ENABLED)
 
-.PHONY: build build-go web check check-all lint fmt-check fmt vet test test-integration test-integration-offline test-cover cover install install-tools setup clean web-check web-test audit audit-web
+.PHONY: build build-go web check check-all lint fmt-check fmt vet test test-integration test-integration-offline test-cover cover install install-tools setup clean web-check web-test audit audit-web railway-sync-vars test-scripts
 
 ## web: build web UI (requires bun)
 web:
@@ -123,6 +123,14 @@ audit:
 audit-web:
 	cd web && bun audit
 
+## railway-sync-vars: preview Railway OTLP env sync from .env.production.example
+railway-sync-vars:
+	python3 scripts/railway_sync_vars.py --env-file .env.production.example --service wasteland --environment production --shared-env-var OTLP_SHARED_TOKEN --dry-run
+
+## test-scripts: run repository script unit tests
+test-scripts:
+	python3 -m unittest discover -s scripts -p 'test_*.py'
+
 ## help: show this help
 help:
 	@grep -E '^## ' $(MAKEFILE_LIST) | sed 's/## //' | column -t -s ':'

README.md

@@ -543,6 +543,30 @@ browser and server telemetry can share the same ingress token. Use
 `WL_BROWSER_OTLP_TRACES_TARGET` or `WL_BROWSER_OTLP_HEADERS` only when the
 browser path needs different routing from the server exporters.
 
+For Railway, the repo now includes a root
+[`.env.production.example`](./.env.production.example) template that Railway
+can suggest/import into the linked service. It keeps the real token out of git
+by referencing a shared Railway variable:
+
+```bash
+OTEL_EXPORTER_OTLP_HEADERS=X-OTLP-Shared-Token=${{shared.OTLP_SHARED_TOKEN}}
+WL_BROWSER_OTLP_HEADERS=X-OTLP-Shared-Token=${{shared.OTLP_SHARED_TOKEN}}
+```
+
+Sync it into Railway with the Railway GraphQL API. Export the live shared OTLP
+token locally first, then run:
+
+```bash
+export OTLP_SHARED_TOKEN=<shared-token>
+python3 scripts/railway_sync_vars.py --service wasteland --environment production --shared-env-var OTLP_SHARED_TOKEN --dry-run
+python3 scripts/railway_sync_vars.py --service wasteland --environment production --shared-env-var OTLP_SHARED_TOKEN --no-skip-deploys
+```
+
+The sync script uses `RAILWAY_TOKEN` or `RAILWAY_API_TOKEN`, auto-discovers the
+project when the token has a single match, and upserts both the shared Railway
+variable and the service-scoped OTLP variables. By default it stages changes
+with `skipDeploys`; pass `--no-skip-deploys` to roll them out immediately.
+
 ## Development
 
 ```bash

internal/observability/otel_test.go

@@ -3,8 +3,11 @@ package observability
 import (
 	"context"
 	"errors"
+	"io"
 	"net/http"
+	"net/http/httptest"
 	"testing"
+	"time"
 
 	"go.opentelemetry.io/otel"
 	"go.opentelemetry.io/otel/propagation"
@@ -143,3 +146,93 @@ func TestInit_RestoresProvidersWhenMetricExporterInitFails(t *testing.T) {
 		t.Fatal("Init() did not restore the original propagator")
 	}
 }
+
+func TestInit_ExportsTracesAndMetricsToConfiguredOTLPEndpoints(t *testing.T) {
+	originalTracerProvider := otel.GetTracerProvider()
+	originalMeterProvider := otel.GetMeterProvider()
+	originalPropagator := otel.GetTextMapPropagator()
+	t.Cleanup(func() {
+		otel.SetTextMapPropagator(originalPropagator)
+		otel.SetTracerProvider(originalTracerProvider)
+		otel.SetMeterProvider(originalMeterProvider)
+	})
+
+	type requestRecord struct {
+		path        string
+		sharedToken string
+		bodyLen     int
+	}
+
+	requests := make(chan requestRecord, 8)
+	collector := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		body, err := io.ReadAll(r.Body)
+		if err != nil {
+			t.Fatalf("read collector body: %v", err)
+		}
+		requests <- requestRecord{
+			path:        r.URL.Path,
+			sharedToken: r.Header.Get("X-OTLP-Shared-Token"),
+			bodyLen:     len(body),
+		}
+		w.WriteHeader(http.StatusAccepted)
+	}))
+	defer collector.Close()
+
+	t.Setenv("OTEL_EXPORTER_OTLP_TRACES_ENDPOINT", collector.URL+"/v1/traces")
+	t.Setenv("OTEL_EXPORTER_OTLP_METRICS_ENDPOINT", collector.URL+"/v1/metrics")
+	t.Setenv("OTEL_EXPORTER_OTLP_HEADERS", "X-OTLP-Shared-Token=abc123TOKEN")
+
+	shutdown, enabled, err := Init(context.Background(), Config{
+		ServiceName:      "wasteland-hosted",
+		ServiceNamespace: "wasteland",
+		ServiceVersion:   "test",
+		Environment:      "test",
+	})
+	if err != nil {
+		t.Fatalf("Init() error = %v", err)
+	}
+	if !enabled {
+		t.Fatal("Init() enabled = false, want true")
+	}
+
+	tracer := otel.Tracer("test")
+	_, span := tracer.Start(context.Background(), "wasteland.otlp.proof")
+	span.End()
+
+	meter := otel.Meter("test")
+	counter, err := meter.Int64Counter("wasteland_otlp_proof_total")
+	if err != nil {
+		t.Fatalf("Int64Counter() error = %v", err)
+	}
+	counter.Add(context.Background(), 1)
+
+	shutdownCtx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
+	defer cancel()
+	if err := shutdown(shutdownCtx); err != nil {
+		t.Fatalf("shutdown() error = %v", err)
+	}
+
+	seenPaths := map[string]bool{}
+	timeout := time.After(5 * time.Second)
+	for len(seenPaths) < 2 {
+		select {
+		case req := <-requests:
+			if req.sharedToken != "abc123TOKEN" {
+				t.Fatalf("collector shared token = %q, want %q", req.sharedToken, "abc123TOKEN")
+			}
+			if req.bodyLen == 0 {
+				t.Fatalf("collector body for %s was empty", req.path)
+			}
+			seenPaths[req.path] = true
+		case <-timeout:
+			t.Fatalf("timed out waiting for OTLP exports, saw %v", seenPaths)
+		}
+	}
+
+	if !seenPaths["/v1/traces"] {
+		t.Fatal("expected OTLP trace export to hit /v1/traces")
+	}
+	if !seenPaths["/v1/metrics"] {
+		t.Fatal("expected OTLP metrics export to hit /v1/metrics")
+	}
+}