config: add allow_incompatible_key_usage TLS option

Let's Encrypt announced they will stop issuing certificates with the TLS
Client Authentication Extended Key Usage (EKU) in 2026. Modern Go TLS
rejects peer certificates that do not carry the expected EKU, causing
failures in components that use the same LE certificate for both client
and server roles (e.g. Alertmanager cluster/gossip mTLS).

Add AllowIncompatibleKeyUsage to TLSConfig. When true the built-in EKU
check is bypassed while the full certificate chain trust, expiry, and
(on client connections) hostname verification are preserved.

Implementation: setting InsecureSkipVerify=true prevents Go's TLS stack
from running its EKU assertion, while a VerifyPeerCertificate callback
re-implements the remaining checks with x509.ExtKeyUsageAny so the EKU
field is accepted regardless of its value.

The new bool serialises as allow_incompatible_key_usage in both YAML
and JSON config files.

Fixes prometheus/alertmanager#5151

Signed-off-by: Ali <alliasgher123@gmail.com>

Author: Ali

config/http_config.go

@@ -1192,6 +1192,45 @@ func NewTLSConfigWithContext(ctx context.Context, cfg *TLSConfig, optFuncs ...TL
 		}
 	}
 
+	if cfg.AllowIncompatibleKeyUsage && !cfg.InsecureSkipVerify {
+		// Go's TLS library always checks Extended Key Usage (EKU) as part of
+		// its built-in peer certificate verification. To skip only the EKU
+		// check while preserving chain trust, hostname, and expiry validation,
+		// we must set InsecureSkipVerify=true (which disables all built-in
+		// verification) and then re-implement the verification ourselves via
+		// VerifyPeerCertificate — omitting the KeyUsages constraint.
+		serverName := tlsConfig.ServerName
+		tlsConfig.InsecureSkipVerify = true //nolint:gosec // EKU-only bypass; chain+hostname still verified below.
+		tlsConfig.VerifyPeerCertificate = func(rawCerts [][]byte, _ [][]*x509.Certificate) error {
+			if len(rawCerts) == 0 {
+				return nil
+			}
+			certs := make([]*x509.Certificate, len(rawCerts))
+			for i, raw := range rawCerts {
+				cert, err := x509.ParseCertificate(raw)
+				if err != nil {
+					return fmt.Errorf("tls: failed to parse peer certificate: %w", err)
+				}
+				certs[i] = cert
+			}
+			opts := x509.VerifyOptions{
+				// Access tlsConfig.RootCAs at call time so callers who
+				// modify RootCAs after NewTLSConfig returns are respected.
+				Roots:         tlsConfig.RootCAs,
+				Intermediates: x509.NewCertPool(),
+				DNSName:       serverName,
+				// ExtKeyUsageAny bypasses EKU checking. An empty slice
+				// defaults to ExtKeyUsageServerAuth per the x509 package docs.
+				KeyUsages: []x509.ExtKeyUsage{x509.ExtKeyUsageAny},
+			}
+			for _, cert := range certs[1:] {
+				opts.Intermediates.AddCert(cert)
+			}
+			_, err := certs[0].Verify(opts)
+			return err
+		}
+	}
+
 	return tlsConfig, nil
 }
 
@@ -1222,6 +1261,16 @@ type TLSConfig struct {
 	ServerName string `yaml:"server_name,omitempty" json:"server_name,omitempty"`
 	// Disable target certificate validation.
 	InsecureSkipVerify bool `yaml:"insecure_skip_verify" json:"insecure_skip_verify"`
+	// AllowIncompatibleKeyUsage disables the Extended Key Usage (EKU) check on
+	// peer certificates while still verifying the certificate chain, expiry,
+	// and (for client connections) the server hostname.
+	//
+	// This is useful when connecting to services that use a certificate without
+	// the expected EKU — for example Let's Encrypt certificates after they
+	// dropped TLS Client Authentication support in 2026. The full certificate
+	// chain trust and hostname verification still apply; only the EKU assertion
+	// is skipped.
+	AllowIncompatibleKeyUsage bool `yaml:"allow_incompatible_key_usage,omitempty" json:"allow_incompatible_key_usage,omitempty"`
 	// Minimum TLS version.
 	MinVersion TLSVersion `yaml:"min_version,omitempty" json:"min_version,omitempty"`
 	// Maximum TLS version.

config/http_config_test.go

@@ -15,13 +15,18 @@ package config
 
 import (
 	"context"
+	"crypto/ecdsa"
+	"crypto/elliptic"
+	"crypto/rand"
 	"crypto/tls"
 	"crypto/x509"
+	"crypto/x509/pkix"
 	"encoding/base64"
 	"encoding/json"
 	"errors"
 	"fmt"
 	"io"
+	"math/big"
 	"net"
 	"net/http"
 	"net/http/httptest"
@@ -2321,3 +2326,95 @@ func TestMultipleHeaders(t *testing.T) {
 	_, err = client.Get(ts.URL)
 	require.NoErrorf(t, err, "can't fetch URL: %v", err)
 }
+
+// TestTLSConfigAllowIncompatibleKeyUsage verifies that when
+// AllowIncompatibleKeyUsage is set, a TLS connection to a server whose
+// certificate lacks the expected Extended Key Usage (e.g. a Let's Encrypt
+// certificate used for mutual TLS after LE dropped clientAuth EKU support)
+// succeeds where it would otherwise fail.
+func TestTLSConfigAllowIncompatibleKeyUsage(t *testing.T) {
+	// Generate a self-signed CA + server cert that has ONLY serverAuth EKU
+	// (no clientAuth). This simulates a Let's Encrypt-style cert.
+	caKey, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader)
+	require.NoError(t, err)
+	caTemplate := &x509.Certificate{
+		SerialNumber:          big.NewInt(1),
+		Subject:               pkix.Name{CommonName: "test-ca"},
+		NotBefore:             time.Now().Add(-time.Hour),
+		NotAfter:              time.Now().Add(time.Hour),
+		IsCA:                  true,
+		KeyUsage:              x509.KeyUsageCertSign,
+		BasicConstraintsValid: true,
+	}
+	caDER, err := x509.CreateCertificate(rand.Reader, caTemplate, caTemplate, &caKey.PublicKey, caKey)
+	require.NoError(t, err)
+	caCert, err := x509.ParseCertificate(caDER)
+	require.NoError(t, err)
+
+	srvKey, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader)
+	require.NoError(t, err)
+	srvTemplate := &x509.Certificate{
+		SerialNumber: big.NewInt(2),
+		Subject:      pkix.Name{CommonName: "127.0.0.1"},
+		IPAddresses:  []net.IP{net.ParseIP("127.0.0.1")},
+		NotBefore:    time.Now().Add(-time.Hour),
+		NotAfter:     time.Now().Add(time.Hour),
+		// clientAuth only — no serverAuth. This triggers the EKU mismatch a
+		// TLS client sees when the peer cert lacks ExtKeyUsageServerAuth, which
+		// is the scenario for LE certs used in mutual-TLS gossip rings.
+		ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth},
+	}
+	srvDER, err := x509.CreateCertificate(rand.Reader, srvTemplate, caCert, &srvKey.PublicKey, caKey)
+	require.NoError(t, err)
+
+	srvTLSCert := tls.Certificate{
+		Certificate: [][]byte{srvDER},
+		PrivateKey:  srvKey,
+	}
+	caPool := x509.NewCertPool()
+	caPool.AddCert(caCert)
+
+	// Start a TLS test server using the serverAuth-only certificate.
+	serverTLSCfg := &tls.Config{Certificates: []tls.Certificate{srvTLSCert}}
+	listener, err := tls.Listen("tcp", "127.0.0.1:0", serverTLSCfg)
+	require.NoError(t, err)
+	srv := &http.Server{Handler: http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
+		w.WriteHeader(http.StatusOK)
+	})}
+	go srv.Serve(listener) //nolint:errcheck
+	defer srv.Close()
+	addr := "https://" + listener.Addr().String()
+
+	t.Run("without AllowIncompatibleKeyUsage fails on EKU mismatch", func(t *testing.T) {
+		cfg := HTTPClientConfig{
+			TLSConfig: TLSConfig{
+				// CA is trusted but EKU check will reject the server cert.
+				InsecureSkipVerify: false,
+			},
+		}
+		// Use the generated CA so hostname+chain pass; only EKU should fail.
+		tlsCfg, err := NewTLSConfig(&cfg.TLSConfig)
+		require.NoError(t, err)
+		tlsCfg.RootCAs = caPool
+		client := &http.Client{Transport: &http.Transport{TLSClientConfig: tlsCfg}}
+		_, err = client.Get(addr)
+		require.Error(t, err, "expected EKU error without AllowIncompatibleKeyUsage")
+		require.Contains(t, err.Error(), "incompatible key usage")
+	})
+
+	t.Run("with AllowIncompatibleKeyUsage succeeds", func(t *testing.T) {
+		cfg := TLSConfig{
+			AllowIncompatibleKeyUsage: true,
+			// ServerName override for IP-addressed connection.
+		}
+		tlsCfg, err := NewTLSConfig(&cfg)
+		require.NoError(t, err)
+		require.True(t, tlsCfg.InsecureSkipVerify, "InsecureSkipVerify should be true internally")
+		require.NotNil(t, tlsCfg.VerifyPeerCertificate, "VerifyPeerCertificate should be set")
+		tlsCfg.RootCAs = caPool
+		client := &http.Client{Transport: &http.Transport{TLSClientConfig: tlsCfg}}
+		resp, err := client.Get(addr)
+		require.NoError(t, err, "connection should succeed when AllowIncompatibleKeyUsage=true")
+		resp.Body.Close()
+	})
+}