Skip to content
Open
Show file tree
Hide file tree
Changes from 7 commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -207,7 +207,7 @@ The Function Runner supports custom TLS certificates for secure registry connect

**Certificate loading:**
- Reads from mounted secret path
- Supports ca.crt or ca.pem filenames
- Supports common certificate filenames
- Parses PEM-encoded certificates
- Creates certificate pool with custom CA

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -125,7 +125,12 @@ type: kubernetes.io/tls
```

{{% alert title="Note" color="primary" %}}
The certificate must be in PEM format and the key must be named `ca.crt` or `ca.pem`.
The certificate must be in PEM format and the key must be named one of the following:
- `ca.crt`
- `ca.pem`
- `cacert.pem`
- `ca-bundle.crt`
- `root.crt`
{{% /alert %}}

### 2. Mount TLS Secret
Expand Down
1 change: 1 addition & 0 deletions func/internal/executableevaluator_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,7 @@ const (
defaultKRMImagePrefix = "ghcr.io/kptdev/krm-functions-catalog/"
setImageFunction = "set-image"
starlarkFunction = "starlark"
testImageName = "test-image"
)

func getFunctionConfigStore(binaryDir string) *reconciler.FunctionConfigStore {
Expand Down
69 changes: 37 additions & 32 deletions func/internal/podevaluator.go
Original file line number Diff line number Diff line change
Expand Up @@ -136,40 +136,45 @@ func NewPodEvaluator(ctx context.Context, o PodEvaluatorOptions, cl client.Clien
readyCh := make(chan *podReadyResponse, channelBufferSize)
evictCh := make(chan *podEvictionRequest, channelBufferSize)

podMgr := &podManager{
kubeClient: cl,
namespace: o.PodNamespace,
wrapperServerImage: o.WrapperServerImage,
podReadyCh: readyCh,
podReadyTimeout: 60 * time.Second,
managerNamespace: managerNs,
maxGrpcMessageSize: o.MaxGrpcMessageSize,

enablePrivateRegistries: o.EnablePrivateRegistries,
registryAuthSecretPath: o.RegistryAuthSecretPath,
registryAuthSecretName: o.RegistryAuthSecretName,
enablePrivateRegistriesTls: o.EnablePrivateRegistriesTls,
tlsSecretPath: o.TlsSecretPath,
imageResolver: runneroptions.ResolveToImageForCLIFunc(o.DefaultImagePrefix),
tagResolver: runtime.TagResolver{}, // TODO: no resolvers, kpt needs to expose these better
}

pcm := &podCacheManager{
gcScanInterval: o.GcScanInterval,
podTTL: o.PodTTL,
connectionRequestCh: reqCh,
podReadyCh: readyCh,
evictionCh: evictCh,
functions: map[string]*functionInfo{},
maxWaitlistLength: maxWaitlist,
maxParallelPodsPerFunction: maxPods,
functionConfigMap: functionConfigStore,

podManager: podMgr,
}

pe := &podEvaluator{
requestCh: reqCh,
evictionCh: evictCh,
maxGrpcRetries: maxRetries,
podCacheManager: &podCacheManager{
gcScanInterval: o.GcScanInterval,
podTTL: o.PodTTL,
connectionRequestCh: reqCh,
podReadyCh: readyCh,
evictionCh: evictCh,
functions: map[string]*functionInfo{},
maxWaitlistLength: maxWaitlist,
maxParallelPodsPerFunction: maxPods,
functionConfigMap: functionConfigStore,

podManager: &podManager{
kubeClient: cl,
namespace: o.PodNamespace,
wrapperServerImage: o.WrapperServerImage,
podReadyCh: readyCh,
podReadyTimeout: 60 * time.Second,
managerNamespace: managerNs,
maxGrpcMessageSize: o.MaxGrpcMessageSize,

enablePrivateRegistries: o.EnablePrivateRegistries,
registryAuthSecretPath: o.RegistryAuthSecretPath,
registryAuthSecretName: o.RegistryAuthSecretName,
enablePrivateRegistriesTls: o.EnablePrivateRegistriesTls,
tlsSecretPath: o.TlsSecretPath,
imageResolver: runneroptions.ResolveToImageForCLIFunc(o.DefaultImagePrefix),
tagResolver: runtime.TagResolver{},
},
},
requestCh: reqCh,
evictionCh: evictCh,
maxGrpcRetries: maxRetries,
podCacheManager: pcm,
}

go pe.podCacheManager.podCacheManager(ctx)

err = pe.podCacheManager.retrieveFunctionPods(context.Background())
Expand Down
4 changes: 0 additions & 4 deletions func/internal/podevaluator_tag_resolution_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -32,10 +32,6 @@ import (
"sigs.k8s.io/controller-runtime/pkg/client"
)

const (
testImageName = "test-image"
)

type fakeLister struct {
tags map[string][]string
err string
Expand Down
26 changes: 26 additions & 0 deletions func/internal/podevaluator_unit_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,8 @@ import (
"time"

"github.com/kptdev/kpt/pkg/fn/runtime"
"github.com/kptdev/kpt/pkg/lib/runneroptions"
fnconf "github.com/kptdev/porch/controllers/functionconfigs/reconciler"
pb "github.com/kptdev/porch/func/evaluator"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
Expand All @@ -31,6 +33,7 @@ import (
"google.golang.org/grpc/credentials/insecure"
"google.golang.org/grpc/status"
"sigs.k8s.io/controller-runtime/pkg/client"
"sigs.k8s.io/controller-runtime/pkg/client/fake"
)

// startFakeEvalServer starts a gRPC function evaluator server on a dynamic port.
Expand All @@ -50,6 +53,29 @@ func startFakeEvalServer(t *testing.T, evalFunc func(ctx context.Context, req *p
}
}

func TestNewPodEvaluator(t *testing.T) {
ctx, cancel := context.WithCancel(context.Background())
t.Cleanup(cancel)

kubeClient := fake.NewClientBuilder().Build()
store := fnconf.NewFunctionConfigStore(runneroptions.GHCRImagePrefix, "/functions")

eval, err := NewPodEvaluator(ctx, PodEvaluatorOptions{
PodNamespace: "test-ns",
WrapperServerImage: "ghcr.io/kptdev/wrapper-server:latest",
GcScanInterval: time.Minute,
PodTTL: time.Minute,
}, kubeClient, store)
require.NoError(t, err)
require.NotNil(t, eval)

pe, ok := eval.(*podEvaluator)
require.True(t, ok)
assert.Equal(t, defaultMaxWaitlistLength, pe.podCacheManager.maxWaitlistLength)
assert.Equal(t, defaultMaxParallelPodsPerFunction, pe.podCacheManager.maxParallelPodsPerFunction)
assert.Equal(t, defaultMaxGrpcRetries, pe.maxGrpcRetries)
}

func TestEvaluateFunction_ErrorInResponse(t *testing.T) {
reqCh := make(chan *connectionRequest, 1)
pe := &podEvaluator{requestCh: reqCh,
Expand Down
47 changes: 27 additions & 20 deletions func/internal/podmanager.go
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,6 @@ import (
"encoding/json"
"fmt"
"net"
"net/http"
"os"
"path/filepath"
"strconv"
Expand All @@ -36,7 +35,9 @@ import (
"github.com/kptdev/kpt/pkg/fn/runtime"
"github.com/kptdev/kpt/pkg/lib/runneroptions"
configapi "github.com/kptdev/porch/api/porchconfig/v1alpha1"
"github.com/kptdev/porch/pkg/httpclient"
"go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc"
"go.uber.org/multierr"
"google.golang.org/grpc"
"google.golang.org/grpc/connectivity"
"google.golang.org/grpc/credentials/insecure"
Expand Down Expand Up @@ -488,24 +489,15 @@ func (pm *podManager) getImage(ctx context.Context, ref name.Reference, auth aut
if !pm.enablePrivateRegistries || strings.HasPrefix(image, defaultRegistry) || !pm.enablePrivateRegistriesTls {
return remote.Image(ref, remote.WithAuth(auth), remote.WithContext(ctx))
}
tlsFile := "ca.crt"
// Check if mounted secret location contains CA file.
if _, err := os.Stat(pm.tlsSecretPath); os.IsNotExist(err) {
caCertPath, err := tlsCACertPath(pm.tlsSecretPath)
if err != nil {
return nil, err
}
if _, errCRT := os.Stat(filepath.Join(pm.tlsSecretPath, "ca.crt")); os.IsNotExist(errCRT) {
if _, errPEM := os.Stat(filepath.Join(pm.tlsSecretPath, "ca.pem")); os.IsNotExist(errPEM) {
return nil, fmt.Errorf("ca.crt not found: %v, and ca.pem also not found: %w", errCRT, errPEM)
}
tlsFile = "ca.pem"
}
// Load the custom TLS configuration
tlsConfig, err := loadTLSConfig(filepath.Join(pm.tlsSecretPath, tlsFile))
tlsConfig, err := loadTLSConfig(caCertPath)
if err != nil {
return nil, err
}
// Create a custom HTTPS transport
transport := createTransport(tlsConfig)
transport := httpclient.RegistryTransport(tlsConfig)

// Attempt image pull with given custom TLS cert
img, tlsErr := remote.Image(ref, remote.WithAuth(auth), remote.WithContext(ctx), remote.WithTransport(transport))
Expand All @@ -521,6 +513,27 @@ func (pm *podManager) getImage(ctx context.Context, ref name.Reference, auth aut
return img, nil
}

func tlsCACertPath(tlsSecretPath string) (string, error) {
if _, err := os.Stat(tlsSecretPath); err != nil {
return "", fmt.Errorf("tls secret folder %q could not be reached: %w", tlsSecretPath, err)
}

var multiErr error

candidates := []string{"ca.crt", "ca.pem", "cacert.pem", "ca-bundle.crt", "root.crt"}
for _, file := range candidates {
path := filepath.Join(tlsSecretPath, file)
if _, err := os.Stat(path); err == nil {
return path, nil
} else {
multierr.AppendInto(&multiErr, err)
}
}
Comment thread
mozesl-nokia marked this conversation as resolved.

return "", fmt.Errorf("no CA certificate found in %q (candidates: [%s]): %w",
tlsSecretPath, strings.Join(candidates, ", "), multiErr)
}

func loadTLSConfig(caCertPath string) (*tls.Config, error) {
// Read the CA certificate file
caCert, err := os.ReadFile(caCertPath)
Expand All @@ -540,12 +553,6 @@ func loadTLSConfig(caCertPath string) (*tls.Config, error) {
return tlsConfig, nil
}

func createTransport(tlsConfig *tls.Config) *http.Transport {
return &http.Transport{
TLSClientConfig: tlsConfig,
}
}

// CreatePod creates a pod for an image.
func (pm *podManager) CreatePod(ctx context.Context, image string, postFix int, config *configapi.PodExecutorConfig, useGenerateName bool) (*corev1.Pod, error) {
var de *digestAndEntrypoint
Expand Down
54 changes: 35 additions & 19 deletions func/internal/podmanager_unit_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -377,6 +377,41 @@ func TestAppendImagePullSecret(t *testing.T) {
})
}

func TestTlsCACertPath(t *testing.T) {
t.Run("prefers ca.crt over ca.pem", func(t *testing.T) {
dir := t.TempDir()
require.NoError(t, os.WriteFile(filepath.Join(dir, "ca.crt"), []byte("crt"), 0o600))
require.NoError(t, os.WriteFile(filepath.Join(dir, "ca.pem"), []byte("pem"), 0o600))

path, err := tlsCACertPath(dir)
require.NoError(t, err)
assert.Equal(t, filepath.Join(dir, "ca.crt"), path)
})

t.Run("falls back to ca.pem", func(t *testing.T) {
dir := t.TempDir()
require.NoError(t, os.WriteFile(filepath.Join(dir, "ca.pem"), []byte("pem"), 0o600))

path, err := tlsCACertPath(dir)
require.NoError(t, err)
assert.Equal(t, filepath.Join(dir, "ca.pem"), path)
})

t.Run("returns error when mount path is missing", func(t *testing.T) {
_, err := tlsCACertPath(filepath.Join(t.TempDir(), "missing"))
require.Error(t, err)
assert.ErrorContains(t, err, "tls secret folder")
})

t.Run("returns error when no candidate files exist", func(t *testing.T) {
dir := t.TempDir()
_, err := tlsCACertPath(dir)
require.Error(t, err)
assert.ErrorContains(t, err, "no CA certificate found")
assert.ErrorContains(t, err, "ca.crt")
})
}

func TestLoadTLSConfig(t *testing.T) {
t.Run("valid PEM certificate", func(t *testing.T) {
// Generate a self-signed certificate for testing
Expand Down Expand Up @@ -416,25 +451,6 @@ func TestLoadTLSConfig(t *testing.T) {
})
}

func TestCreateTransport(t *testing.T) {
certPEM := generateSelfSignedCertPEM(t)

tmpFile, err := os.CreateTemp("", "transport-cert-*.pem")
require.NoError(t, err)
defer os.Remove(tmpFile.Name())

_, err = tmpFile.Write(certPEM)
require.NoError(t, err)
require.NoError(t, tmpFile.Close())

tlsConfig, err := loadTLSConfig(tmpFile.Name())
require.NoError(t, err)

transport := createTransport(tlsConfig)
assert.NotNil(t, transport)
assert.Equal(t, tlsConfig, transport.TLSClientConfig)
}

func TestFindPodsForService(t *testing.T) {
t.Run("returns matching pods", func(t *testing.T) {
svc := &corev1.Service{
Expand Down
4 changes: 2 additions & 2 deletions go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -65,7 +65,6 @@ require (
k8s.io/kube-aggregator v0.36.1
k8s.io/kubectl v0.36.1
k8s.io/utils v0.0.0-20260507154919-ff6756f316d2
sigs.k8s.io/cli-utils v0.37.2
sigs.k8s.io/controller-runtime v0.24.1
sigs.k8s.io/kustomize/kyaml v0.21.1
sigs.k8s.io/yaml v1.6.0
Expand Down Expand Up @@ -208,7 +207,7 @@ require (
go.opentelemetry.io/otel/log v0.19.0 // indirect
go.opentelemetry.io/otel/sdk/log v0.19.0 // indirect
go.starlark.net v0.0.0-20260522144826-ec58d4b459e2 // indirect
go.uber.org/multierr v1.11.0 // indirect
go.uber.org/multierr v1.11.0
go.uber.org/zap v1.28.0 // indirect
go.yaml.in/yaml/v2 v2.4.4 // indirect
go.yaml.in/yaml/v3 v3.0.4 // indirect
Expand All @@ -234,6 +233,7 @@ require (
k8s.io/kube-openapi v0.0.0-20260520065146-aa012df4f4af // indirect
k8s.io/streaming v0.36.1 // indirect
sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.36.0 // indirect
sigs.k8s.io/cli-utils v0.37.2 // indirect
sigs.k8s.io/json v0.0.0-20250730193827-2d320260d730 // indirect
sigs.k8s.io/kustomize/api v0.21.1 // indirect
sigs.k8s.io/randfill v1.0.0 // indirect
Expand Down
3 changes: 0 additions & 3 deletions internal/telemetry/otel.go
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,6 @@ import (

prombridge "go.opentelemetry.io/contrib/bridges/prometheus"
"go.opentelemetry.io/contrib/exporters/autoexport"
"go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp"
"go.opentelemetry.io/contrib/propagators/autoprop"
"go.opentelemetry.io/otel"
otelprometheus "go.opentelemetry.io/otel/exporters/prometheus"
Expand Down Expand Up @@ -116,8 +115,6 @@ func SetupOpenTelemetry(ctx context.Context) (*OTelResources, error) {
return nil, err
}

http.DefaultTransport = otelhttp.NewTransport(http.DefaultTransport)

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

My problem with this one is that it will disable monitoring http metrics for the following paths:
external -> porch server
porch-server -> kube-apiserver
porch-server -> git
function-runner -> oci
function-runner -> kube-apiserver

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

We can add these individually, probably

How can an external service call porch server directly btw? Shouldn't it just be through the kube apiserver?

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I will add these individual wrappers after #1111 is merged and this is rebased.

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

  • external -> porch server [the http listen is handled by the api-server framework, not sure if setting the rest config to use the otel wrapper handles this case]
  • porch-server -> kube-apiserver
  • porch-server -> git [go-git requires registering a global transport, if that is not an http.Transport it cannot inject TLS config into it per request]
  • function-runner -> oci
  • function-runner -> kube-apiserver

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I'm ok with the current level of setup, but please pick up tasks for the 2 that are being removed.

http.DefaultClient.Transport = http.DefaultTransport
klog.Infof("OpenTelemetry initialized in %s", time.Since(setupTiming))
return res, nil
}
Expand Down
Loading
Loading