feat: introduce HTTP policies for sandboxed HTTP Client (#968)

* feat: introduce HTTP policies for connection validation and enhance security testing

* refactor(http): convert Policies fields to private to enforce encapsulation

* refactor(http): remove package qualifier for `Policies` usage in tests to simplify code

* refactor(http): simplify tests by removing package qualifier for client usage

* refactor(http): consolidate Policies into Policy, add method and header validation, normalize API

* refactor(http): replace fmt.Errorf with typed errors and add tests

* refactor(http): update test context initialization and enhance error handling

* refactor(http): introduce `MultiPolicyConfigurationError` for aggregated errors, enhance validation logic, and update tests

* refactor(tests): enhance loopback HTTP policy tests with improved structure, explicit error checks, and added localhost opt-in validation

* refactor(http): unify request handling by consolidating policy evaluation and preparation logic, improve validation and error handling, and update tests

* refactor(http): ensure context is always initialized in `toStdRequest` to prevent `nil` context usage

* refactor(http): remove redundant context initialization in `toStdRequest`

* refactor(http): add nil check for `CloseIdleConnections` to improve safety

* refactor(net): move `CloseIdleNetworkConnections` to `helpers.go` for better package organization and add nil check for improved safety

Author: ziflex

engine.go

@@ -8,6 +8,7 @@ import (
 	"github.com/MontFerret/ferret/v2/pkg/bytecode"
 	"github.com/MontFerret/ferret/v2/pkg/bytecode/artifact"
 	"github.com/MontFerret/ferret/v2/pkg/compiler"
+	ferretnet "github.com/MontFerret/ferret/v2/pkg/net"
 	"github.com/MontFerret/ferret/v2/pkg/source"
 	"github.com/MontFerret/ferret/v2/pkg/vm"
 )
@@ -22,6 +23,7 @@ type Engine struct {
 	limiter       *sessionLimiter
 	idleCap       int
 	totalCap      int
+	ownsNetwork   bool
 }
 
 // New constructs an Engine from the provided options, registers all modules,
@@ -33,14 +35,22 @@ func New(setters ...Option) (*Engine, error) {
 		return nil, err
 	}
 
+	ownsNetwork := opts.network == nil
+
 	boot, err := newBootstrap(opts)
 	if err != nil {
 		return nil, fmt.Errorf("bootstrap: %w", err)
 	}
 
 	for _, m := range opts.modules {
 		if err := m.Register(boot); err != nil {
-			if closeErr := boot.hooks.engine.runCloseHooks(); closeErr != nil {
+			closeErr := boot.hooks.engine.runCloseHooks()
+
+			if ownsNetwork {
+				ferretnet.CloseIdleNetworkConnections(boot.host.Network())
+			}
+
+			if closeErr != nil {
 				return nil, errors.Join(err, fmt.Errorf("close hooks: %w", closeErr))
 			}
 
@@ -50,7 +60,13 @@ func New(setters ...Option) (*Engine, error) {
 
 	h, err := boot.host.Build()
 	if err != nil {
-		if closeErr := boot.hooks.engine.runCloseHooks(); closeErr != nil {
+		closeErr := boot.hooks.engine.runCloseHooks()
+
+		if ownsNetwork {
+			ferretnet.CloseIdleNetworkConnections(boot.host.Network())
+		}
+
+		if closeErr != nil {
 			return nil, errors.Join(err, fmt.Errorf("close hooks: %w", closeErr))
 		}
 
@@ -62,8 +78,13 @@ func New(setters ...Option) (*Engine, error) {
 	// Run init hooks after bootstrap is finalized and before returning the engine.
 	if err := hooks.engine.runInitHooks(); err != nil {
 		initErr := fmt.Errorf("init hooks: %w", err)
+		closeErr := hooks.engine.runCloseHooks()
 
-		if closeErr := hooks.engine.runCloseHooks(); closeErr != nil {
+		if ownsNetwork {
+			ferretnet.CloseIdleNetworkConnections(h.network)
+		}
+
+		if closeErr != nil {
 			return nil, errors.Join(initErr, fmt.Errorf("close hooks: %w", closeErr))
 		}
 
@@ -79,6 +100,7 @@ func New(setters ...Option) (*Engine, error) {
 		limiter:       newSessionLimiter(opts.maxActiveSessions),
 		idleCap:       opts.maxIdleVMsPerPlan,
 		totalCap:      opts.maxVMsPerPlan,
+		ownsNetwork:   ownsNetwork,
 	}, nil
 }
 
@@ -175,7 +197,13 @@ func (e *Engine) Run(ctx context.Context, src *source.Source, opts ...SessionOpt
 
 // Close runs the engine close hooks and releases engine-scoped resources.
 func (e *Engine) Close() error {
-	if err := e.hooks.engine.runCloseHooks(); err != nil {
+	err := e.hooks.engine.runCloseHooks()
+
+	if e.ownsNetwork {
+		ferretnet.CloseIdleNetworkConnections(e.host.network)
+	}
+
+	if err != nil {
 		return fmt.Errorf("close hooks: %w", err)
 	}
 

engine_lifecycle_test.go

@@ -9,6 +9,7 @@ import (
 
 	"github.com/MontFerret/ferret/v2/pkg/logging"
 	"github.com/MontFerret/ferret/v2/pkg/module"
+	ferretnet "github.com/MontFerret/ferret/v2/pkg/net"
 	"github.com/MontFerret/ferret/v2/pkg/runtime"
 	"github.com/MontFerret/ferret/v2/pkg/source"
 	"github.com/MontFerret/ferret/v2/pkg/vm"
@@ -50,10 +51,17 @@ func TestNewRunsCloseHooksWhenHostBuildFails(t *testing.T) {
 		moduleRegistered bool
 		closeHookCalled  bool
 	)
+	client := &recordingHTTPClient{}
 
 	mod := testModule{
 		registerFn: func(boot module.Bootstrap) error {
 			moduleRegistered = true
+			internal, ok := boot.(*bootstrap)
+			if !ok {
+				t.Fatalf("expected internal bootstrap, got %T", boot)
+			}
+			internal.host.network = mustNewTestNetwork(t, ferretnet.WithHTTPClient(client))
+
 			boot.Hooks().Engine().OnClose(func() error {
 				closeHookCalled = true
 				return nil
@@ -78,6 +86,10 @@ func TestNewRunsCloseHooksWhenHostBuildFails(t *testing.T) {
 	if !closeHookCalled {
 		t.Fatal("expected engine close hooks to run on host build failure")
 	}
+
+	if got := client.idleCloseCount(); got != 1 {
+		t.Fatalf("expected host-build-failure cleanup, got %d calls", got)
+	}
 }
 
 func TestNewReturnsJoinedErrorWhenInitAndCloseHooksFail(t *testing.T) {
@@ -115,6 +127,119 @@ func TestNewReturnsJoinedErrorWhenInitAndCloseHooksFail(t *testing.T) {
 	}
 }
 
+func TestEngineCloseClosesOwnedNetworkIdleConnections(t *testing.T) {
+	t.Parallel()
+
+	client := &recordingHTTPClient{}
+	eng := mustNewEngine(t)
+	eng.host.network = mustNewTestNetwork(t, ferretnet.WithHTTPClient(client))
+
+	if err := eng.Close(); err != nil {
+		t.Fatalf("close engine: %v", err)
+	}
+
+	if got := client.idleCloseCount(); got != 1 {
+		t.Fatalf("expected one idle-connection cleanup, got %d", got)
+	}
+}
+
+func TestEngineCloseCleansOwnedNetworkAfterHookFailure(t *testing.T) {
+	t.Parallel()
+
+	hookErr := errors.New("close hook failed")
+	client := &recordingHTTPClient{}
+	eng := mustNewEngine(t, WithEngineCloseHook(func() error {
+		return hookErr
+	}))
+	eng.host.network = mustNewTestNetwork(t, ferretnet.WithHTTPClient(client))
+
+	err := eng.Close()
+	if !errors.Is(err, hookErr) {
+		t.Fatalf("expected close hook error, got %v", err)
+	}
+
+	if got := client.idleCloseCount(); got != 1 {
+		t.Fatalf("expected cleanup after hook failure, got %d calls", got)
+	}
+}
+
+func TestEngineCloseDoesNotCleanInjectedNetwork(t *testing.T) {
+	t.Parallel()
+
+	client := &recordingHTTPClient{}
+	network := mustNewTestNetwork(t, ferretnet.WithHTTPClient(client))
+	eng := mustNewEngine(t, WithNetwork(network))
+
+	if err := eng.Close(); err != nil {
+		t.Fatalf("close engine: %v", err)
+	}
+
+	if got := client.idleCloseCount(); got != 0 {
+		t.Fatalf("expected injected network ownership to remain with caller, got %d cleanup calls", got)
+	}
+}
+
+func TestNewCleansOwnedNetworkOnRegistrationFailure(t *testing.T) {
+	t.Parallel()
+
+	registerErr := errors.New("register failed")
+	client := &recordingHTTPClient{}
+	mod := testModule{
+		registerFn: func(boot module.Bootstrap) error {
+			internal, ok := boot.(*bootstrap)
+			if !ok {
+				t.Fatalf("expected internal bootstrap, got %T", boot)
+			}
+
+			internal.host.network = mustNewTestNetwork(t, ferretnet.WithHTTPClient(client))
+
+			return registerErr
+		},
+	}
+
+	_, err := New(WithModules(mod))
+	if !errors.Is(err, registerErr) {
+		t.Fatalf("expected registration error, got %v", err)
+	}
+
+	if got := client.idleCloseCount(); got != 1 {
+		t.Fatalf("expected construction-failure cleanup, got %d calls", got)
+	}
+}
+
+func TestNewCleansOwnedNetworkOnInitFailure(t *testing.T) {
+	t.Parallel()
+
+	initErr := errors.New("init failed")
+	client := &recordingHTTPClient{}
+	mod := testModule{
+		registerFn: func(boot module.Bootstrap) error {
+			internal, ok := boot.(*bootstrap)
+			if !ok {
+				t.Fatalf("expected internal bootstrap, got %T", boot)
+			}
+
+			internal.host.network = mustNewTestNetwork(t, ferretnet.WithHTTPClient(client))
+
+			return nil
+		},
+	}
+
+	_, err := New(
+		WithModules(mod),
+		WithEngineInitHook(func() error {
+			return initErr
+		}),
+	)
+	if !errors.Is(err, initErr) {
+		t.Fatalf("expected init error, got %v", err)
+	}
+
+	if got := client.idleCloseCount(); got != 1 {
+		t.Fatalf("expected init-failure cleanup, got %d calls", got)
+	}
+}
+
 func TestRunClosesPlanWhenSessionCreationFails(t *testing.T) {
 	t.Parallel()
 

host.go

@@ -1,6 +1,8 @@
 package ferret
 
 import (
+	"fmt"
+
 	"github.com/MontFerret/ferret/v2/pkg/encoding"
 	"github.com/MontFerret/ferret/v2/pkg/fs"
 	"github.com/MontFerret/ferret/v2/pkg/logging"
@@ -37,7 +39,10 @@ func newHostContext(opts *options) (*hostContext, error) {
 
 	network := opts.network
 	if network == nil {
-		network = ferretnet.New()
+		network, err = ferretnet.New()
+		if err != nil {
+			return nil, fmt.Errorf("network: %w", err)
+		}
 	}
 
 	return &hostContext{

network_context_test.go

@@ -26,7 +26,8 @@ func TestSessionRunInjectsConfiguredNetwork(t *testing.T) {
 	t.Parallel()
 
 	client := &recordingHTTPClient{body: []byte("session-network")}
-	engine := mustNewEngine(t, WithNetwork(ferretnet.New(ferretnet.WithHTTPClient(client))))
+	network := mustNewTestNetwork(t, ferretnet.WithHTTPClient(client))
+	engine := mustNewEngine(t, WithNetwork(network))
 	defer func() { _ = engine.Close() }()
 
 	out, err := engine.Run(context.Background(), source.NewAnonymous(`
@@ -55,7 +56,8 @@ func TestDebugSessionRunInjectsConfiguredNetwork(t *testing.T) {
 	t.Parallel()
 
 	client := &recordingHTTPClient{body: []byte("debug-network")}
-	engine := mustNewEngine(t, WithNetwork(ferretnet.New(ferretnet.WithHTTPClient(client))))
+	network := mustNewTestNetwork(t, ferretnet.WithHTTPClient(client))
+	engine := mustNewEngine(t, WithNetwork(network))
 	defer func() { _ = engine.Close() }()
 
 	plan, err := engine.CompileDebug(context.Background(), source.New("debug-network.fql", `

network_http_client_test.go

@@ -8,10 +8,11 @@ import (
 )
 
 type recordingHTTPClient struct {
-	lastURL string
-	body    []byte
-	calls   int
-	mu      sync.Mutex
+	lastURL    string
+	body       []byte
+	calls      int
+	idleCloses int
+	mu         sync.Mutex
 }
 
 func (c *recordingHTTPClient) Do(_ context.Context, req *nethttp.Request) (*nethttp.Response, error) {
@@ -29,3 +30,17 @@ func (c *recordingHTTPClient) Do(_ context.Context, req *nethttp.Request) (*neth
 		Body:       append([]byte(nil), c.body...),
 	}, nil
 }
+
+func (c *recordingHTTPClient) CloseIdleConnections() {
+	c.mu.Lock()
+	defer c.mu.Unlock()
+
+	c.idleCloses++
+}
+
+func (c *recordingHTTPClient) idleCloseCount() int {
+	c.mu.Lock()
+	defer c.mu.Unlock()
+
+	return c.idleCloses
+}

network_test_helpers_test.go

@@ -0,0 +1,18 @@
+package ferret
+
+import (
+	"testing"
+
+	ferretnet "github.com/MontFerret/ferret/v2/pkg/net"
+)
+
+func mustNewTestNetwork(t testing.TB, setters ...ferretnet.Option) ferretnet.Network {
+	t.Helper()
+
+	network, err := ferretnet.New(setters...)
+	if err != nil {
+		t.Fatalf("create test network: %v", err)
+	}
+
+	return network
+}

pkg/net/context_test.go

@@ -7,7 +7,10 @@ import (
 )
 
 func TestWithNetworkRoundTrip(t *testing.T) {
-	network := New()
+	network, err := New()
+	if err != nil {
+		t.Fatalf("create network: %v", err)
+	}
 	ctx := WithNetwork(context.Background(), network)
 
 	resolved, err := NetworkFrom(ctx)
@@ -22,7 +25,10 @@ func TestWithNetworkRoundTrip(t *testing.T) {
 
 func TestHTTPClientFrom(t *testing.T) {
 	client := stubHTTPClient{}
-	network := New(WithHTTPClient(client))
+	network, err := New(WithHTTPClient(client))
+	if err != nil {
+		t.Fatalf("create network: %v", err)
+	}
 	ctx := WithNetwork(context.Background(), network)
 
 	resolved, err := HTTPClientFrom(ctx)
@@ -48,3 +54,24 @@ func TestNetworkFromError(t *testing.T) {
 		t.Fatalf("expected ErrNotFound for missing http client, got %v", err)
 	}
 }
+
+func TestDefaultNetworkForwardsIdleConnectionCleanup(t *testing.T) {
+	t.Parallel()
+
+	client := &trackingHTTPClient{}
+	network, err := New(WithHTTPClient(client))
+	if err != nil {
+		t.Fatalf("create network: %v", err)
+	}
+	closer, ok := network.(interface{ CloseIdleConnections() })
+	if !ok {
+		t.Fatalf("expected default network to expose idle-connection cleanup")
+	}
+
+	closer.CloseIdleConnections()
+	closer.CloseIdleConnections()
+
+	if got := client.idleCloseCount(); got != 2 {
+		t.Fatalf("expected cleanup to be forwarded twice, got %d", got)
+	}
+}