fix: reuse compiled templates in thread-safe engine

lib/multi.go

@@ -44,11 +44,7 @@ func createEphemeralObjects(ctx context.Context, base *NucleiEngine, opts *types
 		ResumeCfg:    types.NewResumeCfg(),
 		Parser:       base.parser,
 		Browser:      base.browserInstance,
-		// Thread-safe executes can run concurrently with different Output writers.
-		// The compiled-template cache shallow-copies requests and mutates shared
-		// ExecutorOptions.Output via UpdateOptions/ApplyNewEngineOptions, which
-		// would otherwise route all findings to whichever call last won the race.
-		DoNotCache: true,
+		DoNotCache:   opts.DoNotCacheTemplates,
 	}
 	if opts.ShouldUseHostError() && base.hostErrCache != nil {
 		u.executerOpts.HostErrorsCache = base.hostErrCache

lib/parser_lifecycle_test.go

@@ -2,10 +2,14 @@ package nuclei
 
 import (
 	"context"
+	"net/http"
+	"net/http/httptest"
 	"os"
 	"path/filepath"
+	"sync"
 	"testing"
 
+	"github.com/projectdiscovery/nuclei/v3/pkg/output"
 	"github.com/projectdiscovery/nuclei/v3/pkg/templates"
 	"github.com/projectdiscovery/nuclei/v3/pkg/types"
 	"github.com/stretchr/testify/require"
@@ -68,3 +72,87 @@ http:
 		require.NotNil(t, cached)
 	}
 }
+
+func TestThreadSafeExecuteUsesSharedCompiledCache(t *testing.T) {
+	templatePath := filepath.Join(t.TempDir(), "template.yaml")
+	require.NoError(t, os.WriteFile(templatePath, []byte(`id: thread-safe-shared-cache
+
+info:
+  name: Thread safe shared cache
+  author: pdteam
+  severity: info
+  tags: thread-safe-shared-cache
+
+http:
+  - method: GET
+    path:
+      - "{{BaseURL}}"
+    matchers:
+      - type: word
+        words:
+          - "thread-safe-shared-cache"
+`), 0o600))
+
+	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
+		_, _ = w.Write([]byte("thread-safe-shared-cache"))
+	}))
+	t.Cleanup(server.Close)
+
+	engine, err := NewThreadSafeNucleiEngineCtx(context.Background())
+	require.NoError(t, err)
+	t.Cleanup(engine.Close)
+
+	const executions = 4
+	var waitGroup sync.WaitGroup
+	errors := make([]error, executions)
+	for i := range executions {
+		waitGroup.Add(1)
+		go func() {
+			defer waitGroup.Done()
+			errors[i] = engine.ExecuteNucleiWithOptsCtx(context.Background(), []string{server.URL},
+				WithTemplatesOrWorkflows(TemplateSources{Templates: []string{templatePath}}),
+				WithTemplateFilters(TemplateFilters{Tags: []string{"thread-safe-shared-cache"}}),
+				WithResultCallback(func(*output.ResultEvent) {}),
+			)
+		}()
+	}
+	waitGroup.Wait()
+
+	for _, err := range errors {
+		require.NoError(t, err)
+	}
+	require.Equal(t, 1, engine.eng.GetParser().CompiledCount())
+}
+
+func TestThreadSafeExecuteHonorsDisableTemplateCache(t *testing.T) {
+	templatePath := filepath.Join(t.TempDir(), "template.yaml")
+	require.NoError(t, os.WriteFile(templatePath, []byte(`id: thread-safe-disable-cache
+
+info:
+  name: Thread safe disable cache
+  author: pdteam
+  severity: info
+  tags: thread-safe-disable-cache
+
+http:
+  - method: GET
+    path:
+      - "{{BaseURL}}"
+`), 0o600))
+
+	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
+		_, _ = w.Write([]byte("ok"))
+	}))
+	t.Cleanup(server.Close)
+
+	engine, err := NewThreadSafeNucleiEngineCtx(context.Background(), DisableTemplateCache())
+	require.NoError(t, err)
+	t.Cleanup(engine.Close)
+
+	err = engine.ExecuteNucleiWithOptsCtx(context.Background(), []string{server.URL},
+		WithTemplatesOrWorkflows(TemplateSources{Templates: []string{templatePath}}),
+		WithTemplateFilters(TemplateFilters{Tags: []string{"thread-safe-disable-cache"}}),
+	)
+	require.NoError(t, err)
+	require.Equal(t, 0, engine.eng.GetParser().CompiledCount())
+}

pkg/protocols/code/code.go

@@ -490,6 +490,15 @@ func prettyPrint(templateId string, buff string) {
 
 // UpdateOptions replaces this request's options with a new copy
 func (r *Request) UpdateOptions(opts *protocols.ExecutorOptions) {
+	r.Operators = protocols.CloneOperators(r.Operators)
+	r.CompiledOperators = nil
+	if r.options == nil {
+		if opts != nil {
+			r.options = opts.Copy()
+		}
+		return
+	}
+	r.options = r.options.Copy()
 	r.options.ApplyNewEngineOptions(opts)
 }
 

pkg/protocols/dns/dns.go

@@ -348,5 +348,14 @@ func classToInt(class string) uint16 {
 
 // UpdateOptions replaces this request's options with a new copy
 func (r *Request) UpdateOptions(opts *protocols.ExecutorOptions) {
+	r.Operators = protocols.CloneOperators(r.Operators)
+	r.CompiledOperators = nil
+	if r.options == nil {
+		if opts != nil {
+			r.options = opts.Copy()
+		}
+		return
+	}
+	r.options = r.options.Copy()
 	r.options.ApplyNewEngineOptions(opts)
 }

pkg/protocols/file/file.go

@@ -230,5 +230,14 @@ func (request *Request) Requests() int {
 
 // UpdateOptions replaces this request's options with a new copy
 func (r *Request) UpdateOptions(opts *protocols.ExecutorOptions) {
+	r.Operators = protocols.CloneOperators(r.Operators)
+	r.CompiledOperators = nil
+	if r.options == nil {
+		if opts != nil {
+			r.options = opts.Copy()
+		}
+		return
+	}
+	r.options = r.options.Copy()
 	r.options.ApplyNewEngineOptions(opts)
 }

pkg/protocols/headless/headless.go

@@ -174,6 +174,15 @@ func (request *Request) Requests() int {
 
 // UpdateOptions replaces this request's options with a new copy
 func (r *Request) UpdateOptions(opts *protocols.ExecutorOptions) {
+	r.Operators = protocols.CloneOperators(r.Operators)
+	r.CompiledOperators = nil
+	if r.options == nil {
+		if opts != nil {
+			r.options = opts.Copy()
+		}
+		return
+	}
+	r.options = r.options.Copy()
 	r.options.ApplyNewEngineOptions(opts)
 }
 

pkg/protocols/http/http.go

@@ -605,6 +605,16 @@ func init() {
 
 // UpdateOptions replaces this request's options with a new copy
 func (r *Request) UpdateOptions(opts *protocols.ExecutorOptions) {
+	r.Operators = protocols.CloneOperators(r.Operators)
+	r.CompiledOperators = nil
+	r.FuzzPreCondition = protocols.CloneMatchers(r.FuzzPreCondition)
+	if r.options == nil {
+		if opts != nil {
+			r.options = opts.Copy()
+		}
+		return
+	}
+	r.options = r.options.Copy()
 	r.options.ApplyNewEngineOptions(opts)
 }
 

pkg/protocols/javascript/js.go

@@ -959,5 +959,14 @@ func prettyPrint(templateId string, buff string) {
 
 // UpdateOptions replaces this request's options with a new copy
 func (r *Request) UpdateOptions(opts *protocols.ExecutorOptions) {
+	r.Operators = protocols.CloneOperators(r.Operators)
+	r.CompiledOperators = nil
+	if r.options == nil {
+		if opts != nil {
+			r.options = opts.Copy()
+		}
+		return
+	}
+	r.options = r.options.Copy()
 	r.options.ApplyNewEngineOptions(opts)
 }

pkg/protocols/network/network.go

@@ -288,5 +288,14 @@ func (request *Request) SetDialer(dialer *fastdialer.Dialer) {
 
 // UpdateOptions replaces this request's options with a new copy
 func (r *Request) UpdateOptions(opts *protocols.ExecutorOptions) {
+	r.Operators = protocols.CloneOperators(r.Operators)
+	r.CompiledOperators = nil
+	if r.options == nil {
+		if opts != nil {
+			r.options = opts.Copy()
+		}
+		return
+	}
+	r.options = r.options.Copy()
 	r.options.ApplyNewEngineOptions(opts)
 }

pkg/protocols/offlinehttp/request.go

@@ -162,5 +162,7 @@ func getURLFromRequest(req *http.Request) string {
 
 // UpdateOptions replaces this request's options with a new copy
 func (r *Request) UpdateOptions(opts *protocols.ExecutorOptions) {
-	r.options = opts
+	if opts != nil {
+		r.options = opts.Copy()
+	}
 }

pkg/protocols/protocols.go

@@ -3,6 +3,7 @@ package protocols
 import (
 	"context"
 	"encoding/base64"
+	"reflect"
 	"sync/atomic"
 
 	"github.com/projectdiscovery/fastdialer/fastdialer"
@@ -344,6 +345,85 @@ func (e *ExecutorOptions) Copy() *ExecutorOptions {
 	return copy
 }
 
+// CloneOperators returns a copy of operators with only template-authored fields.
+// Compiled matcher/extractor state is rebuilt per execution by Request.Compile.
+func CloneOperators(src operators.Operators) operators.Operators {
+	value := cloneExportedValue(reflect.ValueOf(src))
+	cloned := value.Interface().(operators.Operators)
+	cloned.TemplateID = ""
+	cloned.ExcludeMatchers = nil
+	return cloned
+}
+
+// CloneMatchers returns a copy of matcher definitions without compiled matcher
+// state so callers can safely compile them for a single execution.
+func CloneMatchers(src []*matchers.Matcher) []*matchers.Matcher {
+	if src == nil {
+		return nil
+	}
+	value := cloneExportedValue(reflect.ValueOf(src))
+	return value.Interface().([]*matchers.Matcher)
+}
+
+func cloneExportedValue(value reflect.Value) reflect.Value {
+	if !value.IsValid() {
+		return value
+	}
+
+	switch value.Kind() {
+	case reflect.Interface:
+		if value.IsNil() {
+			return reflect.Zero(value.Type())
+		}
+		cloned := cloneExportedValue(value.Elem())
+		result := reflect.New(value.Type()).Elem()
+		result.Set(cloned)
+		return result
+	case reflect.Pointer:
+		if value.IsNil() {
+			return reflect.Zero(value.Type())
+		}
+		cloned := reflect.New(value.Type().Elem())
+		cloned.Elem().Set(cloneExportedValue(value.Elem()))
+		return cloned
+	case reflect.Struct:
+		cloned := reflect.New(value.Type()).Elem()
+		for i := 0; i < value.NumField(); i++ {
+			if value.Type().Field(i).IsExported() {
+				cloned.Field(i).Set(cloneExportedValue(value.Field(i)))
+			}
+		}
+		return cloned
+	case reflect.Slice:
+		if value.IsNil() {
+			return reflect.Zero(value.Type())
+		}
+		cloned := reflect.MakeSlice(value.Type(), value.Len(), value.Len())
+		for i := 0; i < value.Len(); i++ {
+			cloned.Index(i).Set(cloneExportedValue(value.Index(i)))
+		}
+		return cloned
+	case reflect.Map:
+		if value.IsNil() {
+			return reflect.Zero(value.Type())
+		}
+		cloned := reflect.MakeMapWithSize(value.Type(), value.Len())
+		iterator := value.MapRange()
+		for iterator.Next() {
+			cloned.SetMapIndex(cloneExportedValue(iterator.Key()), cloneExportedValue(iterator.Value()))
+		}
+		return cloned
+	case reflect.Array:
+		cloned := reflect.New(value.Type()).Elem()
+		for i := 0; i < value.Len(); i++ {
+			cloned.Index(i).Set(cloneExportedValue(value.Index(i)))
+		}
+		return cloned
+	default:
+		return value
+	}
+}
+
 // Request is an interface implemented any protocol based request generator.
 type Request interface {
 	// Compile compiles the request generators preparing any requests possible.