@@ -662,6 +662,280 @@ func TestExpandedPostingsCacheFuzz(t *testing.T) {
662662 }
663663}
664664
665+ // TestLazyMatchersFuzz fuzzes PromQL queries against two cortex instances with
666+ // identical data:
667+ // - cortex-1: head expanded-postings cache enabled, lazy matcher DISABLED
668+ // (the eager path - regex applied during postings lookup).
669+ // - cortex-2: head expanded-postings cache enabled, lazy matcher ENABLED
670+ // with aggressive thresholds (cardinality=1, both cost ratios=1) so the
671+ // optimization fires on every regex matcher.
672+ //
673+ // The test verifies:
674+ // 1. Query results match between the two instances (correctness).
675+ // 2. The cortex_ingester_expanded_postings_lazy_matcher_queries_total counter
676+ // is incremented on cortex-2 (the optimization actually triggers).
677+ func TestLazyMatchersFuzz (t * testing.T ) {
678+ s , err := e2e .NewScenario (networkName )
679+ require .NoError (t , err )
680+ defer s .Close ()
681+
682+ // Start dependencies.
683+ consul1 := e2edb .NewConsulWithName ("consul1" )
684+ consul2 := e2edb .NewConsulWithName ("consul2" )
685+ require .NoError (t , s .StartAndWaitReady (consul1 , consul2 ))
686+
687+ baseFlags := mergeFlags (
688+ AlertmanagerLocalFlags (),
689+ map [string ]string {
690+ "-store.engine" : blocksStorageEngine ,
691+ "-blocks-storage.backend" : "filesystem" ,
692+ "-blocks-storage.tsdb.head-compaction-interval" : "4m" ,
693+ "-blocks-storage.tsdb.block-ranges-period" : "2h" ,
694+ "-blocks-storage.tsdb.ship-interval" : "1h" ,
695+ "-blocks-storage.bucket-store.sync-interval" : "15m" ,
696+ "-blocks-storage.tsdb.retention-period" : "2h" ,
697+ "-blocks-storage.bucket-store.index-cache.backend" : tsdb .IndexCacheBackendInMemory ,
698+ "-blocks-storage.bucket-store.bucket-index.enabled" : "true" ,
699+ "-blocks-storage.expanded_postings_cache.head.enabled" : "true" ,
700+ "-blocks-storage.expanded_postings_cache.block.enabled" : "true" ,
701+ "-distributor.replication-factor" : "1" ,
702+ "-store-gateway.sharding-enabled" : "false" ,
703+ "-alertmanager.web.external-url" : "http://localhost/alertmanager" ,
704+ // The alertmanager initializes a memberlist gossip ring that auto-
705+ // detects a private RFC1918 IP. On Docker networks where containers
706+ // get non-private IPs (e.g. the 240.0.0.0/4 reserved range), this
707+ // detection hard-fails. Setting an explicit advertise address skips
708+ // the autodetection — the value is unused since we don't enable HA
709+ // peers, but presence of the flag is enough.
710+ "-alertmanager.cluster.advertise-address" : "127.0.0.1:9094" ,
711+ },
712+ )
713+
714+ // cortex-1: eager path. Lazy matcher disabled (default).
715+ flags1 := mergeFlags (baseFlags , map [string ]string {
716+ "-ring.store" : "consul" ,
717+ "-consul.hostname" : consul1 .NetworkHTTPEndpoint (),
718+ "-ingester.matchers-cache-max-items" : "10000" ,
719+ })
720+
721+ // cortex-2: lazy path. Aggressive thresholds force the optimization to
722+ // fire on essentially every regex matcher, so we exercise the lazy code
723+ // path repeatedly for correctness verification.
724+ flags2 := mergeFlags (baseFlags , map [string ]string {
725+ "-ring.store" : "consul" ,
726+ "-consul.hostname" : consul2 .NetworkHTTPEndpoint (),
727+ "-ingester.matchers-cache-max-items" : "10000" ,
728+ "-blocks-storage.expanded_postings_cache.head.lazy-matcher-max-cardinality" : "1" ,
729+ "-blocks-storage.expanded_postings_cache.head.lazy-matcher-simple-cost-ratio" : "1" ,
730+ "-blocks-storage.expanded_postings_cache.head.lazy-matcher-complex-cost-ratio" : "1" ,
731+ })
732+
733+ require .NoError (t , writeFileToSharedDir (s , "alertmanager_configs" , []byte {}))
734+
735+ path1 := path .Join (s .SharedDir (), "cortex-1" )
736+ path2 := path .Join (s .SharedDir (), "cortex-2" )
737+ flags1 = mergeFlags (flags1 , map [string ]string {"-blocks-storage.filesystem.dir" : path1 })
738+ flags2 = mergeFlags (flags2 , map [string ]string {"-blocks-storage.filesystem.dir" : path2 })
739+
740+ // Both instances use the local build.
741+ cortex1 := e2ecortex .NewSingleBinary ("cortex-1" , flags1 , "" )
742+ cortex2 := e2ecortex .NewSingleBinary ("cortex-2" , flags2 , "" )
743+ require .NoError (t , s .StartAndWaitReady (cortex1 , cortex2 ))
744+
745+ require .NoError (t , cortex1 .WaitSumMetrics (e2e .Equals (float64 (512 )), "cortex_ring_tokens_total" ))
746+ require .NoError (t , cortex2 .WaitSumMetrics (e2e .Equals (float64 (512 )), "cortex_ring_tokens_total" ))
747+
748+ c1 , err := e2ecortex .NewClient (cortex1 .HTTPEndpoint (), cortex1 .HTTPEndpoint (), "" , "" , "user-1" )
749+ require .NoError (t , err )
750+ c2 , err := e2ecortex .NewClient (cortex2 .HTTPEndpoint (), cortex2 .HTTPEndpoint (), "" , "" , "user-1" )
751+ require .NoError (t , err )
752+
753+ now := time .Now ()
754+ start := now .Add (- 24 * time .Hour )
755+ scrapeInterval := 30 * time .Second
756+
757+ // Build a fixture with multiple labels, including a high-cardinality
758+ // "pod"-style label so regex matchers from promqlsmith actually exercise
759+ // the deferral path. With lazy-matcher-max-cardinality=1, any label with
760+ // >1 unique value is eligible.
761+ numSeries := 10
762+ numberOfLabelsPerSeries := 5
763+ numSamples := 10
764+ ss := make ([]prompb.TimeSeries , numSeries * numberOfLabelsPerSeries )
765+ lbls := make ([]labels.Labels , numSeries * numberOfLabelsPerSeries )
766+
767+ for i := 0 ; i < numSeries ; i ++ {
768+ for j := 0 ; j < numberOfLabelsPerSeries ; j ++ {
769+ series := e2e .GenerateSeriesWithSamples (
770+ fmt .Sprintf ("test_series_%d" , i ),
771+ start ,
772+ scrapeInterval ,
773+ i * numSamples ,
774+ numSamples ,
775+ prompb.Label {Name : "test_label" , Value : fmt .Sprintf ("test_label_value_%d" , j )},
776+ prompb.Label {Name : "pod" , Value : fmt .Sprintf ("test_pod_%d_%d" , i , j )},
777+ )
778+ ss [i * numberOfLabelsPerSeries + j ] = series
779+
780+ builder := labels .NewBuilder (labels .EmptyLabels ())
781+ for _ , lbl := range series .Labels {
782+ builder .Set (lbl .Name , lbl .Value )
783+ }
784+ lbls [i * numberOfLabelsPerSeries + j ] = builder .Labels ()
785+ }
786+ }
787+
788+ for _ , client := range []* e2ecortex.Client {c1 , c2 } {
789+ res , err := client .Push (ss )
790+ require .NoError (t , err )
791+ require .Equal (t , 200 , res .StatusCode )
792+ }
793+
794+ rnd := rand .New (rand .NewSource (now .Unix ()))
795+ opts := []promqlsmith.Option {
796+ promqlsmith .WithEnabledAggrs (enabledAggrs ),
797+ }
798+ ps := promqlsmith .New (rnd , lbls , opts ... )
799+
800+ // Regex patterns that exercise different cost classes in the lazy matcher gate.
801+ // Each pattern matches a SUBSET of pods (not all), so both =~ and !~ queries
802+ // return non-empty results, verifying correctness with actual data.
803+ regexPatterns := []string {
804+ ".*_0_.*" , // single contains (simple) — 5/50 pods
805+ ".*_[0-4]_[0-2]" , // character class (complex) — 15/50 pods
806+ "test_pod_[5-9]_.*" , // prefix + class (complex) — 25/50 pods
807+ ".*pod_3.*" , // single contains (simple) — 5/50 pods
808+ "(test_pod_1|test_pod_2)_.*" , // alternation (complex) — 10/50 pods
809+ }
810+
811+ testRun := 300
812+ queries := make ([]string , 0 , testRun * 2 )
813+ matchers := make ([]string , 0 , testRun )
814+ for i := 0 ; i < testRun ; i ++ {
815+ expr := ps .WalkRangeQuery ()
816+ if ! isValidQuery (expr , true ) {
817+ continue
818+ }
819+ queries = append (queries , expr .Pretty (0 ))
820+
821+ // Each matcher set includes a __name__= anchor + a regex on pod,
822+ // guaranteeing the lazy matcher optimization fires on every cache miss.
823+ regex := regexPatterns [i % len (regexPatterns )]
824+ matchers = append (matchers , storepb .PromMatchersToString (
825+ append (
826+ ps .WalkSelectors (),
827+ labels .MustNewMatcher (labels .MatchEqual , "__name__" , fmt .Sprintf ("test_series_%d" , i % numSeries )),
828+ labels .MustNewMatcher (labels .MatchRegexp , "pod" , regex ),
829+ )... ))
830+
831+ // Also generate a direct PromQL query with the regex so the instant/range
832+ // query path exercises the lazy matcher too. Include iteration index in
833+ // a != matcher to force unique cache keys (cache miss on every query).
834+ queries = append (queries , fmt .Sprintf (`test_series_%d{pod=~"%s",test_label!="iter_%d"}` , i % numSeries , regex , i ))
835+ // Also test negative regex (!~) to exercise that code path.
836+ queries = append (queries , fmt .Sprintf (`test_series_%d{pod!~"%s",test_label!="iter_%d_neg"}` , i % numSeries , regex , i ))
837+ }
838+
839+ type testCase struct {
840+ query string
841+ qt string
842+ res1 , res2 model.Value
843+ sres1 , sres2 []model.LabelSet
844+ err1 , err2 error
845+ }
846+
847+ cases := make ([]* testCase , 0 , len (queries )* 2 + len (matchers ))
848+
849+ // Data spans [start, start + (numSamples-1)*scrapeInterval]. Constrain
850+ // fuzzed timestamps to this window so queries actually hit the head block.
851+ dataEnd := start .Add (scrapeInterval * time .Duration (numSamples - 1 ))
852+ dataWindowMs := dataEnd .Sub (start ).Milliseconds ()
853+
854+ for _ , query := range queries {
855+ fuzzyTime := time .Duration (rand .Int63n (dataWindowMs ))
856+ queryEnd := start .Add (fuzzyTime * time .Millisecond )
857+ res1 , err1 := c1 .Query (query , queryEnd )
858+ res2 , err2 := c2 .Query (query , queryEnd )
859+ cases = append (cases , & testCase {
860+ query : query , qt : "instant" ,
861+ res1 : res1 , res2 : res2 , err1 : err1 , err2 : err2 ,
862+ })
863+ res1 , err1 = c1 .QueryRange (query , start , queryEnd , scrapeInterval )
864+ res2 , err2 = c2 .QueryRange (query , start , queryEnd , scrapeInterval )
865+ cases = append (cases , & testCase {
866+ query : query , qt : "range query" ,
867+ res1 : res1 , res2 : res2 , err1 : err1 , err2 : err2 ,
868+ })
869+ }
870+
871+ for _ , m := range matchers {
872+ fuzzyTime := time .Duration (rand .Int63n (dataWindowMs ))
873+ queryEnd := start .Add (fuzzyTime * time .Millisecond )
874+ res1 , err := c1 .Series ([]string {m }, start , queryEnd )
875+ require .NoError (t , err )
876+ res2 , err := c2 .Series ([]string {m }, start , queryEnd )
877+ require .NoError (t , err )
878+ cases = append (cases , & testCase {
879+ query : m , qt : "get series" ,
880+ sres1 : res1 , sres2 : res2 ,
881+ })
882+ }
883+
884+ failures := 0
885+ for i , tc := range cases {
886+ if tc .err1 != nil || tc .err2 != nil {
887+ if ! cmp .Equal (tc .err1 , tc .err2 ) {
888+ t .Logf ("case %d error mismatch.\n %s: %s\n err1: %v\n err2: %v\n " , i , tc .qt , tc .query , tc .err1 , tc .err2 )
889+ failures ++
890+ }
891+ } else if shouldUseSampleNumComparer (tc .query ) {
892+ if ! cmp .Equal (tc .res1 , tc .res2 , sampleNumComparer ) {
893+ t .Logf ("case %d # of samples mismatch.\n %s: %s\n res1: %s\n res2: %s\n " , i , tc .qt , tc .query , tc .res1 .String (), tc .res2 .String ())
894+ failures ++
895+ }
896+ } else if ! cmp .Equal (tc .res1 , tc .res2 , comparer ) {
897+ t .Logf ("case %d results mismatch.\n %s: %s\n res1: %s\n res2: %s\n " , i , tc .qt , tc .query , tc .res1 .String (), tc .res2 .String ())
898+ failures ++
899+ } else if ! cmp .Equal (tc .sres1 , tc .sres2 , labelSetsComparer ) {
900+ t .Logf ("case %d series results mismatch.\n %s: %s\n sres1: %s\n sres2: %s\n " , i , tc .qt , tc .query , tc .sres1 , tc .sres2 )
901+ failures ++
902+ }
903+ }
904+ if failures > 0 {
905+ require .Failf (t , "finished lazy matcher fuzzing tests" , "%d test cases failed" , failures )
906+ }
907+
908+ // Verify the lazy-matcher optimization was actually triggered on cortex-2.
909+ // If the gate is misconfigured or the test fixture doesn't exercise the
910+ // path, this guards against silent regressions where the optimization
911+ // becomes a no-op.
912+
913+ // Diagnostic: print related counters before the assertion so failures
914+ // can be debugged from the test output.
915+ for _ , m := range []string {
916+ "cortex_ingester_queries" ,
917+ "cortex_ingester_queried_series" ,
918+ "cortex_ingester_queried_chunks" ,
919+ "cortex_ingester_expanded_postings_cache_requests_total" ,
920+ "cortex_ingester_expanded_postings_cache_hits_total" ,
921+ "cortex_ingester_expanded_postings_non_cacheable_queries_total" ,
922+ "cortex_ingester_expanded_postings_lazy_matcher_queries_total" ,
923+ } {
924+ v , _ := cortex2 .SumMetrics ([]string {m })
925+ t .Logf ("cortex-2 %s = %v" , m , v )
926+ }
927+
928+ require .NoError (t , cortex2 .WaitSumMetrics (e2e .Greater (0 ),
929+ "cortex_ingester_expanded_postings_lazy_matcher_queries_total" ))
930+
931+ // Sanity check: cortex-1 (eager) should NEVER increment this counter.
932+ c1Lazy , err := cortex1 .SumMetrics ([]string {"cortex_ingester_expanded_postings_lazy_matcher_queries_total" })
933+ if err == nil && len (c1Lazy ) > 0 {
934+ require .Equal (t , float64 (0 ), c1Lazy [0 ],
935+ "cortex-1 has lazy matcher disabled but the metric is non-zero" )
936+ }
937+ }
938+
665939func TestVerticalShardingFuzz (t * testing.T ) {
666940 s , err := e2e .NewScenario (networkName )
667941 require .NoError (t , err )
0 commit comments