Skip to content

Commit cf43350

Browse files
kyasbalkyasbal
authored
Improved container ID discovery tasks and implemented pod uid discovery tasks (#396)
* Added new fieldset related tasks and history modifiers for error audit logs * Adding k8s audit log parser tasks * Added LogSorterTask that sorts logs before ingesting them to the manifest generator * Added ChangeTargetGrouperTask that groups logs by the resource paths actually modified with the audit log * Added NonSuccessLogGrouperTask that groups logs by resource paths for non succeeded audit logs * fix issues pointed by gemini-code-assist * fix issues pointed by gemini-code-assist * Adding grouping related tasks and tasks for gathering k8s audit logs from Cloud Logging (#378) * Added new fieldset related tasks and history modifiers for error audit logs * fix issues pointed by gemini-code-assist * Added new fieldset related tasks and history modifiers for error audit logs * Adding k8s audit log parser tasks * Added LogSorterTask that sorts logs before ingesting them to the manifest generator * Added ChangeTargetGrouperTask that groups logs by the resource paths actually modified with the audit log * Added NonSuccessLogGrouperTask that groups logs by resource paths for non succeeded audit logs * fix issues pointed by gemini-code-assist * Adding k8s audit log parser tasks * Added LogSorterTask that sorts logs before ingesting them to the manifest generator * Added ChangeTargetGrouperTask that groups logs by the resource paths actually modified with the audit log * Added NonSuccessLogGrouperTask that groups logs by resource paths for non succeeded audit logs * fix issues pointed by gemini-code-assist * Adding k8s audit log parser tasks * Added LogSorterTask that sorts logs before ingesting them to the manifest generator * Added ChangeTargetGrouperTask that groups logs by the resource paths actually modified with the audit log * Added NonSuccessLogGrouperTask that groups logs by resource paths for non succeeded audit logs * fix issues pointed by gemini-code-assist * Added several test asserter for changeset testing * fix issues pointed by gemini-code-assist * fix issues pointed by gemini-code-assist * fix issue pointed by gemini-code-assist * Migrate containerID discovery tasks to use the inventory task and implemented resource UID inventory * Improved containerd,kubelet and controlplane ID matchers to use inventory tasks
1 parent 9742f71 commit cf43350

26 files changed

Lines changed: 638 additions & 151 deletions

‎pkg/common/patternfinder/finder.go‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,8 @@
1414

1515
package patternfinder
1616

17+
import "fmt"
18+
1719
// PatternMatchResult represents a single match found within a larger text.
1820
// It includes the start and end positions of the match.
1921
type PatternMatchResult[T any] struct {
@@ -22,6 +24,14 @@ type PatternMatchResult[T any] struct {
2224
End int
2325
}
2426

27+
// GetMatchedString extracts the matched string from the original string.
28+
func (p *PatternMatchResult[T]) GetMatchedString(original string) (string, error) {
29+
if p.Start < 0 || p.End > len(original) {
30+
return "", fmt.Errorf("invalid match range: start=%d, end=%d", p.Start, p.End)
31+
}
32+
return original[p.Start:p.End], nil
33+
}
34+
2535
// FindAllWithStarterRunes finds all occurrences of patterns within a search text.
2636
// The search for a pattern only begins after encountering one of the specified starterRunes.
2737
//

‎pkg/common/patternfinder/finder_test.go‎

Lines changed: 46 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -133,6 +133,52 @@ func TestFindAllWithStarterRunes(t *testing.T) {
133133
}
134134
}
135135

136+
func TestGetMatchedString(t *testing.T) {
137+
testCases := []struct {
138+
desc string
139+
result PatternMatchResult[struct{}]
140+
original string
141+
want string
142+
wantErr bool
143+
}{
144+
{
145+
desc: "simple case",
146+
result: PatternMatchResult[struct{}]{
147+
Value: struct{}{},
148+
Start: 0,
149+
End: 5,
150+
},
151+
original: "hello",
152+
want: "hello",
153+
},
154+
{
155+
desc: "original message has short length than result",
156+
result: PatternMatchResult[struct{}]{
157+
Value: struct{}{},
158+
Start: 0,
159+
End: 5,
160+
},
161+
original: "",
162+
want: "",
163+
wantErr: true,
164+
},
165+
}
166+
for _, tc := range testCases {
167+
t.Run(tc.desc, func(t *testing.T) {
168+
got, err := tc.result.GetMatchedString(tc.original)
169+
if tc.wantErr {
170+
if err == nil {
171+
t.Errorf("GetMatchedString() = %v, want error %v", got, tc.want)
172+
}
173+
} else {
174+
if got != tc.want {
175+
t.Errorf("GetMatchedString() = %v, want %v", got, tc.want)
176+
}
177+
}
178+
})
179+
}
180+
}
181+
136182
func BenchmarkFindAllWithStarterRunes(b *testing.B) {
137183
finders := []struct {
138184
name string

‎pkg/core/inspection/taskbase/inventory_task.go‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -108,7 +108,7 @@ func (s *InventoryTaskBuilder[T]) DiscoveryTask(taskID taskid.TaskImplementation
108108
// InventoryMergerStrategy defines the strategy how the generated InventoryTask merges results received from multiple discovery tasks.
109109
type InventoryMergerStrategy[T any] interface {
110110

111-
// Merge defines the logic to combine multiple results from various discovery tasks
111+
// Merge defines the logic to combine multiple results from various InventoryDiscoveryTasks
112112
// into a single, consolidated result.
113113
Merge(results []T) (T, error)
114114
}

‎pkg/core/inspection/test/testutil.go‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -22,8 +22,10 @@ import (
2222
"github.com/GoogleCloudPlatform/khi/pkg/common/typedmap"
2323
inspectionmetadata "github.com/GoogleCloudPlatform/khi/pkg/core/inspection/metadata"
2424
coretask "github.com/GoogleCloudPlatform/khi/pkg/core/task"
25+
"github.com/GoogleCloudPlatform/khi/pkg/core/task/taskid"
2526
tasktest "github.com/GoogleCloudPlatform/khi/pkg/core/task/test"
2627
"github.com/GoogleCloudPlatform/khi/pkg/model/history"
28+
core_contract "github.com/GoogleCloudPlatform/khi/pkg/task/core/contract"
2729
inspectioncore_contract "github.com/GoogleCloudPlatform/khi/pkg/task/inspection/inspectioncore/contract"
2830
)
2931

@@ -39,6 +41,17 @@ func WithDefaultTestInspectionTaskContext(baseContext context.Context) context.C
3941
taskCtx = khictx.WithValue(taskCtx, inspectioncore_contract.GlobalSharedMap, typedmap.NewTypedMap())
4042
taskCtx = khictx.WithValue(taskCtx, inspectioncore_contract.InspectionSharedMap, typedmap.NewTypedMap())
4143

44+
// If this context is used with the task runner, it should have the task result map. But if not, then this must complement the value with the default value.
45+
_, err := khictx.GetValue(taskCtx, core_contract.TaskResultMapContextKey)
46+
if err != nil {
47+
taskCtx = khictx.WithValue(taskCtx, core_contract.TaskResultMapContextKey, typedmap.NewTypedMap())
48+
}
49+
_, err = khictx.GetValue(taskCtx, core_contract.TaskImplementationIDContextKey)
50+
if err != nil {
51+
fakeTaskID := taskid.NewDefaultImplementationID[struct{}]("khi.google.com/fake-test-id")
52+
taskCtx = khictx.WithValue(taskCtx, core_contract.TaskImplementationIDContextKey, fakeTaskID.(taskid.UntypedTaskImplementationID))
53+
}
54+
4255
ioConfig, err := inspectioncore_contract.NewIOConfigForTest()
4356
if err != nil {
4457
panic("Failed to create test IOConfig: " + err.Error())

‎pkg/task/inspection/commonlogk8sauditv2/contract/inventory.go‎

Lines changed: 15 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -19,7 +19,21 @@ import (
1919
"github.com/GoogleCloudPlatform/khi/pkg/core/task/taskid"
2020
)
2121

22-
var NodeNameInventoryTaskID = taskid.NewDefaultImplementationID[[]string]("node-name-inventory-merger")
22+
var NodeNameInventoryTaskID = taskid.NewDefaultImplementationID[[]string](TaskIDPrefix + "node-name-inventory")
2323

2424
// NodeNameInventoryBuilder is the inventory tasks builder for gathering node names
2525
var NodeNameInventoryBuilder = inspectiontaskbase.NewInventoryTaskBuilder[[]string](NodeNameInventoryTaskID)
26+
27+
type UIDToResourceIdentity = map[string]*ResourceIdentity
28+
29+
var ResourceUIDInventoryTaskID = taskid.NewDefaultImplementationID[UIDToResourceIdentity](TaskIDPrefix + "resource-uid-inventory")
30+
31+
// ResourceUIDInventoryBuilder is the inventory tasks builder for gathering resource uids
32+
var ResourceUIDInventoryBuilder = inspectiontaskbase.NewInventoryTaskBuilder[UIDToResourceIdentity](ResourceUIDInventoryTaskID)
33+
34+
type ContainerIDToContainerIdentity = map[string]*ContainerIdentity
35+
36+
var ContainerIDInventoryTaskID = taskid.NewDefaultImplementationID[ContainerIDToContainerIdentity](TaskIDPrefix + "container-id-inventory")
37+
38+
// ContainerIDInventoryBuilder is the inventory tasks builder for gathering the relationship between container id and container identity
39+
var ContainerIDInventoryBuilder = inspectiontaskbase.NewInventoryTaskBuilder[ContainerIDToContainerIdentity](ContainerIDInventoryTaskID)

‎pkg/task/inspection/commonlogk8sauditv2/contract/taskid.go‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,7 @@
1515
package commonlogk8sauditv2_contract
1616

1717
import (
18+
"github.com/GoogleCloudPlatform/khi/pkg/common/patternfinder"
1819
inspectiontaskbase "github.com/GoogleCloudPlatform/khi/pkg/core/inspection/taskbase"
1920
"github.com/GoogleCloudPlatform/khi/pkg/core/task/taskid"
2021
"github.com/GoogleCloudPlatform/khi/pkg/model/log"
@@ -86,3 +87,15 @@ var ConditionHistoryModifierTaskID = taskid.NewDefaultImplementationID[struct{}]
8687

8788
// NodeNameDiscoveryTaskID is the task ID for extracting node names from audit logs.
8889
var NodeNameDiscoveryTaskID = taskid.NewDefaultImplementationID[[]string](TaskIDPrefix + "node-name-discovery")
90+
91+
// ResourceUIDDiscoveryTaskID is the task ID for extracting resource uids from audit logs.
92+
var ResourceUIDDiscoveryTaskID = taskid.NewDefaultImplementationID[UIDToResourceIdentity](TaskIDPrefix + "resource-uid-discovery")
93+
94+
// ResourceUIDPatternFinderTaskID is the task ID to build the PatternFinder from aggregated UIDs obtained from the inventory task.
95+
var ResourceUIDPatternFinderTaskID = taskid.NewDefaultImplementationID[patternfinder.PatternFinder[*ResourceIdentity]](TaskIDPrefix + "resource-uid-pattern-finder")
96+
97+
// ContainerIDDiscoveryTaskID is the task ID for extracting container ids from audit logs.
98+
var ContainerIDDiscoveryTaskID = taskid.NewDefaultImplementationID[ContainerIDToContainerIdentity](TaskIDPrefix + "container-id-discovery")
99+
100+
// ContainerIDPatternFinderTaskID is the task ID to build the PatternFinder from aggregated container ids obtained from the inventory task.
101+
var ContainerIDPatternFinderTaskID = taskid.NewDefaultImplementationID[patternfinder.PatternFinder[*ContainerIdentity]](TaskIDPrefix + "container-id-pattern-finder")

‎pkg/task/inspection/commonlogk8sauditv2/contract/type.go‎

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -113,6 +113,30 @@ func (r *ResourceIdentity) ParentIdentity() *ResourceIdentity {
113113
}
114114
}
115115

116+
type ContainerIdentity struct {
117+
ContainerID string
118+
ContainerName string
119+
PodSandboxID string
120+
}
121+
122+
func (c *ContainerIdentity) Merge(other *ContainerIdentity) *ContainerIdentity {
123+
result := *c
124+
if other.ContainerID != "" {
125+
result.ContainerID = other.ContainerID
126+
}
127+
if other.ContainerName != "" {
128+
result.ContainerName = other.ContainerName
129+
}
130+
if other.PodSandboxID != "" {
131+
result.PodSandboxID = other.PodSandboxID
132+
}
133+
return &result
134+
}
135+
136+
func (c *ContainerIdentity) ResourcePath(podNamespace string, podName string) resourcepath.ResourcePath {
137+
return resourcepath.Container(podNamespace, podName, c.ContainerName)
138+
}
139+
116140
// ResourceLogGroup is the group of the logs associated with k8s resource.
117141
type ResourceLogGroup struct {
118142
// Resource is the resource identity.

‎pkg/task/inspection/commonlogk8sauditv2/impl/container_historymodifier_task.go‎

Lines changed: 7 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -40,15 +40,15 @@ const (
4040
ContainerTypeEphemeral containerType = "ephemeral"
4141
)
4242

43-
type containerIdentity struct {
43+
type containerStatusIdentity struct {
4444
// containerName is the name of the container.
4545
containerName string
4646
// containerType is the type of the container.
4747
containerType containerType
4848
}
4949

5050
// String returns the string representation of the container identity.
51-
func (c *containerIdentity) String() string {
51+
func (c *containerStatusIdentity) String() string {
5252
switch c.containerType {
5353
case ContainerTypeContainer:
5454
return c.containerName
@@ -66,7 +66,7 @@ var ContainerHistoryModifierTask = commonlogk8sauditv2_contract.NewManifestHisto
6666

6767
type containerHistoryModifierTaskState struct {
6868
// containerIdentities is the map of container identities.
69-
containerIdentities map[string]*containerIdentity
69+
containerIdentities map[string]*containerStatusIdentity
7070
// containerStateWalkers is the map of container state walkers.
7171
containerStateWalkers map[string]*containerStateWalker
7272
}
@@ -98,7 +98,7 @@ func (c *containerHistoryModifierTaskSetting) PassCount() int {
9898
func (c *containerHistoryModifierTaskSetting) Process(ctx context.Context, passIndex int, event commonlogk8sauditv2_contract.ResourceChangeEvent, cs *history.ChangeSet, builder *history.Builder, state *containerHistoryModifierTaskState) (*containerHistoryModifierTaskState, error) {
9999
if state == nil {
100100
state = &containerHistoryModifierTaskState{
101-
containerIdentities: map[string]*containerIdentity{},
101+
containerIdentities: map[string]*containerStatusIdentity{},
102102
containerStateWalkers: map[string]*containerStateWalker{},
103103
}
104104
}
@@ -124,7 +124,7 @@ func (c *containerHistoryModifierTaskSetting) processFirstPass(ctx context.Conte
124124
for _, status := range statuses.Children() {
125125
name, err := status.ReadString("name")
126126
if err == nil {
127-
identity := &containerIdentity{
127+
identity := &containerStatusIdentity{
128128
containerName: name,
129129
containerType: containerType,
130130
}
@@ -148,7 +148,7 @@ func (c *containerHistoryModifierTaskSetting) processSecondPass(ctx context.Cont
148148
for _, status := range statuses.Children() {
149149
name, err := status.ReadString("name")
150150
if err == nil {
151-
identity := containerIdentity{
151+
identity := containerStatusIdentity{
152152
containerName: name,
153153
containerType: containerType,
154154
}
@@ -201,7 +201,7 @@ var _ commonlogk8sauditv2_contract.ManifestHistoryModifierTaskSetting[*container
201201

202202
type containerStateWalker struct {
203203
// containerIdentity is the identity of the container.
204-
containerIdentity *containerIdentity
204+
containerIdentity *containerStatusIdentity
205205
// podNamespace is the namespace of the pod.
206206
podNamespace string
207207
// podName is the name of the pod.

‎pkg/task/inspection/commonlogk8sauditv2/impl/container_historymodifier_task_test.go‎

Lines changed: 12 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -352,7 +352,7 @@ state:
352352
for _, tt := range tests {
353353
t.Run(tt.name, func(t *testing.T) {
354354
walker := &containerStateWalker{
355-
containerIdentity: &containerIdentity{
355+
containerIdentity: &containerStatusIdentity{
356356
containerName: containerName,
357357
containerType: ContainerTypeContainer,
358358
},
@@ -417,7 +417,7 @@ status:
417417
`,
418418
initialState: nil,
419419
wantState: &containerHistoryModifierTaskState{
420-
containerIdentities: map[string]*containerIdentity{
420+
containerIdentities: map[string]*containerStatusIdentity{
421421
"main-container": {
422422
containerName: "main-container",
423423
containerType: ContainerTypeContainer,
@@ -440,7 +440,7 @@ status:
440440
pass: 0,
441441
nilBody: true,
442442
wantState: &containerHistoryModifierTaskState{
443-
containerIdentities: map[string]*containerIdentity{},
443+
containerIdentities: map[string]*containerStatusIdentity{},
444444
containerStateWalkers: map[string]*containerStateWalker{},
445445
},
446446
asserters: []testchangeset.ChangeSetAsserter{},
@@ -458,7 +458,7 @@ status:
458458
ready: true
459459
`,
460460
initialState: &containerHistoryModifierTaskState{
461-
containerIdentities: map[string]*containerIdentity{
461+
containerIdentities: map[string]*containerStatusIdentity{
462462
"main-container": {
463463
containerName: "main-container",
464464
containerType: ContainerTypeContainer,
@@ -467,15 +467,15 @@ status:
467467
containerStateWalkers: map[string]*containerStateWalker{},
468468
},
469469
wantState: &containerHistoryModifierTaskState{
470-
containerIdentities: map[string]*containerIdentity{
470+
containerIdentities: map[string]*containerStatusIdentity{
471471
"main-container": {
472472
containerName: "main-container",
473473
containerType: ContainerTypeContainer,
474474
},
475475
},
476476
containerStateWalkers: map[string]*containerStateWalker{
477477
"main-container": {
478-
containerIdentity: &containerIdentity{
478+
containerIdentity: &containerStatusIdentity{
479479
containerName: "main-container",
480480
containerType: ContainerTypeContainer,
481481
},
@@ -508,7 +508,7 @@ status:
508508
containerStatuses: []
509509
`,
510510
initialState: &containerHistoryModifierTaskState{
511-
containerIdentities: map[string]*containerIdentity{
511+
containerIdentities: map[string]*containerStatusIdentity{
512512
"main-container": {
513513
containerName: "main-container",
514514
containerType: ContainerTypeContainer,
@@ -517,15 +517,15 @@ status:
517517
containerStateWalkers: map[string]*containerStateWalker{},
518518
},
519519
wantState: &containerHistoryModifierTaskState{
520-
containerIdentities: map[string]*containerIdentity{
520+
containerIdentities: map[string]*containerStatusIdentity{
521521
"main-container": {
522522
containerName: "main-container",
523523
containerType: ContainerTypeContainer,
524524
},
525525
},
526526
containerStateWalkers: map[string]*containerStateWalker{
527527
"main-container": {
528-
containerIdentity: &containerIdentity{
528+
containerIdentity: &containerStatusIdentity{
529529
containerName: "main-container",
530530
containerType: ContainerTypeContainer,
531531
},
@@ -555,7 +555,7 @@ status:
555555
pass: 1,
556556
nilBody: true,
557557
initialState: &containerHistoryModifierTaskState{
558-
containerIdentities: map[string]*containerIdentity{
558+
containerIdentities: map[string]*containerStatusIdentity{
559559
"main-container": {
560560
containerName: "main-container",
561561
containerType: ContainerTypeContainer,
@@ -564,7 +564,7 @@ status:
564564
containerStateWalkers: map[string]*containerStateWalker{},
565565
},
566566
wantState: &containerHistoryModifierTaskState{
567-
containerIdentities: map[string]*containerIdentity{
567+
containerIdentities: map[string]*containerStatusIdentity{
568568
"main-container": {
569569
containerName: "main-container",
570570
containerType: ContainerTypeContainer,
@@ -610,7 +610,7 @@ status:
610610
t.Fatalf("Process(%d) failed: %v", tc.pass, err)
611611
}
612612

613-
if diff := cmp.Diff(tc.wantState, nextState, cmp.AllowUnexported(containerHistoryModifierTaskState{}, containerIdentity{}, containerStateWalker{})); diff != "" {
613+
if diff := cmp.Diff(tc.wantState, nextState, cmp.AllowUnexported(containerHistoryModifierTaskState{}, containerStatusIdentity{}, containerStateWalker{})); diff != "" {
614614
t.Errorf("state mismatch (-want +got):\n%s", diff)
615615
}
616616

0 commit comments

Comments
 (0)