Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@ package commonlogk8sauditv2_impl
import (
"context"
"slices"
"sort"
"time"

"github.com/GoogleCloudPlatform/khi/pkg/common/structured"
Expand Down Expand Up @@ -82,26 +83,42 @@ func (c *conditionHistoryModifierTaskSetting) ResourcePairs(ctx context.Context,

// processFirstPass collects all available condition types from the log.
// This is necessary because some conditions might appear later in the history, and we need to know about them upfront to track their state correctly.
func (c *conditionHistoryModifierTaskSetting) processFirstPass(ctx context.Context, event commonlogk8sauditv2_contract.ResourceChangeEvent, cs *history.ChangeSet, builder *history.Builder, prevResource *conditionHistoryModifierTaskState) (*conditionHistoryModifierTaskState, error) {
if prevResource == nil {
prevResource = &conditionHistoryModifierTaskState{
func (c *conditionHistoryModifierTaskSetting) processFirstPass(ctx context.Context, event commonlogk8sauditv2_contract.ResourceChangeEvent, cs *history.ChangeSet, builder *history.Builder, state *conditionHistoryModifierTaskState) (*conditionHistoryModifierTaskState, error) {
if state == nil {
state = &conditionHistoryModifierTaskState{
AvailableTypes: map[string]struct{}{},
ConditionWalkers: map[string]*conditionWalker{},
}
}
commonFieldSet := log.MustGetFieldSet(event.Log, &log.CommonFieldSet{})
k8sFieldSet := log.MustGetFieldSet(event.Log, &commonlogk8sauditv2_contract.K8sAuditLogFieldSet{})
ownerPath := resourcepath.ResourcePath{
Path: event.EventTargetResource.ResourcePathString(),
ParentRelationship: enum.RelationshipChild,
}
if event.EventTargetBodyReader != nil {
conditionsReader, err := event.EventTargetBodyReader.GetReader("status.conditions")
if err != nil {
return prevResource, nil
return state, nil
}
for _, child := range conditionsReader.Children() {
conditionType, err := child.ReadString("type")
if err == nil {
prevResource.AvailableTypes[conditionType] = struct{}{}
state.AvailableTypes[conditionType] = struct{}{}
walker := state.ConditionWalkers[conditionType]
if walker == nil {
walker = newConditionWalker(ownerPath, conditionType)
state.ConditionWalkers[conditionType] = walker
}
var condition model.K8sResourceStatusCondition
if err := structured.ReadReflect(&child, "", &condition); err != nil {
continue
}
walker.checkLastTransitionTimes(commonFieldSet, k8sFieldSet, &condition)
}
}
}
return prevResource, nil
return state, nil
}

// processSecondPass generates revisions for each condition type based on the collected available types.
Expand Down Expand Up @@ -231,16 +248,29 @@ type conditionWalker struct {
// minChangeTime is the minimum change time.
// This is used not to create a revision too ealier for the resource retaining the condition after recreation.
minChangeTime *time.Time

lastTransitionStates map[string]*model.K8sResourceStatusCondition

lastTransitionTimeSorted []*time.Time
}

// newConditionWalker creates a new conditionWalker for a specific condition type.
func newConditionWalker(parentResource resourcepath.ResourcePath, stateType string) *conditionWalker {
return &conditionWalker{
parentResource: parentResource,
conditionType: stateType,
lastStatus: "",
lastTransitionTime: "",
lastProbeLikeTime: "",
parentResource: parentResource,
conditionType: stateType,
lastStatus: "",
lastTransitionTime: "",
lastProbeLikeTime: "",
lastTransitionStates: map[string]*model.K8sResourceStatusCondition{},
lastTransitionTimeSorted: []*time.Time{},
}
}

// checkLastTransitionTimes memorizes the last transition time of the condition. This value is used for complementing values for logs without the full status information.
func (c *conditionWalker) checkLastTransitionTimes(commonLog *log.CommonFieldSet, k8sAuditLog *commonlogk8sauditv2_contract.K8sAuditLogFieldSet, condition *model.K8sResourceStatusCondition) {
if condition != nil && condition.Status != "" && condition.LastTransitionTime != "" {
c.lastTransitionStates[condition.LastTransitionTime] = condition
}
}

Expand Down Expand Up @@ -281,6 +311,21 @@ func (c *conditionWalker) CheckAndRecord(commonLog *log.CommonFieldSet, k8sAudit
probeLikeTime, err := condition.ProbeLikeTime()
if err == nil {
if c.lastProbeLikeTime != probeLikeTime.Format(time.RFC3339) {
if condition.Status == "" {
referenceCondition := c.getLastCondition(probeLikeTime)
if referenceCondition != nil {
condition.Status = referenceCondition.Status
if condition.LastTransitionTime == "" {
condition.LastTransitionTime = referenceCondition.LastTransitionTime
}
if condition.Message == "" {
condition.Message = referenceCondition.Message
}
if condition.Reason == "" {
condition.Reason = referenceCondition.Reason
}
}
}
state := conditionStateToRevisionState(condition.Status)
body := c.serializeCondition(condition)
cs.AddRevision(c.conditionPath(), &history.StagingResourceRevision{
Expand Down Expand Up @@ -309,6 +354,37 @@ func (c *conditionWalker) conditionPath() resourcepath.ResourcePath {
return resourcepath.Condition(c.parentResource, c.conditionType)
}

func (c *conditionWalker) getLastCondition(beforeThan time.Time) *model.K8sResourceStatusCondition {
if len(c.lastTransitionTimeSorted) != len(c.lastTransitionStates) {
times := make([]*time.Time, 0, len(c.lastTransitionStates))
for k := range c.lastTransitionStates {
t, err := time.Parse(time.RFC3339, k)
if err != nil {
continue
}
times = append(times, &t)
}
sort.Slice(times, func(i, j int) bool {
return times[i].Before(*times[j])
})
c.lastTransitionTimeSorted = times
}
if len(c.lastTransitionTimeSorted) == 0 {
return nil
}

if c.lastTransitionTimeSorted[0].After(beforeThan) {
return nil
}
idx := sort.Search(len(c.lastTransitionTimeSorted), func(i int) bool {
return c.lastTransitionTimeSorted[i].After(beforeThan)
})
if idx > 0 {
return c.lastTransitionStates[c.lastTransitionTimeSorted[idx-1].Format(time.RFC3339)]
}
return nil
Comment thread
kyasbal marked this conversation as resolved.
}

// serializeCondition serializes the K8sResourceStatusCondition to a YAML string for storage in the revision body.
func (c *conditionWalker) serializeCondition(condition *model.K8sResourceStatusCondition) string {
var conditionBody string
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -246,6 +246,7 @@ func TestConditionHistoryModifierTask_Process(t *testing.T) {
Path: "core/v1#pod#default#nginx",
ParentRelationship: enum.RelationshipChild,
}
oldTime := time.Date(2023, time.December, 31, 12, 0, 0, 0, time.UTC)

testCases := []struct {
name string
Expand All @@ -259,7 +260,7 @@ func TestConditionHistoryModifierTask_Process(t *testing.T) {
asserters []testchangeset.ChangeSetAsserter
}{
{
name: "processFirstPass/Collect AvailableTypes",
name: "processFirstPass/Collect AvailableTypes and LastTransitionTime",
pass: 0,
yaml: `
status:
Expand All @@ -276,8 +277,17 @@ status:
ConditionWalkers: map[string]*conditionWalker{},
},
wantState: &conditionHistoryModifierTaskState{
AvailableTypes: map[string]struct{}{"Ready": {}},
ConditionWalkers: map[string]*conditionWalker{},
AvailableTypes: map[string]struct{}{"Ready": {}},
ConditionWalkers: map[string]*conditionWalker{
"Ready": {
parentResource: resourcepath.ResourcePath{Path: "core/v1#pod#default#nginx"},
conditionType: "Ready",
lastTransitionStates: map[string]*model.K8sResourceStatusCondition{
"2024-01-01T00:00:00Z": {Type: "Ready", LastTransitionTime: "2024-01-01T00:00:00Z", Status: "True"},
},
lastTransitionTimeSorted: []*time.Time{},
},
},
},
asserters: []testchangeset.ChangeSetAsserter{
&testchangeset.MatchResourcePathSet{
Expand All @@ -293,15 +303,21 @@ status:
conditions:
- type: Ready
status: "True"
lastTransitionTime: "2024-01-01T00:00:00Z"
`,
eventType: commonlogk8sauditv2_contract.ChangeEventTypeTargetModification,
operation: enum.RevisionVerbUpdate,
timestamp: time.Date(2024, 1, 1, 0, 0, 0, 0, time.UTC),
initialState: nil,
wantState: &conditionHistoryModifierTaskState{
AvailableTypes: map[string]struct{}{"Ready": {}},
ConditionWalkers: map[string]*conditionWalker{},
AvailableTypes: map[string]struct{}{"Ready": {}},
ConditionWalkers: map[string]*conditionWalker{
"Ready": {
parentResource: resourcepath.ResourcePath{Path: "core/v1#pod#default#nginx"},
conditionType: "Ready",
lastTransitionStates: map[string]*model.K8sResourceStatusCondition{},
lastTransitionTimeSorted: []*time.Time{},
},
},
},
asserters: []testchangeset.ChangeSetAsserter{
&testchangeset.MatchResourcePathSet{
Expand All @@ -328,8 +344,21 @@ status:
ConditionWalkers: map[string]*conditionWalker{},
},
wantState: &conditionHistoryModifierTaskState{
AvailableTypes: map[string]struct{}{"Ready": {}, "Scheduled": {}},
ConditionWalkers: map[string]*conditionWalker{},
AvailableTypes: map[string]struct{}{"Ready": {}, "Scheduled": {}},
ConditionWalkers: map[string]*conditionWalker{
"Ready": {
parentResource: resourcepath.ResourcePath{Path: "core/v1#pod#default#nginx"},
conditionType: "Ready",
lastTransitionStates: map[string]*model.K8sResourceStatusCondition{},
lastTransitionTimeSorted: []*time.Time{},
},
"Scheduled": {
parentResource: resourcepath.ResourcePath{Path: "core/v1#pod#default#nginx"},
conditionType: "Scheduled",
lastTransitionStates: map[string]*model.K8sResourceStatusCondition{},
lastTransitionTimeSorted: []*time.Time{},
},
},
},
asserters: []testchangeset.ChangeSetAsserter{
&testchangeset.MatchResourcePathSet{
Expand Down Expand Up @@ -358,10 +387,12 @@ status:
AvailableTypes: map[string]struct{}{"Ready": {}},
ConditionWalkers: map[string]*conditionWalker{
"Ready": {
parentResource: parentPath,
conditionType: "Ready",
lastStatus: "True",
lastTransitionTime: "2024-01-01T00:00:00Z",
parentResource: parentPath,
conditionType: "Ready",
lastStatus: "True",
lastTransitionTime: "2024-01-01T00:00:00Z",
lastTransitionStates: map[string]*model.K8sResourceStatusCondition{},
lastTransitionTimeSorted: []*time.Time{},
},
},
},
Expand All @@ -378,6 +409,82 @@ status:
},
},
},
{
name: "processSecondPass/complement condition from other logs",
pass: 1,
yaml: `
status:
conditions:
- type: Ready
lastProbeTime: "2024-01-01T00:00:00Z"
`,
eventType: commonlogk8sauditv2_contract.ChangeEventTypeTargetModification,
operation: enum.RevisionVerbUpdate,
timestamp: time.Date(2024, 1, 1, 0, 0, 0, 0, time.UTC),
initialState: &conditionHistoryModifierTaskState{
AvailableTypes: map[string]struct{}{"Ready": {}},
ConditionWalkers: map[string]*conditionWalker{
"Ready": {
parentResource: parentPath,
conditionType: "Ready",
lastStatus: "",
lastTransitionTime: "",
lastTransitionStates: map[string]*model.K8sResourceStatusCondition{
"2023-12-31T12:00:00Z": {
Type: "Ready",
Status: "False",
Reason: "Process is not responsive",
Message: "Something is wrong",
LastTransitionTime: "2023-12-31T12:00:00Z",
},
},
lastTransitionTimeSorted: []*time.Time{},
},
},
},
wantState: &conditionHistoryModifierTaskState{
AvailableTypes: map[string]struct{}{"Ready": {}},
ConditionWalkers: map[string]*conditionWalker{
"Ready": {
parentResource: parentPath,
conditionType: "Ready",
lastStatus: "",
lastTransitionTime: "",
lastProbeLikeTime: "2024-01-01T00:00:00Z",
lastTransitionStates: map[string]*model.K8sResourceStatusCondition{
"2023-12-31T12:00:00Z": {
Type: "Ready",
Status: "False",
Reason: "Process is not responsive",
Message: "Something is wrong",
LastTransitionTime: "2023-12-31T12:00:00Z",
},
},
lastTransitionTimeSorted: []*time.Time{
&oldTime,
},
},
},
},
asserters: []testchangeset.ChangeSetAsserter{
&testchangeset.HasRevision{
ResourcePath: resourcepath.Condition(parentPath, "Ready").Path,
WantRevision: history.StagingResourceRevision{
Verb: enum.RevisionVerbUpdate,
State: enum.RevisionStateConditionFalse,
ChangeTime: time.Date(2024, 1, 1, 0, 0, 0, 0, time.UTC),
Requestor: "user-1",
Body: `lastProbeTime: "2024-01-01T00:00:00Z"
lastTransitionTime: "2023-12-31T12:00:00Z"
message: Something is wrong
reason: Process is not responsive
status: "False"
type: Ready
`,
},
},
},
},
{
name: "processSecondPass/Inferred Creation",
pass: 1,
Expand All @@ -398,10 +505,12 @@ status:
AvailableTypes: map[string]struct{}{"Ready": {}},
ConditionWalkers: map[string]*conditionWalker{
"Ready": {
parentResource: parentPath,
conditionType: "Ready",
lastStatus: "n/a",
minChangeTime: testutil.P(time.Date(2024, 1, 1, 0, 0, 0, 0, time.UTC)),
parentResource: parentPath,
conditionType: "Ready",
lastStatus: "n/a",
minChangeTime: testutil.P(time.Date(2024, 1, 1, 0, 0, 0, 0, time.UTC)),
lastTransitionStates: map[string]*model.K8sResourceStatusCondition{},
lastTransitionTimeSorted: []*time.Time{},
},
},
},
Expand Down Expand Up @@ -436,10 +545,12 @@ status:
AvailableTypes: map[string]struct{}{"Ready": {}},
ConditionWalkers: map[string]*conditionWalker{
"Ready": {
parentResource: parentPath,
conditionType: "Ready",
lastStatus: "", // Reset() clears this
minChangeTime: testutil.P(time.Date(2024, 1, 1, 0, 0, 0, 0, time.UTC)),
parentResource: parentPath,
conditionType: "Ready",
lastStatus: "", // Reset() clears this
minChangeTime: testutil.P(time.Date(2024, 1, 1, 0, 0, 0, 0, time.UTC)),
lastTransitionStates: map[string]*model.K8sResourceStatusCondition{},
lastTransitionTimeSorted: []*time.Time{},
},
},
},
Expand Down