Skip to content

Commit 26fd4a8

Browse files
shuo-wushuo-wu
committed
chore: improve instance handler logging
Signed-off-by: Shuo Wu <shuo.wu@suse.com>
1 parent e72e927 commit 26fd4a8

1 file changed

Lines changed: 23 additions & 14 deletions

File tree

‎controller/instance_handler.go‎

Lines changed: 23 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -49,7 +49,7 @@ func NewInstanceHandler(ds *datastore.DataStore, instanceManagerHandler Instance
4949
}
5050
}
5151

52-
func (h *InstanceHandler) syncStatusWithInstanceManager(im *longhorn.InstanceManager, instanceName string, spec *longhorn.InstanceSpec, status *longhorn.InstanceStatus, instances map[string]longhorn.InstanceProcess) {
52+
func (h *InstanceHandler) syncStatusWithInstanceManager(log *logrus.Entry, im *longhorn.InstanceManager, instanceName string, spec *longhorn.InstanceSpec, status *longhorn.InstanceStatus, instances map[string]longhorn.InstanceProcess) {
5353
defer func() {
5454
if status.CurrentState == longhorn.InstanceStateStopped {
5555
status.InstanceManagerName = ""
@@ -64,7 +64,7 @@ func (h *InstanceHandler) syncStatusWithInstanceManager(im *longhorn.InstanceMan
6464
if im == nil || im.Status.CurrentState == longhorn.InstanceManagerStateUnknown || isDelinquent {
6565
if status.Started {
6666
if status.CurrentState != longhorn.InstanceStateUnknown {
67-
logrus.Warnf("Marking the instance as state UNKNOWN since the related node %v of instance %v is down or deleted", spec.NodeID, instanceName)
67+
log.Warnf("Marking the instance as state UNKNOWN since the related node %v of instance %v is down or deleted", spec.NodeID, instanceName)
6868
}
6969
status.CurrentState = longhorn.InstanceStateUnknown
7070
} else {
@@ -122,7 +122,7 @@ func (h *InstanceHandler) syncStatusWithInstanceManager(im *longhorn.InstanceMan
122122
if !exists {
123123
if status.Started {
124124
if status.CurrentState != longhorn.InstanceStateError {
125-
logrus.Warnf("Marking the instance as state ERROR since failed to find the instance status in instance manager %v for the running instance %v", im.Name, instanceName)
125+
log.Warnf("Marking the instance as state ERROR since failed to find the instance status in instance manager %v for the running instance %v", im.Name, instanceName)
126126
}
127127
status.CurrentState = longhorn.InstanceStateError
128128
} else {
@@ -139,7 +139,7 @@ func (h *InstanceHandler) syncStatusWithInstanceManager(im *longhorn.InstanceMan
139139
}
140140

141141
if status.InstanceManagerName != "" && status.InstanceManagerName != im.Name {
142-
logrus.Errorf("The related process of instance %v is found in the instance manager %v, but the instance manager name in the instance status is %v. "+
142+
log.Errorf("The related process of instance %v is found in the instance manager %v, but the instance manager name in the instance status is %v. "+
143143
"The instance manager name shouldn't change except for cleanup",
144144
instanceName, im.Name, status.InstanceManagerName)
145145
}
@@ -164,28 +164,34 @@ func (h *InstanceHandler) syncStatusWithInstanceManager(im *longhorn.InstanceMan
164164

165165
imPod, err := h.ds.GetPodRO(im.Namespace, im.Name)
166166
if err != nil {
167-
logrus.WithError(err).Errorf("Failed to get instance manager pod from %v", im.Name)
167+
log.WithError(err).Errorf("Failed to get instance manager pod from %v", im.Name)
168168
return
169169
}
170170

171171
if imPod == nil {
172-
logrus.Warnf("Instance manager pod from %v not exist in datastore", im.Name)
172+
log.Warnf("Instance manager pod from %v not exist in datastore", im.Name)
173173
return
174174
}
175175

176176
storageIP := h.ds.GetStorageIPFromPod(imPod)
177177
if status.StorageIP != storageIP {
178+
if status.StorageIP != "" {
179+
log.Warnf("Instance %v is state running in instance manager %s, but its status Storage IP %s does not match the instance manager recorded Storage IP %s", instanceName, im.Name, status.StorageIP, storageIP)
180+
}
178181
status.StorageIP = storageIP
179-
logrus.Warnf("Instance %v starts running, Storage IP %v", instanceName, status.StorageIP)
180182
}
181183

182184
if status.IP != im.Status.IP {
185+
if status.IP != "" {
186+
log.Warnf("Instance %v is state running in instance manager %s, but its status IP %s does not match the instance manager recorded IP %s", instanceName, im.Name, status.IP, im.Status.IP)
187+
}
183188
status.IP = im.Status.IP
184-
logrus.Warnf("Instance %v starts running, IP %v", instanceName, status.IP)
185189
}
186190
if status.Port != int(instance.Status.PortStart) {
191+
if status.Port != 0 {
192+
log.Warnf("Instance %v is state running in instance manager %s, but its status Port %d does not match the instance manager recorded Port %d", instanceName, im.Name, status.Port, instance.Status.PortStart)
193+
}
187194
status.Port = int(instance.Status.PortStart)
188-
logrus.Warnf("Instance %v starts running, Port %d", instanceName, status.Port)
189195
}
190196
if status.UblkID != instance.Status.UblkID {
191197
status.UblkID = instance.Status.UblkID
@@ -230,7 +236,7 @@ func (h *InstanceHandler) syncStatusWithInstanceManager(im *longhorn.InstanceMan
230236
h.resetInstanceErrorCondition(status)
231237
default:
232238
if status.CurrentState != longhorn.InstanceStateError {
233-
logrus.Warnf("Instance %v is state %v, error message: %v", instanceName, instance.Status.State, instance.Status.ErrorMsg)
239+
log.Warnf("Instance %v is state %v, error message: %v", instanceName, instance.Status.State, instance.Status.ErrorMsg)
234240
}
235241
status.CurrentState = longhorn.InstanceStateError
236242
status.CurrentImage = ""
@@ -278,7 +284,7 @@ func (h *InstanceHandler) ReconcileInstanceState(obj interface{}, spec *longhorn
278284
return err
279285
}
280286

281-
log := logrus.WithField("instance", instanceName)
287+
log := logrus.WithFields(logrus.Fields{"instance": instanceName, "volumeName": spec.VolumeName, "dataEngine": spec.DataEngine, "specNodeID": spec.NodeID})
282288

283289
var im *longhorn.InstanceManager
284290
if status.InstanceManagerName != "" {
@@ -310,6 +316,9 @@ func (h *InstanceHandler) ReconcileInstanceState(obj interface{}, spec *longhorn
310316
}
311317
}
312318
}
319+
if im != nil {
320+
log = log.WithFields(logrus.Fields{"instanceManager": im.Name})
321+
}
313322

314323
if spec.LogRequested {
315324
if !status.LogFetched {
@@ -388,7 +397,7 @@ func (h *InstanceHandler) ReconcileInstanceState(obj interface{}, spec *longhorn
388397
return fmt.Errorf("unknown instance desire state: desire %v", spec.DesireState)
389398
}
390399

391-
h.syncStatusWithInstanceManager(im, instanceName, spec, status, instances)
400+
h.syncStatusWithInstanceManager(log, im, instanceName, spec, status, instances)
392401

393402
switch status.CurrentState {
394403
case longhorn.InstanceStateRunning:
@@ -417,10 +426,10 @@ func (h *InstanceHandler) ReconcileInstanceState(obj interface{}, spec *longhorn
417426
}
418427

419428
if types.IsDataEngineV1(instance.Spec.DataEngine) {
420-
logrus.Warnf("Instance %v crashed on Instance Manager %v at %v, getting log",
429+
log.Warnf("Instance %v crashed on Instance Manager %v at %v, getting log",
421430
instanceName, im.Name, im.Spec.NodeID)
422431
if err := h.printInstanceLogs(instanceName, runtimeObj); err != nil {
423-
logrus.WithError(err).Warnf("failed to get crash log for instance %v on Instance Manager %v at %v",
432+
log.WithError(err).Warnf("failed to get crash log for instance %v on Instance Manager %v at %v",
424433
instanceName, im.Name, im.Spec.NodeID)
425434
}
426435
}

0 commit comments

Comments
 (0)