if p.backoffQ.has(pInfo) {
return
}
- if p.unschedulablePods.get(pInfo.Pod) != nil {
- return
+
+ if p.unschedulablePods.get(pInfo.Pod) == nil {
+ logger.V(5).Info("Pod moved to an internal scheduling queue, because the pod is gated", "pod", klog.KObj(pInfo.Pod), "event", event, "queue", unschedulableQ)
}
- p.unschedulablePods.addOrUpdate(pInfo, event)
- logger.V(5).Info("Pod moved to an internal scheduling queue, because the pod is gated", "pod", klog.KObj(pInfo.Pod), "event", event, "queue", unschedulableQ)
+ p.unschedulablePods.addOrUpdate(pInfo, gatedBefore, event)
return
}
if pInfo.InitialAttemptTimestamp == nil {
p.runPreEnqueuePlugins(context.Background(), pInfo)
if pInfo.Gated() {
if p.unschedulablePods.get(pInfo.Pod) == nil {
- p.unschedulablePods.addOrUpdate(pInfo, event)
logger.V(5).Info("Pod moved to an internal scheduling queue", "pod", klog.KObj(pInfo.Pod), "event", event, "queue", unschedulableQ)
}
+ p.unschedulablePods.addOrUpdate(pInfo, gatedBefore, event)
return false
}
}
}
}
} else {
- p.unschedulablePods.addOrUpdate(pInfo, framework.ScheduleAttemptFailure)
+ p.unschedulablePods.addOrUpdate(pInfo, false /* the Pod was absent from all queues */, framework.ScheduleAttemptFailure)
logger.V(5).Info("Pod moved to an internal scheduling queue", "pod", klog.KObj(pod), "event", framework.ScheduleAttemptFailure, "queue", unschedulableQ)
}
}
// Pod update didn't make it schedulable, keep it in the unschedulable queue.
- p.unschedulablePods.addOrUpdate(pInfo, framework.EventUnscheduledPodUpdate.Label())
+ // Use pInfo.Gated() to avoid double-counting "Gated" metrics during an in-place update.
+ p.unschedulablePods.addOrUpdate(pInfo, pInfo.Gated(), framework.EventUnscheduledPodUpdate.Label())
return
}
// If pod is not in any of the queues, we put it in the active queue.
// NOTE: this function assumes lock has been acquired in caller
func (p *PriorityQueue) requeuePodWithQueueingStrategy(logger klog.Logger, pInfo *framework.QueuedPodInfo, strategy queueingStrategy, event string) string {
if strategy == queueSkip {
- p.unschedulablePods.addOrUpdate(pInfo, event)
+ // Current Gate status is required for already exisiting pods. For new pods, this parameter is ignored/unused by addOrUpdate.
+ p.unschedulablePods.addOrUpdate(pInfo, pInfo.Gated(), event)
return unschedulableQ
}
q.backoffQ.add(logger, errorBackoffPodInfo, framework.EventUnscheduledPodAdd.Label())
// Add pod to the unschedulablePods
unschedulablePodInfo := q.newQueuedPodInfo(unschedulablePodInfo.Pod, "plugin")
- q.unschedulablePods.addOrUpdate(unschedulablePodInfo, framework.EventUnscheduledPodAdd.Label())
+ q.unschedulablePods.addOrUpdate(unschedulablePodInfo, false, framework.EventUnscheduledPodAdd.Label())
var gotPods []string
for i := 0; i < len(tt.wantPods)+1; i++ {
schedulingHintsEnablement: []bool{false, true},
},
{
- name: "when updating a pod which is in unschedulable queue and is backing off, it will be moved to backoff queue",
+ name: "when updating a pod in unschedulablePods, if its backoff timer has not yet expired, it moves to backoffQ",
wantQ: backoffQ,
prepareFunc: func(t *testing.T, logger klog.Logger, q *PriorityQueue) (oldPod, newPod *v1.Pod) {
pInfo := q.newQueuedPodInfo(medPriorityPodInfo.Pod, queuePlugin)
// needs to increment to make the pod backing off
pInfo.UnschedulableCount++
- q.unschedulablePods.addOrUpdate(pInfo, framework.EventUnscheduledPodAdd.Label())
+ q.unschedulablePods.addOrUpdate(pInfo, false, framework.EventUnscheduledPodAdd.Label())
updatedPod := medPriorityPodInfo.Pod.DeepCopy()
updatedPod.Annotations["foo"] = "test"
return medPriorityPodInfo.Pod, updatedPod
schedulingHintsEnablement: []bool{false, true},
},
{
- name: "when updating a pod which is in unschedulable queue and is not backing off, it will be moved to active queue",
+ name: "when updating a pod in unschedulablePods, if its backoff timer has expired, it moves to activeQ",
wantQ: activeQ,
prepareFunc: func(t *testing.T, logger klog.Logger, q *PriorityQueue) (oldPod, newPod *v1.Pod) {
pInfo := q.newQueuedPodInfo(medPriorityPodInfo.Pod, queuePlugin)
// needs to increment to make the pod backing off
pInfo.UnschedulableCount++
- q.unschedulablePods.addOrUpdate(pInfo, framework.EventUnscheduledPodAdd.Label())
+ q.unschedulablePods.addOrUpdate(pInfo, false, framework.EventUnscheduledPodAdd.Label())
updatedPod := medPriorityPodInfo.Pod.DeepCopy()
updatedPod.Annotations["foo"] = "test1"
// Move clock by podMaxBackoffDuration, so that pods in the unschedulablePods would pass the backing off,
schedulingHintsEnablement: []bool{false, true},
},
{
- name: "when updating a pod which is in unschedulable pods but the plugin returns skip, it will remain in unschedulablePods",
+ name: "when updating a pod in unschedulablePods, if the scheduling hint returns QueueSkip, it remains in unschedulablePods",
wantQ: unschedulableQ,
prepareFunc: func(t *testing.T, logger klog.Logger, q *PriorityQueue) (oldPod, newPod *v1.Pod) {
- q.unschedulablePods.addOrUpdate(q.newQueuedPodInfo(medPriorityPodInfo.Pod, skipPlugin), framework.EventUnscheduledPodAdd.Label())
+ q.unschedulablePods.addOrUpdate(q.newQueuedPodInfo(medPriorityPodInfo.Pod, skipPlugin), false, framework.EventUnscheduledPodAdd.Label())
updatedPod := medPriorityPodInfo.Pod.DeepCopy()
updatedPod.Annotations["foo"] = "test1"
return medPriorityPodInfo.Pod, updatedPod
}
for _, qPodInfo := range tt.qPodInfoInUnschedulablePods {
- q.unschedulablePods.addOrUpdate(qPodInfo, framework.EventUnscheduledPodAdd.Label())
+ q.unschedulablePods.addOrUpdate(qPodInfo, false, framework.EventUnscheduledPodAdd.Label())
}
for _, qPodInfo := range tt.qPodInfoInBackoffQ {
// needs to increment it to make it backoff
pInfo.UnschedulableCount++
}
- queue.unschedulablePods.addOrUpdate(pInfo, framework.EventUnscheduledPodAdd.Label())
+ queue.unschedulablePods.addOrUpdate(pInfo, false, framework.EventUnscheduledPodAdd.Label())
}
deletePod = func(t *testing.T, _ klog.Logger, queue *PriorityQueue, pInfo *framework.QueuedPodInfo) {
queue.Delete(pInfo.Pod)
queue.clock.(*testingclock.FakeClock).Step(queue.podMaxInUnschedulablePodsDuration)
queue.flushUnschedulablePodsLeftover(logger)
}
+ updatePluginToGateAllPods = func(t *testing.T, logger klog.Logger, queue *PriorityQueue, _ *framework.QueuedPodInfo) {
+ queue.preEnqueuePluginMap[""]["preEnqueuePlugin"] = &preEnqueuePlugin{allowlists: []string{""}}
+ }
+ updatePluginToUngateAllPods = func(t *testing.T, logger klog.Logger, queue *PriorityQueue, _ *framework.QueuedPodInfo) {
+ queue.preEnqueuePluginMap[""]["preEnqueuePlugin"] = &preEnqueuePlugin{allowlists: []string{"queueable"}}
+ }
)
// TestPodTimestamp tests the operations related to QueuedPodInfo.
operands [][]*framework.QueuedPodInfo
metricsName string
pluginMetricsSamplePercent int
+ disablePopFromBackoffQ bool
wants string
}{
{
scheduler_plugin_execution_duration_seconds_count{extension_point="PreEnqueue",plugin="preEnqueuePlugin",status="Success"} 1
`, // the observed value will always be 0, because we don't proceed the fake clock.
},
+ {
+ name: "Gated metric should be 1 when Ungated to Gated transition into moveToActiveQ",
+ operations: []operation{
+ addPodUnschedulablePods,
+ moveClockForward,
+ updatePluginToGateAllPods,
+ updatePodQueueable,
+ },
+ operands: [][]*framework.QueuedPodInfo{
+ pInfos[:1],
+ {nil},
+ {nil},
+ pInfos[:1],
+ },
+ metricsName: "scheduler_pending_pods",
+ pluginMetricsSamplePercent: 100,
+ wants: `
+# HELP scheduler_pending_pods [STABLE] Number of pending pods, by the queue type. 'active' means number of pods in activeQ; 'backoff' means number of pods in backoffQ; 'unschedulable' means number of pods in unschedulablePods that the scheduler attempted to schedule and failed; 'gated' is the number of unschedulable pods that the scheduler never attempted to schedule because they are gated.
+# TYPE scheduler_pending_pods gauge
+scheduler_pending_pods{queue="active"} 0
+scheduler_pending_pods{queue="backoff"} 0
+scheduler_pending_pods{queue="gated"} 1
+scheduler_pending_pods{queue="unschedulable"} 0
+`,
+ },
+ {
+ name: "Gated metric should be 1 when Ungated to Gated transition into moveToBackoffQ",
+ operations: []operation{
+ addPodUnschedulablePods,
+ updatePluginToGateAllPods,
+ updatePodQueueable,
+ },
+ operands: [][]*framework.QueuedPodInfo{
+ pInfos[:1],
+ {nil},
+ pInfos[:1],
+ },
+ metricsName: "scheduler_pending_pods",
+ pluginMetricsSamplePercent: 100,
+ wants: `
+# HELP scheduler_pending_pods [STABLE] Number of pending pods, by the queue type. 'active' means number of pods in activeQ; 'backoff' means number of pods in backoffQ; 'unschedulable' means number of pods in unschedulablePods that the scheduler attempted to schedule and failed; 'gated' is the number of unschedulable pods that the scheduler never attempted to schedule because they are gated.
+# TYPE scheduler_pending_pods gauge
+scheduler_pending_pods{queue="active"} 0
+scheduler_pending_pods{queue="backoff"} 0
+scheduler_pending_pods{queue="gated"} 1
+scheduler_pending_pods{queue="unschedulable"} 0
+`,
+ },
+ {
+ name: "Gated metric should be 1 when Ungated to Gated transition when popFromBackoffQ is disabled",
+ operations: []operation{
+ addPodUnschedulablePods,
+ moveClockForward,
+ updatePluginToGateAllPods,
+ updatePodQueueable,
+ },
+ operands: [][]*framework.QueuedPodInfo{
+ pInfos[:1],
+ {nil},
+ {nil},
+ pInfos[:1],
+ },
+ metricsName: "scheduler_pending_pods",
+ pluginMetricsSamplePercent: 100,
+ disablePopFromBackoffQ: true,
+ wants: `
+# HELP scheduler_pending_pods [STABLE] Number of pending pods, by the queue type. 'active' means number of pods in activeQ; 'backoff' means number of pods in backoffQ; 'unschedulable' means number of pods in unschedulablePods that the scheduler attempted to schedule and failed; 'gated' is the number of unschedulable pods that the scheduler never attempted to schedule because they are gated.
+# TYPE scheduler_pending_pods gauge
+scheduler_pending_pods{queue="active"} 0
+scheduler_pending_pods{queue="backoff"} 0
+scheduler_pending_pods{queue="gated"} 1
+scheduler_pending_pods{queue="unschedulable"} 0
+`,
+ },
+ {
+ name: "Gated metric should be 0 when Ungated -> Gated -> Ungated (ActiveQ) transition",
+ operations: []operation{
+ addPodUnschedulablePods,
+ moveClockForward,
+ updatePluginToGateAllPods,
+ updatePodQueueable,
+ updatePluginToUngateAllPods,
+ updatePodQueueable,
+ },
+ operands: [][]*framework.QueuedPodInfo{
+ pInfos[:1],
+ {nil},
+ {nil},
+ pInfos[:1],
+ {nil},
+ pInfos[:1],
+ },
+ pluginMetricsSamplePercent: 100,
+ metricsName: "scheduler_pending_pods",
+ wants: `
+# HELP scheduler_pending_pods [STABLE] Number of pending pods, by the queue type. 'active' means number of pods in activeQ; 'backoff' means number of pods in backoffQ; 'unschedulable' means number of pods in unschedulablePods that the scheduler attempted to schedule and failed; 'gated' is the number of unschedulable pods that the scheduler never attempted to schedule because they are gated.
+# TYPE scheduler_pending_pods gauge
+scheduler_pending_pods{queue="active"} 1
+scheduler_pending_pods{queue="backoff"} 0
+scheduler_pending_pods{queue="gated"} 0
+scheduler_pending_pods{queue="unschedulable"} 0
+`,
+ },
+ {
+ name: "Gated metric should be 0 when Ungated -> Gated -> Ungated (BackoffQ) transition",
+ operations: []operation{
+ addPodUnschedulablePods,
+ updatePluginToGateAllPods,
+ updatePodQueueable,
+ updatePluginToUngateAllPods,
+ updatePodQueueable,
+ },
+ operands: [][]*framework.QueuedPodInfo{
+ pInfos[:1],
+ {nil},
+ pInfos[:1],
+ {nil},
+ pInfos[:1],
+ },
+ pluginMetricsSamplePercent: 100,
+ metricsName: "scheduler_pending_pods",
+ wants: `
+# HELP scheduler_pending_pods [STABLE] Number of pending pods, by the queue type. 'active' means number of pods in activeQ; 'backoff' means number of pods in backoffQ; 'unschedulable' means number of pods in unschedulablePods that the scheduler attempted to schedule and failed; 'gated' is the number of unschedulable pods that the scheduler never attempted to schedule because they are gated.
+# TYPE scheduler_pending_pods gauge
+scheduler_pending_pods{queue="active"} 0
+scheduler_pending_pods{queue="backoff"} 1
+scheduler_pending_pods{queue="gated"} 0
+scheduler_pending_pods{queue="unschedulable"} 0
+`,
+ },
+ {
+ name: "Gated metric should be 0 when Ungated -> Gated -> Ungated transition, when popFromBackoffQ is disabled",
+ operations: []operation{
+ addPodUnschedulablePods,
+ moveClockForward,
+ updatePluginToGateAllPods,
+ updatePodQueueable,
+ updatePluginToUngateAllPods,
+ updatePodQueueable,
+ },
+ operands: [][]*framework.QueuedPodInfo{
+ pInfos[:1],
+ {nil},
+ {nil},
+ pInfos[:1],
+ {nil},
+ pInfos[:1],
+ },
+ pluginMetricsSamplePercent: 100,
+ disablePopFromBackoffQ: true,
+ wants: `
+# HELP scheduler_pending_pods [STABLE] Number of pending pods, by the queue type. 'active' means number of pods in activeQ; 'backoff' means number of pods in backoffQ; 'unschedulable' means number of pods in unschedulablePods that the scheduler attempted to schedule and failed; 'gated' is the number of unschedulable pods that the scheduler never attempted to schedule because they are gated.
+# TYPE scheduler_pending_pods gauge
+scheduler_pending_pods{queue="active"} 1
+scheduler_pending_pods{queue="backoff"} 0
+scheduler_pending_pods{queue="gated"} 0
+scheduler_pending_pods{queue="unschedulable"} 0
+`,
+ },
}
resetMetrics := func() {
preenq := map[string]map[string]fwk.PreEnqueuePlugin{"": {(&preEnqueuePlugin{}).Name(): &preEnqueuePlugin{allowlists: []string{queueable}}}}
recorder := metrics.NewMetricsAsyncRecorder(3, 20*time.Microsecond, ctx.Done())
queue := NewTestQueue(ctx, newDefaultQueueSort(), WithClock(testingclock.NewFakeClock(timestamp)), WithPreEnqueuePluginMap(preenq), WithPluginMetricsSamplePercent(test.pluginMetricsSamplePercent), WithMetricsRecorder(recorder), WithQueueingHintMapPerProfile(m))
+ queue.isPopFromBackoffQEnabled = !test.disablePopFromBackoffQ
for i, op := range test.operations {
for _, pInfo := range test.operands[i] {
op(t, logger, queue, pInfo)
unlockedActiveQ.add(logger, newQueuedPodInfoForLookup(activeQPod), framework.EventUnscheduledPodAdd.Label())
})
q.backoffQ.add(logger, newQueuedPodInfoForLookup(backoffQPod), framework.EventUnscheduledPodAdd.Label())
- q.unschedulablePods.addOrUpdate(newQueuedPodInfoForLookup(unschedPod), framework.EventUnscheduledPodAdd.Label())
+ q.unschedulablePods.addOrUpdate(newQueuedPodInfoForLookup(unschedPod), false, framework.EventUnscheduledPodAdd.Label())
tests := []struct {
name string