PodEvictor: turn an exceeded limit into an error

When checking for node limit getting exceeded the pod eviction
never fails. Thus, ignoring the metric reporting when a pod fails
to be evicted due to node limit constrains.

The error also allows plugin to react on other limits getting
exceeded. E.g. the limit on the number of pods evicted per namespace.

pkg/framework/plugins/removepodsviolatingnodeaffinity/node_affinity.go

@@ -134,11 +134,18 @@ func (d *RemovePodsViolatingNodeAffinity) processNodes(ctx context.Context, node
 			}
 		}
 
+	loop:
 		for _, pod := range pods {
 			klog.V(1).InfoS("Evicting pod", "pod", klog.KObj(pod))
-			d.handle.Evictor().Evict(ctx, pod, evictions.EvictOptions{StrategyName: PluginName})
-			if d.handle.Evictor().NodeLimitExceeded(node) {
-				break
+			err := d.handle.Evictor().Evict(ctx, pod, evictions.EvictOptions{StrategyName: PluginName})
+			if err == nil {
+				continue
+			}
+			switch err.(type) {
+			case *evictions.EvictionNodeLimitError:
+				break loop
+			default:
+				klog.Errorf("eviction failed: %v", err)
 			}
 		}
 	}