From e7c42794a0eedf6fbe2bc1cc8ffe11bbf55a1eb5 Mon Sep 17 00:00:00 2001 From: Mike Dame Date: Wed, 1 Apr 2020 16:10:56 -0400 Subject: [PATCH 1/2] Add RemovePodsHavingTooManyRestarts strategy --- .gitignore | 2 + README.md | 16 ++ examples/policy.yaml | 6 + pkg/api/types.go | 6 + pkg/api/v1alpha1/types.go | 6 + pkg/api/v1alpha1/zz_generated.conversion.go | 38 ++++ pkg/api/v1alpha1/zz_generated.deepcopy.go | 17 ++ pkg/api/zz_generated.deepcopy.go | 17 ++ pkg/descheduler/descheduler.go | 1 + pkg/descheduler/strategies/toomanyrestarts.go | 77 +++++++ .../strategies/toomanyrestarts_test.go | 191 ++++++++++++++++++ 11 files changed, 377 insertions(+) create mode 100644 pkg/descheduler/strategies/toomanyrestarts.go create mode 100644 pkg/descheduler/strategies/toomanyrestarts_test.go diff --git a/.gitignore b/.gitignore index eb94dbb3e..65f7e6348 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,5 @@ _output/ _tmp/ vendordiff.patch +.idea/ + diff --git a/README.md b/README.md index 3ea9a147e..4df278c17 100644 --- a/README.md +++ b/README.md @@ -182,6 +182,22 @@ strategies: enabled: true ```` +### RemovePodsHavingTooManyRestarts + +This strategy makes sure that pods having too many restarts are removed from nodes. For example a pod with EBS/PD that can't get the volume/disk attached to the instance, then the pod should be re-scheduled to other nodes. + +``` +apiVersion: "descheduler/v1alpha1" +kind: "DeschedulerPolicy" +strategies: + "RemovePodsHavingTooManyRestarts": + enabled: true + params: + podsHavingTooManyRestarts: + podRestartThreshold: 100 + includingInitContainers: true +``` + ## Pod Evictions When the descheduler decides to evict pods from a node, it employs the following general mechanism: diff --git a/examples/policy.yaml b/examples/policy.yaml index 38fd0d8f3..e1aaf624b 100644 --- a/examples/policy.yaml +++ b/examples/policy.yaml @@ -17,3 +17,9 @@ strategies: "cpu" : 50 "memory": 50 "pods": 50 + "RemovePodsHavingTooManyRestarts": + enabled: true + params: + podsHavingTooManyRestarts: + podRestartThresholds: 100 + includingInitContainers: true diff --git a/pkg/api/types.go b/pkg/api/types.go index 5b1d1c30c..5d814f55c 100644 --- a/pkg/api/types.go +++ b/pkg/api/types.go @@ -48,6 +48,7 @@ type DeschedulerStrategy struct { type StrategyParameters struct { NodeResourceUtilizationThresholds NodeResourceUtilizationThresholds NodeAffinityType []string + PodsHavingTooManyRestarts PodsHavingTooManyRestarts } type Percentage float64 @@ -58,3 +59,8 @@ type NodeResourceUtilizationThresholds struct { TargetThresholds ResourceThresholds NumberOfNodes int } + +type PodsHavingTooManyRestarts struct { + PodRestartThreshold int32 + IncludingInitContainers bool +} diff --git a/pkg/api/v1alpha1/types.go b/pkg/api/v1alpha1/types.go index 1ca0cac9b..213b63fcf 100644 --- a/pkg/api/v1alpha1/types.go +++ b/pkg/api/v1alpha1/types.go @@ -48,6 +48,7 @@ type DeschedulerStrategy struct { type StrategyParameters struct { NodeResourceUtilizationThresholds NodeResourceUtilizationThresholds `json:"nodeResourceUtilizationThresholds,omitempty"` NodeAffinityType []string `json:"nodeAffinityType,omitempty"` + PodsHavingTooManyRestarts PodsHavingTooManyRestarts `json:"podsHavingTooManyRestarts,omitempty"` } type Percentage float64 @@ -58,3 +59,8 @@ type NodeResourceUtilizationThresholds struct { TargetThresholds ResourceThresholds `json:"targetThresholds,omitempty"` NumberOfNodes int `json:"numberOfNodes,omitempty"` } + +type PodsHavingTooManyRestarts struct { + PodRestartThreshold int32 `json:"podRestartThreshold,omitempty"` + IncludingInitContainers bool `json:"includingInitContainers,omitempty"` +} diff --git a/pkg/api/v1alpha1/zz_generated.conversion.go b/pkg/api/v1alpha1/zz_generated.conversion.go index 829370b0a..d251ac64a 100644 --- a/pkg/api/v1alpha1/zz_generated.conversion.go +++ b/pkg/api/v1alpha1/zz_generated.conversion.go @@ -65,6 +65,16 @@ func RegisterConversions(s *runtime.Scheme) error { }); err != nil { return err } + if err := s.AddGeneratedConversionFunc((*PodsHavingTooManyRestarts)(nil), (*api.PodsHavingTooManyRestarts)(nil), func(a, b interface{}, scope conversion.Scope) error { + return Convert_v1alpha1_PodsHavingTooManyRestarts_To_api_PodsHavingTooManyRestarts(a.(*PodsHavingTooManyRestarts), b.(*api.PodsHavingTooManyRestarts), scope) + }); err != nil { + return err + } + if err := s.AddGeneratedConversionFunc((*api.PodsHavingTooManyRestarts)(nil), (*PodsHavingTooManyRestarts)(nil), func(a, b interface{}, scope conversion.Scope) error { + return Convert_api_PodsHavingTooManyRestarts_To_v1alpha1_PodsHavingTooManyRestarts(a.(*api.PodsHavingTooManyRestarts), b.(*PodsHavingTooManyRestarts), scope) + }); err != nil { + return err + } if err := s.AddGeneratedConversionFunc((*StrategyParameters)(nil), (*api.StrategyParameters)(nil), func(a, b interface{}, scope conversion.Scope) error { return Convert_v1alpha1_StrategyParameters_To_api_StrategyParameters(a.(*StrategyParameters), b.(*api.StrategyParameters), scope) }); err != nil { @@ -150,11 +160,36 @@ func Convert_api_NodeResourceUtilizationThresholds_To_v1alpha1_NodeResourceUtili return autoConvert_api_NodeResourceUtilizationThresholds_To_v1alpha1_NodeResourceUtilizationThresholds(in, out, s) } +func autoConvert_v1alpha1_PodsHavingTooManyRestarts_To_api_PodsHavingTooManyRestarts(in *PodsHavingTooManyRestarts, out *api.PodsHavingTooManyRestarts, s conversion.Scope) error { + out.PodRestartThreshold = in.PodRestartThreshold + out.IncludingInitContainers = in.IncludingInitContainers + return nil +} + +// Convert_v1alpha1_PodsHavingTooManyRestarts_To_api_PodsHavingTooManyRestarts is an autogenerated conversion function. +func Convert_v1alpha1_PodsHavingTooManyRestarts_To_api_PodsHavingTooManyRestarts(in *PodsHavingTooManyRestarts, out *api.PodsHavingTooManyRestarts, s conversion.Scope) error { + return autoConvert_v1alpha1_PodsHavingTooManyRestarts_To_api_PodsHavingTooManyRestarts(in, out, s) +} + +func autoConvert_api_PodsHavingTooManyRestarts_To_v1alpha1_PodsHavingTooManyRestarts(in *api.PodsHavingTooManyRestarts, out *PodsHavingTooManyRestarts, s conversion.Scope) error { + out.PodRestartThreshold = in.PodRestartThreshold + out.IncludingInitContainers = in.IncludingInitContainers + return nil +} + +// Convert_api_PodsHavingTooManyRestarts_To_v1alpha1_PodsHavingTooManyRestarts is an autogenerated conversion function. +func Convert_api_PodsHavingTooManyRestarts_To_v1alpha1_PodsHavingTooManyRestarts(in *api.PodsHavingTooManyRestarts, out *PodsHavingTooManyRestarts, s conversion.Scope) error { + return autoConvert_api_PodsHavingTooManyRestarts_To_v1alpha1_PodsHavingTooManyRestarts(in, out, s) +} + func autoConvert_v1alpha1_StrategyParameters_To_api_StrategyParameters(in *StrategyParameters, out *api.StrategyParameters, s conversion.Scope) error { if err := Convert_v1alpha1_NodeResourceUtilizationThresholds_To_api_NodeResourceUtilizationThresholds(&in.NodeResourceUtilizationThresholds, &out.NodeResourceUtilizationThresholds, s); err != nil { return err } out.NodeAffinityType = *(*[]string)(unsafe.Pointer(&in.NodeAffinityType)) + if err := Convert_v1alpha1_PodsHavingTooManyRestarts_To_api_PodsHavingTooManyRestarts(&in.PodsHavingTooManyRestarts, &out.PodsHavingTooManyRestarts, s); err != nil { + return err + } return nil } @@ -168,6 +203,9 @@ func autoConvert_api_StrategyParameters_To_v1alpha1_StrategyParameters(in *api.S return err } out.NodeAffinityType = *(*[]string)(unsafe.Pointer(&in.NodeAffinityType)) + if err := Convert_api_PodsHavingTooManyRestarts_To_v1alpha1_PodsHavingTooManyRestarts(&in.PodsHavingTooManyRestarts, &out.PodsHavingTooManyRestarts, s); err != nil { + return err + } return nil } diff --git a/pkg/api/v1alpha1/zz_generated.deepcopy.go b/pkg/api/v1alpha1/zz_generated.deepcopy.go index af41198ce..37420ba13 100644 --- a/pkg/api/v1alpha1/zz_generated.deepcopy.go +++ b/pkg/api/v1alpha1/zz_generated.deepcopy.go @@ -103,6 +103,22 @@ func (in *NodeResourceUtilizationThresholds) DeepCopy() *NodeResourceUtilization return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *PodsHavingTooManyRestarts) DeepCopyInto(out *PodsHavingTooManyRestarts) { + *out = *in + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new PodsHavingTooManyRestarts. +func (in *PodsHavingTooManyRestarts) DeepCopy() *PodsHavingTooManyRestarts { + if in == nil { + return nil + } + out := new(PodsHavingTooManyRestarts) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in ResourceThresholds) DeepCopyInto(out *ResourceThresholds) { { @@ -156,6 +172,7 @@ func (in *StrategyParameters) DeepCopyInto(out *StrategyParameters) { *out = make([]string, len(*in)) copy(*out, *in) } + out.PodsHavingTooManyRestarts = in.PodsHavingTooManyRestarts return } diff --git a/pkg/api/zz_generated.deepcopy.go b/pkg/api/zz_generated.deepcopy.go index 1af80dbca..e046e70c1 100644 --- a/pkg/api/zz_generated.deepcopy.go +++ b/pkg/api/zz_generated.deepcopy.go @@ -103,6 +103,22 @@ func (in *NodeResourceUtilizationThresholds) DeepCopy() *NodeResourceUtilization return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *PodsHavingTooManyRestarts) DeepCopyInto(out *PodsHavingTooManyRestarts) { + *out = *in + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new PodsHavingTooManyRestarts. +func (in *PodsHavingTooManyRestarts) DeepCopy() *PodsHavingTooManyRestarts { + if in == nil { + return nil + } + out := new(PodsHavingTooManyRestarts) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in ResourceThresholds) DeepCopyInto(out *ResourceThresholds) { { @@ -156,6 +172,7 @@ func (in *StrategyParameters) DeepCopyInto(out *StrategyParameters) { *out = make([]string, len(*in)) copy(*out, *in) } + out.PodsHavingTooManyRestarts = in.PodsHavingTooManyRestarts return } diff --git a/pkg/descheduler/descheduler.go b/pkg/descheduler/descheduler.go index 9ff210500..1f77ceeac 100644 --- a/pkg/descheduler/descheduler.go +++ b/pkg/descheduler/descheduler.go @@ -90,6 +90,7 @@ func RunDeschedulerStrategies(rs *options.DeschedulerServer, deschedulerPolicy * strategies.RemovePodsViolatingInterPodAntiAffinity(rs, deschedulerPolicy.Strategies["RemovePodsViolatingInterPodAntiAffinity"], nodes, podEvictor) strategies.RemovePodsViolatingNodeAffinity(rs, deschedulerPolicy.Strategies["RemovePodsViolatingNodeAffinity"], nodes, podEvictor) strategies.RemovePodsViolatingNodeTaints(rs, deschedulerPolicy.Strategies["RemovePodsViolatingNodeTaints"], nodes, podEvictor) + strategies.RemovePodsHavingTooManyRestarts(rs, deschedulerPolicy.Strategies["RemovePodsHavingTooManyRestarts"], nodes, podEvictor) // If there was no interval specified, send a signal to the stopChannel to end the wait.Until loop after 1 iteration if rs.DeschedulingInterval.Seconds() == 0 { diff --git a/pkg/descheduler/strategies/toomanyrestarts.go b/pkg/descheduler/strategies/toomanyrestarts.go new file mode 100644 index 000000000..ab919c237 --- /dev/null +++ b/pkg/descheduler/strategies/toomanyrestarts.go @@ -0,0 +1,77 @@ +/* +Copyright 2018 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package strategies + +import ( + "k8s.io/api/core/v1" + "k8s.io/klog" + + "sigs.k8s.io/descheduler/cmd/descheduler/app/options" + "sigs.k8s.io/descheduler/pkg/api" + "sigs.k8s.io/descheduler/pkg/descheduler/evictions" + podutil "sigs.k8s.io/descheduler/pkg/descheduler/pod" +) + +// RemovePodsHavingTooManyRestarts removes the pods that have too many restarts on node. +// There are too many cases leading this issue: Volume mount failed, app error due to nodes' different settings. +// As of now, this strategy won't evict daemonsets, mirror pods, critical pods and pods with local storages. +func RemovePodsHavingTooManyRestarts(ds *options.DeschedulerServer, strategy api.DeschedulerStrategy, nodes []*v1.Node, podEvictor *evictions.PodEvictor) { + if !strategy.Enabled || strategy.Params.PodsHavingTooManyRestarts.PodRestartThreshold < 1 { + return + } + removePodsHavingTooManyRestarts(ds, strategy, nodes, podEvictor, ds.EvictLocalStoragePods) +} + +func removePodsHavingTooManyRestarts(ds *options.DeschedulerServer, strategy api.DeschedulerStrategy, nodes []*v1.Node, podEvictor *evictions.PodEvictor, evictLocalStoragePods bool) { + for _, node := range nodes { + klog.V(1).Infof("Processing node: %s", node.Name) + pods, err := podutil.ListEvictablePodsOnNode(ds.Client, node, evictLocalStoragePods) + if err != nil { + klog.Errorf("Error when list pods at node %s", node.Name) + continue + } + + for i, pod := range pods { + restarts, initRestarts := calcContainerRestarts(pod) + if strategy.Params.PodsHavingTooManyRestarts.IncludingInitContainers { + if restarts+initRestarts < strategy.Params.PodsHavingTooManyRestarts.PodRestartThreshold { + continue + } + } else if restarts < strategy.Params.PodsHavingTooManyRestarts.PodRestartThreshold { + continue + } + if _, err := podEvictor.EvictPod(pods[i], node); err != nil { + break + } + } + } +} + +// calcContainerRestarts get container restarts and init container restarts. +func calcContainerRestarts(pod *v1.Pod) (int32, int32) { + var restarts, initRestarts int32 + + for _, cs := range pod.Status.ContainerStatuses { + restarts += cs.RestartCount + } + + for _, cs := range pod.Status.InitContainerStatuses { + initRestarts += cs.RestartCount + } + + return restarts, initRestarts +} diff --git a/pkg/descheduler/strategies/toomanyrestarts_test.go b/pkg/descheduler/strategies/toomanyrestarts_test.go new file mode 100644 index 000000000..9948d9df7 --- /dev/null +++ b/pkg/descheduler/strategies/toomanyrestarts_test.go @@ -0,0 +1,191 @@ +/* +Copyright 2018 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package strategies + +import ( + "testing" + + "fmt" + "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/client-go/kubernetes/fake" + core "k8s.io/client-go/testing" + "sigs.k8s.io/descheduler/cmd/descheduler/app/options" + "sigs.k8s.io/descheduler/pkg/api" + "sigs.k8s.io/descheduler/pkg/apis/componentconfig" + "sigs.k8s.io/descheduler/pkg/descheduler/evictions" + "sigs.k8s.io/descheduler/test" +) + +func initPods(node *v1.Node) []v1.Pod { + pods := make([]v1.Pod, 0) + + for i := int32(0); i <= 9; i++ { + pod := test.BuildTestPod(fmt.Sprintf("pod-%d", i), 100, 0, node.Name, nil) + pod.ObjectMeta.OwnerReferences = test.GetNormalPodOwnerRefList() + + // pod at index i will have 25 * i restarts. + pod.Status = v1.PodStatus{ + InitContainerStatuses: []v1.ContainerStatus{ + { + RestartCount: 5 * i, + }, + }, + ContainerStatuses: []v1.ContainerStatus{ + { + RestartCount: 10 * i, + }, + { + RestartCount: 10 * i, + }, + }, + } + pods = append(pods, *pod) + } + + // The following 3 pods won't get evicted. + // A daemonset. + pods[6].ObjectMeta.OwnerReferences = test.GetDaemonSetOwnerRefList() + // A pod with local storage. + pods[7].ObjectMeta.OwnerReferences = test.GetNormalPodOwnerRefList() + pods[7].Spec.Volumes = []v1.Volume{ + { + Name: "sample", + VolumeSource: v1.VolumeSource{ + HostPath: &v1.HostPathVolumeSource{Path: "somePath"}, + EmptyDir: &v1.EmptyDirVolumeSource{ + SizeLimit: resource.NewQuantity(int64(10), resource.BinarySI)}, + }, + }, + } + // A Mirror Pod. + pods[8].Annotations = test.GetMirrorPodAnnotation() + + return pods +} + +func TestRemovePodsHavingTooManyRestarts(t *testing.T) { + createStrategy := func(enabled, includingInitContainers bool, restartThresholds int32) api.DeschedulerStrategy { + return api.DeschedulerStrategy{ + Enabled: enabled, + Params: api.StrategyParameters{ + PodsHavingTooManyRestarts: api.PodsHavingTooManyRestarts{ + PodRestartThreshold: restartThresholds, + IncludingInitContainers: includingInitContainers, + }, + }, + } + } + + node := test.BuildTestNode("node1", 2000, 3000, 10, nil) + pods := initPods(node) + + fakeClient := &fake.Clientset{} + + fakeClient.Fake.AddReactor("list", "pods", func(action core.Action) (bool, runtime.Object, error) { + return true, &v1.PodList{Items: pods}, nil + }) + + tests := []struct { + description string + pods []v1.Pod + strategy api.DeschedulerStrategy + expectedEvictedPodCount int + maxPodsToEvict int + }{ + { + description: "All pods have total restarts under threshold, no pod evictions", + strategy: createStrategy(true, true, 10000), + expectedEvictedPodCount: 0, + maxPodsToEvict: 0, + }, + { + description: "Some pods have total restarts bigger than threshold", + strategy: createStrategy(true, true, 1), + expectedEvictedPodCount: 6, + maxPodsToEvict: 0, + }, + { + description: "Nine pods have total restarts equals threshold(includingInitContainers=true), 6 pods evictions", + strategy: createStrategy(true, true, 1*25), + expectedEvictedPodCount: 6, + maxPodsToEvict: 0, + }, + { + description: "Nine pods have total restarts equals threshold(includingInitContainers=false), 5 pods evictions", + strategy: createStrategy(true, false, 1*25), + expectedEvictedPodCount: 5, + maxPodsToEvict: 0, + }, + { + description: "All pods have total restarts equals threshold(includingInitContainers=true), 6 pods evictions", + strategy: createStrategy(true, true, 1*20), + expectedEvictedPodCount: 6, + maxPodsToEvict: 0, + }, + { + description: "Nine pods have total restarts equals threshold(includingInitContainers=false), 6 pods evictions", + strategy: createStrategy(true, false, 1*20), + expectedEvictedPodCount: 6, + maxPodsToEvict: 0, + }, + { + description: "Five pods have total restarts bigger than threshold(includingInitContainers=true), but only 1 pod eviction", + strategy: createStrategy(true, true, 5*25+1), + expectedEvictedPodCount: 1, + maxPodsToEvict: 0, + }, + { + description: "Five pods have total restarts bigger than threshold(includingInitContainers=false), but only 1 pod eviction", + strategy: createStrategy(true, false, 5*20+1), + expectedEvictedPodCount: 1, + maxPodsToEvict: 0, + }, + { + description: "All pods have total restarts equals threshold(maxPodsToEvict=3), 3 pods evictions", + strategy: createStrategy(true, true, 1), + expectedEvictedPodCount: 3, + maxPodsToEvict: 3, + }, + } + + for _, tc := range tests { + + ds := options.DeschedulerServer{ + DeschedulerConfiguration: componentconfig.DeschedulerConfiguration{ + MaxNoOfPodsToEvictPerNode: tc.maxPodsToEvict, + }, + Client: fakeClient, + } + + podEvictor := evictions.NewPodEvictor( + fakeClient, + "v1", + false, + tc.maxPodsToEvict, + []*v1.Node{node}, + ) + + removePodsHavingTooManyRestarts(&ds, tc.strategy, []*v1.Node{node}, podEvictor, ds.EvictLocalStoragePods) + actualEvictedPodCount := podEvictor.TotalEvicted() + if actualEvictedPodCount != tc.expectedEvictedPodCount { + t.Errorf("Test %#v failed, expected %v pod evictions, but got %v pod evictions\n", tc.description, tc.expectedEvictedPodCount, actualEvictedPodCount) + } + } + +} From c2d7e2274991dd8f8381d565ef3f0fb1c9aa9024 Mon Sep 17 00:00:00 2001 From: Mike Dame Date: Wed, 8 Apr 2020 12:16:47 -0400 Subject: [PATCH 2/2] make gen && go mod vendor --- go.mod | 3 + go.sum | 9 ++ .../github.com/gogo/protobuf/proto/encode.go | 2 + vendor/github.com/gogo/protobuf/proto/lib.go | 18 ++- .../gogo/protobuf/proto/properties.go | 71 +++++++----- .../gogo/protobuf/proto/table_marshal.go | 17 +-- .../gogo/protobuf/proto/table_merge.go | 19 +++ .../gogo/protobuf/proto/table_unmarshal.go | 22 ++-- vendor/github.com/gogo/protobuf/proto/text.go | 6 +- vendor/gopkg.in/yaml.v2/.travis.yml | 18 +-- vendor/gopkg.in/yaml.v2/decode.go | 14 ++- vendor/gopkg.in/yaml.v2/scannerc.go | 109 +++++++++--------- vendor/gopkg.in/yaml.v2/yaml.go | 2 +- vendor/gopkg.in/yaml.v2/yamlh.go | 1 + vendor/modules.txt | 8 +- vendor/sigs.k8s.io/yaml/.travis.yml | 15 ++- vendor/sigs.k8s.io/yaml/OWNERS | 2 + vendor/sigs.k8s.io/yaml/README.md | 14 ++- vendor/sigs.k8s.io/yaml/go.mod | 8 ++ vendor/sigs.k8s.io/yaml/go.sum | 9 ++ vendor/sigs.k8s.io/yaml/yaml.go | 61 ++++++++++ 21 files changed, 287 insertions(+), 141 deletions(-) create mode 100644 vendor/sigs.k8s.io/yaml/go.mod create mode 100644 vendor/sigs.k8s.io/yaml/go.sum diff --git a/go.mod b/go.mod index 2dba8b5ba..c93820bd8 100644 --- a/go.mod +++ b/go.mod @@ -3,6 +3,7 @@ module sigs.k8s.io/descheduler go 1.13 require ( + github.com/gogo/protobuf v1.3.1 // indirect github.com/spf13/cobra v0.0.5 github.com/spf13/pflag v1.0.5 k8s.io/api v0.17.0 @@ -11,4 +12,6 @@ require ( k8s.io/client-go v0.17.0 k8s.io/component-base v0.17.0 k8s.io/klog v1.0.0 + k8s.io/kube-openapi v0.0.0-20200121204235-bf4fb3bd569c // indirect + sigs.k8s.io/yaml v1.2.0 // indirect ) diff --git a/go.sum b/go.sum index 28f79fbee..f4de7f480 100644 --- a/go.sum +++ b/go.sum @@ -81,6 +81,8 @@ github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7a github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4= github.com/gogo/protobuf v1.2.2-0.20190723190241-65acae22fc9d h1:3PaI8p3seN09VjbTYC/QWlUZdZ1qS1zGjy7LH2Wt07I= github.com/gogo/protobuf v1.2.2-0.20190723190241-65acae22fc9d/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o= +github.com/gogo/protobuf v1.3.1 h1:DqDEcV5aeaTmdFBePNpYsp3FlcVH/2ISVVM9Qf8PSls= +github.com/gogo/protobuf v1.3.1/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= github.com/golang/groupcache v0.0.0-20160516000752-02826c3e7903 h1:LbsanbbD6LieFkXbj9YNNBupiGHJgFeLpO0j0Fza1h8= github.com/golang/groupcache v0.0.0-20160516000752-02826c3e7903/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= @@ -325,6 +327,8 @@ gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.4 h1:/eiJrUcujPVeJ3xlSWaiNi3uSVmDGBK1pDHUHAnao1I= gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.8 h1:obN1ZagJSUGI0Ek/LBmuj4SNLPfIny3KsKFopxRdj10= +gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gotest.tools v2.2.0+incompatible/go.mod h1:DsYFclhRJ6vuDpmuTbkuFWG+y2sxOXAzmJt81HFBacw= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= @@ -347,9 +351,14 @@ k8s.io/klog v1.0.0 h1:Pt+yjF5aB1xDSVbau4VsWe+dQNzA0qv1LlXdC2dF6Q8= k8s.io/klog v1.0.0/go.mod h1:4Bi6QPql/J/LkTDqv7R/cd3hPo4k2DG6Ptcz060Ez5I= k8s.io/kube-openapi v0.0.0-20191107075043-30be4d16710a h1:UcxjrRMyNx/i/y8G7kPvLyy7rfbeuf1PYyBf973pgyU= k8s.io/kube-openapi v0.0.0-20191107075043-30be4d16710a/go.mod h1:1TqjTSzOxsLGIKfj0lK8EeCP7K1iUG65v09OM0/WG5E= +k8s.io/kube-openapi v0.0.0-20200121204235-bf4fb3bd569c h1:/KUFqjjqAcY4Us6luF5RDNZ16KJtb49HfR3ZHB9qYXM= +k8s.io/kube-openapi v0.0.0-20200121204235-bf4fb3bd569c/go.mod h1:GRQhZsXIAJ1xR0C9bd8UpWHZ5plfAS9fzPjJuQ6JL3E= k8s.io/utils v0.0.0-20191114184206-e782cd3c129f h1:GiPwtSzdP43eI1hpPCbROQCCIgCuiMMNF8YUVLF3vJo= k8s.io/utils v0.0.0-20191114184206-e782cd3c129f/go.mod h1:sZAwmy6armz5eXlNoLmJcl4F1QuKu7sr+mFQ0byX7Ew= sigs.k8s.io/structured-merge-diff v0.0.0-20190525122527-15d366b2352e/go.mod h1:wWxsB5ozmmv/SG7nM11ayaAW51xMvak/t1r0CSlcokI= sigs.k8s.io/structured-merge-diff v1.0.1-0.20191108220359-b1b620dd3f06/go.mod h1:/ULNhyfzRopfcjskuui0cTITekDduZ7ycKN3oUT9R18= +sigs.k8s.io/structured-merge-diff/v3 v3.0.0-20200116222232-67a7b8c61874/go.mod h1:PlARxl6Hbt/+BC80dRLi1qAmnMqwqDg62YvvVkZjemw= sigs.k8s.io/yaml v1.1.0 h1:4A07+ZFc2wgJwo8YNlQpr1rVlgUDlxXHhPJciaPY5gs= sigs.k8s.io/yaml v1.1.0/go.mod h1:UJmg0vDUVViEyp3mgSv9WPwZCDxu4rQW1olrI1uml+o= +sigs.k8s.io/yaml v1.2.0 h1:kr/MCeFWJWTwyaHoR9c8EjH9OumOmoF9YGiZd7lFm/Q= +sigs.k8s.io/yaml v1.2.0/go.mod h1:yfXDCHCao9+ENCvLSE62v9VSji2MKu5jeNfTrofGhJc= diff --git a/vendor/github.com/gogo/protobuf/proto/encode.go b/vendor/github.com/gogo/protobuf/proto/encode.go index 3abfed2cf..9581ccd30 100644 --- a/vendor/github.com/gogo/protobuf/proto/encode.go +++ b/vendor/github.com/gogo/protobuf/proto/encode.go @@ -189,6 +189,8 @@ type Marshaler interface { // prefixed by a varint-encoded length. func (p *Buffer) EncodeMessage(pb Message) error { siz := Size(pb) + sizVar := SizeVarint(uint64(siz)) + p.grow(siz + sizVar) p.EncodeVarint(uint64(siz)) return p.Marshal(pb) } diff --git a/vendor/github.com/gogo/protobuf/proto/lib.go b/vendor/github.com/gogo/protobuf/proto/lib.go index d17f80209..80db1c155 100644 --- a/vendor/github.com/gogo/protobuf/proto/lib.go +++ b/vendor/github.com/gogo/protobuf/proto/lib.go @@ -948,13 +948,19 @@ func isProto3Zero(v reflect.Value) bool { return false } -// ProtoPackageIsVersion2 is referenced from generated protocol buffer files -// to assert that that code is compatible with this version of the proto package. -const GoGoProtoPackageIsVersion2 = true +const ( + // ProtoPackageIsVersion3 is referenced from generated protocol buffer files + // to assert that that code is compatible with this version of the proto package. + GoGoProtoPackageIsVersion3 = true -// ProtoPackageIsVersion1 is referenced from generated protocol buffer files -// to assert that that code is compatible with this version of the proto package. -const GoGoProtoPackageIsVersion1 = true + // ProtoPackageIsVersion2 is referenced from generated protocol buffer files + // to assert that that code is compatible with this version of the proto package. + GoGoProtoPackageIsVersion2 = true + + // ProtoPackageIsVersion1 is referenced from generated protocol buffer files + // to assert that that code is compatible with this version of the proto package. + GoGoProtoPackageIsVersion1 = true +) // InternalMessageInfo is a type used internally by generated .pb.go files. // This type is not intended to be used by non-generated code. diff --git a/vendor/github.com/gogo/protobuf/proto/properties.go b/vendor/github.com/gogo/protobuf/proto/properties.go index c9e5fa020..28da1475f 100644 --- a/vendor/github.com/gogo/protobuf/proto/properties.go +++ b/vendor/github.com/gogo/protobuf/proto/properties.go @@ -43,7 +43,6 @@ package proto import ( "fmt" "log" - "os" "reflect" "sort" "strconv" @@ -205,7 +204,7 @@ func (p *Properties) Parse(s string) { // "bytes,49,opt,name=foo,def=hello!" fields := strings.Split(s, ",") // breaks def=, but handled below. if len(fields) < 2 { - fmt.Fprintf(os.Stderr, "proto: tag has too few fields: %q\n", s) + log.Printf("proto: tag has too few fields: %q", s) return } @@ -225,7 +224,7 @@ func (p *Properties) Parse(s string) { p.WireType = WireBytes // no numeric converter for non-numeric types default: - fmt.Fprintf(os.Stderr, "proto: tag has unknown wire type: %q\n", s) + log.Printf("proto: tag has unknown wire type: %q", s) return } @@ -400,6 +399,15 @@ func GetProperties(t reflect.Type) *StructProperties { return sprop } +type ( + oneofFuncsIface interface { + XXX_OneofFuncs() (func(Message, *Buffer) error, func(Message, int, int, *Buffer) (bool, error), func(Message) int, []interface{}) + } + oneofWrappersIface interface { + XXX_OneofWrappers() []interface{} + } +) + // getPropertiesLocked requires that propertiesMu is held. func getPropertiesLocked(t reflect.Type) *StructProperties { if prop, ok := propertiesMap[t]; ok { @@ -441,37 +449,40 @@ func getPropertiesLocked(t reflect.Type) *StructProperties { // Re-order prop.order. sort.Sort(prop) - type oneofMessage interface { - XXX_OneofFuncs() (func(Message, *Buffer) error, func(Message, int, int, *Buffer) (bool, error), func(Message) int, []interface{}) - } - if om, ok := reflect.Zero(reflect.PtrTo(t)).Interface().(oneofMessage); isOneofMessage && ok { + if isOneofMessage { var oots []interface{} - _, _, _, oots = om.XXX_OneofFuncs() - - // Interpret oneof metadata. - prop.OneofTypes = make(map[string]*OneofProperties) - for _, oot := range oots { - oop := &OneofProperties{ - Type: reflect.ValueOf(oot).Type(), // *T - Prop: new(Properties), - } - sft := oop.Type.Elem().Field(0) - oop.Prop.Name = sft.Name - oop.Prop.Parse(sft.Tag.Get("protobuf")) - // There will be exactly one interface field that - // this new value is assignable to. - for i := 0; i < t.NumField(); i++ { - f := t.Field(i) - if f.Type.Kind() != reflect.Interface { - continue + switch m := reflect.Zero(reflect.PtrTo(t)).Interface().(type) { + case oneofFuncsIface: + _, _, _, oots = m.XXX_OneofFuncs() + case oneofWrappersIface: + oots = m.XXX_OneofWrappers() + } + if len(oots) > 0 { + // Interpret oneof metadata. + prop.OneofTypes = make(map[string]*OneofProperties) + for _, oot := range oots { + oop := &OneofProperties{ + Type: reflect.ValueOf(oot).Type(), // *T + Prop: new(Properties), } - if !oop.Type.AssignableTo(f.Type) { - continue + sft := oop.Type.Elem().Field(0) + oop.Prop.Name = sft.Name + oop.Prop.Parse(sft.Tag.Get("protobuf")) + // There will be exactly one interface field that + // this new value is assignable to. + for i := 0; i < t.NumField(); i++ { + f := t.Field(i) + if f.Type.Kind() != reflect.Interface { + continue + } + if !oop.Type.AssignableTo(f.Type) { + continue + } + oop.Field = i + break } - oop.Field = i - break + prop.OneofTypes[oop.Prop.OrigName] = oop } - prop.OneofTypes[oop.Prop.OrigName] = oop } } diff --git a/vendor/github.com/gogo/protobuf/proto/table_marshal.go b/vendor/github.com/gogo/protobuf/proto/table_marshal.go index 9b1538d05..f8babdefa 100644 --- a/vendor/github.com/gogo/protobuf/proto/table_marshal.go +++ b/vendor/github.com/gogo/protobuf/proto/table_marshal.go @@ -389,8 +389,13 @@ func (u *marshalInfo) computeMarshalInfo() { // get oneof implementers var oneofImplementers []interface{} // gogo: isOneofMessage is needed for embedded oneof messages, without a marshaler and unmarshaler - if m, ok := reflect.Zero(reflect.PtrTo(t)).Interface().(oneofMessage); ok && isOneofMessage { - _, _, _, oneofImplementers = m.XXX_OneofFuncs() + if isOneofMessage { + switch m := reflect.Zero(reflect.PtrTo(t)).Interface().(type) { + case oneofFuncsIface: + _, _, _, oneofImplementers = m.XXX_OneofFuncs() + case oneofWrappersIface: + oneofImplementers = m.XXX_OneofWrappers() + } } // normal fields @@ -519,10 +524,6 @@ func (fi *marshalFieldInfo) computeOneofFieldInfo(f *reflect.StructField, oneofI } } -type oneofMessage interface { - XXX_OneofFuncs() (func(Message, *Buffer) error, func(Message, int, int, *Buffer) (bool, error), func(Message) int, []interface{}) -} - // wiretype returns the wire encoding of the type. func wiretype(encoding string) uint64 { switch encoding { @@ -2968,7 +2969,9 @@ func (p *Buffer) Marshal(pb Message) error { if m, ok := pb.(newMarshaler); ok { siz := m.XXX_Size() p.grow(siz) // make sure buf has enough capacity - p.buf, err = m.XXX_Marshal(p.buf, p.deterministic) + pp := p.buf[len(p.buf) : len(p.buf) : len(p.buf)+siz] + pp, err = m.XXX_Marshal(pp, p.deterministic) + p.buf = append(p.buf, pp...) return err } if m, ok := pb.(Marshaler); ok { diff --git a/vendor/github.com/gogo/protobuf/proto/table_merge.go b/vendor/github.com/gogo/protobuf/proto/table_merge.go index f520106e0..60dcf70d1 100644 --- a/vendor/github.com/gogo/protobuf/proto/table_merge.go +++ b/vendor/github.com/gogo/protobuf/proto/table_merge.go @@ -530,6 +530,25 @@ func (mi *mergeInfo) computeMergeInfo() { } case reflect.Struct: switch { + case isSlice && !isPointer: // E.g. []pb.T + mergeInfo := getMergeInfo(tf) + zero := reflect.Zero(tf) + mfi.merge = func(dst, src pointer) { + // TODO: Make this faster? + dstsp := dst.asPointerTo(f.Type) + dsts := dstsp.Elem() + srcs := src.asPointerTo(f.Type).Elem() + for i := 0; i < srcs.Len(); i++ { + dsts = reflect.Append(dsts, zero) + srcElement := srcs.Index(i).Addr() + dstElement := dsts.Index(dsts.Len() - 1).Addr() + mergeInfo.merge(valToPointer(dstElement), valToPointer(srcElement)) + } + if dsts.IsNil() { + dsts = reflect.MakeSlice(f.Type, 0, 0) + } + dstsp.Elem().Set(dsts) + } case !isPointer: mergeInfo := getMergeInfo(tf) mfi.merge = func(dst, src pointer) { diff --git a/vendor/github.com/gogo/protobuf/proto/table_unmarshal.go b/vendor/github.com/gogo/protobuf/proto/table_unmarshal.go index bb2622f28..937229386 100644 --- a/vendor/github.com/gogo/protobuf/proto/table_unmarshal.go +++ b/vendor/github.com/gogo/protobuf/proto/table_unmarshal.go @@ -371,15 +371,18 @@ func (u *unmarshalInfo) computeUnmarshalInfo() { } // Find any types associated with oneof fields. - // TODO: XXX_OneofFuncs returns more info than we need. Get rid of some of it? - fn := reflect.Zero(reflect.PtrTo(t)).MethodByName("XXX_OneofFuncs") // gogo: len(oneofFields) > 0 is needed for embedded oneof messages, without a marshaler and unmarshaler - if fn.IsValid() && len(oneofFields) > 0 { - res := fn.Call(nil)[3] // last return value from XXX_OneofFuncs: []interface{} - for i := res.Len() - 1; i >= 0; i-- { - v := res.Index(i) // interface{} - tptr := reflect.ValueOf(v.Interface()).Type() // *Msg_X - typ := tptr.Elem() // Msg_X + if len(oneofFields) > 0 { + var oneofImplementers []interface{} + switch m := reflect.Zero(reflect.PtrTo(t)).Interface().(type) { + case oneofFuncsIface: + _, _, _, oneofImplementers = m.XXX_OneofFuncs() + case oneofWrappersIface: + oneofImplementers = m.XXX_OneofWrappers() + } + for _, v := range oneofImplementers { + tptr := reflect.TypeOf(v) // *Msg_X + typ := tptr.Elem() // Msg_X f := typ.Field(0) // oneof implementers have one field baseUnmarshal := fieldUnmarshaler(&f) @@ -407,11 +410,12 @@ func (u *unmarshalInfo) computeUnmarshalInfo() { u.setTag(fieldNum, of.field, unmarshal, 0, name) } } + } } // Get extension ranges, if any. - fn = reflect.Zero(reflect.PtrTo(t)).MethodByName("ExtensionRangeArray") + fn := reflect.Zero(reflect.PtrTo(t)).MethodByName("ExtensionRangeArray") if fn.IsValid() { if !u.extensions.IsValid() && !u.oldExtensions.IsValid() && !u.bytesExtensions.IsValid() { panic("a message with extensions, but no extensions field in " + t.Name()) diff --git a/vendor/github.com/gogo/protobuf/proto/text.go b/vendor/github.com/gogo/protobuf/proto/text.go index 0407ba85d..87416afe9 100644 --- a/vendor/github.com/gogo/protobuf/proto/text.go +++ b/vendor/github.com/gogo/protobuf/proto/text.go @@ -476,6 +476,8 @@ func (tm *TextMarshaler) writeStruct(w *textWriter, sv reflect.Value) error { return nil } +var textMarshalerType = reflect.TypeOf((*encoding.TextMarshaler)(nil)).Elem() + // writeAny writes an arbitrary field. func (tm *TextMarshaler) writeAny(w *textWriter, v reflect.Value, props *Properties) error { v = reflect.Indirect(v) @@ -589,8 +591,8 @@ func (tm *TextMarshaler) writeAny(w *textWriter, v reflect.Value, props *Propert // mutating this value. v = v.Addr() } - if etm, ok := v.Interface().(encoding.TextMarshaler); ok { - text, err := etm.MarshalText() + if v.Type().Implements(textMarshalerType) { + text, err := v.Interface().(encoding.TextMarshaler).MarshalText() if err != nil { return err } diff --git a/vendor/gopkg.in/yaml.v2/.travis.yml b/vendor/gopkg.in/yaml.v2/.travis.yml index 9f556934d..055480b9e 100644 --- a/vendor/gopkg.in/yaml.v2/.travis.yml +++ b/vendor/gopkg.in/yaml.v2/.travis.yml @@ -1,12 +1,16 @@ language: go go: - - 1.4 - - 1.5 - - 1.6 - - 1.7 - - 1.8 - - 1.9 - - tip + - "1.4.x" + - "1.5.x" + - "1.6.x" + - "1.7.x" + - "1.8.x" + - "1.9.x" + - "1.10.x" + - "1.11.x" + - "1.12.x" + - "1.13.x" + - "tip" go_import_path: gopkg.in/yaml.v2 diff --git a/vendor/gopkg.in/yaml.v2/decode.go b/vendor/gopkg.in/yaml.v2/decode.go index 531087655..129bc2a97 100644 --- a/vendor/gopkg.in/yaml.v2/decode.go +++ b/vendor/gopkg.in/yaml.v2/decode.go @@ -319,10 +319,14 @@ func (d *decoder) prepare(n *node, out reflect.Value) (newout reflect.Value, unm } const ( - // 400,000 decode operations is ~500kb of dense object declarations, or ~5kb of dense object declarations with 10000% alias expansion + // 400,000 decode operations is ~500kb of dense object declarations, or + // ~5kb of dense object declarations with 10000% alias expansion alias_ratio_range_low = 400000 - // 4,000,000 decode operations is ~5MB of dense object declarations, or ~4.5MB of dense object declarations with 10% alias expansion + + // 4,000,000 decode operations is ~5MB of dense object declarations, or + // ~4.5MB of dense object declarations with 10% alias expansion alias_ratio_range_high = 4000000 + // alias_ratio_range is the range over which we scale allowed alias ratios alias_ratio_range = float64(alias_ratio_range_high - alias_ratio_range_low) ) @@ -784,8 +788,7 @@ func (d *decoder) merge(n *node, out reflect.Value) { case mappingNode: d.unmarshal(n, out) case aliasNode: - an, ok := d.doc.anchors[n.value] - if ok && an.kind != mappingNode { + if n.alias != nil && n.alias.kind != mappingNode { failWantMap() } d.unmarshal(n, out) @@ -794,8 +797,7 @@ func (d *decoder) merge(n *node, out reflect.Value) { for i := len(n.children) - 1; i >= 0; i-- { ni := n.children[i] if ni.kind == aliasNode { - an, ok := d.doc.anchors[ni.value] - if ok && an.kind != mappingNode { + if ni.alias != nil && ni.alias.kind != mappingNode { failWantMap() } } else if ni.kind != mappingNode { diff --git a/vendor/gopkg.in/yaml.v2/scannerc.go b/vendor/gopkg.in/yaml.v2/scannerc.go index 570b8ecd1..0b9bb6030 100644 --- a/vendor/gopkg.in/yaml.v2/scannerc.go +++ b/vendor/gopkg.in/yaml.v2/scannerc.go @@ -626,30 +626,17 @@ func trace(args ...interface{}) func() { func yaml_parser_fetch_more_tokens(parser *yaml_parser_t) bool { // While we need more tokens to fetch, do it. for { - // Check if we really need to fetch more tokens. - need_more_tokens := false - - if parser.tokens_head == len(parser.tokens) { - // Queue is empty. - need_more_tokens = true - } else { - // Check if any potential simple key may occupy the head position. - if !yaml_parser_stale_simple_keys(parser) { + if parser.tokens_head != len(parser.tokens) { + // If queue is non-empty, check if any potential simple key may + // occupy the head position. + head_tok_idx, ok := parser.simple_keys_by_tok[parser.tokens_parsed] + if !ok { + break + } else if valid, ok := yaml_simple_key_is_valid(parser, &parser.simple_keys[head_tok_idx]); !ok { return false + } else if !valid { + break } - - for i := range parser.simple_keys { - simple_key := &parser.simple_keys[i] - if simple_key.possible && simple_key.token_number == parser.tokens_parsed { - need_more_tokens = true - break - } - } - } - - // We are finished. - if !need_more_tokens { - break } // Fetch the next token. if !yaml_parser_fetch_next_token(parser) { @@ -678,11 +665,6 @@ func yaml_parser_fetch_next_token(parser *yaml_parser_t) bool { return false } - // Remove obsolete potential simple keys. - if !yaml_parser_stale_simple_keys(parser) { - return false - } - // Check the indentation level against the current column. if !yaml_parser_unroll_indent(parser, parser.mark.column) { return false @@ -837,29 +819,30 @@ func yaml_parser_fetch_next_token(parser *yaml_parser_t) bool { "found character that cannot start any token") } -// Check the list of potential simple keys and remove the positions that -// cannot contain simple keys anymore. -func yaml_parser_stale_simple_keys(parser *yaml_parser_t) bool { - // Check for a potential simple key for each flow level. - for i := range parser.simple_keys { - simple_key := &parser.simple_keys[i] - - // The specification requires that a simple key - // - // - is limited to a single line, - // - is shorter than 1024 characters. - if simple_key.possible && (simple_key.mark.line < parser.mark.line || simple_key.mark.index+1024 < parser.mark.index) { - - // Check if the potential simple key to be removed is required. - if simple_key.required { - return yaml_parser_set_scanner_error(parser, - "while scanning a simple key", simple_key.mark, - "could not find expected ':'") - } - simple_key.possible = false - } +func yaml_simple_key_is_valid(parser *yaml_parser_t, simple_key *yaml_simple_key_t) (valid, ok bool) { + if !simple_key.possible { + return false, true } - return true + + // The 1.2 specification says: + // + // "If the ? indicator is omitted, parsing needs to see past the + // implicit key to recognize it as such. To limit the amount of + // lookahead required, the “:” indicator must appear at most 1024 + // Unicode characters beyond the start of the key. In addition, the key + // is restricted to a single line." + // + if simple_key.mark.line < parser.mark.line || simple_key.mark.index+1024 < parser.mark.index { + // Check if the potential simple key to be removed is required. + if simple_key.required { + return false, yaml_parser_set_scanner_error(parser, + "while scanning a simple key", simple_key.mark, + "could not find expected ':'") + } + simple_key.possible = false + return false, true + } + return true, true } // Check if a simple key may start at the current position and add it if @@ -879,13 +862,14 @@ func yaml_parser_save_simple_key(parser *yaml_parser_t) bool { possible: true, required: required, token_number: parser.tokens_parsed + (len(parser.tokens) - parser.tokens_head), + mark: parser.mark, } - simple_key.mark = parser.mark if !yaml_parser_remove_simple_key(parser) { return false } parser.simple_keys[len(parser.simple_keys)-1] = simple_key + parser.simple_keys_by_tok[simple_key.token_number] = len(parser.simple_keys) - 1 } return true } @@ -900,9 +884,10 @@ func yaml_parser_remove_simple_key(parser *yaml_parser_t) bool { "while scanning a simple key", parser.simple_keys[i].mark, "could not find expected ':'") } + // Remove the key from the stack. + parser.simple_keys[i].possible = false + delete(parser.simple_keys_by_tok, parser.simple_keys[i].token_number) } - // Remove the key from the stack. - parser.simple_keys[i].possible = false return true } @@ -912,7 +897,12 @@ const max_flow_level = 10000 // Increase the flow level and resize the simple key list if needed. func yaml_parser_increase_flow_level(parser *yaml_parser_t) bool { // Reset the simple key on the next level. - parser.simple_keys = append(parser.simple_keys, yaml_simple_key_t{}) + parser.simple_keys = append(parser.simple_keys, yaml_simple_key_t{ + possible: false, + required: false, + token_number: parser.tokens_parsed + (len(parser.tokens) - parser.tokens_head), + mark: parser.mark, + }) // Increase the flow level. parser.flow_level++ @@ -928,7 +918,9 @@ func yaml_parser_increase_flow_level(parser *yaml_parser_t) bool { func yaml_parser_decrease_flow_level(parser *yaml_parser_t) bool { if parser.flow_level > 0 { parser.flow_level-- - parser.simple_keys = parser.simple_keys[:len(parser.simple_keys)-1] + last := len(parser.simple_keys) - 1 + delete(parser.simple_keys_by_tok, parser.simple_keys[last].token_number) + parser.simple_keys = parser.simple_keys[:last] } return true } @@ -1005,6 +997,8 @@ func yaml_parser_fetch_stream_start(parser *yaml_parser_t) bool { // Initialize the simple key stack. parser.simple_keys = append(parser.simple_keys, yaml_simple_key_t{}) + parser.simple_keys_by_tok = make(map[int]int) + // A simple key is allowed at the beginning of the stream. parser.simple_key_allowed = true @@ -1286,7 +1280,11 @@ func yaml_parser_fetch_value(parser *yaml_parser_t) bool { simple_key := &parser.simple_keys[len(parser.simple_keys)-1] // Have we found a simple key? - if simple_key.possible { + if valid, ok := yaml_simple_key_is_valid(parser, simple_key); !ok { + return false + + } else if valid { + // Create the KEY token and insert it into the queue. token := yaml_token_t{ typ: yaml_KEY_TOKEN, @@ -1304,6 +1302,7 @@ func yaml_parser_fetch_value(parser *yaml_parser_t) bool { // Remove the simple key. simple_key.possible = false + delete(parser.simple_keys_by_tok, simple_key.token_number) // A simple key cannot follow another simple key. parser.simple_key_allowed = false diff --git a/vendor/gopkg.in/yaml.v2/yaml.go b/vendor/gopkg.in/yaml.v2/yaml.go index de85aa4cd..89650e293 100644 --- a/vendor/gopkg.in/yaml.v2/yaml.go +++ b/vendor/gopkg.in/yaml.v2/yaml.go @@ -89,7 +89,7 @@ func UnmarshalStrict(in []byte, out interface{}) (err error) { return unmarshal(in, out, true) } -// A Decorder reads and decodes YAML values from an input stream. +// A Decoder reads and decodes YAML values from an input stream. type Decoder struct { strict bool parser *parser diff --git a/vendor/gopkg.in/yaml.v2/yamlh.go b/vendor/gopkg.in/yaml.v2/yamlh.go index e25cee563..f6a9c8e34 100644 --- a/vendor/gopkg.in/yaml.v2/yamlh.go +++ b/vendor/gopkg.in/yaml.v2/yamlh.go @@ -579,6 +579,7 @@ type yaml_parser_t struct { simple_key_allowed bool // May a simple key occur at the current position? simple_keys []yaml_simple_key_t // The stack of simple keys. + simple_keys_by_tok map[int]int // possible simple_key indexes indexed by token_number // Parser stuff diff --git a/vendor/modules.txt b/vendor/modules.txt index aa10a1873..08264ef6c 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -17,7 +17,7 @@ github.com/davecgh/go-spew/spew github.com/dgrijalva/jwt-go # github.com/evanphx/json-patch v4.2.0+incompatible github.com/evanphx/json-patch -# github.com/gogo/protobuf v1.2.2-0.20190723190241-65acae22fc9d +# github.com/gogo/protobuf v1.3.1 github.com/gogo/protobuf/proto github.com/gogo/protobuf/sortkeys # github.com/golang/groupcache v0.0.0-20160516000752-02826c3e7903 @@ -103,7 +103,7 @@ google.golang.org/appengine/internal/urlfetch google.golang.org/appengine/urlfetch # gopkg.in/inf.v0 v0.9.1 gopkg.in/inf.v0 -# gopkg.in/yaml.v2 v2.2.4 +# gopkg.in/yaml.v2 v2.2.8 gopkg.in/yaml.v2 # k8s.io/api v0.17.0 k8s.io/api/admissionregistration/v1 @@ -409,11 +409,11 @@ k8s.io/component-base/featuregate k8s.io/component-base/logs # k8s.io/klog v1.0.0 k8s.io/klog -# k8s.io/kube-openapi v0.0.0-20191107075043-30be4d16710a +# k8s.io/kube-openapi v0.0.0-20200121204235-bf4fb3bd569c k8s.io/kube-openapi/pkg/util/proto # k8s.io/utils v0.0.0-20191114184206-e782cd3c129f k8s.io/utils/buffer k8s.io/utils/integer k8s.io/utils/trace -# sigs.k8s.io/yaml v1.1.0 +# sigs.k8s.io/yaml v1.2.0 sigs.k8s.io/yaml diff --git a/vendor/sigs.k8s.io/yaml/.travis.yml b/vendor/sigs.k8s.io/yaml/.travis.yml index 03ddc7318..d20e23eff 100644 --- a/vendor/sigs.k8s.io/yaml/.travis.yml +++ b/vendor/sigs.k8s.io/yaml/.travis.yml @@ -1,14 +1,13 @@ language: go dist: xenial go: - - 1.9.x - - 1.10.x - - 1.11.x + - 1.12.x + - 1.13.x script: - - go get -t -v ./... - - diff -u <(echo -n) <(gofmt -d .) + - diff -u <(echo -n) <(gofmt -d *.go) - diff -u <(echo -n) <(golint $(go list -e ./...) | grep -v YAMLToJSON) - - go tool vet . - - go test -v -race ./... + - GO111MODULE=on go vet . + - GO111MODULE=on go test -v -race ./... + - git diff --exit-code install: - - go get golang.org/x/lint/golint + - GO111MODULE=off go get golang.org/x/lint/golint diff --git a/vendor/sigs.k8s.io/yaml/OWNERS b/vendor/sigs.k8s.io/yaml/OWNERS index 11ad7ce1a..325b40b07 100644 --- a/vendor/sigs.k8s.io/yaml/OWNERS +++ b/vendor/sigs.k8s.io/yaml/OWNERS @@ -1,3 +1,5 @@ +# See the OWNERS docs at https://go.k8s.io/owners + approvers: - dims - lavalamp diff --git a/vendor/sigs.k8s.io/yaml/README.md b/vendor/sigs.k8s.io/yaml/README.md index 0200f75b4..5a651d916 100644 --- a/vendor/sigs.k8s.io/yaml/README.md +++ b/vendor/sigs.k8s.io/yaml/README.md @@ -1,12 +1,14 @@ # YAML marshaling and unmarshaling support for Go -[![Build Status](https://travis-ci.org/ghodss/yaml.svg)](https://travis-ci.org/ghodss/yaml) +[![Build Status](https://travis-ci.org/kubernetes-sigs/yaml.svg)](https://travis-ci.org/kubernetes-sigs/yaml) + +kubernetes-sigs/yaml is a permanent fork of [ghodss/yaml](https://github.com/ghodss/yaml). ## Introduction A wrapper around [go-yaml](https://github.com/go-yaml/yaml) designed to enable a better way of handling YAML when marshaling to and from structs. -In short, this library first converts YAML to JSON using go-yaml and then uses `json.Marshal` and `json.Unmarshal` to convert to or from the struct. This means that it effectively reuses the JSON struct tags as well as the custom JSON methods `MarshalJSON` and `UnmarshalJSON` unlike go-yaml. For a detailed overview of the rationale behind this method, [see this blog post](http://ghodss.com/2014/the-right-way-to-handle-yaml-in-golang/). +In short, this library first converts YAML to JSON using go-yaml and then uses `json.Marshal` and `json.Unmarshal` to convert to or from the struct. This means that it effectively reuses the JSON struct tags as well as the custom JSON methods `MarshalJSON` and `UnmarshalJSON` unlike go-yaml. For a detailed overview of the rationale behind this method, [see this blog post](http://web.archive.org/web/20190603050330/http://ghodss.com/2014/the-right-way-to-handle-yaml-in-golang/). ## Compatibility @@ -32,13 +34,13 @@ GOOD: To install, run: ``` -$ go get github.com/ghodss/yaml +$ go get sigs.k8s.io/yaml ``` And import using: ``` -import "github.com/ghodss/yaml" +import "sigs.k8s.io/yaml" ``` Usage is very similar to the JSON library: @@ -49,7 +51,7 @@ package main import ( "fmt" - "github.com/ghodss/yaml" + "sigs.k8s.io/yaml" ) type Person struct { @@ -93,7 +95,7 @@ package main import ( "fmt" - "github.com/ghodss/yaml" + "sigs.k8s.io/yaml" ) func main() { diff --git a/vendor/sigs.k8s.io/yaml/go.mod b/vendor/sigs.k8s.io/yaml/go.mod new file mode 100644 index 000000000..7224f3497 --- /dev/null +++ b/vendor/sigs.k8s.io/yaml/go.mod @@ -0,0 +1,8 @@ +module sigs.k8s.io/yaml + +go 1.12 + +require ( + github.com/davecgh/go-spew v1.1.1 + gopkg.in/yaml.v2 v2.2.8 +) diff --git a/vendor/sigs.k8s.io/yaml/go.sum b/vendor/sigs.k8s.io/yaml/go.sum new file mode 100644 index 000000000..76e49483a --- /dev/null +++ b/vendor/sigs.k8s.io/yaml/go.sum @@ -0,0 +1,9 @@ +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw= +gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.7 h1:VUgggvou5XRW9mHwD/yXxIYSMtY0zoKQf/v226p2nyo= +gopkg.in/yaml.v2 v2.2.7/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.8 h1:obN1ZagJSUGI0Ek/LBmuj4SNLPfIny3KsKFopxRdj10= +gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= diff --git a/vendor/sigs.k8s.io/yaml/yaml.go b/vendor/sigs.k8s.io/yaml/yaml.go index 024596112..efbc535d4 100644 --- a/vendor/sigs.k8s.io/yaml/yaml.go +++ b/vendor/sigs.k8s.io/yaml/yaml.go @@ -317,3 +317,64 @@ func convertToJSONableObject(yamlObj interface{}, jsonTarget *reflect.Value) (in return yamlObj, nil } } + +// JSONObjectToYAMLObject converts an in-memory JSON object into a YAML in-memory MapSlice, +// without going through a byte representation. A nil or empty map[string]interface{} input is +// converted to an empty map, i.e. yaml.MapSlice(nil). +// +// interface{} slices stay interface{} slices. map[string]interface{} becomes yaml.MapSlice. +// +// int64 and float64 are down casted following the logic of github.com/go-yaml/yaml: +// - float64s are down-casted as far as possible without data-loss to int, int64, uint64. +// - int64s are down-casted to int if possible without data-loss. +// +// Big int/int64/uint64 do not lose precision as in the json-yaml roundtripping case. +// +// string, bool and any other types are unchanged. +func JSONObjectToYAMLObject(j map[string]interface{}) yaml.MapSlice { + if len(j) == 0 { + return nil + } + ret := make(yaml.MapSlice, 0, len(j)) + for k, v := range j { + ret = append(ret, yaml.MapItem{Key: k, Value: jsonToYAMLValue(v)}) + } + return ret +} + +func jsonToYAMLValue(j interface{}) interface{} { + switch j := j.(type) { + case map[string]interface{}: + if j == nil { + return interface{}(nil) + } + return JSONObjectToYAMLObject(j) + case []interface{}: + if j == nil { + return interface{}(nil) + } + ret := make([]interface{}, len(j)) + for i := range j { + ret[i] = jsonToYAMLValue(j[i]) + } + return ret + case float64: + // replicate the logic in https://github.com/go-yaml/yaml/blob/51d6538a90f86fe93ac480b35f37b2be17fef232/resolve.go#L151 + if i64 := int64(j); j == float64(i64) { + if i := int(i64); i64 == int64(i) { + return i + } + return i64 + } + if ui64 := uint64(j); j == float64(ui64) { + return ui64 + } + return j + case int64: + if i := int(j); j == int64(i) { + return i + } + return j + } + return j +}