diff --git a/CHANGELOG-1.14.md b/CHANGELOG-1.14.md index 076bf1342b104..53de5d6bf2bfc 100644 --- a/CHANGELOG-1.14.md +++ b/CHANGELOG-1.14.md @@ -1,23 +1,30 @@ -- [v1.14.2](#v1142) - - [Downloads for v1.14.2](#downloads-for-v1142) +- [v1.14.3](#v1143) + - [Downloads for v1.14.3](#downloads-for-v1143) - [Client Binaries](#client-binaries) - [Server Binaries](#server-binaries) - [Node Binaries](#node-binaries) - - [Changelog since v1.14.1](#changelog-since-v1141) + - [Changelog since v1.14.2](#changelog-since-v1142) - [Other notable changes](#other-notable-changes) -- [v1.14.1](#v1141) - - [Downloads for v1.14.1](#downloads-for-v1141) +- [v1.14.2](#v1142) + - [Downloads for v1.14.2](#downloads-for-v1142) - [Client Binaries](#client-binaries-1) - [Server Binaries](#server-binaries-1) - [Node Binaries](#node-binaries-1) - - [Changelog since v1.14.0](#changelog-since-v1140) + - [Changelog since v1.14.1](#changelog-since-v1141) - [Other notable changes](#other-notable-changes-1) -- [v1.14.0](#v1140) - - [Downloads for v1.14.0](#downloads-for-v1140) +- [v1.14.1](#v1141) + - [Downloads for v1.14.1](#downloads-for-v1141) - [Client Binaries](#client-binaries-2) - [Server Binaries](#server-binaries-2) - [Node Binaries](#node-binaries-2) + - [Changelog since v1.14.0](#changelog-since-v1140) + - [Other notable changes](#other-notable-changes-2) +- [v1.14.0](#v1140) + - [Downloads for v1.14.0](#downloads-for-v1140) + - [Client Binaries](#client-binaries-3) + - [Server Binaries](#server-binaries-3) + - [Node Binaries](#node-binaries-3) - [Kubernetes v1.14 Release Notes](#kubernetes-v114-release-notes) - [1.14 What’s New](#114-whats-new) - [Known Issues](#known-issues) @@ -49,57 +56,124 @@ - [External Dependencies](#external-dependencies) - [v1.14.0-rc.1](#v1140-rc1) - [Downloads for v1.14.0-rc.1](#downloads-for-v1140-rc1) - - [Client Binaries](#client-binaries-3) - - [Server Binaries](#server-binaries-3) - - [Node Binaries](#node-binaries-3) + - [Client Binaries](#client-binaries-4) + - [Server Binaries](#server-binaries-4) + - [Node Binaries](#node-binaries-4) - [Changelog since v1.14.0-beta.2](#changelog-since-v1140-beta2) - [Action Required](#action-required) - - [Other notable changes](#other-notable-changes-2) + - [Other notable changes](#other-notable-changes-3) - [v1.14.0-beta.2](#v1140-beta2) - [Downloads for v1.14.0-beta.2](#downloads-for-v1140-beta2) - - [Client Binaries](#client-binaries-4) - - [Server Binaries](#server-binaries-4) - - [Node Binaries](#node-binaries-4) + - [Client Binaries](#client-binaries-5) + - [Server Binaries](#server-binaries-5) + - [Node Binaries](#node-binaries-5) - [Changelog since v1.14.0-beta.1](#changelog-since-v1140-beta1) - [Action Required](#action-required-1) - - [Other notable changes](#other-notable-changes-3) + - [Other notable changes](#other-notable-changes-4) - [v1.14.0-beta.1](#v1140-beta1) - [Downloads for v1.14.0-beta.1](#downloads-for-v1140-beta1) - - [Client Binaries](#client-binaries-5) - - [Server Binaries](#server-binaries-5) - - [Node Binaries](#node-binaries-5) + - [Client Binaries](#client-binaries-6) + - [Server Binaries](#server-binaries-6) + - [Node Binaries](#node-binaries-6) - [Changelog since v1.14.0-alpha.3](#changelog-since-v1140-alpha3) - [Action Required](#action-required-2) - - [Other notable changes](#other-notable-changes-4) + - [Other notable changes](#other-notable-changes-5) - [v1.14.0-alpha.3](#v1140-alpha3) - [Downloads for v1.14.0-alpha.3](#downloads-for-v1140-alpha3) - - [Client Binaries](#client-binaries-6) - - [Server Binaries](#server-binaries-6) - - [Node Binaries](#node-binaries-6) + - [Client Binaries](#client-binaries-7) + - [Server Binaries](#server-binaries-7) + - [Node Binaries](#node-binaries-7) - [Changelog since v1.14.0-alpha.2](#changelog-since-v1140-alpha2) - [Action Required](#action-required-3) - - [Other notable changes](#other-notable-changes-5) + - [Other notable changes](#other-notable-changes-6) - [v1.14.0-alpha.2](#v1140-alpha2) - [Downloads for v1.14.0-alpha.2](#downloads-for-v1140-alpha2) - - [Client Binaries](#client-binaries-7) - - [Server Binaries](#server-binaries-7) - - [Node Binaries](#node-binaries-7) + - [Client Binaries](#client-binaries-8) + - [Server Binaries](#server-binaries-8) + - [Node Binaries](#node-binaries-8) - [Changelog since v1.14.0-alpha.1](#changelog-since-v1140-alpha1) - [Action Required](#action-required-4) - - [Other notable changes](#other-notable-changes-6) + - [Other notable changes](#other-notable-changes-7) - [v1.14.0-alpha.1](#v1140-alpha1) - [Downloads for v1.14.0-alpha.1](#downloads-for-v1140-alpha1) - - [Client Binaries](#client-binaries-8) - - [Server Binaries](#server-binaries-8) - - [Node Binaries](#node-binaries-8) + - [Client Binaries](#client-binaries-9) + - [Server Binaries](#server-binaries-9) + - [Node Binaries](#node-binaries-9) - [Changelog since v1.13.0](#changelog-since-v1130) - [Action Required](#action-required-5) - - [Other notable changes](#other-notable-changes-7) + - [Other notable changes](#other-notable-changes-8) +# v1.14.3 + +[Documentation](https://docs.k8s.io) + +## Downloads for v1.14.3 + + +filename | sha512 hash +-------- | ----------- +[kubernetes.tar.gz](https://dl.k8s.io/v1.14.3/kubernetes.tar.gz) | `7e82dc2070b55ca8f4746579a66d9dcd61e750f8c20632f901cbeb8c9c5bc28b7c428531d96375a5491e22397e4da3abaad2b1399250736380c586a3163fc960` +[kubernetes-src.tar.gz](https://dl.k8s.io/v1.14.3/kubernetes-src.tar.gz) | `58db2d138b1a0575dfdc9be1a742788c56626275f588d262ab64840627e4055c68480f3b2b96c0059967e13f53028cc0bae5882f4abf1767bfcea15ebb761c73` + +### Client Binaries + +filename | sha512 hash +-------- | ----------- +[kubernetes-client-darwin-386.tar.gz](https://dl.k8s.io/v1.14.3/kubernetes-client-darwin-386.tar.gz) | `29ee18e669171002a11b39fe5046577d91472dfb089de95ad23499a8eb686777887c0bcc81a1681b7ad28f35303ed638bfef36ec94fc19caec9982ba94cd9c74` +[kubernetes-client-darwin-amd64.tar.gz](https://dl.k8s.io/v1.14.3/kubernetes-client-darwin-amd64.tar.gz) | `2cf1680e9dfae84dbf4d1f10e9142a087ac9be56c9c5233b0cea669f9dae59ca645e23109a5f039c174dea42e035180e9e3b5298affcba4aa980760af16ba2ff` +[kubernetes-client-linux-386.tar.gz](https://dl.k8s.io/v1.14.3/kubernetes-client-linux-386.tar.gz) | `39ace58bdb85397b856b52d54e97eb9f489169995f55a55b278c1c923829c0c264113a3251f01b705e53f35a213d420570feff41c130935aca3ce9e0c1bccc5b` +[kubernetes-client-linux-amd64.tar.gz](https://dl.k8s.io/v1.14.3/kubernetes-client-linux-amd64.tar.gz) | `dcb6eeebf618a01f274091998984f8afca567f7a20ffff810bef97e9654af5e4b912f83c97e6ed13dd1b93fcea2c14d7cde4fd55b174f32aa1f2f005baadf2d7` +[kubernetes-client-linux-arm.tar.gz](https://dl.k8s.io/v1.14.3/kubernetes-client-linux-arm.tar.gz) | `4a7dd3310fa96e842ab3505f7815b0778af621fd639ab8296b4994428d930d63057f073e81ebc29852beaa4f6e55271289342c90d46d1b5a70564c58f1bd61c7` +[kubernetes-client-linux-arm64.tar.gz](https://dl.k8s.io/v1.14.3/kubernetes-client-linux-arm64.tar.gz) | `7f091cff9c63d58623b5b49967b8824e02139a347c9c5eb9c9fa178b3825320614623afc2e0a2414d29b3cb8f70c8ecfbb23c907e351c63312cd4bc4a57d4610` +[kubernetes-client-linux-ppc64le.tar.gz](https://dl.k8s.io/v1.14.3/kubernetes-client-linux-ppc64le.tar.gz) | `f7f43d71663a19c2526ae8496b76b2e9df2285ae31f94ba054e45704cc94e2e04cc563fdfce6afebd532ecaf2fb2e68e7ed1bd2c39641ac10bdf910cb17df3ca` +[kubernetes-client-linux-s390x.tar.gz](https://dl.k8s.io/v1.14.3/kubernetes-client-linux-s390x.tar.gz) | `23e782ab1fe8e5d30e1cdff8594de4cc4750ab178ae3c71c177c49acf9f35d908bdf8281ecb174a52664b9841ece206ec8140901334ef7836f556fbe66c04572` +[kubernetes-client-windows-386.tar.gz](https://dl.k8s.io/v1.14.3/kubernetes-client-windows-386.tar.gz) | `25c05c105996c0d75443e75b0809b7bf7c33d6d8d1d24d2b0b9c2f872838277030870c02c96bcd4ac378c66a7ad59e2165293a6ac6feea12691d68749dfe1c5b` +[kubernetes-client-windows-amd64.tar.gz](https://dl.k8s.io/v1.14.3/kubernetes-client-windows-amd64.tar.gz) | `df0da8deca3a354406a4be3f025bd9b6729f68c4414126bdcf58a649228196de42d8b94227963bece055801cf883a5de3cbb68b940bfdc98c56eab26e3015029` + +### Server Binaries + +filename | sha512 hash +-------- | ----------- +[kubernetes-server-linux-amd64.tar.gz](https://dl.k8s.io/v1.14.3/kubernetes-server-linux-amd64.tar.gz) | `8d7192d1e74a139feca532d9956f6d3dc5289eb5c3eff643fd13d02815ed11a12dca5580e830d69cb142e1b5e99e0ecf8939d9e260888bca184f9e013f2b5948` +[kubernetes-server-linux-arm.tar.gz](https://dl.k8s.io/v1.14.3/kubernetes-server-linux-arm.tar.gz) | `89c26c83562e32340ab58a0aa835c0b586c6626e811231129645205edfc0bf563aba4125bb932761a27757a0f205ea013fea30210448eac68cd5b25787b60bc8` +[kubernetes-server-linux-arm64.tar.gz](https://dl.k8s.io/v1.14.3/kubernetes-server-linux-arm64.tar.gz) | `f74ef5b4a680167251b93d346c9864436f3906f72d676483c9ce27ea0f42e366eab0f85aebd138b915fae2e893bed579a755be9869bb0fce43c0460cd629738c` +[kubernetes-server-linux-ppc64le.tar.gz](https://dl.k8s.io/v1.14.3/kubernetes-server-linux-ppc64le.tar.gz) | `6bfc3cbd4604112de5a09fa9c8e35ad4637d1fedc024ef20fafb9ef5adf5f160656a6e7c3298d7f39f741872dc496e5f1b3dc422b5e6f24be7b1d20704e80f6b` +[kubernetes-server-linux-s390x.tar.gz](https://dl.k8s.io/v1.14.3/kubernetes-server-linux-s390x.tar.gz) | `a1355a9db2e34b062b5e335c52e7a5e2b6e55490896bf9ec1f3900b048b73d4c9eeac6fd770efed269a7fbdcdc613ba10fce3ce4d22aa6d86609790b91c321e9` + +### Node Binaries + +filename | sha512 hash +-------- | ----------- +[kubernetes-node-linux-amd64.tar.gz](https://dl.k8s.io/v1.14.3/kubernetes-node-linux-amd64.tar.gz) | `1eb8610f93153b5e274ae61f68f9ccd30bdb5373b6098b03a0f885da4b389bfd2f060208aedab663c03311bcae853feed50070e55497d353b243e781d7cef62a` +[kubernetes-node-linux-arm.tar.gz](https://dl.k8s.io/v1.14.3/kubernetes-node-linux-arm.tar.gz) | `cedef35f9e84c93d099117a413de39bb5a55e532de1b24dd6c53bcf849ed3100321a0edbd1b895d0a82c0cbdf8d567a44818c9b5d7fd064a168f42a300c834e0` +[kubernetes-node-linux-arm64.tar.gz](https://dl.k8s.io/v1.14.3/kubernetes-node-linux-arm64.tar.gz) | `d1f4152ece68cd3d96ff6632fd2229f7da630d5256fd8edd225ee75a4fc60976ad0778dfc71dee8bafdf6b1373a365a157f17faec00fdf8ce87664ace99998f2` +[kubernetes-node-linux-ppc64le.tar.gz](https://dl.k8s.io/v1.14.3/kubernetes-node-linux-ppc64le.tar.gz) | `dbfc833c385c46118aac8d651b9053e48ee9f961fee0668756db0a1254d178d7c9d6c160e6dc9c14e643a1831a9d0e989a00a1b78b455b822c3db8ba29f6b223` +[kubernetes-node-linux-s390x.tar.gz](https://dl.k8s.io/v1.14.3/kubernetes-node-linux-s390x.tar.gz) | `d715253ab6d2b6b6478d19f4ee4b69d4ba4ae5f20643c3b908fe6e481900f83440ded286c955aa1b189bf4ea6461a0d9e4348d61205dc45e1ef26f8b2c03326b` +[kubernetes-node-windows-amd64.tar.gz](https://dl.k8s.io/v1.14.3/kubernetes-node-windows-amd64.tar.gz) | `3e111f2121c4d87d9492c8ba13812537cce11bf2c52852715d0ee5a3d475981567a058b0a17de9f65e1ba09cb5e1b4b1a78c530d78850a0edfe610b77a7dca18` + +## Changelog since v1.14.2 + +### Other notable changes + +* Fix vSphere SAML token auth when using Zones ([#78180](https://github.com/kubernetes/kubernetes/pull/78180), [@dougm](https://github.com/dougm)) +* IPVS: Disable graceful termination for UDP traffic to solve issues with high number of UDP connections (DNS / syslog in particular) ([#77802](https://github.com/kubernetes/kubernetes/pull/77802), [@lbernail](https://github.com/lbernail)) +* Fix broken detection of non-root image user ID ([#78261](https://github.com/kubernetes/kubernetes/pull/78261), [@tallclair](https://github.com/tallclair)) +* fix azure retry issue when return 2XX with error ([#78298](https://github.com/kubernetes/kubernetes/pull/78298), [@andyzhangx](https://github.com/andyzhangx)) +* kubelet: fix fail to close kubelet->API connections on heartbeat failure when bootstrapping or client certificate rotation is disabled ([#78016](https://github.com/kubernetes/kubernetes/pull/78016), [@gaorong](https://github.com/gaorong)) +* Active watches of custom resources now terminate properly if the CRD is modified. ([#78029](https://github.com/kubernetes/kubernetes/pull/78029), [@liggitt](https://github.com/liggitt)) +* client-go and kubectl no longer write cached discovery files with world-accessible file permissions ([#77874](https://github.com/kubernetes/kubernetes/pull/77874), [@yuchengwu](https://github.com/yuchengwu)) +* Fix panic logspam when running kubelet in standalone mode. ([#77888](https://github.com/kubernetes/kubernetes/pull/77888), [@tallclair](https://github.com/tallclair)) +* Fixed a bug in the apiserver storage that could cause just-added finalizers to be ignored on an immediately following delete request, leading to premature deletion. ([#77619](https://github.com/kubernetes/kubernetes/pull/77619), [@caesarxuchao](https://github.com/caesarxuchao)) +* Fixes a bug where dry-run is not honored for pod/eviction sub-resource. ([#76969](https://github.com/kubernetes/kubernetes/pull/76969), [@apelisse](https://github.com/apelisse)) +* fix incorrect prometheus azure metrics ([#77722](https://github.com/kubernetes/kubernetes/pull/77722), [@andyzhangx](https://github.com/andyzhangx)) +* fix KUBE_SERVER_PLATFORMS null error when cross compiling kubectl for non-linux platform ([#78059](https://github.com/kubernetes/kubernetes/pull/78059), [@figo](https://github.com/figo)) + + + # v1.14.2 [Documentation](https://docs.k8s.io) @@ -375,6 +449,7 @@ kubernetes/enhancements: [kep](https://github.com/kubernetes/enhancements/blob/m - Default RBAC policy no longer grants access to discovery and permission-checking APIs (used by `kubectl auth can-i`) to *unauthenticated* users. Upgraded clusters preserve prior behavior, but cluster administrators wishing to grant unauthenticated users access in new clusters will need to explicitly opt-in to expose the discovery and/or permission-checking APIs: - `kubectl create clusterrolebinding anonymous-discovery --clusterrole=system:discovery --group=system:unauthenticated` - `kubectl create clusterrolebinding anonymous-access-review --clusterrole=system:basic-user --group=system:unauthenticated` + ([#73807](https://github.com/kubernetes/kubernetes/pull/73807), [@dekkagaijin](https://github.com/dekkagaijin)) - The deprecated --storage-versions flag has been removed. The storage versions will always be the default value built-in the kube-apiserver binary. ([#67678](https://github.com/kubernetes/kubernetes/pull/67678), [@caesarxuchao](https://github.com/caesarxuchao)) - The deprecated `--repair-malformed-updates` flag has been removed ([#73663](https://github.com/kubernetes/kubernetes/pull/73663), [@danielqsj](https://github.com/danielqsj)) - The `/swaggerapi/*` schema docs, deprecated since 1.7, have been removed in favor of the /openapi/v2 schema docs. ([#72924](https://github.com/kubernetes/kubernetes/pull/72924), [@liggitt](https://github.com/liggitt)) @@ -966,12 +1041,7 @@ filename | sha512 hash * While this is a backwards-incompatible change, it would have been impossible to setup reliable monitoring around these metrics since the labels were not stable. * Add a configuration field to shorten the timeout of validating/mutating admission webhook call. The timeout value must be between 1 and 30 seconds. Default to 30 seconds when unspecified. ([#74562](https://github.com/kubernetes/kubernetes/pull/74562), [@roycaihw](https://github.com/roycaihw)) * client-go: PortForwarder.GetPorts() now contain correct local port if no local port was initially specified when setting up the port forwarder ([#73676](https://github.com/kubernetes/kubernetes/pull/73676), [@martin-helmich](https://github.com/martin-helmich)) -* # Apply resources from a directory containing kustomization.yaml ([#74140](https://github.com/kubernetes/kubernetes/pull/74140), [@Liujingfang1](https://github.com/Liujingfang1)) - * kubectl apply -k dir - * # Delete resources from a directory containing kustomization.yaml. - * kubectl delete -k dir - * # List resources from a directory containing kustomization.yaml - * kubectl get -k dir +* The examples in kubectl apply/get/delete are updated to support `-k` which uses a `kustomization.yaml` file. ([#74140](https://github.com/kubernetes/kubernetes/pull/74140), [@Liujingfang1](https://github.com/Liujingfang1)) * kubeadm: Allow to download certificate secrets uploaded by `init` or `upload-certs` phase, allowing to transfer certificate secrets (certificates and keys) from the cluster to other master machines when creating HA deployments. ([#74168](https://github.com/kubernetes/kubernetes/pull/74168), [@ereslibre](https://github.com/ereslibre)) * Fixes an issue with missing apiVersion/kind in object data sent to admission webhooks ([#74448](https://github.com/kubernetes/kubernetes/pull/74448), [@liggitt](https://github.com/liggitt)) * client-go: the deprecated versionless API group accessors (like `clientset.Apps()` have been removed). Use an explicit version instead (like `clientset.AppsV1()`) ([#74422](https://github.com/kubernetes/kubernetes/pull/74422), [@liggitt](https://github.com/liggitt)) diff --git a/api/openapi-spec/swagger.json b/api/openapi-spec/swagger.json index 759a438fc3150..00dd3e060a388 100644 --- a/api/openapi-spec/swagger.json +++ b/api/openapi-spec/swagger.json @@ -18300,7 +18300,7 @@ }, "info": { "title": "Kubernetes", - "version": "v1.14.3" + "version": "v1.14.4" }, "paths": { "/api/": { diff --git a/cluster/addons/ip-masq-agent/ip-masq-agent.yaml b/cluster/addons/ip-masq-agent/ip-masq-agent.yaml index 39f709f798124..d9ca3744b49fd 100644 --- a/cluster/addons/ip-masq-agent/ip-masq-agent.yaml +++ b/cluster/addons/ip-masq-agent/ip-masq-agent.yaml @@ -8,7 +8,7 @@ metadata: kubernetes.io/cluster-service: "true" addonmanager.kubernetes.io/mode: Reconcile --- -# https://github.com/kubernetes-incubator/ip-masq-agent/blob/v2.0.0/README.md +# https://github.com/kubernetes-incubator/ip-masq-agent/blob/v2.3.0/README.md apiVersion: apps/v1 kind: DaemonSet metadata: @@ -32,7 +32,7 @@ spec: hostNetwork: true containers: - name: ip-masq-agent - image: k8s.gcr.io/ip-masq-agent-amd64:v2.1.1 + image: k8s.gcr.io/ip-masq-agent-amd64:v2.3.0 args: - --masq-chain=IP-MASQ resources: diff --git a/cmd/kube-controller-manager/app/core.go b/cmd/kube-controller-manager/app/core.go index 7d9d3bfdfd142..aa4be25a0fee6 100644 --- a/cmd/kube-controller-manager/app/core.go +++ b/cmd/kube-controller-manager/app/core.go @@ -224,6 +224,7 @@ func startAttachDetachController(ctx ControllerContext) (http.Handler, bool, err ctx.InformerFactory.Core().V1().PersistentVolumeClaims(), ctx.InformerFactory.Core().V1().PersistentVolumes(), ctx.InformerFactory.Storage().V1beta1().CSINodes(), + ctx.InformerFactory.Storage().V1beta1().CSIDrivers(), ctx.Cloud, ProbeAttachableVolumePlugins(), GetDynamicPluginProber(ctx.ComponentConfig.PersistentVolumeBinderController.VolumeConfiguration), diff --git a/cmd/kubelet/app/server.go b/cmd/kubelet/app/server.go index e5557ead4e201..815dbfd8cb644 100644 --- a/cmd/kubelet/app/server.go +++ b/cmd/kubelet/app/server.go @@ -708,9 +708,10 @@ func run(s *options.KubeletServer, kubeDeps *kubelet.Dependencies, stopCh <-chan } if s.HealthzPort > 0 { - healthz.DefaultHealthz() + mux := http.NewServeMux() + healthz.InstallHandler(mux) go wait.Until(func() { - err := http.ListenAndServe(net.JoinHostPort(s.HealthzBindAddress, strconv.Itoa(int(s.HealthzPort))), nil) + err := http.ListenAndServe(net.JoinHostPort(s.HealthzBindAddress, strconv.Itoa(int(s.HealthzPort))), mux) if err != nil { klog.Errorf("Starting health server failed: %v", err) } diff --git a/pkg/api/pod/util.go b/pkg/api/pod/util.go index 3bffd1bf292b4..3b21ae8196611 100644 --- a/pkg/api/pod/util.go +++ b/pkg/api/pod/util.go @@ -406,12 +406,22 @@ func dropDisabledProcMountField(podSpec, oldPodSpec *api.PodSpec) { defaultProcMount := api.DefaultProcMount for i := range podSpec.Containers { if podSpec.Containers[i].SecurityContext != nil { - podSpec.Containers[i].SecurityContext.ProcMount = &defaultProcMount + if podSpec.Containers[i].SecurityContext.ProcMount != nil { + // The ProcMount field was improperly forced to non-nil in 1.12. + // If the feature is disabled, and the existing object is not using any non-default values, and the ProcMount field is present in the incoming object, force to the default value. + // Note: we cannot force the field to nil when the feature is disabled because it causes a diff against previously persisted data. + podSpec.Containers[i].SecurityContext.ProcMount = &defaultProcMount + } } } for i := range podSpec.InitContainers { if podSpec.InitContainers[i].SecurityContext != nil { - podSpec.InitContainers[i].SecurityContext.ProcMount = &defaultProcMount + if podSpec.InitContainers[i].SecurityContext.ProcMount != nil { + // The ProcMount field was improperly forced to non-nil in 1.12. + // If the feature is disabled, and the existing object is not using any non-default values, and the ProcMount field is present in the incoming object, force to the default value. + // Note: we cannot force the field to nil when the feature is disabled because it causes a diff against previously persisted data. + podSpec.InitContainers[i].SecurityContext.ProcMount = &defaultProcMount + } } } } @@ -473,7 +483,7 @@ func runtimeClassInUse(podSpec *api.PodSpec) bool { return false } -// procMountInUse returns true if the pod spec is non-nil and has a SecurityContext's ProcMount field set +// procMountInUse returns true if the pod spec is non-nil and has a SecurityContext's ProcMount field set to a non-default value func procMountInUse(podSpec *api.PodSpec) bool { if podSpec == nil { return false diff --git a/pkg/api/pod/util_test.go b/pkg/api/pod/util_test.go index aff0726cd7267..7b0b3e3474a0f 100644 --- a/pkg/api/pod/util_test.go +++ b/pkg/api/pod/util_test.go @@ -616,7 +616,7 @@ func TestDropProcMount(t *testing.T) { }, } } - podWithoutProcMount := func() *api.Pod { + podWithDefaultProcMount := func() *api.Pod { return &api.Pod{ Spec: api.PodSpec{ RestartPolicy: api.RestartPolicyNever, @@ -625,6 +625,15 @@ func TestDropProcMount(t *testing.T) { }, } } + podWithoutProcMount := func() *api.Pod { + return &api.Pod{ + Spec: api.PodSpec{ + RestartPolicy: api.RestartPolicyNever, + Containers: []api.Container{{Name: "container1", Image: "testimage", SecurityContext: &api.SecurityContext{ProcMount: nil}}}, + InitContainers: []api.Container{{Name: "container1", Image: "testimage", SecurityContext: &api.SecurityContext{ProcMount: nil}}}, + }, + } + } podInfo := []struct { description string @@ -636,6 +645,11 @@ func TestDropProcMount(t *testing.T) { hasProcMount: true, pod: podWithProcMount, }, + { + description: "has default ProcMount", + hasProcMount: false, + pod: podWithDefaultProcMount, + }, { description: "does not have ProcMount", hasProcMount: false, @@ -683,8 +697,8 @@ func TestDropProcMount(t *testing.T) { t.Errorf("new pod was not changed") } // new pod should not have ProcMount - if !reflect.DeepEqual(newPod, podWithoutProcMount()) { - t.Errorf("new pod had ProcMount: %v", diff.ObjectReflectDiff(newPod, podWithoutProcMount())) + if procMountInUse(&newPod.Spec) { + t.Errorf("new pod had ProcMount: %#v", &newPod.Spec) } default: // new pod should not need to be changed diff --git a/pkg/apis/apps/v1/zz_generated.defaults.go b/pkg/apis/apps/v1/zz_generated.defaults.go index 1c63c1917c4a5..4b541a3b15a5e 100644 --- a/pkg/apis/apps/v1/zz_generated.defaults.go +++ b/pkg/apis/apps/v1/zz_generated.defaults.go @@ -136,9 +136,6 @@ func SetObjectDefaults_DaemonSet(in *v1.DaemonSet) { } } } - if a.SecurityContext != nil { - corev1.SetDefaults_SecurityContext(a.SecurityContext) - } } for i := range in.Spec.Template.Spec.Containers { a := &in.Spec.Template.Spec.Containers[i] @@ -181,9 +178,6 @@ func SetObjectDefaults_DaemonSet(in *v1.DaemonSet) { } } } - if a.SecurityContext != nil { - corev1.SetDefaults_SecurityContext(a.SecurityContext) - } } } @@ -289,9 +283,6 @@ func SetObjectDefaults_Deployment(in *v1.Deployment) { } } } - if a.SecurityContext != nil { - corev1.SetDefaults_SecurityContext(a.SecurityContext) - } } for i := range in.Spec.Template.Spec.Containers { a := &in.Spec.Template.Spec.Containers[i] @@ -334,9 +325,6 @@ func SetObjectDefaults_Deployment(in *v1.Deployment) { } } } - if a.SecurityContext != nil { - corev1.SetDefaults_SecurityContext(a.SecurityContext) - } } } @@ -442,9 +430,6 @@ func SetObjectDefaults_ReplicaSet(in *v1.ReplicaSet) { } } } - if a.SecurityContext != nil { - corev1.SetDefaults_SecurityContext(a.SecurityContext) - } } for i := range in.Spec.Template.Spec.Containers { a := &in.Spec.Template.Spec.Containers[i] @@ -487,9 +472,6 @@ func SetObjectDefaults_ReplicaSet(in *v1.ReplicaSet) { } } } - if a.SecurityContext != nil { - corev1.SetDefaults_SecurityContext(a.SecurityContext) - } } } @@ -595,9 +577,6 @@ func SetObjectDefaults_StatefulSet(in *v1.StatefulSet) { } } } - if a.SecurityContext != nil { - corev1.SetDefaults_SecurityContext(a.SecurityContext) - } } for i := range in.Spec.Template.Spec.Containers { a := &in.Spec.Template.Spec.Containers[i] @@ -640,9 +619,6 @@ func SetObjectDefaults_StatefulSet(in *v1.StatefulSet) { } } } - if a.SecurityContext != nil { - corev1.SetDefaults_SecurityContext(a.SecurityContext) - } } for i := range in.Spec.VolumeClaimTemplates { a := &in.Spec.VolumeClaimTemplates[i] diff --git a/pkg/apis/apps/v1beta1/zz_generated.defaults.go b/pkg/apis/apps/v1beta1/zz_generated.defaults.go index 4a31e2e169906..656f61edc9972 100644 --- a/pkg/apis/apps/v1beta1/zz_generated.defaults.go +++ b/pkg/apis/apps/v1beta1/zz_generated.defaults.go @@ -132,9 +132,6 @@ func SetObjectDefaults_Deployment(in *v1beta1.Deployment) { } } } - if a.SecurityContext != nil { - v1.SetDefaults_SecurityContext(a.SecurityContext) - } } for i := range in.Spec.Template.Spec.Containers { a := &in.Spec.Template.Spec.Containers[i] @@ -177,9 +174,6 @@ func SetObjectDefaults_Deployment(in *v1beta1.Deployment) { } } } - if a.SecurityContext != nil { - v1.SetDefaults_SecurityContext(a.SecurityContext) - } } } @@ -285,9 +279,6 @@ func SetObjectDefaults_StatefulSet(in *v1beta1.StatefulSet) { } } } - if a.SecurityContext != nil { - v1.SetDefaults_SecurityContext(a.SecurityContext) - } } for i := range in.Spec.Template.Spec.Containers { a := &in.Spec.Template.Spec.Containers[i] @@ -330,9 +321,6 @@ func SetObjectDefaults_StatefulSet(in *v1beta1.StatefulSet) { } } } - if a.SecurityContext != nil { - v1.SetDefaults_SecurityContext(a.SecurityContext) - } } for i := range in.Spec.VolumeClaimTemplates { a := &in.Spec.VolumeClaimTemplates[i] diff --git a/pkg/apis/apps/v1beta2/zz_generated.defaults.go b/pkg/apis/apps/v1beta2/zz_generated.defaults.go index 847a56b0d08c6..713bcaa7819ed 100644 --- a/pkg/apis/apps/v1beta2/zz_generated.defaults.go +++ b/pkg/apis/apps/v1beta2/zz_generated.defaults.go @@ -136,9 +136,6 @@ func SetObjectDefaults_DaemonSet(in *v1beta2.DaemonSet) { } } } - if a.SecurityContext != nil { - v1.SetDefaults_SecurityContext(a.SecurityContext) - } } for i := range in.Spec.Template.Spec.Containers { a := &in.Spec.Template.Spec.Containers[i] @@ -181,9 +178,6 @@ func SetObjectDefaults_DaemonSet(in *v1beta2.DaemonSet) { } } } - if a.SecurityContext != nil { - v1.SetDefaults_SecurityContext(a.SecurityContext) - } } } @@ -289,9 +283,6 @@ func SetObjectDefaults_Deployment(in *v1beta2.Deployment) { } } } - if a.SecurityContext != nil { - v1.SetDefaults_SecurityContext(a.SecurityContext) - } } for i := range in.Spec.Template.Spec.Containers { a := &in.Spec.Template.Spec.Containers[i] @@ -334,9 +325,6 @@ func SetObjectDefaults_Deployment(in *v1beta2.Deployment) { } } } - if a.SecurityContext != nil { - v1.SetDefaults_SecurityContext(a.SecurityContext) - } } } @@ -442,9 +430,6 @@ func SetObjectDefaults_ReplicaSet(in *v1beta2.ReplicaSet) { } } } - if a.SecurityContext != nil { - v1.SetDefaults_SecurityContext(a.SecurityContext) - } } for i := range in.Spec.Template.Spec.Containers { a := &in.Spec.Template.Spec.Containers[i] @@ -487,9 +472,6 @@ func SetObjectDefaults_ReplicaSet(in *v1beta2.ReplicaSet) { } } } - if a.SecurityContext != nil { - v1.SetDefaults_SecurityContext(a.SecurityContext) - } } } @@ -595,9 +577,6 @@ func SetObjectDefaults_StatefulSet(in *v1beta2.StatefulSet) { } } } - if a.SecurityContext != nil { - v1.SetDefaults_SecurityContext(a.SecurityContext) - } } for i := range in.Spec.Template.Spec.Containers { a := &in.Spec.Template.Spec.Containers[i] @@ -640,9 +619,6 @@ func SetObjectDefaults_StatefulSet(in *v1beta2.StatefulSet) { } } } - if a.SecurityContext != nil { - v1.SetDefaults_SecurityContext(a.SecurityContext) - } } for i := range in.Spec.VolumeClaimTemplates { a := &in.Spec.VolumeClaimTemplates[i] diff --git a/pkg/apis/batch/v1/zz_generated.defaults.go b/pkg/apis/batch/v1/zz_generated.defaults.go index 8c0f02b469ced..201e3b2285257 100644 --- a/pkg/apis/batch/v1/zz_generated.defaults.go +++ b/pkg/apis/batch/v1/zz_generated.defaults.go @@ -130,9 +130,6 @@ func SetObjectDefaults_Job(in *v1.Job) { } } } - if a.SecurityContext != nil { - corev1.SetDefaults_SecurityContext(a.SecurityContext) - } } for i := range in.Spec.Template.Spec.Containers { a := &in.Spec.Template.Spec.Containers[i] @@ -175,9 +172,6 @@ func SetObjectDefaults_Job(in *v1.Job) { } } } - if a.SecurityContext != nil { - corev1.SetDefaults_SecurityContext(a.SecurityContext) - } } } diff --git a/pkg/apis/batch/v1beta1/zz_generated.defaults.go b/pkg/apis/batch/v1beta1/zz_generated.defaults.go index a9a1a5d067564..d90031c32aafe 100644 --- a/pkg/apis/batch/v1beta1/zz_generated.defaults.go +++ b/pkg/apis/batch/v1beta1/zz_generated.defaults.go @@ -131,9 +131,6 @@ func SetObjectDefaults_CronJob(in *v1beta1.CronJob) { } } } - if a.SecurityContext != nil { - v1.SetDefaults_SecurityContext(a.SecurityContext) - } } for i := range in.Spec.JobTemplate.Spec.Template.Spec.Containers { a := &in.Spec.JobTemplate.Spec.Template.Spec.Containers[i] @@ -176,9 +173,6 @@ func SetObjectDefaults_CronJob(in *v1beta1.CronJob) { } } } - if a.SecurityContext != nil { - v1.SetDefaults_SecurityContext(a.SecurityContext) - } } } @@ -283,9 +277,6 @@ func SetObjectDefaults_JobTemplate(in *v1beta1.JobTemplate) { } } } - if a.SecurityContext != nil { - v1.SetDefaults_SecurityContext(a.SecurityContext) - } } for i := range in.Template.Spec.Template.Spec.Containers { a := &in.Template.Spec.Template.Spec.Containers[i] @@ -328,8 +319,5 @@ func SetObjectDefaults_JobTemplate(in *v1beta1.JobTemplate) { } } } - if a.SecurityContext != nil { - v1.SetDefaults_SecurityContext(a.SecurityContext) - } } } diff --git a/pkg/apis/batch/v2alpha1/zz_generated.defaults.go b/pkg/apis/batch/v2alpha1/zz_generated.defaults.go index 8156eaac3f2e9..367c56b15ed3e 100644 --- a/pkg/apis/batch/v2alpha1/zz_generated.defaults.go +++ b/pkg/apis/batch/v2alpha1/zz_generated.defaults.go @@ -131,9 +131,6 @@ func SetObjectDefaults_CronJob(in *v2alpha1.CronJob) { } } } - if a.SecurityContext != nil { - v1.SetDefaults_SecurityContext(a.SecurityContext) - } } for i := range in.Spec.JobTemplate.Spec.Template.Spec.Containers { a := &in.Spec.JobTemplate.Spec.Template.Spec.Containers[i] @@ -176,9 +173,6 @@ func SetObjectDefaults_CronJob(in *v2alpha1.CronJob) { } } } - if a.SecurityContext != nil { - v1.SetDefaults_SecurityContext(a.SecurityContext) - } } } @@ -283,9 +277,6 @@ func SetObjectDefaults_JobTemplate(in *v2alpha1.JobTemplate) { } } } - if a.SecurityContext != nil { - v1.SetDefaults_SecurityContext(a.SecurityContext) - } } for i := range in.Template.Spec.Template.Spec.Containers { a := &in.Template.Spec.Template.Spec.Containers[i] @@ -328,8 +319,5 @@ func SetObjectDefaults_JobTemplate(in *v2alpha1.JobTemplate) { } } } - if a.SecurityContext != nil { - v1.SetDefaults_SecurityContext(a.SecurityContext) - } } } diff --git a/pkg/apis/core/fuzzer/fuzzer.go b/pkg/apis/core/fuzzer/fuzzer.go index 2a6b66e456e59..391226ba00780 100644 --- a/pkg/apis/core/fuzzer/fuzzer.go +++ b/pkg/apis/core/fuzzer/fuzzer.go @@ -354,10 +354,6 @@ var Funcs = func(codecs runtimeserializer.CodecFactory) []interface{} { c.Fuzz(&sc.Capabilities.Add) c.Fuzz(&sc.Capabilities.Drop) } - if sc.ProcMount == nil { - defProcMount := core.DefaultProcMount - sc.ProcMount = &defProcMount - } }, func(s *core.Secret, c fuzz.Continue) { c.FuzzNoCustom(s) // fuzz self without calling this function again diff --git a/pkg/apis/core/v1/defaults.go b/pkg/apis/core/v1/defaults.go index d06356a89dcbd..4f681a5a7748c 100644 --- a/pkg/apis/core/v1/defaults.go +++ b/pkg/apis/core/v1/defaults.go @@ -421,10 +421,3 @@ func SetDefaults_HostPathVolumeSource(obj *v1.HostPathVolumeSource) { obj.Type = &typeVol } } - -func SetDefaults_SecurityContext(obj *v1.SecurityContext) { - if obj.ProcMount == nil { - defProcMount := v1.DefaultProcMount - obj.ProcMount = &defProcMount - } -} diff --git a/pkg/apis/core/v1/zz_generated.defaults.go b/pkg/apis/core/v1/zz_generated.defaults.go index 0ea5e0fae0503..00e0b384aa863 100644 --- a/pkg/apis/core/v1/zz_generated.defaults.go +++ b/pkg/apis/core/v1/zz_generated.defaults.go @@ -263,9 +263,6 @@ func SetObjectDefaults_Pod(in *v1.Pod) { } } } - if a.SecurityContext != nil { - SetDefaults_SecurityContext(a.SecurityContext) - } } for i := range in.Spec.Containers { a := &in.Spec.Containers[i] @@ -308,9 +305,6 @@ func SetObjectDefaults_Pod(in *v1.Pod) { } } } - if a.SecurityContext != nil { - SetDefaults_SecurityContext(a.SecurityContext) - } } } @@ -415,9 +409,6 @@ func SetObjectDefaults_PodTemplate(in *v1.PodTemplate) { } } } - if a.SecurityContext != nil { - SetDefaults_SecurityContext(a.SecurityContext) - } } for i := range in.Template.Spec.Containers { a := &in.Template.Spec.Containers[i] @@ -460,9 +451,6 @@ func SetObjectDefaults_PodTemplate(in *v1.PodTemplate) { } } } - if a.SecurityContext != nil { - SetDefaults_SecurityContext(a.SecurityContext) - } } } @@ -569,9 +557,6 @@ func SetObjectDefaults_ReplicationController(in *v1.ReplicationController) { } } } - if a.SecurityContext != nil { - SetDefaults_SecurityContext(a.SecurityContext) - } } for i := range in.Spec.Template.Spec.Containers { a := &in.Spec.Template.Spec.Containers[i] @@ -614,9 +599,6 @@ func SetObjectDefaults_ReplicationController(in *v1.ReplicationController) { } } } - if a.SecurityContext != nil { - SetDefaults_SecurityContext(a.SecurityContext) - } } } } diff --git a/pkg/apis/extensions/v1beta1/zz_generated.defaults.go b/pkg/apis/extensions/v1beta1/zz_generated.defaults.go index ce23c6b801c23..0fa4c321c292a 100644 --- a/pkg/apis/extensions/v1beta1/zz_generated.defaults.go +++ b/pkg/apis/extensions/v1beta1/zz_generated.defaults.go @@ -138,9 +138,6 @@ func SetObjectDefaults_DaemonSet(in *v1beta1.DaemonSet) { } } } - if a.SecurityContext != nil { - v1.SetDefaults_SecurityContext(a.SecurityContext) - } } for i := range in.Spec.Template.Spec.Containers { a := &in.Spec.Template.Spec.Containers[i] @@ -183,9 +180,6 @@ func SetObjectDefaults_DaemonSet(in *v1beta1.DaemonSet) { } } } - if a.SecurityContext != nil { - v1.SetDefaults_SecurityContext(a.SecurityContext) - } } } @@ -291,9 +285,6 @@ func SetObjectDefaults_Deployment(in *v1beta1.Deployment) { } } } - if a.SecurityContext != nil { - v1.SetDefaults_SecurityContext(a.SecurityContext) - } } for i := range in.Spec.Template.Spec.Containers { a := &in.Spec.Template.Spec.Containers[i] @@ -336,9 +327,6 @@ func SetObjectDefaults_Deployment(in *v1beta1.Deployment) { } } } - if a.SecurityContext != nil { - v1.SetDefaults_SecurityContext(a.SecurityContext) - } } } @@ -466,9 +454,6 @@ func SetObjectDefaults_ReplicaSet(in *v1beta1.ReplicaSet) { } } } - if a.SecurityContext != nil { - v1.SetDefaults_SecurityContext(a.SecurityContext) - } } for i := range in.Spec.Template.Spec.Containers { a := &in.Spec.Template.Spec.Containers[i] @@ -511,9 +496,6 @@ func SetObjectDefaults_ReplicaSet(in *v1beta1.ReplicaSet) { } } } - if a.SecurityContext != nil { - v1.SetDefaults_SecurityContext(a.SecurityContext) - } } } diff --git a/pkg/cloudprovider/providers/aws/aws.go b/pkg/cloudprovider/providers/aws/aws.go index c51e606175605..624600714f4f2 100644 --- a/pkg/cloudprovider/providers/aws/aws.go +++ b/pkg/cloudprovider/providers/aws/aws.go @@ -1351,14 +1351,17 @@ func (c *Cloud) NodeAddresses(ctx context.Context, name types.NodeName) ([]v1.No addresses = append(addresses, v1.NodeAddress{Type: v1.NodeExternalIP, Address: externalIP}) } - internalDNS, err := c.metadata.GetMetadata("local-hostname") - if err != nil || len(internalDNS) == 0 { + localHostname, err := c.metadata.GetMetadata("local-hostname") + if err != nil || len(localHostname) == 0 { //TODO: It would be nice to be able to determine the reason for the failure, // but the AWS client masks all failures with the same error description. klog.V(4).Info("Could not determine private DNS from AWS metadata.") } else { - addresses = append(addresses, v1.NodeAddress{Type: v1.NodeInternalDNS, Address: internalDNS}) - addresses = append(addresses, v1.NodeAddress{Type: v1.NodeHostName, Address: internalDNS}) + hostname, internalDNS := parseMetadataLocalHostname(localHostname) + addresses = append(addresses, v1.NodeAddress{Type: v1.NodeHostName, Address: hostname}) + for _, d := range internalDNS { + addresses = append(addresses, v1.NodeAddress{Type: v1.NodeInternalDNS, Address: d}) + } } externalDNS, err := c.metadata.GetMetadata("public-hostname") @@ -1380,6 +1383,26 @@ func (c *Cloud) NodeAddresses(ctx context.Context, name types.NodeName) ([]v1.No return extractNodeAddresses(instance) } +// parseMetadataLocalHostname parses the output of "local-hostname" metadata. +// If a DHCP option set is configured for a VPC and it has multiple domain names, GetMetadata +// returns a string containing first the hostname followed by additional domain names, +// space-separated. For example, if the DHCP option set has: +// domain-name = us-west-2.compute.internal a.a b.b c.c d.d; +// $ curl http://169.254.169.254/latest/meta-data/local-hostname +// ip-192-168-111-51.us-west-2.compute.internal a.a b.b c.c d.d +func parseMetadataLocalHostname(metadata string) (string, []string) { + localHostnames := strings.Fields(metadata) + hostname := localHostnames[0] + internalDNS := []string{hostname} + + privateAddress := strings.Split(hostname, ".")[0] + for _, h := range localHostnames[1:] { + internalDNSAddress := privateAddress + "." + h + internalDNS = append(internalDNS, internalDNSAddress) + } + return hostname, internalDNS +} + // extractNodeAddresses maps the instance information from EC2 to an array of NodeAddresses func extractNodeAddresses(instance *ec2.Instance) ([]v1.NodeAddress, error) { // Not clear if the order matters here, but we might as well indicate a sensible preference order diff --git a/pkg/cloudprovider/providers/aws/aws_test.go b/pkg/cloudprovider/providers/aws/aws_test.go index d37630e837c07..84db7aeba769f 100644 --- a/pkg/cloudprovider/providers/aws/aws_test.go +++ b/pkg/cloudprovider/providers/aws/aws_test.go @@ -681,6 +681,41 @@ func TestNodeAddressesWithMetadata(t *testing.T) { testHasNodeAddress(t, addrs, v1.NodeExternalIP, "2.3.4.5") } +func TestParseMetadataLocalHostname(t *testing.T) { + tests := []struct { + name string + metadata string + hostname string + internalDNS []string + }{ + { + "single hostname", + "ip-172-31-16-168.us-west-2.compute.internal", + "ip-172-31-16-168.us-west-2.compute.internal", + []string{"ip-172-31-16-168.us-west-2.compute.internal"}, + }, + { + "dhcp options set with three additional domain names", + "ip-172-31-16-168.us-west-2.compute.internal example.com example.ca example.org", + "ip-172-31-16-168.us-west-2.compute.internal", + []string{"ip-172-31-16-168.us-west-2.compute.internal", "ip-172-31-16-168.example.com", "ip-172-31-16-168.example.ca", "ip-172-31-16-168.example.org"}, + }, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + hostname, internalDNS := parseMetadataLocalHostname(test.metadata) + if hostname != test.hostname { + t.Errorf("got hostname %v, expected %v", hostname, test.hostname) + } + for i, v := range internalDNS { + if v != test.internalDNS[i] { + t.Errorf("got an internalDNS %v, expected %v", v, test.internalDNS[i]) + } + } + }) + } +} + func TestGetRegion(t *testing.T) { aws := mockAvailabilityZone("us-west-2e") zones, ok := aws.Zones() diff --git a/pkg/cloudprovider/providers/azure/azure_controller_standard.go b/pkg/cloudprovider/providers/azure/azure_controller_standard.go index 8e10fc750802e..71f26265b04f1 100644 --- a/pkg/cloudprovider/providers/azure/azure_controller_standard.go +++ b/pkg/cloudprovider/providers/azure/azure_controller_standard.go @@ -17,7 +17,6 @@ limitations under the License. package azure import ( - "fmt" "net/http" "strings" @@ -132,7 +131,8 @@ func (as *availabilitySet) DetachDisk(diskName, diskURI string, nodeName types.N } if !bFoundDisk { - return nil, fmt.Errorf("detach azure disk failure, disk %s not found, diskURI: %s", diskName, diskURI) + // only log here, next action is to update VM status with original meta data + klog.Errorf("detach azure disk: disk %s not found, diskURI: %s", diskName, diskURI) } newVM := compute.VirtualMachine{ diff --git a/pkg/cloudprovider/providers/azure/azure_controller_vmss.go b/pkg/cloudprovider/providers/azure/azure_controller_vmss.go index db914c2c9a045..8c6b8f34638bf 100644 --- a/pkg/cloudprovider/providers/azure/azure_controller_vmss.go +++ b/pkg/cloudprovider/providers/azure/azure_controller_vmss.go @@ -17,7 +17,6 @@ limitations under the License. package azure import ( - "fmt" "net/http" "strings" @@ -136,7 +135,8 @@ func (ss *scaleSet) DetachDisk(diskName, diskURI string, nodeName types.NodeName } if !bFoundDisk { - return nil, fmt.Errorf("detach azure disk failure, disk %s not found, diskURI: %s", diskName, diskURI) + // only log here, next action is to update VM status with original meta data + klog.Errorf("detach azure disk: disk %s not found, diskURI: %s", diskName, diskURI) } newVM := compute.VirtualMachineScaleSetVM{ diff --git a/pkg/cloudprovider/providers/azure/azure_loadbalancer.go b/pkg/cloudprovider/providers/azure/azure_loadbalancer.go index ccb9dedd13ae0..e632775400e32 100644 --- a/pkg/cloudprovider/providers/azure/azure_loadbalancer.go +++ b/pkg/cloudprovider/providers/azure/azure_loadbalancer.go @@ -1542,7 +1542,10 @@ func findSecurityRule(rules []network.SecurityRule, rule network.SecurityRule) b func (az *Cloud) getPublicIPAddressResourceGroup(service *v1.Service) string { if resourceGroup, found := service.Annotations[ServiceAnnotationLoadBalancerResourceGroup]; found { - return resourceGroup + resourceGroupName := strings.TrimSpace(resourceGroup) + if len(resourceGroupName) > 0 { + return resourceGroupName + } } return az.ResourceGroup diff --git a/pkg/cloudprovider/providers/azure/azure_loadbalancer_test.go b/pkg/cloudprovider/providers/azure/azure_loadbalancer_test.go index d7fe868ce3078..d949092cc3b69 100644 --- a/pkg/cloudprovider/providers/azure/azure_loadbalancer_test.go +++ b/pkg/cloudprovider/providers/azure/azure_loadbalancer_test.go @@ -368,3 +368,35 @@ func TestEnsureLoadBalancerDeleted(t *testing.T) { assert.Equal(t, len(result), 0, "TestCase[%d]: %s", i, c.desc) } } + +func TestGetPublicIPAddressResourceGroup(t *testing.T) { + az := getTestCloud() + + for i, c := range []struct { + desc string + annotations map[string]string + expected string + }{ + { + desc: "no annotation", + expected: "rg", + }, + { + desc: "annoation with empty string resource group", + annotations: map[string]string{ServiceAnnotationLoadBalancerResourceGroup: ""}, + expected: "rg", + }, + { + desc: "annoation with non-empty resource group ", + annotations: map[string]string{ServiceAnnotationLoadBalancerResourceGroup: "rg2"}, + expected: "rg2", + }, + } { + t.Run(c.desc, func(t *testing.T) { + s := &v1.Service{} + s.Annotations = c.annotations + real := az.getPublicIPAddressResourceGroup(s) + assert.Equal(t, c.expected, real, "TestCase[%d]: %s", i, c.desc) + }) + } +} diff --git a/pkg/cloudprovider/providers/vsphere/BUILD b/pkg/cloudprovider/providers/vsphere/BUILD index 45e49e2a73b05..a1a173ab35a0f 100644 --- a/pkg/cloudprovider/providers/vsphere/BUILD +++ b/pkg/cloudprovider/providers/vsphere/BUILD @@ -32,6 +32,7 @@ go_library( "//staging/src/k8s.io/cloud-provider:go_default_library", "//staging/src/k8s.io/cloud-provider/node/helpers:go_default_library", "//staging/src/k8s.io/cloud-provider/volume/helpers:go_default_library", + "//vendor/github.com/vmware/govmomi/find:go_default_library", "//vendor/github.com/vmware/govmomi/object:go_default_library", "//vendor/github.com/vmware/govmomi/property:go_default_library", "//vendor/github.com/vmware/govmomi/vapi/rest:go_default_library", diff --git a/pkg/cloudprovider/providers/vsphere/nodemanager.go b/pkg/cloudprovider/providers/vsphere/nodemanager.go index f245e336f65c3..ceb23e409492d 100644 --- a/pkg/cloudprovider/providers/vsphere/nodemanager.go +++ b/pkg/cloudprovider/providers/vsphere/nodemanager.go @@ -39,6 +39,11 @@ type NodeInfo struct { zone *cloudprovider.Zone } +func (n NodeInfo) String() string { + return fmt.Sprintf("{datacenter: %v, vm: %v, vcServer: %s, vmUUID: %s, zone: %v}", + *n.dataCenter, n.vm.Reference(), n.vcServer, n.vmUUID, *n.zone) +} + type NodeManager struct { // TODO: replace map with concurrent map when k8s supports go v1.9 diff --git a/pkg/cloudprovider/providers/vsphere/vclib/connection.go b/pkg/cloudprovider/providers/vsphere/vclib/connection.go index f0c7f5cb74b8d..44cc163545ad1 100644 --- a/pkg/cloudprovider/providers/vsphere/vclib/connection.go +++ b/pkg/cloudprovider/providers/vsphere/vclib/connection.go @@ -111,6 +111,7 @@ func (connection *VSphereConnection) Signer(ctx context.Context, client *vim25.C req := sts.TokenRequest{ Certificate: &cert, + Delegatable: true, } signer, err := tokens.Issue(ctx, req) diff --git a/pkg/cloudprovider/providers/vsphere/vclib/constants.go b/pkg/cloudprovider/providers/vsphere/vclib/constants.go index a8a4e5f420ea9..2209898c6f6a9 100644 --- a/pkg/cloudprovider/providers/vsphere/vclib/constants.go +++ b/pkg/cloudprovider/providers/vsphere/vclib/constants.go @@ -52,6 +52,7 @@ const ( DatacenterType = "Datacenter" ClusterComputeResourceType = "ClusterComputeResource" HostSystemType = "HostSystem" + NameProperty = "name" ) // Test Constants diff --git a/pkg/cloudprovider/providers/vsphere/vclib/datacenter.go b/pkg/cloudprovider/providers/vsphere/vclib/datacenter.go index bbd60cc620a31..c757be1d5d86f 100644 --- a/pkg/cloudprovider/providers/vsphere/vclib/datacenter.go +++ b/pkg/cloudprovider/providers/vsphere/vclib/datacenter.go @@ -187,6 +187,28 @@ func (dc *Datacenter) GetDatastoreByName(ctx context.Context, name string) (*Dat return &datastore, nil } +// GetDatastoreInfoByName gets the Datastore object for the given datastore name +func (dc *Datacenter) GetDatastoreInfoByName(ctx context.Context, name string) (*DatastoreInfo, error) { + finder := getFinder(dc) + ds, err := finder.Datastore(ctx, name) + if err != nil { + klog.Errorf("Failed while searching for datastore: %s. err: %+v", name, err) + return nil, err + } + datastore := Datastore{ds, dc} + var dsMo mo.Datastore + pc := property.DefaultCollector(dc.Client()) + properties := []string{DatastoreInfoProperty} + err = pc.RetrieveOne(ctx, ds.Reference(), properties, &dsMo) + if err != nil { + klog.Errorf("Failed to get Datastore managed objects from datastore reference."+ + " dsRef: %+v, err: %+v", ds.Reference(), err) + return nil, err + } + klog.V(9).Infof("Result dsMo: %+v", dsMo) + return &DatastoreInfo{Datastore: &datastore, Info: dsMo.Info.GetDatastoreInfo()}, nil +} + // GetResourcePool gets the resource pool for the given path func (dc *Datacenter) GetResourcePool(ctx context.Context, resourcePoolPath string) (*object.ResourcePool, error) { finder := getFinder(dc) diff --git a/pkg/cloudprovider/providers/vsphere/vclib/datastore.go b/pkg/cloudprovider/providers/vsphere/vclib/datastore.go index 859e56239f5c9..00fc1f42e2606 100644 --- a/pkg/cloudprovider/providers/vsphere/vclib/datastore.go +++ b/pkg/cloudprovider/providers/vsphere/vclib/datastore.go @@ -96,8 +96,14 @@ func (ds *Datastore) GetDatastoreHostMounts(ctx context.Context) ([]types.Manage return nil, err } hosts := make([]types.ManagedObjectReference, len(dsMo.Host)) - for _, dsHostMount := range dsMo.Host { - hosts = append(hosts, dsHostMount.Key) + for i, dsHostMount := range dsMo.Host { + hosts[i] = dsHostMount.Key } return hosts, nil } + +// Exists returns whether the given file exists in this datastore +func (ds *Datastore) Exists(ctx context.Context, file string) bool { + _, err := ds.Datastore.Stat(ctx, file) + return err == nil +} diff --git a/pkg/cloudprovider/providers/vsphere/vclib/diskmanagers/vmdm.go b/pkg/cloudprovider/providers/vsphere/vclib/diskmanagers/vmdm.go index 637ac514bfd65..b48e303f3652d 100644 --- a/pkg/cloudprovider/providers/vsphere/vclib/diskmanagers/vmdm.go +++ b/pkg/cloudprovider/providers/vsphere/vclib/diskmanagers/vmdm.go @@ -211,7 +211,7 @@ func (vmdisk vmDiskManager) createDummyVM(ctx context.Context, datacenter *vclib } // CleanUpDummyVMs deletes stale dummyVM's -func CleanUpDummyVMs(ctx context.Context, folder *vclib.Folder, dc *vclib.Datacenter) error { +func CleanUpDummyVMs(ctx context.Context, folder *vclib.Folder) error { vmList, err := folder.GetVirtualMachines(ctx) if err != nil { klog.V(4).Infof("Failed to get virtual machines in the kubernetes cluster: %s, err: %+v", folder.InventoryPath, err) @@ -230,7 +230,7 @@ func CleanUpDummyVMs(ctx context.Context, folder *vclib.Folder, dc *vclib.Datace continue } if strings.HasPrefix(vmName, vclib.DummyVMPrefixName) { - vmObj := vclib.VirtualMachine{VirtualMachine: object.NewVirtualMachine(dc.Client(), vm.Reference()), Datacenter: dc} + vmObj := vclib.VirtualMachine{VirtualMachine: object.NewVirtualMachine(folder.Client(), vm.Reference())} dummyVMList = append(dummyVMList, &vmObj) } } diff --git a/pkg/cloudprovider/providers/vsphere/vclib/pbm.go b/pkg/cloudprovider/providers/vsphere/vclib/pbm.go index 8070f20042398..05eff9eb3c7b5 100644 --- a/pkg/cloudprovider/providers/vsphere/vclib/pbm.go +++ b/pkg/cloudprovider/providers/vsphere/vclib/pbm.go @@ -85,7 +85,7 @@ func (pbmClient *PbmClient) IsDatastoreCompatible(ctx context.Context, storagePo // GetCompatibleDatastores filters and returns compatible list of datastores for given storage policy id // For Non Compatible Datastores, fault message with the Datastore Name is also returned -func (pbmClient *PbmClient) GetCompatibleDatastores(ctx context.Context, dc *Datacenter, storagePolicyID string, datastores []*DatastoreInfo) ([]*DatastoreInfo, string, error) { +func (pbmClient *PbmClient) GetCompatibleDatastores(ctx context.Context, storagePolicyID string, datastores []*DatastoreInfo) ([]*DatastoreInfo, string, error) { var ( dsMorNameMap = getDsMorNameMap(ctx, datastores) localizedMessagesForNotCompatibleDatastores = "" diff --git a/pkg/cloudprovider/providers/vsphere/vclib/virtualmachine.go b/pkg/cloudprovider/providers/vsphere/vclib/virtualmachine.go index 878822c32dbb9..07f1f28144e9c 100644 --- a/pkg/cloudprovider/providers/vsphere/vclib/virtualmachine.go +++ b/pkg/cloudprovider/providers/vsphere/vclib/virtualmachine.go @@ -235,7 +235,7 @@ func (vm *VirtualMachine) GetAllAccessibleDatastores(ctx context.Context) ([]*Da var dsMoList []mo.Datastore pc := property.DefaultCollector(vm.Client()) - properties := []string{DatastoreInfoProperty} + properties := []string{DatastoreInfoProperty, NameProperty} err = pc.Retrieve(ctx, dsRefList, properties, &dsMoList) if err != nil { klog.Errorf("Failed to get Datastore managed objects from datastore objects."+ @@ -243,12 +243,19 @@ func (vm *VirtualMachine) GetAllAccessibleDatastores(ctx context.Context) ([]*Da return nil, err } klog.V(9).Infof("Result dsMoList: %+v", dsMoList) + finder := getFinder(vm.Datacenter) var dsObjList []*DatastoreInfo for _, dsMo := range dsMoList { + // use the finder so that InventoryPath is set correctly in ds + ds, err := finder.Datastore(ctx, dsMo.Name) + if err != nil { + klog.Errorf("Failed finding datastore: %s. err: %+v", dsMo.Name, err) + return nil, err + } + datastore := Datastore{ds, vm.Datacenter} dsObjList = append(dsObjList, &DatastoreInfo{ - &Datastore{object.NewDatastore(vm.Client(), dsMo.Reference()), - vm.Datacenter}, + &datastore, dsMo.Info.GetDatastoreInfo()}) } return dsObjList, nil diff --git a/pkg/cloudprovider/providers/vsphere/vsphere.go b/pkg/cloudprovider/providers/vsphere/vsphere.go index ff72c36d3c79f..54ab036dc4913 100644 --- a/pkg/cloudprovider/providers/vsphere/vsphere.go +++ b/pkg/cloudprovider/providers/vsphere/vsphere.go @@ -1164,15 +1164,9 @@ func (vs *VSphere) DisksAreAttached(nodeVolumes map[k8stypes.NodeName][]string) func (vs *VSphere) CreateVolume(volumeOptions *vclib.VolumeOptions) (canonicalVolumePath string, err error) { klog.V(1).Infof("Starting to create a vSphere volume with volumeOptions: %+v", volumeOptions) createVolumeInternal := func(volumeOptions *vclib.VolumeOptions) (canonicalVolumePath string, err error) { - var datastore string + var datastoreInfo *vclib.DatastoreInfo var dsList []*vclib.DatastoreInfo - // If datastore not specified, then use default datastore - if volumeOptions.Datastore == "" { - datastore = vs.cfg.Workspace.DefaultDatastore - } else { - datastore = volumeOptions.Datastore - } - datastore = strings.TrimSpace(datastore) + // Create context ctx, cancel := context.WithCancel(context.Background()) defer cancel() @@ -1180,16 +1174,29 @@ func (vs *VSphere) CreateVolume(volumeOptions *vclib.VolumeOptions) (canonicalVo if err != nil { return "", err } - dc, err := vclib.GetDatacenter(ctx, vsi.conn, vs.cfg.Workspace.Datacenter) + // If datastore not specified, then use default datastore + datastoreName := strings.TrimSpace(volumeOptions.Datastore) + if datastoreName == "" { + datastoreName = strings.TrimSpace(vs.cfg.Workspace.DefaultDatastore) + } + // The given datastoreName may be present in more than one datacenter + candidateDatastoreInfos, err := vs.FindDatastoreByName(ctx, datastoreName) if err != nil { return "", err } + // Each of the datastores found is a candidate for Volume creation. + // One of these will be selected based on given policy and/or zone. + candidateDatastores := make(map[string]*vclib.DatastoreInfo) + for _, dsInfo := range candidateDatastoreInfos { + candidateDatastores[dsInfo.Info.Url] = dsInfo + } + var vmOptions *vclib.VMOptions if volumeOptions.VSANStorageProfileData != "" || volumeOptions.StoragePolicyName != "" { // If datastore and zone are specified, first validate if the datastore is in the provided zone. if len(volumeOptions.Zone) != 0 && volumeOptions.Datastore != "" { klog.V(4).Infof("Specified zone : %s, datastore : %s", volumeOptions.Zone, volumeOptions.Datastore) - dsList, err = getDatastoresForZone(ctx, dc, vs.nodeManager, volumeOptions.Zone) + dsList, err = getDatastoresForZone(ctx, vs.nodeManager, volumeOptions.Zone) if err != nil { return "", err } @@ -1197,8 +1204,7 @@ func (vs *VSphere) CreateVolume(volumeOptions *vclib.VolumeOptions) (canonicalVo // Validate if the datastore provided belongs to the zone. If not, fail the operation. found := false for _, ds := range dsList { - if ds.Info.Name == volumeOptions.Datastore { - found = true + if datastoreInfo, found = candidateDatastores[ds.Info.Url]; found { break } } @@ -1220,19 +1226,14 @@ func (vs *VSphere) CreateVolume(volumeOptions *vclib.VolumeOptions) (canonicalVo cleanUpRoutineInitialized = true } cleanUpRoutineInitLock.Unlock() - vmOptions, err = vs.setVMOptions(ctx, dc, vs.cfg.Workspace.ResourcePoolPath) - if err != nil { - klog.Errorf("Failed to set VM options requires to create a vsphere volume. err: %+v", err) - return "", err - } } if volumeOptions.StoragePolicyName != "" && volumeOptions.Datastore == "" { if len(volumeOptions.Zone) == 0 { klog.V(4).Infof("Selecting a shared datastore as per the storage policy %s", volumeOptions.StoragePolicyName) - datastore, err = getPbmCompatibleDatastore(ctx, dc, volumeOptions.StoragePolicyName, vs.nodeManager) + datastoreInfo, err = getPbmCompatibleDatastore(ctx, vsi.conn.Client, volumeOptions.StoragePolicyName, vs.nodeManager) } else { // If zone is specified, first get the datastores in the zone. - dsList, err = getDatastoresForZone(ctx, dc, vs.nodeManager, volumeOptions.Zone) + dsList, err = getDatastoresForZone(ctx, vs.nodeManager, volumeOptions.Zone) if err != nil { klog.Errorf("Failed to find a shared datastore matching zone %s. err: %+v", volumeOptions.Zone, err) @@ -1248,18 +1249,18 @@ func (vs *VSphere) CreateVolume(volumeOptions *vclib.VolumeOptions) (canonicalVo klog.V(4).Infof("Specified zone : %s. Picking a datastore as per the storage policy %s among the zoned datastores : %s", volumeOptions.Zone, volumeOptions.StoragePolicyName, dsList) // Among the compatible datastores, select the one based on the maximum free space. - datastore, err = getPbmCompatibleZonedDatastore(ctx, dc, volumeOptions.StoragePolicyName, dsList) + datastoreInfo, err = getPbmCompatibleZonedDatastore(ctx, vsi.conn.Client, volumeOptions.StoragePolicyName, dsList) } - klog.V(1).Infof("Datastore selected as per policy : %s", datastore) if err != nil { klog.Errorf("Failed to get pbm compatible datastore with storagePolicy: %s. err: %+v", volumeOptions.StoragePolicyName, err) return "", err } + klog.V(1).Infof("Datastore selected as per policy : %s", datastoreInfo.Info.Name) } else { // If zone is specified, pick the datastore in the zone with maximum free space within the zone. if volumeOptions.Datastore == "" && len(volumeOptions.Zone) != 0 { klog.V(4).Infof("Specified zone : %s", volumeOptions.Zone) - dsList, err = getDatastoresForZone(ctx, dc, vs.nodeManager, volumeOptions.Zone) + dsList, err = getDatastoresForZone(ctx, vs.nodeManager, volumeOptions.Zone) if err != nil { klog.Errorf("Failed to find a shared datastore matching zone %s. err: %+v", volumeOptions.Zone, err) @@ -1272,40 +1273,40 @@ func (vs *VSphere) CreateVolume(volumeOptions *vclib.VolumeOptions) (canonicalVo return "", err } - datastore, err = getMostFreeDatastoreName(ctx, nil, dsList) + datastoreInfo, err = getMostFreeDatastore(ctx, nil, dsList) if err != nil { klog.Errorf("Failed to get shared datastore: %+v", err) return "", err } - klog.V(1).Infof("Specified zone : %s. Selected datastore : %s", volumeOptions.StoragePolicyName, datastore) + klog.V(1).Infof("Specified zone : %s. Selected datastore : %s", volumeOptions.Zone, datastoreInfo.Info.Name) } else { var sharedDsList []*vclib.DatastoreInfo var err error if len(volumeOptions.Zone) == 0 { // If zone is not provided, get the shared datastore across all node VMs. - klog.V(4).Infof("Validating if datastore %s is shared across all node VMs", datastore) - sharedDsList, err = getSharedDatastoresInK8SCluster(ctx, dc, vs.nodeManager) + klog.V(4).Infof("Validating if datastore %s is shared across all node VMs", datastoreName) + sharedDsList, err = getSharedDatastoresInK8SCluster(ctx, vs.nodeManager) if err != nil { klog.Errorf("Failed to get shared datastore: %+v", err) return "", err } // Prepare error msg to be used later, if required. - err = fmt.Errorf("The specified datastore %s is not a shared datastore across node VMs", datastore) + err = fmt.Errorf("The specified datastore %s is not a shared datastore across node VMs", datastoreName) } else { // If zone is provided, get the shared datastores in that zone. - klog.V(4).Infof("Validating if datastore %s is in zone %s ", datastore, volumeOptions.Zone) - sharedDsList, err = getDatastoresForZone(ctx, dc, vs.nodeManager, volumeOptions.Zone) + klog.V(4).Infof("Validating if datastore %s is in zone %s ", datastoreName, volumeOptions.Zone) + sharedDsList, err = getDatastoresForZone(ctx, vs.nodeManager, volumeOptions.Zone) if err != nil { klog.Errorf("Failed to find a shared datastore matching zone %s. err: %+v", volumeOptions.Zone, err) return "", err } // Prepare error msg to be used later, if required. - err = fmt.Errorf("The specified datastore %s does not match the provided zones : %s", datastore, volumeOptions.Zone) + err = fmt.Errorf("The specified datastore %s does not match the provided zones : %s", datastoreName, volumeOptions.Zone) } found := false // Check if the selected datastore belongs to the list of shared datastores computed. for _, sharedDs := range sharedDsList { - if datastore == sharedDs.Info.Name { + if datastoreInfo, found = candidateDatastores[sharedDs.Info.Url]; found { klog.V(4).Infof("Datastore validation succeeded") found = true break @@ -1317,11 +1318,19 @@ func (vs *VSphere) CreateVolume(volumeOptions *vclib.VolumeOptions) (canonicalVo } } } - ds, err := dc.GetDatastoreByName(ctx, datastore) + + // if datastoreInfo is still not determined, it is an error condition + if datastoreInfo == nil { + klog.Errorf("Ambigous datastore name %s, cannot be found among: %v", datastoreName, candidateDatastoreInfos) + return "", fmt.Errorf("Ambigous datastore name %s", datastoreName) + } + ds := datastoreInfo.Datastore + volumeOptions.Datastore = datastoreInfo.Info.Name + vmOptions, err = vs.setVMOptions(ctx, vsi.conn, ds) if err != nil { + klog.Errorf("Failed to set VM options required to create a vsphere volume. err: %+v", err) return "", err } - volumeOptions.Datastore = datastore kubeVolsPath := filepath.Clean(ds.Path(VolDir)) + "/" err = ds.CreateDirectory(ctx, kubeVolsPath, false) if err != nil && err != vclib.ErrFileAlreadyExist { @@ -1336,18 +1345,18 @@ func (vs *VSphere) CreateVolume(volumeOptions *vclib.VolumeOptions) (canonicalVo } volumePath, err = disk.Create(ctx, ds) if err != nil { - klog.Errorf("Failed to create a vsphere volume with volumeOptions: %+v on datastore: %s. err: %+v", volumeOptions, datastore, err) + klog.Errorf("Failed to create a vsphere volume with volumeOptions: %+v on datastore: %s. err: %+v", volumeOptions, ds, err) return "", err } // Get the canonical path for the volume path. - canonicalVolumePath, err = getcanonicalVolumePath(ctx, dc, volumePath) + canonicalVolumePath, err = getcanonicalVolumePath(ctx, datastoreInfo.Datacenter, volumePath) if err != nil { - klog.Errorf("Failed to get canonical vsphere volume path for volume: %s with volumeOptions: %+v on datastore: %s. err: %+v", volumePath, volumeOptions, datastore, err) + klog.Errorf("Failed to get canonical vsphere volume path for volume: %s with volumeOptions: %+v on datastore: %s. err: %+v", volumePath, volumeOptions, ds, err) return "", err } - if filepath.Base(datastore) != datastore { + if filepath.Base(datastoreName) != datastoreName { // If datastore is within cluster, add cluster path to the volumePath - canonicalVolumePath = strings.Replace(canonicalVolumePath, filepath.Base(datastore), datastore, 1) + canonicalVolumePath = strings.Replace(canonicalVolumePath, filepath.Base(datastoreName), datastoreName, 1) } return canonicalVolumePath, nil } @@ -1576,12 +1585,29 @@ func (vs *VSphere) GetVolumeLabels(volumePath string) (map[string]string, error) return nil, nil } + // Find the datastore on which this volume resides datastorePathObj, err := vclib.GetDatastorePathObjFromVMDiskPath(volumePath) if err != nil { klog.Errorf("Failed to get datastore for volume: %v: %+v", volumePath, err) return nil, err } - dsZones, err := vs.GetZonesForDatastore(ctx, datastorePathObj.Datastore) + dsInfos, err := vs.FindDatastoreByName(ctx, datastorePathObj.Datastore) + if err != nil { + klog.Errorf("Failed to get datastore by name: %v: %+v", datastorePathObj.Datastore, err) + return nil, err + } + var datastore *vclib.Datastore + for _, dsInfo := range dsInfos { + if dsInfo.Datastore.Exists(ctx, datastorePathObj.Path) { + datastore = dsInfo.Datastore + } + } + if datastore == nil { + klog.Errorf("Could not find %s among %v", volumePath, dsInfos) + return nil, fmt.Errorf("Could not find the datastore for volume: %s", volumePath) + } + + dsZones, err := vs.GetZonesForDatastore(ctx, datastore) if err != nil { klog.Errorf("Failed to get zones for datastore %v: %+v", datastorePathObj.Datastore, err) return nil, err @@ -1619,25 +1645,16 @@ func (vs *VSphere) collapseZonesInRegion(ctx context.Context, zones []cloudprovi } // GetZonesForDatastore returns all the zones from which this datastore is visible -func (vs *VSphere) GetZonesForDatastore(ctx context.Context, datastore string) ([]cloudprovider.Zone, error) { +func (vs *VSphere) GetZonesForDatastore(ctx context.Context, datastore *vclib.Datastore) ([]cloudprovider.Zone, error) { vsi, err := vs.getVSphereInstanceForServer(vs.cfg.Workspace.VCenterIP, ctx) if err != nil { klog.Errorf("Failed to get vSphere instance: %+v", err) return nil, err } - dc, err := vclib.GetDatacenter(ctx, vsi.conn, vs.cfg.Workspace.Datacenter) - if err != nil { - klog.Errorf("Failed to get datacenter: %+v", err) - return nil, err - } + // get the hosts mounted on this datastore // datastore -> ["host-1", "host-2", "host-3", ...] - ds, err := dc.GetDatastoreByName(ctx, datastore) - if err != nil { - klog.Errorf("Failed to get datastore by name: %v: %+v", datastore, err) - return nil, err - } - dsHosts, err := ds.GetDatastoreHostMounts(ctx) + dsHosts, err := datastore.GetDatastoreHostMounts(ctx) if err != nil { klog.Errorf("Failed to get datastore host mounts for %v: %+v", datastore, err) return nil, err diff --git a/pkg/cloudprovider/providers/vsphere/vsphere_util.go b/pkg/cloudprovider/providers/vsphere/vsphere_util.go index ff0677a80d8c3..615cd84cd8bb6 100644 --- a/pkg/cloudprovider/providers/vsphere/vsphere_util.go +++ b/pkg/cloudprovider/providers/vsphere/vsphere_util.go @@ -20,20 +20,22 @@ import ( "context" "errors" "fmt" + "math/rand" "os" "path/filepath" "strings" "time" + "github.com/vmware/govmomi/find" "github.com/vmware/govmomi/object" "github.com/vmware/govmomi/property" "github.com/vmware/govmomi/vim25" "github.com/vmware/govmomi/vim25/mo" "github.com/vmware/govmomi/vim25/soap" "github.com/vmware/govmomi/vim25/types" + v1 "k8s.io/api/core/v1" "k8s.io/klog" - "k8s.io/api/core/v1" k8stypes "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/version" "k8s.io/kubernetes/pkg/cloudprovider/providers/vsphere/vclib" @@ -43,6 +45,7 @@ import ( const ( DatastoreProperty = "datastore" DatastoreInfoProperty = "info" + DatastoreNameProperty = "name" Folder = "Folder" VirtualMachine = "VirtualMachine" DummyDiskName = "kube-dummyDisk.vmdk" @@ -121,7 +124,7 @@ func getAccessibleDatastores(ctx context.Context, nodeVmDetail *NodeDetails, nod } // Get all datastores accessible for the virtual machine object. -func getSharedDatastoresInK8SCluster(ctx context.Context, dc *vclib.Datacenter, nodeManager *NodeManager) ([]*vclib.DatastoreInfo, error) { +func getSharedDatastoresInK8SCluster(ctx context.Context, nodeManager *NodeManager) ([]*vclib.DatastoreInfo, error) { nodeVmDetails, err := nodeManager.GetNodeDetails() if err != nil { klog.Errorf("Error while obtaining Kubernetes node nodeVmDetail details. error : %+v", err) @@ -155,12 +158,6 @@ func getSharedDatastoresInK8SCluster(ctx context.Context, dc *vclib.Datacenter, } } klog.V(9).Infof("sharedDatastores : %+v", sharedDatastores) - sharedDatastores, err = getDatastoresForEndpointVC(ctx, dc, sharedDatastores) - if err != nil { - klog.Errorf("Failed to get shared datastores from endpoint VC. err: %+v", err) - return nil, err - } - klog.V(9).Infof("sharedDatastores at endpoint VC: %+v", sharedDatastores) return sharedDatastores, nil } @@ -182,7 +179,7 @@ func intersect(list1 []*vclib.DatastoreInfo, list2 []*vclib.DatastoreInfo) []*vc } // getMostFreeDatastore gets the best fit compatible datastore by free space. -func getMostFreeDatastoreName(ctx context.Context, client *vim25.Client, dsInfoList []*vclib.DatastoreInfo) (string, error) { +func getMostFreeDatastore(ctx context.Context, client *vim25.Client, dsInfoList []*vclib.DatastoreInfo) (*vclib.DatastoreInfo, error) { var curMax int64 curMax = -1 var index int @@ -193,65 +190,46 @@ func getMostFreeDatastoreName(ctx context.Context, client *vim25.Client, dsInfoL index = i } } - return dsInfoList[index].Info.GetDatastoreInfo().Name, nil + return dsInfoList[index], nil } -// Returns the datastores in the given datacenter by performing lookup based on datastore URL. -func getDatastoresForEndpointVC(ctx context.Context, dc *vclib.Datacenter, sharedDsInfos []*vclib.DatastoreInfo) ([]*vclib.DatastoreInfo, error) { - var datastores []*vclib.DatastoreInfo - allDsInfoMap, err := dc.GetAllDatastores(ctx) +func getPbmCompatibleDatastore(ctx context.Context, vcClient *vim25.Client, storagePolicyName string, nodeManager *NodeManager) (*vclib.DatastoreInfo, error) { + pbmClient, err := vclib.NewPbmClient(ctx, vcClient) if err != nil { return nil, err } - for _, sharedDsInfo := range sharedDsInfos { - dsInfo, ok := allDsInfoMap[sharedDsInfo.Info.Url] - if ok { - datastores = append(datastores, dsInfo) - } else { - klog.V(4).Infof("Warning: Shared datastore with URL %s does not exist in endpoint VC", sharedDsInfo.Info.Url) - } - } - klog.V(9).Infof("Datastore from endpoint VC: %+v", datastores) - return datastores, nil -} - -func getPbmCompatibleDatastore(ctx context.Context, dc *vclib.Datacenter, storagePolicyName string, nodeManager *NodeManager) (string, error) { - pbmClient, err := vclib.NewPbmClient(ctx, dc.Client()) - if err != nil { - return "", err - } storagePolicyID, err := pbmClient.ProfileIDByName(ctx, storagePolicyName) if err != nil { klog.Errorf("Failed to get Profile ID by name: %s. err: %+v", storagePolicyName, err) - return "", err + return nil, err } - sharedDs, err := getSharedDatastoresInK8SCluster(ctx, dc, nodeManager) + sharedDs, err := getSharedDatastoresInK8SCluster(ctx, nodeManager) if err != nil { klog.Errorf("Failed to get shared datastores. err: %+v", err) - return "", err + return nil, err } if len(sharedDs) == 0 { msg := "No shared datastores found in the endpoint virtual center" klog.Errorf(msg) - return "", errors.New(msg) + return nil, errors.New(msg) } - compatibleDatastores, _, err := pbmClient.GetCompatibleDatastores(ctx, dc, storagePolicyID, sharedDs) + compatibleDatastores, _, err := pbmClient.GetCompatibleDatastores(ctx, storagePolicyID, sharedDs) if err != nil { klog.Errorf("Failed to get compatible datastores from datastores : %+v with storagePolicy: %s. err: %+v", sharedDs, storagePolicyID, err) - return "", err + return nil, err } klog.V(9).Infof("compatibleDatastores : %+v", compatibleDatastores) - datastore, err := getMostFreeDatastoreName(ctx, dc.Client(), compatibleDatastores) + datastore, err := getMostFreeDatastore(ctx, vcClient, compatibleDatastores) if err != nil { klog.Errorf("Failed to get most free datastore from compatible datastores: %+v. err: %+v", compatibleDatastores, err) - return "", err + return nil, err } - klog.V(4).Infof("Most free datastore : %+s", datastore) + klog.V(4).Infof("Most free datastore : %+s", datastore.Info.Name) return datastore, err } -func getDatastoresForZone(ctx context.Context, dc *vclib.Datacenter, nodeManager *NodeManager, selectedZones []string) ([]*vclib.DatastoreInfo, error) { +func getDatastoresForZone(ctx context.Context, nodeManager *NodeManager, selectedZones []string) ([]*vclib.DatastoreInfo, error) { var sharedDatastores []*vclib.DatastoreInfo @@ -279,7 +257,7 @@ func getDatastoresForZone(ctx context.Context, dc *vclib.Datacenter, nodeManager var dsMoList []mo.Datastore pc := property.DefaultCollector(host.Client()) - properties := []string{DatastoreInfoProperty} + properties := []string{DatastoreInfoProperty, DatastoreNameProperty} err = pc.Retrieve(ctx, dsRefList, properties, &dsMoList) if err != nil { klog.Errorf("Failed to get Datastore managed objects from datastore objects."+ @@ -288,12 +266,37 @@ func getDatastoresForZone(ctx context.Context, dc *vclib.Datacenter, nodeManager } klog.V(9).Infof("Datastore mo details: %+v", dsMoList) + // find the Datacenter parent for this host + mes, err := mo.Ancestors(ctx, host.Client(), pc.Reference(), host.Reference()) + if err != nil { + return nil, err + } + var dcMoref *types.ManagedObjectReference + for i := len(mes) - 1; i > 0; i-- { + if mes[i].Self.Type == "Datacenter" { + dcMoref = &mes[i].Self + break + } + } + if dcMoref == nil { + return nil, fmt.Errorf("Failed to find the Datacenter of host %s", host) + } + + dc := object.NewDatacenter(host.Client(), *dcMoref) + finder := find.NewFinder(host.Client(), false) + finder.SetDatacenter(dc) var dsObjList []*vclib.DatastoreInfo for _, dsMo := range dsMoList { + // use the finder so that InventoryPath is set correctly in dsObj + dsObj, err := finder.Datastore(ctx, dsMo.Name) + if err != nil { + klog.Errorf("Failed to find datastore named %s in datacenter %s", dsMo.Name, dc) + return nil, err + } dsObjList = append(dsObjList, &vclib.DatastoreInfo{ - Datastore: &vclib.Datastore{Datastore: object.NewDatastore(host.Client(), dsMo.Reference()), - Datacenter: nil}, + Datastore: &vclib.Datastore{Datastore: dsObj, + Datacenter: &vclib.Datacenter{Datacenter: dc}}, Info: dsMo.Info.GetDatastoreInfo()}) } @@ -317,40 +320,48 @@ func getDatastoresForZone(ctx context.Context, dc *vclib.Datacenter, nodeManager return sharedDatastores, nil } -func getPbmCompatibleZonedDatastore(ctx context.Context, dc *vclib.Datacenter, storagePolicyName string, zonedDatastores []*vclib.DatastoreInfo) (string, error) { - pbmClient, err := vclib.NewPbmClient(ctx, dc.Client()) +func getPbmCompatibleZonedDatastore(ctx context.Context, vcClient *vim25.Client, storagePolicyName string, zonedDatastores []*vclib.DatastoreInfo) (*vclib.DatastoreInfo, error) { + pbmClient, err := vclib.NewPbmClient(ctx, vcClient) if err != nil { - return "", err + return nil, err } storagePolicyID, err := pbmClient.ProfileIDByName(ctx, storagePolicyName) if err != nil { klog.Errorf("Failed to get Profile ID by name: %s. err: %+v", storagePolicyName, err) - return "", err + return nil, err } - compatibleDatastores, _, err := pbmClient.GetCompatibleDatastores(ctx, dc, storagePolicyID, zonedDatastores) + compatibleDatastores, _, err := pbmClient.GetCompatibleDatastores(ctx, storagePolicyID, zonedDatastores) if err != nil { klog.Errorf("Failed to get compatible datastores from datastores : %+v with storagePolicy: %s. err: %+v", zonedDatastores, storagePolicyID, err) - return "", err + return nil, err } klog.V(9).Infof("compatibleDatastores : %+v", compatibleDatastores) - datastore, err := getMostFreeDatastoreName(ctx, dc.Client(), compatibleDatastores) + datastore, err := getMostFreeDatastore(ctx, vcClient, compatibleDatastores) if err != nil { klog.Errorf("Failed to get most free datastore from compatible datastores: %+v. err: %+v", compatibleDatastores, err) - return "", err + return nil, err } - klog.V(4).Infof("Most free datastore : %+s", datastore) + klog.V(4).Infof("Most free datastore : %+s", datastore.Info.Name) return datastore, err } -func (vs *VSphere) setVMOptions(ctx context.Context, dc *vclib.Datacenter, resourcePoolPath string) (*vclib.VMOptions, error) { +func (vs *VSphere) setVMOptions(ctx context.Context, connection *vclib.VSphereConnection, ds *vclib.Datastore) (*vclib.VMOptions, error) { var vmOptions vclib.VMOptions - resourcePool, err := dc.GetResourcePool(ctx, resourcePoolPath) + dsHosts, err := ds.GetDatastoreHostMounts(ctx) if err != nil { + klog.Errorf("Failed to get datastore host mounts for %v: %+v", ds, err) return nil, err } - klog.V(9).Infof("Resource pool path %s, resourcePool %+v", resourcePoolPath, resourcePool) - folder, err := dc.GetFolderByPath(ctx, vs.cfg.Workspace.Folder) + // pick a host at random to use for Volume creation + dsHostMoref := dsHosts[rand.New(rand.NewSource(time.Now().UnixNano())).Intn(len(dsHosts))] + dummyVMHost := object.NewHostSystem(connection.Client, dsHostMoref) + resourcePool, err := dummyVMHost.ResourcePool(ctx) + if err != nil { + klog.Errorf("Failed to get resource pool from host %v", dummyVMHost) + return nil, err + } + folder, err := ds.Datacenter.GetFolderByPath(ctx, vs.cfg.Workspace.Folder) if err != nil { return nil, err } @@ -366,32 +377,30 @@ func (vs *VSphere) cleanUpDummyVMs(dummyVMPrefix string) { defer cancel() for { time.Sleep(CleanUpDummyVMRoutineInterval * time.Minute) - vsi, err := vs.getVSphereInstanceForServer(vs.cfg.Workspace.VCenterIP, ctx) - if err != nil { - klog.V(4).Infof("Failed to get VSphere instance with err: %+v. Retrying again...", err) - continue - } - dc, err := vclib.GetDatacenter(ctx, vsi.conn, vs.cfg.Workspace.Datacenter) + datacenters, err := vs.GetWorkspaceDatacenters(ctx) if err != nil { - klog.V(4).Infof("Failed to get the datacenter: %s from VC. err: %+v", vs.cfg.Workspace.Datacenter, err) + klog.V(4).Infof("Failed to get datacenters from VC. err: %+v", err) continue } - // Get the folder reference for global working directory where the dummy VM needs to be created. - vmFolder, err := dc.GetFolderByPath(ctx, vs.cfg.Workspace.Folder) - if err != nil { - klog.V(4).Infof("Unable to get the kubernetes folder: %q reference. err: %+v", vs.cfg.Workspace.Folder, err) - continue - } - // A write lock is acquired to make sure the cleanUp routine doesn't delete any VM's created by ongoing PVC requests. - cleanUpDummyVMs := func() { - cleanUpDummyVMLock.Lock() - defer cleanUpDummyVMLock.Unlock() - err = diskmanagers.CleanUpDummyVMs(ctx, vmFolder, dc) + // Clean up dummy VMs in each datacenter + for _, dc := range datacenters { + // Get the folder reference for global working directory where the dummy VM needs to be created. + vmFolder, err := dc.GetFolderByPath(ctx, vs.cfg.Workspace.Folder) if err != nil { - klog.V(4).Infof("Unable to clean up dummy VM's in the kubernetes cluster: %q. err: %+v", vs.cfg.Workspace.Folder, err) + klog.V(4).Infof("Unable to get the kubernetes folder: %q reference. err: %+v", vs.cfg.Workspace.Folder, err) + continue + } + // A write lock is acquired to make sure the cleanUp routine doesn't delete any VM's created by ongoing PVC requests. + cleanUpDummyVMs := func() { + cleanUpDummyVMLock.Lock() + defer cleanUpDummyVMLock.Unlock() + err = diskmanagers.CleanUpDummyVMs(ctx, vmFolder) + if err != nil { + klog.V(4).Infof("Unable to clean up dummy VM's in the kubernetes cluster: %q. err: %+v", vs.cfg.Workspace.Folder, err) + } } + cleanUpDummyVMs() } - cleanUpDummyVMs() } } @@ -586,43 +595,6 @@ func (vs *VSphere) checkDiskAttached(ctx context.Context, nodes []k8stypes.NodeN return nodesToRetry, nil } -func (vs *VSphere) IsDummyVMPresent(vmName string) (bool, error) { - isDummyVMPresent := false - - // Create context - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - - vsi, err := vs.getVSphereInstanceForServer(vs.cfg.Workspace.VCenterIP, ctx) - if err != nil { - return isDummyVMPresent, err - } - - dc, err := vclib.GetDatacenter(ctx, vsi.conn, vs.cfg.Workspace.Datacenter) - if err != nil { - return isDummyVMPresent, err - } - - vmFolder, err := dc.GetFolderByPath(ctx, vs.cfg.Workspace.Folder) - if err != nil { - return isDummyVMPresent, err - } - - vms, err := vmFolder.GetVirtualMachines(ctx) - if err != nil { - return isDummyVMPresent, err - } - - for _, vm := range vms { - if vm.Name() == vmName { - isDummyVMPresent = true - break - } - } - - return isDummyVMPresent, nil -} - func (vs *VSphere) GetNodeNameFromProviderID(providerID string) (string, error) { var nodeName string nodes, err := vs.nodeManager.GetNodeDetails() @@ -698,3 +670,63 @@ func GetVMUUID() (string, error) { uuid = fmt.Sprintf("%s-%s-%s-%s-%s", uuid[0:8], uuid[8:12], uuid[12:16], uuid[16:20], uuid[20:32]) return uuid, nil } + +// GetWorkspaceDatacenters returns the Datacenter objects that VCP has access to. +// User can configure the list of datacenters in vsphere.conf. Otherwise all the +// Datacenters in the configured list of VCs are returned. +func (vs *VSphere) GetWorkspaceDatacenters(ctx context.Context) ([]*vclib.Datacenter, error) { + var datacenterObjs []*vclib.Datacenter + for vc, vsi := range vs.vsphereInstanceMap { + // ensure connection to VC + err := vs.nodeManager.vcConnect(ctx, vsi) + if err != nil { + return nil, err + } + if vsi.cfg.Datacenters == "" { + vcDatacenterObjs, err := vclib.GetAllDatacenter(ctx, vsi.conn) + if err != nil { + klog.Errorf("Error fetching list of datacenters from VC %s: %+v", vc, err) + return nil, err + } + datacenterObjs = append(datacenterObjs, vcDatacenterObjs...) + } else { + datacenters := strings.Split(vsi.cfg.Datacenters, ",") + for _, dc := range datacenters { + dc = strings.TrimSpace(dc) + if dc == "" { + continue + } + datacenterObj, err := vclib.GetDatacenter(ctx, vsi.conn, dc) + if err != nil { + klog.Errorf("Error fetching datacenter %s from VC %s: %+v", dc, vc, err) + return nil, err + } + datacenterObjs = append(datacenterObjs, datacenterObj) + } + } + } + return datacenterObjs, nil +} + +// FindDatastoreByName looks for the given datastore by name across all available datacenters. +// If more than one Datacenter has a Datastore with the given name, then returns reference to all of them. +func (vs *VSphere) FindDatastoreByName(ctx context.Context, datastoreName string) ([]*vclib.DatastoreInfo, error) { + datacenters, err := vs.GetWorkspaceDatacenters(ctx) + if err != nil { + return nil, err + } + var datastoreInfos []*vclib.DatastoreInfo + for _, dc := range datacenters { + datastoreInfo, err := dc.GetDatastoreInfoByName(ctx, datastoreName) + if err != nil { + klog.V(9).Infof("Did not find datastore %s in datacenter %s, still looking.", datastoreName, dc.Name()) + continue + } + datastoreInfos = append(datastoreInfos, datastoreInfo) + } + if len(datastoreInfos) == 0 { + return nil, fmt.Errorf("Datastore '%s' not found", datastoreName) + } + klog.V(4).Infof("Found datastore infos %v for datastore %s", datastoreInfos, datastoreName) + return datastoreInfos, nil +} diff --git a/pkg/controller/podautoscaler/horizontal.go b/pkg/controller/podautoscaler/horizontal.go index c6a31532f196d..0394df5fcd71f 100644 --- a/pkg/controller/podautoscaler/horizontal.go +++ b/pkg/controller/podautoscaler/horizontal.go @@ -23,7 +23,7 @@ import ( autoscalingv1 "k8s.io/api/autoscaling/v1" autoscalingv2 "k8s.io/api/autoscaling/v2beta2" - "k8s.io/api/core/v1" + v1 "k8s.io/api/core/v1" apiequality "k8s.io/apimachinery/pkg/api/equality" "k8s.io/apimachinery/pkg/api/errors" apimeta "k8s.io/apimachinery/pkg/api/meta" @@ -235,7 +235,8 @@ func (a *HorizontalController) processNextWorkItem() bool { func (a *HorizontalController) computeReplicasForMetrics(hpa *autoscalingv2.HorizontalPodAutoscaler, scale *autoscalingv1.Scale, metricSpecs []autoscalingv2.MetricSpec) (replicas int32, metric string, statuses []autoscalingv2.MetricStatus, timestamp time.Time, err error) { - currentReplicas := scale.Status.Replicas + specReplicas := scale.Spec.Replicas + statusReplicas := scale.Status.Replicas statuses = make([]autoscalingv2.MetricStatus, len(metricSpecs)) @@ -267,7 +268,7 @@ func (a *HorizontalController) computeReplicasForMetrics(hpa *autoscalingv2.Hori setCondition(hpa, autoscalingv2.ScalingActive, v1.ConditionFalse, "FailedGetObjectMetric", "the HPA was unable to compute the replica count: %v", err) return 0, "", nil, time.Time{}, fmt.Errorf("failed to get object metric value: %v", err) } - replicaCountProposal, timestampProposal, metricNameProposal, err = a.computeStatusForObjectMetric(currentReplicas, metricSpec, hpa, selector, &statuses[i], metricSelector) + replicaCountProposal, timestampProposal, metricNameProposal, err = a.computeStatusForObjectMetric(specReplicas, statusReplicas, metricSpec, hpa, selector, &statuses[i], metricSelector) if err != nil { return 0, "", nil, time.Time{}, fmt.Errorf("failed to get object metric value: %v", err) } @@ -278,17 +279,17 @@ func (a *HorizontalController) computeReplicasForMetrics(hpa *autoscalingv2.Hori setCondition(hpa, autoscalingv2.ScalingActive, v1.ConditionFalse, "FailedGetPodsMetric", "the HPA was unable to compute the replica count: %v", err) return 0, "", nil, time.Time{}, fmt.Errorf("failed to get pods metric value: %v", err) } - replicaCountProposal, timestampProposal, metricNameProposal, err = a.computeStatusForPodsMetric(currentReplicas, metricSpec, hpa, selector, &statuses[i], metricSelector) + replicaCountProposal, timestampProposal, metricNameProposal, err = a.computeStatusForPodsMetric(specReplicas, metricSpec, hpa, selector, &statuses[i], metricSelector) if err != nil { return 0, "", nil, time.Time{}, fmt.Errorf("failed to get object metric value: %v", err) } case autoscalingv2.ResourceMetricSourceType: - replicaCountProposal, timestampProposal, metricNameProposal, err = a.computeStatusForResourceMetric(currentReplicas, metricSpec, hpa, selector, &statuses[i]) + replicaCountProposal, timestampProposal, metricNameProposal, err = a.computeStatusForResourceMetric(specReplicas, metricSpec, hpa, selector, &statuses[i]) if err != nil { return 0, "", nil, time.Time{}, err } case autoscalingv2.ExternalMetricSourceType: - replicaCountProposal, timestampProposal, metricNameProposal, err = a.computeStatusForExternalMetric(currentReplicas, metricSpec, hpa, selector, &statuses[i]) + replicaCountProposal, timestampProposal, metricNameProposal, err = a.computeStatusForExternalMetric(specReplicas, statusReplicas, metricSpec, hpa, selector, &statuses[i]) if err != nil { return 0, "", nil, time.Time{}, err } @@ -326,9 +327,9 @@ func (a *HorizontalController) reconcileKey(key string) (deleted bool, err error } // computeStatusForObjectMetric computes the desired number of replicas for the specified metric of type ObjectMetricSourceType. -func (a *HorizontalController) computeStatusForObjectMetric(currentReplicas int32, metricSpec autoscalingv2.MetricSpec, hpa *autoscalingv2.HorizontalPodAutoscaler, selector labels.Selector, status *autoscalingv2.MetricStatus, metricSelector labels.Selector) (int32, time.Time, string, error) { +func (a *HorizontalController) computeStatusForObjectMetric(specReplicas, statusReplicas int32, metricSpec autoscalingv2.MetricSpec, hpa *autoscalingv2.HorizontalPodAutoscaler, selector labels.Selector, status *autoscalingv2.MetricStatus, metricSelector labels.Selector) (int32, time.Time, string, error) { if metricSpec.Object.Target.Type == autoscalingv2.ValueMetricType { - replicaCountProposal, utilizationProposal, timestampProposal, err := a.replicaCalc.GetObjectMetricReplicas(currentReplicas, metricSpec.Object.Target.Value.MilliValue(), metricSpec.Object.Metric.Name, hpa.Namespace, &metricSpec.Object.DescribedObject, selector, metricSelector) + replicaCountProposal, utilizationProposal, timestampProposal, err := a.replicaCalc.GetObjectMetricReplicas(specReplicas, metricSpec.Object.Target.Value.MilliValue(), metricSpec.Object.Metric.Name, hpa.Namespace, &metricSpec.Object.DescribedObject, selector, metricSelector) if err != nil { a.eventRecorder.Event(hpa, v1.EventTypeWarning, "FailedGetObjectMetric", err.Error()) setCondition(hpa, autoscalingv2.ScalingActive, v1.ConditionFalse, "FailedGetObjectMetric", "the HPA was unable to compute the replica count: %v", err) @@ -349,7 +350,7 @@ func (a *HorizontalController) computeStatusForObjectMetric(currentReplicas int3 } return replicaCountProposal, timestampProposal, fmt.Sprintf("%s metric %s", metricSpec.Object.DescribedObject.Kind, metricSpec.Object.Metric.Name), nil } else if metricSpec.Object.Target.Type == autoscalingv2.AverageValueMetricType { - replicaCountProposal, utilizationProposal, timestampProposal, err := a.replicaCalc.GetObjectPerPodMetricReplicas(currentReplicas, metricSpec.Object.Target.AverageValue.MilliValue(), metricSpec.Object.Metric.Name, hpa.Namespace, &metricSpec.Object.DescribedObject, metricSelector) + replicaCountProposal, utilizationProposal, timestampProposal, err := a.replicaCalc.GetObjectPerPodMetricReplicas(statusReplicas, metricSpec.Object.Target.AverageValue.MilliValue(), metricSpec.Object.Metric.Name, hpa.Namespace, &metricSpec.Object.DescribedObject, metricSelector) if err != nil { a.eventRecorder.Event(hpa, v1.EventTypeWarning, "FailedGetObjectMetric", err.Error()) setCondition(hpa, autoscalingv2.ScalingActive, v1.ConditionFalse, "FailedGetObjectMetric", "the HPA was unable to compute the replica count: %v", err) @@ -452,9 +453,9 @@ func (a *HorizontalController) computeStatusForResourceMetric(currentReplicas in } // computeStatusForExternalMetric computes the desired number of replicas for the specified metric of type ExternalMetricSourceType. -func (a *HorizontalController) computeStatusForExternalMetric(currentReplicas int32, metricSpec autoscalingv2.MetricSpec, hpa *autoscalingv2.HorizontalPodAutoscaler, selector labels.Selector, status *autoscalingv2.MetricStatus) (int32, time.Time, string, error) { +func (a *HorizontalController) computeStatusForExternalMetric(specReplicas, statusReplicas int32, metricSpec autoscalingv2.MetricSpec, hpa *autoscalingv2.HorizontalPodAutoscaler, selector labels.Selector, status *autoscalingv2.MetricStatus) (int32, time.Time, string, error) { if metricSpec.External.Target.AverageValue != nil { - replicaCountProposal, utilizationProposal, timestampProposal, err := a.replicaCalc.GetExternalPerPodMetricReplicas(currentReplicas, metricSpec.External.Target.AverageValue.MilliValue(), metricSpec.External.Metric.Name, hpa.Namespace, metricSpec.External.Metric.Selector) + replicaCountProposal, utilizationProposal, timestampProposal, err := a.replicaCalc.GetExternalPerPodMetricReplicas(statusReplicas, metricSpec.External.Target.AverageValue.MilliValue(), metricSpec.External.Metric.Name, hpa.Namespace, metricSpec.External.Metric.Selector) if err != nil { a.eventRecorder.Event(hpa, v1.EventTypeWarning, "FailedGetExternalMetric", err.Error()) setCondition(hpa, autoscalingv2.ScalingActive, v1.ConditionFalse, "FailedGetExternalMetric", "the HPA was unable to compute the replica count: %v", err) @@ -475,7 +476,7 @@ func (a *HorizontalController) computeStatusForExternalMetric(currentReplicas in return replicaCountProposal, timestampProposal, fmt.Sprintf("external metric %s(%+v)", metricSpec.External.Metric.Name, metricSpec.External.Metric.Selector), nil } if metricSpec.External.Target.Value != nil { - replicaCountProposal, utilizationProposal, timestampProposal, err := a.replicaCalc.GetExternalMetricReplicas(currentReplicas, metricSpec.External.Target.Value.MilliValue(), metricSpec.External.Metric.Name, hpa.Namespace, metricSpec.External.Metric.Selector, selector) + replicaCountProposal, utilizationProposal, timestampProposal, err := a.replicaCalc.GetExternalMetricReplicas(specReplicas, metricSpec.External.Target.Value.MilliValue(), metricSpec.External.Metric.Name, hpa.Namespace, metricSpec.External.Metric.Selector, selector) if err != nil { a.eventRecorder.Event(hpa, v1.EventTypeWarning, "FailedGetExternalMetric", err.Error()) setCondition(hpa, autoscalingv2.ScalingActive, v1.ConditionFalse, "FailedGetExternalMetric", "the HPA was unable to compute the replica count: %v", err) @@ -550,7 +551,7 @@ func (a *HorizontalController) reconcileAutoscaler(hpav1Shared *autoscalingv1.Ho return fmt.Errorf("failed to query scale subresource for %s: %v", reference, err) } setCondition(hpa, autoscalingv2.AbleToScale, v1.ConditionTrue, "SucceededGetScale", "the HPA controller was able to get the target's current scale") - currentReplicas := scale.Status.Replicas + currentReplicas := scale.Spec.Replicas a.recordInitialRecommendation(currentReplicas, key) var metricStatuses []autoscalingv2.MetricStatus diff --git a/pkg/controller/podautoscaler/replica_calculator.go b/pkg/controller/podautoscaler/replica_calculator.go index ace9803bdf3d7..f498a0ddeb665 100644 --- a/pkg/controller/podautoscaler/replica_calculator.go +++ b/pkg/controller/podautoscaler/replica_calculator.go @@ -22,7 +22,7 @@ import ( "time" autoscaling "k8s.io/api/autoscaling/v2beta2" - "k8s.io/api/core/v1" + v1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/labels" "k8s.io/apimachinery/pkg/util/sets" @@ -333,7 +333,7 @@ func (c *ReplicaCalculator) GetExternalMetricReplicas(currentReplicas int32, tar // GetExternalPerPodMetricReplicas calculates the desired replica count based on a // target metric value per pod (as a milli-value) for the external metric in the // given namespace, and the current replica count. -func (c *ReplicaCalculator) GetExternalPerPodMetricReplicas(currentReplicas int32, targetUtilizationPerPod int64, metricName, namespace string, metricSelector *metav1.LabelSelector) (replicaCount int32, utilization int64, timestamp time.Time, err error) { +func (c *ReplicaCalculator) GetExternalPerPodMetricReplicas(statusReplicas int32, targetUtilizationPerPod int64, metricName, namespace string, metricSelector *metav1.LabelSelector) (replicaCount int32, utilization int64, timestamp time.Time, err error) { metricLabelSelector, err := metav1.LabelSelectorAsSelector(metricSelector) if err != nil { return 0, 0, time.Time{}, err @@ -347,13 +347,13 @@ func (c *ReplicaCalculator) GetExternalPerPodMetricReplicas(currentReplicas int3 utilization = utilization + val } - replicaCount = currentReplicas + replicaCount = statusReplicas usageRatio := float64(utilization) / (float64(targetUtilizationPerPod) * float64(replicaCount)) if math.Abs(1.0-usageRatio) > c.tolerance { // update number of replicas if the change is large enough replicaCount = int32(math.Ceil(float64(utilization) / float64(targetUtilizationPerPod))) } - utilization = int64(math.Ceil(float64(utilization) / float64(currentReplicas))) + utilization = int64(math.Ceil(float64(utilization) / float64(statusReplicas))) return replicaCount, utilization, timestamp, nil } diff --git a/pkg/controller/podautoscaler/replica_calculator_test.go b/pkg/controller/podautoscaler/replica_calculator_test.go index 0638f89341ffd..3968103b92d6d 100644 --- a/pkg/controller/podautoscaler/replica_calculator_test.go +++ b/pkg/controller/podautoscaler/replica_calculator_test.go @@ -23,7 +23,7 @@ import ( "time" autoscalingv2 "k8s.io/api/autoscaling/v2beta2" - "k8s.io/api/core/v1" + v1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/meta/testrestmapper" "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" diff --git a/pkg/controller/volume/attachdetach/attach_detach_controller.go b/pkg/controller/volume/attachdetach/attach_detach_controller.go index 77ffbc7fb390d..fa7086fdc16c4 100644 --- a/pkg/controller/volume/attachdetach/attach_detach_controller.go +++ b/pkg/controller/volume/attachdetach/attach_detach_controller.go @@ -106,6 +106,7 @@ func NewAttachDetachController( pvcInformer coreinformers.PersistentVolumeClaimInformer, pvInformer coreinformers.PersistentVolumeInformer, csiNodeInformer storageinformers.CSINodeInformer, + csiDriverInformer storageinformers.CSIDriverInformer, cloud cloudprovider.Interface, plugins []volume.VolumePlugin, prober volume.DynamicPluginProber, @@ -147,6 +148,11 @@ func NewAttachDetachController( adc.csiNodeSynced = csiNodeInformer.Informer().HasSynced } + if utilfeature.DefaultFeatureGate.Enabled(features.CSIDriverRegistry) { + adc.csiDriverLister = csiDriverInformer.Lister() + adc.csiDriversSynced = csiDriverInformer.Informer().HasSynced + } + if err := adc.volumePluginMgr.InitPlugins(plugins, prober, adc); err != nil { return nil, fmt.Errorf("Could not initialize volume plugins for Attach/Detach Controller: %+v", err) } @@ -271,6 +277,12 @@ type attachDetachController struct { csiNodeLister storagelisters.CSINodeLister csiNodeSynced kcache.InformerSynced + // csiDriverLister is the shared CSIDriver lister used to fetch and store + // CSIDriver objects from the API server. It is shared with other controllers + // and therefore the CSIDriver objects in its store should be treated as immutable. + csiDriverLister storagelisters.CSIDriverLister + csiDriversSynced kcache.InformerSynced + // cloud provider used by volume host cloud cloudprovider.Interface @@ -327,6 +339,9 @@ func (adc *attachDetachController) Run(stopCh <-chan struct{}) { if adc.csiNodeSynced != nil { synced = append(synced, adc.csiNodeSynced) } + if adc.csiDriversSynced != nil { + synced = append(synced, adc.csiDriversSynced) + } if !controller.WaitForCacheSync("attach detach", stopCh, synced...) { return @@ -669,6 +684,10 @@ func (adc *attachDetachController) CSINodeLister() storagelisters.CSINodeLister return adc.csiNodeLister } +func (adc *attachDetachController) CSIDriverLister() storagelisters.CSIDriverLister { + return adc.csiDriverLister +} + func (adc *attachDetachController) IsAttachDetachController() bool { return true } @@ -793,3 +812,7 @@ func (adc *attachDetachController) GetSubpather() subpath.Interface { // Subpaths not needed in attachdetach controller return nil } + +func (adc *attachDetachController) GetCSIDriverLister() storagelisters.CSIDriverLister { + return adc.csiDriverLister +} diff --git a/pkg/controller/volume/attachdetach/attach_detach_controller_test.go b/pkg/controller/volume/attachdetach/attach_detach_controller_test.go index 028d868db4db8..80518e6f579a6 100644 --- a/pkg/controller/volume/attachdetach/attach_detach_controller_test.go +++ b/pkg/controller/volume/attachdetach/attach_detach_controller_test.go @@ -45,6 +45,7 @@ func Test_NewAttachDetachController_Positive(t *testing.T) { informerFactory.Core().V1().PersistentVolumeClaims(), informerFactory.Core().V1().PersistentVolumes(), informerFactory.Storage().V1beta1().CSINodes(), + informerFactory.Storage().V1beta1().CSIDrivers(), nil, /* cloud */ nil, /* plugins */ nil, /* prober */ @@ -220,6 +221,7 @@ func attachDetachRecoveryTestCase(t *testing.T, extraPods1 []*v1.Pod, extraPods2 informerFactory.Core().V1().PersistentVolumeClaims(), informerFactory.Core().V1().PersistentVolumes(), informerFactory.Storage().V1beta1().CSINodes(), + informerFactory.Storage().V1beta1().CSIDrivers(), nil, /* cloud */ plugins, prober, diff --git a/pkg/kubelet/BUILD b/pkg/kubelet/BUILD index fb1e04671ba6f..4673a16fed7c8 100644 --- a/pkg/kubelet/BUILD +++ b/pkg/kubelet/BUILD @@ -132,9 +132,11 @@ go_library( "//staging/src/k8s.io/apimachinery/pkg/util/validation:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/util/wait:go_default_library", "//staging/src/k8s.io/apiserver/pkg/util/feature:go_default_library", + "//staging/src/k8s.io/client-go/informers:go_default_library", "//staging/src/k8s.io/client-go/kubernetes:go_default_library", "//staging/src/k8s.io/client-go/kubernetes/typed/core/v1:go_default_library", "//staging/src/k8s.io/client-go/listers/core/v1:go_default_library", + "//staging/src/k8s.io/client-go/listers/storage/v1beta1:go_default_library", "//staging/src/k8s.io/client-go/tools/cache:go_default_library", "//staging/src/k8s.io/client-go/tools/record:go_default_library", "//staging/src/k8s.io/client-go/util/certificate:go_default_library", diff --git a/pkg/kubelet/cm/cgroup_manager_linux.go b/pkg/kubelet/cm/cgroup_manager_linux.go index b08bab132d54a..7797e9b0c20bd 100644 --- a/pkg/kubelet/cm/cgroup_manager_linux.go +++ b/pkg/kubelet/cm/cgroup_manager_linux.go @@ -326,7 +326,7 @@ func getSupportedSubsystems() map[subsystem]bool { supportedSubsystems := map[subsystem]bool{ &cgroupfs.MemoryGroup{}: true, &cgroupfs.CpuGroup{}: true, - &cgroupfs.PidsGroup{}: true, + &cgroupfs.PidsGroup{}: false, } // not all hosts support hugetlb cgroup, and in the absent of hugetlb, we will fail silently by reporting no capacity. supportedSubsystems[&cgroupfs.HugetlbGroup{}] = false diff --git a/pkg/kubelet/kuberuntime/kuberuntime_manager.go b/pkg/kubelet/kuberuntime/kuberuntime_manager.go index b6313fb8bdf2a..4676a3548acd9 100644 --- a/pkg/kubelet/kuberuntime/kuberuntime_manager.go +++ b/pkg/kubelet/kuberuntime/kuberuntime_manager.go @@ -477,11 +477,15 @@ func (m *kubeGenericRuntimeManager) computePodActions(pod *v1.Pod, podStatus *ku // If we need to (re-)create the pod sandbox, everything will need to be // killed and recreated, and init containers should be purged. if createPodSandbox { - if !shouldRestartOnFailure(pod) && attempt != 0 { + if !shouldRestartOnFailure(pod) && attempt != 0 && len(podStatus.ContainerStatuses) != 0 { // Should not restart the pod, just return. // we should not create a sandbox for a pod if it is already done. // if all containers are done and should not be started, there is no need to create a new sandbox. // this stops confusing logs on pods whose containers all have exit codes, but we recreate a sandbox before terminating it. + // + // If ContainerStatuses is empty, we assume that we've never + // successfully created any containers. In this case, we should + // retry creating the sandbox. changes.CreateSandbox = false return changes } diff --git a/pkg/kubelet/kuberuntime/kuberuntime_manager_test.go b/pkg/kubelet/kuberuntime/kuberuntime_manager_test.go index 8de68a7e4bc5f..5102cec441cea 100644 --- a/pkg/kubelet/kuberuntime/kuberuntime_manager_test.go +++ b/pkg/kubelet/kuberuntime/kuberuntime_manager_test.go @@ -852,6 +852,26 @@ func TestComputePodActions(t *testing.T) { ContainersToKill: map[kubecontainer.ContainerID]containerToKillInfo{}, }, }, + "Verify we create a pod sandbox if no ready sandbox for pod with RestartPolicy=Never and no containers have ever been created": { + mutatePodFn: func(pod *v1.Pod) { + pod.Spec.RestartPolicy = v1.RestartPolicyNever + }, + mutateStatusFn: func(status *kubecontainer.PodStatus) { + // no ready sandbox + status.SandboxStatuses[0].State = runtimeapi.PodSandboxState_SANDBOX_NOTREADY + status.SandboxStatuses[0].Metadata.Attempt = uint32(2) + // no visible containers + status.ContainerStatuses = []*kubecontainer.ContainerStatus{} + }, + actions: podActions{ + SandboxID: baseStatus.SandboxStatuses[0].Id, + Attempt: uint32(3), + CreateSandbox: true, + KillPod: true, + ContainersToStart: []int{0, 1, 2}, + ContainersToKill: map[kubecontainer.ContainerID]containerToKillInfo{}, + }, + }, "Kill and recreate the container if the container is in unknown state": { mutatePodFn: func(pod *v1.Pod) { pod.Spec.RestartPolicy = v1.RestartPolicyNever }, mutateStatusFn: func(status *kubecontainer.PodStatus) { @@ -1028,6 +1048,36 @@ func TestComputePodActionsWithInitContainers(t *testing.T) { ContainersToKill: getKillMapWithInitContainers(basePod, baseStatus, []int{}), }, }, + "Pod sandbox not ready, init container failed, but RestartPolicy == Never; kill pod only": { + mutatePodFn: func(pod *v1.Pod) { pod.Spec.RestartPolicy = v1.RestartPolicyNever }, + mutateStatusFn: func(status *kubecontainer.PodStatus) { + status.SandboxStatuses[0].State = runtimeapi.PodSandboxState_SANDBOX_NOTREADY + }, + actions: podActions{ + KillPod: true, + CreateSandbox: false, + SandboxID: baseStatus.SandboxStatuses[0].Id, + Attempt: uint32(1), + ContainersToStart: []int{}, + ContainersToKill: getKillMapWithInitContainers(basePod, baseStatus, []int{}), + }, + }, + "Pod sandbox not ready, and RestartPolicy == Never, but no visible init containers; create a new pod sandbox": { + mutatePodFn: func(pod *v1.Pod) { pod.Spec.RestartPolicy = v1.RestartPolicyNever }, + mutateStatusFn: func(status *kubecontainer.PodStatus) { + status.SandboxStatuses[0].State = runtimeapi.PodSandboxState_SANDBOX_NOTREADY + status.ContainerStatuses = []*kubecontainer.ContainerStatus{} + }, + actions: podActions{ + KillPod: true, + CreateSandbox: true, + SandboxID: baseStatus.SandboxStatuses[0].Id, + Attempt: uint32(1), + NextInitContainerToStart: &basePod.Spec.InitContainers[0], + ContainersToStart: []int{}, + ContainersToKill: getKillMapWithInitContainers(basePod, baseStatus, []int{}), + }, + }, } { pod, status := makeBasePodAndStatusWithInitContainers() if test.mutatePodFn != nil { diff --git a/pkg/kubelet/stats/cri_stats_provider_windows.go b/pkg/kubelet/stats/cri_stats_provider_windows.go index cc9934e7827d1..d1892e929984e 100644 --- a/pkg/kubelet/stats/cri_stats_provider_windows.go +++ b/pkg/kubelet/stats/cri_stats_provider_windows.go @@ -19,6 +19,7 @@ limitations under the License. package stats import ( + "fmt" "time" "github.com/Microsoft/hcsshim" @@ -40,18 +41,11 @@ func (p *criStatsProvider) listContainerNetworkStats() (map[string]*statsapi.Net stats := make(map[string]*statsapi.NetworkStats) for _, c := range containers { - container, err := hcsshim.OpenContainer(c.ID) + cstats, err := fetchContainerStats(c) if err != nil { - klog.V(4).Infof("Failed to open container %q with error '%v', continue to get stats for other containers", c.ID, err) + klog.V(4).Infof("Failed to fetch statistics for container %q with error '%v', continue to get stats for other containers", c.ID, err) continue } - - cstats, err := container.Statistics() - if err != nil { - klog.V(4).Infof("Failed to get statistics for container %q with error '%v', continue to get stats for other containers", c.ID, err) - continue - } - if len(cstats.Network) > 0 { stats[c.ID] = hcsStatsToNetworkStats(cstats.Timestamp, cstats.Network) } @@ -60,6 +54,27 @@ func (p *criStatsProvider) listContainerNetworkStats() (map[string]*statsapi.Net return stats, nil } +func fetchContainerStats(c hcsshim.ContainerProperties) (stats hcsshim.Statistics, err error) { + var ( + container hcsshim.Container + ) + container, err = hcsshim.OpenContainer(c.ID) + if err != nil { + return + } + defer func() { + if closeErr := container.Close(); closeErr != nil { + if err != nil { + err = fmt.Errorf("failed to close container after error %v; close error: %v", err, closeErr) + } else { + err = closeErr + } + } + }() + + return container.Statistics() +} + // hcsStatsToNetworkStats converts hcsshim.Statistics.Network to statsapi.NetworkStats func hcsStatsToNetworkStats(timestamp time.Time, hcsStats []hcsshim.NetworkStats) *statsapi.NetworkStats { result := &statsapi.NetworkStats{ diff --git a/pkg/kubelet/volume_host.go b/pkg/kubelet/volume_host.go index 5b2077f348d78..4b2e51844cfb1 100644 --- a/pkg/kubelet/volume_host.go +++ b/pkg/kubelet/volume_host.go @@ -26,8 +26,12 @@ import ( authenticationv1 "k8s.io/api/authentication/v1" "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/types" + "k8s.io/apimachinery/pkg/util/wait" utilfeature "k8s.io/apiserver/pkg/util/feature" + "k8s.io/client-go/informers" clientset "k8s.io/client-go/kubernetes" + storagelisters "k8s.io/client-go/listers/storage/v1beta1" + "k8s.io/client-go/tools/cache" "k8s.io/client-go/tools/record" cloudprovider "k8s.io/cloud-provider" "k8s.io/kubernetes/pkg/features" @@ -56,6 +60,24 @@ func NewInitializedVolumePluginMgr( plugins []volume.VolumePlugin, prober volume.DynamicPluginProber) (*volume.VolumePluginMgr, error) { + // Initialize csiDriverLister before calling InitPlugins + var informerFactory informers.SharedInformerFactory + var csiDriverLister storagelisters.CSIDriverLister + var csiDriversSynced cache.InformerSynced + const resyncPeriod = 0 + if utilfeature.DefaultFeatureGate.Enabled(features.CSIDriverRegistry) { + // Don't initialize if kubeClient is nil + if kubelet.kubeClient != nil { + informerFactory = informers.NewSharedInformerFactory(kubelet.kubeClient, resyncPeriod) + csiDriverInformer := informerFactory.Storage().V1beta1().CSIDrivers() + csiDriverLister = csiDriverInformer.Lister() + csiDriversSynced = csiDriverInformer.Informer().HasSynced + + } else { + klog.Warning("kubeClient is nil. Skip initialization of CSIDriverLister") + } + } + mountPodManager, err := mountpod.NewManager(kubelet.getRootDir(), kubelet.podManager) if err != nil { return nil, err @@ -67,6 +89,9 @@ func NewInitializedVolumePluginMgr( configMapManager: configMapManager, tokenManager: tokenManager, mountPodManager: mountPodManager, + informerFactory: informerFactory, + csiDriverLister: csiDriverLister, + csiDriversSynced: csiDriversSynced, } if err := kvh.volumePluginMgr.InitPlugins(plugins, prober, kvh); err != nil { @@ -93,6 +118,9 @@ type kubeletVolumeHost struct { tokenManager *token.Manager configMapManager configmap.Manager mountPodManager mountpod.Manager + informerFactory informers.SharedInformerFactory + csiDriverLister storagelisters.CSIDriverLister + csiDriversSynced cache.InformerSynced } func (kvh *kubeletVolumeHost) SetKubeletError(err error) { @@ -131,6 +159,34 @@ func (kvh *kubeletVolumeHost) GetSubpather() subpath.Interface { return kvh.kubelet.subpather } +func (kvh *kubeletVolumeHost) GetInformerFactory() informers.SharedInformerFactory { + return kvh.informerFactory +} + +func (kvh *kubeletVolumeHost) CSIDriverLister() storagelisters.CSIDriverLister { + return kvh.csiDriverLister +} + +func (kvh *kubeletVolumeHost) CSIDriversSynced() cache.InformerSynced { + return kvh.csiDriversSynced +} + +// WaitForCacheSync is a helper function that waits for cache sync for CSIDriverLister +func (kvh *kubeletVolumeHost) WaitForCacheSync() error { + if kvh.csiDriversSynced == nil { + klog.Error("csiDriversSynced not found on KubeletVolumeHost") + return fmt.Errorf("csiDriversSynced not found on KubeletVolumeHost") + } + + synced := []cache.InformerSynced{kvh.csiDriversSynced} + if !cache.WaitForCacheSync(wait.NeverStop, synced...) { + klog.Warning("failed to wait for cache sync for CSIDriverLister") + return fmt.Errorf("failed to wait for cache sync for CSIDriverLister") + } + + return nil +} + func (kvh *kubeletVolumeHost) NewWrapperMounter( volName string, spec volume.Spec, diff --git a/pkg/kubelet/volumemanager/volume_manager.go b/pkg/kubelet/volumemanager/volume_manager.go index 561733d592c98..652e67a69f950 100644 --- a/pkg/kubelet/volumemanager/volume_manager.go +++ b/pkg/kubelet/volumemanager/volume_manager.go @@ -250,6 +250,11 @@ func (vm *volumeManager) Run(sourcesReady config.SourcesReady, stopCh <-chan str metrics.Register(vm.actualStateOfWorld, vm.desiredStateOfWorld, vm.volumePluginMgr) + if vm.kubeClient != nil { + // start informer for CSIDriver + vm.volumePluginMgr.Run(stopCh) + } + <-stopCh klog.Infof("Shutting down Kubelet Volume Manager") } diff --git a/pkg/proxy/BUILD b/pkg/proxy/BUILD index a7969c25c8640..0da6d0a31016f 100644 --- a/pkg/proxy/BUILD +++ b/pkg/proxy/BUILD @@ -17,6 +17,7 @@ go_library( importpath = "k8s.io/kubernetes/pkg/proxy", deps = [ "//pkg/api/v1/service:go_default_library", + "//pkg/proxy/metrics:go_default_library", "//pkg/proxy/util:go_default_library", "//staging/src/k8s.io/api/core/v1:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/types:go_default_library", diff --git a/pkg/proxy/endpoints.go b/pkg/proxy/endpoints.go index 21f10cdcbf42f..954abdb0f3da1 100644 --- a/pkg/proxy/endpoints.go +++ b/pkg/proxy/endpoints.go @@ -29,6 +29,7 @@ import ( "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/sets" "k8s.io/client-go/tools/record" + "k8s.io/kubernetes/pkg/proxy/metrics" utilproxy "k8s.io/kubernetes/pkg/proxy/util" utilnet "k8s.io/utils/net" ) @@ -127,6 +128,7 @@ func (ect *EndpointChangeTracker) Update(previous, current *v1.Endpoints) bool { if endpoints == nil { return false } + metrics.EndpointChangesTotal.Inc() namespacedName := types.NamespacedName{Namespace: endpoints.Namespace, Name: endpoints.Name} ect.lock.Lock() @@ -154,6 +156,8 @@ func (ect *EndpointChangeTracker) Update(previous, current *v1.Endpoints) bool { // should be exported. delete(ect.lastChangeTriggerTimes, namespacedName) } + + metrics.EndpointChangesPending.Set(float64(len(ect.items))) return len(ect.items) > 0 } @@ -295,6 +299,7 @@ func (endpointsMap EndpointsMap) apply(changes *EndpointChangeTracker, staleEndp detectStaleConnections(change.previous, change.current, staleEndpoints, staleServiceNames) } changes.items = make(map[types.NamespacedName]*endpointsChange) + metrics.EndpointChangesPending.Set(0) for _, lastChangeTriggerTime := range changes.lastChangeTriggerTimes { *lastChangeTriggerTimes = append(*lastChangeTriggerTimes, lastChangeTriggerTime...) } diff --git a/pkg/proxy/iptables/proxier.go b/pkg/proxy/iptables/proxier.go index 1a19eab5a9e48..5d18aa9b354dc 100644 --- a/pkg/proxy/iptables/proxier.go +++ b/pkg/proxy/iptables/proxier.go @@ -1387,6 +1387,7 @@ func (proxier *Proxier) syncProxyRules() { if proxier.healthzServer != nil { proxier.healthzServer.UpdateTimestamp() } + metrics.SyncProxyRulesLastTimestamp.SetToCurrentTime() // Update healthchecks. The endpoints list might include services that are // not "OnlyLocal", but the services list will not, and the healthChecker diff --git a/pkg/proxy/ipvs/graceful_termination.go b/pkg/proxy/ipvs/graceful_termination.go index 712ba0377819b..1076fe4794d66 100644 --- a/pkg/proxy/ipvs/graceful_termination.go +++ b/pkg/proxy/ipvs/graceful_termination.go @@ -18,6 +18,7 @@ package ipvs import ( "fmt" + "strings" "sync" "time" @@ -167,7 +168,7 @@ func (m *GracefulTerminationManager) deleteRsFunc(rsToDelete *listItem) (bool, e // For UDP traffic, no graceful termination, we immediately delete the RS // (existing connections will be deleted on the next packet because sysctlExpireNoDestConn=1) // For other protocols, don't delete until all connections have expired) - if rsToDelete.VirtualServer.Protocol != "udp" && rs.ActiveConn+rs.InactiveConn != 0 { + if strings.ToUpper(rsToDelete.VirtualServer.Protocol) != "UDP" && rs.ActiveConn+rs.InactiveConn != 0 { klog.Infof("Not deleting, RS %v: %v ActiveConn, %v InactiveConn", rsToDelete.String(), rs.ActiveConn, rs.InactiveConn) return false, nil } diff --git a/pkg/proxy/ipvs/proxier.go b/pkg/proxy/ipvs/proxier.go index 0ec34753d740b..3e19753d5883b 100644 --- a/pkg/proxy/ipvs/proxier.go +++ b/pkg/proxy/ipvs/proxier.go @@ -1245,6 +1245,7 @@ func (proxier *Proxier) syncProxyRules() { if proxier.healthzServer != nil { proxier.healthzServer.UpdateTimestamp() } + metrics.SyncProxyRulesLastTimestamp.SetToCurrentTime() // Update healthchecks. The endpoints list might include services that are // not "OnlyLocal", but the services list will not, and the healthChecker diff --git a/pkg/proxy/metrics/metrics.go b/pkg/proxy/metrics/metrics.go index 38924387ff8e7..54d7f0a4ca46e 100644 --- a/pkg/proxy/metrics/metrics.go +++ b/pkg/proxy/metrics/metrics.go @@ -46,6 +46,16 @@ var ( }, ) + // SyncProxyRulesLastTimestamp is the timestamp proxy rules were last + // successfully synced. + SyncProxyRulesLastTimestamp = prometheus.NewGauge( + prometheus.GaugeOpts{ + Subsystem: kubeProxySubsystem, + Name: "sync_proxy_rules_last_timestamp_seconds", + Help: "The last time proxy rules were successfully synced", + }, + ) + // NetworkProgrammingLatency is defined as the time it took to program the network - from the time // the service or pod has changed to the time the change was propagated and the proper kube-proxy // rules were synced. Exported for each endpoints object that were part of the rules sync. @@ -63,6 +73,46 @@ var ( Buckets: prometheus.ExponentialBuckets(0.001, 2, 20), }, ) + + // EndpointChangesPending is the number of pending endpoint changes that + // have not yet been synced to the proxy. + EndpointChangesPending = prometheus.NewGauge( + prometheus.GaugeOpts{ + Subsystem: kubeProxySubsystem, + Name: "sync_proxy_rules_endpoint_changes_pending", + Help: "Pending proxy rules Endpoint changes", + }, + ) + + // EndpointChangesTotal is the number of endpoint changes that the proxy + // has seen. + EndpointChangesTotal = prometheus.NewCounter( + prometheus.CounterOpts{ + Subsystem: kubeProxySubsystem, + Name: "sync_proxy_rules_endpoint_changes_total", + Help: "Cumulative proxy rules Endpoint changes", + }, + ) + + // ServiceChangesPending is the number of pending service changes that + // have not yet been synced to the proxy. + ServiceChangesPending = prometheus.NewGauge( + prometheus.GaugeOpts{ + Subsystem: kubeProxySubsystem, + Name: "sync_proxy_rules_service_changes_pending", + Help: "Pending proxy rules Service changes", + }, + ) + + // ServiceChangesTotal is the number of service changes that the proxy has + // seen. + ServiceChangesTotal = prometheus.NewCounter( + prometheus.CounterOpts{ + Subsystem: kubeProxySubsystem, + Name: "sync_proxy_rules_service_changes_total", + Help: "Cumulative proxy rules Service changes", + }, + ) ) var registerMetricsOnce sync.Once @@ -72,7 +122,12 @@ func RegisterMetrics() { registerMetricsOnce.Do(func() { prometheus.MustRegister(SyncProxyRulesLatency) prometheus.MustRegister(DeprecatedSyncProxyRulesLatency) + prometheus.MustRegister(SyncProxyRulesLastTimestamp) prometheus.MustRegister(NetworkProgrammingLatency) + prometheus.MustRegister(EndpointChangesPending) + prometheus.MustRegister(EndpointChangesTotal) + prometheus.MustRegister(ServiceChangesPending) + prometheus.MustRegister(ServiceChangesTotal) }) } diff --git a/pkg/proxy/service.go b/pkg/proxy/service.go index 2d5bd352f5350..02e500d0ef60b 100644 --- a/pkg/proxy/service.go +++ b/pkg/proxy/service.go @@ -30,6 +30,7 @@ import ( "k8s.io/apimachinery/pkg/util/sets" "k8s.io/client-go/tools/record" apiservice "k8s.io/kubernetes/pkg/api/v1/service" + "k8s.io/kubernetes/pkg/proxy/metrics" utilproxy "k8s.io/kubernetes/pkg/proxy/util" utilnet "k8s.io/utils/net" ) @@ -198,6 +199,7 @@ func (sct *ServiceChangeTracker) Update(previous, current *v1.Service) bool { if svc == nil { return false } + metrics.ServiceChangesTotal.Inc() namespacedName := types.NamespacedName{Namespace: svc.Namespace, Name: svc.Name} sct.lock.Lock() @@ -214,6 +216,7 @@ func (sct *ServiceChangeTracker) Update(previous, current *v1.Service) bool { if reflect.DeepEqual(change.previous, change.current) { delete(sct.items, namespacedName) } + metrics.ServiceChangesPending.Set(float64(len(sct.items))) return len(sct.items) > 0 } @@ -296,6 +299,7 @@ func (serviceMap *ServiceMap) apply(changes *ServiceChangeTracker, UDPStaleClust } // clear changes after applying them to ServiceMap. changes.items = make(map[types.NamespacedName]*serviceChange) + metrics.ServiceChangesPending.Set(0) return } diff --git a/pkg/proxy/winkernel/metrics.go b/pkg/proxy/winkernel/metrics.go index 61cf962ee0eee..729cc5e626f61 100644 --- a/pkg/proxy/winkernel/metrics.go +++ b/pkg/proxy/winkernel/metrics.go @@ -43,6 +43,16 @@ var ( Buckets: prometheus.ExponentialBuckets(1000, 2, 15), }, ) + + // SyncProxyRulesLastTimestamp is the timestamp proxy rules were last + // successfully synced. + SyncProxyRulesLastTimestamp = prometheus.NewGauge( + prometheus.GaugeOpts{ + Subsystem: kubeProxySubsystem, + Name: "sync_proxy_rules_last_timestamp_seconds", + Help: "The last time proxy rules were successfully synced", + }, + ) ) var registerMetricsOnce sync.Once @@ -51,6 +61,7 @@ func RegisterMetrics() { registerMetricsOnce.Do(func() { prometheus.MustRegister(SyncProxyRulesLatency) prometheus.MustRegister(DeprecatedSyncProxyRulesLatency) + prometheus.MustRegister(SyncProxyRulesLastTimestamp) }) } diff --git a/pkg/proxy/winkernel/proxier.go b/pkg/proxy/winkernel/proxier.go index 7f6eec06cf999..a6f10812c01cc 100644 --- a/pkg/proxy/winkernel/proxier.go +++ b/pkg/proxy/winkernel/proxier.go @@ -1197,6 +1197,7 @@ func (proxier *Proxier) syncProxyRules() { if proxier.healthzServer != nil { proxier.healthzServer.UpdateTimestamp() } + SyncProxyRulesLastTimestamp.SetToCurrentTime() // Update healthchecks. The endpoints list might include services that are // not "OnlyLocal", but the services list will not, and the healthChecker diff --git a/pkg/util/mount/mount_linux.go b/pkg/util/mount/mount_linux.go index 9ffd766a51b35..a8a52241a7d68 100644 --- a/pkg/util/mount/mount_linux.go +++ b/pkg/util/mount/mount_linux.go @@ -234,7 +234,7 @@ func (mounter *Mounter) IsLikelyNotMountPoint(file string) (bool, error) { if err != nil { return true, err } - rootStat, err := os.Lstat(filepath.Dir(strings.TrimSuffix(file, "/"))) + rootStat, err := os.Stat(filepath.Dir(strings.TrimSuffix(file, "/"))) if err != nil { return true, err } diff --git a/pkg/volume/BUILD b/pkg/volume/BUILD index dc4a16cbd6196..fd001aee03594 100644 --- a/pkg/volume/BUILD +++ b/pkg/volume/BUILD @@ -18,6 +18,7 @@ go_library( importpath = "k8s.io/kubernetes/pkg/volume", visibility = ["//visibility:public"], deps = [ + "//pkg/features:go_default_library", "//pkg/util/mount:go_default_library", "//pkg/volume/util/fs:go_default_library", "//pkg/volume/util/recyclerclient:go_default_library", @@ -29,8 +30,11 @@ go_library( "//staging/src/k8s.io/apimachinery/pkg/types:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/util/errors:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/util/validation:go_default_library", + "//staging/src/k8s.io/apiserver/pkg/util/feature:go_default_library", + "//staging/src/k8s.io/client-go/informers:go_default_library", "//staging/src/k8s.io/client-go/kubernetes:go_default_library", "//staging/src/k8s.io/client-go/listers/storage/v1beta1:go_default_library", + "//staging/src/k8s.io/client-go/tools/cache:go_default_library", "//staging/src/k8s.io/client-go/tools/record:go_default_library", "//staging/src/k8s.io/cloud-provider:go_default_library", "//vendor/k8s.io/klog:go_default_library", diff --git a/pkg/volume/csi/BUILD b/pkg/volume/csi/BUILD index 71e07796af6e8..93de7cbc1dd32 100644 --- a/pkg/volume/csi/BUILD +++ b/pkg/volume/csi/BUILD @@ -31,8 +31,6 @@ go_library( "//staging/src/k8s.io/apimachinery/pkg/util/wait:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/watch:go_default_library", "//staging/src/k8s.io/apiserver/pkg/util/feature:go_default_library", - "//staging/src/k8s.io/client-go/informers:go_default_library", - "//staging/src/k8s.io/client-go/informers/storage/v1beta1:go_default_library", "//staging/src/k8s.io/client-go/kubernetes:go_default_library", "//staging/src/k8s.io/client-go/listers/storage/v1beta1:go_default_library", "//staging/src/k8s.io/csi-translation-lib/plugins:go_default_library", @@ -73,6 +71,7 @@ go_test( "//staging/src/k8s.io/apimachinery/pkg/watch:go_default_library", "//staging/src/k8s.io/apiserver/pkg/util/feature:go_default_library", "//staging/src/k8s.io/apiserver/pkg/util/feature/testing:go_default_library", + "//staging/src/k8s.io/client-go/informers:go_default_library", "//staging/src/k8s.io/client-go/kubernetes:go_default_library", "//staging/src/k8s.io/client-go/kubernetes/fake:go_default_library", "//staging/src/k8s.io/client-go/testing:go_default_library", diff --git a/pkg/volume/csi/csi_attacher_test.go b/pkg/volume/csi/csi_attacher_test.go index 1de09af066224..688963ec77044 100644 --- a/pkg/volume/csi/csi_attacher_test.go +++ b/pkg/volume/csi/csi_attacher_test.go @@ -33,6 +33,7 @@ import ( "k8s.io/apimachinery/pkg/watch" utilfeature "k8s.io/apiserver/pkg/util/feature" utilfeaturetesting "k8s.io/apiserver/pkg/util/feature/testing" + "k8s.io/client-go/informers" clientset "k8s.io/client-go/kubernetes" fakeclient "k8s.io/client-go/kubernetes/fake" core "k8s.io/client-go/testing" @@ -1435,12 +1436,20 @@ func newTestWatchPlugin(t *testing.T, fakeClient *fakeclient.Clientset) (*csiPlu fakeClient = fakeclient.NewSimpleClientset() } fakeWatcher := watch.NewRaceFreeFake() - fakeClient.Fake.AddWatchReactor("*", core.DefaultWatchReactor(fakeWatcher, nil)) + fakeClient.Fake.PrependWatchReactor("volumeattachments", core.DefaultWatchReactor(fakeWatcher, nil)) + + // Start informer for CSIDrivers. + factory := informers.NewSharedInformerFactory(fakeClient, csiResyncPeriod) + csiDriverInformer := factory.Storage().V1beta1().CSIDrivers() + csiDriverLister := csiDriverInformer.Lister() + go factory.Start(wait.NeverStop) + host := volumetest.NewFakeVolumeHostWithCSINodeName( tmpDir, fakeClient, nil, "node", + csiDriverLister, ) plugMgr := &volume.VolumePluginMgr{} plugMgr.InitPlugins(ProbeVolumePlugins(), nil /* prober */, host) @@ -1458,7 +1467,7 @@ func newTestWatchPlugin(t *testing.T, fakeClient *fakeclient.Clientset) (*csiPlu if utilfeature.DefaultFeatureGate.Enabled(features.CSIDriverRegistry) { // Wait until the informer in CSI volume plugin has all CSIDrivers. wait.PollImmediate(testInformerSyncPeriod, testInformerSyncTimeout, func() (bool, error) { - return csiPlug.csiDriverInformer.Informer().HasSynced(), nil + return csiDriverInformer.Informer().HasSynced(), nil }) } diff --git a/pkg/volume/csi/csi_block_test.go b/pkg/volume/csi/csi_block_test.go index 9bc2b8c63fc15..f8090310e51f0 100644 --- a/pkg/volume/csi/csi_block_test.go +++ b/pkg/volume/csi/csi_block_test.go @@ -216,6 +216,7 @@ func TestBlockMapperSetupDevice(t *testing.T) { fakeClient, nil, "fakeNode", + nil, ) plug.host = host @@ -282,6 +283,7 @@ func TestBlockMapperMapDevice(t *testing.T) { fakeClient, nil, "fakeNode", + nil, ) plug.host = host @@ -364,6 +366,7 @@ func TestBlockMapperTearDownDevice(t *testing.T) { fakeClient, nil, "fakeNode", + nil, ) plug.host = host diff --git a/pkg/volume/csi/csi_mounter.go b/pkg/volume/csi/csi_mounter.go index ccdce2ace2fa3..c64c91baa144a 100644 --- a/pkg/volume/csi/csi_mounter.go +++ b/pkg/volume/csi/csi_mounter.go @@ -19,7 +19,6 @@ package csi import ( "context" "crypto/sha256" - "errors" "fmt" "os" "path" @@ -292,8 +291,14 @@ func (c *csiMountMgr) podAttributes() (map[string]string, error) { if !utilfeature.DefaultFeatureGate.Enabled(features.CSIDriverRegistry) { return nil, nil } + + kletHost, ok := c.plugin.host.(volume.KubeletVolumeHost) + if ok { + kletHost.WaitForCacheSync() + } + if c.plugin.csiDriverLister == nil { - return nil, errors.New("CSIDriver lister does not exist") + return nil, fmt.Errorf("CSIDriverLister not found") } csiDriver, err := c.plugin.csiDriverLister.Get(string(c.driverName)) diff --git a/pkg/volume/csi/csi_mounter_test.go b/pkg/volume/csi/csi_mounter_test.go index e7f3edc7411b7..3e1ecd8f125e1 100644 --- a/pkg/volume/csi/csi_mounter_test.go +++ b/pkg/volume/csi/csi_mounter_test.go @@ -33,7 +33,6 @@ import ( storagev1beta1 "k8s.io/api/storage/v1beta1" meta "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" - "k8s.io/apimachinery/pkg/util/wait" utilfeature "k8s.io/apiserver/pkg/util/feature" utilfeaturetesting "k8s.io/apiserver/pkg/util/feature/testing" fakeclient "k8s.io/client-go/kubernetes/fake" @@ -155,13 +154,6 @@ func MounterSetUpTests(t *testing.T, podInfoEnabled bool) { plug, tmpDir := newTestPlugin(t, fakeClient) defer os.RemoveAll(tmpDir) - if utilfeature.DefaultFeatureGate.Enabled(features.CSIDriverRegistry) { - // Wait until the informer in CSI volume plugin has all CSIDrivers. - wait.PollImmediate(testInformerSyncPeriod, testInformerSyncTimeout, func() (bool, error) { - return plug.csiDriverInformer.Informer().HasSynced(), nil - }) - } - registerFakePlugin(test.driver, "endpoint", []string{"1.0.0"}, t) pv := makeTestPV("test-pv", 10, test.driver, testVol) pv.Spec.CSI.VolumeAttributes = test.volumeContext @@ -391,6 +383,7 @@ func TestMounterSetUpSimple(t *testing.T) { }) } } + func TestMounterSetUpWithInline(t *testing.T) { defer utilfeaturetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.CSIInlineVolume, true)() @@ -527,6 +520,7 @@ func TestMounterSetUpWithInline(t *testing.T) { }) } } + func TestMounterSetUpWithFSGroup(t *testing.T) { fakeClient := fakeclient.NewSimpleClientset() plug, tmpDir := newTestPlugin(t, fakeClient) diff --git a/pkg/volume/csi/csi_plugin.go b/pkg/volume/csi/csi_plugin.go index 3590b5726b0c3..c43ec006e01af 100644 --- a/pkg/volume/csi/csi_plugin.go +++ b/pkg/volume/csi/csi_plugin.go @@ -37,10 +37,8 @@ import ( utilversion "k8s.io/apimachinery/pkg/util/version" "k8s.io/apimachinery/pkg/util/wait" utilfeature "k8s.io/apiserver/pkg/util/feature" - csiapiinformer "k8s.io/client-go/informers" - csiinformer "k8s.io/client-go/informers/storage/v1beta1" clientset "k8s.io/client-go/kubernetes" - csilister "k8s.io/client-go/listers/storage/v1beta1" + storagelisters "k8s.io/client-go/listers/storage/v1beta1" csitranslationplugins "k8s.io/csi-translation-lib/plugins" "k8s.io/kubernetes/pkg/features" "k8s.io/kubernetes/pkg/volume" @@ -68,10 +66,9 @@ const ( var deprecatedSocketDirVersions = []string{"0.1.0", "0.2.0", "0.3.0", "0.4.0"} type csiPlugin struct { - host volume.VolumeHost - blockEnabled bool - csiDriverLister csilister.CSIDriverLister - csiDriverInformer csiinformer.CSIDriverInformer + host volume.VolumeHost + blockEnabled bool + csiDriverLister storagelisters.CSIDriverLister } //TODO (vladimirvivien) add this type to storage api @@ -217,11 +214,21 @@ func (p *csiPlugin) Init(host volume.VolumeHost) error { if csiClient == nil { klog.Warning(log("kubeclient not set, assuming standalone kubelet")) } else { - // Start informer for CSIDrivers. - factory := csiapiinformer.NewSharedInformerFactory(csiClient, csiResyncPeriod) - p.csiDriverInformer = factory.Storage().V1beta1().CSIDrivers() - p.csiDriverLister = p.csiDriverInformer.Lister() - go factory.Start(wait.NeverStop) + // set CSIDriverLister + adcHost, ok := host.(volume.AttachDetachVolumeHost) + if ok { + p.csiDriverLister = adcHost.CSIDriverLister() + if p.csiDriverLister == nil { + klog.Error(log("CSIDriverLister not found on AttachDetachVolumeHost")) + } + } + kletHost, ok := host.(volume.KubeletVolumeHost) + if ok { + p.csiDriverLister = kletHost.CSIDriverLister() + if p.csiDriverLister == nil { + klog.Error(log("CSIDriverLister not found on KubeletVolumeHost")) + } + } } } @@ -752,6 +759,12 @@ func (p *csiPlugin) skipAttach(driver string) (bool, error) { if !utilfeature.DefaultFeatureGate.Enabled(features.CSIDriverRegistry) { return false, nil } + + kletHost, ok := p.host.(volume.KubeletVolumeHost) + if ok { + kletHost.WaitForCacheSync() + } + if p.csiDriverLister == nil { return false, errors.New("CSIDriver lister does not exist") } diff --git a/pkg/volume/csi/csi_plugin_test.go b/pkg/volume/csi/csi_plugin_test.go index 4747ba4db36bd..4043178e01bac 100644 --- a/pkg/volume/csi/csi_plugin_test.go +++ b/pkg/volume/csi/csi_plugin_test.go @@ -31,6 +31,7 @@ import ( "k8s.io/apimachinery/pkg/util/wait" utilfeature "k8s.io/apiserver/pkg/util/feature" utilfeaturetesting "k8s.io/apiserver/pkg/util/feature/testing" + "k8s.io/client-go/informers" fakeclient "k8s.io/client-go/kubernetes/fake" utiltesting "k8s.io/client-go/util/testing" "k8s.io/kubernetes/pkg/features" @@ -48,11 +49,19 @@ func newTestPlugin(t *testing.T, client *fakeclient.Clientset) (*csiPlugin, stri if client == nil { client = fakeclient.NewSimpleClientset() } + + // Start informer for CSIDrivers. + factory := informers.NewSharedInformerFactory(client, csiResyncPeriod) + csiDriverInformer := factory.Storage().V1beta1().CSIDrivers() + csiDriverLister := csiDriverInformer.Lister() + go factory.Start(wait.NeverStop) + host := volumetest.NewFakeVolumeHostWithCSINodeName( tmpDir, client, nil, "fakeNode", + csiDriverLister, ) plugMgr := &volume.VolumePluginMgr{} plugMgr.InitPlugins(ProbeVolumePlugins(), nil /* prober */, host) @@ -70,7 +79,7 @@ func newTestPlugin(t *testing.T, client *fakeclient.Clientset) (*csiPlugin, stri if utilfeature.DefaultFeatureGate.Enabled(features.CSIDriverRegistry) { // Wait until the informer in CSI volume plugin has all CSIDrivers. wait.PollImmediate(testInformerSyncPeriod, testInformerSyncTimeout, func() (bool, error) { - return csiPlug.csiDriverInformer.Informer().HasSynced(), nil + return csiDriverInformer.Informer().HasSynced(), nil }) } diff --git a/pkg/volume/csi/nodeinfomanager/nodeinfomanager_test.go b/pkg/volume/csi/nodeinfomanager/nodeinfomanager_test.go index 71946f8edbb49..7429448e3c643 100644 --- a/pkg/volume/csi/nodeinfomanager/nodeinfomanager_test.go +++ b/pkg/volume/csi/nodeinfomanager/nodeinfomanager_test.go @@ -740,6 +740,7 @@ func TestInstallCSIDriverExistingAnnotation(t *testing.T) { client, nil, nodeName, + nil, ) nim := NewNodeInfoManager(types.NodeName(nodeName), host, nil) @@ -799,6 +800,7 @@ func test(t *testing.T, addNodeInfo bool, csiNodeInfoEnabled bool, testcases []t client, nil, nodeName, + nil, ) nim := NewNodeInfoManager(types.NodeName(nodeName), host, nil) diff --git a/pkg/volume/flexvolume/unmounter.go b/pkg/volume/flexvolume/unmounter.go index 3da5f77e81f76..9435f2c2c4623 100644 --- a/pkg/volume/flexvolume/unmounter.go +++ b/pkg/volume/flexvolume/unmounter.go @@ -42,15 +42,15 @@ func (f *flexVolumeUnmounter) TearDown() error { } func (f *flexVolumeUnmounter) TearDownAt(dir string) error { - pathExists, pathErr := mount.PathExists(dir) - if !pathExists { - klog.Warningf("Warning: Unmount skipped because path does not exist: %v", dir) - return nil - } - - if pathErr != nil && !mount.IsCorruptedMnt(pathErr) { - return fmt.Errorf("Error checking path: %v", pathErr) + if pathErr != nil { + // only log warning here since plugins should anyways have to deal with errors + klog.Warningf("Error checking path: %v", pathErr) + } else { + if !pathExists { + klog.Warningf("Warning: Unmount skipped because path does not exist: %v", dir) + return nil + } } call := f.plugin.NewDriverCall(unmountCmd) diff --git a/pkg/volume/plugins.go b/pkg/volume/plugins.go index c26bc9a4f2832..9a77ae070e9e5 100644 --- a/pkg/volume/plugins.go +++ b/pkg/volume/plugins.go @@ -29,11 +29,15 @@ import ( "k8s.io/apimachinery/pkg/types" utilerrors "k8s.io/apimachinery/pkg/util/errors" "k8s.io/apimachinery/pkg/util/validation" + utilfeature "k8s.io/apiserver/pkg/util/feature" + "k8s.io/client-go/informers" clientset "k8s.io/client-go/kubernetes" storagelisters "k8s.io/client-go/listers/storage/v1beta1" + "k8s.io/client-go/tools/cache" "k8s.io/client-go/tools/record" cloudprovider "k8s.io/cloud-provider" "k8s.io/klog" + "k8s.io/kubernetes/pkg/features" "k8s.io/kubernetes/pkg/util/mount" "k8s.io/kubernetes/pkg/volume/util/recyclerclient" "k8s.io/kubernetes/pkg/volume/util/subpath" @@ -319,6 +323,15 @@ type KubeletVolumeHost interface { // SetKubeletError lets plugins set an error on the Kubelet runtime status // that will cause the Kubelet to post NotReady status with the error message provided SetKubeletError(err error) + + // GetInformerFactory returns the informer factory for CSIDriverLister + GetInformerFactory() informers.SharedInformerFactory + // CSIDriverLister returns the informer lister for the CSIDriver API Object + CSIDriverLister() storagelisters.CSIDriverLister + // CSIDriverSynced returns the informer synced for the CSIDriver API Object + CSIDriversSynced() cache.InformerSynced + // WaitForCacheSync is a helper function that waits for cache sync for CSIDriverLister + WaitForCacheSync() error } // AttachDetachVolumeHost is a AttachDetach Controller specific interface that plugins can use @@ -327,6 +340,9 @@ type AttachDetachVolumeHost interface { // CSINodeLister returns the informer lister for the CSINode API Object CSINodeLister() storagelisters.CSINodeLister + // CSIDriverLister returns the informer lister for the CSIDriver API Object + CSIDriverLister() storagelisters.CSIDriverLister + // IsAttachDetachController is an interface marker to strictly tie AttachDetachVolumeHost // to the attachDetachController IsAttachDetachController() bool @@ -1004,6 +1020,17 @@ func (pm *VolumePluginMgr) FindNodeExpandablePluginByName(name string) (NodeExpa return nil, nil } +func (pm *VolumePluginMgr) Run(stopCh <-chan struct{}) { + kletHost, ok := pm.Host.(KubeletVolumeHost) + if ok { + // start informer for CSIDriver + if utilfeature.DefaultFeatureGate.Enabled(features.CSIDriverRegistry) { + informerFactory := kletHost.GetInformerFactory() + go informerFactory.Start(stopCh) + } + } +} + // NewPersistentVolumeRecyclerPodTemplate creates a template for a recycler // pod. By default, a recycler pod simply runs "rm -rf" on a volume and tests // for emptiness. Most attributes of the template will be correct for most diff --git a/pkg/volume/testing/BUILD b/pkg/volume/testing/BUILD index b454ee05f5bbe..0994e3df2161d 100644 --- a/pkg/volume/testing/BUILD +++ b/pkg/volume/testing/BUILD @@ -26,6 +26,7 @@ go_library( "//staging/src/k8s.io/apimachinery/pkg/types:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/util/uuid:go_default_library", "//staging/src/k8s.io/client-go/kubernetes:go_default_library", + "//staging/src/k8s.io/client-go/listers/storage/v1beta1:go_default_library", "//staging/src/k8s.io/client-go/tools/record:go_default_library", "//staging/src/k8s.io/client-go/util/testing:go_default_library", "//staging/src/k8s.io/cloud-provider:go_default_library", diff --git a/pkg/volume/testing/testing.go b/pkg/volume/testing/testing.go index b7b5396d2e5f2..2201b9b681daf 100644 --- a/pkg/volume/testing/testing.go +++ b/pkg/volume/testing/testing.go @@ -34,6 +34,7 @@ import ( "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/uuid" clientset "k8s.io/client-go/kubernetes" + storagelisters "k8s.io/client-go/listers/storage/v1beta1" "k8s.io/client-go/tools/record" utiltesting "k8s.io/client-go/util/testing" cloudprovider "k8s.io/cloud-provider" @@ -62,17 +63,21 @@ const ( // fakeVolumeHost is useful for testing volume plugins. type fakeVolumeHost struct { - rootDir string - kubeClient clientset.Interface - pluginMgr VolumePluginMgr - cloud cloudprovider.Interface - mounter mount.Interface - exec mount.Exec - nodeLabels map[string]string - nodeName string - subpather subpath.Interface + rootDir string + kubeClient clientset.Interface + pluginMgr VolumePluginMgr + cloud cloudprovider.Interface + mounter mount.Interface + exec mount.Exec + nodeLabels map[string]string + nodeName string + subpather subpath.Interface + csiDriverLister storagelisters.CSIDriverLister } +var _ VolumeHost = &fakeVolumeHost{} +var _ AttachDetachVolumeHost = &fakeVolumeHost{} + func NewFakeVolumeHost(rootDir string, kubeClient clientset.Interface, plugins []VolumePlugin) *fakeVolumeHost { return newFakeVolumeHost(rootDir, kubeClient, plugins, nil, nil) } @@ -87,9 +92,12 @@ func NewFakeVolumeHostWithNodeLabels(rootDir string, kubeClient clientset.Interf return volHost } -func NewFakeVolumeHostWithCSINodeName(rootDir string, kubeClient clientset.Interface, plugins []VolumePlugin, nodeName string) *fakeVolumeHost { +func NewFakeVolumeHostWithCSINodeName(rootDir string, kubeClient clientset.Interface, plugins []VolumePlugin, nodeName string, driverLister storagelisters.CSIDriverLister) *fakeVolumeHost { volHost := newFakeVolumeHost(rootDir, kubeClient, plugins, nil, nil) volHost.nodeName = nodeName + if driverLister != nil { + volHost.csiDriverLister = driverLister + } return volHost } @@ -1469,3 +1477,16 @@ func ContainsAccessMode(modes []v1.PersistentVolumeAccessMode, mode v1.Persisten } return false } + +func (f *fakeVolumeHost) CSIDriverLister() storagelisters.CSIDriverLister { + return f.csiDriverLister +} + +func (f *fakeVolumeHost) CSINodeLister() storagelisters.CSINodeLister { + // not needed for testing + return nil +} + +func (f *fakeVolumeHost) IsAttachDetachController() bool { + return true +} diff --git a/pkg/volume/util/subpath/subpath_linux.go b/pkg/volume/util/subpath/subpath_linux.go index 977f7d2da4d29..bc5f697929b76 100644 --- a/pkg/volume/util/subpath/subpath_linux.go +++ b/pkg/volume/util/subpath/subpath_linux.go @@ -398,7 +398,7 @@ func doSafeMakeDir(pathname string, base string, perm os.FileMode) error { return fmt.Errorf("cannot create directory %s: %s", currentPath, err) } // Dive into the created directory - childFD, err := syscall.Openat(parentFD, dir, nofollowFlags, 0) + childFD, err = syscall.Openat(parentFD, dir, nofollowFlags, 0) if err != nil { return fmt.Errorf("cannot open %s: %s", currentPath, err) } diff --git a/staging/src/k8s.io/apiserver/pkg/server/config_selfclient_test.go b/staging/src/k8s.io/apiserver/pkg/server/config_selfclient_test.go index 9374403ff9dcc..053ac4de02441 100644 --- a/staging/src/k8s.io/apiserver/pkg/server/config_selfclient_test.go +++ b/staging/src/k8s.io/apiserver/pkg/server/config_selfclient_test.go @@ -59,7 +59,9 @@ func TestLoopbackHostPort(t *testing.T) { if err != nil { t.Fatalf("unexpected error: %v", err) } - if ip := net.ParseIP(host); ip == nil || !ip.IsLoopback() || ip.To4() != nil { + if host == "localhost" { + // can happen on machines without IPv6 + } else if ip := net.ParseIP(host); ip == nil || !ip.IsLoopback() || ip.To4() != nil { t.Fatalf("expected IPv6 host to be loopback, got %q", host) } diff --git a/staging/src/k8s.io/apiserver/pkg/server/healthz/doc.go b/staging/src/k8s.io/apiserver/pkg/server/healthz/doc.go index 06e67f6fe3cce..d938caa371392 100644 --- a/staging/src/k8s.io/apiserver/pkg/server/healthz/doc.go +++ b/staging/src/k8s.io/apiserver/pkg/server/healthz/doc.go @@ -17,5 +17,5 @@ limitations under the License. // Package healthz implements basic http server health checking. // Usage: // import "k8s.io/apiserver/pkg/server/healthz" -// healthz.DefaultHealthz() +// healthz.InstallHandler(mux) package healthz // import "k8s.io/apiserver/pkg/server/healthz" diff --git a/staging/src/k8s.io/apiserver/pkg/server/healthz/healthz.go b/staging/src/k8s.io/apiserver/pkg/server/healthz/healthz.go index ebb3dadfb79e8..a54dcdb75e809 100644 --- a/staging/src/k8s.io/apiserver/pkg/server/healthz/healthz.go +++ b/staging/src/k8s.io/apiserver/pkg/server/healthz/healthz.go @@ -37,15 +37,6 @@ type HealthzChecker interface { Check(req *http.Request) error } -var defaultHealthz = sync.Once{} - -// DefaultHealthz installs the default healthz check to the http.DefaultServeMux. -func DefaultHealthz(checks ...HealthzChecker) { - defaultHealthz.Do(func() { - InstallHandler(http.DefaultServeMux, checks...) - }) -} - // PingHealthz returns true automatically when checked var PingHealthz HealthzChecker = ping{} diff --git a/test/e2e/framework/deployment_util.go b/test/e2e/framework/deployment_util.go index a525d47ecc6e3..9c84aefdd80cf 100644 --- a/test/e2e/framework/deployment_util.go +++ b/test/e2e/framework/deployment_util.go @@ -107,8 +107,9 @@ func NewDeployment(deploymentName string, replicas int32, podLabels map[string]s TerminationGracePeriodSeconds: &zero, Containers: []v1.Container{ { - Name: imageName, - Image: image, + Name: imageName, + Image: image, + SecurityContext: &v1.SecurityContext{}, }, }, }, diff --git a/test/e2e/framework/jobs_util.go b/test/e2e/framework/jobs_util.go index a5839dc36f8a7..e54be2829e262 100644 --- a/test/e2e/framework/jobs_util.go +++ b/test/e2e/framework/jobs_util.go @@ -83,6 +83,7 @@ func NewTestJob(behavior, name string, rPol v1.RestartPolicy, parallelism, compl Name: "data", }, }, + SecurityContext: &v1.SecurityContext{}, }, }, }, diff --git a/test/e2e/framework/rs_util.go b/test/e2e/framework/rs_util.go index cfdf0975ad522..93bb2e43902b3 100644 --- a/test/e2e/framework/rs_util.go +++ b/test/e2e/framework/rs_util.go @@ -148,8 +148,9 @@ func NewReplicaSet(name, namespace string, replicas int32, podLabels map[string] Spec: v1.PodSpec{ Containers: []v1.Container{ { - Name: imageName, - Image: image, + Name: imageName, + Image: image, + SecurityContext: &v1.SecurityContext{}, }, }, }, diff --git a/test/e2e/framework/statefulset_utils.go b/test/e2e/framework/statefulset_utils.go index 24d8a39964a18..6bcde3c0b2efe 100644 --- a/test/e2e/framework/statefulset_utils.go +++ b/test/e2e/framework/statefulset_utils.go @@ -807,9 +807,10 @@ func NewStatefulSet(name, ns, governingSvcName string, replicas int32, statefulP Spec: v1.PodSpec{ Containers: []v1.Container{ { - Name: "nginx", - Image: imageutils.GetE2EImage(imageutils.Nginx), - VolumeMounts: mounts, + Name: "nginx", + Image: imageutils.GetE2EImage(imageutils.Nginx), + VolumeMounts: mounts, + SecurityContext: &v1.SecurityContext{}, }, }, Volumes: vols, diff --git a/test/e2e/framework/volume_util.go b/test/e2e/framework/volume_util.go index 45fe4f1ce6bcc..e9948a72d5ddb 100644 --- a/test/e2e/framework/volume_util.go +++ b/test/e2e/framework/volume_util.go @@ -522,10 +522,8 @@ func InjectHtml(client clientset.Interface, config VolumeTestConfig, fsGroup *in SecurityContext: GenerateSecurityContext(true), }, }, - SecurityContext: &v1.PodSecurityContext{ - FSGroup: fsGroup, - }, - RestartPolicy: v1.RestartPolicyNever, + SecurityContext: GeneratePodSecurityContext(fsGroup, nil), + RestartPolicy: v1.RestartPolicyNever, Volumes: []v1.Volume{ { Name: volMountName, diff --git a/test/e2e/kubectl/BUILD b/test/e2e/kubectl/BUILD index eb665e57e7c45..954c436309f61 100644 --- a/test/e2e/kubectl/BUILD +++ b/test/e2e/kubectl/BUILD @@ -37,6 +37,7 @@ go_library( "//test/utils/crd:go_default_library", "//test/utils/image:go_default_library", "//vendor/github.com/elazarl/goproxy:go_default_library", + "//vendor/github.com/googleapis/gnostic/OpenAPIv2:go_default_library", "//vendor/github.com/onsi/ginkgo:go_default_library", "//vendor/github.com/onsi/gomega:go_default_library", "//vendor/golang.org/x/net/websocket:go_default_library", diff --git a/test/e2e/kubectl/kubectl.go b/test/e2e/kubectl/kubectl.go index 3f197b2b64b9f..e959c8ed1e0b1 100644 --- a/test/e2e/kubectl/kubectl.go +++ b/test/e2e/kubectl/kubectl.go @@ -40,6 +40,7 @@ import ( "time" "github.com/elazarl/goproxy" + openapi_v2 "github.com/googleapis/gnostic/OpenAPIv2" "sigs.k8s.io/yaml" "k8s.io/api/core/v1" @@ -821,6 +822,56 @@ metadata: }) }) + // definitionMatchesGVK returns true if the specified GVK is listed as an x-kubernetes-group-version-kind extension + definitionMatchesGVK := func(extensions []*openapi_v2.NamedAny, desiredGVK schema.GroupVersionKind) bool { + for _, extension := range extensions { + if extension.GetValue().GetYaml() == "" || + extension.GetName() != "x-kubernetes-group-version-kind" { + continue + } + var values []map[string]string + err := yaml.Unmarshal([]byte(extension.GetValue().GetYaml()), &values) + if err != nil { + framework.Logf("%v\n%s", err, string(extension.GetValue().GetYaml())) + continue + } + for _, value := range values { + if value["group"] != desiredGVK.Group { + continue + } + if value["version"] != desiredGVK.Version { + continue + } + if value["kind"] != desiredGVK.Kind { + continue + } + return true + } + } + return false + } + + // schemaForGVK returns a schema (if defined) for the specified GVK + schemaForGVK := func(desiredGVK schema.GroupVersionKind) *openapi_v2.Schema { + d, err := f.ClientSet.Discovery().OpenAPISchema() + if err != nil { + framework.Failf("%v", err) + } + if d == nil || d.Definitions == nil { + return nil + } + for _, p := range d.Definitions.AdditionalProperties { + if p == nil || p.Value == nil { + continue + } + if !definitionMatchesGVK(p.Value.VendorExtension, desiredGVK) { + continue + } + return p.Value + } + return nil + } + framework.KubeDescribe("Kubectl client-side validation", func() { ginkgo.It("should create/apply a CR with unknown fields for CRD with no validation schema", func() { ginkgo.By("create CRD with no validation schema") @@ -875,10 +926,28 @@ metadata: ginkgo.By("sleep for 10s to wait for potential crd openapi publishing alpha feature") time.Sleep(10 * time.Second) + publishedSchema := schemaForGVK(schema.GroupVersionKind{Group: crd.APIGroup, Version: crd.Versions[0].Name, Kind: crd.Kind}) + expectSuccess := false + if publishedSchema == nil || publishedSchema.Properties == nil || publishedSchema.Properties.AdditionalProperties == nil || len(publishedSchema.Properties.AdditionalProperties) == 0 { + // expect success in the following cases: + // - no schema was published + // - a schema was published with no properties + expectSuccess = true + framework.Logf("no schema with properties found, expect apply with extra properties to succeed") + } else { + framework.Logf("schema with properties found, expect apply with extra properties to fail") + } + meta := fmt.Sprintf(metaPattern, crd.Kind, crd.APIGroup, crd.Versions[0].Name, "test-cr") validArbitraryCR := fmt.Sprintf(`{%s,"spec":{"bars":[{"name":"test-bar"}],"extraProperty":"arbitrary-value"}}`, meta) if err := createApplyCustomResource(validArbitraryCR, f.Namespace.Name, "test-cr", crd); err != nil { - framework.Failf("%v", err) + if expectSuccess { + framework.Failf("%v", err) + } + } else { + if !expectSuccess { + framework.Failf("expected error, got none") + } } }) diff --git a/test/e2e/network/dns_configmap.go b/test/e2e/network/dns_configmap.go index 5426daba44319..52418fda6e74d 100644 --- a/test/e2e/network/dns_configmap.go +++ b/test/e2e/network/dns_configmap.go @@ -35,7 +35,7 @@ type dnsFederationsConfigMapTest struct { } var ( - googleDnsHostname = "google-public-dns-a.google.com" + googleDnsHostname = "dns.google" // The ConfigMap update mechanism takes longer than the standard // wait.ForeverTestTimeout. moreForeverTestTimeout = 2 * 60 * time.Second diff --git a/test/e2e/storage/vsphere/vsphere_volume_datastore.go b/test/e2e/storage/vsphere/vsphere_volume_datastore.go index dfd497fa551ea..137852636dbcd 100644 --- a/test/e2e/storage/vsphere/vsphere_volume_datastore.go +++ b/test/e2e/storage/vsphere/vsphere_volume_datastore.go @@ -70,7 +70,7 @@ var _ = utils.SIGDescribe("Volume Provisioning on Datastore [Feature:vsphere]", scParameters[DiskFormat] = ThinDisk err := invokeInvalidDatastoreTestNeg(client, namespace, scParameters) Expect(err).To(HaveOccurred()) - errorMsg := `Failed to provision volume with StorageClass \"` + DatastoreSCName + `\": The specified datastore ` + InvalidDatastore + ` is not a shared datastore across node VMs` + errorMsg := `Failed to provision volume with StorageClass \"` + DatastoreSCName + `\": Datastore ` + InvalidDatastore + ` not found` if !strings.Contains(err.Error(), errorMsg) { Expect(err).NotTo(HaveOccurred(), errorMsg) } diff --git a/test/e2e/storage/vsphere/vsphere_zone_support.go b/test/e2e/storage/vsphere/vsphere_zone_support.go index 5ad8cf1b49cf5..0d270f3310e55 100644 --- a/test/e2e/storage/vsphere/vsphere_zone_support.go +++ b/test/e2e/storage/vsphere/vsphere_zone_support.go @@ -176,6 +176,13 @@ var _ = utils.SIGDescribe("Zone Support", func() { verifyPVCAndPodCreationSucceeds(client, namespace, scParameters, zones) }) + It("Verify a pod is created on a non-Workspace zone and attached to a dynamically created PV, based on the allowed zones and storage policy specified in storage class", func() { + By(fmt.Sprintf("Creating storage class with zone :%s and storage policy :%s", zoneB, compatPolicy)) + scParameters[SpbmStoragePolicy] = compatPolicy + zones = append(zones, zoneB) + verifyPVCAndPodCreationSucceeds(client, namespace, scParameters, zones) + }) + It("Verify PVC creation with incompatible storagePolicy and zone combination specified in storage class fails", func() { By(fmt.Sprintf("Creating storage class with zone :%s and storage policy :%s", zoneA, nonCompatPolicy)) scParameters[SpbmStoragePolicy] = nonCompatPolicy diff --git a/test/e2e/upgrades/apps/daemonsets.go b/test/e2e/upgrades/apps/daemonsets.go index 98997480a4b86..4529a25a8b30e 100644 --- a/test/e2e/upgrades/apps/daemonsets.go +++ b/test/e2e/upgrades/apps/daemonsets.go @@ -65,9 +65,10 @@ func (t *DaemonSetUpgradeTest) Setup(f *framework.Framework) { }, Containers: []v1.Container{ { - Name: daemonSetName, - Image: image, - Ports: []v1.ContainerPort{{ContainerPort: 9376}}, + Name: daemonSetName, + Image: image, + Ports: []v1.ContainerPort{{ContainerPort: 9376}}, + SecurityContext: &v1.SecurityContext{}, }, }, }, diff --git a/test/integration/volume/BUILD b/test/integration/volume/BUILD index 27785393155c6..4325654c414ff 100644 --- a/test/integration/volume/BUILD +++ b/test/integration/volume/BUILD @@ -21,6 +21,7 @@ go_test( "//pkg/controller/volume/attachdetach/cache:go_default_library", "//pkg/controller/volume/persistentvolume:go_default_library", "//pkg/controller/volume/persistentvolume/options:go_default_library", + "//pkg/features:go_default_library", "//pkg/volume:go_default_library", "//pkg/volume/testing:go_default_library", "//pkg/volume/util:go_default_library", @@ -31,6 +32,7 @@ go_test( "//staging/src/k8s.io/apimachinery/pkg/runtime/schema:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/util/wait:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/watch:go_default_library", + "//staging/src/k8s.io/apiserver/pkg/util/feature:go_default_library", "//staging/src/k8s.io/client-go/informers:go_default_library", "//staging/src/k8s.io/client-go/kubernetes:go_default_library", "//staging/src/k8s.io/client-go/rest:go_default_library", diff --git a/test/integration/volume/attach_detach_test.go b/test/integration/volume/attach_detach_test.go index 39722abc4daaa..6f1bb9c47cd05 100644 --- a/test/integration/volume/attach_detach_test.go +++ b/test/integration/volume/attach_detach_test.go @@ -27,7 +27,8 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/apimachinery/pkg/util/wait" - "k8s.io/client-go/informers" + utilfeature "k8s.io/apiserver/pkg/util/feature" + clientgoinformers "k8s.io/client-go/informers" clientset "k8s.io/client-go/kubernetes" restclient "k8s.io/client-go/rest" "k8s.io/client-go/tools/cache" @@ -36,6 +37,7 @@ import ( volumecache "k8s.io/kubernetes/pkg/controller/volume/attachdetach/cache" "k8s.io/kubernetes/pkg/controller/volume/persistentvolume" persistentvolumeoptions "k8s.io/kubernetes/pkg/controller/volume/persistentvolume/options" + "k8s.io/kubernetes/pkg/features" "k8s.io/kubernetes/pkg/volume" volumetest "k8s.io/kubernetes/pkg/volume/testing" "k8s.io/kubernetes/pkg/volume/util" @@ -179,6 +181,7 @@ func TestPodDeletionWithDswp(t *testing.T) { stopCh := make(chan struct{}) go informers.Core().V1().PersistentVolumeClaims().Informer().Run(stopCh) go informers.Core().V1().PersistentVolumes().Informer().Run(stopCh) + initCSIObjects(stopCh, informers) go ctrl.Run(stopCh) waitToObservePods(t, podInformer, 1) @@ -207,6 +210,16 @@ func TestPodDeletionWithDswp(t *testing.T) { close(stopCh) } +func initCSIObjects(stopCh chan struct{}, informers clientgoinformers.SharedInformerFactory) { + if utilfeature.DefaultFeatureGate.Enabled(features.CSIMigration) && + utilfeature.DefaultFeatureGate.Enabled(features.CSINodeInfo) { + go informers.Storage().V1beta1().CSINodes().Informer().Run(stopCh) + } + if utilfeature.DefaultFeatureGate.Enabled(features.CSIDriverRegistry) { + go informers.Storage().V1beta1().CSIDrivers().Informer().Run(stopCh) + } +} + func TestPodUpdateWithWithADC(t *testing.T) { _, server, closeFn := framework.RunAMaster(framework.NewIntegrationTestMasterConfig()) defer closeFn() @@ -246,6 +259,7 @@ func TestPodUpdateWithWithADC(t *testing.T) { stopCh := make(chan struct{}) go informers.Core().V1().PersistentVolumeClaims().Informer().Run(stopCh) go informers.Core().V1().PersistentVolumes().Informer().Run(stopCh) + initCSIObjects(stopCh, informers) go ctrl.Run(stopCh) waitToObservePods(t, podInformer, 1) @@ -314,6 +328,7 @@ func TestPodUpdateWithKeepTerminatedPodVolumes(t *testing.T) { stopCh := make(chan struct{}) go informers.Core().V1().PersistentVolumeClaims().Informer().Run(stopCh) go informers.Core().V1().PersistentVolumes().Informer().Run(stopCh) + initCSIObjects(stopCh, informers) go ctrl.Run(stopCh) waitToObservePods(t, podInformer, 1) @@ -383,7 +398,7 @@ func waitForPodFuncInDSWP(t *testing.T, dswp volumecache.DesiredStateOfWorld, ch } } -func createAdClients(ns *v1.Namespace, t *testing.T, server *httptest.Server, syncPeriod time.Duration, timers attachdetach.TimerConfig) (*clientset.Clientset, attachdetach.AttachDetachController, *persistentvolume.PersistentVolumeController, informers.SharedInformerFactory) { +func createAdClients(ns *v1.Namespace, t *testing.T, server *httptest.Server, syncPeriod time.Duration, timers attachdetach.TimerConfig) (*clientset.Clientset, attachdetach.AttachDetachController, *persistentvolume.PersistentVolumeController, clientgoinformers.SharedInformerFactory) { config := restclient.Config{ Host: server.URL, ContentConfig: restclient.ContentConfig{GroupVersion: &schema.GroupVersion{Group: "", Version: "v1"}}, @@ -408,7 +423,7 @@ func createAdClients(ns *v1.Namespace, t *testing.T, server *httptest.Server, sy } plugins := []volume.VolumePlugin{plugin} cloud := &fakecloud.FakeCloud{} - informers := informers.NewSharedInformerFactory(testClient, resyncPeriod) + informers := clientgoinformers.NewSharedInformerFactory(testClient, resyncPeriod) ctrl, err := attachdetach.NewAttachDetachController( testClient, informers.Core().V1().Pods(), @@ -416,6 +431,7 @@ func createAdClients(ns *v1.Namespace, t *testing.T, server *httptest.Server, sy informers.Core().V1().PersistentVolumeClaims(), informers.Core().V1().PersistentVolumes(), informers.Storage().V1beta1().CSINodes(), + informers.Storage().V1beta1().CSIDrivers(), cloud, plugins, nil, /* prober */ @@ -491,6 +507,7 @@ func TestPodAddedByDswp(t *testing.T) { stopCh := make(chan struct{}) go informers.Core().V1().PersistentVolumeClaims().Informer().Run(stopCh) go informers.Core().V1().PersistentVolumes().Informer().Run(stopCh) + initCSIObjects(stopCh, informers) go ctrl.Run(stopCh) waitToObservePods(t, podInformer, 1) @@ -576,6 +593,7 @@ func TestPVCBoundWithADC(t *testing.T) { stopCh := make(chan struct{}) informers.Start(stopCh) informers.WaitForCacheSync(stopCh) + initCSIObjects(stopCh, informers) go ctrl.Run(stopCh) go pvCtrl.Run(stopCh)