diff --git a/go.mod b/go.mod index 0463550c5d..90003fe2b1 100644 --- a/go.mod +++ b/go.mod @@ -32,7 +32,7 @@ require ( k8s.io/kubernetes v1.26.0 k8s.io/mount-utils v0.0.0 k8s.io/utils v0.0.0-20221128185143-99ec85e7a448 - sigs.k8s.io/cloud-provider-azure v1.26.1-0.20230112064854-e451894aa471 + sigs.k8s.io/cloud-provider-azure v1.26.1-0.20230117121033-188ebdf2a417 sigs.k8s.io/yaml v1.3.0 ) diff --git a/go.sum b/go.sum index 1bc77d9219..e3229a0a22 100644 --- a/go.sum +++ b/go.sum @@ -871,8 +871,8 @@ rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.0.33 h1:LYqFq+6Cj2D0gFfrJvL7iElD4ET6ir3VDdhDdTK7rgc= sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.0.33/go.mod h1:soWkSNf2tZC7aMibXEqVhCd73GOY5fJikn8qbdzemB0= -sigs.k8s.io/cloud-provider-azure v1.26.1-0.20230112064854-e451894aa471 h1:/QNVf/3CB10y5iNHAZivrjdHQJoIiXBGn2kJ9fvwceU= -sigs.k8s.io/cloud-provider-azure v1.26.1-0.20230112064854-e451894aa471/go.mod h1:8sMPMqXtfiD4MgBmkpiXHzTEtYVWqS3wBuVUZb/mRmk= +sigs.k8s.io/cloud-provider-azure v1.26.1-0.20230117121033-188ebdf2a417 h1:RkEi97kDD73Ae0sKNQ75lFamksv25vQtuNkixoYR+og= +sigs.k8s.io/cloud-provider-azure v1.26.1-0.20230117121033-188ebdf2a417/go.mod h1:8sMPMqXtfiD4MgBmkpiXHzTEtYVWqS3wBuVUZb/mRmk= sigs.k8s.io/json v0.0.0-20220713155537-f223a00ba0e2 h1:iXTIw73aPyC+oRdyqqvVJuloN1p0AC/kzH07hu3NE+k= sigs.k8s.io/json v0.0.0-20220713155537-f223a00ba0e2/go.mod h1:B8JuhiUyNFVKdsE8h686QcCxMaH6HrOAZj4vswFpcB0= sigs.k8s.io/structured-merge-diff/v4 v4.2.3 h1:PRbqxJClWWYMNV1dhaG4NsibJbArud9kFxnAMREiWFE= diff --git a/vendor/modules.txt b/vendor/modules.txt index f773a46233..72d71b1495 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -1227,7 +1227,7 @@ k8s.io/utils/trace ## explicit; go 1.17 sigs.k8s.io/apiserver-network-proxy/konnectivity-client/pkg/client sigs.k8s.io/apiserver-network-proxy/konnectivity-client/proto/client -# sigs.k8s.io/cloud-provider-azure v1.26.1-0.20230112064854-e451894aa471 +# sigs.k8s.io/cloud-provider-azure v1.26.1-0.20230117121033-188ebdf2a417 ## explicit; go 1.19 sigs.k8s.io/cloud-provider-azure/pkg/azureclients sigs.k8s.io/cloud-provider-azure/pkg/azureclients/armclient diff --git a/vendor/sigs.k8s.io/cloud-provider-azure/pkg/consts/consts.go b/vendor/sigs.k8s.io/cloud-provider-azure/pkg/consts/consts.go index 80d26c353b..6fac548c91 100644 --- a/vendor/sigs.k8s.io/cloud-provider-azure/pkg/consts/consts.go +++ b/vendor/sigs.k8s.io/cloud-provider-azure/pkg/consts/consts.go @@ -360,9 +360,6 @@ const ( // LoadBalancerBackendPoolConfigurationTypePODIP is the lb backend pool config type pod ip // TODO (nilo19): support pod IP in the future LoadBalancerBackendPoolConfigurationTypePODIP = "podIP" - - // To get pip, we need both resource group name and pip name, key in cache has format: pip_rg:pip_name - PIPCacheKeySeparator = ":" ) // error messages diff --git a/vendor/sigs.k8s.io/cloud-provider-azure/pkg/provider/azure.go b/vendor/sigs.k8s.io/cloud-provider-azure/pkg/provider/azure.go index 2047d7da9c..d852c4cc05 100644 --- a/vendor/sigs.k8s.io/cloud-provider-azure/pkg/provider/azure.go +++ b/vendor/sigs.k8s.io/cloud-provider-azure/pkg/provider/azure.go @@ -363,6 +363,9 @@ type Cloud struct { lbCache *azcache.TimedCache nsgCache *azcache.TimedCache rtCache *azcache.TimedCache + // public ip cache + // key: [resourceGroupName] + // Value: sync.Map of [pipName]*PublicIPAddress pipCache *azcache.TimedCache // use LB frontEndIpConfiguration ID as the key and search for PLS attached to the frontEnd plsCache *azcache.TimedCache diff --git a/vendor/sigs.k8s.io/cloud-provider-azure/pkg/provider/azure_backoff.go b/vendor/sigs.k8s.io/cloud-provider-azure/pkg/provider/azure_backoff.go index 396fb956cc..d1a824cd76 100644 --- a/vendor/sigs.k8s.io/cloud-provider-azure/pkg/provider/azure_backoff.go +++ b/vendor/sigs.k8s.io/cloud-provider-azure/pkg/provider/azure_backoff.go @@ -385,25 +385,6 @@ func (az *Cloud) ListLB(service *v1.Service) ([]network.LoadBalancer, error) { return allLBs, nil } -// ListPIP list the PIP resources in the given resource group -func (az *Cloud) ListPIP(service *v1.Service, pipResourceGroup string) ([]network.PublicIPAddress, error) { - ctx, cancel := getContextWithCancel() - defer cancel() - - allPIPs, rerr := az.PublicIPAddressesClient.List(ctx, pipResourceGroup) - if rerr != nil { - if rerr.IsNotFound() { - return nil, nil - } - az.Event(service, v1.EventTypeWarning, "ListPublicIPs", rerr.Error().Error()) - klog.Errorf("PublicIPAddressesClient.List(%v) failure with err=%v", pipResourceGroup, rerr) - return nil, rerr.Error() - } - - klog.V(2).Infof("PublicIPAddressesClient.List(%v) success", pipResourceGroup) - return allPIPs, nil -} - // CreateOrUpdatePIP invokes az.PublicIPAddressesClient.CreateOrUpdate with exponential backoff retry func (az *Cloud) CreateOrUpdatePIP(service *v1.Service, pipResourceGroup string, pip network.PublicIPAddress) error { ctx, cancel := getContextWithCancel() @@ -413,7 +394,7 @@ func (az *Cloud) CreateOrUpdatePIP(service *v1.Service, pipResourceGroup string, klog.V(10).Infof("PublicIPAddressesClient.CreateOrUpdate(%s, %s): end", pipResourceGroup, pointer.StringDeref(pip.Name, "")) if rerr == nil { // Invalidate the cache right after updating - _ = az.pipCache.Delete(az.getPIPCacheKey(pipResourceGroup, pointer.StringDeref(pip.Name, ""))) + _ = az.pipCache.Delete(pipResourceGroup) return nil } @@ -424,14 +405,14 @@ func (az *Cloud) CreateOrUpdatePIP(service *v1.Service, pipResourceGroup string, // Invalidate the cache because ETAG precondition mismatch. if rerr.HTTPStatusCode == http.StatusPreconditionFailed { klog.V(3).Infof("PublicIP cache for (%s, %s) is cleanup because of http.StatusPreconditionFailed", pipResourceGroup, pointer.StringDeref(pip.Name, "")) - _ = az.pipCache.Delete(az.getPIPCacheKey(pipResourceGroup, pointer.StringDeref(pip.Name, ""))) + _ = az.pipCache.Delete(pipResourceGroup) } retryErrorMessage := rerr.Error().Error() // Invalidate the cache because another new operation has canceled the current request. if strings.Contains(strings.ToLower(retryErrorMessage), consts.OperationCanceledErrorMessage) { klog.V(3).Infof("PublicIP cache for (%s, %s) is cleanup because CreateOrUpdate is canceled by another operation", pipResourceGroup, pointer.StringDeref(pip.Name, "")) - _ = az.pipCache.Delete(az.getPIPCacheKey(pipResourceGroup, pointer.StringDeref(pip.Name, ""))) + _ = az.pipCache.Delete(pipResourceGroup) } return rerr.Error() @@ -471,7 +452,7 @@ func (az *Cloud) DeletePublicIP(service *v1.Service, pipResourceGroup string, pi } // Invalidate the cache right after deleting - _ = az.pipCache.Delete(az.getPIPCacheKey(pipResourceGroup, pipName)) + _ = az.pipCache.Delete(pipResourceGroup) return nil } diff --git a/vendor/sigs.k8s.io/cloud-provider-azure/pkg/provider/azure_loadbalancer.go b/vendor/sigs.k8s.io/cloud-provider-azure/pkg/provider/azure_loadbalancer.go index ebd8c96076..3d839f967f 100644 --- a/vendor/sigs.k8s.io/cloud-provider-azure/pkg/provider/azure_loadbalancer.go +++ b/vendor/sigs.k8s.io/cloud-provider-azure/pkg/provider/azure_loadbalancer.go @@ -923,7 +923,7 @@ func (az *Cloud) findMatchedPIPByLoadBalancerIP(service *v1.Service, loadBalance // this should not happen return nil, fmt.Errorf("findMatchedPIPByLoadBalancerIP: nil pip list passed") } else if *pips == nil { - pipList, err := az.ListPIP(service, pipResourceGroup) + pipList, err := az.listPIP(pipResourceGroup) if err != nil { return nil, err } @@ -3032,7 +3032,7 @@ func (az *Cloud) reconcilePublicIP(clusterName string, service *v1.Service, lbNa pipResourceGroup := az.getPublicIPAddressResourceGroup(service) - pips, err := az.ListPIP(service, pipResourceGroup) + pips, err := az.listPIP(pipResourceGroup) if err != nil { return nil, err } diff --git a/vendor/sigs.k8s.io/cloud-provider-azure/pkg/provider/azure_loadbalancer_backendpool.go b/vendor/sigs.k8s.io/cloud-provider-azure/pkg/provider/azure_loadbalancer_backendpool.go index 68362de2af..8471f11992 100644 --- a/vendor/sigs.k8s.io/cloud-provider-azure/pkg/provider/azure_loadbalancer_backendpool.go +++ b/vendor/sigs.k8s.io/cloud-provider-azure/pkg/provider/azure_loadbalancer_backendpool.go @@ -34,6 +34,7 @@ import ( "sigs.k8s.io/cloud-provider-azure/pkg/cache" "sigs.k8s.io/cloud-provider-azure/pkg/consts" + "sigs.k8s.io/cloud-provider-azure/pkg/metrics" ) type BackendPool interface { @@ -141,9 +142,7 @@ func (bc *backendPoolTypeNodeIPConfig) ReconcileBackendPools(clusterName string, newBackendPools = *lb.BackendAddressPools } - foundBackendPool := false - changed := false - shouldRefreshLB := false + var foundBackendPool, changed, shouldRefreshLB, isOperationSucceeded, isMigration bool lbName := *lb.Name serviceName := getServiceName(service) @@ -152,6 +151,8 @@ func (bc *backendPoolTypeNodeIPConfig) ReconcileBackendPools(clusterName string, vmSetName := bc.mapLoadBalancerNameToVMSet(lbName, clusterName) isBackendPoolPreConfigured := bc.isBackendPoolPreConfigured(service) + mc := metrics.NewMetricContext("services", "migrate_to_ip_based_backend_pool", bc.ResourceGroup, bc.getNetworkResourceSubscriptionID(), serviceName) + for i := len(newBackendPools) - 1; i >= 0; i-- { bp := newBackendPools[i] if strings.EqualFold(*bp.Name, lbBackendPoolName) { @@ -169,6 +170,8 @@ func (bc *backendPoolTypeNodeIPConfig) ReconcileBackendPools(clusterName string, if bp.BackendAddressPoolPropertiesFormat != nil && bp.LoadBalancerBackendAddresses != nil && len(*bp.LoadBalancerBackendAddresses) > 0 { + isMigration = true + if removeNodeIPAddressesFromBackendPool(bp, []string{}, true) { if err := bc.CreateOrUpdateLBBackendPool(lbName, bp); err != nil { klog.Errorf("bc.ReconcileBackendPools for service (%s): failed to cleanup IP based backend pool %s: %s", serviceName, lbBackendPoolName, err.Error()) @@ -246,6 +249,13 @@ func (bc *backendPoolTypeNodeIPConfig) ReconcileBackendPools(clusterName string, changed = true } + if isMigration { + defer func() { + mc.ObserveOperationWithResult(isOperationSucceeded) + }() + } + + isOperationSucceeded = true return isBackendPoolPreConfigured, changed, err } @@ -449,9 +459,7 @@ func (bi *backendPoolTypeNodeIP) ReconcileBackendPools(clusterName string, servi newBackendPools = *lb.BackendAddressPools } - foundBackendPool := false - changed := false - shouldRefreshLB := false + var foundBackendPool, changed, shouldRefreshLB, isOperationSucceeded, isMigration bool lbName := *lb.Name serviceName := getServiceName(service) lbBackendPoolName := getBackendPoolName(clusterName, service) @@ -459,6 +467,8 @@ func (bi *backendPoolTypeNodeIP) ReconcileBackendPools(clusterName string, servi lbBackendPoolID := bi.getBackendPoolID(pointer.StringDeref(lb.Name, ""), bi.getLoadBalancerResourceGroup(), getBackendPoolName(clusterName, service)) isBackendPoolPreConfigured := bi.isBackendPoolPreConfigured(service) + mc := metrics.NewMetricContext("services", "migrate_to_nic_based_backend_pool", bi.ResourceGroup, bi.getNetworkResourceSubscriptionID(), serviceName) + var err error for i := len(newBackendPools) - 1; i >= 0; i-- { bp := newBackendPools[i] @@ -489,6 +499,8 @@ func (bi *backendPoolTypeNodeIP) ReconcileBackendPools(clusterName string, servi } } if len(nodeIPAddressesToBeDeleted) > 0 { + isMigration = true + updated := removeNodeIPAddressesFromBackendPool(bp, nodeIPAddressesToBeDeleted, false) if updated { (*lb.BackendAddressPools)[i] = bp @@ -516,6 +528,13 @@ func (bi *backendPoolTypeNodeIP) ReconcileBackendPools(clusterName string, servi changed = true } + if isMigration { + defer func() { + mc.ObserveOperationWithResult(isOperationSucceeded) + }() + } + + isOperationSucceeded = true return isBackendPoolPreConfigured, changed, nil } diff --git a/vendor/sigs.k8s.io/cloud-provider-azure/pkg/provider/azure_storageaccount.go b/vendor/sigs.k8s.io/cloud-provider-azure/pkg/provider/azure_storageaccount.go index fc6a3a2d9b..9493c10e8e 100644 --- a/vendor/sigs.k8s.io/cloud-provider-azure/pkg/provider/azure_storageaccount.go +++ b/vendor/sigs.k8s.io/cloud-provider-azure/pkg/provider/azure_storageaccount.go @@ -642,17 +642,11 @@ func isTagsEqual(account storage.Account, accountOptions *AccountOptions) bool { } func isHnsPropertyEqual(account storage.Account, accountOptions *AccountOptions) bool { - if accountOptions.IsHnsEnabled == nil { - return true - } - return *accountOptions.IsHnsEnabled == pointer.BoolDeref(account.IsHnsEnabled, false) + return pointer.BoolDeref(accountOptions.IsHnsEnabled, false) == pointer.BoolDeref(account.IsHnsEnabled, false) } func isEnableNfsV3PropertyEqual(account storage.Account, accountOptions *AccountOptions) bool { - if accountOptions.EnableNfsV3 == nil { - return true - } - return *accountOptions.EnableNfsV3 == pointer.BoolDeref(account.EnableNfsV3, false) + return pointer.BoolDeref(accountOptions.EnableNfsV3, false) == pointer.BoolDeref(account.EnableNfsV3, false) } func isPrivateEndpointAsExpected(account storage.Account, accountOptions *AccountOptions) bool { @@ -666,27 +660,19 @@ func isPrivateEndpointAsExpected(account storage.Account, accountOptions *Accoun } func isAllowBlobPublicAccessEqual(account storage.Account, accountOptions *AccountOptions) bool { - if accountOptions.AllowBlobPublicAccess == nil { - return true - } - return *accountOptions.AllowBlobPublicAccess == pointer.BoolDeref(account.AllowBlobPublicAccess, false) + return pointer.BoolDeref(accountOptions.AllowBlobPublicAccess, false) == pointer.BoolDeref(account.AllowBlobPublicAccess, false) } func isRequireInfrastructureEncryptionEqual(account storage.Account, accountOptions *AccountOptions) bool { - if accountOptions.RequireInfrastructureEncryption == nil { - return true - } + requireInfraEncryption := pointer.BoolDeref(accountOptions.RequireInfrastructureEncryption, false) if account.Encryption == nil { - return !*accountOptions.RequireInfrastructureEncryption + return !requireInfraEncryption } - return *accountOptions.RequireInfrastructureEncryption == pointer.BoolDeref(account.Encryption.RequireInfrastructureEncryption, false) + return requireInfraEncryption == pointer.BoolDeref(account.Encryption.RequireInfrastructureEncryption, false) } func isAllowSharedKeyAccessEqual(account storage.Account, accountOptions *AccountOptions) bool { - if accountOptions.AllowSharedKeyAccess == nil { - return true - } - return *accountOptions.AllowSharedKeyAccess == pointer.BoolDeref(account.AllowSharedKeyAccess, false) + return pointer.BoolDeref(accountOptions.AllowSharedKeyAccess, false) == pointer.BoolDeref(account.AllowSharedKeyAccess, false) } func isAccessTierEqual(account storage.Account, accountOptions *AccountOptions) bool { diff --git a/vendor/sigs.k8s.io/cloud-provider-azure/pkg/provider/azure_wrap.go b/vendor/sigs.k8s.io/cloud-provider-azure/pkg/provider/azure_wrap.go index aa86e44f95..036053e47b 100644 --- a/vendor/sigs.k8s.io/cloud-provider-azure/pkg/provider/azure_wrap.go +++ b/vendor/sigs.k8s.io/cloud-provider-azure/pkg/provider/azure_wrap.go @@ -21,6 +21,7 @@ import ( "net/http" "regexp" "strings" + "sync" "time" "github.com/Azure/azure-sdk-for-go/services/compute/mgmt/2022-03-01/compute" @@ -34,6 +35,7 @@ import ( azcache "sigs.k8s.io/cloud-provider-azure/pkg/cache" "sigs.k8s.io/cloud-provider-azure/pkg/consts" "sigs.k8s.io/cloud-provider-azure/pkg/retry" + "sigs.k8s.io/cloud-provider-azure/pkg/util/deepcopy" ) var ( @@ -98,27 +100,44 @@ func (az *Cloud) getRouteTable(crt azcache.AzureCacheReadType) (routeTable netwo return *(cachedRt.(*network.RouteTable)), true, nil } -func (az *Cloud) getPIPCacheKey(pipResourceGroup string, pipName string) string { - resourceGroup := az.ResourceGroup - if pipResourceGroup != "" { - resourceGroup = pipResourceGroup - } - return fmt.Sprintf("%s%s%s", resourceGroup, consts.PIPCacheKeySeparator, pipName) -} - func (az *Cloud) getPublicIPAddress(pipResourceGroup string, pipName string, crt azcache.AzureCacheReadType) (network.PublicIPAddress, bool, error) { - pip := network.PublicIPAddress{} - cacheKey := az.getPIPCacheKey(pipResourceGroup, pipName) - cachedPIP, err := az.pipCache.GetWithDeepCopy(cacheKey, crt) + cached, err := az.pipCache.Get(pipResourceGroup, crt) if err != nil { - return pip, false, err + return network.PublicIPAddress{}, false, err } - if cachedPIP == nil { - return pip, false, nil + pips := cached.(*sync.Map) + pip, ok := pips.Load(pipName) + if !ok { + // pip not found, refresh cache and retry + cached, err = az.pipCache.Get(pipResourceGroup, azcache.CacheReadTypeForceRefresh) + if err != nil { + return network.PublicIPAddress{}, false, err + } + pips = cached.(*sync.Map) + pip, ok = pips.Load(pipName) + if !ok { + return network.PublicIPAddress{}, false, nil + } } - return *(cachedPIP.(*network.PublicIPAddress)), true, nil + pip = pip.(*network.PublicIPAddress) + return *(deepcopy.Copy(pip).(*network.PublicIPAddress)), true, nil +} + +func (az *Cloud) listPIP(pipResourceGroup string) ([]network.PublicIPAddress, error) { + cached, err := az.pipCache.Get(pipResourceGroup, azcache.CacheReadTypeDefault) + if err != nil { + return nil, err + } + pips := cached.(*sync.Map) + var ret []network.PublicIPAddress + pips.Range(func(key, value interface{}) bool { + pip := value.(*network.PublicIPAddress) + ret = append(ret, *pip) + return true + }) + return ret, nil } func (az *Cloud) getSubnet(virtualNetworkName string, subnetName string) (network.Subnet, bool, error) { @@ -304,24 +323,18 @@ func (az *Cloud) newPIPCache() (*azcache.TimedCache, error) { ctx, cancel := getContextWithCancel() defer cancel() - parsedKey := strings.Split(strings.TrimSpace(key), consts.PIPCacheKeySeparator) - if len(parsedKey) != 2 { - return nil, fmt.Errorf("failed to parse public ip rg and name from cache key %q", key) - } - pipResourceGroup, pipName := strings.TrimSpace(parsedKey[0]), strings.TrimSpace(parsedKey[1]) - - pip, err := az.PublicIPAddressesClient.Get(ctx, pipResourceGroup, pipName, "") - exists, rerr := checkResourceExistsFromError(err) + pipResourceGroup := key + pipList, rerr := az.PublicIPAddressesClient.List(ctx, pipResourceGroup) if rerr != nil { return nil, rerr.Error() } - if !exists { - klog.V(2).Infof("Public IP %q in rg %q not found", pipName, pipResourceGroup) - return nil, nil + pipMap := &sync.Map{} + for _, pip := range pipList { + pip := pip + pipMap.Store(pointer.StringDeref(pip.Name, ""), &pip) } - - return &pip, nil + return pipMap, nil } if az.PublicIPCacheTTLInSeconds == 0 {