Skip to content

Commit

Permalink
Calculate disks limit per node (#17)
Browse files Browse the repository at this point in the history
Signed-off-by: Vitaliy Snurnitsin <vitaliy.snurnitsin@flant.com>
  • Loading branch information
vitaliy-sn authored Sep 1, 2023
1 parent 69c8309 commit be968a8
Show file tree
Hide file tree
Showing 5 changed files with 182 additions and 30 deletions.
4 changes: 2 additions & 2 deletions Dockerfile
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.

FROM golang:1.13-alpine as build
FROM golang:1.15-alpine as build
RUN apk add git

WORKDIR /go/src/app
Expand All @@ -31,7 +31,7 @@ RUN CGO_ENABLED=0 GOOS=$OS GOARCH=$ARCH go build -a \
-o /go/bin/yandex-csi-driver \
github.com/deckhouse/yandex-csi-driver/cmd/yandex-csi-driver

FROM alpine:3.10
FROM alpine:3.18

RUN apk add --no-cache ca-certificates \
e2fsprogs \
Expand Down
75 changes: 72 additions & 3 deletions driver/node.go
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,7 @@ import (

"github.com/container-storage-interface/spec/lib/go/csi"
"github.com/sirupsen/logrus"
"github.com/yandex-cloud/go-genproto/yandex/cloud/compute/v1"
"google.golang.org/grpc/codes"
"google.golang.org/grpc/status"
"k8s.io/klog"
Expand All @@ -36,9 +37,6 @@ import (
const (
diskIDPath = "/dev/disk/by-id"

// See: https://cloud.yandex.ru/docs/compute/concepts/limits
maxVolumesPerNode = 7

volumeModeBlock = "block"
volumeModeFilesystem = "filesystem"

Expand Down Expand Up @@ -369,6 +367,77 @@ func (d *Driver) NodeGetCapabilities(_ context.Context, _ *csi.NodeGetCapabiliti
// by the CO in ControllerPublishVolume.
func (d *Driver) NodeGetInfo(_ context.Context, _ *csi.NodeGetInfoRequest) (*csi.NodeGetInfoResponse, error) {
d.log.WithField("method", "node_get_info").Info("node get info called")

instance, err := d.sdk.Compute().Instance().Get(context.TODO(), &compute.GetInstanceRequest{
InstanceId: d.hostID,
View: 0,
})
if err != nil {
return nil, err
}

const bootDisksCount = 1
networkInterfacesCount := int64(len(instance.GetNetworkInterfaces()))
secondaryDisksCount := int64(len(instance.GetSecondaryDisks()))
localDisksCount := int64(len(instance.GetLocalDisks()))
filesystemsCount := int64(len(instance.GetFilesystems()))

var nonCSISecondaryDisksCount = localDisksCount + filesystemsCount
for _, disk := range instance.GetSecondaryDisks() {
if !strings.HasPrefix(disk.DeviceName, "csi-") {
nonCSISecondaryDisksCount++
}
}

// See: https://cloud.yandex.ru/docs/compute/concepts/limits
// When a VM is starting, a maximum of 14 devices, including the boot disk and a NIC, can be connected to it. Other devices must be connected to a running VM. Please note: If you restart a VM with more than 14 devices connected, it will not be able to load and run.
var disksLimit int64

switch instance.GetPlatformId() {
// Intel Broadwell, Intel Broadwell with NVIDIA® Tesla® V100
case "standard-v1", "gpu-standard-v1":
if instance.GetResources().Cores > 18 {
disksLimit = 14
} else {
disksLimit = 8
}
// Intel Cascade Lake, Intel Cascade Lake with NVIDIA® Tesla® V100
case "standard-v2", "gpu-standard-v2":
if instance.GetResources().Cores > 20 {
disksLimit = 14
} else {
disksLimit = 8
}
// Intel Ice Lake, Ice Lake Compute-optimized
case "standard-v3", "highfreq-v3":
if instance.GetResources().Cores > 32 {
disksLimit = 14
} else {
disksLimit = 8
}
// other
default:
disksLimit = 8
}

var maxVolumesPerNode int64

if bootDisksCount+secondaryDisksCount+networkInterfacesCount < 14 {
maxVolumesPerNode = disksLimit - bootDisksCount - nonCSISecondaryDisksCount
} else {
maxVolumesPerNode = disksLimit - bootDisksCount - nonCSISecondaryDisksCount - networkInterfacesCount
}

d.log.WithFields(logrus.Fields{
"secondaryDisksCount": secondaryDisksCount,
"localDisksCount": localDisksCount,
"filesystemsCount": filesystemsCount,
"networkInterfacesCount": networkInterfacesCount,
"nonCSISecondaryDisksCount": nonCSISecondaryDisksCount,
"disksLimit": disksLimit,
"maxVolumesPerNode": maxVolumesPerNode,
}).Info()

return &csi.NodeGetInfoResponse{
NodeId: d.hostID,
MaxVolumesPerNode: maxVolumesPerNode,
Expand Down
19 changes: 9 additions & 10 deletions go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -2,23 +2,22 @@ module github.com/deckhouse/yandex-csi-driver

require (
github.com/container-storage-interface/spec v1.2.0
github.com/golang/protobuf v1.3.2
github.com/hashicorp/go-multierror v1.0.0 // indirect
github.com/mitchellh/go-testing-interface v1.0.0 // indirect
github.com/golang/protobuf v1.5.2
github.com/sirupsen/logrus v1.4.2
github.com/yandex-cloud/go-genproto v0.0.0-20200113111713-27a750dfd05a
github.com/yandex-cloud/go-sdk v0.0.0-20200113201139-dc3c759a1204
github.com/yandex-cloud/go-genproto v0.0.0-20230828085742-c924b3bfee75
github.com/yandex-cloud/go-sdk v0.0.0-20230828093337-d401bc1eeaba
golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e
golang.org/x/sys v0.0.0-20190826190057-c7b8b68b1456
google.golang.org/genproto v0.0.0-20190502173448-54afdca5d873
google.golang.org/grpc v1.23.1
golang.org/x/sys v0.0.0-20210510120138-977fb7262007
google.golang.org/genproto v0.0.0-20211021150943-2b146023228c
google.golang.org/grpc v1.41.0
google.golang.org/protobuf v1.31.0
k8s.io/apimachinery v0.17.1
k8s.io/kubernetes v1.17.1
k8s.io/klog v1.0.0
k8s.io/kubernetes v1.17.1
k8s.io/utils v0.0.0-20191114184206-e782cd3c129f
)

go 1.13
go 1.15

replace k8s.io/api => k8s.io/api v0.17.1

Expand Down
Loading

0 comments on commit be968a8

Please # to comment.