From 418c1887fe278e649c37d6988cbe0d80643c8100 Mon Sep 17 00:00:00 2001 From: Cedric Kring Date: Sat, 5 Oct 2019 11:44:45 +0200 Subject: [PATCH 1/4] Allow volume binds to specific nodes with `--volume src:dest@node-name` --- cli/commands.go | 31 +++++++++++++++++++++++++++++-- cli/container.go | 26 +++++++++++++++++++------- cli/util.go | 5 +++++ 3 files changed, 53 insertions(+), 9 deletions(-) diff --git a/cli/commands.go b/cli/commands.go index 40227456c..cc1db240f 100644 --- a/cli/commands.go +++ b/cli/commands.go @@ -147,7 +147,34 @@ func CreateCluster(c *cli.Context) error { return err } volumes := c.StringSlice("volume") - volumes = append(volumes, fmt.Sprintf("%s:/images", imageVolume.Name)) + + volumesSpec := Volumes{ + DefaultVolumes: []string{}, + NodeSpecificVolumes: make(map[string][]string), + } + for _, volume := range volumes { + if strings.Contains(volume, "@") { + split := strings.Split(volume, "@") + if len(split) != 2 { + return fmt.Errorf("invalid node volume spec: %s", volume) + } + + nodeVolumes := split[0] + node := split[1] + if len(node) == 0 { + return fmt.Errorf("invalid node volume spec: %s", volume) + } + + if _, ok := volumesSpec.NodeSpecificVolumes[node]; !ok { + volumesSpec.NodeSpecificVolumes[node] = []string{} + } + volumesSpec.NodeSpecificVolumes[node] = append(volumesSpec.NodeSpecificVolumes[node], nodeVolumes) + } else { + volumesSpec.DefaultVolumes = append(volumesSpec.DefaultVolumes, volume) + } + } + + volumesSpec.DefaultVolumes = append(volumesSpec.DefaultVolumes, fmt.Sprintf("%s:/images", imageVolume.Name)) clusterSpec := &ClusterSpec{ AgentArgs: k3AgentArgs, @@ -159,7 +186,7 @@ func CreateCluster(c *cli.Context) error { NodeToPortSpecMap: portmap, PortAutoOffset: c.Int("port-auto-offset"), ServerArgs: k3sServerArgs, - Volumes: volumes, + Volumes: volumesSpec, } // create the server diff --git a/cli/container.go b/cli/container.go index baba8cb3f..c883dae56 100644 --- a/cli/container.go +++ b/cli/container.go @@ -30,7 +30,7 @@ type ClusterSpec struct { NodeToPortSpecMap map[string][]string PortAutoOffset int ServerArgs []string - Volumes []string + Volumes Volumes } func startContainer(config *container.Config, hostConfig *container.HostConfig, networkingConfig *network.NetworkingConfig, containerName string) (string, error) { @@ -118,9 +118,7 @@ func createServer(spec *ClusterSpec) (string, error) { hostConfig.RestartPolicy.Name = "unless-stopped" } - if len(spec.Volumes) > 0 && spec.Volumes[0] != "" { - hostConfig.Binds = spec.Volumes - } + addVolumesToHostConfig(spec.Volumes, containerName, hostConfig) networkingConfig := &network.NetworkingConfig{ EndpointsConfig: map[string]*network.EndpointSettings{ @@ -187,9 +185,7 @@ func createWorker(spec *ClusterSpec, postfix int) (string, error) { hostConfig.RestartPolicy.Name = "unless-stopped" } - if len(spec.Volumes) > 0 && spec.Volumes[0] != "" { - hostConfig.Binds = spec.Volumes - } + addVolumesToHostConfig(spec.Volumes, containerName, hostConfig) networkingConfig := &network.NetworkingConfig{ EndpointsConfig: map[string]*network.EndpointSettings{ @@ -234,3 +230,19 @@ func removeContainer(ID string) error { } return nil } + +// addVolumesToHostConfig adds all default volumes and node specific volumes to a HostConfig +func addVolumesToHostConfig(volumesSpec Volumes, containerName string, hostConfig *container.HostConfig) { + if len(volumesSpec.DefaultVolumes) > 0 { + volumes := volumesSpec.DefaultVolumes + + // add node specific volumes if present + if v, ok := volumesSpec.NodeSpecificVolumes[containerName]; ok { + for _, volume := range v { + volumes = append(volumes, volume) + } + } + + hostConfig.Binds = volumes + } +} diff --git a/cli/util.go b/cli/util.go index 49bef40c7..000b9182b 100644 --- a/cli/util.go +++ b/cli/util.go @@ -15,6 +15,11 @@ type apiPort struct { Port string } +type Volumes struct { + DefaultVolumes []string + NodeSpecificVolumes map[string][]string +} + const letterBytes = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ" const ( letterIdxBits = 6 // 6 bits to represent a letter index From 1e0aac19f60755163f5e9e7dce06ca6bb29f2cc6 Mon Sep 17 00:00:00 2001 From: Cedric Kring Date: Sat, 5 Oct 2019 12:30:08 +0200 Subject: [PATCH 2/4] Replace loop with simple append & spread --- cli/container.go | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/cli/container.go b/cli/container.go index c883dae56..a983e72ef 100644 --- a/cli/container.go +++ b/cli/container.go @@ -238,9 +238,7 @@ func addVolumesToHostConfig(volumesSpec Volumes, containerName string, hostConfi // add node specific volumes if present if v, ok := volumesSpec.NodeSpecificVolumes[containerName]; ok { - for _, volume := range v { - volumes = append(volumes, volume) - } + volumes = append(volumes, v...) } hostConfig.Binds = volumes From 9b888c82165515895bd856f0c852d9748462f4e2 Mon Sep 17 00:00:00 2001 From: Cedric Kring Date: Wed, 9 Oct 2019 19:35:13 +0200 Subject: [PATCH 3/4] Added support for group based node volume binds + refactoring --- cli/commands.go | 27 ++---------------- cli/container.go | 20 ++----------- cli/port.go | 9 ------ cli/util.go | 14 ++++++---- cli/volume.go | 73 ++++++++++++++++++++++++++++++++++++++++++++++++ 5 files changed, 88 insertions(+), 55 deletions(-) diff --git a/cli/commands.go b/cli/commands.go index cc1db240f..55f86db17 100644 --- a/cli/commands.go +++ b/cli/commands.go @@ -148,30 +148,9 @@ func CreateCluster(c *cli.Context) error { } volumes := c.StringSlice("volume") - volumesSpec := Volumes{ - DefaultVolumes: []string{}, - NodeSpecificVolumes: make(map[string][]string), - } - for _, volume := range volumes { - if strings.Contains(volume, "@") { - split := strings.Split(volume, "@") - if len(split) != 2 { - return fmt.Errorf("invalid node volume spec: %s", volume) - } - - nodeVolumes := split[0] - node := split[1] - if len(node) == 0 { - return fmt.Errorf("invalid node volume spec: %s", volume) - } - - if _, ok := volumesSpec.NodeSpecificVolumes[node]; !ok { - volumesSpec.NodeSpecificVolumes[node] = []string{} - } - volumesSpec.NodeSpecificVolumes[node] = append(volumesSpec.NodeSpecificVolumes[node], nodeVolumes) - } else { - volumesSpec.DefaultVolumes = append(volumesSpec.DefaultVolumes, volume) - } + volumesSpec, err := NewVolumes(volumes) + if err != nil { + return err } volumesSpec.DefaultVolumes = append(volumesSpec.DefaultVolumes, fmt.Sprintf("%s:/images", imageVolume.Name)) diff --git a/cli/container.go b/cli/container.go index a983e72ef..be09ce605 100644 --- a/cli/container.go +++ b/cli/container.go @@ -30,7 +30,7 @@ type ClusterSpec struct { NodeToPortSpecMap map[string][]string PortAutoOffset int ServerArgs []string - Volumes Volumes + Volumes *Volumes } func startContainer(config *container.Config, hostConfig *container.HostConfig, networkingConfig *network.NetworkingConfig, containerName string) (string, error) { @@ -118,7 +118,7 @@ func createServer(spec *ClusterSpec) (string, error) { hostConfig.RestartPolicy.Name = "unless-stopped" } - addVolumesToHostConfig(spec.Volumes, containerName, hostConfig) + spec.Volumes.addVolumesToHostConfig(containerName, "server", hostConfig) networkingConfig := &network.NetworkingConfig{ EndpointsConfig: map[string]*network.EndpointSettings{ @@ -185,7 +185,7 @@ func createWorker(spec *ClusterSpec, postfix int) (string, error) { hostConfig.RestartPolicy.Name = "unless-stopped" } - addVolumesToHostConfig(spec.Volumes, containerName, hostConfig) + spec.Volumes.addVolumesToHostConfig(containerName, "worker", hostConfig) networkingConfig := &network.NetworkingConfig{ EndpointsConfig: map[string]*network.EndpointSettings{ @@ -230,17 +230,3 @@ func removeContainer(ID string) error { } return nil } - -// addVolumesToHostConfig adds all default volumes and node specific volumes to a HostConfig -func addVolumesToHostConfig(volumesSpec Volumes, containerName string, hostConfig *container.HostConfig) { - if len(volumesSpec.DefaultVolumes) > 0 { - volumes := volumesSpec.DefaultVolumes - - // add node specific volumes if present - if v, ok := volumesSpec.NodeSpecificVolumes[containerName]; ok { - volumes = append(volumes, v...) - } - - hostConfig.Binds = volumes - } -} diff --git a/cli/port.go b/cli/port.go index 505eb4b21..a730a604d 100644 --- a/cli/port.go +++ b/cli/port.go @@ -14,15 +14,6 @@ type PublishedPorts struct { PortBindings map[nat.Port][]nat.PortBinding } -// defaultNodes describes the type of nodes on which a port should be exposed by default -const defaultNodes = "server" - -// mapping a node role to groups that should be applied to it -var nodeRuleGroupsMap = map[string][]string{ - "worker": {"all", "workers"}, - "server": {"all", "server", "master"}, -} - // mapNodesToPortSpecs maps nodes to portSpecs func mapNodesToPortSpecs(specs []string, createdNodes []string) (map[string][]string, error) { diff --git a/cli/util.go b/cli/util.go index 000b9182b..5a6563db4 100644 --- a/cli/util.go +++ b/cli/util.go @@ -15,11 +15,6 @@ type apiPort struct { Port string } -type Volumes struct { - DefaultVolumes []string - NodeSpecificVolumes map[string][]string -} - const letterBytes = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ" const ( letterIdxBits = 6 // 6 bits to represent a letter index @@ -27,6 +22,15 @@ const ( letterIdxMax = 63 / letterIdxBits // # of letter indices fitting in 63 bits ) +// defaultNodes describes the default node group (master) +const defaultNodes = "server" + +// mapping a node role to groups that should be applied to it +var nodeRuleGroupsMap = map[string][]string{ + "worker": {"all", "workers"}, + "server": {"all", "server", "master"}, +} + var src = rand.NewSource(time.Now().UnixNano()) // GenerateRandomString thanks to https://stackoverflow.com/a/31832326/6450189 diff --git a/cli/volume.go b/cli/volume.go index cb3043dbd..1eb1aecfc 100644 --- a/cli/volume.go +++ b/cli/volume.go @@ -3,13 +3,21 @@ package run import ( "context" "fmt" + "strings" "github.com/docker/docker/api/types" + "github.com/docker/docker/api/types/container" "github.com/docker/docker/api/types/filters" "github.com/docker/docker/api/types/volume" "github.com/docker/docker/client" ) +type Volumes struct { + DefaultVolumes []string + NodeSpecificVolumes map[string][]string + GroupSpecificVolumes map[string][]string +} + // createImageVolume will create a new docker volume used for storing image tarballs that can be loaded into the clusters func createImageVolume(clusterName string) (types.Volume, error) { @@ -90,3 +98,68 @@ func getImageVolume(clusterName string) (types.Volume, error) { return vol, nil } + +func NewVolumes(volumes []string) (*Volumes, error) { + volumesSpec := &Volumes{ + DefaultVolumes: []string{}, + NodeSpecificVolumes: make(map[string][]string), + GroupSpecificVolumes: make(map[string][]string), + } + for _, volume := range volumes { + if strings.Contains(volume, "@") { + split := strings.Split(volume, "@") + if len(split) != 2 { + return nil, fmt.Errorf("invalid node volume spec: %s", volume) + } + + nodeVolumes := split[0] + node := strings.ToLower(split[1]) + if len(node) == 0 { + return nil, fmt.Errorf("invalid node volume spec: %s", volume) + } + + // check if node selector is a node group + if _, ok := nodeRuleGroupsMap[node]; ok { + volumesSpec.addGroupSpecificVolume(node, nodeVolumes) + } + + // otherwise this is a volume for a specific node + volumesSpec.addNodeSpecificVolume(node, nodeVolumes) + } else { + volumesSpec.DefaultVolumes = append(volumesSpec.DefaultVolumes, volume) + } + } + + return volumesSpec, nil +} + +// addVolumesToHostConfig adds all default volumes and node / group specific volumes to a HostConfig +func (v Volumes) addVolumesToHostConfig(containerName string, groupName string, hostConfig *container.HostConfig) { + volumes := v.DefaultVolumes + + if v, ok := v.NodeSpecificVolumes[containerName]; ok { + volumes = append(volumes, v...) + } + + if v, ok := v.GroupSpecificVolumes[groupName]; ok { + volumes = append(volumes, v...) + } + + if len(volumes) > 0 { + hostConfig.Binds = volumes + } +} + +func (v *Volumes) addNodeSpecificVolume(node, volume string) { + if _, ok := v.NodeSpecificVolumes[node]; !ok { + v.NodeSpecificVolumes[node] = []string{} + } + v.NodeSpecificVolumes[node] = append(v.NodeSpecificVolumes[node], volume) +} + +func (v *Volumes) addGroupSpecificVolume(group, volume string) { + if _, ok := v.GroupSpecificVolumes[group]; !ok { + v.GroupSpecificVolumes[group] = []string{} + } + v.GroupSpecificVolumes[group] = append(v.GroupSpecificVolumes[group], volume) +} From 33c3f7cbdab6ba0f648e8a1f8b41f89c8aa4b443 Mon Sep 17 00:00:00 2001 From: Cedric Kring Date: Wed, 9 Oct 2019 19:51:24 +0200 Subject: [PATCH 4/4] Use group values instead of keys for lookup --- cli/volume.go | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/cli/volume.go b/cli/volume.go index 1eb1aecfc..a172ab208 100644 --- a/cli/volume.go +++ b/cli/volume.go @@ -105,6 +105,8 @@ func NewVolumes(volumes []string) (*Volumes, error) { NodeSpecificVolumes: make(map[string][]string), GroupSpecificVolumes: make(map[string][]string), } + +volumes: for _, volume := range volumes { if strings.Contains(volume, "@") { split := strings.Split(volume, "@") @@ -119,8 +121,20 @@ func NewVolumes(volumes []string) (*Volumes, error) { } // check if node selector is a node group - if _, ok := nodeRuleGroupsMap[node]; ok { - volumesSpec.addGroupSpecificVolume(node, nodeVolumes) + for group, names := range nodeRuleGroupsMap { + added := false + + for _, name := range names { + if name == node { + volumesSpec.addGroupSpecificVolume(group, nodeVolumes) + added = true + break + } + } + + if added { + continue volumes + } } // otherwise this is a volume for a specific node