Build lockc on OpenStack with openSUSE Leap image

contrib/terraform/libvirt/worker.tf

@@ -146,11 +146,6 @@ resource "null_resource" "worker_provision_k8s_containerd" {
     type = "ssh"
   }
 
-  provisioner "file" {
-    source =      "../../../target/debug/lockc.tar.gz"
-    destination = "/home/opensuse/lockc.tar.gz"
-  }
-
   provisioner "remote-exec" {
     script = "provision-k8s-containerd.sh"
   }
@@ -177,13 +172,13 @@ export sshopts="-o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -oCo
 if ! ssh $sshopts $user@$host 'sudo needs-restarting -r'; then
     ssh $sshopts $user@$host sudo reboot || :
     export delay=5
-    # # wait for node reboot completed
-    # # lol, doesn't work
-    # while ! ssh $sshopts $user@$host 'sudo needs-restarting -r'; do
-    #     sleep $delay
-    #     delay=$((delay+1))
-    #     [ $delay -gt 60 ] && exit 1
-    # done
+    # wait for node reboot completed
+    while ! ssh $sshopts $user@$host 'sudo needs-restarting -r'; do
+        sleep $delay
+        delay=$((delay+1))
+        [ $delay -gt 60 ] && exit 1
+        ssh $sshopts $user@$host 'sudo needs-restarting -r'
+    done
 fi
 EOT
   }

contrib/terraform/openstack/cloud-init/common.tpl

@@ -1,10 +1,13 @@
 #cloud-config
 
+# set hostname
+hostname: ${hostname}
+
 # set locale
-locale: en_US.UTF-8
+locale: ${locale} # en_US.UTF-8
 
 # set timezone
-timezone: Etc/UTC
+timezone: ${timezone} # Etc/UTC
 
 # Inject the public keys
 ssh_authorized_keys:
@@ -27,11 +30,6 @@ ${repositories}
     solver.onlyRequires: "true"
     download.use_deltarpm: "true"
 
-#packages:
-
-# set hostname
-hostname: ${hostname}
-
 runcmd:
   # workaround for bsc#1119397 . If this is not called, /etc/resolv.conf is empty
   - netconfig -f update
@@ -42,8 +40,11 @@ runcmd:
   - sshd -t || echo "ssh syntax failure"
   - systemctl restart sshd
   # Set node's hostname from DHCP server
-  - sed -i -e '/^DHCLIENT_SET_HOSTNAME/s/^.*$/DHCLIENT_SET_HOSTNAME=\"${hostname_from_dhcp}\"/' /etc/sysconfig/network/dhcp
+  - sed -i -e '/^DHCLIENT_SET_HOSTNAME/s/^.*$/DHCLIENT_SET_HOSTNAME=\"yes\"/' /etc/sysconfig/network/dhcp
   - systemctl restart wicked
+  # Refresh repos and upgrade
+  - zypper ref
+  - zypper dup -y --allow-vendor-change --replacefiles
 ${commands}
 
 final_message: "The system is finally up, after $UPTIME seconds"

contrib/terraform/openstack/cloud-init/repository.tpl

@@ -1,6 +1,7 @@
     - id: ${repository_name}
       name: ${repository_name}
       baseurl: ${repository_url}
+      priority: 90
       enabled: 1
       autorefresh: 1
       gpgcheck: 0

contrib/terraform/openstack/deploy-kubernetes.sh

@@ -1,70 +1,65 @@
 #!/bin/bash
 #shellcheck disable=SC2145,SC2016
+
+set -eux
+
 log()   { (>&1 echo -e "$@") ; }
-cmd()   { log "$@" ; }
 info()  { log "[ INFO ] $@" ; }
 error() { (>&2 echo -e "[ ERROR ] $@") ;}
 
 if [ -z "${TR_STACK}" ] || [ -z "${TR_LB_IP}" ] || \
-   [ -z "$TR_MASTER_IPS" ] || [ -z "$TR_WORKER_IPS" ] || \
-   [ -z "${TR_USERNAME}" ]; then
-    error '$TR_STACK $TR_LB_IP $TR_MASTER_IPS $TR_WORKER_IPS $TR_USERNAME must be specified'
+   [ -z "$TR_MASTER_IPS" ] || [ -z "${TR_USERNAME}" ]; then
+    error '$TR_STACK $TR_LB_IP $TR_MASTER_IPS $TR_USERNAME must be specified'
     exit 1
 fi
 
+sleep 5
+
+CILIUM_VERSION=$(curl -s https://api.github.com/repos/cilium/cilium/releases/latest | jq -r '.tag_name' | sed -e 's/^v//')
+
 info "### Run following commands to bootstrap Kubernetes cluster:\\n"
-cmd ""
 
 i=0
 for MASTER in $TR_MASTER_IPS; do
-    cmd "ssh -o 'StrictHostKeyChecking no' -l ${TR_USERNAME} ${MASTER} /bin/bash <<EOF"
-    cmd ""
-
     if [ $i -eq "0" ]; then
-        cmd "  sudo kubeadm init --cri-socket /run/containerd/containerd.sock --control-plane-endpoint ${TR_LB_IP}:6443 | tee kubeadm-init.log"
-        cmd ""
-        cmd "  mkdir -p /home/${TR_USERNAME}/.kube"
-        cmd "  sudo cp /etc/kubernetes/admin.conf /home/${TR_USERNAME}/.kube/config"
-        cmd "  sudo chown ${TR_USERNAME}:users /home/${TR_USERNAME}/.kube/config"
-        cmd "EOF"
-
         ssh -o 'StrictHostKeyChecking no' -l ${TR_USERNAME} ${MASTER} /bin/bash <<-EOF
-          sudo kubeadm init --cri-socket /run/containerd/containerd.sock --control-plane-endpoint ${TR_LB_IP}:6443 | tee kubeadm-init.log
+          set -eux
+          sudo kubeadm init --cri-socket /run/containerd/containerd.sock --control-plane-endpoint ${MASTER}:6443 --upload-certs | tee kubeadm-init.log
           mkdir -p /home/${TR_USERNAME}/.kube
           sudo cp /etc/kubernetes/admin.conf /home/${TR_USERNAME}/.kube/config
           sudo chown ${TR_USERNAME}:users /home/${TR_USERNAME}/.kube/config
+          helm repo add cilium https://helm.cilium.io/
+          helm install cilium cilium/cilium --version ${CILIUM_VERSION} --namespace kube-system
 EOF
 
-        cmd ""
-        export KUBEADM_JOIN=`ssh -l ${TR_USERNAME} ${MASTER} tail -n2 kubeadm-init.log`
-        export KUBEADM_CMD=`echo $KUBEADM_JOIN | sed -e 's/\\\ //'`
-        echo $KUBEADM_CMD
+        export KUBEADM_MASTER_JOIN=`ssh -o 'StrictHostKeyChecking no' -l ${TR_USERNAME} ${MASTER} tail -n12 kubeadm-init.log | head -n3`
+        export KUBEADM_WORKER_JOIN=`ssh -o 'StrictHostKeyChecking no' -l ${TR_USERNAME} ${MASTER} tail -n2 kubeadm-init.log`
     else
-        cmd ""
-        cmd " sudo kubeadm join"
-        cmd "EOF"
-        cmd ""
+        ssh -o 'StrictHostKeyChecking no' -l ${TR_USERNAME} ${MASTER} /bin/bash <<-EOF
+          set -eux
+          sudo ${KUBEADM_MASTER_JOIN}
+          mkdir -p /home/${TR_USERNAME}/.kube
+          sudo cp /etc/kubernetes/admin.conf /home/${TR_USERNAME}/.kube/config
+          sudo chown ${TR_USERNAME}:users /home/${TR_USERNAME}/.kube/config
+EOF
     fi
     ((++i))
 done
 
 i=0
 for WORKER in $TR_WORKER_IPS; do
-    cmd "ssh -o 'StrictHostKeyChecking no' -l ${TR_USERNAME} ${WORKER} sudo ${KUBEADM_CMD}"
-    ssh -o 'StrictHostKeyChecking no' -l ${TR_USERNAME} ${WORKER} sudo ${KUBEADM_CMD}
+    ssh -o 'StrictHostKeyChecking no' -l ${TR_USERNAME} ${WORKER} /bin/bash <<-EOF
+      set -eux
+      sudo ${KUBEADM_WORKER_JOIN}
+EOF
     ((++i))
 done
 
-cmd ""
-cmd "scp ${TR_USERNAME}@${MASTER}:/home/${TR_USERNAME}/.kube/config ./admin.conf"
-cmd "export KUBECONFIG=`pwd`/admin.conf"
-cmd ""
-
 scp ${TR_USERNAME}@${MASTER}:/home/${TR_USERNAME}/.kube/config ./admin.conf
 export KUBECONFIG=`pwd`/admin.conf
 kubectl get nodes
 
-cmd ""
-cmd "WARNING!!! To start with K8s cluster please run following command:"
-cmd "export KUBECONFIG=`pwd`/admin.conf"
-cmd ""
+log ""
+log "WARNING!!! To start with K8s cluster please run following command:"
+log "export KUBECONFIG=`pwd`/admin.conf"
+log ""

contrib/terraform/openstack/master-instance.tf

@@ -10,25 +10,26 @@ data "template_file" "master_repositories" {
 
 data "template_file" "master_commands" {
   template = file("cloud-init/commands.tpl")
-  count    = join("", var.packages) == "" ? 0 : 1
+  count    = length(var.packages)
 
   vars = {
     packages = join(", ", var.packages)
   }
 }
 
-data "template_file" "master-cloud-init" {
+data "template_file" "master_cloud_init" {
   template = file("cloud-init/common.tpl")
   count    = var.masters
 
   vars = {
+    hostname           = "${var.stack_name}-k8s-master${count.index}"
+    locale             = var.locale
+    timezone           = var.timezone
+    username           = var.username
     authorized_keys    = join("\n", formatlist("  - %s", var.authorized_keys))
     repositories       = join("\n", data.template_file.master_repositories.*.rendered)
     commands           = join("\n", data.template_file.master_commands.*.rendered)
-    username           = var.username
     ntp_servers        = join("\n", formatlist("    - %s", var.ntp_servers))
-    hostname           = "${var.stack_name}-k8s-master${count.index}"
-    hostname_from_dhcp = var.hostname_from_dhcp
   }
 }
 
@@ -54,7 +55,7 @@ resource "openstack_compute_instance_v2" "master" {
     openstack_networking_secgroup_v2.master_nodes.id,
   ]
 
-  user_data = data.template_file.master-cloud-init[count.index].rendered
+  user_data = data.template_file.master_cloud_init[count.index].rendered
 }
 
 resource "openstack_networking_floatingip_v2" "master_ext" {
@@ -95,9 +96,51 @@ resource "null_resource" "master_wait_cloudinit" {
   }
 }
 
-resource "null_resource" "master_reboot" {
+resource "null_resource" "master_provision" {
   depends_on = [null_resource.master_wait_cloudinit]
   count      = var.masters
+  connection {
+    host = element(
+      openstack_compute_floatingip_associate_v2.master_ext_ip.*.floating_ip,
+      count.index
+    )
+    user = var.username
+    type = "ssh"
+  }
+
+  provisioner "remote-exec" {
+    script = "provision.sh"
+  }
+}
+
+resource "null_resource" "master_provision_k8s_containerd" {
+  depends_on = [null_resource.master_provision]
+  count      = var.masters
+
+  connection {
+    host = element(
+      openstack_compute_floatingip_associate_v2.master_ext_ip.*.floating_ip,
+      count.index
+    )
+    user = var.username
+    type = "ssh"
+  }
+
+  provisioner "remote-exec" {
+    script = "provision-k8s-containerd.sh"
+  }
+
+  provisioner "remote-exec" {
+    script = "provision-k8s-containerd-cp.sh"
+  }
+}
+
+
+resource "null_resource" "master_reboot" {
+  depends_on = [
+    null_resource.master_provision_k8s_containerd,
+  ]
+  count      = var.masters
 
   provisioner "local-exec" {
     environment = {
@@ -118,6 +161,7 @@ if ! ssh $sshopts $user@$host 'sudo needs-restarting -r'; then
         sleep $delay
         delay=$((delay+1))
         [ $delay -gt 30 ] && exit 1
+        ssh $sshopts $user@$host 'sudo needs-restarting -r'
     done
 fi
 EOT

contrib/terraform/openstack/provision-k8s-containerd-cp.sh

@@ -0,0 +1,9 @@
+#!/bin/bash
+
+# ensure running as root
+if [ "$(id -u)" != "0" ]; then
+  exec sudo "$0" "$@"
+fi
+
+HELM_VERSION=$(curl -s https://api.github.com/repos/helm/helm/releases/latest | jq -r '.tag_name')
+curl -sSL "https://get.helm.sh/helm-${HELM_VERSION}-linux-amd64.tar.gz" | sudo tar -C /usr/local/bin --strip-components=1 -xzf - linux-amd64/helm

contrib/terraform/openstack/provision-k8s-containerd.sh

@@ -0,0 +1,31 @@
+#!/bin/bash
+
+# ensure running as root
+if [ "$(id -u)" != "0" ]; then
+  exec sudo "$0" "$@"
+fi
+
+CONTAINERD_URL=$(curl -s https://api.github.com/repos/containerd/containerd/releases/latest | jq -r '.assets[] | select(.browser_download_url | contains("cri-containerd-cni") and endswith("linux-amd64.tar.gz")) | .browser_download_url')
+curl -L "${CONTAINERD_URL}" | sudo tar --no-overwrite-dir -C / -xz
+
+systemctl enable containerd
+
+CNI_VERSION=$(curl -s https://api.github.com/repos/containernetworking/plugins/releases/latest | jq -r '.tag_name')
+ARCH="amd64"
+mkdir -p /opt/cni/bin
+curl -L "https://github.com/containernetworking/plugins/releases/download/${CNI_VERSION}/cni-plugins-linux-${ARCH}-${CNI_VERSION}.tgz" | sudo tar -C /opt/cni/bin -xz
+
+DOWNLOAD_DIR=/usr/local/bin
+mkdir -p $DOWNLOAD_DIR
+
+RELEASE="$(curl -sSL https://dl.k8s.io/release/stable.txt)"
+cd $DOWNLOAD_DIR
+curl -L --remote-name-all https://storage.googleapis.com/kubernetes-release/release/${RELEASE}/bin/linux/amd64/{kubeadm,kubelet,kubectl}
+chmod +x {kubeadm,kubelet,kubectl}
+
+RELEASE_VERSION=$(curl -s https://api.github.com/repos/kubernetes/release/releases/latest | jq -r '.name')
+curl -sSL "https://raw.githubusercontent.com/kubernetes/release/${RELEASE_VERSION}/cmd/kubepkg/templates/latest/deb/kubelet/lib/systemd/system/kubelet.service" | sed "s:/usr/bin:${DOWNLOAD_DIR}:g" | tee /etc/systemd/system/kubelet.service
+mkdir -p /etc/systemd/system/kubelet.service.d
+curl -sSL "https://raw.githubusercontent.com/kubernetes/release/${RELEASE_VERSION}/cmd/kubepkg/templates/latest/deb/kubeadm/10-kubeadm.conf" | sed "s:/usr/bin:${DOWNLOAD_DIR}:g" | tee /etc/systemd/system/kubelet.service.d/10-kubeadm.conf
+
+systemctl enable kubelet

contrib/terraform/openstack/provision.sh

@@ -0,0 +1,37 @@
+#!/bin/bash
+
+# ensure running as root
+if [ "$(id -u)" != "0" ]; then
+  exec sudo "$0" "$@"
+fi
+
+set -eux
+
+# TODO(vadorovsky): Include BPF as an enabled LSM in openSUSE kernel config.
+sed -i -e "s/GRUB_CMDLINE_LINUX=.*/GRUB_CMDLINE_LINUX=\"lsm=bpf,integrity\"/" \
+    /etc/default/grub
+grub2-mkconfig -o /boot/grub2/grub.cfg
+
+# Load br_netfilter
+cat >> /etc/modules-load.d/99-k8s.conf << EOF
+br_netfilter
+EOF
+
+# Network-related sysctls
+cat >> /etc/sysctl.d/99-k8s.conf << EOF
+net.bridge.bridge-nf-call-ip6tables = 1
+net.bridge.bridge-nf-call-iptables = 1
+net.ipv4.ip_forward = 1
+net.ipv4.conf.all.forwarding = 1
+EOF
+
+# Add 9p drivers to dracut
+cat >> /etc/dracut.conf.d/90-9p.conf << EOF
+# Add 9p 9pnet and 9pnet_virtio modules
+add_drivers+=" 9p 9pnet 9pnet_virtio "
+EOF
+
+# Rebuild initrd with dracut
+mkinitrd
+
+exit 0