diff --git a/contrib/terraform/libvirt/worker.tf b/contrib/terraform/libvirt/worker.tf index 4a76cb5..c532547 100644 --- a/contrib/terraform/libvirt/worker.tf +++ b/contrib/terraform/libvirt/worker.tf @@ -146,11 +146,6 @@ resource "null_resource" "worker_provision_k8s_containerd" { type = "ssh" } - provisioner "file" { - source = "../../../target/debug/lockc.tar.gz" - destination = "/home/opensuse/lockc.tar.gz" - } - provisioner "remote-exec" { script = "provision-k8s-containerd.sh" } @@ -177,13 +172,13 @@ export sshopts="-o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -oCo if ! ssh $sshopts $user@$host 'sudo needs-restarting -r'; then ssh $sshopts $user@$host sudo reboot || : export delay=5 - # # wait for node reboot completed - # # lol, doesn't work - # while ! ssh $sshopts $user@$host 'sudo needs-restarting -r'; do - # sleep $delay - # delay=$((delay+1)) - # [ $delay -gt 60 ] && exit 1 - # done + # wait for node reboot completed + while ! ssh $sshopts $user@$host 'sudo needs-restarting -r'; do + sleep $delay + delay=$((delay+1)) + [ $delay -gt 60 ] && exit 1 + ssh $sshopts $user@$host 'sudo needs-restarting -r' + done fi EOT } diff --git a/contrib/terraform/openstack/cloud-init/common.tpl b/contrib/terraform/openstack/cloud-init/common.tpl index 461f9c6..e82fb3b 100644 --- a/contrib/terraform/openstack/cloud-init/common.tpl +++ b/contrib/terraform/openstack/cloud-init/common.tpl @@ -1,10 +1,13 @@ #cloud-config +# set hostname +hostname: ${hostname} + # set locale -locale: en_US.UTF-8 +locale: ${locale} # en_US.UTF-8 # set timezone -timezone: Etc/UTC +timezone: ${timezone} # Etc/UTC # Inject the public keys ssh_authorized_keys: @@ -27,11 +30,6 @@ ${repositories} solver.onlyRequires: "true" download.use_deltarpm: "true" -#packages: - -# set hostname -hostname: ${hostname} - runcmd: # workaround for bsc#1119397 . If this is not called, /etc/resolv.conf is empty - netconfig -f update @@ -42,8 +40,11 @@ runcmd: - sshd -t || echo "ssh syntax failure" - systemctl restart sshd # Set node's hostname from DHCP server - - sed -i -e '/^DHCLIENT_SET_HOSTNAME/s/^.*$/DHCLIENT_SET_HOSTNAME=\"${hostname_from_dhcp}\"/' /etc/sysconfig/network/dhcp + - sed -i -e '/^DHCLIENT_SET_HOSTNAME/s/^.*$/DHCLIENT_SET_HOSTNAME=\"yes\"/' /etc/sysconfig/network/dhcp - systemctl restart wicked + # Refresh repos and upgrade + - zypper ref + - zypper dup -y --allow-vendor-change --replacefiles ${commands} final_message: "The system is finally up, after $UPTIME seconds" diff --git a/contrib/terraform/openstack/cloud-init/repository.tpl b/contrib/terraform/openstack/cloud-init/repository.tpl index 678e4a8..9ca3467 100644 --- a/contrib/terraform/openstack/cloud-init/repository.tpl +++ b/contrib/terraform/openstack/cloud-init/repository.tpl @@ -1,6 +1,7 @@ - id: ${repository_name} name: ${repository_name} baseurl: ${repository_url} + priority: 90 enabled: 1 autorefresh: 1 gpgcheck: 0 diff --git a/contrib/terraform/openstack/deploy-kubernetes.sh b/contrib/terraform/openstack/deploy-kubernetes.sh index 7a93c98..b105919 100644 --- a/contrib/terraform/openstack/deploy-kubernetes.sh +++ b/contrib/terraform/openstack/deploy-kubernetes.sh @@ -1,70 +1,65 @@ #!/bin/bash #shellcheck disable=SC2145,SC2016 + +set -eux + log() { (>&1 echo -e "$@") ; } -cmd() { log "$@" ; } info() { log "[ INFO ] $@" ; } error() { (>&2 echo -e "[ ERROR ] $@") ;} if [ -z "${TR_STACK}" ] || [ -z "${TR_LB_IP}" ] || \ - [ -z "$TR_MASTER_IPS" ] || [ -z "$TR_WORKER_IPS" ] || \ - [ -z "${TR_USERNAME}" ]; then - error '$TR_STACK $TR_LB_IP $TR_MASTER_IPS $TR_WORKER_IPS $TR_USERNAME must be specified' + [ -z "$TR_MASTER_IPS" ] || [ -z "${TR_USERNAME}" ]; then + error '$TR_STACK $TR_LB_IP $TR_MASTER_IPS $TR_USERNAME must be specified' exit 1 fi +sleep 5 + +CILIUM_VERSION=$(curl -s https://api.github.com/repos/cilium/cilium/releases/latest | jq -r '.tag_name' | sed -e 's/^v//') + info "### Run following commands to bootstrap Kubernetes cluster:\\n" -cmd "" i=0 for MASTER in $TR_MASTER_IPS; do - cmd "ssh -o 'StrictHostKeyChecking no' -l ${TR_USERNAME} ${MASTER} /bin/bash <> /etc/modules-load.d/99-k8s.conf << EOF +br_netfilter +EOF + +# Network-related sysctls +cat >> /etc/sysctl.d/99-k8s.conf << EOF +net.bridge.bridge-nf-call-ip6tables = 1 +net.bridge.bridge-nf-call-iptables = 1 +net.ipv4.ip_forward = 1 +net.ipv4.conf.all.forwarding = 1 +EOF + +# Add 9p drivers to dracut +cat >> /etc/dracut.conf.d/90-9p.conf << EOF +# Add 9p 9pnet and 9pnet_virtio modules +add_drivers+=" 9p 9pnet 9pnet_virtio " +EOF + +# Rebuild initrd with dracut +mkinitrd + +exit 0 diff --git a/contrib/terraform/openstack/variables.tf b/contrib/terraform/openstack/variables.tf index 4b9edfa..7549951 100644 --- a/contrib/terraform/openstack/variables.tf +++ b/contrib/terraform/openstack/variables.tf @@ -79,6 +79,16 @@ variable "stack_name" { description = "Identifier to make all your resources unique and avoid clashes with other users of this terraform project" } +variable "locale" { + description = "System locales to set on all the nodes" + default = "en_US.UTF-8" +} + +variable "timezone" { + description = "Timezone to set on all the nodes" + default = "Etc/UTC" +} + variable "authorized_keys" { type = list(string) default = [] @@ -121,8 +131,3 @@ variable "ca_file" { default = "" description = "Used to specify the path to your custom CA file" } - -variable "hostname_from_dhcp" { - default = true - description = "Set node's hostname from DHCP server" -} diff --git a/contrib/terraform/openstack/worker-instance.tf b/contrib/terraform/openstack/worker-instance.tf index b587be3..67ac903 100644 --- a/contrib/terraform/openstack/worker-instance.tf +++ b/contrib/terraform/openstack/worker-instance.tf @@ -10,25 +10,26 @@ data "template_file" "worker_repositories" { data "template_file" "worker_commands" { template = file("cloud-init/commands.tpl") - count = join("", var.packages) == "" ? 0 : 1 + count = length(var.packages) vars = { packages = join(", ", var.packages) } } -data "template_file" "worker-cloud-init" { +data "template_file" "worker_cloud_init" { template = file("cloud-init/common.tpl") count = var.workers vars = { + hostname = "${var.stack_name}-k8s-worker${count.index}" + locale = var.locale + timezone = var.timezone + username = var.username authorized_keys = join("\n", formatlist(" - %s", var.authorized_keys)) repositories = join("\n", data.template_file.worker_repositories.*.rendered) commands = join("\n", data.template_file.worker_commands.*.rendered) - username = var.username ntp_servers = join("\n", formatlist(" - %s", var.ntp_servers)) - hostname = "${var.stack_name}-k8s-worker${count.index}" - hostname_from_dhcp = var.hostname_from_dhcp } } @@ -41,6 +42,7 @@ resource "openstack_blockstorage_volume_v2" "worker_vol" { resource "openstack_compute_volume_attach_v2" "worker_vol_attach" { count = var.workers_vol_enabled ? var.workers : 0 instance_id = element(openstack_compute_instance_v2.worker.*.id, count.index) + volume_id = element( openstack_blockstorage_volume_v2.worker_vol.*.id, count.index, @@ -68,7 +70,7 @@ resource "openstack_compute_instance_v2" "worker" { openstack_networking_secgroup_v2.common.id, ] - user_data = data.template_file.worker-cloud-init[count.index].rendered + user_data = data.template_file.worker_cloud_init[count.index].rendered } resource "openstack_networking_floatingip_v2" "worker_ext" { @@ -79,10 +81,12 @@ resource "openstack_networking_floatingip_v2" "worker_ext" { resource "openstack_compute_floatingip_associate_v2" "worker_ext_ip" { depends_on = [openstack_compute_instance_v2.worker] count = var.workers + floating_ip = element( openstack_networking_floatingip_v2.worker_ext.*.address, count.index, ) + instance_id = element(openstack_compute_instance_v2.worker.*.id, count.index) } @@ -109,8 +113,55 @@ resource "null_resource" "worker_wait_cloudinit" { } } +resource "null_resource" "worker_provision" { + depends_on = [ + null_resource.worker_wait_cloudinit + ] + count = var.workers + + connection { + host = element( + openstack_compute_floatingip_associate_v2.worker_ext_ip.*.floating_ip, + count.index + ) + user = "opensuse" + type = "ssh" + } + + provisioner "remote-exec" { + script = "provision.sh" + } +} + +resource "null_resource" "worker_provision_k8s_containerd" { + depends_on = [ + null_resource.worker_provision + ] + count = var.workers + + connection { + host = element( + openstack_compute_floatingip_associate_v2.worker_ext_ip.*.floating_ip, + count.index + ) + user = var.username + type = "ssh" + } + + provisioner "remote-exec" { + script = "provision-k8s-containerd.sh" + } + + provisioner "remote-exec" { + script = "provision-k8s-containerd-cp.sh" + } +} + + resource "null_resource" "worker_reboot" { - depends_on = [null_resource.worker_wait_cloudinit] + depends_on = [ + null_resource.worker_provision_k8s_containerd, + ] count = var.workers provisioner "local-exec" { @@ -132,6 +183,7 @@ if ! ssh $sshopts $user@$host 'sudo needs-restarting -r'; then sleep $delay delay=$((delay+1)) [ $delay -gt 30 ] && exit 1 + ssh $sshopts $user@$host 'sudo needs-restarting -r' done fi EOT