configure

#!/usr/bin/env bash

set -eux

source $SNAP/actions/common/utils.sh

use_snap_env

# Make sure either the install hook has run or we are refreshing an already existing snap as indicated
# by the existence of certificates.
if [ ! -f "${SNAP_DATA}/var/lock/installed.lock" ] && [ ! -f ${SNAP_DATA}/certs/csr.conf.template ]
then
  exit 0
fi

if is_strict
then
  echo "Checking snap interfaces..."

  check_snap_interfaces # Check for interfaces but do not start until this script has run.
else
  # In classic we do not make use of the status flag. Here we clean any "blocked" message that may have
  # come from a failed strict deployment
  snapctl set-health okay
fi

need_api_restart=false
need_cluster_agent_restart=false
need_proxy_restart=false
need_kubelet_restart=false
need_controller_restart=false
need_scheduler_restart=false

# Try to symlink /var/lib/kubelet so that most kubelet device plugins work out of the box.
if ! [ -e /var/lib/kubelet ] && ln -s $SNAP_COMMON/var/lib/kubelet /var/lib/kubelet; then
  echo "/var/lib/kubelet linked to $SNAP_COMMON"
fi

SNAP_DATA_CURRENT=`echo "${SNAP_DATA}" | sed -e "s,${SNAP_REVISION},current,"`

# Try to symlink /var/lib/calico so that the Calico CNI plugin picks up the mtu configuration.
if ! [ -e /var/lib/calico ]; then
  if ln -s $SNAP_DATA_CURRENT/var/lib/calico /var/lib/calico; then
    echo "/var/lib/calico linked to $SNAP_DATA_CURRENT/var/lib/calico"
  fi
fi

# Try to symlink standard CNI Kubernetes directories.
if ! [ -e /etc/cni/net.d ]; then
  if mkdir -p /etc/cni && ln -s $SNAP_DATA_CURRENT/args/cni-network /etc/cni/net.d; then
    echo "/etc/cni/net.d linked to $SNAP_DATA_CURRENT/args/cni-network"
  fi
fi
if ! [ -e /opt/cni/bin ]; then
  if mkdir -p /opt/cni && ln -s $SNAP_DATA_CURRENT/opt/cni/bin /opt/cni/bin; then
    echo "/opt/cni/bin linked to $SNAP_DATA_CURRENT/opt/cni/bin"
  fi
fi

# If the configurations directory is missing from SNAP_COMMON, we are upgrading from an older MicroK8s version.
if [ ! -d "${SNAP_COMMON}/etc/launcher" ]
then
  mkdir -p "${SNAP_COMMON}/etc/launcher"
fi

# snap set microk8s config="$(cat config.yaml)"
config="$(snapctl get config || true)"
if [ ! -z "${config}" ]
then
  # Only write config file if not already applied.
  applied_config="$(cat ${SNAP_COMMON}/etc/launcher/snap-set.yaml.applied || true)"
  if [ -z "${applied_config}" ] || [ "${config}" != "${applied_config}" ]
  then
    echo "${config}" > "${SNAP_COMMON}/etc/launcher/snap-set.yaml"
  fi
fi

# If the addons directory is missing from SNAP_COMMON, then we are upgrading from an older MicroK8s version.
if [ ! -d "${SNAP_COMMON}/addons" ]
then
  mkdir -p ${SNAP_COMMON}/addons
  snap_current=`echo "${SNAP}" | sed -e "s,${SNAP_REVISION},current,"`
  for addon in $(cat "${SNAP}/addons/.auto-add"); do
    "${SNAP}/git.wrapper" clone "${snap_current}/addons/${addon}" "${SNAP_COMMON}/addons/${addon}"
  done
fi

if [ ! -d "${SNAP_COMMON}/plugins" ]
then
  mkdir -p ${SNAP_COMMON}/plugins
fi

#Allow the ability to add external IPs to the csr, by moving the csr.conf.template to SNAP_DATA
# TODO(neoaggelos): investigate if this is needed
if [ ! -f ${SNAP_DATA}/certs/csr.conf.template ]
then
   cp ${SNAP}/certs/csr.conf.template ${SNAP_DATA}/certs/csr.conf.template
fi

# Enable the aggregation layer
if ! grep "requestheader-client-ca-file" ${SNAP_DATA}/args/kube-apiserver
then
  echo "Patching requestheader-client-ca-file argument"
  # Add a new line at the end
  echo "" >> ${SNAP_DATA}/args/kube-apiserver
  echo "--requestheader-client-ca-file=\${SNAP_DATA}/certs/front-proxy-ca.crt" >> ${SNAP_DATA}/args/kube-apiserver
  need_api_restart=true
fi

# Enable the aggregation layer (continue)
if ! grep -E -- '--(requestheader-allowed-names|requestheader-extra-headers-prefix|requestheader-group-headers|requestheader-username-headers|proxy-client-cert-file|proxy-client-key-file)=' ${SNAP_DATA}/args/kube-apiserver
then
  echo "Enabling Enable the aggregation layer"
  echo "" >> ${SNAP_DATA}/args/kube-apiserver
  echo '--requestheader-allowed-names=front-proxy-client' >> ${SNAP_DATA}/args/kube-apiserver
  echo '--requestheader-extra-headers-prefix=X-Remote-Extra-' >> ${SNAP_DATA}/args/kube-apiserver
  echo '--requestheader-group-headers=X-Remote-Group' >> ${SNAP_DATA}/args/kube-apiserver
  echo '--requestheader-username-headers=X-Remote-User' >> ${SNAP_DATA}/args/kube-apiserver
  echo '--proxy-client-cert-file=${SNAP_DATA}/certs/front-proxy-client.crt' >> ${SNAP_DATA}/args/kube-apiserver
  echo '--proxy-client-key-file=${SNAP_DATA}/certs/front-proxy-client.key' >> ${SNAP_DATA}/args/kube-apiserver
  need_api_restart=true
fi

# Patch for issue: https://github.com/canonical/microk8s/issues/121
if grep -e  "requestheader-client-ca-file=/var/snap/microk8s/.../certs/ca.crt"  ${SNAP_DATA}/args/kube-apiserver
then
  "$SNAP/bin/sed" -i 's@requestheader-client-ca-file=/var/snap/microk8s/.../certs/ca.crt@requestheader-client-ca-file=\${SNAP_DATA}/certs/ca.crt@g' ${SNAP_DATA}/args/kube-apiserver
fi

# Patch for issue: https://github.com/canonical/microk8s/issues/721
if grep -F 'requestheader-client-ca-file=${SNAP_DATA}/certs/ca.crt' ${SNAP_DATA}/args/kube-apiserver
then
  "$SNAP/bin/sed" -i 's@requestheader-client-ca-file=${SNAP_DATA}/certs/ca.crt@requestheader-client-ca-file=${SNAP_DATA}/certs/front-proxy-ca.crt@g' ${SNAP_DATA}/args/kube-apiserver
fi

# Create the locks directory
mkdir -p ${SNAP_DATA}/var/lock/

# Create tmp directory
mkdir -p ${SNAP_DATA}/tmp/

# This will allow us to refresh the snap to the more secure version.
# We need to make sure the client certificate used in microk8s kubectl is available under $SNAP_DATA
# TODO(neoaggelos): investigate whether this is needed
if [ ! -f ${SNAP_DATA}/credentials/client.config ]
then
  echo "Patching client config location"
  mkdir -p ${SNAP_DATA}/credentials/
  cp ${SNAP}/client.config ${SNAP_DATA}/credentials/
fi

# copy kubectl-env
if [ ! -e ${SNAP_DATA}/args/kubectl-env ] && grep -e "\-\-kubeconfig=\${SNAP_DATA}/credentials/client.config" ${SNAP_DATA}/args/kubectl
then
  echo "Making sure we have kubectl environment file"
  cp ${SNAP}/default-args/kubectl-env ${SNAP_DATA}/args/kubectl-env
  skip_opt_in_config kubeconfig kubectl
fi

# copy kubectl
if [ ! -e ${SNAP_DATA}/args/kubectl ]
then
  echo "Making sure we have kubectl arguments file"
  cp ${SNAP}/default-args/kubectl ${SNAP_DATA}/args/kubectl
fi

# copy traefik
if [ ! -e ${SNAP_DATA}/args/traefik ]
then
  echo "Making sure we have traefik configuration"
  cp -r ${SNAP}/default-args/traefik ${SNAP_DATA}/args/
fi

# copy apiserver-proxy
if [ ! -e ${SNAP_DATA}/args/apiserver-proxy ]
then
  echo "Making sure we have apiserver-proxy configuration"
  cp -r ${SNAP}/default-args/apiserver-proxy ${SNAP_DATA}/args/
fi

# (1.24 -> 1.25) migrate from traefik to apiserver-proxy
if [ -e ${SNAP_DATA}/var/lock/no-traefik ]
then
  touch ${SNAP_DATA}/var/lock/no-apiserver-proxy
fi

# Upgrading to containerd
if [ ! -e ${SNAP_DATA}/args/containerd ] ||
   grep -e "\-\-docker unix://\${SNAP_DATA}/docker.sock" ${SNAP_DATA}/args/kubelet
then
  echo "Making sure we have containerd file"
  cp ${SNAP_DATA}/args/containerd ${SNAP_DATA}/args/containerd.backup || true
  cp ${SNAP}/default-args/containerd ${SNAP_DATA}/args/containerd
  cp ${SNAP_DATA}/args/containerd-template.toml ${SNAP_DATA}/args/containerd-template.toml.backup || true
  cp ${SNAP}/default-args/containerd-template.toml ${SNAP_DATA}/args/containerd-template.toml
  cp ${SNAP_DATA}/args/containerd-env ${SNAP_DATA}/args/containerd-env.backup || true
  cp ${SNAP}/default-args/containerd-env ${SNAP_DATA}/args/containerd-env

  cp -r ${SNAP}/default-args/cni-network ${SNAP_DATA}/args/

  cp ${SNAP}/default-args/ctr ${SNAP_DATA}/args/ctr

  refresh_opt_in_config container-runtime remote kubelet
  refresh_opt_in_config container-runtime-endpoint \${SNAP_COMMON}/run/containerd.sock kubelet

  skip_opt_in_config docker-root kubelet
  skip_opt_in_config docker kubelet
  skip_opt_in_config docker-endpoint kubelet

  snapctl restart ${SNAP_NAME}.daemon-containerd
  need_kubelet_restart=true

  if [ -e ${SNAP_DATA}/args/dockerd ] && grep -e "default-runtime=nvidia" ${SNAP_DATA}/args/dockerd
  then
    # Deployment used to run docker with nvidia enabled we need to enable nvidia on containerd
    # Allow for kubelet and containerd to restart
    sleep 10
    ${SNAP}/microk8s-enable.wrapper gpu
  fi

fi

# Install default-hooks
if ! [ -d "${SNAP_COMMON}/hooks" ]
then
  cp -r --preserve=mode ${SNAP}/default-hooks ${SNAP_COMMON}/hooks
fi

# (1.28 -> 1.29) Install 10-pods-restart hook if missing
if ! [ -e "${SNAP_COMMON}/hooks/reconcile.d/10-pods-restart" ]; then
  cp -r --preserve=mode "${SNAP}/default-hooks/reconcile.d/10-pods-restart" "${SNAP_COMMON}/hooks/reconcile.d/10-pods-restart"
fi

# Make sure the server certificate includes the IP we are using
if [ "$(produce_certs)" == "1" ]
then
    rm -rf .srl
    need_api_restart=true
    need_proxy_restart=true
    need_cluster_agent_restart=true
fi

# Make containerd stream server listen to localhost
if [ -e ${SNAP_DATA}/args/containerd-template.toml ] && grep -e "stream_server_address = \"\"" ${SNAP_DATA}/args/containerd-template.toml
then
    "$SNAP/bin/sed" -i 's@stream_server_address = ""@stream_server_address = "127.0.0.1"@g' ${SNAP_DATA}/args/containerd-template.toml
    if grep -e "stream_server_port = \"10010\"" ${SNAP_DATA}/args/containerd-template.toml
    then
        "$SNAP/bin/sed" -i 's@stream_server_port = "10010"@stream_server_port = "0"@g' ${SNAP_DATA}/args/containerd-template.toml
    fi
    snapctl restart ${SNAP_NAME}.daemon-containerd
    need_kubelet_restart=true
fi

# With v1.15 allow-privileged is removed from kubelet
if grep -e "\-\-allow-privileged" ${SNAP_DATA}/args/kubelet
then
  echo "Patching 1.15 allow-privileged"
  "${SNAP}/bin/sed" -i '/allow-privileged/d' ${SNAP_DATA}/args/kubelet
    need_kubelet_restart=true
fi

# Add option to support kata containers
if [ -e "${SNAP_DATA}/args/containerd-env" ] &&
   ! grep -e "KATA_PATH" ${SNAP_DATA}/args/containerd-env
then
  echo "" >> "${SNAP_DATA}/args/containerd-env"
  echo "# You can set the of the kata containers runtime here." >> "${SNAP_DATA}/args/containerd-env"
  echo "#" >> "${SNAP_DATA}/args/containerd-env"
  echo "# KATA_PATH=" >> "${SNAP_DATA}/args/containerd-env"
  echo "#" >> "${SNAP_DATA}/args/containerd-env"
  echo "PATH=\$PATH:\$KATA_PATH" >> "${SNAP_DATA}/args/containerd-env"
fi

# Add option to support kata containers
if [ -e "${SNAP_DATA}/args/containerd-template.toml" ] &&
   ! grep -e "io.containerd.kata.v2" ${SNAP_DATA}/args/containerd-template.toml
then
  KATA_HANDLER_BEFORE='\[plugins."io.containerd.grpc.v1.cri".containerd.runtimes.nvidia-container-runtime\]'
  KATA_HANDLER_AFTER='    [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.kata]
      runtime_type = "io.containerd.kata.v2"
      [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.kata.options]
        BinaryName = "kata-runtime"
        '
  CD_TOML="${SNAP_DATA}/args/containerd-template.toml"
  CD_TOML_TMP="${SNAP_DATA}/args/containerd-template.toml.tmp"

  "$SNAP/usr/bin/gawk" -v kata="${KATA_HANDLER_AFTER}" '/'${KATA_HANDLER_BEFORE}'/{print kata }1' ${CD_TOML} > ${CD_TOML_TMP}
  mv ${CD_TOML_TMP} ${CD_TOML}
fi

for dir in ${SNAP_COMMON}/plugins ${SNAP_COMMON}/addons ${SNAP_DATA}/credentials/ ${SNAP_DATA}/certs/ ${SNAP_DATA}/args/ ${SNAP_DATA}/var/lock ${SNAP_DATA}/tmp/ ${SNAP_COMMON}/hooks
do
  chmod -R ug+rwX ${dir}
  chmod -R o-rwX ${dir}
done

group=$(get_microk8s_or_cis_group)

# Try to create the snap_microk8s group. Do not fail the installation if something goes wrong
if ! getent group ${group} >/dev/null 2>&1
then
  groupadd ${group} || true
fi

if getent group ${group} >/dev/null 2>&1 && ! [ -e "${SNAP_DATA}/var/lock/cis-hardening" ]
then
  chgrp ${group} -R ${SNAP_COMMON}/plugins ${SNAP_COMMON}/addons ${SNAP_DATA}/credentials/ ${SNAP_DATA}/certs/ ${SNAP_DATA}/args/ ${SNAP_DATA}/var/lock/ ${SNAP_DATA}/var/kubernetes/backend/ ${SNAP_DATA}/tmp/ ${SNAP_COMMON}/hooks || true
fi

if [ -e "${SNAP_DATA}/var/lock/cis-hardening" ]
then
  chmod -R g-wr ${SNAP_COMMON}/plugins ${SNAP_COMMON}/addons ${SNAP_DATA}/credentials/ ${SNAP_DATA}/certs/ ${SNAP_DATA}/args/ ${SNAP_DATA}/var/lock/ ${SNAP_DATA}/var/kubernetes/backend/ ${SNAP_DATA}/tmp/ ${SNAP_COMMON}/hooks || true
  chmod -R o-wr ${SNAP_COMMON}/plugins ${SNAP_COMMON}/addons ${SNAP_DATA}/credentials/ ${SNAP_DATA}/certs/ ${SNAP_DATA}/args/ ${SNAP_DATA}/var/lock/ ${SNAP_DATA}/var/kubernetes/backend/ ${SNAP_DATA}/tmp/ ${SNAP_COMMON}/hooks || true

  if ! is_strict && [ -e /etc/systemd/system/snap.microk8s.daemon-kubelite.service ]
  then
    chmod -R g-wr /etc/systemd/system/snap.microk8s.daemon-kubelite.service
    chmod -R o-wr /etc/systemd/system/snap.microk8s.daemon-kubelite.service
  fi
fi

if ! is_strict
then
  try_copy_users_to_snap_microk8s
fi

# as only one cni bin dir can be used we will use the one in SNAP_DATA but have links to
# the real CNI plugins we distribute in SNAP
mkdir -p "${SNAP_DATA}/opt/cni/bin/"
(
  cd "${SNAP}/opt/cni/bin/"
  MY_SNAP_DIR=$(dirname "${SNAP}")
  for i in *; do ln -sf "${MY_SNAP_DIR}/current/opt/cni/bin/$i" "${SNAP_DATA}/opt/cni/bin/${i}"; done
)

if ! [ -e "${SNAP_DATA}/opt/cni/bin/flanneld" ]
then
  # cover situation where cilium was installed prior to this update
  if [ -f "${SNAP_DATA}/opt/cni/bin/loopback" ] && [ -f "${SNAP}/opt/cni/bin/loopback" ]; then
    rm -f "${SNAP_DATA}/opt/cni/bin/loopback"
  fi
fi

if ! [ -f "${SNAP_DATA}/args/flanneld" ]
then
  mkdir -p ${SNAP_DATA}/args/cni-network/
  cp -r ${SNAP}/default-args/cni-network/flannel.conflist ${SNAP_DATA}/args/cni-network/
  cp ${SNAP}/default-args/flanneld ${SNAP_DATA}/args/
  cp ${SNAP}/default-args/flannel-template.conflist ${SNAP_DATA}/args/
  cp ${SNAP}/default-args/flannel-network-mgr-config ${SNAP_DATA}/args/
  snapctl restart ${SNAP_NAME}.daemon-etcd
  snapctl restart ${SNAP_NAME}.daemon-containerd
  snapctl restart ${SNAP_NAME}.daemon-flanneld
fi

if grep -e "etcd.socket:2379" ${SNAP_DATA}/args/etcd
then
  echo "Using a port for etcd"
  # TODO: Do something smart in selecting a port
  refresh_opt_in_config advertise-client-urls https://\${DEFAULT_INTERFACE_IP_ADDR}:12379 etcd
  refresh_opt_in_config listen-client-urls https://0.0.0.0:12379 etcd
  refresh_opt_in_config client-cert-auth true etcd
  refresh_opt_in_config trusted-ca-file \${SNAP_DATA}/certs/ca.crt etcd
  refresh_opt_in_config cert-file \${SNAP_DATA}/certs/server.crt etcd
  refresh_opt_in_config key-file \${SNAP_DATA}/certs/server.key etcd
  snapctl restart ${SNAP_NAME}.daemon-etcd

  refresh_opt_in_config etcd-servers https://127.0.0.1:12379 kube-apiserver
  refresh_opt_in_config etcd-cafile \${SNAP_DATA}/certs/ca.crt kube-apiserver
  refresh_opt_in_config etcd-certfile \${SNAP_DATA}/certs/server.crt kube-apiserver
  refresh_opt_in_config etcd-keyfile \${SNAP_DATA}/certs/server.key kube-apiserver
  need_api_restart=true
fi

if ! grep -e "service-account-issuer" ${SNAP_DATA}/args/kube-apiserver
then
  echo "--service-account-issuer='https://kubernetes.default.svc'" >> ${SNAP_DATA}/args/kube-apiserver
  need_api_restart=true
fi

if ! grep -e "service-account-signing-key-file" ${SNAP_DATA}/args/kube-apiserver
then
  echo '--service-account-signing-key-file=${SNAP_DATA}/certs/serviceaccount.key' >> ${SNAP_DATA}/args/kube-apiserver
  need_api_restart=true
fi

# Disable the watchlist feature until k8s-dqlite can handle it
if ! grep -e "WatchList" ${SNAP_DATA}/args/kube-apiserver
then
  if ! grep -e "--feature-gates=" ${SNAP_DATA}/args/kube-apiserver
  then
    echo "" >> ${SNAP_DATA}/args/kube-apiserver
    echo '--feature-gates=WatchList=false' >> ${SNAP_DATA}/args/kube-apiserver
  else
    # Find the line starting with --feature-gates= and append ,WatchList=false
    "${SNAP}/bin/sed" -i '/^--feature-gates=/ s/$/,WatchList=false/' "$SNAP_DATA/args/kube-apiserver"
  fi

  need_api_restart=true
fi

# RemoveSelfLink feature flag is removed after 1.24
if grep -e "feature-gates=RemoveSelfLink" ${SNAP_DATA}/args/kube-apiserver
then
  "${SNAP}/bin/sed" -i '/feature-gates=RemoveSelfLink/d' "$SNAP_DATA/args/kube-apiserver"
  need_api_restart=true
fi

if remove_docker_specific_args
then
  need_kubelet_restart=true
fi

# scheduler --address flag is removed after 1.24
if grep -e "--address=" ${SNAP_DATA}/args/kube-scheduler
then
  "${SNAP}/bin/sed" -i '/--address=/d' "$SNAP_DATA/args/kube-scheduler"
fi

# controller-manager --address flag is removed after 1.24
if grep -e "--address=" ${SNAP_DATA}/args/kube-controller-manager
then
  "${SNAP}/bin/sed" -i '/--address=/d' "$SNAP_DATA/args/kube-controller-manager"
  need_api_restart=true
fi

if ! grep '\-\-enable\-v2' ${SNAP_DATA}/args/etcd
then
  refresh_opt_in_config enable-v2 true etcd
  snapctl restart ${SNAP_NAME}.daemon-etcd
fi

if [ -e ${SNAP_DATA}/var/lock/clustered.lock ]
then
  if grep -e "\-\-etcd-cafile /var/snap/microk8s/.*/ca.remote.crt" ${SNAP_DATA}/args/flanneld
  then
    skip_opt_in_config etcd-cafile flanneld
    refresh_opt_in_config etcd-cafile \${SNAP_DATA}/certs/ca.remote.crt flanneld
  fi

  if grep -e "\-\-etcd-certfile /var/snap/microk8s/.*/server.remote.crt" ${SNAP_DATA}/args/flanneld
  then
    skip_opt_in_config etcd-certfile flanneld
    refresh_opt_in_config etcd-certfile \${SNAP_DATA}/certs/server.remote.crt flanneld
  fi
fi

# This patches flanneld conf template by adding cniversion if it does not exist.
if [ -e ${SNAP_DATA}/args/flannel-template.conflist ] && ! grep -e "cniVersion" ${SNAP_DATA}/args/flannel-template.conflist
then
    "$SNAP/bin/sed" -i 's@"name": "microk8s-flannel-network",@"name": "microk8s-flannel-network",\n    "cniVersion": "0.3.1",@g' ${SNAP_DATA}/args/flannel-template.conflist
    snapctl restart ${SNAP_NAME}.daemon-flanneld
    snapctl restart ${SNAP_NAME}.daemon-containerd
fi

if [ ! -f ${SNAP_DATA}/args/cluster-agent ]
then
   cp ${SNAP}/default-args/cluster-agent ${SNAP_DATA}/args/cluster-agent
fi

if ! grep -e "\-\-timeout" ${SNAP_DATA}/args/cluster-agent
then
  refresh_opt_in_config timeout 240 cluster-agent
  snapctl restart ${SNAP_NAME}.daemon-containerd
fi

if ! grep -e "\-\-ip-masq" ${SNAP_DATA}/args/flanneld
then
  refresh_opt_in_config ip-masq true flanneld
  snapctl restart ${SNAP_NAME}.daemon-flanneld
fi

if grep -e "\-\-cluster-cidr=10.152.183.0/24" ${SNAP_DATA}/args/kube-proxy
then
  refresh_opt_in_config cluster-cidr 10.1.0.0/16 kube-proxy
  need_proxy_restart=true
fi

if [ -e ${SNAP_DATA}/var/lock/stopped.lock ]
then
  snapctl stop ${SNAP_NAME}.daemon-kubelite
fi

# Enable kubelite
if ! [ -e ${SNAP_DATA}/var/lock/lite.lock ]
then
  touch "${SNAP_DATA}/var/lock/lite.lock"
  if ! [ -e ${SNAP_DATA}/args/kubelite ]
  then
    cp ${SNAP}/default-args/kubelite ${SNAP_DATA}/args/kubelite
  fi
  date
  if [ -e ${SNAP_DATA}/var/lock/stopped.lock ]
  then
    snapctl stop ${SNAP_NAME}.daemon-kubelite
  else
    snapctl start ${SNAP_NAME}.daemon-kubelite
  fi
fi

if ! [ -e ${SNAP_DATA}/args/kubelite ]
then
  cp ${SNAP}/default-args/kubelite ${SNAP_DATA}/args/kubelite
  need_api_restart=true
fi

# Removed --insecure-port argument
if grep -e "\-\-insecure\-port" ${SNAP_DATA}/args/kube-apiserver
then
  $SNAP/bin/sed -i '/\-\-insecure\-port/d' ${SNAP_DATA}/args/kube-apiserver
  need_api_restart=true
fi

# Are we using etcd or some other non-dqlite datastore?
if ! [ -e ${SNAP_DATA}/args/k8s-dqlite ] &&
   ! grep -e "\-\-storage-backend=dqlite" ${SNAP_DATA}/args/kube-apiserver
then
  set_service_not_expected_to_start k8s-dqlite
  snapctl stop ${SNAP_NAME}.daemon-k8s-dqlite
fi

# Configure the API sever to talk to the external dqlite
if [ -e ${SNAP}/default-args/k8s-dqlite ] &&
   ! [ -e ${SNAP_DATA}/args/k8s-dqlite ] &&
   grep -e "\-\-storage-backend=dqlite" ${SNAP_DATA}/args/kube-apiserver
then
  echo "Reconfiguring the API server for dqlite"
  cp ${SNAP}/default-args/k8s-dqlite ${SNAP_DATA}/args/k8s-dqlite
  cp ${SNAP}/default-args/k8s-dqlite-env ${SNAP_DATA}/args/k8s-dqlite-env
  need_api_restart=true
  snapctl stop ${SNAP_NAME}.daemon-kubelite

  refresh_opt_in_local_config etcd-servers unix://\${SNAP_DATA}/var/kubernetes/backend/kine.sock:12379 kube-apiserver
  $SNAP/bin/sed -i '/\-\-storage\-backend=dqlite/d' ${SNAP_DATA}/args/kube-apiserver

  storage_dir="$(get_opt_in_config '--storage-dir' 'kube-apiserver')"
  if ! [ -z $storage_dir ]
  then
    refresh_opt_in_local_config storage-dir "$storage_dir" k8s-dqlite
  fi
  $SNAP/bin/sed -i '/\-\-storage\-dir/d' ${SNAP_DATA}/args/kube-apiserver

  snapctl restart ${SNAP_NAME}.daemon-k8s-dqlite
fi

# Fix hard-coded snap revision numbers in worker node services of existing clusters
# https://github.com/canonical/microk8s/pull/3554
for svc in kubelet proxy; do
  cfg="${SNAP_DATA}/credentials/${svc}.config"
  if [ -e ${cfg} ]; then
    sed -i 's,/var/snap/microk8s/[x0-9]*/,/var/snap/microk8s/current/,' "${cfg}" || true
  fi
done

# (1.26) Removed --log-dir argument from kubelet
if grep -e "\-\-log\-dir" "${SNAP_DATA}/args/kubelet"
then
  "${SNAP}/bin/sed" -i '/\-\-log\-dir/d' "${SNAP_DATA}/args/kubelet"
  need_api_restart=true
fi

# Refresh calico if needed
refresh_calico_if_needed

# Refresh apiserver proxy
snapctl restart "${SNAP_NAME}.daemon-apiserver-proxy"

if is_strict
then
  enable_snap
fi

# if we are refreshing in a no-flanneld we need to restart the CNI pods because they mount parts of $SNAP_DATA
if [ -e "${SNAP_DATA}/var/lock/no-flanneld" ]
then
  touch "${SNAP_DATA}/var/lock/snapdata-mounts-need-reload"
fi

# Restart reconfigured services
if ${need_api_restart} ||
   ${need_proxy_restart} ||
   ${need_controller_restart} ||
   ${need_kubelet_restart}
then
  if [ -e ${SNAP_DATA}/var/lock/lite.lock ]
  then
    snapctl restart ${SNAP_NAME}.daemon-kubelite
  else
    echo "Unable to restart service"
    exit 1
  fi
fi
if ${need_cluster_agent_restart}
then
  snapctl restart ${SNAP_NAME}.daemon-cluster-agent
fi