feat: faster and simpler DNS checks, better ip-address determination

CHANGELOG.md

@@ -2,6 +2,10 @@
 
 ## untagged
 
+- Make DNS-checking faster and run it fully during "cmdeploy run",
+  introducing a generic mechanism for remote ssh-based python function execution. 
+  ([#346](https://github.com/deltachat/chatmail/pull/346))
+
 - Don't fix file owner ship of /home/vmail 
   ([#345](https://github.com/deltachat/chatmail/pull/345))
 

cmdeploy/pyproject.toml

@@ -18,6 +18,7 @@ dependencies = [
   "ruff",
   "pytest",
   "pytest-xdist", 
+  "execnet",
   "imap_tools",
 ]
 

cmdeploy/src/cmdeploy/__init__.py

@@ -630,5 +630,3 @@ def deploy_chatmail(config_path: Path) -> None:
         name="Ensure cron is installed",
         packages=["cron"],
     )
-
-

cmdeploy/src/cmdeploy/cmdeploy.py

@@ -15,7 +15,9 @@
 from chatmaild.config import read_config, write_initial_config
 from termcolor import colored
 
-from cmdeploy.dns import check_necessary_dns, show_dns
+from . import remote_funcs
+from .dns import show_dns
+from .sshexec import SSHExec
 
 #
 # cmdeploy sub commands and options
@@ -51,12 +53,7 @@ def run_cmd_options(parser):
 
 def run_cmd(args, out):
     """Deploy chatmail services on the remote server."""
-    mail_domain = args.config.mail_domain
-    if not check_necessary_dns(
-        out,
-        mail_domain,
-    ):
-        sys.exit(1)
+    show_dns(args, out)
 
     env = os.environ.copy()
     env["CHATMAIL_INI"] = args.inipath
@@ -85,18 +82,16 @@ def dns_cmd(args, out):
 def status_cmd(args, out):
     """Display status for online chatmail instance."""
 
-    ssh = f"ssh root@{args.config.mail_domain}"
+    sshexec = SSHExec(args.config.mail_domain, remote_funcs)
 
     out.green(f"chatmail domain: {args.config.mail_domain}")
     if args.config.privacy_mail:
         out.green("privacy settings: present")
     else:
         out.red("no privacy settings")
 
-    s1 = "systemctl --type=service --state=running"
-    for line in out.shell_output(f"{ssh} -- {s1}").split("\n"):
-        if line.startswith("  "):
-            print(line)
+    for line in sshexec(remote_funcs.get_systemd_running):
+        print(line)
 
 
 def test_cmd_options(parser):
@@ -208,16 +203,6 @@ def __call__(self, msg, red=False, green=False, file=sys.stdout):
         color = "red" if red else ("green" if green else None)
         print(colored(msg, color), file=file)
 
-    def shell_output(self, arg, no_print=False, timeout=10):
-        if not no_print:
-            self(f"[$ {arg}]", file=sys.stderr)
-            output = subprocess.STDOUT
-        else:
-            output = subprocess.DEVNULL
-        return subprocess.check_output(
-            arg, shell=True, timeout=timeout, stderr=output
-        ).decode()
-
     def check_call(self, arg, env=None, quiet=False):
         if not quiet:
             self(f"[$ {arg}]", file=sys.stderr)

cmdeploy/src/cmdeploy/dns.py

@@ -1,209 +1,62 @@
 import datetime
 import importlib
-import subprocess
-import sys
 
-import requests
-
-
-class DNS:
-    def __init__(self, out, mail_domain):
-        self.session = requests.Session()
-        self.out = out
-        self.ssh = f"ssh root@{mail_domain} -- "
-        self.out.shell_output(
-            f"{ self.ssh }'apt-get update && apt-get install -y dnsutils'",
-            timeout=60,
-            no_print=True,
-        )
-        try:
-            self.shell(f"unbound-control flush_zone {mail_domain}")
-        except subprocess.CalledProcessError:
-            pass
-
-    def shell(self, cmd):
-        try:
-            return self.out.shell_output(f"{self.ssh}{cmd}", no_print=True)
-        except (subprocess.CalledProcessError, subprocess.TimeoutExpired) as e:
-            if "exit status 255" in str(e) or "timed out" in str(e):
-                self.out.red(f"Error: can't reach the server with: {self.ssh[:-4]}")
-                sys.exit(1)
-            else:
-                raise
-
-    def get_ipv4(self):
-        cmd = "ip a | grep 'inet ' | grep 'scope global' | grep -oE '[0-9]{1,3}.[0-9]{1,3}.[0-9]{1,3}.[0-9]{1,3}' | head -1"
-        return self.shell(cmd).strip()
-
-    def get_ipv6(self):
-        cmd = "ip a | grep inet6 | grep 'scope global' | sed -e 's#/64 scope global##' | sed -e 's#inet6##'"
-        return self.shell(cmd).strip()
-
-    def get(self, typ: str, domain: str) -> str:
-        """Get a DNS entry or empty string if there is none."""
-        dig_result = self.shell(f"dig -r -q {domain} -t {typ} +short")
-        line = dig_result.partition("\n")[0]
-        return line
-
-    def check_ptr_record(self, ip: str, mail_domain) -> bool:
-        """Check the PTR record for an IPv4 or IPv6 address."""
-        result = self.shell(f"dig -r -x {ip} +short").rstrip()
-        return result == f"{mail_domain}."
+from . import remote_funcs
+from .sshexec import SSHExec
 
 
 def show_dns(args, out) -> int:
     """Check existing DNS records, optionally write them to zone file, return exit code 0 or 1."""
+    print("Checking your DKIM keys and DNS entries...")
     template = importlib.resources.files(__package__).joinpath("chatmail.zone.f")
     mail_domain = args.config.mail_domain
-    ssh = f"ssh root@{mail_domain}"
-    dns = DNS(out, mail_domain)
 
-    print("Checking your DKIM keys and DNS entries...")
-    try:
-        acme_account_url = out.shell_output(f"{ssh} -- acmetool account-url")
-    except subprocess.CalledProcessError:
-        print("Please run `cmdeploy run` first.")
-        return 1
+    sshexec = SSHExec(mail_domain, remote_funcs)
 
-    dkim_selector = "opendkim"
-    dkim_pubkey = out.shell_output(
-        ssh + f" -- openssl rsa -in /etc/dkimkeys/{dkim_selector}.private"
-        " -pubout 2>/dev/null | awk '/-/{next}{printf(\"%s\",$0)}'"
-    )
-    dkim_entry_value = f"v=DKIM1;k=rsa;p={dkim_pubkey};s=email;t=s"
-    dkim_entry_str = ""
-    while len(dkim_entry_value) >= 255:
-        dkim_entry_str += '"' + dkim_entry_value[:255] + '" '
-        dkim_entry_value = dkim_entry_value[255:]
-    dkim_entry_str += '"' + dkim_entry_value + '"'
-    dkim_entry = f"{dkim_selector}._domainkey.{mail_domain}. TXT {dkim_entry_str}"
+    remote_data = sshexec(remote_funcs.perform_initial_checks, mail_domain=mail_domain)
 
-    ipv6 = dns.get_ipv6()
-    reverse_ipv6 = dns.check_ptr_record(ipv6, mail_domain)
-    ipv4 = dns.get_ipv4()
-    reverse_ipv4 = dns.check_ptr_record(ipv4, mail_domain)
-    to_print = []
+    assert remote_data["ipv4"] or remote_data["ipv6"]
 
     with open(template, "r") as f:
         zonefile = (
             f.read()
             .format(
-                acme_account_url=acme_account_url,
+                acme_account_url=remote_data["acme_account_url"],
+                dkim_entry=remote_data["dkim_entry"],
+                ipv6=remote_data["ipv6"],
+                ipv4=remote_data["ipv4"],
                 sts_id=datetime.datetime.now().strftime("%Y%m%d%H%M"),
                 chatmail_domain=args.config.mail_domain,
-                dkim_entry=dkim_entry,
-                ipv6=ipv6,
-                ipv4=ipv4,
             )
             .strip()
         )
-        try:
-            with open(args.zonefile, "w+") as zf:
-                zf.write(zonefile)
-                print(f"DNS records successfully written to: {args.zonefile}")
-            return 0
-        except TypeError:
-            pass
-        for raw_line in zonefile.splitlines():
-            line = raw_line.format(
-                acme_account_url=acme_account_url,
-                sts_id=datetime.datetime.now().strftime("%Y%m%d%H%M"),
-                chatmail_domain=args.config.mail_domain,
-                dkim_entry=dkim_entry,
-                ipv6=ipv6,
-            ).strip()
-            for typ in ["A", "AAAA", "CNAME", "CAA"]:
-                if f" {typ} " in line:
-                    domain, value = line.split(f" {typ} ")
-                    current = dns.get(typ, domain.strip()[:-1])
-                    if current != value.strip():
-                        to_print.append(line)
-            if " MX " in line:
-                domain, typ, prio, value = line.split()
-                current = dns.get(typ, domain[:-1])
-                if not current:
-                    to_print.append(line)
-                elif current.split()[1] != value:
-                    print(line.replace(prio, str(int(current[0]) + 1)))
-            if " SRV " in line:
-                domain, typ, prio, weight, port, value = line.split()
-                current = dns.get("SRV", domain[:-1])
-                if current != f"{prio} {weight} {port} {value}":
-                    to_print.append(line)
-            if " TXT " in line:
-                domain, value = line.split(" TXT ")
-                current = dns.get("TXT", domain.strip()[:-1])
-                if domain.startswith("_mta-sts."):
-                    if current:
-                        if current.split("id=")[0] == value.split("id=")[0]:
-                            continue
+    if getattr(args, "zonefile", None):
+        with open(args.zonefile, "w+") as zf:
+            zf.write(zonefile)
+        print(f"DNS records successfully written to: {args.zonefile}")
 
-                # TXT records longer than 255 bytes
-                # are split into multiple <character-string>s.
-                # This typically happens with DKIM record
-                # which contains long RSA key.
-                #
-                # Removing `" "` before comparison
-                # to get back a single string.
-                if current.replace('" "', "") != value.replace('" "', ""):
-                    to_print.append(line)
+    to_print = sshexec(remote_funcs.check_zonefile, zonefile=zonefile)
 
-    exit_code = 0
     if to_print:
         to_print.insert(
-            0, "You should configure the following DNS entries at your provider:\n"
+            0, "You should configure the following entries at your DNS provider:\n"
         )
         to_print.append(
             "\nIf you already configured the DNS entries, wait a bit until the DNS entries propagate to the Internet."
         )
-        print("\n".join(to_print))
-        exit_code = 1
+        out.red("\n".join(to_print))
     else:
-        out.green("Great! All your DNS entries are correct.")
+        out.green("Great! All your DNS entries are verified and correct.")
 
     to_print = []
-    if not reverse_ipv4:
-        to_print.append(f"\tIPv4:\t{ipv4}\t{args.config.mail_domain}")
-    if not reverse_ipv6:
-        to_print.append(f"\tIPv6:\t{ipv6}\t{args.config.mail_domain}")
+    if not remote_data["reverse_ipv4"]:
+        to_print.append(f"\tIPv4:\t{remote_data['ipv4']}\t{args.config.mail_domain}")
+    if not remote_data["reverse_ipv6"]:
+        to_print.append(f"\tIPv6:\t{remote_data['ipv6']}\t{args.config.mail_domain}")
     if len(to_print) > 0:
-        if len(to_print) == 1:
-            warning = "You should add the following PTR/reverse DNS entry:"
-        else:
-            warning = "You should add the following PTR/reverse DNS entries:"
-        out.red(warning)
+        out.red("You need to set the following PTR/reverse DNS data:")
         for entry in to_print:
             print(entry)
-        print(
+        out.red(
             "You can do so at your hosting provider (maybe this isn't your DNS provider)."
         )
-        exit_code = 1
-    return exit_code
-
-
-def check_necessary_dns(out, mail_domain):
-    """Check whether $mail_domain and mta-sts.$mail_domain resolve."""
-    print("Checking necessary DNS records... ")
-    dns = DNS(out, mail_domain)
-    ipv4 = dns.get("A", mail_domain)
-    ipv6 = dns.get("AAAA", mail_domain)
-    mta_entry = dns.get("CNAME", "mta-sts." + mail_domain)
-    www_entry = dns.get("CNAME", "www." + mail_domain)
-    to_print = []
-    if not (ipv4 or ipv6):
-        to_print.append(f"\t{mail_domain}.\t\t\tA<your server's IPv4 address>")
-    if mta_entry != mail_domain + ".":
-        to_print.append(f"\tmta-sts.{mail_domain}.\tCNAME\t{mail_domain}.")
-    if www_entry != mail_domain + ".":
-        to_print.append(f"\twww.{mail_domain}.\tCNAME\t{mail_domain}.")
-    if to_print:
-        to_print.insert(
-            0,
-            "\nFor chatmail to work, you need to configure this at your DNS provider:\n",
-        )
-        for line in to_print:
-            print(line)
-        print()
-    else:
-        dns.out.green("All necessary DNS records seem to be set.")
-        return True