remove all hardcoding of passdb and user mailbox dirs

CHANGELOG.md

@@ -2,6 +2,14 @@
 
 ## untagged
 
+- BREAKING: new required chatmail.ini values:
+
+  mailboxes_dir = /home/vmail/mail/{mail_domain}
+  passdb = /home/vmail/passdb.sqlite
+
+  reducing hardcoding these two paths all over the files, also improving testability. 
+  ([#351](https://github.com/deltachat/chatmail/pull/351))
+
 - BREAKING: new required chatmail.ini value 'delete_inactive_users_after = 100'
   which removes users from database and mails after 100 days without any login. 
   ([#350](https://github.com/deltachat/chatmail/pull/350))

chatmaild/src/chatmaild/config.py

@@ -3,17 +3,15 @@
 import iniconfig
 
 
-def read_config(inipath, mail_basedir=None):
+def read_config(inipath):
     assert Path(inipath).exists(), inipath
     cfg = iniconfig.IniConfig(inipath)
     params = cfg.sections["params"]
-    if mail_basedir is None:
-        mail_basedir = Path(f"/home/vmail/mail/{params['mail_domain']}")
-    return Config(inipath, params=params, mail_basedir=mail_basedir)
+    return Config(inipath, params=params)
 
 
 class Config:
-    def __init__(self, inipath, params, mail_basedir: Path):
+    def __init__(self, inipath, params):
         self._inipath = inipath
         self.mail_domain = params["mail_domain"]
         self.max_user_send_per_minute = int(params["max_user_send_per_minute"])
@@ -25,33 +23,49 @@ def __init__(self, inipath, params, mail_basedir: Path):
         self.password_min_length = int(params["password_min_length"])
         self.passthrough_senders = params["passthrough_senders"].split()
         self.passthrough_recipients = params["passthrough_recipients"].split()
+        self.mailboxes_dir = Path(params["mailboxes_dir"].strip())
+        self.passdb_path = Path(params["passdb_path"].strip())
         self.filtermail_smtp_port = int(params["filtermail_smtp_port"])
         self.postfix_reinject_port = int(params["postfix_reinject_port"])
         self.iroh_relay = params.get("iroh_relay")
         self.privacy_postal = params.get("privacy_postal")
         self.privacy_mail = params.get("privacy_mail")
         self.privacy_pdo = params.get("privacy_pdo")
         self.privacy_supervisor = params.get("privacy_supervisor")
-        self.mail_basedir = mail_basedir
 
     def _getbytefile(self):
         return open(self._inipath, "rb")
 
     def get_user_maildir(self, addr):
         if addr and addr != "." and "/" not in addr:
-            res = self.mail_basedir.joinpath(addr).resolve()
-            if res.is_relative_to(self.mail_basedir):
+            res = self.mailboxes_dir.joinpath(addr).resolve()
+            if res.is_relative_to(self.mailboxes_dir):
                 return res
         raise ValueError(f"invalid address {addr!r}")
 
 
-def write_initial_config(inipath, mail_domain):
+def write_initial_config(inipath, mail_domain, overrides):
+    """Write out default config file, using the specified config value overrides."""
     from importlib.resources import files
 
     inidir = files(__package__).joinpath("ini")
-    content = (
-        inidir.joinpath("chatmail.ini.f").read_text().format(mail_domain=mail_domain)
-    )
+    source_inipath = inidir.joinpath("chatmail.ini.f")
+    content = source_inipath.read_text().format(mail_domain=mail_domain)
+
+    # apply config overrides
+    new_lines = []
+    for line in content.split("\n"):
+        new_line = line.strip()
+        if new_line and new_line[0] not in "#[":
+            name, value = map(str.strip, new_line.split("=", maxsplit=1))
+            value = overrides.get(name, value)
+            new_line = f"{name} = {value}"
+        new_lines.append(new_line)
+
+    content = "\n".join(new_lines)
+
+    # apply testrun privacy overrides
+
     if mail_domain.endswith(".testrun.org"):
         override_inipath = inidir.joinpath("override-testrun.ini")
         privacy = iniconfig.IniConfig(override_inipath)["privacy"]

chatmaild/src/chatmaild/delete_inactive_users.py

@@ -27,6 +27,7 @@ def delete_inactive_users(db, config, CHUNK=100):
 
 
 def main():
-    db = Database(sys.argv[1])
-    config = read_config(sys.argv[2])
+    (cfgpath,) = sys.argv[1:]
+    config = read_config(cfgpath)
+    db = Database(config.passdb_path)
     delete_inactive_users(db, config)

chatmaild/src/chatmaild/doveauth.py

@@ -68,15 +68,15 @@ def is_allowed_to_create(config: Config, user, cleartext_password) -> bool:
 def get_user_data(db, config: Config, user):
     if user == f"echo@{config.mail_domain}":
         return dict(
-            home=f"/home/vmail/mail/{config.mail_domain}/echo@{config.mail_domain}",
+            home=str(config.get_user_maildir(user)),
             uid="vmail",
             gid="vmail",
         )
 
     with db.read_connection() as conn:
         result = conn.get_user(user)
     if result:
-        result["home"] = f"/home/vmail/mail/{config.mail_domain}/{user}"
+        result["home"] = str(config.get_user_maildir(user))
         result["uid"] = "vmail"
         result["gid"] = "vmail"
     return result
@@ -96,7 +96,7 @@ def lookup_passdb(db, config: Config, user, cleartext_password, last_login=None)
             return None
 
         return dict(
-            home=f"/home/vmail/mail/{config.mail_domain}/echo@{config.mail_domain}",
+            home=str(config.get_user_maildir(user)),
             uid="vmail",
             gid="vmail",
             password=encrypt_password(password),
@@ -114,7 +114,7 @@ def lookup_passdb(db, config: Config, user, cleartext_password, last_login=None)
                 "UPDATE users SET last_login=? WHERE addr=?", (last_login, user)
             )
 
-            userdata["home"] = f"/home/vmail/mail/{config.mail_domain}/{user}"
+            userdata["home"] = str(config.get_user_maildir(user))
             userdata["uid"] = "vmail"
             userdata["gid"] = "vmail"
             return userdata
@@ -127,7 +127,7 @@ def lookup_passdb(db, config: Config, user, cleartext_password, last_login=None)
         conn.execute(q, (user, encrypted_password, last_login))
         print(f"Created address: {user}", file=sys.stderr)
         return dict(
-            home=f"/home/vmail/mail/{config.mail_domain}/{user}",
+            home=str(config.get_user_maildir(user)),
             uid="vmail",
             gid="vmail",
             password=encrypted_password,
@@ -245,9 +245,9 @@ class ThreadedUnixStreamServer(ThreadingMixIn, UnixStreamServer):
 
 
 def main():
-    socket = sys.argv[1]
-    db = Database(sys.argv[2])
-    config = read_config(sys.argv[3])
+    socket, cfgpath = sys.argv[1:]
+    config = read_config(cfgpath)
+    db = Database(config.passdb_path)
 
     class Handler(StreamRequestHandler):
         def handle(self):

chatmaild/src/chatmaild/ini/chatmail.ini.f

@@ -42,6 +42,12 @@
 # Deployment Details
 #
 
+# Directory where user mailboxes are stored
+mailboxes_dir = /home/vmail/mail/{mail_domain}
+
+# user address sqlite database path
+passdb_path = /home/vmail/passdb.sqlite
+
 # where the filtermail SMTP service listens
 filtermail_smtp_port = 10080
 
@@ -63,4 +69,3 @@
 
 # postal address of the privacy supervisor
 privacy_supervisor =
-

chatmaild/src/chatmaild/metadata.py

@@ -1,7 +1,6 @@
 import logging
 import os
 import sys
-from pathlib import Path
 from socketserver import (
     StreamRequestHandler,
     ThreadingMixIn,
@@ -128,12 +127,12 @@ class ThreadedUnixStreamServer(ThreadingMixIn, UnixStreamServer):
 
 
 def main():
-    socket, vmail_dir, config_path = sys.argv[1:]
+    socket, config_path = sys.argv[1:]
 
     config = read_config(config_path)
     iroh_relay = config.iroh_relay
 
-    vmail_dir = Path(vmail_dir)
+    vmail_dir = config.mailboxes_dir
     if not vmail_dir.exists():
         logging.error("vmail dir does not exist: %r", vmail_dir)
         return 1

chatmaild/src/chatmaild/tests/plugin.py

@@ -16,10 +16,12 @@ def make_config(tmp_path):
     inipath = tmp_path.joinpath("chatmail.ini")
 
     def make_conf(mail_domain):
-        write_initial_config(inipath, mail_domain=mail_domain)
         basedir = tmp_path.joinpath(f"vmail/{mail_domain}")
         basedir.mkdir(parents=True, exist_ok=True)
-        return read_config(inipath, mail_basedir=basedir)
+        passdb = tmp_path.joinpath("vmail/passdb.sqlite")
+        overrides = dict(mailboxes_dir=str(basedir), passdb_path=str(passdb))
+        write_initial_config(inipath, mail_domain, overrides=overrides)
+        return read_config(inipath)
 
     return make_conf
 

chatmaild/src/chatmaild/tests/test_config.py

@@ -33,13 +33,17 @@ def test_read_config_testrun(make_config):
     assert config.passthrough_senders == []
 
 
-def test_get_user_maildir(make_config):
+def test_config_userstate_paths(make_config, tmp_path):
     config = make_config("something.testrun.org")
-    assert config.mail_basedir.name == "something.testrun.org"
+    mailboxes_dir = config.mailboxes_dir
+    passdb_path = config.passdb_path
+    assert mailboxes_dir.name == "something.testrun.org"
+    assert passdb_path.name == "passdb.sqlite"
+    assert passdb_path.is_relative_to(tmp_path)
     assert config.mail_domain == "something.testrun.org"
     path = config.get_user_maildir("user1@something.testrun.org")
     assert not path.exists()
-    assert path == config.mail_basedir.joinpath("user1@something.testrun.org")
+    assert path == mailboxes_dir.joinpath("user1@something.testrun.org")
 
     with pytest.raises(ValueError):
         config.get_user_maildir("")

chatmaild/src/chatmaild/tests/test_delete_inactive_users.py

@@ -37,13 +37,13 @@ def create_user(addr, last_login):
 
     delete_inactive_users(db, example_config)
 
-    for p in example_config.mail_basedir.iterdir():
+    for p in example_config.mailboxes_dir.iterdir():
         assert not p.name.startswith("old")
 
     for addr in to_remove:
+        assert not example_config.get_user_maildir(addr).exists()
         with db.read_connection() as conn:
             assert not conn.get_user(addr)
-            assert not example_config.get_user_maildir(addr).exists()
 
     for addr in remain:
         assert example_config.get_user_maildir(addr).exists()

chatmaild/src/chatmaild/tests/test_doveauth.py

@@ -114,10 +114,7 @@ def test_handle_dovecot_request(db, example_config):
     assert res
     assert res[0] == "O" and res.endswith("\n")
     userdata = json.loads(res[1:].strip())
-    assert (
-        userdata["home"]
-        == "/home/vmail/mail/chat.example.org/some42123@chat.example.org"
-    )
+    assert userdata["home"].endswith("chat.example.org/some42123@chat.example.org")
     assert userdata["uid"] == userdata["gid"] == "vmail"
     assert userdata["password"].startswith("{SHA512-CRYPT}")
 

cmdeploy/src/cmdeploy/__init__.py

@@ -92,7 +92,7 @@ def _install_remote_venv_with_chatmaild(config) -> None:
         group="root",
         mode="644",
         config={
-            "mail_domain": config.mail_domain,
+            "mailboxes_dir": config.mailboxes_dir,
             "execpath": f"{remote_venv_dir}/bin/chatmail-metrics",
         },
     )

cmdeploy/src/cmdeploy/cmdeploy.py

@@ -38,7 +38,7 @@ def init_cmd(args, out):
     if args.inipath.exists():
         print(f"Path exists, not modifying: {args.inipath}")
     else:
-        write_initial_config(args.inipath, mail_domain)
+        write_initial_config(args.inipath, mail_domain, overrides={})
         out.green(f"created config file for {mail_domain} in {args.inipath}")
 
 

cmdeploy/src/cmdeploy/dovecot/dovecot.conf.j2

@@ -67,7 +67,7 @@ userdb {
 ##
 
 # Mailboxes are stored in the "mail" directory of the vmail user home.
-mail_location = maildir:/home/vmail/mail/%d/%u
+mail_location = maildir:{{ config.mailboxes_dir }}/%u
 
 namespace inbox {
   inbox = yes

cmdeploy/src/cmdeploy/dovecot/expunge.cron.j2

@@ -1,12 +1,12 @@
 # delete all mails after {{ config.delete_mails_after }} days, in the Inbox
-2 0 * * * vmail find /home/vmail/mail/{{ config.mail_domain }} -path '*/cur/*' -mtime +{{ config.delete_mails_after }} -type f -delete
+2 0 * * * vmail find {{ config.mailboxes_dir }} -path '*/cur/*' -mtime +{{ config.delete_mails_after }} -type f -delete
 # or in any IMAP subfolder
-2 0 * * * vmail find /home/vmail/mail/{{ config.mail_domain }} -path '*/.*/cur/*' -mtime +{{ config.delete_mails_after }} -type f -delete
+2 0 * * * vmail find {{ config.mailboxes_dir }} -path '*/.*/cur/*' -mtime +{{ config.delete_mails_after }} -type f -delete
 # even if they are unseen
-2 0 * * * vmail find /home/vmail/mail/{{ config.mail_domain }} -path '*/new/*' -mtime +{{ config.delete_mails_after }} -type f -delete
-2 0 * * * vmail find /home/vmail/mail/{{ config.mail_domain }} -path '*/.*/new/*' -mtime +{{ config.delete_mails_after }} -type f -delete
+2 0 * * * vmail find {{ config.mailboxes_dir }} -path '*/new/*' -mtime +{{ config.delete_mails_after }} -type f -delete
+2 0 * * * vmail find {{ config.mailboxes_dir }} -path '*/.*/new/*' -mtime +{{ config.delete_mails_after }} -type f -delete
 # or only temporary (but then they shouldn't be around after {{ config.delete_mails_after }} days anyway).
-2 0 * * * vmail find /home/vmail/mail/{{ config.mail_domain }} -path '*/tmp/*' -mtime +{{ config.delete_mails_after }} -type f -delete
-2 0 * * * vmail find /home/vmail/mail/{{ config.mail_domain }} -path '*/.*/tmp/*' -mtime +{{ config.delete_mails_after }} -type f -delete
-3 0 * * * vmail find /home/vmail/mail/{{ config.mail_domain }} -name 'maildirsize' -type f -delete
-4 0 * * * vmail /usr/local/lib/chatmaild/venv/bin/delete_inactive_users /home/vmail/passdb.sqlite /usr/local/lib/chatmaild/chatmail.ini 
+2 0 * * * vmail find {{ config.mailboxes_dir }} -path '*/tmp/*' -mtime +{{ config.delete_mails_after }} -type f -delete
+2 0 * * * vmail find {{ config.mailboxes_dir }} -path '*/.*/tmp/*' -mtime +{{ config.delete_mails_after }} -type f -delete
+3 0 * * * vmail find {{ config.mailboxes_dir }} -name 'maildirsize' -type f -delete
+4 0 * * * vmail /usr/local/lib/chatmaild/venv/bin/delete_inactive_users /usr/local/lib/chatmaild/chatmail.ini 

cmdeploy/src/cmdeploy/metrics.cron.j2

@@ -1 +1 @@
-*/5 * * * * root {{ config.execpath }} /home/vmail/mail/{{ config.mail_domain }} >/var/www/html/metrics
+*/5 * * * * root {{ config.execpath }} {{ config.mailboxes_dir }} >/var/www/html/metrics

cmdeploy/src/cmdeploy/service/chatmail-metadata.service.f

@@ -2,7 +2,7 @@
 Description=Chatmail dict proxy for IMAP METADATA
 
 [Service]
-ExecStart={execpath} /run/chatmail-metadata/metadata.socket /home/vmail/mail/{mail_domain} {config_path}
+ExecStart={execpath} /run/chatmail-metadata/metadata.socket {config_path}
 Restart=always
 RestartSec=30
 User=vmail

cmdeploy/src/cmdeploy/service/doveauth.service.f

@@ -2,7 +2,7 @@
 Description=Chatmail dict authentication proxy for dovecot
 
 [Service]
-ExecStart={execpath} /run/doveauth/doveauth.socket /home/vmail/passdb.sqlite {config_path}
+ExecStart={execpath} /run/doveauth/doveauth.socket {config_path}
 Restart=always
 RestartSec=30
 User=vmail

cmdeploy/src/cmdeploy/tests/online/test_1_basic.py

@@ -108,12 +108,12 @@ def test_exceed_rate_limit(cmsetup, gencreds, maildata, chatmail_config):
 def test_expunged(remote, chatmail_config):
     outdated_days = int(chatmail_config.delete_mails_after) + 1
     find_cmds = [
-        f"find /home/vmail/mail/{chatmail_config.mail_domain} -path '*/cur/*' -mtime +{outdated_days} -type f",
-        f"find /home/vmail/mail/{chatmail_config.mail_domain} -path '*/.*/cur/*' -mtime +{outdated_days} -type f",
-        f"find /home/vmail/mail/{chatmail_config.mail_domain} -path '*/new/*' -mtime +{outdated_days} -type f",
-        f"find /home/vmail/mail/{chatmail_config.mail_domain} -path '*/.*/new/*' -mtime +{outdated_days} -type f",
-        f"find /home/vmail/mail/{chatmail_config.mail_domain} -path '*/tmp/*' -mtime +{outdated_days} -type f",
-        f"find /home/vmail/mail/{chatmail_config.mail_domain} -path '*/.*/tmp/*' -mtime +{outdated_days} -type f",
+        f"find {chatmail_config.mailboxes_dir} -path '*/cur/*' -mtime +{outdated_days} -type f",
+        f"find {chatmail_config.mailboxes_dir} -path '*/.*/cur/*' -mtime +{outdated_days} -type f",
+        f"find {chatmail_config.mailboxes_dir} -path '*/new/*' -mtime +{outdated_days} -type f",
+        f"find {chatmail_config.mailboxes_dir} -path '*/.*/new/*' -mtime +{outdated_days} -type f",
+        f"find {chatmail_config.mailboxes_dir} -path '*/tmp/*' -mtime +{outdated_days} -type f",
+        f"find {chatmail_config.mailboxes_dir} -path '*/.*/tmp/*' -mtime +{outdated_days} -type f",
     ]
     for cmd in find_cmds:
         for line in remote.iter_output(cmd):