Fix 5 bugs found via dry-run provision of a real Linode

- spamd, not spamassassin, is the actual systemd unit name
- /etc/chasquid/certs ships non-empty by default (README.certs), breaking
  the symlink-to-certs-orig conversion
- /etc/letsencrypt/renewal-hooks/deploy doesn't exist until certbot has
  actually run once - can't assume it from just installing the package
- Ubuntu's default nginx site conflicts with our own default_server
- chasquid task ordering: domains/certs must exist before chasquid.conf
  is deployed (and its restart triggered), or chasquid crash-loops fast
  enough to hit systemd's restart backoff before domains/ gets populated
  a moment later - reproduces the exact bug from the start of this project

diff --git a/ansible/roles/base/tasks/main.yml b/ansible/roles/base/tasks/main.yml
index d2131a6fef5d294be9e1e2885b1c8083c4084625..58a3083943810d7b9f30a409adc6c49f7eca92d5 100644 (file)
--- a/ansible/roles/base/tasks/main.yml
+++ b/ansible/roles/base/tasks/main.yml
@@ -39,8 +39,13 @@
       APT::Periodic::Update-Package-Lists "1";
       APT::Periodic::Unattended-Upgrade "1";
 
       APT::Periodic::Update-Package-Lists "1";
       APT::Periodic::Unattended-Upgrade "1";
 

-- name: Enable spamassassin service
+# The systemd unit is 'spamd', not 'spamassassin' (package name != unit
+# name) - confirmed against a real fresh Ubuntu 24.04 install via dry run.
+# It's already enabled/running out of the box on this Ubuntu version, this
+# task just makes that an explicit, checked assertion rather than relying
+# on package-default behavior.
+- name: Enable spamd service

   systemd:
   systemd:

-    name: spamassassin
+    name: spamd

     enabled: true
     state: started
     enabled: true
     state: started

diff --git a/ansible/roles/certbot/tasks/main.yml b/ansible/roles/certbot/tasks/main.yml
index 23a9f4a794bb56740b5dcbd050c2ba05abec774b..0059e24a9a67e8443695158b8102bf6a7f225aa1 100644 (file)
--- a/ansible/roles/certbot/tasks/main.yml
+++ b/ansible/roles/certbot/tasks/main.yml
@@ -1,4 +1,16 @@
 ---
 ---

+# /etc/letsencrypt/renewal-hooks/deploy/ doesn't exist just from installing
+# the certbot package - it's only created the first time certbot actually
+# runs. Since cert issuance itself is deliberately manual here (see below),
+# this has to be created explicitly rather than assumed - confirmed via
+# dry run against a fresh install.
+- name: Ensure certbot deploy-hooks directory exists
+  file:
+    path: /etc/letsencrypt/renewal-hooks/deploy
+    state: directory
+    owner: root
+    group: root
+

 - name: Deploy chasquid cert-sync deploy hook
   template:
     src: chasquid-deploy-hook.sh.j2
 - name: Deploy chasquid cert-sync deploy hook
   template:
     src: chasquid-deploy-hook.sh.j2

diff --git a/ansible/roles/chasquid/tasks/main.yml b/ansible/roles/chasquid/tasks/main.yml
index 52e76e45393b7b9cbd757ab3d7be3412ac441232..11d8c8cadc732bafdd2f6c2344d8c690d68aa98b 100644 (file)
--- a/ansible/roles/chasquid/tasks/main.yml
+++ b/ansible/roles/chasquid/tasks/main.yml
@@ -1,10 +1,13 @@
 ---
 ---

-- name: Deploy chasquid.conf
-  template:
-    src: chasquid.conf.j2
-    dest: /etc/chasquid/chasquid.conf
-  notify: restart chasquid
-
+# Domain directories and certs must exist BEFORE chasquid.conf is deployed
+# (and its restart triggered) - not after. chasquid refuses to start with
+# zero domains configured ("No entries found in domains/"), and if the
+# restart fires while domains/ is still empty, chasquid crash-loops fast
+# enough to hit systemd's restart backoff ("start request repeated too
+# quickly") - at which point systemd won't retry again on its own even
+# after domains/ is populated a moment later. This exact failure mode bit
+# us once for real on the live box, and was reproduced here by getting the
+# task order wrong the first time this role was written. Order matters.

 - name: Create domain directories (chasquid accepts mail for any domain with a dir here)
   file:
     path: "/etc/chasquid/domains/{{ item }}"
 - name: Create domain directories (chasquid accepts mail for any domain with a dir here)
   file:
     path: "/etc/chasquid/domains/{{ item }}"


@@ -15,22 +18,29 @@
     - "{{ primary_domain }}"
     - "{{ secondary_domain }}"
 
     - "{{ primary_domain }}"
     - "{{ secondary_domain }}"
 

-# chasquid will refuse to start with zero domains configured ("No entries
-# found in domains/") - this bit us once on the live box, worth remembering
-# if this playbook is ever run against an empty /etc/chasquid/domains/.
-

 # NOTE: /etc/chasquid/certs is a symlink to /etc/chasquid/certs-orig on the
 # live box, populated by the certbot role's deploy hook (chasquid runs
 # unprivileged and can't read /etc/letsencrypt/{live,archive} directly,
 # NOTE: /etc/chasquid/certs is a symlink to /etc/chasquid/certs-orig on the
 # live box, populated by the certbot role's deploy hook (chasquid runs
 # unprivileged and can't read /etc/letsencrypt/{live,archive} directly,

-# which are root:root 0700). This role does not populate certs itself -
-# chasquid will fail to start until the certbot role has run at least once
-# per domain that needs a cert (mail.{{ primary_domain }} at minimum).
-- name: Symlink chasquid cert dir to certs-orig
+# which are root:root 0700). This role does not populate certs itself.
+#
+# IMPORTANT, confirmed via dry run: chasquid requires certs/ to be
+# non-empty to start at all - "No entries found in certs/" is a fatal
+# error, exactly like the empty-domains/ case, not just a missing cert for
+# one specific hostname. That means chasquid WILL NOT be running at the
+# end of a fresh run of this playbook, full stop, until certbot has
+# actually issued at least one real cert (which needs DNS already pointed
+# at this box - a chicken-and-egg this role can't resolve on its own). A
+# freshly-provisioned box with chasquid down is the correct, expected
+# state, not a bug - don't "fix" this by faking a placeholder cert.
+#
+# The package ships /etc/chasquid/certs as a real, non-empty directory
+# (just a README.certs placeholder) - confirmed via dry run against a
+# fresh install. Ansible's file module correctly refuses to convert a
+# non-empty directory into a symlink, so it has to be cleared first.
+- name: Remove package-default certs directory (will become a symlink)

   file:
   file:

-    src: /etc/chasquid/certs-orig
-    dest: /etc/chasquid/certs
-    state: link
-    force: true
+    path: /etc/chasquid/certs
+    state: absent

 
 - name: Ensure certs-orig exists
   file:
 
 - name: Ensure certs-orig exists
   file:


@@ -38,3 +48,37 @@
     state: directory
     owner: root
     group: root
     state: directory
     owner: root
     group: root

+
+- name: Symlink chasquid cert dir to certs-orig
+  file:
+    src: /etc/chasquid/certs-orig
+    dest: /etc/chasquid/certs
+    state: link
+    force: true
+
+- name: Deploy chasquid.conf
+  template:
+    src: chasquid.conf.j2
+    dest: /etc/chasquid/chasquid.conf
+  notify: restart chasquid
+
+# Defensive, not just reactive: clear any systemd restart backoff from a
+# previous failed attempt (e.g. a re-run after fixing something), so that
+# once certs actually exist, chasquid gets a fresh start attempt rather
+# than sitting in a backoff state from before. `notify` only fires on a
+# task reporting "changed" - on a re-run where chasquid.conf is already
+# correct, nothing would otherwise force a retry.
+- name: Clear any systemd restart backoff
+  command: systemctl reset-failed chasquid.service
+  changed_when: false
+  failed_when: false
+
+# This attempt is expected to fail on a fresh box with no certs yet - see
+# the note above. failed_when: false because that's a legitimate outcome
+# here, not a playbook error; re-run this role (or just `systemctl start
+# chasquid`) after certs are issued.
+- name: Attempt to start chasquid (expected to fail until certs exist)
+  systemd:
+    name: chasquid
+    state: started
+  failed_when: false

diff --git a/ansible/roles/nginx/tasks/main.yml b/ansible/roles/nginx/tasks/main.yml
index 9db5caa404e297189b5ef1c559105e3f90dfa45b..87147b50dc93c04b026cd6981babc606d5f8ece8 100644 (file)
--- a/ansible/roles/nginx/tasks/main.yml
+++ b/ansible/roles/nginx/tasks/main.yml
@@ -1,4 +1,15 @@
 ---
 ---

+# Ubuntu's nginx package ships its own default_server site, which conflicts
+# with our own default_server declaration ("duplicate default server for
+# 0.0.0.0:80") - confirmed via dry run. On the live box this was removed
+# manually at some point, outside any documented step; do it explicitly
+# here instead of relying on that tribal knowledge.
+- name: Remove distro default nginx site
+  file:
+    path: /etc/nginx/sites-enabled/default
+    state: absent
+  notify: reload nginx
+

 - name: Create web roots
   file:
     path: "/var/www/{{ item }}"
 - name: Create web roots
   file:
     path: "/var/www/{{ item }}"