make caddy a L4 proxy to terminate TLS for HiveMQ

nix/hosts/mqtt-ingest.nix

@@ -21,13 +21,16 @@
     # GEBOS_POSTGRES_PASSWORD sourced from /run/secrets/gebos-env (sops-nix).
   };
 
-  # Caddy fronts TLS for `ingest.gebos.online` and forwards to HiveMQ.
+  # Caddy (layer4) terminates TLS on :8883 and forwards cleartext MQTT to
+  # HiveMQ's loopback listener on :1883. tcpProxy is the upstream backend.
   services.gebos.caddy = {
     enable = true;
     sites = {
-      "ingest.gebos.online" = { tcpProxy = "127.0.0.1:8883"; };
+      "ingest.gebos.online" = { tcpProxy = "127.0.0.1:1883"; };
     };
   };
 
-  networking.firewall.allowedTCPPorts = [ 8883 ];
+  # 8883: MQTTS for senders. 80/443: ACME challenge so Caddy can obtain/renew
+  # the ingest.gebos.online cert that the layer4 TLS handler serves.
+  networking.firewall.allowedTCPPorts = [ 80 443 8883 ];
 }

nix/modules/gebos-caddy.nix

@@ -3,7 +3,24 @@
 let
   cfg = config.services.gebos.caddy;
 
-  siteBlock = host: site:
+  tcpProxySites = lib.filterAttrs (_: s: s ? tcpProxy) cfg.sites;
+  httpSites = lib.filterAttrs (_: s: !(s ? tcpProxy)) cfg.sites;
+  needsLayer4 = tcpProxySites != { };
+
+  # External MQTTS port the senders connect to (mqtt_port = 8883, SSL).
+  tlsListenPort = 8883;
+
+  # Caddy built with the layer4 module (caddy-l4) — needed to TLS-terminate raw
+  # MQTT and proxy it to HiveMQ, which Caddy's HTTP-only core can't do. Only
+  # built when a host actually has a tcpProxy site; app-host's static/upstream
+  # sites run on stock Caddy so they don't pay for the custom build.
+  caddyWithL4 = pkgs.caddy.withPlugins {
+    plugins = [ "github.com/mholt/caddy-l4@v0.1.1" ];
+    hash = "sha256-O6GuC2q1mA/Fa0utb2Yg7ZE73iq13oVYhJI1IVyOvog=";
+  };
+
+  # HTTP-style sites: static file server or reverse proxy.
+  httpSiteBlock = host: site:
     if site ? staticRoot then ''
       ${host} {
         root * ${site.staticRoot}
@@ -18,14 +35,38 @@ let
         encode zstd gzip
       }
     ''
-    else if site ? tcpProxy then ''
-      ${host}:8883 {
-        # TLS termination for MQTT — Caddy's `layer4` app would be ideal here;
-        # for now, document that the upstream HiveMQ port is ${site.tcpProxy}.
-        # TODO: switch to the caddy-l4 module or terminate TLS in HiveMQ directly.
-      }
-    ''
     else throw "site `${host}` needs one of: staticRoot, upstream, tcpProxy";
+
+  # A tcpProxy site needs TWO Caddyfile pieces:
+  #   1. a layer4 listener (in the global options block) that terminates TLS on
+  #      :8883 and proxies cleartext to the upstream (HiveMQ on 127.0.0.1:1883);
+  #   2. a normal HTTPS site block for the host, purely so Caddy's automatic
+  #      HTTPS OBTAINS the ACME cert that the layer4 `tls` handler then serves by
+  #      SNI — caddy-l4's `tls` handler only terminates, it never provisions
+  #      certs. This is why ports 80/443 must stay open for the ACME challenge.
+  layer4Listener = _host: site: ''
+		:${toString tlsListenPort} {
+			route {
+				tls
+				proxy ${site.tcpProxy}
+			}
+		}
+'';
+
+  certSiteBlock = host: _site: ''
+    ${host} {
+    	respond "gebos MQTT ingest — connect MQTTS on :${toString tlsListenPort}" 200
+    }
+  '';
+
+  # Caddyfile global options block must come first and exist once; we fold every
+  # layer4 listener into a single `layer4 { ... }` inside it.
+  globalOptions = lib.optionalString needsLayer4 ''
+    {
+    	layer4 {
+    ${lib.concatStrings (lib.mapAttrsToList layer4Listener tcpProxySites)}	}
+    }
+  '';
 in
 {
   options.services.gebos.caddy = {
@@ -39,9 +80,13 @@ in
   config = lib.mkIf cfg.enable {
     services.caddy = {
       enable = true;
+      package = if needsLayer4 then caddyWithL4 else pkgs.caddy;
       email = "ops@gebos.online";  # TODO: confirm ACME contact
-      extraConfig = lib.concatStringsSep "\n"
-        (lib.mapAttrsToList siteBlock cfg.sites);
+      extraConfig = lib.concatStringsSep "\n" (
+        [ globalOptions ]
+        ++ lib.mapAttrsToList httpSiteBlock httpSites
+        ++ lib.mapAttrsToList certSiteBlock tcpProxySites
+      );
     };
   };
 }

nix/modules/gebos-secrets.nix

@@ -17,12 +17,30 @@ let
   ];
   ingesterKeys = [ "ingester_postgres_password" ];
   postgresAdminKeys = [ "postgres_admin_password" ];
-  hivemqKeys = [ "hivemq_ingester_password" "hivemq_admin_password" ];
+  hivemqKeys = [ "hivemq_ingester_password" "hivemq_admin_password" "hivemq_device_password" ];
 
   # HiveMQ file-RBAC credentials.xml. Only the passwords are secret; the role/
-  # topic policy is structural. `ingester` may subscribe to the whole telemetry
-  # tree (t/<tenant>/d/<device>/<metric>); `admin` is an operational superuser.
-  hivemqCredentialsXml = ingesterPassword: adminPassword: ''
+  # topic policy is structural. Devices publish under `acrios/<IMSI>/<metric>`
+  # (set by the sender firmware's mqtt_topic_base), so that is the namespace the
+  # ingester consumes and the device role is scoped to.
+  #
+  # Roles:
+  #   ingest    — the Go ingester; SUBSCRIBE the whole device tree.
+  #   device    — physical senders; full access under acrios/ (publish telemetry
+  #               + the LWT/status and any downlink topics the firmware uses).
+  #   superuser — operational break-glass.
+  #
+  # Option A (current): one shared `device` user (`bender`) for the whole fleet.
+  # Tenant isolation is enforced downstream by the ingester's IMSI→tenant
+  # registry lookup, NOT at the broker — any device could publish under any IMSI.
+  #
+  # TODO(Option B / per-device auth): give each sender its own RBAC user with
+  # username == IMSI and scope its role to `acrios/${{username}}/#`, so the
+  # broker enforces that a device can only publish under its own IMSI (real
+  # anti-spoofing + per-device revocation). Requires setting mqtt_user = <IMSI>
+  # on each device and a per-device password provisioning flow (generate → sops
+  # → push to device). file-RBAC hot-reloads, so added users need no restart.
+  hivemqCredentialsXml = ingesterPassword: adminPassword: devicePassword: ''
     <?xml version="1.0" encoding="UTF-8" standalone="yes"?>
     <file-rbac>
         <users>
@@ -36,17 +54,30 @@ let
                 <password>${adminPassword}</password>
                 <roles><id>superuser</id></roles>
             </user>
+            <user>
+                <name>bender</name>
+                <password>${devicePassword}</password>
+                <roles><id>device</id></roles>
+            </user>
         </users>
         <roles>
             <role>
                 <id>ingest</id>
                 <permissions>
                     <permission>
-                        <topic>t/#</topic>
+                        <topic>acrios/#</topic>
                         <activity>SUBSCRIBE</activity>
                     </permission>
                 </permissions>
             </role>
+            <role>
+                <id>device</id>
+                <permissions>
+                    <!-- No <activity> = PUBLISH and SUBSCRIBE; covers telemetry,
+                         the LWT/status topic, and any downlink under acrios/. -->
+                    <permission><topic>acrios/#</topic></permission>
+                </permissions>
+            </role>
             <role>
                 <id>superuser</id>
                 <permissions>
@@ -112,7 +143,7 @@ in
         if hasSecrets && cfg.hivemq
         then config.sops.templates."hivemq-credentials".path
         else "${pkgs.writeText "hivemq-credentials-dev.xml"
-          (hivemqCredentialsXml "dev-ingester" "dev-admin")}";
+          (hivemqCredentialsXml "dev-ingester" "dev-admin" "dev-device")}";
     };
 
     envFile = lib.mkOption {
@@ -157,7 +188,8 @@ in
       templates."hivemq-credentials" = lib.mkIf cfg.hivemq {
         content = hivemqCredentialsXml
           config.sops.placeholder.hivemq_ingester_password
-          config.sops.placeholder.hivemq_admin_password;
+          config.sops.placeholder.hivemq_admin_password
+          config.sops.placeholder.hivemq_device_password;
         mode = "0400";
         owner = "gebos-hivemq";
       };